From 0df88f56034f578b3350db481acfccd3ba51ff9f Mon Sep 17 00:00:00 2001 From: Elliot Chen <2340896+cyfyifanchen@users.noreply.github.com> Date: Wed, 24 Jun 2026 23:17:23 +0800 Subject: [PATCH] chore(release): update EverOS to 1.1.0 (#307) --- .claude/skills/add-memory-kind/SKILL.md | 308 ++++ .env.example | 20 +- .gitignore | 192 ++- .pre-commit-config.yaml | 30 +- CHANGELOG.md | 136 +- CITATION.md | 8 +- Makefile | 2 +- SECURITY.md | 8 +- config.example.toml | 2 +- docs/api.md | 78 +- docs/architecture.md | 57 +- docs/cascade_runbook.md | 2 +- docs/cli.md | 26 +- docs/configuration.md | 228 +++ docs/datetime.md | 20 +- docs/engineering.md | 308 ++-- docs/how-memory-works.md | 31 +- docs/index.md | 5 +- docs/knowledge.md | 668 ++++++++ docs/locomo_benchmark.md | 21 +- docs/multimodal.md | 57 +- docs/openapi.json | 1511 ++++++++++++++++- docs/prompt_slots.md | 14 +- docs/reflection.md | 359 ++++ docs/release-notes-1.1.0.md | 52 + docs/storage_layout.md | 60 +- pyproject.toml | 29 +- scripts/check_consistency.py | 2 +- scripts/check_deprecated_names.py | 2 + scripts/e2e_memorize/README.md | 4 +- src/everos/component/embedding/__init__.py | 6 +- .../component/embedding/openai_provider.py | 4 +- src/everos/component/embedding/protocol.py | 11 +- src/everos/component/parser/__init__.py | 17 + src/everos/component/parser/_core.py | 67 + src/everos/component/rerank/__init__.py | 9 +- src/everos/component/rerank/_errors.py | 45 + .../component/rerank/deepinfra_provider.py | 27 +- src/everos/component/rerank/factory.py | 2 +- src/everos/component/rerank/protocol.py | 5 +- src/everos/component/rerank/vllm_provider.py | 24 +- src/everos/component/tokenizer/__init__.py | 2 - src/everos/component/tokenizer/factory.py | 6 +- src/everos/config/__init__.py | 4 +- src/everos/config/default.toml | 40 +- src/everos/config/default_ome.toml | 26 +- src/everos/config/settings.py | 125 +- src/everos/core/errors.py | 183 +- src/everos/core/middleware/__init__.py | 6 +- .../core/middleware/global_exception.py | 143 -- .../core/persistence/lancedb/repository.py | 27 +- .../core/persistence/markdown/__init__.py | 8 + .../core/persistence/markdown/frontmatter.py | 44 + .../core/persistence/markdown/writer.py | 79 +- src/everos/core/persistence/memory_root.py | 42 +- src/everos/entrypoints/api/app.py | 15 +- .../entrypoints/api/exception_handlers.py | 362 ++++ src/everos/entrypoints/api/routes/get.py | 9 +- .../entrypoints/api/routes/knowledge.py | 643 +++++++ src/everos/entrypoints/api/routes/memorize.py | 53 +- src/everos/entrypoints/api/routes/ome.py | 47 + src/everos/entrypoints/api/routes/search.py | 11 +- src/everos/entrypoints/api/utils.py | 13 + .../entrypoints/cli/commands/cascade.py | 14 + .../entrypoints/cli/commands/config_cmd.py | 85 + .../entrypoints/cli/commands/init_cmd.py | 186 +- src/everos/entrypoints/cli/commands/server.py | 93 +- src/everos/entrypoints/cli/main.py | 3 +- .../infra/ome/_background/config_reloader.py | 5 + src/everos/infra/ome/_dispatch/registry.py | 21 +- src/everos/infra/ome/_dispatch/runner.py | 32 +- src/everos/infra/ome/_stores/run_record.py | 19 +- src/everos/infra/ome/_stores/storage.py | 24 +- src/everos/infra/ome/context.py | 27 +- src/everos/infra/ome/decorator.py | 103 +- src/everos/infra/ome/engine.py | 101 +- src/everos/infra/ome/records.py | 1 + src/everos/infra/ome/testing/fakes.py | 39 +- src/everos/infra/ome/testing/harness.py | 9 +- .../infra/persistence/lancedb/__init__.py | 7 + .../persistence/lancedb/repos/__init__.py | 3 + .../lancedb/repos/knowledge_topic.py | 22 + .../persistence/lancedb/tables/__init__.py | 3 + .../lancedb/tables/_parent_type.py | 25 +- .../persistence/lancedb/tables/atomic_fact.py | 7 +- .../persistence/lancedb/tables/episode.py | 7 +- .../persistence/lancedb/tables/foresight.py | 2 +- .../lancedb/tables/knowledge_topic.py | 36 + .../infra/persistence/markdown/__init__.py | 10 + .../persistence/markdown/mds/__init__.py | 6 + .../persistence/markdown/mds/atomic_fact.py | 3 + .../infra/persistence/markdown/mds/episode.py | 3 + .../markdown/mds/knowledge_document.py | 26 + .../markdown/mds/knowledge_topic.py | 32 + .../persistence/markdown/readers/__init__.py | 4 + .../markdown/readers/taxonomy_reader.py | 217 +++ .../persistence/markdown/writers/__init__.py | 2 + .../persistence/markdown/writers/base.py | 12 + .../markdown/writers/knowledge_writer.py | 246 +++ .../infra/persistence/sqlite/__init__.py | 20 + .../persistence/sqlite/repos/__init__.py | 12 + .../infra/persistence/sqlite/repos/cluster.py | 147 +- .../persistence/sqlite/repos/knowledge.py | 342 ++++ .../sqlite/repos/reflection_report.py | 78 + .../persistence/sqlite/tables/__init__.py | 6 + .../persistence/sqlite/tables/cluster.py | 8 +- .../persistence/sqlite/tables/knowledge.py | 69 + .../sqlite/tables/reflection_report.py | 42 + src/everos/memory/_partition_locks.py | 66 + .../memory/cascade/handlers/__init__.py | 12 +- .../cascade/handlers/_daily_log_base.py | 163 +- .../memory/cascade/handlers/atomic_fact.py | 2 +- src/everos/memory/cascade/handlers/episode.py | 2 +- .../memory/cascade/handlers/foresight.py | 2 +- .../cascade/handlers/knowledge_document.py | 94 + .../cascade/handlers/knowledge_topic.py | 227 +++ src/everos/memory/cascade/registry.py | 24 +- src/everos/memory/events.py | 2 + src/everos/memory/extract/ingest/service.py | 3 +- .../memory/extract/parser/availability.py | 8 +- src/everos/memory/extract/parser/enrich.py | 43 +- .../memory/extract/pipeline/user_memory.py | 2 + src/everos/memory/get/filters_adapter.py | 2 + src/everos/memory/get/manager.py | 1 + src/everos/memory/models.py | 28 +- src/everos/memory/reflection/__init__.py | 14 + src/everos/memory/reflection/orchestrator.py | 1093 ++++++++++++ src/everos/memory/search/agentic.py | 84 +- src/everos/memory/search/dto.py | 13 +- src/everos/memory/search/filters.py | 9 +- src/everos/memory/search/hierarchy.py | 147 +- src/everos/memory/search/manager.py | 34 +- src/everos/memory/search/recall/__init__.py | 3 + .../memory/search/recall/atomic_fact.py | 116 +- src/everos/memory/search/recall/episode.py | 53 +- .../memory/search/recall/knowledge_topic.py | 128 ++ src/everos/memory/search/shaper.py | 2 +- src/everos/memory/strategies/__init__.py | 3 + .../memory/strategies/extract_agent_skill.py | 6 +- .../memory/strategies/extract_atomic_facts.py | 92 +- .../memory/strategies/extract_user_profile.py | 32 +- .../memory/strategies/reflect_episodes.py | 89 + .../strategies/trigger_profile_clustering.py | 29 +- .../strategies/trigger_skill_clustering.py | 2 +- src/everos/service/__init__.py | 63 + src/everos/service/knowledge.py | 1329 +++++++++++++++ src/everos/service/memorize.py | 2 + src/everos/templates/env.template | 20 +- tests/_consistency_assertions.py | 2 +- tests/e2e/conftest.py | 10 +- tests/e2e/test_get_endpoint_e2e.py | 4 +- tests/e2e/test_knowledge_e2e.py | 283 +++ tests/e2e/test_search_endpoint_e2e.py | 52 +- tests/fixtures/_dump_search_seed.py | 6 +- tests/helpers/__init__.py | 1 + tests/helpers/knowledge_md.py | 68 + tests/integration/search/_helpers.py | 9 +- tests/integration/search/_rerun_probes.py | 2 +- tests/integration/search/_run_full_report.py | 2 +- tests/integration/search/conftest.py | 11 +- tests/integration/search/test_search_e2e.py | 3 +- .../test_cascade_all_kinds_consistency.py | 3 +- .../test_cascade_cli_integration.py | 20 +- .../test_cascade_fsevents_repro.py | 3 +- tests/integration/test_cascade_integration.py | 3 +- tests/integration/test_cascade_scenarios.py | 3 +- .../integration/test_knowledge_integration.py | 1132 ++++++++++++ tests/integration/test_memorize_agent_mode.py | 1 + .../test_memorize_concurrent_session_lock.py | 1 + .../integration/test_memorize_integration.py | 1 + .../test_memorize_window_segmentation.py | 1 + .../test_ome_strategies_integration.py | 38 +- .../test_reflection_integration.py | 826 +++++++++ tests/run_locomo_10x3.sh | 6 +- tests/run_locomo_batch.sh | 4 +- tests/run_locomo_full.sh | 156 ++ tests/test_locomo.py | 238 ++- tests/test_reflection_e2e.py | 810 +++++++++ .../test_component/test_llm/test_client.py | 2 +- .../test_rerank/test_deepinfra_provider.py | 15 +- .../test_rerank/test_vllm_provider.py | 12 +- .../test_tokenizer/test_jieba.py | 54 +- .../test_utils/test_datetime.py | 10 +- .../test_config/test_knowledge_settings.py | 17 + tests/unit/test_config/test_settings.py | 173 +- tests/unit/test_core/test_errors.py | 110 ++ .../test_middleware/test_global_exception.py | 106 -- .../test_frontmatter_knowledge.py | 56 + .../test_persistence/test_locking.py | 128 +- .../test_markdown/test_writer.py | 8 + .../test_writer_patch_frontmatter.py | 102 ++ .../test_persistence/test_memory_root.py | 61 +- .../test_api/test_exception_handlers.py | 288 ++++ .../test_api/test_lifespans/test_cascade.py | 4 +- .../test_api/test_lifespans/test_ome.py | 1 + .../test_api/test_lifespans/test_storage.py | 2 +- .../test_routes/test_get_route_validation.py | 2 +- .../test_routes/test_knowledge_api.py | 541 ++++++ .../test_memorize_route_validation.py | 31 +- .../test_routes/test_metrics_route.py | 2 +- .../test_search_route_validation.py | 2 +- .../test_cli/test_cascade_command.py | 4 +- .../test_cli/test_init_command.py | 262 +-- .../test_entrypoints/test_cli/test_main.py | 1 + .../test_cli/test_server_command.py | 74 +- .../test_lancedb/test_lancedb_manager.py | 2 +- .../test_repos/test_agent_skill.py | 2 +- .../test_knowledge_topic_schema.py | 39 + .../test_mds/test_knowledge_frontmatter.py | 80 + .../test_readers/test_taxonomy_reader.py | 56 + .../test_writers/test_knowledge_writer.py | 306 ++++ .../test_ome/test_crash_recovery.py | 3 + .../test_infra/test_ome/test_decorator.py | 11 +- .../test_ome/test_engine_event_id.py | 82 + .../unit/test_infra/test_ome/test_records.py | 26 +- .../test_ome/test_run_record_store.py | 65 + tests/unit/test_infra/test_ome/test_runner.py | 19 +- .../test_ome/test_storage_migration.py | 61 + .../test_sqlite/test_knowledge_tables.py | 39 + .../test_sqlite/test_repos/test_cluster.py | 186 ++ .../test_repos/test_reflection_report.py | 119 ++ .../test_sqlite/test_sqlite_manager.py | 2 +- .../test_handler_knowledge_document.py | 213 +++ .../test_handler_knowledge_topic.py | 318 ++++ .../test_cascade/test_orchestrator.py | 2 +- .../test_memory/test_cascade/test_registry.py | 4 +- .../test_cascade/test_registry_knowledge.py | 39 + .../test_cascade/test_scanner_unit.py | 17 +- .../test_extract/test_parser/test_enrich.py | 98 +- .../test_pipeline/test_user_memory_emits.py | 4 +- .../test_get/test_filters_adapter.py | 10 +- .../test_memory/test_reflection/__init__.py | 0 .../test_reflection/test_orchestrator.py | 436 +++++ .../test_memory/test_search/test_agentic.py | 24 +- .../unit/test_memory/test_search/test_dto.py | 14 +- .../test_memory/test_search/test_filters.py | 14 +- .../test_memory/test_search/test_hierarchy.py | 347 +++- .../test_memory/test_search/test_manager.py | 24 +- .../test_search/test_recall_agent_skill.py | 2 +- .../test_search/test_recall_atomic_fact.py | 109 +- .../test_search/test_recall_episode.py | 116 +- .../test_recall_knowledge_topic.py | 227 +++ .../test_search/test_recall_or_semantics.py | 2 +- .../test_search/test_recall_profile.py | 2 +- .../test_extract_agent_case.py | 2 +- .../test_extract_agent_skill.py | 8 +- .../test_extract_atomic_facts.py | 228 +-- .../test_strategies/test_extract_foresight.py | 2 +- .../test_extract_user_profile.py | 49 +- .../test_strategies/test_partition_locks.py | 4 +- .../test_strategies/test_reflect_episodes.py | 31 + .../test_strategies/test_registration.py | 5 +- .../test_strategies_persistence.py | 21 +- .../test_strategy_to_handler_contract.py | 21 +- .../test_trigger_profile_clustering.py | 49 +- .../test_trigger_skill_clustering.py | 6 +- .../test_service/test_knowledge_create.py | 281 +++ .../unit/test_service/test_knowledge_crud.py | 345 ++++ .../test_service/test_knowledge_search.py | 442 +++++ .../test_knowledge_search_degradation.py | 41 + .../test_original_file_storage.py | 448 +++++ uv.lock | 78 +- 262 files changed, 20901 insertions(+), 2595 deletions(-) create mode 100644 .claude/skills/add-memory-kind/SKILL.md create mode 100644 docs/configuration.md create mode 100644 docs/knowledge.md create mode 100644 docs/reflection.md create mode 100644 docs/release-notes-1.1.0.md create mode 100644 src/everos/component/parser/__init__.py create mode 100644 src/everos/component/parser/_core.py create mode 100644 src/everos/component/rerank/_errors.py delete mode 100644 src/everos/core/middleware/global_exception.py create mode 100644 src/everos/entrypoints/api/exception_handlers.py create mode 100644 src/everos/entrypoints/api/routes/knowledge.py create mode 100644 src/everos/entrypoints/api/routes/ome.py create mode 100644 src/everos/entrypoints/api/utils.py create mode 100644 src/everos/entrypoints/cli/commands/config_cmd.py create mode 100644 src/everos/infra/persistence/lancedb/repos/knowledge_topic.py create mode 100644 src/everos/infra/persistence/lancedb/tables/knowledge_topic.py create mode 100644 src/everos/infra/persistence/markdown/mds/knowledge_document.py create mode 100644 src/everos/infra/persistence/markdown/mds/knowledge_topic.py create mode 100644 src/everos/infra/persistence/markdown/readers/taxonomy_reader.py create mode 100644 src/everos/infra/persistence/markdown/writers/knowledge_writer.py create mode 100644 src/everos/infra/persistence/sqlite/repos/knowledge.py create mode 100644 src/everos/infra/persistence/sqlite/repos/reflection_report.py create mode 100644 src/everos/infra/persistence/sqlite/tables/knowledge.py create mode 100644 src/everos/infra/persistence/sqlite/tables/reflection_report.py create mode 100644 src/everos/memory/_partition_locks.py create mode 100644 src/everos/memory/cascade/handlers/knowledge_document.py create mode 100644 src/everos/memory/cascade/handlers/knowledge_topic.py create mode 100644 src/everos/memory/reflection/__init__.py create mode 100644 src/everos/memory/reflection/orchestrator.py create mode 100644 src/everos/memory/search/recall/knowledge_topic.py create mode 100644 src/everos/memory/strategies/reflect_episodes.py create mode 100644 src/everos/service/knowledge.py create mode 100644 tests/e2e/test_knowledge_e2e.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/knowledge_md.py create mode 100644 tests/integration/test_knowledge_integration.py create mode 100644 tests/integration/test_reflection_integration.py create mode 100644 tests/run_locomo_full.sh create mode 100644 tests/test_reflection_e2e.py create mode 100644 tests/unit/test_config/test_knowledge_settings.py create mode 100644 tests/unit/test_core/test_errors.py delete mode 100644 tests/unit/test_core/test_middleware/test_global_exception.py create mode 100644 tests/unit/test_core/test_persistence/test_frontmatter_knowledge.py create mode 100644 tests/unit/test_core/test_persistence/test_markdown/test_writer_patch_frontmatter.py create mode 100644 tests/unit/test_entrypoints/test_api/test_exception_handlers.py create mode 100644 tests/unit/test_entrypoints/test_api/test_routes/test_knowledge_api.py create mode 100644 tests/unit/test_infra/test_lancedb/test_tables/test_knowledge_topic_schema.py create mode 100644 tests/unit/test_infra/test_markdown/test_mds/test_knowledge_frontmatter.py create mode 100644 tests/unit/test_infra/test_markdown/test_readers/test_taxonomy_reader.py create mode 100644 tests/unit/test_infra/test_markdown/test_writers/test_knowledge_writer.py create mode 100644 tests/unit/test_infra/test_ome/test_engine_event_id.py create mode 100644 tests/unit/test_infra/test_ome/test_storage_migration.py create mode 100644 tests/unit/test_infra/test_sqlite/test_knowledge_tables.py create mode 100644 tests/unit/test_infra/test_sqlite/test_repos/test_reflection_report.py create mode 100644 tests/unit/test_memory/test_cascade/test_handler_knowledge_document.py create mode 100644 tests/unit/test_memory/test_cascade/test_handler_knowledge_topic.py create mode 100644 tests/unit/test_memory/test_cascade/test_registry_knowledge.py create mode 100644 tests/unit/test_memory/test_reflection/__init__.py create mode 100644 tests/unit/test_memory/test_reflection/test_orchestrator.py create mode 100644 tests/unit/test_memory/test_search/test_recall_knowledge_topic.py create mode 100644 tests/unit/test_memory/test_strategies/test_reflect_episodes.py create mode 100644 tests/unit/test_service/test_knowledge_create.py create mode 100644 tests/unit/test_service/test_knowledge_crud.py create mode 100644 tests/unit/test_service/test_knowledge_search.py create mode 100644 tests/unit/test_service/test_knowledge_search_degradation.py create mode 100644 tests/unit/test_service/test_original_file_storage.py diff --git a/.claude/skills/add-memory-kind/SKILL.md b/.claude/skills/add-memory-kind/SKILL.md new file mode 100644 index 0000000..6a58fcd --- /dev/null +++ b/.claude/skills/add-memory-kind/SKILL.md @@ -0,0 +1,308 @@ +--- +name: add-memory-kind +description: Add a new business memory kind end-to-end. Pick the storage combination (Markdown / SQLite / LanceDB), pick the markdown strategy (daily-log / skill-named / single-file), then wire up the schema(s), repo(s), and writer(s). +--- + +# /add-memory-kind — Add a new business memory kind + +## When to invoke + +Adding a new persisted business entity (Episode, Case, Skill, AtomicFact, +Foresight, Profile, or something custom). Multiple storage layers may be +involved; this skill walks the decision then the wiring. + +## 1. Decide the storage combination + +A memory kind **does not have to use all three** layers. Pick by what +the kind actually needs: + +| Need | Markdown | SQLite | LanceDB | +|---|:-:|:-:|:-:| +| Human-readable / agent-editable source-of-truth text | ✅ | | | +| Structured state, ACID transactions, joins, predicates | | ✅ | | +| Vector / BM25 / hybrid retrieval | | | ✅ | + +Common combinations seen in EverOS: + +| Combo | Example | Rationale | +|---|---|---| +| **md only** | scratch notes / dump bins | text-of-truth, no index needed | +| **md + lancedb** | episode / memcell / case | text-of-truth + semantic retrieval | +| **md + sqlite** | profile / playbook / soul.md state | text-of-truth + structured state to query | +| **md + sqlite + lancedb** | full-blown business records | when you need *both* transactional state AND retrieval | +| **sqlite only** | audit log / task queue / LSN watermark | system state, never user-facing | +| **lancedb only** | rare; usually you still want md | derived embeddings without text-of-truth | + +Rule of thumb: **markdown is the truth**; sqlite and lancedb are derived +indexes that can be rebuilt from md. Drop md only when the kind has no +human-readable form (pure system state). + +## 2. Pick the markdown storage strategy (if md is in your combo) + +Three strategies — declared in the EverOS Markdown First spec: + +| Strategy | Filename | Mutation | Examples | +|---|---|---|---| +| **Daily-log append** | `-YYYY-MM-DD.md` | append entries | memcell / episode / case / atomic_fact / foresight | +| **Skill-named in-place** | `skill_.md` | overwrite the file | skills (procedural memory) | +| **Single-file rewrite** | `user.md` / `agent.md` / `soul.md` / `behaviors.md` / `tools.md` | overwrite the file | profiles / playbooks | + +This skill currently has a **complete recipe for daily-log append**. +Skill-named and single-file recipes are sketched at the bottom — their +base writers (`BaseSkillWriter` / `BaseProfileWriter`) land later in the +project; until then build a thin wrapper over `MarkdownWriter` +directly. + +--- + +## 3. Markdown daily-log: 4 steps + +### 3.1 Frontmatter schema — `infra/persistence/markdown/mds/.py` + +```python +"""Episode daily-log frontmatter.""" + +from __future__ import annotations + +import datetime as _dt +from typing import ClassVar, Literal + +from everos.core.persistence.markdown import UserScopedFrontmatter + + +class UserEpisodeDailyFrontmatter(UserScopedFrontmatter): + """``users//episodes/episode-.md``.""" + + ENTRY_ID_PREFIX: ClassVar[str] = "ep" + DIR_NAME: ClassVar[str] = "episodes" + FILE_PREFIX: ClassVar[str] = "episode" + + type: Literal["user_episode_daily"] = "user_episode_daily" + date: _dt.date + entry_count: int = 0 + last_appended_at: _dt.datetime | None = None +``` + +For agent-track kinds subclass `AgentScopedFrontmatter` instead. If +user-track and agent-track share a kind name (e.g. `memcell`), give +each a **distinct** `ENTRY_ID_PREFIX` (e.g. `umc` vs `amc`) so reverse +lookup is unambiguous. + +### 3.2 Re-export — `mds/__init__.py` + +```python +from .episode import UserEpisodeDailyFrontmatter as UserEpisodeDailyFrontmatter +``` + +### 3.3 Business writer — `infra/persistence/markdown/writers/.py` + +```python +"""Episode appender.""" + +from __future__ import annotations + +from pathlib import Path + +from everos.core.persistence import MarkdownReader + +from ..mds import UserEpisodeDailyFrontmatter +from .base import BaseDailyWriter + + +class UserEpisodeAppender(BaseDailyWriter): + schema = UserEpisodeDailyFrontmatter + + # OPTIONAL: override the count strategy. Default is len(entries); + # override to trust the frontmatter field instead. + def _current_count(self, path: Path) -> int: + if not path.exists(): + return 0 + return MarkdownReader.read(path).frontmatter.get("entry_count", 0) +``` + +### 3.4 Re-export — `writers/__init__.py` + +```python +from .episode import UserEpisodeAppender as UserEpisodeAppender +``` + +### Done — usage + +```python +from everos.infra.persistence.markdown.writers import UserEpisodeAppender + +appender = UserEpisodeAppender(memory_root) +eid = appender.append("u_jason", "I went to the doctor today.") +# → users/u_jason/episodes/episode-.md +# → entry markers carry an auto-generated EntryId (e.g. ep_20260507_001) +``` + +--- + +## 4. (Optional) SQLite table — 4 steps + +Skip this section if the kind doesn't need structured state beyond markdown. + +### 4.1 Schema — `infra/persistence/sqlite/tables/.py` + +```python +from everos.core.persistence.sqlite import BaseTable, Field + + +class EpisodeState(BaseTable, table=True): + __tablename__ = "episode_state" # type: ignore[assignment] + + id: int | None = Field(default=None, primary_key=True) + entry_id: str = Field(index=True, unique=True) + cluster_id: str | None = Field(default=None, index=True) + status: str = Field(default="active") +``` + +`BaseTable` already provides `created_at` / `updated_at` (auto-bumped). + +### 4.2 Re-export — `tables/__init__.py` + +```python +from .episode import EpisodeState as EpisodeState +``` + +### 4.3 Repo — `infra/persistence/sqlite/repos/.py` + +```python +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from everos.core.persistence.sqlite import RepoBase + +from ..sqlite_manager import get_session_factory +from ..tables import EpisodeState + + +class _EpisodeStateRepo(RepoBase[EpisodeState]): + model = EpisodeState + + def _factory_lookup(self) -> async_sessionmaker[AsyncSession]: + return get_session_factory() + + +episode_state_repo = _EpisodeStateRepo() +``` + +### 4.4 Re-export — `repos/__init__.py` + +```python +from .episode import episode_state_repo as episode_state_repo +``` + +--- + +## 5. (Optional) LanceDB index — 4 steps + +Skip this section if the kind doesn't need vector / BM25 / hybrid retrieval. + +### 5.1 Schema — `infra/persistence/lancedb/tables/.py` + +```python +from everos.core.persistence.lancedb import BaseLanceTable, Vector + + +class EpisodeIndex(BaseLanceTable): + entry_id: str + text: str + tags: list[str] + vector: Vector(384) # type: ignore[valid-type] +``` + +`Vector(N)` must match your embedding dimension. + +### 5.2 Re-export — `tables/__init__.py` + +```python +from .episode import EpisodeIndex as EpisodeIndex +``` + +### 5.3 Repo — `infra/persistence/lancedb/repos/.py` + +```python +from lancedb import AsyncTable + +from everos.core.persistence.lancedb import LanceRepoBase + +from ..lancedb_manager import get_table +from ..tables import EpisodeIndex + + +class _EpisodeIndexRepo(LanceRepoBase[EpisodeIndex]): + schema = EpisodeIndex + table_name = "episode_index" + + async def _table_lookup(self) -> AsyncTable: + return await get_table(self.table_name, self.schema) + + +episode_index_repo = _EpisodeIndexRepo() +``` + +### 5.4 Re-export — `repos/__init__.py` + +```python +from .episode import episode_index_repo as episode_index_repo +``` + +--- + +## 6. (Future) Skill-named & single-file markdown strategies + +When the new memory kind needs: + +- **skill-named** files (one file per named skill, in-place rewrite) — wait + for `BaseSkillWriter`, or use `MarkdownWriter.write_markdown` directly + with a thin wrapper. +- **single-file rewrite** (one fixed file like `user.md`) — wait for + `BaseProfileWriter`, same fallback. + +These strategies do **not** use entry markers; their frontmatter schema +does not need `ENTRY_ID_PREFIX` (only `id` / `type` / `schema_version` plus +the scope mixin fields). + +--- + +## 7. Verification checklist + +- [ ] `make lint` — ruff + import-linter clean +- [ ] `make test` — existing manager / lifespan / writer tests still pass +- [ ] When the kind crosses **multiple** layers: + - [ ] markdown entry id is the join key for the sqlite / lancedb rows + - [ ] business code reads only via the repo singleton (no raw engine + access in service / memory) + - [ ] cascade daemon (when it lands) can rebuild sqlite / lancedb + from md alone — keep md as the truth + +Tests by layer: + +| Tests for | Location | +|---|---| +| Markdown frontmatter schema | `tests/unit/test_infra/test_markdown/test_mds/` | +| Markdown business appender | `tests/unit/test_infra/test_markdown/test_writers/` | +| SQLite RepoBase logic | `tests/unit/test_core/test_persistence/test_sqlite/` | +| SQLite manager / lifespan | `tests/unit/test_infra/test_sqlite/` | +| LanceDB LanceRepoBase logic | `tests/unit/test_core/test_persistence/test_lancedb/` | +| LanceDB manager / lifespan | `tests/unit/test_infra/test_lancedb/` | + +## 8. Common pitfalls + +| Mistake | Symptom | Fix | +|---|---|---| +| Forgot `ENTRY_ID_PREFIX` / `DIR_NAME` / `FILE_PREFIX` on a daily-log schema | `BaseDailyWriter.__init__` raises `TypeError` | Add all three ClassVars | +| Same `ENTRY_ID_PREFIX` on user + agent variants | `MemoryLayout.locate_for_entry` collision error | Use distinct prefixes (e.g. `umc` vs `amc`) | +| Imported `RepoBase` from `infra.persistence.sqlite` | `ImportError` | Lives in `core.persistence.sqlite` (moved earlier) | +| Skipped one of the four files (schema / writer / table / repo) | One side silently absent | Re-export both/all from each `__init__.py` | +| `Vector(N)` mismatched with embedding dim | LanceDB raises on insert | Make `N` exactly match the model output | +| Imported `MemoryLayout` from a writer (infra) | `import-linter` fails (`infra → memory` reverse dep) | Use `MemoryRoot` (in core) and let the schema's ClassVars drive paths | +| Hand-rolling `datetime.now()` instead of `today_with_timezone()` | Day-boundary drift across timezones | Always go through `everos.component.utils.datetime` | + +## Background + +- Architecture: [../../rules/architecture.md](../../rules/architecture.md) +- `__init__.py` re-export rules: [../../rules/init-py-and-reexport.md](../../rules/init-py-and-reexport.md) +- Async programming: [../../rules/async-programming.md](../../rules/async-programming.md) +- Datetime handling: [../../rules/datetime-handling.md](../../rules/datetime-handling.md) diff --git a/.env.example b/.env.example index fe392d2..11ada68 100644 --- a/.env.example +++ b/.env.example @@ -4,23 +4,19 @@ # ===================================================== # # Setup: -# 1. Create .env with `everos init` or `cp .env.example .env` +# 1. cp env.template .env # 2. Edit .env with your values # 3. .env is gitignored (never commit) # # Override priority (low → high): # src/everos/config/default.toml (shipped baseline) # ↓ -# ~/.everos/config.toml (user-level overrides; optional) +# /everos.toml (user config; optional; root resolved +# by EVEROS_ROOT env > ~/.everos) # ↓ -# .env (this file; gitignored) -# ↓ -# EVEROS_
__ process envs +# EVEROS_
__ process envs (this file sources these) # ↓ # programmatic init args / CLI flags -# -# The user-level toml path defaults to ~/.everos/config.toml; override -# with EVEROS_CONFIG_FILE=/path/to/your.toml. Missing file is skipped. # ===================================================== @@ -65,7 +61,7 @@ EVEROS_MULTIMODAL__BASE_URL=https://openrouter.ai/api/v1 # ─── Embedding (OpenAI-protocol /embeddings) ───────── # Any OpenAI-compatible embedding endpoint plugs in via base_url. # model / api_key / base_url have no shipped default — set them here -# or in ~/.everos/config.toml before the embedding capability is used. +# or in /everos.toml before the embedding capability is used. EVEROS_EMBEDDING__MODEL=Qwen/Qwen3-Embedding-4B EVEROS_EMBEDDING__API_KEY= @@ -109,10 +105,10 @@ EVEROS_RERANK__BASE_URL=https://api.deepinfra.com/v1/inference # ─── Storage paths ─────────────────────────────────── # memory-root holds md files + .index/ (LanceDB) + .system.db (SQLite) + ... -# Override the default with EVEROS_MEMORY__ROOT (note the double-underscore -# for nested config keys); see config/default.toml for all tunables. +# Override the default (~/.everos) with EVEROS_ROOT; also controls which +# everos.toml is loaded. See config/default.toml for all other tunables. -# EVEROS_MEMORY__ROOT=~/.everos +# EVEROS_ROOT=~/.everos # ─── HTTP API ──────────────────────────────────────── diff --git a/.gitignore b/.gitignore index af95e07..6bd1a98 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# VS Code workspace files +*.code-workspace + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -50,10 +53,6 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ -.ruff_cache/ -.import_linter_cache/ -.uv-cache/ -.package-smoke/ # Translations *.mo @@ -107,6 +106,7 @@ celerybeat.pid # Environments .env +.env.* .venv env/ venv/ @@ -132,73 +132,139 @@ dmypy.json # Pyre type checker .pyre/ -# ─────────────────────────────────────────────────────────────────────────── # Project specific -# ─────────────────────────────────────────────────────────────────────────── +outputs/ +.pytest_cache/ + +evaluation/static_memory_evaluation/logs/ +evaluation/static_memory_evaluation/retrieval/ +evaluation/static_memory_evaluation/retrieval_bk/ +evaluation/static_memory_evaluation/analysis/ +memory_base/static_memory_base/bm25_index/ +memory_base/static_memory_base/hipporag/ +memory_base/static_memory_base/vector_search/ + +# Demo related files +demo/.chat_history +demo/chat_history/ +demo/memcell_outputs/ # macOS .DS_Store - -# Editor / IDE -.trae/ -.cursor/ - -# Claude Code: track team-shared rules/skills/settings; ignore personal + runtime -.claude/* -!.claude/rules/ -!.claude/skills/ -!.claude/settings.json -.claude/settings.local.json -.claude/worktrees/ -.worktrees/ -worktrees/ - -# Runtime data (the default memory root + local databases) -.everos/ *.duckdb -# Large / generated artefacts that should never be committed + +# Evaluation and memory data files +evaluation_memory_offline/data/ +evaluation_memory_offline/temporary/ + +# Dynamic memory files (runtime generated) +memory_base/dynamic_memory_base/memory/ +# Evaluation results and large files +evaluation/dynamic_memory_evaluation/locomo_results/*.json +evaluation/dynamic_memory_evaluation/locomo_results/hipporag_index*/ +evaluation/dynamic_memory_evaluation/locomo_results/vector_index*/ +evaluation/dynamic_memory_evaluation/locomo_results/storages* +# Large data files +memory_base/static_memory_base/raw_data/*.json *.tar *.zip -tmp/ -outputs/ -logs/ + +# Large evaluation files +evaluation/memory_evaluation/converted_messages_all_from_1_1.jsonl +evaluation/memory_evaluation/*.jsonl +evaluation/memory_evaluation/*.json + +# Large log files +*.log +su_ser.log +service.log +service_*.log nohup.out +mongodb_backup/ -# Repository media policy: do not commit images, videos, or asset/media folders. -# Use external hosting, release artifacts, or approved storage and link from docs. -asset/ -assets/ -image/ -images/ -img/ -media/ -video/ -videos/ -*.avif -*.bmp -*.gif -*.heic -*.heif -*.icns -*.ico -*.jpeg -*.jpg -*.png -*.svg -*.tif -*.tiff -*.webp -*.avi -*.flv -*.m4v -*.mkv -*.mov -*.mp4 -*.mpeg -*.mpg -*.webm -*.wmv +# Log directories +logs/ +log/ -# Use-cases: exclude lock files to keep the repo lean -use-cases/**/package-lock.json +# Apollo config +apollo_config/ + +# Exclude specific files from git tracking +simple_query_memcell.py +evaluation/memory_evaluation/extract_room_data.py +evaluation/memory_evaluation/test.py +evaluation/results/ + +# Large test files +unit_test/memcell_outputs/ +unit_test/profile_outputs/ +unit_test/memcell_outputs.zip + +#temp test files +export_mongodb_data.py + +#backup files +src/memory_layer/memory_extractor/profile_memory_extractor keep llm merge.py + +# JetBrains IDE +.idea/ + +# Local design plans +docs/plans/ + +# Git worktrees +.worktrees/ + +#LLM related +AGENTS.mk +.cursor/* + +# .claude/ team-shared (rules, skills, settings.json, CLAUDE.md) goes IN git +# Only personal overrides are ignored: +.claude/settings.local.json +CLAUDE.local.md + +# Work context: personal design drafts, not part of the codebase +.work_context/ + +# Local personal workspace: spec drafts, plans, scratch, recovery scripts +local/ + +#tmp_data +demo/memcell_outputs/ +demo/results/ +evaluation/locomo_evaluation/results/ +evaluation/locomo_evaluation/results_ref/ +tmp/ +evaluation/locomo_evaluation/results_ref/demo/results/ + +# i18n translation progress +.translation_progress.json +.review_progress.json + +# Single "output/" directory (plural "outputs/" already covered above) +output/ +demo/output/ + +# Adapter prompt dumps (runtime generated, sometimes several MB) +evaluation/src/adapters/*/prompts/profile/*.json + +# Locomo source dataset (downloadable, not source code) +data/locomo10.json +evaluation/data/locomo/locomo10.json +evaluation/locomo_evaluation/data/locomo10.json + +# Legacy src kept locally for migration reference; not under version control. +src_old/ + +# Benchmark checkpoints — Phase-level intermediate JSON dumps produced by +# tests/test_locomo.py / tests/run_locomo_batch.sh. Per-conv final results +# (benchmark_results/run_*/convN.json) and reports (REPORT.md / REPORT.html) +# stay tracked; checkpoints are large, regeneratable, and not useful in +# code review. +benchmark_results/ +benchmark_checkpoints/ + +# Local everos runtime data (memory root, indexes, OME state) +.everos/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea470ca..844b658 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,16 @@ -# Local quality gate, run before each commit. Mirrors the checks CI enforces so -# failures surface locally first. Install with `make install` (sets up both the -# pre-commit and commit-msg hook stages). -# -# Run manually across the repo: uv run pre-commit run --all-files -default_install_hook_types: [pre-commit, commit-msg] +default_language_version: + python: python3.12 repos: - # ruff version is kept in sync with the `ruff` dev dependency in uv.lock. - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.12 hooks: - id: ruff - name: ruff (lint + autofix) args: [--fix] - id: ruff-format - name: ruff (format) - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -25,25 +18,12 @@ repos: - id: check-toml - id: check-added-large-files args: [--maxkb=1024] - - id: detect-private-key - id: check-merge-conflict + - id: detect-private-key - - repo: local - hooks: - - id: no-repo-assets - name: block committed images, videos, and asset directories - entry: python3 scripts/check_repo_assets.py - language: system - pass_filenames: false - - id: no-deprecated-product-names - name: block deprecated product names - entry: python3 scripts/check_deprecated_names.py - language: system - pass_filenames: false - + # Commit message format: gitlint runs in commit-msg stage (no Node required). - repo: https://github.com/jorisroovers/gitlint rev: v0.19.1 hooks: - id: gitlint - name: gitlint (commit message format) stages: [commit-msg] diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fabd11..68355ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,31 +7,134 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -_Unreleased changes on `dev` will be listed here._ +## [1.1.0] - 2026-06-24 + +### Added + +- **Knowledge base subsystem** — full-stack document management exposed via + `/api/v1/knowledge/*`. Upload documents (PDF / HTML / DOCX via multimodal + parser), CRUD operations, and hybrid search (BM25 + vector + rerank + + category boost). Ships with a 20-category default taxonomy + (`.taxonomy.md`, auto-generated on first use). Original uploaded files are + preserved alongside extracted Markdown. New settings group: + `knowledge.*` (search tuning, `max_upload_bytes`, etc.). +- **Reflection V1** — offline memory self-improvement engine. + Select → Merge → Re-extract → Deprecate: clusters related episodes within + existing 7-day windows, merges them via LLM, re-extracts consolidated + episodes, and deprecates the originals. Runs as an OME strategy + (`reflect_episodes`); configure via `ome.toml` + (`[strategies.reflect_episodes]`, cron `0 2 * * 1`), changes are + hot-reloaded within ~2 s, no restart needed; **disabled by default**. + Requires `everalgo-user-memory>=0.3.1`. +- **Standardized error response contract.** All API errors now return a + canonical envelope with a semantic `ErrorCode` (10 codes: `NOT_FOUND`, + `CONFLICT`, `INVALID_INPUT`, `EXTRACTION_EMPTY`, `UNSUPPORTED_FORMAT`, + `EXTERNAL_SERVICE_UNAVAILABLE`, `CAPABILITY_UNAVAILABLE`, + `CONFIGURATION_ERROR`, `INTERNAL_ERROR`, `BAD_REQUEST`), per-type + exception handlers with MRO dispatch, and an `ErrorResponse` Pydantic + model visible in OpenAPI docs. Replaces the v1.0 two-code scheme + (`HTTP_ERROR` / `SYSTEM_ERROR`). +- **Search: hierarchical fact eviction** (Layer-4) with `min_score` floor — + low-confidence atomic facts are evicted before fusion, improving + precision. +- **Search degradation guidance** — when embedding or rerank providers fail, + the response now includes a `degradation` field explaining which + capability is unavailable and how results are affected. +- **Knowledge topic recaller** — dual-column BM25 recall for knowledge + topics, integrated into the search manager alongside existing recall + types. + +### Changed + +- **`everos init` now generates `gpt-4.1-mini`** as the default LLM model + (was `gpt-4o-mini`). Existing user configurations are not affected. +- **API error `code` values have changed.** v1.0 returned only `HTTP_ERROR` + (all 4xx) and `SYSTEM_ERROR` (all 5xx). v1.1 returns fine-grained + semantic codes (see Added above). Clients that match on `error.code` + string values need to update. The envelope structure + (`request_id` + `error.{code, message, timestamp, path}`) is unchanged. +- **DDD-aligned exception hierarchy** — domain errors reorganized: + `ValidationError` → `InvalidInputError`; + `DocumentAlreadyExistsError` → `DuplicateDocumentError`; + `EmbeddingError` → `EmbeddingServiceError`; + `RerankError` → `RerankServiceError`; + `LLMError` → `LLMServiceError` (at the boundary); + `MultimodalError` split into `UnsupportedModalityError` (domain) + + `MultimodalNotEnabledError` (infrastructure). + New base classes: `CapabilityError`, `ConfigurationError`. +- **`infra/` restructured** — storage adapters moved under + `infra/persistence/{markdown,sqlite,lancedb}`; each sub-package's + `__init__.py` is the sole public API (enforced by import-linter). +- **Parser capability extracted** to `component/parser` (shared by memorize + and knowledge upload paths). + +### Fixed + +- **Knowledge search no longer returns a bare `500 INTERNAL_ERROR` when the + embedding or rerank provider is unconfigured.** `_require_search_providers` + now raises `ConfigurationError` → `500 CONFIGURATION_ERROR`. A provider + that is configured but fails at call time still surfaces as + `503 EXTERNAL_SERVICE_UNAVAILABLE`. +- **Knowledge document uploads are capped** at `knowledge.max_upload_bytes` + (default 50 MiB); oversized uploads are rejected with `422` before parsing. +- **Knowledge search `query` is bounded** to 2000 chars. +- **`GET /knowledge/documents?sort_by=updated_at`** is now accepted. +- **`POST /knowledge/documents` returns `original_file_path`** so callers no + longer need a follow-up `GET` to locate the preserved upload. +- **Rerank providers no longer echo the upstream HTTP response body** into the + client-facing `503` message (vLLM / DeepInfra); the body is logged instead. +- **Knowledge FK cascade race** — removed the foreign key on + `knowledge_topics.doc_id` that caused delete-order race conditions; + cascade cleanup handled at application level. +- **Knowledge `replace_document`** — atomic PUT: backup old Markdown before + re-extraction; removed explicit SQLite delete for atomicity. +- **Knowledge duplicate `doc_id`** rejected on create; title collision + resolved by appending `doc_id` to directory name. +- **Knowledge `md_path` resolution** fixed in `delete_document` (was not + resolved against `memory_root`). +- **OME file-handle leak** — portalocker file handle is now closed on lock + contention instead of being left open. +- **jieba / Python 3.12 compatibility** — deferred jieba import to avoid + `SyntaxError` from invalid escape sequences; suppressed + `DeprecationWarning` in tests. +- **Test isolation** — tests no longer leak `.env` state or depend on module + import ordering. + +### Documentation + +- Added knowledge base technical documentation. +- Corrected the onboarding flow: `everos init` writes `everos.toml` + + `ome.toml` (TOML), not a `.env` file; removed the nonexistent + `--xdg` / `--env-file` options and the false `0600`-permissions claim + from `README.md` / `QUICKSTART.md`; fixed the stable-version line + (`v1.0.1`) and completed the `docs/cli.md` command tree. +- Updated error handling docs to match the new DDD exception hierarchy. ## [1.0.1] - 2026-06-16 ### Security - **Path-traversal hardening for caller-supplied identifiers.** `sender_id` - now carries the same path-safety guard as `app_id` / `project_id`: a - character whitelist plus rejection of the `.` / `..` tokens. The whitelist - admits `@` and `+` so email-style ids and plus-addressing still pass. + (which flows through to `owner_id` and becomes a directory segment on the + episode write path) now carries the same path-safety guard as `app_id` / + `project_id`: a character whitelist plus rejection of the `.` / `..` tokens. + The whitelist admits `@` and `+` so real-world ids (email-style, + plus-addressing) still pass. - **Defense-in-depth write containment.** `MarkdownWriter` now rejects any - write target that resolves outside the configured memory root before reading, - creating parent directories, or writing files. The API layer maps this - backstop error to HTTP 400. + write target that resolves outside the configured memory root, before any + filesystem touch (both the write `mkdir` and the append read-modify-write + read). This backstop holds even if an identifier reaches the writer + unsanitised (e.g. an `owner_id` set in the extract pipeline rather than from + the DTO). The API layer maps the resulting error to HTTP 400. ### Documentation - Add a multimodal usage guide and correct the multimodal error semantics after end-to-end verification. -- Document the upcoming Knowledge Wiki and idle/offline Reflection/Dreaming - roadmap in the README and documentation set. -- Rename outdated algorithm-library references to `everalgo` across docs and - code comments; no code identifiers changed. -- Fix accuracy drift found in a documentation audit; reflect the `everalgo` - packages being published and the v1.0.0 stable status. +- Rename the algorithm library from the previous package name to `everalgo` + across docs and code comments (no code identifiers changed). +- Fix accuracy drift found in an adversarial doc audit; reflect the + `everalgo` packages being published and the v1.0.0 stable status. ## [1.0.0] - 2026-06-03 @@ -60,6 +163,7 @@ for AI agents. - **Decoupled algorithms** — memory extraction algorithms live in the standalone `everalgo-*` libraries published on PyPI. -[Unreleased]: https://github.com/EverMind-AI/EverOS/compare/v1.0.1...HEAD -[1.0.1]: https://github.com/EverMind-AI/EverOS/releases/tag/v1.0.1 -[1.0.0]: https://github.com/EverMind-AI/EverOS/releases/tag/v1.0.0 +[Unreleased]: https://github.com/EverMind-AI/everos/compare/v1.1.0...HEAD +[1.1.0]: https://github.com/EverMind-AI/everos/compare/v1.0.1...v1.1.0 +[1.0.1]: https://github.com/EverMind-AI/everos/releases/tag/v1.0.1 +[1.0.0]: https://github.com/EverMind-AI/everos/releases/tag/v1.0.0 diff --git a/CITATION.md b/CITATION.md index e797b0c..3db0c90 100644 --- a/CITATION.md +++ b/CITATION.md @@ -33,8 +33,8 @@ To cite the software itself: ``` EverOS: md-first memory extraction framework for AI agents -Version: 1.0.1 -URL: https://github.com/EverMind-AI/EverOS +Version: 0.1.0 +URL: https://github.com/EverMind-AI/everos License: Apache 2.0 ``` @@ -52,9 +52,9 @@ If you use EverOS, we appreciate: ## Stay updated -- Watch the [GitHub repository](https://github.com/EverMind-AI/EverOS) for paper announcements +- Watch the [GitHub repository](https://github.com/EverMind-AI/everos) for paper announcements - Follow [@EverMindAI](https://x.com/EverMindAI) on X / Twitter -- Join [GitHub Discussions](https://github.com/EverMind-AI/EverOS/discussions) +- Join [GitHub Discussions](https://github.com/EverMind-AI/everos/discussions) --- diff --git a/Makefile b/Makefile index 905641b..d6f633a 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ help: @echo "Targets:" @echo " install Install deps + pre-commit hooks (full dev setup)" @echo " install-deps Install deps only (uv sync --frozen, used by CI)" - @echo " lint ruff + import-linter + repo hygiene + datetime discipline + openapi drift" + @echo " lint ruff (check + format-check) + import-linter + datetime discipline + openapi drift" @echo " docs-check Validate Markdown links, use-case banners, and issue template YAML" @echo " check-commits Validate Conventional Commit subjects for a git range" @echo " check-pr-title Validate PR title uses Conventional Commit format" diff --git a/SECURITY.md b/SECURITY.md index cdd5751..60f6e22 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,13 +2,13 @@ ## Supported Versions -EverOS is in active alpha development. Security fixes are applied to the latest -release line only. +EverOS is released and at v1.0.0 (stable). Security fixes are applied to the +latest release line only. | Version | Supported | |---------|-----------| -| 0.1.x | ✅ | -| < 0.1 | ❌ | +| 1.x | ✅ | +| < 1.0 | ❌ | ## Reporting a Vulnerability diff --git a/config.example.toml b/config.example.toml index 1a3d69d..f804312 100644 --- a/config.example.toml +++ b/config.example.toml @@ -21,7 +21,7 @@ # ── LLM ─────────────────────────────────────────────── # OpenAI-protocol chat-completions endpoint used by the algo extractors. [llm] -model = "gpt-4o-mini" +model = "gpt-4.1-mini" api_key = "sk-..." base_url = "https://api.openai.com/v1" diff --git a/docs/api.md b/docs/api.md index d0f05c7..e9a0a53 100644 --- a/docs/api.md +++ b/docs/api.md @@ -30,6 +30,7 @@ business semantics the raw spec does not carry. - [POST /api/v1/memory/flush](#post-apiv1memoryflush) - [POST /api/v1/memory/search](#post-apiv1memorysearch) - [POST /api/v1/memory/get](#post-apiv1memoryget) + - [POST /api/v1/ome/trigger](#post-apiv1ometrigger) - [OpenAPI spec source](#openapi-spec-source) ## Overview @@ -166,26 +167,39 @@ the top level (mirroring the success envelope) alongside a nested { "request_id": "<32-char hex>", "error": { - "code": "HTTP_ERROR", - "message": "Value error, exactly one of user_id / agent_id must be provided", + "code": "NOT_FOUND", + "message": "Document 'abc123' not found", "timestamp": "2026-06-01T12:24:46+00:00", - "path": "/api/v1/memory/search" + "path": "/api/v1/knowledge/documents/abc123" } } ``` -| HTTP | `error.code` | `error.message` | When | +### error.code values + +`error.code` is a machine-readable `ErrorCode` enum. Clients can switch +on this value to decide retry / display / routing behaviour without +parsing the human-readable `message` field. + +| `error.code` | HTTP | Retryable? | When | |---|---|---|---| -| `415 Unsupported Media Type` | `HTTP_ERROR` | the parse-failure reason | `/add` only — a `ContentItem` could not be parsed (unsupported modality for the configured multimodal LLM, or a payload that cannot be fetched / dispatched) | -| `422 Unprocessable Entity` | `HTTP_ERROR` | the **first** validation error (see below) | Request-body validation failure. Also covers `/search` / `/get` filter-DSL compile errors — the compile reason rides in `message` | -| `500 Internal Server Error` | `SYSTEM_ERROR` | `"Internal server error"` (fixed; internal details are logged, never leaked) | Unhandled exception caught by the global handler | +| `NOT_FOUND` | `404` | No | Requested resource does not exist | +| `CONFLICT` | `409` | No | Operation conflicts with existing state (e.g. duplicate document) | +| `INVALID_INPUT` | `422` | No | Request-body validation failure. Also covers `/search` / `/get` filter-DSL compile errors — the compile reason rides in `message` | +| `EXTRACTION_EMPTY` | `422` | No | Document extraction produced no topics (empty or whitespace-only content) | +| `BAD_REQUEST` | `400` | No | Path traversal attempt or other malformed input | +| `UNSUPPORTED_FORMAT` | `415` | No | File format or modality not supported (e.g. unsupported `ContentItem` type, missing `ext` for `base64`) | +| `EXTERNAL_SERVICE_UNAVAILABLE` | `503` | **Yes** | An external service (LLM, embedding, rerank) returned an error or timed out | +| `CAPABILITY_UNAVAILABLE` | `503` | No | A required server-side capability is missing (e.g. `everos[multimodal]` extra not installed, LibreOffice absent) — requires admin action, not retry | +| `CONFIGURATION_ERROR` | `500` | No | A required configuration is missing or invalid (e.g. embedding model not set) | +| `INTERNAL_ERROR` | `500` | No | Unhandled exception (internal details are logged, never leaked) | ### error object | Field | Type | Description | |---|---|---| -| `code` | `string` | `"HTTP_ERROR"` for 4xx (validation / business / `HTTPException`); `"SYSTEM_ERROR"` for 5xx | -| `message` | `string` | Human-readable reason. For `422`, **only the first** validation error is surfaced, formatted `": "` with the leading `body` segment stripped (e.g. `"Field required: messages"`); a model-level validator with no field location surfaces just `""` (e.g. the XOR example above) | +| `code` | `string` | One of the `ErrorCode` values listed above | +| `message` | `string` | Human-readable reason. For `INVALID_INPUT` from request validation, **only the first** validation error is surfaced, formatted `": "` with the leading `body` segment stripped (e.g. `"Field required: messages"`); a model-level validator with no field location surfaces just `""` (e.g. `"Value error, exactly one of user_id / agent_id must be provided"`) | | `timestamp` | `string` | ISO-8601 with timezone offset (display tz) | | `path` | `string` | Request path, e.g. `/api/v1/memory/add` | @@ -193,6 +207,15 @@ the top level (mirroring the success envelope) alongside a nested > returned — only the first error's message. A client that needs the > offending field can read the `` suffix in `message`. +### Search degradation + +When a `/search` call uses `method: "vector"` or `"hybrid"` and the +embedding or rerank service is temporarily unavailable, the server does +**not** return `503`. Instead, it degrades gracefully — the response +suggests an alternative method in the error detail so the client can +retry with `"keyword"` (which requires no embedding). This keeps search +available during transient provider outages. + ## Common types ### MessageItem @@ -793,7 +816,7 @@ attribution, so `session_id` is the only meaningful query dimension. | `sender_id` | `string` | Original sender id from `/add` | | `sender_name` | `string \| null` | Original sender name; `null` if not provided | | `role` | `"user" \| "assistant" \| "tool"` | Original role | -| `content` | `string \| array` | `string` for the single-text shorthand, `array` of opaque content items for the original multimodal payload (mirrors [MessageItem.content](#messageitem)) | +| `content` | `string \| array` | `string` for the single-text shorthand, `array` of opaque content items for the original multimodal payload (mirrors [MessageItem.content](#addmessage)) | | `timestamp` | `string` | ISO-8601 with timezone offset — see [Conventions](#conventions) | | `tool_calls` | `array \| null` | Original tool_calls payload if any | | `tool_call_id` | `string \| null` | Original tool_call_id if any | @@ -1021,6 +1044,41 @@ Response (real capture): } ``` +### POST /api/v1/ome/trigger + +Manually trigger a registered OME strategy. + +#### Request body + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | `string` | yes | — | Strategy name (e.g. `reflect_episodes`) | +| `timeout` | `float` | no | `120.0` | Max seconds to wait for completion | +| `force` | `bool` | no | `false` | Bypass the `enabled` gate in `ome.toml` | + +#### Response body + +`200 OK` returns: + +| Field | Type | Notes | +|---|---|---| +| `status` | `"ok" \| "timeout"` | Whether the strategy completed within the timeout | +| `name` | `string` | Echoes the requested strategy name | + +#### Errors + +- `404` — strategy name not found in the OME registry. + +#### cURL example + +```bash +curl -X POST http://127.0.0.1:8000/api/v1/ome/trigger \ + -H 'Content-Type: application/json' \ + -d '{"name": "reflect_episodes", "force": true}' +``` + +--- + ## OpenAPI spec source This document mirrors the OpenAPI 3.x spec that FastAPI auto-generates diff --git a/docs/architecture.md b/docs/architecture.md index 901227b..1a7ca66 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,7 @@ │ cli + api │ ├──────────────────────────────────────────────────────┤ │ service/ (Application — Use Case orchestration) │ -│ memorize / retrieve / evolve / manage │ +│ memorize / search / get / knowledge │ ├──────────────────────────────────────────────────────┤ │ memory/ (Domain — Business core) │ │ models + extract + search + cascade + prompt_slots │ @@ -20,8 +20,8 @@ └──────────────────────────────────────────────────────┘ Cross-cutting (used by all layers, depends on none): - component/ ← Injectable providers (LLM / Embedding / config / utils) - core/ ← Runtime base (observability / lifespan / context) + component/ ← Injectable providers (LLM / Embedding / parser / config / utils) + core/ ← Runtime base (observability / lifespan / context / errors) config/ ← Configuration data (Settings schema + default.toml) ``` @@ -170,7 +170,7 @@ Three-piece observability: ## Markdown layout ``` -~/.everos/ # memory root (default; EVEROS_MEMORY__ROOT) +~/.everos/ # memory root (default; EVEROS_ROOT) └── // # scope ("default" → default_app/default_project) ├── users// │ ├── user.md # profile (single-file rewrite) @@ -209,6 +209,55 @@ everalgo is: This boundary lets everalgo be reused across product forms (this open-source build, EverOS Cloud, OpenClaw plugins, etc.). +## Error handling architecture + +### Exception hierarchy + +All application exceptions derive from `AppError` (`core/errors.py`), +split into four branches by nature: + +``` +AppError +├── DomainError (client-side / business-rule violations) +│ ├── NotFoundError → 404 NOT_FOUND +│ │ ├── DocumentNotFoundError +│ │ └── TopicNotFoundError +│ ├── ConflictError → 409 CONFLICT +│ │ └── DuplicateDocumentError +│ ├── InvalidInputError → 422 INVALID_INPUT +│ │ ├── ExtractionEmptyError → 422 EXTRACTION_EMPTY +│ │ └── FilterError +│ ├── PathTraversalError → 400 BAD_REQUEST +│ └── UnsupportedModalityError → 415 UNSUPPORTED_FORMAT +├── InfrastructureError (transient, retryable) → 503 +│ ├── StorageError +│ ├── VectorStoreError +│ └── ExternalServiceError +│ ├── LLMServiceError +│ ├── EmbeddingServiceError +│ └── RerankServiceError +├── CapabilityError (permanent, not retryable) → 503 +│ └── MultimodalNotEnabledError +└── ConfigurationError (misconfiguration) → 500 +``` + +### Error propagation strategy + +Exceptions are raised at the layer where the error is detected and +propagate naturally — **service and route layers do not catch-and-wrap**. +The entrypoints layer registers per-type exception handlers +(`entrypoints/api/exception_handlers.py`) via Starlette's MRO dispatch. +Each handler converts the exception into a canonical error envelope with +an `ErrorCode` enum value and the appropriate HTTP status code. + +### Boundary translation + +Third-party exception types are translated at the component boundary to +prevent external types from leaking into upper layers: + +- `everalgo.llm.LLMError` → `LLMServiceError` at `component/parser/_core.py` +- Embedding / rerank provider errors → `EmbeddingServiceError` / `RerankServiceError` at their respective protocol modules + ## Further reading - [docs/overview.md](overview.md) — vision and scope diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index 74e1b03..d5f628f 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -204,7 +204,7 @@ Lives in `LanceDBSettings`; overridable via the `EVEROS_LANCEDB__INDEX_CACHE_SIZE_BYTES` environment variable. This is the only knob that bounds the steady-state file-descriptor count of a long-running EverOS daemon — see -[Recovery paths § FD exhaustion](#fd-exhaustion-os-error-24--emfile) +[Recovery paths § FD exhaustion](#fd-exhaustion-os-error-24-emfile) for why nothing else (prune, rebuild, `drop_index`) helps. Measured cap → FD ceiling (30 add+optimize cycles + 100-query stress diff --git a/docs/cli.md b/docs/cli.md index 3c664e2..e24e6e5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,8 +1,9 @@ # CLI The `everos` command-line entry point covers **setup and operations** — -generate a starter `.env` (`init`), run the HTTP API server (`server -start`), and operate the md → LanceDB index queue (`cascade`). Hot-path +generate starter config files (`init`), run the HTTP API server (`server +start`), inspect effective config (`config show`), and operate the +md → LanceDB index queue (`cascade`). Hot-path business (`/add` `/flush` `/search` `/get`) is the **HTTP API**, not the CLI. @@ -26,13 +27,15 @@ a [Typer](https://typer.tiangolo.com/) app. ``` everos -├── init Generate a starter .env from the packaged template +├── init [--root PATH] [--force] [--print] Generate starter config files (everos.toml + ome.toml) +├── config +│ └── show [--root PATH] Show effective configuration ├── server -│ └── start Start the HTTP API server (uvicorn) -└── cascade Inspect / operate the md → LanceDB sync queue +│ └── start [--host] [--port] [--root] [--reload] [--log-level] Start the HTTP API server (uvicorn) +└── cascade [--root PATH] Inspect / operate the md → LanceDB sync queue ├── status Queue / LSN summary - ├── sync Drain the queue now (force md → LanceDB) - └── fix List failed rows / re-enqueue retryable ones + ├── sync [PATH] Drain the queue now (optional PATH force-enqueues) + └── fix [--apply] List failed rows / re-enqueue retryable ones ``` Each subcommand lives in its own module under @@ -54,7 +57,7 @@ everos server start \ --host 127.0.0.1 \ --port 8000 \ --log-level info \ - --env-file .env + --root ~/.everos ``` | Flag | Env var | Default | @@ -62,7 +65,7 @@ everos server start \ | `--host` | `EVEROS_API__HOST` | `127.0.0.1` (loopback only; binding `0.0.0.0` logs a warning — EverOS ships no auth) | | `--port` | `EVEROS_API__PORT` | `8000` | | `--log-level` | `EVEROS_LOG_LEVEL` | `INFO` | -| `--env-file` | — | searched: `./.env` → `$XDG_CONFIG_HOME/everos/.env` → `~/.everos/.env` | +| `--root` | `EVEROS_ROOT` | `~/.everos` | | `--reload` | — | off (use in development) | Lifespan startup wires the storage backends (SQLite engine + LanceDB @@ -75,7 +78,7 @@ Both CLI and HTTP server read configuration from `pydantic-settings`: | Env var | Settings field | |---|---| -| `EVEROS_MEMORY__ROOT` | `Settings.memory.root` (memory-root path) | +| `EVEROS_ROOT` | memory-root path (default `~/.everos`) | | `EVEROS_MEMORY__TIMEZONE` | `Settings.memory.timezone` (e.g. `Asia/Shanghai`) | | `EVEROS_SQLITE__BUSY_TIMEOUT_MS` | `Settings.sqlite.busy_timeout_ms` | | `EVEROS_LANCEDB__READ_CONSISTENCY_SECONDS` | `Settings.lancedb.read_consistency_seconds` | @@ -98,7 +101,8 @@ everos server start --log-level debug # see all sql / lance traffic | Responsibility | API | CLI | |---|---|---| | Hot-path business (`/add` `/flush` `/search` `/get`) | ✅ | — (HTTP only) | -| Setup (generate `.env`) | — | `everos init` | +| Setup (generate config files) | — | `everos init` | +| Inspect effective config | — | `everos config show` | | Run the server | — | `everos server start` | | Index ops (drain / inspect / fix the cascade queue) | — | `everos cascade {status,sync,fix}` | | Health probe | `GET /health` | (use HTTP) | diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..f0334cf --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,228 @@ +# Configuration + +EverOS uses a two-file TOML configuration system with environment variable +overrides for container deployments. + +## Quick Start + +```bash +# 1. Generate config files in the default root (~/.everos/) +everos init + +# 2. Edit the config — fill in API keys +$EDITOR ~/.everos/everos.toml + +# 3. Start the server +everos server start +``` + +## Configuration Files + +### `everos.toml` — Application Settings + +Located at `/everos.toml`. Controls all application behavior: API +bind address, LLM/embedding/rerank provider credentials, SQLite pragmas, +search strategy, memorize mode, and clustering tunables. + +Generated by `everos init` from the shipped `config/default.toml` +template. Changes require a server restart. + +### `ome.toml` — Strategy Configuration + +Located at `/ome.toml`. Controls the Offline Memory Engine (OME) +strategy scheduling: which strategies are enabled, cron expressions, +gate thresholds, and retry limits. + +Generated by `everos init` from the shipped `config/default_ome.toml` +template. Changes are **hot-reloaded** within ~2 seconds — no server +restart needed. + +### Source Priority + +Settings are resolved in this order (later wins): + +1. `config/default.toml` — shipped defaults (lowest priority) +2. `/everos.toml` — user configuration (optional) +3. `EVEROS_*` environment variables — container/CI overrides +4. Programmatic init args (highest priority; internal use) + +## Memory Root Resolution + +The memory root directory is resolved from three sources (first wins): + +| Source | Example | +|---|---| +| `--root` CLI flag | `everos server start --root /data/everos` | +| `EVEROS_ROOT` env var | `export EVEROS_ROOT=/data/everos` | +| Default | `~/.everos` | + +All CLI commands that interact with storage accept `--root`: + +```bash +everos server start --root /data/everos +everos cascade status --root /data/everos +everos config show --root /data/everos +everos init --root /data/everos +``` + +## Configuration Reference + +### `[memory]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `timezone` | string | `"UTC"` | Effective timezone for date buckets and timestamps. Validated against `zoneinfo.ZoneInfo`. | + +### `[api]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `host` | string | `"127.0.0.1"` | HTTP server bind address. | +| `port` | int | `8000` | HTTP server bind port (1–65535). | + +### `[sqlite]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `journal_mode` | string | `"WAL"` | PRAGMA journal_mode. Options: WAL, DELETE, MEMORY, OFF, TRUNCATE, PERSIST. | +| `synchronous` | string | `"NORMAL"` | PRAGMA synchronous. Options: FULL, NORMAL, OFF, EXTRA. | +| `foreign_keys` | bool | `true` | PRAGMA foreign_keys. | +| `temp_store` | string | `"MEMORY"` | PRAGMA temp_store. Options: DEFAULT, FILE, MEMORY. | +| `busy_timeout_ms` | int | `5000` | PRAGMA busy_timeout in milliseconds. | +| `journal_size_limit_bytes` | int | `67108864` | PRAGMA journal_size_limit (~64 MB). | +| `cache_size_kb` | int | `2048` | PRAGMA cache_size in KB (per connection). | + +### `[lancedb]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `read_consistency_seconds` | float \| null | `null` | Read consistency interval. `null` = no check, `0` = strict, `>0` = eventual. | +| `index_cache_size_bytes` | int | `16777216` | Upper bound on LanceDB index cache (16 MB default). | + +### `[llm]` + +| Field | Type | Default | Required | Description | +|---|---|---|---|---| +| `model` | string | `"gpt-4.1-mini"` | No | LLM model identifier. | +| `api_key` | string | — | **Yes** | API key for the LLM provider. | +| `base_url` | string | — | No | Custom endpoint URL (OpenAI-compatible). | + +### `[multimodal]` + +| Field | Type | Default | Required | Description | +|---|---|---|---|---| +| `model` | string | `"google/gemini-3-flash-preview"` | No | Multimodal parsing model. | +| `api_key` | string | — | **Yes** | API key. | +| `base_url` | string | — | No | Custom endpoint URL. | +| `max_concurrency` | int | `4` | No | Max parallel parsing requests. | + +### `[embedding]` + +| Field | Type | Default | Required | Description | +|---|---|---|---|---| +| `model` | string | — | **Yes** | Embedding model identifier. | +| `api_key` | string | — | **Yes** | API key. | +| `base_url` | string | — | **Yes** | Embedding endpoint URL. | +| `timeout_seconds` | float | `30.0` | No | Request timeout. | +| `max_retries` | int | `3` | No | Retry count on failure. | +| `batch_size` | int | `10` | No | Texts per batch request. | +| `max_concurrent` | int | `5` | No | Max parallel batch requests. | + +### `[rerank]` + +| Field | Type | Default | Required | Description | +|---|---|---|---|---| +| `provider` | string | `"deepinfra"` | No | Rerank provider: `deepinfra` or `vllm`. | +| `model` | string | — | **Yes** | Reranker model identifier. | +| `api_key` | string | — | **Yes** | API key. | +| `base_url` | string | — | **Yes** | Rerank endpoint URL. | +| `timeout_seconds` | float | `30.0` | No | Request timeout. | +| `max_retries` | int | `3` | No | Retry count on failure. | +| `batch_size` | int | `10` | No | Documents per batch. | +| `max_concurrent` | int | `5` | No | Max parallel batch requests. | + +### `[boundary_detection]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `hard_token_limit` | int | `65536` | Max tokens before forced boundary. | +| `hard_msg_limit` | int | `500` | Max messages before forced boundary. | + +### `[memorize]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `mode` | string | `"agent"` | Conversation mode: `chat` (user-memory only) or `agent` (user + agent memory). Requires restart. | +| `session_lock_timeout_seconds` | float | `360.0` | Max wall-clock per memorize() invocation. | + +### `[clustering]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `threshold` | float | `0.65` | Cosine similarity threshold for clustering (0–1). | +| `time_window_days` | float | `7.0` | Max age gap between cluster members. | + +### `[search]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `vector_strategy` | string | `"maxsim_atomic"` | Vector retrieval path: `maxsim_atomic` (finer-grained) or `episode` (legacy). | + +### `[knowledge.search]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `recall_n` | int | `200` | Initial recall pool size. | +| `rerank_n` | int | `50` | Candidates sent to reranker. | +| `mass_top_m` | int | `50` | Top-M for mass scoring. | +| `lambda` | float | `0.1` | Interpolation weight. | +| `top_k_cap` | int | `100` | Hard cap on returned results. | + +## Troubleshooting + +Use `everos config show` to inspect the effective configuration: + +```bash +everos config show +everos config show --root /data/everos +``` + +This prints: +- The resolved root directory +- Which config files were found +- All settings sections with their effective values +- API keys are masked in output + +## Advanced: Container Deployment + +In containerized environments, skip `everos init` and use environment +variables directly: + +```dockerfile +ENV EVEROS_ROOT=/data/everos +ENV EVEROS_LLM__API_KEY=sk-... +ENV EVEROS_LLM__MODEL=gpt-4o +ENV EVEROS_EMBEDDING__MODEL=text-embedding-3-large +ENV EVEROS_EMBEDDING__API_KEY=sk-... +ENV EVEROS_EMBEDDING__BASE_URL=https://api.openai.com/v1 +ENV EVEROS_API__HOST=0.0.0.0 +ENV EVEROS_API__PORT=8000 +``` + +Environment variable naming convention: + +``` +EVEROS_
__ +``` + +- Section and key are uppercased +- Double underscore (`__`) separates section from key +- Nested sections use additional `__` separators + +Examples: + +| TOML | Environment Variable | +|---|---| +| `[llm] api_key = "sk-..."` | `EVEROS_LLM__API_KEY=sk-...` | +| `[sqlite] busy_timeout_ms = 10000` | `EVEROS_SQLITE__BUSY_TIMEOUT_MS=10000` | +| `[memory] timezone = "Asia/Tokyo"` | `EVEROS_MEMORY__TIMEZONE=Asia/Tokyo` | diff --git a/docs/datetime.md b/docs/datetime.md index 5c8bad3..a3c4723 100644 --- a/docs/datetime.md +++ b/docs/datetime.md @@ -66,8 +66,8 @@ All helpers live in [`everos.component.utils.datetime`](../src/everos/component/ | Helper | Behaviour | |---|---| | `get_utc_now() -> datetime` | Current UTC instant, `tzinfo=UTC`. Independent of any setting. Use as `default_factory` on any storage field. | -| `ensure_utc(d) -> datetime` | Naive → attach display tz → convert to UTC. Aware → `astimezone(UTC)`. Use at the storage boundary if you receive a datetime you didn't construct. | -| `UtcDatetime` | `Annotated[datetime, AfterValidator(ensure_utc)]`. Apply to any SQLite field. Pydantic auto-runs validation on both INSERT defaults and read-back rows. | +| `ensure_utc(d) -> datetime \| None` | Naive → **assume UTC** (attach `tzinfo=UTC`; no display-tz step). Aware → `astimezone(UTC)`. `None` → `None`. Use at the storage boundary; for caller input that may be naive-in-display-tz, funnel through `from_iso_format` first. | +| `UtcDatetime` | `Annotated[datetime, AfterValidator(ensure_utc)]` — normalises on **construction**. The ORM hydrate path bypasses it; UTC-on-read is handled by the `UtcDateTimeColumn` SQL type (below). | ### Display rail @@ -107,8 +107,9 @@ SQLModel's ORM hydrate path (rows from `select(...)`) **bypasses** the Pydantic validator — SQLAlchemy assigns column values straight to instance attributes. To close that gap, [core/persistence/sqlite/base.py](../src/everos/core/persistence/sqlite/base.py) -registers a SQLAlchemy `load` event listener that re-attaches -`tzinfo=UTC` to every `UtcDatetime` column after hydrate. Net effect: +defines a `TypeDecorator` column type, `UtcDateTimeColumn`, whose +`process_result_value` re-attaches `tzinfo=UTC` on read (and +`process_bind_param` normalises to UTC on write). Net effect: **callers never see a naive datetime from a SQLite repo**, whatever the code path. @@ -154,8 +155,8 @@ event payload. | Backend | Defense | Where | |---|---|---| -| **SQLite** | SQLAlchemy `load` event listener on `BaseTable` re-attaches `tzinfo=UTC` after every ORM hydrate | [core/persistence/sqlite/base.py](../src/everos/core/persistence/sqlite/base.py) | -| **LanceDB** | `BaseLanceTable.to_arrow_schema()` rewrites `UTC_DATETIME_FIELDS` columns to `timestamp[us, tz=UTC]`; PyArrow handles UTC end-to-end | [core/persistence/lancedb/base.py](../src/everos/core/persistence/lancedb/base.py) | +| **SQLite** | `UtcDateTimeColumn` (`TypeDecorator`) re-attaches `tzinfo=UTC` on read (`process_result_value`) and normalises to UTC on write (`process_bind_param`) | [core/persistence/sqlite/base.py](../src/everos/core/persistence/sqlite/base.py) | +| **LanceDB** | `BaseLanceTable.to_arrow_schema()` rewrites **every** `timestamp[us]` column to `timestamp[us, tz=UTC]`; PyArrow handles UTC end-to-end | [core/persistence/lancedb/base.py](../src/everos/core/persistence/lancedb/base.py) | | **CI gate** | `scripts/check_datetime_discipline.py` fails the build on any code that bypasses `component/utils/datetime` | wired into `make lint` | These defenses replace what used to be an "every consumer must call @@ -176,9 +177,10 @@ User input (any zone) ▼ ┌────────────────┬────────────────┐ │ SQLite │ LanceDB │ -│ (UtcDatetime │ (Arrow │ -│ re-attaches │ stripped to │ -│ UTC on read) │ UTC bytes) │ +│ (UtcDateTime- │ (Arrow │ +│ Column re- │ stripped to │ +│ attaches UTC │ UTC bytes) │ +│ on read) │ │ └────────────────┴────────────────┘ │ ▼ diff --git a/docs/engineering.md b/docs/engineering.md index d56ae16..112041e 100644 --- a/docs/engineering.md +++ b/docs/engineering.md @@ -4,7 +4,6 @@ > hard coding constraints live in [../.claude/rules/](../.claude/rules/). > This document covers the surrounding tooling, configuration, and processes > — what we adopted, what role each piece plays, and how they fit together. -> CI runs on GitHub Actions; all checks are invoked through the `Makefile`. --- @@ -56,7 +55,7 @@ Reasons this is documented separately: │ │ .claude/ │ │ │ │ ├── CLAUDE.md subdir context (optional) │ │ │ │ ├── rules/ (10) path-scoped hard coding rules │ │ -│ │ ├── skills/ (3) slash command workflows │ │ +│ │ ├── skills/ (5) slash command workflows │ │ │ │ └── settings.json permissions allowlist │ │ │ │ │ │ │ └────────────────────────────────────────────────────────────┘ │ @@ -69,13 +68,11 @@ Reasons this is documented separately: │ │ ├ check-yaml / check-toml │ │ │ │ ├ check-added-large-files (≥1MB warn) │ │ │ │ ├ detect-private-key │ │ -│ │ ├ no committed images/videos/assets │ │ │ │ └ gitlint (commit-msg stage) │ │ │ │ │ │ │ │ ruff lint + format │ │ │ │ (replaces black / isort / flake8) │ │ │ │ import-linter DDD layer-direction enforcement │ │ -│ │ repo asset gate blocks images/videos/assets in git │ │ │ │ pytest unit / integration │ │ │ │ │ │ │ └────────────────────────────────────────────────────────────┘ │ @@ -88,26 +85,25 @@ Reasons this is documented separately: │ │ uv.lock checked in; CI uses --frozen │ │ │ │ hatchling wheel build backend │ │ │ │ Makefile unified entry; CI calls it │ │ -│ │ src/everos/templates/env.template │ │ -│ │ environment variable template │ │ +│ │ config/default.toml default settings (shipped) │ │ │ │ │ │ │ └────────────────────────────────────────────────────────────┘ │ │ │ -│ ┌─ CI/CD (GitHub Actions) ───────────────────────────────────┐ │ +│ ┌─ Dual-platform CI/CD ──────────────────────────────────────┐ │ │ │ │ │ -│ │ CI: .github/workflows/ci.yml lint / test / integ │ │ -│ │ / package build │ │ -│ │ Docs: .github/workflows/docs.yml Markdown + YAML check │ │ -│ │ Gates invoke Makefile targets; the Makefile is the │ │ +│ │ Primary: GitLab CI .gitlab-ci.yml │ │ +│ │ Mirror: GitHub Actions .github/workflows/ci.yml │ │ +│ │ Both invoke Makefile targets; the Makefile is the │ │ │ │ single source of truth for commands. │ │ │ │ │ │ │ └────────────────────────────────────────────────────────────┘ │ │ │ │ ┌─ Collaboration workflow ───────────────────────────────────┐ │ │ │ │ │ -│ │ Branch model: protected main + short-lived PR branches │ │ -│ │ PR template: .github/PULL_REQUEST_TEMPLATE.md │ │ -│ │ ISSUE_TEMPLATE: bug / feature / use-case / docs / config │ │ +│ │ Branch model: dev / master (GitFlow Lite) │ │ +│ │ PR / MR templates: same template across platforms │ │ +│ │ CODEOWNERS: by DDD layer ownership │ │ +│ │ ISSUE_TEMPLATE: bug / feature / config │ │ │ │ CONTRIBUTING.md: contributor onboarding │ │ │ │ │ │ │ └────────────────────────────────────────────────────────────┘ │ @@ -158,13 +154,15 @@ session start (no manual import): load (~1.5–2K tokens); the rest load on demand when Claude Code reads a matching `.py` file. -### 3.3 Skills (3 slash commands) +### 3.3 Skills (5 slash commands) | Command | Purpose | When to use | |---|---|---| -| `/commit` | Generate a Conventional Commits message | After a focused change, ready to commit | -| `/new-branch` | Create branch from protected main | Starting a new feat / fix / ci branch | -| `/pr` | Open a GitHub PR with the repo template | Ready to merge | +| `/commit` | Generate Gitmoji-format commit message | After a focused change, ready to commit | +| `/new-branch` | Create branch under dev/master strategy | Starting a new feat / fix / hotfix | +| `/pr` | Create GitLab MR or GitHub PR with template | Ready to merge | +| `/add-memory-kind` | Scaffold a new business memory kind end-to-end | Adding a new memory type (md + sqlite + lancedb) | +| `/release` | Cut a PyPI release (rc / stable) | Shipping a version | Skills and rules use **independent loading mechanisms**: rules auto-load into the system prompt, skills only trigger when the user types `/`. @@ -206,26 +204,25 @@ Stage 2: pre-commit (triggered by `git commit`) ├ trailing-whitespace, end-of-file-fixer ├ check-yaml, check-toml ├ check-added-large-files (≥1MB) + ├ check-merge-conflict ├ detect-private-key - ├ no-repo-assets (rejects images/videos/assets in git) └ gitlint (commit-msg stage; rejects malformed messages) │ ▼ Stage 3: local `make ci` (manual, before push) - ├ make lint (ruff + import-linter + repo hygiene gates) + ├ make lint (ruff + format-check + import-linter + datetime + openapi-drift) ├ make test (pytest tests/unit) - ├ make integration (pytest tests/integration) - └ make package (sdist/wheel build + import smoke test) + └ make integration (pytest tests/integration) │ ▼ -Stage 4: CI (GitHub Actions, push + PR triggered) - └ re-runs the same `make lint / test / integration / package` targets +Stage 4: CI (PR triggered, GitLab + GitHub) + └ re-runs the same `make lint / test / integration` targets │ ▼ -Stage 5: PR review +Stage 5: PR / MR review ├ ≥ 1 approval └ all threads resolved + all CI green ``` @@ -260,12 +257,13 @@ everos = "everos.entrypoints.cli.main:app" # exposes CLI command [tool.ruff] # code style [tool.pytest.ini_options] # tests -[tool.coverage.run] # coverage config (gate lives in `make cov`) +[tool.coverage.run] # coverage (unit+integration; make cov gates at 80%) [tool.importlinter] # dependency direction [dependency-groups] dev = ["ruff", "pytest", "pytest-asyncio", "pytest-cov", - "import-linter", "pre-commit", "ipdb"] + "pytest-rerunfailures", "import-linter", "pre-commit", + "pyinstrument", "ipdb"] ``` **Single-file principle**: configuration that used to live in `pylintrc`, @@ -277,60 +275,68 @@ dev = ["ruff", "pytest", "pytest-asyncio", "pytest-cov", make help list all targets make install uv sync --frozen make format ruff fix + format -make lint ruff + import-linter + repo asset/media + datetime discipline + openapi drift +make lint ruff + format --check + import-linter + datetime + openapi-drift make test pytest tests/unit make integration pytest tests/integration -make package build sdist/wheel + smoke-test wheel import -make cov pytest unit + integration, coverage gate (fail under 80%) -make ci lint + test + integration + package +make cov pytest unit+integration with coverage, gate at 80% +make ci lint + test + integration ← CI invokes these targets make clean clear caches ``` -**Single source of truth**: CI only invokes `make `, so local and CI -run identical commands and cannot drift. +> Plus `install-deps` (CI's `uv sync --frozen`), `openapi` (regenerate +> `docs/openapi.json`), `check-openapi` / `check-datetime` (the lint +> sub-gates), and `check-cjk` (advisory) — see the `Makefile` for the +> full list. -### 5.3 env.template (slimmed down) +**Single source of truth**: CI configuration only invokes `make `, +preventing drift between GitHub and GitLab. Local and CI run identical +commands. -The template lives at `src/everos/templates/env.template` (bundled -inside the wheel as package data, copied to `./.env` via `everos init`). -It groups settings by provider, each block sharing the OpenAI-protocol -`MODEL` / `API_KEY` / `BASE_URL` triple: +### 5.3 Configuration model + +Settings are loaded in ascending priority: + +1. `config/default.toml` (shipped with the package; lowest priority) +2. `/everos.toml` (user config; optional) +3. `EVEROS_*` environment variables (highest priority) + +`everos init [--root PATH]` generates starter config files +(`everos.toml` + `ome.toml`) in the memory root. `everos.toml` holds +all config (API keys, model, storage); override individual fields with +`EVEROS_*` env vars. Inspect the effective config with +`everos config show [--root PATH]`. + +Key `EVEROS_*` env vars: ``` -EVEROS_LLM__* # text model (model / api_key / base_url) -EVEROS_MULTIMODAL__* # vision model for image/office inputs -EVEROS_EMBEDDING__* # embedding model (vector index) -EVEROS_RERANK__* # cross-encoder reranker -EVEROS_MEMORY__ROOT # memory-root (md files + .index/{sqlite,lancedb}/) -EVEROS_LOG_LEVEL # DEBUG | INFO | WARNING | ERROR -EVEROS_LOG_FORMAT # json | text -TZ # display timezone (storage is always UTC) +EVEROS_LLM__MODEL # model name (provider-agnostic) +EVEROS_LLM__API_KEY # any OpenAI-protocol API key +EVEROS_LLM__BASE_URL # optional: custom endpoint (Ollama bridge etc.) +EVEROS_ROOT # memory-root (default ~/.everos) +EVEROS_LOG_LEVEL +TZ ``` -Every key has a sensible default except the `API_KEY` fields, which you fill in. - --- -## 6. CI/CD (GitHub Actions) +## 6. Dual-platform CI/CD -### 6.1 Strategy +### 6.1 Dual-platform strategy ``` ┌──────────────────────────────────────────────────────────┐ │ │ -│ GitHub Actions (.github/workflows/) │ -│ ci.yml push (main) + PR │ -│ ├ lint make lint │ -│ ├ unit tests make test │ -│ ├ integration tests make integration │ -│ └ package build make package │ -│ docs.yml Markdown link check + issue-template YAML │ -│ └ make docs-check │ -│ commits.yml Conventional Commit subject check │ -│ └ make check-commits │ +│ Primary: GitLab CI (.gitlab-ci.yml) │ +│ ├ internal team dev stages: lint / test │ +│ ├ MR triggered │ +│ └ uv cache (keyed by uv.lock) │ +│ │ +│ Mirror: GitHub Actions (.github/workflows/ci.yml) │ +│ ├ public OSS mirror same make targets │ +│ ├ push + PR triggered │ +│ └ astral-sh/setup-uv@v3 │ │ │ │ Consistency: │ -│ ├ astral-sh/setup-uv (cache keyed by uv.lock) │ │ ├ Makefile is the single source of CI commands │ │ └ pre-commit runs locally first to reduce CI churn │ │ │ @@ -339,77 +345,102 @@ Every key has a sensible default except the `API_KEY` fields, which you fill in. ### 6.2 CI checklist -| Check | Tool | Failure condition | -|---|---|---| -| Lint | `make lint` (ruff check + ruff format --check) | any error | -| Layer direction | `make lint` (lint-imports inside) | layer violation | -| Repository media | `make lint` (check_repo_assets.py) | images/videos/assets committed | -| Datetime discipline | `make lint` (check_datetime_discipline.py) | bypasses helper module | -| OpenAPI drift | `make lint` (dump_openapi.py --check) | schema ≠ committed openapi.json | -| Unit | `make test` (pytest tests/unit) | any failure | -| Integration | `make integration` (pytest tests/integration) | any failure | -| Package build | `make package` (sdist/wheel + import smoke test) | build or import failure | -| Commit message | `Commit lint` workflow | non-Conventional Commit subject | +| Check | Tool | Platform | Failure condition | +|---|---|---|---| +| Lint | `make lint` (ruff + format-check + import-linter + datetime + openapi-drift) | both | any error | +| Layer direction | `make lint` (lint-imports inside) | both | layer violation | +| Unit | `make test` (pytest tests/unit) | both | any failure | +| Integration | `make integration` (pytest tests/integration) | both | any failure (PR + master/dev push only) | -Integration tests run with a `FakeLLMClient` — no live credentials are needed in CI. -Commit message format is enforced locally via `gitlint` in the `commit-msg` -pre-commit stage and remotely via the `Commit lint` workflow. +Commit message format is enforced **locally** via `gitlint` in the +`commit-msg` pre-commit stage; it does not run in CI. ### 6.3 Branch protection -| Branch | Rule | -|---|---| -| **main** | branch protection: PR + two reviews + green required checks; no direct push | -| feat / fix / docs / ci | contributor branches; merge through PR | +| Branch | GitLab rule | GitHub rule | +|---|---|---| +| **master** | no direct push; MR + 1 approval + green pipeline | branch protection + 1 review + status checks | +| **dev** | same as above | same as above | +| feat / fix / hotfix | free push; rebase parent before merge | same | --- ## 7. Collaboration workflow -### 7.1 Branch model - -EverOS uses a simple protected-main model after the 1.0 history reset: +### 7.1 Branch model (GitFlow Lite) ``` -main ●────●────●────●────► protected, releasable - ▲ ▲ ▲ - │ │ └─ PR from ci/* - │ └────── PR from fix/* - └─────────── PR from feat/* + v0.1 v0.2 v1.0 + ▲ ▲ ▲ + │ release PR │ release PR │ release PR + │ (dev→master+tag) │ (dev→master+tag) │ (dev→master+tag) +master ●──────────────────────●─────────────●──────────────────●──────────────────────────────────●────► stable / released + │ ▲ │ │ + │ │ merge hotfix │ │ + │ │ │ │ + │ ●──●──┘ │ │ + │ │ hotfix branch │ │ + │ │ (cut from master) │ │ + │ │ │ │ + │ ▼ sync to dev │ │ + │ │ │ │ +dev ●──●──●──●──●──●──●──●──●─●──●──●─●──●──●──●──●──●──●──●──●─●──●──●──●──●──●──●──●──●──●──●──●─────► integration + ▲ ↑ ↑ ↑ + │ release point release point release point + feat/A (dev HEAD → (dev HEAD → (dev HEAD → + ●──●──● master + v0.1) master + v0.2) master + v1.0) + + + feat/* : cut from dev → PR → merge into dev + hotfix/* : cut from master → merge into master + sync into dev (double merge) + release : dev → master + tag on master (no separate release branch) + + Vertical │ in the diagram = "dev HEAD merged into master via release PR + v0.x tag" ``` -All work starts from `main`, lands through a pull request, and requires green -checks. Force-pushing `main` is reserved only for repository recovery work. +Details in [../.claude/skills/new-branch/SKILL.md](../.claude/skills/new-branch/SKILL.md). -### 7.2 PR template +### 7.2 PR / MR template (shared across platforms) -A single PR template at [`.github/PULL_REQUEST_TEMPLATE.md`](../.github/PULL_REQUEST_TEMPLATE.md) -with five sections: **Summary / Area / Verification / Checklist / Notes for -Reviewers**. The `/pr` skill fills it in (see -[../.claude/skills/pr/SKILL.md](../.claude/skills/pr/SKILL.md)). +Six sections: changes / target branch / scope / API impact / tests / +checklist. -### 7.3 Commit convention (Conventional Commits) +File locations: -Format: `[(scope)][!]: ` per -[Conventional Commits](https://www.conventionalcommits.org). +- GitLab: `.gitlab/merge_request_templates/default.md` +- GitHub: `.github/PULL_REQUEST_TEMPLATE.md` + +### 7.3 CODEOWNERS (by DDD layer) ``` -feat: new feature -fix: bug fix -refactor: restructuring (no behavior change) -test: add / update tests -docs: documentation -style: formatting -perf: performance optimization -chore: configuration / build / tooling -build: build system or dependencies -ci: CI configuration -revert: revert a previous commit +/src/everos/memory/ @chandler.zhang @libin.zhang001 +/src/everos/infra/ @chandler.zhang @yeanhua +/src/everos/component/ @chandler.zhang +/src/everos/core/ @chandler.zhang +/src/everos/service/ @chandler.zhang @libin.zhang001 +/src/everos/entrypoints/ @chandler.zhang +/.claude/ @chandler.zhang +/.gitlab-ci.yml @chandler.zhang @jianhua.yao ``` -`gitlint` enforces the format locally via its `contrib-title-conventional-commits` -rule in the commit-msg pre-commit stage. GitHub Actions runs the same policy on -pushes to `main` and pull requests. See +At least one owner per directory; two owners for critical modules. Edits +auto-mention the corresponding owners. + +### 7.4 Commit convention (Gitmoji) + +``` +✨ feat: new feature +🐛 fix: bug fix +♻️ refactor: refactoring (no behavior change) +✅ test: add / update tests +📝 docs: documentation +🎨 style: formatting +⚡️ perf: performance optimization +🔧 chore: configuration / build +🚧 wip: work in progress (must not land on master) +``` + +`gitlint` enforces format **locally** (commit-msg pre-commit stage). See [../.claude/skills/commit/SKILL.md](../.claude/skills/commit/SKILL.md). --- @@ -418,11 +449,9 @@ pushes to `main` and pull requests. See ``` .github/ISSUE_TEMPLATE/ -├── bug_report.yml structured bug report (form) -├── feature_request.yml feature proposal (form) -├── use_case.yml share a use case / integration -├── docs.yml documentation issue -└── config.yml disable blank issues + community links +├── bug_report.md software deps: lancedb / sqlite / ruff +├── feature_request.md generic template +└── config.yml disable blank issue + Discord / Discussions links CONTRIBUTING.md contributor onboarding: setup / code style / branch / commit / PR / testing @@ -441,7 +470,7 @@ CONTRIBUTING.md contributor onboarding: setup / code style / │ │ │ context │ │ Team rules │ /.claude/rules/ (10) │ cc unaware │ │ │ │ of conv. │ -│ Team skills │ /.claude/skills/ (3) │ no slash │ +│ Team skills │ /.claude/skills/ (5) │ no slash │ │ │ │ workflows │ │ Permissions │ /.claude/settings.json │ cc prompts │ │ │ │ on each op │ @@ -452,15 +481,18 @@ CONTRIBUTING.md contributor onboarding: setup / code style / │ │ │ entry │ │ pre-commit │ /.pre-commit-config.yaml │ no local │ │ │ │ gate │ -│ env template │ /src/everos/templates/env.template │ newcomers │ -│ │ │ lost on env│ +│ default config │ /src/everos/config/default.toml │ newcomers │ +│ │ │ lost on cfg│ ├─────────────────────┼──────────────────────────────────────┼─────────────┤ -│ CI │ /.github/workflows/ci.yml │ PR cannot │ +│ GitLab CI │ /.gitlab-ci.yml │ MR cannot │ │ │ │ merge │ -│ Docs CI │ /.github/workflows/docs.yml │ broken │ -│ │ │ doc links │ -│ PR template │ /.github/PULL_REQUEST_TEMPLATE.md │ no PR temp │ -│ Issue templates │ /.github/ISSUE_TEMPLATE/ (5) │ scattered │ +│ GitHub Actions │ /.github/workflows/ci.yml │ PR cannot │ +│ │ │ merge │ +│ CODEOWNERS │ /.gitlab/CODEOWNERS │ no auto │ +│ │ │ reviewer │ +│ GitLab MR template │ /.gitlab/merge_request_templates/ │ no MR temp │ +│ GitHub PR template │ /.github/PULL_REQUEST_TEMPLATE.md │ no PR temp │ +│ Issue templates │ /.github/ISSUE_TEMPLATE/ (3) │ scattered │ │ CONTRIBUTING │ /CONTRIBUTING.md │ contrib. │ │ │ │ confused │ └─────────────────────┴──────────────────────────────────────┴─────────────┘ @@ -472,19 +504,22 @@ CONTRIBUTING.md contributor onboarding: setup / code style / ``` Near-term + ☑ Coverage threshold — `make cov` now gates at 80% (unit + integration) □ /new-module skill: scaffold a subpackage that complies with rules + □ /run-eval skill: run behavior-consistency eval □ ruff rule sets: add D (docstring), ANN (annotations) - □ Static type checking (pyright or mypy) once hot paths stabilize -Mid-term +Mid-term (before v0.5) + □ Type checking re-introduction (pyright or mypy) once hot paths stabilize □ release-please / Conventional Commits → automated changelog - □ Automated PyPI wheel upload on tag - □ Multi-Python version matrix (3.12 / 3.13) + □ pre-commit autoupdate cadence □ Performance benchmark CI with historical comparison -Long-term +Long-term (after v1.0) + □ /security-review skill: automated security review □ Mutation testing (mutmut) - □ Coverage ratchet (raise the 80% gate as the suite matures) + □ Multi-Python version matrix (3.12 / 3.13) + □ Automated PyPI wheel upload ``` --- @@ -500,7 +535,8 @@ Long-term │ coding rules + │ │ quality gates (pre-commit + CI) + │ │ automation (Makefile + skills) + │ -│ collaboration (branch + PR) + │ +│ collaboration (branch + PR + │ +│ CODEOWNERS) + │ │ knowledge base (CLAUDE.md + │ │ rules + docs) │ │ │ @@ -518,10 +554,10 @@ Old project vs. new project after this rewrite: | Config files | pyproject + pylintrc + pyrightconfig + pytest.ini | unified pyproject.toml | | pre-commit | basic | adds gitlint commit-msg + import / yaml / private-key checks | | Layer direction | not enforced | import-linter enforced in CI | -| Commit format | freeform | gitlint pre-commit hook (Conventional Commits) | +| Commit format | freeform | gitlint pre-commit hook (Gitmoji) | | Claude Code integration | partial rules | rules + skills + settings (full) | -| CI platform | ad hoc | GitHub Actions calling Makefile targets | -| Tests | basic | unit + integration + e2e + coverage report | +| CI platform | GitLab only | GitLab + GitHub mirror, both calling Makefile | +| Tests | basic | unit + integration + golden + coverage report | These are not perfectionism — they are baseline requirements for **multi-person collaboration, long-term maintenance, and sustainable @@ -542,5 +578,7 @@ evolution**. - gitlint: [jorisroovers.com/gitlint](https://jorisroovers.com/gitlint/) - uv: [docs.astral.sh/uv](https://docs.astral.sh/uv/) - pre-commit: [pre-commit.com](https://pre-commit.com/) -- Conventional Commits: [conventionalcommits.org](https://www.conventionalcommits.org/) +- Gitmoji: [gitmoji.dev](https://gitmoji.dev/) +- GitLab CI: [docs.gitlab.com/ee/ci](https://docs.gitlab.com/ee/ci/) - GitHub Actions: [docs.github.com/en/actions](https://docs.github.com/en/actions) +- CODEOWNERS: [docs.gitlab.com/ee/user/project/codeowners](https://docs.gitlab.com/ee/user/project/codeowners/) diff --git a/docs/how-memory-works.md b/docs/how-memory-works.md index 5d21ea4..da6969d 100644 --- a/docs/how-memory-works.md +++ b/docs/how-memory-works.md @@ -41,9 +41,8 @@ Three embedded pieces, each owning what it is best at. Markdown is the ## Storage paths The default memory root is **`~/.everos/`** (override with -`EVEROS_MEMORY__ROOT` or `[memory] root` in TOML). Configuration (the -`.env` file) is separate from data (the memory root): the server searches -`./.env` → `$XDG_CONFIG_HOME/everos/.env` → `~/.everos/.env`. +`EVEROS_ROOT` env var or `--root` on the CLI). Configuration lives +inside the memory root as `everos.toml` (generated by `everos init`). Memory is partitioned by **`/`** *before* the user-visible directories, so different `(app, project)` spaces never share @@ -52,7 +51,7 @@ a directory or cross in search. The reserved id `"default"` materialises as visually distinct from a user-named one). ``` -~/.everos/ ← memory root (EVEROS_MEMORY__ROOT) +~/.everos/ ← memory root (EVEROS_ROOT) ├── default_app/ ← ("default" → default_app) │ └── default_project/ ← ("default" → default_project) │ ├── users/ ← user-visible (source of truth) @@ -97,8 +96,8 @@ visually distinct from a user-named one). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py); every path above is a property on it. `MemoryRoot.ensure()` creates the runtime dirs -(`.index/{sqlite,lancedb}/`, `.tmp/`) and copies the OME template to -`ome.toml`; user-visible dirs appear on first write. +(`.index/{sqlite,lancedb}/`, `.tmp/`); user-visible dirs appear on first +write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. ## How a memory is born @@ -207,6 +206,10 @@ and write their markdown when ready: - `extract_agent_case` — a reusable agent trajectory (only when the cell is substantive enough; thin trajectories are skipped by design) - `extract_agent_skill` — clusters related cases into a named skill +- `reflect_episodes` (cron, default off) — offline memory consolidation. + Merges fragmented episodes within a cluster into a single coherent + narrative, re-extracts atomic facts, and deprecates the originals. + Enable via `ome.toml`. Strategies are configurable without a code change via **`ome.toml`** at the memory root (hot-reloaded within ~2 s). Example — turn two off: @@ -224,6 +227,20 @@ and its scheduler jobstore in `.index/sqlite/ome.aps.db` (split so the sync APScheduler writer and the async OME writer never contend for one file lock). +### Reflection (offline consolidation) + +The `reflect_episodes` strategy runs on a cron schedule (default +`0 2 * * 1` — every Monday at 02:00, disabled by default). It selects clusters with multiple +members, calls the LLM to merge their episodes into a single narrative, +writes the merged episode to markdown, re-extracts atomic facts, and +deprecates the originals via `deprecated_by`. The merged episode uses +`parent_type=cluster` and `session_id=None`. Enable it in `ome.toml`: + +```toml +[strategies.reflect_episodes] +enabled = true +``` + !!! tip "Implication for clients" After `/flush` returns `extracted`, the **episode** is queryable soon (once cascade indexes it), but **atomic facts / profile / agent cases** @@ -270,7 +287,7 @@ The CLI ([cli.md](cli.md)) is intentionally small: | Command | What it does | |---|---| -| `everos init` | write a starter `.env` | +| `everos init` | generate starter config files (`everos.toml` + `ome.toml`) | | `everos server start` | run the HTTP API (cascade + OME start with it) | | `everos cascade status` | queue / LSN summary | | `everos cascade sync` | drain the cascade queue now (force md → LanceDB) | diff --git a/docs/index.md b/docs/index.md index 0751fbc..dbb21a5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,10 +12,11 @@ already know what you want to do and need to know exactly how. | Doc | Purpose | |---|---| | [api.md](api.md) | HTTP API v1 reference — endpoints, request / response, error contracts | +| [knowledge.md](knowledge.md) | Knowledge base module — upload, search, taxonomy, storage layout | +| [reflection.md](reflection.md) | Reflection — offline memory consolidation: enable, schedule, storage, triggering | | [cli.md](cli.md) | `everos` CLI subcommands + env var conventions | | [storage_layout.md](storage_layout.md) | Memory-root tree + frontmatter chassis + EntryId encoding | | [prompt_slots.md](prompt_slots.md) | YamlConfigLoader + three-layer prompt override | -| [migration-to-1.0.0.md](migration-to-1.0.0.md) | Legacy API and infrastructure migration notes for EverOS 1.0.0 | ## Explanation @@ -37,7 +38,6 @@ specific thing (drain a queue, recover from a stuck row, etc.). | Doc | Purpose | |---|---| | [cascade_runbook.md](cascade_runbook.md) | Cascade subsystem ops — drain queue, recover stuck rows | -| [multimodal.md](multimodal.md) | Ingest images, PDFs, audio, and office docs into memory | ## Engineering / Internal @@ -54,7 +54,6 @@ Top-level project files live next to the repo root: - [README.md](../README.md) — quick start & feature overview - [QUICKSTART.md](../QUICKSTART.md) — 5-minute walkthrough (install → service → search) -- [use-cases.md](use-cases.md) — full use-case gallery and integration examples - [CONTRIBUTING.md](../CONTRIBUTING.md) — how to contribute (issue-only model) - [CHANGELOG.md](../CHANGELOG.md) — release notes - [SECURITY.md](../SECURITY.md) — security policy & private vulnerability reporting diff --git a/docs/knowledge.md b/docs/knowledge.md new file mode 100644 index 0000000..463479f --- /dev/null +++ b/docs/knowledge.md @@ -0,0 +1,668 @@ +# Knowledge Base + +The Knowledge module turns unstructured documents (Markdown, PDF, DOCX, …) +into a searchable topic library. Upload a file, and EverOS extracts a +structured topic tree via LLM, indexes it for keyword + vector search, +and keeps the original file for reference. + +## Quick start + +> The examples below assume EverOS is running on the default port 8000. +> See [README](../README.md) or [QUICKSTART](../QUICKSTART.md) to start +> the server. + +```bash +# Upload a document +curl -s -X POST http://localhost:8000/api/v1/knowledge/documents \ + -F "file=@my-report.pdf" \ + -F "title=Q1 Engineering Report" \ + | jq .data +# → { "doc_id": "d_a1b2c3d4e5f6", "category_id": "Technology", "topic_count": 8, ... } + +# Search +curl -s -X POST http://localhost:8000/api/v1/knowledge/search \ + -H "Content-Type: application/json" \ + -d '{"query": "performance bottleneck", "method": "hybrid"}' \ + | jq '.data.hits[:3] | .[] | {topic_name, score}' +``` + +## Three-tier hierarchy + +Knowledge is organized into three levels, from broadest to most granular: + +``` +L0 Category ← taxonomy bucket (e.g., "Technology", "Finance") +L1 Document ← one uploaded file = one document +L2 Topic ← LLM-extracted section with content +``` + +Each level corresponds to a different granularity of API: + +| Level | Endpoint | Returns | +|-------|----------|---------| +| L0 | `GET /categories` | `category_id`, `description`, `document_count` | +| L1 | `GET /documents` | `doc_id`, `title`, `category_id`, `topic_count`, `created_at` | +| L1 (detail) | `GET /documents/{id}` | Full detail: summary, source info, original file path, topic list | +| L2 | `GET /topics/{id}` | Full topic: content, labels, tree position | + +## Storage layout + +Every document is a self-contained directory. Markdown files are the +single source of truth; SQLite and LanceDB are derived indexes built +automatically by the cascade daemon. + +``` +~/.everos///knowledge/ +├── .taxonomy.md ← category definitions (YAML) +├── Technology/ +│ └── Q1_Engineering_Report_d_a1b2c3d4e5f6/ +│ ├── index.md ← document metadata + summary +│ ├── 1_Performance_Analysis.md ← topic with full content +│ ├── 2_Infrastructure_Costs.md +│ ├── 3_Team_Velocity.md +│ └── _original/ ← original uploaded file +│ └── my-report.pdf +└── Finance/ + └── Budget_Review_d_f6e5d4c3b2a1/ + ├── index.md + ├── 1_Revenue.md + └── _original/ + └── budget.xlsx +``` + +### Storage roles + +``` +Markdown (source of truth) + SQLite (structured state) + LanceDB (vector + BM25 index) +``` + +| Store | What it holds | Role | +|-------|---------------|------| +| Markdown | Document metadata, summaries, topic content, original files | Single source of truth; human-readable and editable | +| SQLite | Document rows, topic rows (with content), change queue | Structured queries, paginated lists, count aggregation | +| LanceDB | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | + +Even if SQLite and LanceDB data is corrupted, as long as the Markdown +files are intact, the indexes can be fully rebuilt via the cascade daemon. + +### Markdown format + +**index.md** (document root): + +```yaml +--- +type: knowledge_document +id: d_a1b2c3d4e5f6 +doc_id: d_a1b2c3d4e5f6 +category_id: Technology +title: Q1 Engineering Report +schema_version: 1 +source_name: my-report.pdf +source_type: file +--- +This report covers Q1 engineering outcomes including performance +analysis, infrastructure costs, and team velocity metrics. +``` + +The body is an LLM-generated summary of the entire document. + +**Topic files** (e.g., `1_Performance_Analysis.md`): + +```yaml +--- +type: knowledge_topic +id: d_a1b2c3d4e5f6_1 +node_id: d_a1b2c3d4e5f6_1 +doc_id: d_a1b2c3d4e5f6 +category_id: Technology +topic_index: 1 +topic_name: Performance Analysis +topic_path: Q1 Engineering Report > Performance Analysis +summary: Analysis of API latency, database query times, and caching hit rates. +depth: 1 +parent_node_id: d_a1b2c3d4e5f6_0 +children_node_ids: [] +content_labels: ["performance", "latency", "caching"] +schema_version: 1 +--- +The P99 API latency dropped from 450ms to 120ms after the Redis +caching layer was deployed in week 6. Database query times improved +by 40% following the index optimization sprint... +``` + +The body is the full extracted content for this topic. + +> The taxonomy file uses `kind` (not `type`) in its frontmatter to +> distinguish it from document and topic files, which use `type`. + +### Original file preservation + +The `_original/` subdirectory stores the uploaded binary file unchanged. +Users can locate the original via the `original_file_path` field returned +by `GET /documents/{doc_id}`. + +The underscore prefix follows the Jekyll/Eleventy convention for +non-content directories that the cascade daemon should skip. + +Lifecycle: +- **POST** (create) — writes `_original/` +- **PUT** (replace) — clears the old directory and writes the new file +- **DELETE** — removes the entire document directory including `_original/` +- **PATCH** (category change) — moves the whole directory; `_original/` follows + +## Taxonomy + +Categories are defined in `.taxonomy.md` at the knowledge root. EverOS +ships with 20 default categories: + +| Category | Description | +|----------|-------------| +| Technology | CS, software, AI/ML, cloud, cybersecurity | +| Science | Physics, chemistry, biology, astronomy | +| Medical | Clinical medicine, drugs, public health | +| Finance | Securities, banking, accounting, fintech | +| Legal | Laws, contracts, compliance, IP | +| Education | Teaching, curriculum, e-learning | +| Business | Strategy, marketing, operations, HR | +| Engineering | Mechanical, civil, electrical engineering | +| Arts | Visual arts, music, literature, film | +| Sports | Athletics, fitness, sports science | +| Travel | Tourism, hospitality, transportation | +| Food | Culinary, nutrition, food safety | +| Environment | Climate, ecology, sustainability | +| Politics | Government, international relations, policy | +| History | Historical events, civilizations, historiography | +| Psychology | Cognitive science, behavioral psychology, mental health | +| Agriculture | Farming, crop science, agribusiness | +| RealEstate | Property development, urban planning, housing | +| Media | Journalism, social media, PR | +| Others | Fallback for unclassified documents | + +### Customization + +Edit `.taxonomy.md` directly to add, remove, or rename categories: + +```yaml +--- +kind: knowledge_taxonomy +categories: + - id: Technology + description: Computer science, software engineering, AI/ML. + - id: InternalOps + description: Company-specific operational procedures and runbooks. + - id: CustomerSuccess + description: Customer onboarding, support playbooks, case studies. +--- +``` + +Taxonomy changes are **hot-reloaded** — no server restart needed. The +system reads `.taxonomy.md` from disk on every upload and category list +request, so edits take effect immediately. + +When a document is uploaded, the LLM selects the best-matching category +from this list. If no category matches, the document falls back to +`Others`. You can also specify `category_id` explicitly in the upload +request to bypass LLM classification. + +## API reference + +All endpoints are under `/api/v1/knowledge`. Responses use the envelope +format `{"request_id": "...", "data": {...}}`. The `request_id` field is +omitted from examples below for brevity. + +### Upload a document + +``` +POST /documents +Content-Type: multipart/form-data +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `file` | file | yes | The document to upload | +| `title` | string | yes | Human-readable title | +| `source_type` | string | no | Provenance type (`"file"`, `"url"`, …) | +| `category_id` | string | no | Skip LLM classification; use this category | +| `app_id` | string | no | Tenant app (default: `"default"`) | +| `project_id` | string | no | Tenant project (default: `"default"`) | + +**Response** (201): + +```json +{ + "data": { + "doc_id": "d_a1b2c3d4e5f6", + "category_id": "Technology", + "topic_count": 8, + "source_name": "my-report.pdf", + "md_path": "/home/user/.everos/default_app/default_project/knowledge/Technology/Q1_Report_d_a1b2c3d4e5f6", + "original_file_path": "/home/user/.everos/.../Q1_Report_d_a1b2c3d4e5f6/_original/my-report.pdf" + } +} +``` + +`original_file_path` is the absolute path to the preserved upload, or `null` +when no binary was stored (e.g. an empty filename). + +**Example — Python**: + +```python +from pathlib import Path + +import httpx + + +async def upload_document(file_path: str, title: str) -> dict: + async with httpx.AsyncClient(base_url="http://localhost:8000") as client: + with open(file_path, "rb") as f: + resp = await client.post( + "/api/v1/knowledge/documents", + files={"file": (Path(file_path).name, f)}, + data={"title": title}, + ) + resp.raise_for_status() + return resp.json()["data"] +``` + +**Example — curl**: + +```bash +curl -X POST http://localhost:8000/api/v1/knowledge/documents \ + -F "file=@report.pdf" \ + -F "title=Quarterly Report" \ + -F "category_id=Finance" +``` + +### Replace a document + +``` +PUT /documents/{doc_id} +Content-Type: multipart/form-data +``` + +Same fields as POST. Atomic operation: if extraction fails, the old +document is restored from backup. + +### Update metadata + +``` +PATCH /documents/{doc_id} +Content-Type: application/json +``` + +```json +{ + "title": "Updated Title", + "category_id": "Finance" +} +``` + +Returns `updated_fields: ["title", "category_id"]`. Changing `category_id` +moves the document directory to the new category folder. + +### Delete a document + +``` +DELETE /documents/{doc_id} +``` + +Returns 204 if the document did not exist (idempotent), or 200 with +`deleted_topics` count. + +### List documents + +``` +GET /documents?page=1&page_size=20&sort_by=created_at&sort_order=desc +``` + +Optional filter: `category_id=Technology`. `sort_by` accepts `created_at` +(default), `updated_at`, or `title`; `sort_order` is `asc` or `desc`. + +```json +{ + "data": { + "documents": [ + { + "doc_id": "d_a1b2c3d4e5f6", + "category_id": "Technology", + "title": "Q1 Engineering Report", + "topic_count": 8, + "created_at": "2026-06-24T10:00:00Z" + } + ], + "total": 42, + "page": 1, + "page_size": 20 + } +} +``` + +### Get document detail + +``` +GET /documents/{doc_id} +``` + +Returns full metadata, summary, original file path, and topic overview list. + +```json +{ + "data": { + "doc_id": "d_a1b2c3d4e5f6", + "category_id": "Technology", + "title": "Q1 Engineering Report", + "summary": "This report covers Q1 engineering outcomes...", + "source_name": "my-report.pdf", + "source_type": "file", + "original_file_path": "/home/user/.everos/.../Q1_Report_d_a1b2c3d4e5f6/_original/my-report.pdf", + "topics": [ + { + "topic_id": "d_a1b2c3d4e5f6_1", + "topic_name": "Performance Analysis", + "topic_path": "Q1 Engineering Report > Performance Analysis", + "depth": 1, + "summary": "Analysis of API latency..." + } + ], + "created_at": "2026-06-24T10:00:00Z", + "updated_at": "2026-06-24T10:00:00Z" + } +} +``` + +`original_file_path` is `null` for documents created before the original +file preservation feature, or when no file was attached. + +### Get topic detail + +``` +GET /topics/{topic_id} +``` + +Returns the full topic content, tree structure, and labels. + +```json +{ + "data": { + "topic_id": "d_a1b2c3d4e5f6_1", + "doc_id": "d_a1b2c3d4e5f6", + "category_id": "Technology", + "topic_name": "Performance Analysis", + "topic_path": "Q1 Engineering Report > Performance Analysis", + "depth": 1, + "summary": "Analysis of API latency, database query times...", + "content": "The P99 API latency dropped from 450ms to 120ms...", + "content_labels": ["performance", "latency", "caching"], + "parent_topic_id": "d_a1b2c3d4e5f6_0", + "children_topic_ids": [], + "created_at": "2026-06-24T10:00:00Z", + "updated_at": "2026-06-24T10:00:00Z" + } +} +``` + +### Search + +``` +POST /search +Content-Type: application/json +``` + +```json +{ + "query": "performance bottleneck", + "method": "hybrid", + "top_k": 10, + "include_content": true, + "score_threshold": 0.5 +} +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `query` | string | — | Search query (required, 1–2000 chars) | +| `method` | string | `"hybrid"` | `"keyword"`, `"vector"`, or `"hybrid"` | +| `top_k` | int | 10 | Max results (1–100) | +| `include_content` | bool | false | Include full topic content in results | +| `score_threshold` | float | null | Drop results below this score | + +**Search methods**: + +- **keyword** — BM25 sparse retrieval over tokenized summary + content +- **vector** — Dense ANN over embedded summary vectors (requires embedding provider) +- **hybrid** — Parallel keyword + vector, fused with Reciprocal Rank Fusion (RRF), then cross-encoder reranking + +All three methods embed the query and apply cross-encoder reranking, so +knowledge search requires **both** an embedding and a rerank provider — +there is no provider-free fallback (this is by design: no silent +degradation). The two failure modes map to distinct status codes: + +- **Provider not configured** → `500 CONFIGURATION_ERROR` (a required + setting is missing; retrying will not help — set `EVEROS_EMBEDDING__*` / + `EVEROS_RERANK__*`). +- **Provider configured but failing/timing out at call time** → + `503 EXTERNAL_SERVICE_UNAVAILABLE` (transient; retryable). + +**Response**: + +```json +{ + "data": { + "hits": [ + { + "topic_id": "d_a1b2c3d4e5f6_1", + "category_id": "Technology", + "topic_name": "Performance Analysis", + "topic_path": "Q1 Engineering Report > Performance Analysis", + "depth": 1, + "summary": "Analysis of API latency...", + "content": "The P99 API latency dropped...", + "score": 0.92, + "retrieval_method": "hybrid", + "document": { + "doc_id": "d_a1b2c3d4e5f6", + "title": "Q1 Engineering Report", + "summary": "This report covers..." + } + } + ], + "total": 3, + "took_ms": 245.6 + } +} +``` + +### List categories + +``` +GET /categories +``` + +```json +{ + "data": { + "categories": [ + {"category_id": "Technology", "description": "Computer science...", "document_count": 12}, + {"category_id": "Finance", "description": "Securities...", "document_count": 5}, + {"category_id": "Others", "description": "Fallback...", "document_count": 0} + ] + } +} +``` + +## Search pipeline + +``` +query ─→ embed ─→ keyword (BM25) ─┐ + vector (ANN) ──┤─→ RRF fusion ─→ rerank ─→ top_k +``` + +1. **Embed** — the query is embedded using the configured embedding provider +2. **Recall** — dual-channel retrieval from LanceDB: + - BM25 channel: keyword matching on `summary_tokens` + `content_tokens` + - ANN channel: nearest-neighbor search on the `vector` column + - In `hybrid` mode, both channels run in parallel +3. **Fuse** — Reciprocal Rank Fusion merges the two candidate lists +4. **Rerank** — cross-encoder reranker rescores the top candidates +5. **Filter** — drop results below `score_threshold` and limit to `top_k` + +### Configuration + +Search tuning parameters in `src/everos/config/default.toml`: + +```toml +[knowledge.search] +recall_n = 200 # initial recall pool size per channel +rerank_n = 50 # candidates sent to reranker +mass_top_m = 50 # category-aware retrieve pool +lambda = 0.1 # category boost weight +top_k_cap = 100 # hard cap on returned results +``` + +Override via environment variables: + +```bash +export EVEROS_KNOWLEDGE__SEARCH__RECALL_N=500 +export EVEROS_KNOWLEDGE__SEARCH__RERANK_N=100 +``` + +## Cascade sync + +The cascade daemon watches the knowledge Markdown directory for file +changes and keeps SQLite + LanceDB in sync. + +``` +md file written + → FSEvents / watchdog detects change + → worker picks up from queue (≤1s poll interval) + → handler dispatched by file type: + index.md → KnowledgeDocumentHandler → SQLite upsert (metadata) + N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + LanceDB upsert +``` + +The topic handler uses a SHA-256 content digest to skip unchanged files — +re-embedding only happens when the content actually changes. + +Typical latency from file write to search availability: **1–3 seconds**. + +## Supported file formats + +EverOS accepts text-based files natively. Binary formats require the +`everos[multimodal]` extra (depends on LibreOffice for document conversion). + +| Category | Formats | Requires `[multimodal]` | +|----------|---------|:-----------------------:| +| Text | `.txt`, `.md`, `.csv`, `.tsv`, `.vtt` | No | +| Documents | `.pdf`, `.docx`, `.doc`, `.rtf`, `.odt`, `.pages` | Yes | +| Spreadsheets | `.xlsx`, `.xls`, `.ods`, `.numbers` | Yes | +| Presentations | `.pptx`, `.ppt`, `.odp`, `.key` | Yes | +| Web | `.html`, `.htm`, `.eml` | Yes | +| Images (OCR) | `.png`, `.jpg`, `.webp`, `.tiff`, `.bmp`, `.svg` | Yes | +| Audio (transcription) | `.mp3`, `.wav`, `.m4a`, `.amr`, `.aiff`, `.aac`, `.ogg`, `.flac` | Yes | + +```bash +pip install everos[multimodal] +``` + +## Error handling + +| HTTP | Error code | Scenario | +|------|-----------|----------| +| 404 | `NOT_FOUND` | Document or topic does not exist | +| 409 | `CONFLICT` | `doc_id` already exists (use PUT to replace) | +| 415 | `UNSUPPORTED_FORMAT` | File format not parseable | +| 422 | `INVALID_INPUT` | Empty/oversized query, empty title, invalid ID format | +| 500 | `CONFIGURATION_ERROR` | Embedding or rerank provider not configured | +| 503 | `EXTERNAL_SERVICE_UNAVAILABLE` | Configured embedding/rerank provider failing at call time | +| 503 | `CAPABILITY_UNAVAILABLE` | `everos[multimodal]` not installed | + +All error responses include a human-readable `message` field: + +```json +{ + "error": { + "code": "NOT_FOUND", + "message": "Document 'd_abc123' not found" + } +} +``` + +## Multi-tenancy + +All endpoints accept `app_id` and `project_id` parameters (default: +`"default"`). Data is fully isolated per tenant pair: + +```bash +# Tenant A uploads +curl -X POST .../documents -F "file=@a.pdf" -F "title=A" \ + -F "app_id=tenant_a" -F "project_id=proj_1" + +# Tenant B cannot see Tenant A's data +curl .../documents?app_id=tenant_b&project_id=proj_1 +# → { "documents": [], "total": 0 } +``` + +Storage paths, SQLite rows, and LanceDB indexes are all scoped by +`app_id` + `project_id`. + +## End-to-end walkthrough + +A complete workflow from upload to search: + +```bash +BASE=http://localhost:8000/api/v1/knowledge + +# 1. List available categories +curl -s "$BASE/categories" | jq '[.data.categories[] | .category_id]' +# → ["Technology", "Science", "Medical", ..., "Others"] + +# 2. Upload a document +DOC_ID=$(curl -s -X POST "$BASE/documents" \ + -F "file=@architecture-guide.md" \ + -F "title=System Architecture Guide" \ + | jq -r .data.doc_id) +echo "Created: $DOC_ID" + +# 3. View document detail (with topic list) +curl -s "$BASE/documents/$DOC_ID" | jq '{ + title: .data.title, + category: .data.category_id, + topics: [.data.topics[] | .topic_name], + original: .data.original_file_path +}' + +# 4. Read a topic — pick the first from the detail response +TOPIC_ID=$(curl -s "$BASE/documents/$DOC_ID" \ + | jq -r '.data.topics[0].topic_id') +curl -s "$BASE/topics/$TOPIC_ID" | jq '{ + name: .data.topic_name, + path: .data.topic_path, + content: .data.content[:200], + labels: .data.content_labels +}' + +# 5. Search (index is typically ready within 1–3 seconds) +sleep 3 +curl -s -X POST "$BASE/search" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "how to handle service failures", + "method": "hybrid", + "top_k": 5, + "include_content": true + }' | jq '.data | { + total, + took_ms, + top_hit: .hits[0] | {topic_name, score, content: .content[:100]} + }' + +# 6. Move document to a different category +curl -s -X PATCH "$BASE/documents/$DOC_ID" \ + -H "Content-Type: application/json" \ + -d '{"category_id": "Engineering"}' \ + | jq .data.updated_fields +# → ["category_id"] + +# 7. Clean up +curl -s -X DELETE "$BASE/documents/$DOC_ID" | jq . +``` diff --git a/docs/locomo_benchmark.md b/docs/locomo_benchmark.md index 36b4a85..f4bc70b 100644 --- a/docs/locomo_benchmark.md +++ b/docs/locomo_benchmark.md @@ -18,8 +18,8 @@ locally using the `hybrid` and `agentic` search methods. ## Prerequisites - Python **3.12**, [uv](https://docs.astral.sh/uv/) -- A `.env` at the repo root with the LLM / embedding credentials EverOS - needs: +- LLM / embedding credentials configured via `everos.toml` in the memory + root (generated by `everos init`) or `EVEROS_*` env vars: - `EVEROS_LLM__MODEL`, `EVEROS_LLM__API_KEY`, `EVEROS_LLM__BASE_URL` - `EVEROS_EMBEDDING__*` - `EVEROS_RERANK__*` @@ -41,11 +41,11 @@ later with `--data-path` if you keep it elsewhere. ## 2. Start the server ```bash -EVEROS_MEMORY__ROOT=~/.everos \ +EVEROS_ROOT=~/.everos \ uv run python -m everos.entrypoints.cli.main server start --port 8000 ``` -`EVEROS_MEMORY__ROOT` isolates one benchmark's corpus from another — +`EVEROS_ROOT` isolates one benchmark's corpus from another — change it (or `rm -rf` it) whenever you want a clean run. Leave the server running in one terminal; run the benchmark from @@ -74,7 +74,7 @@ bash tests/run_locomo_batch.sh \ --concurrency 2 ``` -The wrapper picks up `EVEROS_MEMORY__ROOT` from the environment so the +The wrapper picks up `EVEROS_ROOT` from the environment so the cascade poll path matches the server's data root. If you set them differently, pass `--corpus-path` explicitly. @@ -119,6 +119,17 @@ An aggregate accuracy table prints at the end of the wrapper run. ## Notes +- **Smoke mode**: add `--smoke` for a quick sanity check — 2 sessions + × 5 QA with reduced wait times. Not a scored run, just pipeline + verification. Pass it to `test_locomo.py` directly or via the batch + wrapper's `--` passthrough: + + ```bash + uv run python tests/test_locomo.py --smoke --base-url http://localhost:8000 + # or via the wrapper + bash tests/run_locomo_batch.sh --conv-indices 0 --methods hybrid -- --smoke + ``` + - **Re-running on the same corpus**: add `--skip-add` to skip ingest and reuse what's already in `~/.everos`. Useful when comparing methods side by side. diff --git a/docs/multimodal.md b/docs/multimodal.md index 61aeb76..8551d70 100644 --- a/docs/multimodal.md +++ b/docs/multimodal.md @@ -85,14 +85,15 @@ brew install --cask libreoffice # macOS sudo apt-get install -y libreoffice # Debian / Ubuntu ``` -Without LibreOffice, **office uploads return `415`** with a clear error; -image / PDF / audio / HTML / email parsing is unaffected. +Without LibreOffice, **office uploads return `503`** +(`CAPABILITY_UNAVAILABLE`) with a clear error message; image / PDF / +audio / HTML / email parsing is unaffected. ### Configure the multimodal LLM The parser uses its own LLM section, independent from `[llm]`. The model -must accept OpenAI `image_url` parts. `everos init` writes these into the -generated `.env`: +must accept OpenAI `image_url` parts. Configure these in `everos.toml` +(under `[multimodal]`) or via env vars: ```bash EVEROS_MULTIMODAL__MODEL=google/gemini-3-flash-preview @@ -277,26 +278,38 @@ All fields bind from the environment via the parent `Settings` ## Errors and limits -Two failure classes behave differently. **Deterministic** problems -(nothing to parse, no handler, missing system dependency) **abort the -whole `/add` batch with `415`**. A **transient** multimodal-LLM failure -(timeout, rate-limit, the model rejecting the asset) **degrades just that -item** — the request still returns `200`, the item is marked -`parse_status="failed"` and contributes no text, and the rest of the -batch extracts normally. +Three failure classes behave differently: -| Condition | Result | -|---|---| -| Non-text item carries only `text` (no `uri` / `base64`) | `415` (batch aborted) | -| Extension / modality the parser has no handler for | `415` (batch aborted) | -| `base64` without a resolvable `ext` / MIME to dispatch on | `415` (batch aborted) | -| Office document but no LibreOffice (`soffice`) on host | `415` (batch aborted) | -| `file://` fails a guardrail (missing / non-regular / too large / outside allowlist) | `415` (batch aborted) | -| Multimodal **LLM call** fails (timeout / rate-limit / model rejects the asset) | **`200`** — that item is skipped (`parse_status="failed"`), the rest of the batch still extracts | +**Format errors** — the uploaded file format is invalid or not +recognized. These abort the batch with `415` (`UNSUPPORTED_FORMAT`): -The `415` body uses the standard error envelope with the parse-failure -reason in `error.message` — see -[api.md → POST /add](api.md#post-apiv1memoryadd). +| Condition | HTTP | `error.code` | +|---|---|---| +| Non-text item carries only `text` (no `uri` / `base64`) | `415` | `UNSUPPORTED_FORMAT` | +| Extension / modality the parser has no handler for | `415` | `UNSUPPORTED_FORMAT` | +| `base64` without a resolvable `ext` / MIME to dispatch on | `415` | `UNSUPPORTED_FORMAT` | +| `file://` fails a guardrail (missing / non-regular / too large / outside allowlist) | `415` | `UNSUPPORTED_FORMAT` | + +**Capability errors** — the server is missing a required dependency. +These abort the batch with `503` (`CAPABILITY_UNAVAILABLE`). Unlike +transient errors, retrying will not help — admin action is required: + +| Condition | HTTP | `error.code` | +|---|---|---| +| `everos[multimodal]` extra not installed | `503` | `CAPABILITY_UNAVAILABLE` | +| Office document but no LibreOffice (`soffice`) on host | `503` | `CAPABILITY_UNAVAILABLE` | + +**Transient LLM errors** — the multimodal LLM call failed. These +degrade gracefully — the request still returns `200`, the affected +item is marked `parse_status="failed"` and contributes no text, and the +rest of the batch extracts normally: + +| Condition | HTTP | Result | +|---|---|---| +| Multimodal LLM call fails (timeout / rate-limit / model rejects) | `200` | That item is skipped; the rest of the batch still extracts | + +All error responses use the standard error envelope — see +[api.md → Errors](api.md#errors). ## Searching multimodal memory diff --git a/docs/openapi.json b/docs/openapi.json index 1d4a3b4..1c54825 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "everos", "description": "md-first memory extraction framework", - "version": "1.0.1" + "version": "1.1.0" }, "paths": { "/health": { @@ -219,6 +219,598 @@ } } } + }, + "/api/v1/ome/trigger": { + "post": { + "tags": [ + "ome" + ], + "summary": "Trigger", + "description": "Manually trigger a registered OME strategy and wait for completion.", + "operationId": "trigger_api_v1_ome_trigger_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TriggerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TriggerResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/knowledge/documents": { + "post": { + "tags": [ + "knowledge" + ], + "summary": "Create Document Route", + "description": "Upload a new knowledge document.", + "operationId": "create_document_route_api_v1_knowledge_documents_post", + "requestBody": { + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_create_document_route_api_v1_knowledge_documents_post" + } + } + } + }, + "responses": { + "201": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope_DocumentCreateResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "get": { + "tags": [ + "knowledge" + ], + "summary": "List Documents Route", + "description": "Paginated document listing.", + "operationId": "list_documents_route_api_v1_knowledge_documents_get", + "parameters": [ + { + "name": "app_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "App Id" + } + }, + { + "name": "project_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "Project Id" + } + }, + { + "name": "category_id", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Category Id" + } + }, + { + "name": "page", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "default": 1, + "title": "Page" + } + }, + { + "name": "page_size", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 100, + "minimum": 1, + "default": 20, + "title": "Page Size" + } + }, + { + "name": "sort_by", + "in": "query", + "required": false, + "schema": { + "enum": [ + "created_at", + "updated_at", + "title" + ], + "type": "string", + "default": "created_at", + "title": "Sort By" + } + }, + { + "name": "sort_order", + "in": "query", + "required": false, + "schema": { + "enum": [ + "asc", + "desc" + ], + "type": "string", + "default": "desc", + "title": "Sort Order" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope_DocumentListResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/knowledge/documents/{doc_id}": { + "put": { + "tags": [ + "knowledge" + ], + "summary": "Replace Document Route", + "description": "Replace an existing knowledge document (atomic backup/restore on failure).", + "operationId": "replace_document_route_api_v1_knowledge_documents__doc_id__put", + "parameters": [ + { + "name": "doc_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^d_[a-f0-9]{12,32}$", + "title": "Doc Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_replace_document_route_api_v1_knowledge_documents__doc_id__put" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope_DocumentCreateResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "knowledge" + ], + "summary": "Delete Document Route", + "description": "Remove a knowledge document.", + "operationId": "delete_document_route_api_v1_knowledge_documents__doc_id__delete", + "parameters": [ + { + "name": "doc_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^d_[a-f0-9]{12,32}$", + "title": "Doc Id" + } + }, + { + "name": "app_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "App Id" + } + }, + { + "name": "project_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "Project Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "get": { + "tags": [ + "knowledge" + ], + "summary": "Get Document Route", + "description": "Fetch a single document with its topic list.", + "operationId": "get_document_route_api_v1_knowledge_documents__doc_id__get", + "parameters": [ + { + "name": "doc_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^d_[a-f0-9]{12,32}$", + "title": "Doc Id" + } + }, + { + "name": "app_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "App Id" + } + }, + { + "name": "project_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "Project Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope_DocumentDetailResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "patch": { + "tags": [ + "knowledge" + ], + "summary": "Patch Document Route", + "description": "Update mutable document metadata fields.", + "operationId": "patch_document_route_api_v1_knowledge_documents__doc_id__patch", + "parameters": [ + { + "name": "doc_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^d_[a-f0-9]{12,32}$", + "title": "Doc Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DocumentPatchRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope_DocumentPatchResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/knowledge/topics/{topic_id}": { + "get": { + "tags": [ + "knowledge" + ], + "summary": "Get Topic Route", + "description": "Fetch a single topic with full content.", + "operationId": "get_topic_route_api_v1_knowledge_topics__topic_id__get", + "parameters": [ + { + "name": "topic_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^d_[a-f0-9]{12,32}_\\d+$", + "title": "Topic Id" + } + }, + { + "name": "app_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "App Id" + } + }, + { + "name": "project_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "Project Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope_TopicDetailResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/knowledge/search": { + "post": { + "tags": [ + "knowledge" + ], + "summary": "Search Knowledge Route", + "description": "Knowledge retrieval (keyword / vector / hybrid).", + "operationId": "search_knowledge_route_api_v1_knowledge_search_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/KnowledgeSearchRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope_KnowledgeSearchResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/knowledge/categories": { + "get": { + "tags": [ + "knowledge" + ], + "summary": "List Categories Route", + "description": "List taxonomy categories from ``.taxonomy.md``.", + "operationId": "list_categories_route_api_v1_knowledge_categories_get", + "parameters": [ + { + "name": "app_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "App Id" + } + }, + { + "name": "project_id", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "default", + "title": "Project Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessEnvelope_CategoryListResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } } }, "components": { @@ -245,6 +837,153 @@ ], "title": "AddResponseData" }, + "Body_create_document_route_api_v1_knowledge_documents_post": { + "properties": { + "file": { + "type": "string", + "contentMediaType": "application/octet-stream", + "title": "File" + }, + "title": { + "type": "string", + "minLength": 1, + "pattern": "\\w", + "title": "Title" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Type" + }, + "category_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Category Id" + }, + "app_id": { + "type": "string", + "title": "App Id", + "default": "default" + }, + "project_id": { + "type": "string", + "title": "Project Id", + "default": "default" + } + }, + "type": "object", + "required": [ + "file", + "title" + ], + "title": "Body_create_document_route_api_v1_knowledge_documents_post" + }, + "Body_replace_document_route_api_v1_knowledge_documents__doc_id__put": { + "properties": { + "file": { + "type": "string", + "contentMediaType": "application/octet-stream", + "title": "File" + }, + "title": { + "type": "string", + "minLength": 1, + "pattern": "\\w", + "title": "Title" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Type" + }, + "category_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Category Id" + }, + "app_id": { + "type": "string", + "title": "App Id", + "default": "default" + }, + "project_id": { + "type": "string", + "title": "Project Id", + "default": "default" + } + }, + "type": "object", + "required": [ + "file", + "title" + ], + "title": "Body_replace_document_route_api_v1_knowledge_documents__doc_id__put" + }, + "CategoryDTO": { + "properties": { + "category_id": { + "type": "string", + "title": "Category Id" + }, + "description": { + "type": "string", + "title": "Description" + }, + "document_count": { + "type": "integer", + "title": "Document Count" + } + }, + "type": "object", + "required": [ + "category_id", + "description", + "document_count" + ], + "title": "CategoryDTO", + "description": "One taxonomy category." + }, + "CategoryListResponse": { + "properties": { + "categories": { + "items": { + "$ref": "#/components/schemas/CategoryDTO" + }, + "type": "array", + "title": "Categories" + } + }, + "type": "object", + "required": [ + "categories" + ], + "title": "CategoryListResponse", + "description": "Response for GET /categories." + }, "ContentItemDTO": { "properties": { "type": { @@ -336,6 +1075,305 @@ "title": "ContentItemDTO", "description": "Content piece (v1 API brief appendix A)." }, + "DocumentContextDTO": { + "properties": { + "doc_id": { + "type": "string", + "title": "Doc Id" + }, + "title": { + "type": "string", + "title": "Title" + }, + "summary": { + "type": "string", + "title": "Summary" + } + }, + "type": "object", + "required": [ + "doc_id", + "title", + "summary" + ], + "title": "DocumentContextDTO", + "description": "L1 document metadata attached to every search hit." + }, + "DocumentCreateResponse": { + "properties": { + "doc_id": { + "type": "string", + "title": "Doc Id" + }, + "category_id": { + "type": "string", + "title": "Category Id" + }, + "topic_count": { + "type": "integer", + "title": "Topic Count" + }, + "source_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Name" + }, + "md_path": { + "type": "string", + "title": "Md Path" + }, + "original_file_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Original File Path" + } + }, + "type": "object", + "required": [ + "doc_id", + "category_id", + "topic_count", + "source_name", + "md_path", + "original_file_path" + ], + "title": "DocumentCreateResponse", + "description": "Response for POST/PUT /documents." + }, + "DocumentDetailResponse": { + "properties": { + "doc_id": { + "type": "string", + "title": "Doc Id" + }, + "category_id": { + "type": "string", + "title": "Category Id" + }, + "title": { + "type": "string", + "title": "Title" + }, + "summary": { + "type": "string", + "title": "Summary" + }, + "source_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Name" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Type" + }, + "original_file_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Original File Path" + }, + "topics": { + "items": { + "$ref": "#/components/schemas/TopicOverviewDTO" + }, + "type": "array", + "title": "Topics" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Updated At" + } + }, + "type": "object", + "required": [ + "doc_id", + "category_id", + "title", + "summary", + "source_name", + "source_type", + "original_file_path", + "topics", + "created_at", + "updated_at" + ], + "title": "DocumentDetailResponse", + "description": "Response for GET /documents/{doc_id}." + }, + "DocumentListResponse": { + "properties": { + "documents": { + "items": { + "$ref": "#/components/schemas/DocumentOverviewItemDTO" + }, + "type": "array", + "title": "Documents" + }, + "total": { + "type": "integer", + "title": "Total" + }, + "page": { + "type": "integer", + "title": "Page" + }, + "page_size": { + "type": "integer", + "title": "Page Size" + } + }, + "type": "object", + "required": [ + "documents", + "total", + "page", + "page_size" + ], + "title": "DocumentListResponse", + "description": "Response for GET /documents." + }, + "DocumentOverviewItemDTO": { + "properties": { + "doc_id": { + "type": "string", + "title": "Doc Id" + }, + "category_id": { + "type": "string", + "title": "Category Id" + }, + "title": { + "type": "string", + "title": "Title" + }, + "topic_count": { + "type": "integer", + "title": "Topic Count" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At" + } + }, + "type": "object", + "required": [ + "doc_id", + "category_id", + "title", + "topic_count", + "created_at" + ], + "title": "DocumentOverviewItemDTO", + "description": "One row in the paginated document list." + }, + "DocumentPatchRequest": { + "properties": { + "title": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "\\w" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "category_id": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ], + "title": "Category Id" + }, + "app_id": { + "type": "string", + "title": "App Id", + "default": "default" + }, + "project_id": { + "type": "string", + "title": "Project Id", + "default": "default" + } + }, + "type": "object", + "title": "DocumentPatchRequest", + "description": "Request body for PATCH /documents/{doc_id}." + }, + "DocumentPatchResponse": { + "properties": { + "doc_id": { + "type": "string", + "title": "Doc Id" + }, + "updated_fields": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Updated Fields" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Updated At" + } + }, + "type": "object", + "required": [ + "doc_id", + "updated_fields", + "updated_at" + ], + "title": "DocumentPatchResponse", + "description": "Response for PATCH /documents/{doc_id}." + }, "FilterNode": { "properties": { "AND": { @@ -815,6 +1853,92 @@ "type": "object", "title": "HTTPValidationError" }, + "KnowledgeSearchRequest": { + "properties": { + "query": { + "type": "string", + "maxLength": 2000, + "minLength": 1, + "title": "Query" + }, + "method": { + "type": "string", + "enum": [ + "keyword", + "vector", + "hybrid" + ], + "title": "Method", + "default": "hybrid" + }, + "top_k": { + "type": "integer", + "maximum": 100.0, + "minimum": 1.0, + "title": "Top K", + "default": 10 + }, + "score_threshold": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score Threshold" + }, + "include_content": { + "type": "boolean", + "title": "Include Content", + "default": false + }, + "app_id": { + "type": "string", + "title": "App Id", + "default": "default" + }, + "project_id": { + "type": "string", + "title": "Project Id", + "default": "default" + } + }, + "type": "object", + "required": [ + "query" + ], + "title": "KnowledgeSearchRequest", + "description": "Request body for POST /search." + }, + "KnowledgeSearchResponse": { + "properties": { + "hits": { + "items": { + "$ref": "#/components/schemas/SearchHitDTO" + }, + "type": "array", + "title": "Hits" + }, + "total": { + "type": "integer", + "title": "Total" + }, + "took_ms": { + "type": "number", + "title": "Took Ms" + } + }, + "type": "object", + "required": [ + "hits", + "total", + "took_ms" + ], + "title": "KnowledgeSearchResponse", + "description": "Response for POST /search." + }, "MemorizeAddRequest": { "properties": { "session_id": { @@ -1206,7 +2330,14 @@ "default": "default" }, "session_id": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Session Id" }, "timestamp": { @@ -1255,7 +2386,6 @@ "required": [ "id", "user_id", - "session_id", "timestamp", "summary", "subject", @@ -1266,6 +2396,83 @@ "title": "SearchEpisodeItem", "description": "Episode hit — always user-scoped in the current emission contract.\n\n``type`` is narrowed to ``\"Conversation\"`` because the only emitted\nepisode shape today is conversation-derived; widen when other\nsources ship. Item kind is encoded by class name (no ``owner_type``\nfield on the wire), so episode results never carry ambiguity." }, + "SearchHitDTO": { + "properties": { + "topic_id": { + "type": "string", + "title": "Topic Id" + }, + "category_id": { + "type": "string", + "title": "Category Id" + }, + "topic_name": { + "type": "string", + "title": "Topic Name" + }, + "topic_path": { + "type": "string", + "title": "Topic Path" + }, + "depth": { + "type": "integer", + "title": "Depth" + }, + "summary": { + "type": "string", + "title": "Summary" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "score": { + "type": "number", + "title": "Score" + }, + "retrieval_method": { + "type": "string", + "title": "Retrieval Method" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source" + }, + "document": { + "$ref": "#/components/schemas/DocumentContextDTO" + } + }, + "type": "object", + "required": [ + "topic_id", + "category_id", + "topic_name", + "topic_path", + "depth", + "summary", + "content", + "score", + "retrieval_method", + "source", + "document" + ], + "title": "SearchHitDTO", + "description": "One ranked result from knowledge search." + }, "SearchMethod": { "type": "string", "enum": [ @@ -1394,6 +2601,19 @@ ], "title": "Radius" }, + "min_score": { + "anyOf": [ + { + "type": "number", + "maximum": 1.0, + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Min Score" + }, "include_profile": { "type": "boolean", "title": "Include Profile", @@ -1460,6 +2680,91 @@ ], "title": "SuccessEnvelope[AddResponseData]" }, + "SuccessEnvelope_CategoryListResponse_": { + "properties": { + "request_id": { + "type": "string", + "title": "Request Id" + }, + "data": { + "$ref": "#/components/schemas/CategoryListResponse" + } + }, + "type": "object", + "required": [ + "request_id", + "data" + ], + "title": "SuccessEnvelope[CategoryListResponse]" + }, + "SuccessEnvelope_DocumentCreateResponse_": { + "properties": { + "request_id": { + "type": "string", + "title": "Request Id" + }, + "data": { + "$ref": "#/components/schemas/DocumentCreateResponse" + } + }, + "type": "object", + "required": [ + "request_id", + "data" + ], + "title": "SuccessEnvelope[DocumentCreateResponse]" + }, + "SuccessEnvelope_DocumentDetailResponse_": { + "properties": { + "request_id": { + "type": "string", + "title": "Request Id" + }, + "data": { + "$ref": "#/components/schemas/DocumentDetailResponse" + } + }, + "type": "object", + "required": [ + "request_id", + "data" + ], + "title": "SuccessEnvelope[DocumentDetailResponse]" + }, + "SuccessEnvelope_DocumentListResponse_": { + "properties": { + "request_id": { + "type": "string", + "title": "Request Id" + }, + "data": { + "$ref": "#/components/schemas/DocumentListResponse" + } + }, + "type": "object", + "required": [ + "request_id", + "data" + ], + "title": "SuccessEnvelope[DocumentListResponse]" + }, + "SuccessEnvelope_DocumentPatchResponse_": { + "properties": { + "request_id": { + "type": "string", + "title": "Request Id" + }, + "data": { + "$ref": "#/components/schemas/DocumentPatchResponse" + } + }, + "type": "object", + "required": [ + "request_id", + "data" + ], + "title": "SuccessEnvelope[DocumentPatchResponse]" + }, "SuccessEnvelope_FlushResponseData_": { "properties": { "request_id": { @@ -1477,6 +2782,40 @@ ], "title": "SuccessEnvelope[FlushResponseData]" }, + "SuccessEnvelope_KnowledgeSearchResponse_": { + "properties": { + "request_id": { + "type": "string", + "title": "Request Id" + }, + "data": { + "$ref": "#/components/schemas/KnowledgeSearchResponse" + } + }, + "type": "object", + "required": [ + "request_id", + "data" + ], + "title": "SuccessEnvelope[KnowledgeSearchResponse]" + }, + "SuccessEnvelope_TopicDetailResponse_": { + "properties": { + "request_id": { + "type": "string", + "title": "Request Id" + }, + "data": { + "$ref": "#/components/schemas/TopicDetailResponse" + } + }, + "type": "object", + "required": [ + "request_id", + "data" + ], + "title": "SuccessEnvelope[TopicDetailResponse]" + }, "ToolCallDTO": { "properties": { "id": { @@ -1517,6 +2856,172 @@ ], "title": "ToolFunctionDTO" }, + "TopicDetailResponse": { + "properties": { + "topic_id": { + "type": "string", + "title": "Topic Id" + }, + "doc_id": { + "type": "string", + "title": "Doc Id" + }, + "category_id": { + "type": "string", + "title": "Category Id" + }, + "topic_name": { + "type": "string", + "title": "Topic Name" + }, + "topic_path": { + "type": "string", + "title": "Topic Path" + }, + "depth": { + "type": "integer", + "title": "Depth" + }, + "summary": { + "type": "string", + "title": "Summary" + }, + "content": { + "type": "string", + "title": "Content" + }, + "content_labels": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Content Labels" + }, + "parent_topic_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Parent Topic Id" + }, + "children_topic_ids": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Children Topic Ids" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Updated At" + } + }, + "type": "object", + "required": [ + "topic_id", + "doc_id", + "category_id", + "topic_name", + "topic_path", + "depth", + "summary", + "content", + "content_labels", + "parent_topic_id", + "children_topic_ids", + "created_at", + "updated_at" + ], + "title": "TopicDetailResponse", + "description": "Response for GET /topics/{topic_id}." + }, + "TopicOverviewDTO": { + "properties": { + "topic_id": { + "type": "string", + "title": "Topic Id" + }, + "topic_name": { + "type": "string", + "title": "Topic Name" + }, + "topic_path": { + "type": "string", + "title": "Topic Path" + }, + "depth": { + "type": "integer", + "title": "Depth" + }, + "summary": { + "type": "string", + "title": "Summary" + } + }, + "type": "object", + "required": [ + "topic_id", + "topic_name", + "topic_path", + "depth", + "summary" + ], + "title": "TopicOverviewDTO", + "description": "Minimal topic summary inside a document detail." + }, + "TriggerRequest": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "timeout": { + "type": "number", + "title": "Timeout", + "default": 120.0 + }, + "force": { + "type": "boolean", + "title": "Force", + "default": false + } + }, + "type": "object", + "required": [ + "name" + ], + "title": "TriggerRequest", + "description": "Request body for ``POST /api/v1/ome/trigger``." + }, + "TriggerResponse": { + "properties": { + "status": { + "type": "string", + "title": "Status" + }, + "name": { + "type": "string", + "title": "Name" + } + }, + "type": "object", + "required": [ + "status", + "name" + ], + "title": "TriggerResponse", + "description": "Response body for ``POST /api/v1/ome/trigger``." + }, "UnprocessedMessageDTO": { "properties": { "id": { diff --git a/docs/prompt_slots.md b/docs/prompt_slots.md index 9776344..d66c046 100644 --- a/docs/prompt_slots.md +++ b/docs/prompt_slots.md @@ -25,9 +25,12 @@ layer 3 is supplied at the call site. ## Loader -The category loader lives at -[`src/everos/component/config/loader.py`](../src/everos/component/config/loader.py) -as `YamlConfigLoader`: +The prompt-slots public entry point is +[`PromptLoader`](../src/everos/memory/prompt_slots/loader.py) (re-exported +from `everos.memory.prompt_slots`); it wraps the generic category loader +[`YamlConfigLoader`](../src/everos/component/config/loader.py). The generic +loader is shown below — `PromptLoader` is the prompt-slots-specific wrapper +over the same mechanism: ```python from pathlib import Path @@ -68,7 +71,7 @@ output_schema: participants: { type: array } llm: - model: gpt-4o-mini + model: gpt-4.1-mini temperature: 0.3 max_tokens: 2000 @@ -106,6 +109,7 @@ forcing one model on the other gets clunky. ## See also -- [`src/everos/component/config/loader.py`](../src/everos/component/config/loader.py) +- [`src/everos/memory/prompt_slots/`](../src/everos/memory/prompt_slots/) — `PromptLoader` (prompt-slots public API) +- [`src/everos/component/config/loader.py`](../src/everos/component/config/loader.py) — generic `YamlConfigLoader` - [`tests/unit/test_component/test_config/test_loader.py`](../tests/unit/test_component/test_config/test_loader.py) - [`docs/architecture.md`](architecture.md) — layer placement diff --git a/docs/reflection.md b/docs/reflection.md new file mode 100644 index 0000000..94a56ae --- /dev/null +++ b/docs/reflection.md @@ -0,0 +1,359 @@ +# Reflection + +Reflection periodically consolidates the memory fragments scattered across +many conversations into a single, chronologically-organized narrative. It +runs offline in the background: it merges the multiple Episodes inside one +similarity cluster into one, resolves stale information by keeping the +latest state, and soft-archives the originals it replaces — so memory gets +more accurate and more compact with use, instead of piling up into noise. + +## Prerequisites + +Reflection's merge step calls an LLM, and re-clustering the merged narrative +calls an embedding model. Configure both as you would for the rest of EverOS +— an OpenAI-compatible `[llm]` and `[embedding]` block in `/everos.toml`, +or the matching `EVEROS_LLM__*` / `EVEROS_EMBEDDING__*` environment variables. +If a provider is unavailable, the affected clusters are skipped and logged +rather than failing the run. + +## Quick start + +> The examples below assume EverOS is running on the default port 8000. +> `` is the EverOS memory root (see [QUICKSTART](../QUICKSTART.md)). + +Reflection is off by default. Turn it on in `/ome.toml` — **one line**: + +```toml +[strategies.reflect_episodes] +enabled = true +``` + +Once enabled, it **runs automatically every Monday at 02:00**. This is how +Reflection is meant to work — nothing else to configure. To change the run +time, add a `cron` line (a schedule expression; optional): + +```toml +[strategies.reflect_episodes] +enabled = true +cron = "0 3 * * 0" # optional: change the run time (here: Sundays at 03:00) +``` + +> **Don't run it too often.** Once a week at most is recommended. Each run +> is a lossy LLM merge; repeatedly re-consolidating the same memories can +> make the narrative *worse*, not better — which is why the default is +> weekly. + +Config changes hot-reload (no restart needed). From then on, at each +scheduled time, Reflection consolidates each user's memory once. + +**What does it produce, and where do I see it?** Each run **appends one +merged narrative** to the relevant user's Episode log, and marks the older +fragments it replaces as archived (removed from default search). Markdown is +the source of truth — just open the user's Episode log file: + +``` +/default_app/default_project/users//episodes/episode-.md +``` + +The new entry carries `parent_type: cluster` (Episodes produced by ordinary +conversation are `parent_type: memcell`). It looks roughly like this: + +```markdown +--- +owner_id: u_andrew +timestamp: 2026-10-11T02:00:00+00:00 +parent_type: cluster # <- marks it as a Reflection merge product +parent_id: cl_a1b2c3d4e5f6 +--- +## Subject +Andrew's pet adoption journey + +## Content +Andrew initially had no pets. He later adopted a dog named Toby, and then +adopted another dog named Buddy. He currently has two dogs. +``` + +A search on the topic afterwards returns this single complete narrative +rather than the scattered old fragments. + +> To inspect which clusters were consolidated, how many entries were +> archived, etc., see [Auditing & troubleshooting](#auditing--troubleshooting) +> (advanced; not needed for everyday use). +> For debugging without waiting for the schedule, you can trigger a run by +> hand — see [Triggering a run](#triggering-a-run). + +## How it works + +Reflection runs *offline*, separate from the live conversation path. The +online path keeps extracting Episodes and clustering them; Reflection later +consumes those clusters — it never sits between a user and a response: + +``` +Online (never blocked) Offline (scheduled) +────────────────────── ─────────────────── +conversation → Episode → Cluster ───► Reflection consolidates the clusters +``` + +A scheduled run processes every user across all app/project tenants that have +clusters. + +After each conversation, EverOS extracts an **Episode** (a summary of a +conversation segment), and geometric clustering groups semantically similar, +time-adjacent Episodes into a **Cluster**. The same topic thus ends up +scattered as several point-in-time snapshots within one cluster: + +``` +Cluster cl_xxx +├── ep_0001 "Andrew has no pets yet" (August) +├── ep_0002 "Andrew adopted Toby" (September) +└── ep_0003 "Andrew also adopted Buddy" (October) +``` + +Reflection consolidates one cluster at a time, in four steps: + +``` +Select ─→ Merge ─→ Re-extract ─→ Deprecate +``` + +1. **Select** — pick clusters worth consolidating: not yet consolidated and + holding ≥ 2 members, or already consolidated and since joined by new + members. At most 10 clusters per run, largest first. +2. **Merge** — hand the cluster's Episodes to the LLM in chronological order + and merge them into one narrative: preserve facts, resolve contradictions + by keeping the latest state, restore the timeline, drop duplicates, and + end with the current state. A previously consolidated cluster is updated + incrementally — only the new fragments are folded into the existing + narrative. +3. **Re-extract** — the merged narrative is written to Markdown and triggers + re-extraction of atomic facts, keeping derived data consistent with it. +4. **Deprecate** — the replaced original Episodes and their atomic facts get + `deprecated_by` pointing at the new narrative; cluster membership is + updated; an audit record is written. + +The result: + +``` +Cluster cl_xxx +└── ep_0042 "Andrew initially had no pets. He later adopted a dog named + Toby, then another named Buddy. He currently has two dogs." + (originals ep_0001 / ep_0002 / ep_0003 → deprecated_by = ep_0042) +``` + +The merged narrative is, structurally, just an ordinary Episode +(`parent_type="cluster"`) — transparent to retrieval, no search-pipeline +changes required. Default search excludes any memory carrying `deprecated_by`, +so a query like "how many pets does Andrew have" only hits the one complete +narrative. + +## Storage layout + +Memory uses Markdown as the single source of truth; SQLite and LanceDB are +derived indexes built automatically by the cascade daemon. + +| Store | What it holds | Role | +|---|---|---| +| Markdown | Episode bodies, merged narratives, archive markers | Single source of truth; human-readable and editable | +| SQLite | Clusters and members, consolidation audit records | Structured state and queries | +| LanceDB | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | + +The **merged narrative** is written to the Episode daily-log Markdown; its +frontmatter marks that it came from a cluster: + +```yaml +--- +owner_id: u_andrew +timestamp: 2026-10-10T12:00:00+00:00 +parent_type: cluster +parent_id: cl_a1b2c3d4e5f6 +--- +## Subject +Andrew's pet adoption journey + +## Content +Andrew initially had no pets. He later adopted a dog named Toby, and then +adopted another dog named Buddy. He currently has two dogs. +``` + +The **replaced originals** are not deleted. Their file's frontmatter records +the archive mapping, and the index layer writes `deprecated_by`: + +```yaml +--- +# added to the original Episode file's frontmatter: +deprecated_entries: + ep_20260810_0001: ep_20261010_0042 + ep_20260910_0002: ep_20261010_0042 +--- +``` + +> Soft-archive, not delete: even if SQLite / LanceDB are corrupted, as long +> as the Markdown is intact the indexes can be fully rebuilt — and every +> consolidation remains traceable back to its original content. + +## Configuration + +| Setting | Location | Default | Description | +|---|---|---|---| +| `reflect_episodes.enabled` | `/ome.toml` | `false` | Set to `true` to enable (the only setting needed) | +| `reflect_episodes.cron` | `/ome.toml` | `0 2 * * 1` | Run time, as a standard cron expression (`0 2 * * 1` = Mondays at 02:00); **optional**, omit to use the built-in default. Running more than weekly is not recommended | +| `clustering.threshold` | `/everos.toml` | `0.65` | Clustering similarity threshold | +| `clustering.time_window_days` | `/everos.toml` | `7.0` | Clustering time window (days) | + +Two files, two scopes: `ome.toml` holds OME-strategy config (Reflection's +on/off switch and schedule); `everos.toml` holds general settings (clustering +and the like). Both live under the memory root, and you only write the keys +you want to override — everything else falls back to the shipped defaults in +`config/default.toml`, which you never edit by hand. + +Changes to `ome.toml` hot-reload (~1–2s); no server restart needed. + +> Setting `enabled` back to `false` stops the *next* run from starting; a run +> already in progress finishes normally. + +## API reference + +Reflection's normal mode of operation is the scheduled automatic run (see +[Quick start](#quick-start)). The endpoint below triggers a run **on +demand** — for testing, debugging, or when you want to consolidate +immediately. It is an auxiliary path, not the normal mode (there is no +dedicated CLI command). + +### Triggering a run + +``` +POST /api/v1/ome/trigger +Content-Type: application/json +``` + +| Field | Type | Default | Description | +|---|---|---|---| +| `name` | string | — | Strategy name; use `reflect_episodes` | +| `timeout` | float | 120.0 | Max seconds to wait for the run to finish | +| `force` | bool | false | When `true`, runs even if `enabled=false` | + +**Response**: + +```json +{ "status": "ok", "name": "reflect_episodes" } +``` + +`status` is `"ok"` (finished) or `"timeout"` (did not finish in time); an +unknown strategy name returns 404. + +**Example — Python**: + +```python +import httpx + + +async def trigger_reflection() -> str: + async with httpx.AsyncClient(base_url="http://localhost:8000") as client: + resp = await client.post( + "/api/v1/ome/trigger", + json={"name": "reflect_episodes", "timeout": 120, "force": True}, + ) + resp.raise_for_status() + return resp.json()["status"] +``` + +**Example — curl**: + +```bash +curl -X POST http://localhost:8000/api/v1/ome/trigger \ + -H "Content-Type: application/json" \ + -d '{"name": "reflect_episodes", "timeout": 120, "force": true}' +``` + +## Auditing & troubleshooting + +> This section is **advanced**. For everyday use you don't need it — just +> read the Markdown (see [Quick start](#quick-start)). It's here for +> inspecting consolidation details or diagnosing problems. + +Each run writes one `reflection_report` audit record, useful for reviewing +consolidation history: + +| Field | Description | +|---|---| +| `cluster_id` | The cluster that was consolidated | +| `mode` | `init` (first merge) or `update` (incremental update) | +| `source_count` | Number of fragments merged | +| `merged_entry_id` | The merged-narrative Episode produced | +| `deprecated_fact_count` | Number of atomic facts archived alongside | +| `created_at` | Consolidation time | + +```bash +sqlite3 /.index/sqlite/system.db \ + "SELECT cluster_id, mode, source_count, merged_entry_id + FROM reflection_report ORDER BY created_at DESC LIMIT 10;" +``` + +| Symptom | Likely cause | +|---|---| +| No consolidation records after triggering | No eligible clusters (a cluster needs ≥ 2 members) | +| Response `status: "timeout"` | Downstream re-extraction is slow; raise `timeout` and retry | +| Old fragments still appear in search | Index syncs asynchronously, usually 1–3s; wait and retry | +| 404 returned | Strategy name must be `reflect_episodes` | + +## Design notes + +Why Reflection is shaped the way it is: + +- **Offline and scheduled.** Merging is a heavy, lossy LLM operation, so it + runs off the request path — conversations stay fast — and a weekly cadence + lets enough new fragments accumulate to be worth re-merging. +- **Soft-archive, never delete.** Originals stay in Markdown, so every + consolidation is traceable and the indexes can always be rebuilt from the + Markdown source of truth. +- **A merged narrative is just an Episode.** Reusing the Episode type means + search and every downstream consumer keep working unchanged — Reflection + introduces no new retrieval path. + +## Limitations + +- **Merging is lossy** — LLM consolidation may drop individual details. The + original fragments are retained in storage and remain traceable, but are + not in default search results. +- **Clustering is by similarity** — Reflection consolidates the output of + similarity clustering; a single cluster is not guaranteed to be strictly + one topic. +- **No one-click rollback yet** — originals are fully retained, but there is + currently no endpoint to undo a specific consolidation. + +## End-to-end walkthrough + +The walkthrough triggers a run by hand to demonstrate the full flow; in a +real deployment, once enabled it runs automatically on schedule, so this step +isn't needed. + +```bash +BASE=http://localhost:8000/api/v1 + +# 1. With Reflection enabled (set enabled = true in /ome.toml), +# trigger a run by hand here (for the demo; in production it runs on schedule) +curl -s -X POST "$BASE/ome/trigger" \ + -H "Content-Type: application/json" \ + -d '{"name": "reflect_episodes", "timeout": 120, "force": true}' \ + | jq . +# → { "status": "ok", "name": "reflect_episodes" } + +# 2. Review the consolidation audit +sqlite3 /.index/sqlite/system.db \ + "SELECT mode, source_count, merged_entry_id + FROM reflection_report ORDER BY created_at DESC LIMIT 1;" +# → init|3|ep_20261010_0042 + +# 3. Verify via search: the hit is the merged narrative, not the old fragments +curl -s -X POST "$BASE/memory/search" \ + -H "Content-Type: application/json" \ + -d '{"query": "how many pets does Andrew have", "top_k": 5}' \ + | jq '.data.episodes[0] | {subject, episode, session_id}' +# → session_id is null on a merged narrative (the aggregation-product marker); +# episode holds the full narrative text +``` + +## See also + +- [how-memory-works.md](how-memory-works.md) — Episodes and the memory extraction pipeline +- [storage_layout.md](storage_layout.md) — Markdown + SQLite + LanceDB stack +- [api.md](api.md) — full HTTP API reference diff --git a/docs/release-notes-1.1.0.md b/docs/release-notes-1.1.0.md new file mode 100644 index 0000000..91af831 --- /dev/null +++ b/docs/release-notes-1.1.0.md @@ -0,0 +1,52 @@ +# EverOS 1.1.0 Release Notes + +EverOS 1.1.0 expands the memory system beyond user episodes with first-class +knowledge management, reflection, and stronger operational guarantees around +search, persistence, and API contracts. + +## Highlights + +- Added Knowledge APIs and storage for document creation, listing, patching, + deletion, taxonomy handling, and knowledge-topic search. +- Added Reflection orchestration for periodically merging and refining episode + clusters. +- Expanded OME runtime support with event IDs, run-record storage migration, + configuration reload behavior, and testing harness improvements. +- Improved search and get behavior for agent-owned memory, including agent + cases, agent skills, and owner-type isolation. +- Reworked API error handling around typed exception handlers and consistent + error envelopes. +- Updated docs, OpenAPI schema, configuration examples, and test coverage for + the 1.1.0 surface area. + +## Compatibility Notes + +- Existing local TUI demo registration remains available in this PR. +- Existing DashScope rerank support is preserved; the DashScope provider file + is not replaced by the 1.1.0 archive. +- The update intentionally leaves directories outside the 1.1.0 archive scope, + including existing use-case and iOS demo material, untouched. +- Knowledge search requires configured embedding and rerank providers. Missing + providers now fail explicitly with configuration errors rather than silently + returning degraded results. + +## Upgrade Notes + +- Regenerate or review `docs/openapi.json` after route or DTO changes. +- Run `uv sync --frozen` against the updated `uv.lock`. +- Review `config.example.toml`, `src/everos/config/default.toml`, and + `src/everos/config/default_ome.toml` for new Knowledge and OME settings. +- If running e2e tests without live provider credentials, use dummy provider + environment variables for startup-only checks; live vector and rerank paths + remain behind slow/live markers. + +## Verification + +This PR was checked with: + +- `uv run ruff check .` +- `uv run pytest tests/unit` +- `uv run pytest tests/integration` +- `uv run lint-imports` +- `uv run pytest tests/e2e` with dummy LLM and embedding environment variables +- Targeted search/get regression tests for agent and deprecated-filter handling diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 425ca91..9fea9c9 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -9,8 +9,8 @@ derived indexes that can be rebuilt from markdown alone. ## 1. Memory-root tree A memory-root is a single directory holding all persisted memory. The -default location is `~/.everos/`; override via `EVEROS_MEMORY__ROOT` -env var or `[memory] root` in the TOML config. +default location is `~/.everos/`; override via the `EVEROS_ROOT` +env var or `--root` on the CLI. Memory is partitioned by **`/`** *before* the user-visible scope dirs, so different `(app, project)` spaces never share @@ -64,9 +64,9 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py), exposing every path as a property. `MemoryRoot.ensure()` creates the -runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`) and copies the -OME template to `ome.toml`; the user-visible dirs are *not* pre-created — -they appear on first write. +runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); the +user-visible dirs are *not* pre-created — they appear on first write. +Config files (`everos.toml`, `ome.toml`) are created by `everos init`. > The single-file writer also supports `agent.md` / `soul.md` / `tools.md` > / `behaviors.md`, but no shipped strategy produces those today — only @@ -89,11 +89,8 @@ business-aware writers live in [`infra/persistence/markdown/writers/`](../src/everos/infra/persistence/markdown/writers/) and pick the right strategy via a base class. -To add a new memory kind, define its per-kind frontmatter schema under -[`infra/persistence/markdown/mds/`](../src/everos/infra/persistence/markdown/mds/) -and add a matching writer/reader pair under -[`writers/`](../src/everos/infra/persistence/markdown/writers/) and -[`readers/`](../src/everos/infra/persistence/markdown/readers/). +For a step-by-step recipe to add a new memory kind, see the +[`/add-memory-kind`](../.claude/skills/add-memory-kind/SKILL.md) skill. ## 3. Frontmatter chassis (YAML) @@ -178,20 +175,30 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers ``` .index/ ├── sqlite/ -│ └── system.db state / audit log / task queue / LSN watermark -│ + per-kind business state tables (composite key) +│ └── system.db state / audit / cascade queue + buffer / LSN +│ (system tables: md_change_state, memcell, +│ unprocessed_buffer, conversation_status, cluster) └── lancedb/ - └── .lance/ one Arrow-based table per kind - stores text / vector / tags / metadata + └── .lance/ one Arrow table per business kind — the per-kind + rows (text / vector / tokens / metadata) live here ``` -- **SQLite** schema lives in - [`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/); - every business table that joins back to markdown declares a - `UniqueConstraint("user_id", "entry_id")` (or `agent_id` symmetric). -- **LanceDB** schemas live in - [`infra/persistence/lancedb/tables/`](../src/everos/infra/persistence/lancedb/tables/); - `Vector(N)` dimension matches the embedding model output. +- **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) + holds only system / coordination tables — `md_change_state` (cascade + queue), `memcell` (boundary ledger), `unprocessed_buffer`, + `conversation_status`, `cluster`, `reflection_report` — **not** + per-kind business rows. `reflection_report` is the audit trail for + Reflection merges (cluster_id, mode, source_members, merged_entry_id, + status). +- **LanceDB** ([`infra/persistence/lancedb/tables/`](../src/everos/infra/persistence/lancedb/tables/)) + holds the per-kind business rows, keyed `_` (so + cross-table joins use `(owner_id, entry_id)`); each table's `Vector(N)` + dimension matches the embedding model output. + +Episode and AtomicFact LanceDB tables carry a `deprecated_by: str | None` +column. When an episode is superseded by a Reflection merge, +`deprecated_by` is set to the merged episode's entry_id. Search filters +automatically exclude rows where `deprecated_by IS NOT NULL`. Both layers are **fully derivable from markdown** — wipe `.index/` and the in-process cascade subsystem re-builds everything by scanning the @@ -210,13 +217,14 @@ appends an entry block → atomic write back. The caller passes a full `EntryId` (built via `EntryId.next_for(prefix, date, current_count)`); this primitive is **schema-agnostic** — field-level semantics (`entry_count` / `last_appended_at`) are a business writer's job -(see `BaseDailyAppender._frontmatter_updates` in +(see `BaseDailyWriter._frontmatter_updates` in [`infra/persistence/markdown/writers/base.py`](../src/everos/infra/persistence/markdown/writers/base.py)). ## 7. References +- Skill: [`/add-memory-kind`](../.claude/skills/add-memory-kind/SKILL.md) - Code: - - [`core/persistence/memory_root.py`](../src/everos/core/persistence/memory_root.py) — memory-root resolution - - [`core/persistence/markdown/`](../src/everos/core/persistence/markdown/) — schema-agnostic read/write chassis - - [`infra/persistence/markdown/mds/`](../src/everos/infra/persistence/markdown/mds/) — per-kind frontmatter schemas - - [`infra/persistence/{markdown,sqlite,lancedb}/`](../src/everos/infra/persistence/) — business-aware adapters + - [`core/persistence/memory_root.py`](../src/everos/core/persistence/memory_root.py) + - [`core/persistence/markdown/`](../src/everos/core/persistence/markdown/) + - [`infra/persistence/{markdown,sqlite,lancedb}/`](../src/everos/infra/persistence/) + - [`memory/cascade/`](../src/everos/memory/cascade/) (md → LanceDB sync) diff --git a/pyproject.toml b/pyproject.toml index a06c1a0..2edd079 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "everos" -version = "1.0.1" +version = "1.1.0" description = "EverOS — local-first markdown memory framework for AI agents and user chats; lightweight, dev-friendly, small-team" license = {text = "Apache-2.0"} readme = "README.md" @@ -39,6 +39,7 @@ dependencies = [ # Web framework (entrypoints/api) "fastapi>=0.104.0", + "python-multipart>=0.0.7", # Required by FastAPI for Form / UploadFile endpoints "uvicorn[standard]>=0.24.0", # Observability (core/observability) @@ -59,15 +60,14 @@ dependencies = [ "anyio>=4.0", # Async file I/O (anyio.Path, to_thread.run_sync) for the markdown layer # Algorithm library (everalgo monorepo, published on PyPI). - "everalgo-core==0.2.0", - "everalgo-boundary==0.2.0", - "everalgo-user-memory==0.2.0", - "everalgo-agent-memory==0.2.0", - "everalgo-rank==0.3.0", + "everalgo-user-memory==0.3.1", + "everalgo-agent-memory==0.3.1", + "everalgo-rank==0.4.1", + "everalgo-knowledge==0.1.1", ] [project.optional-dependencies] -multimodal = ["everalgo-parser[svg]>=0.1.0"] # [svg] bundles cairosvg → SVG works by default +multimodal = ["everalgo-parser[svg]>=0.2.1"] # [svg] bundles cairosvg → SVG works by default [build-system] requires = ["hatchling"] @@ -136,6 +136,21 @@ python_files = ["test_*.py"] python_functions = ["test_*"] asyncio_mode = "auto" addopts = "-v --tb=short -m 'not slow and not live_llm'" +filterwarnings = [ + "error", + # jieba 0.42.1: unescaped backslashes in regex strings (upstream + # unmaintained; no fix available). Python 3.12 emits + # SyntaxWarning for invalid escape sequences. + "ignore:invalid escape sequence:SyntaxWarning", + # aiosqlite teardown: connection-worker thread and Connection.__del__ + # raise after pytest closes the event loop. Upstream issue + # (aiosqlite + pytest-asyncio); we cannot fix their lifecycle. + "ignore:Exception in thread.*_connection_worker_thread:pytest.PytestUnhandledThreadExceptionWarning", + # aiosqlite + asyncio teardown: Connection.__del__ and + # BaseEventLoop.__del__ fire after pytest closes the event loop. + # Upstream lifecycle issue; we cannot fix their cleanup order. + "ignore:Exception ignored.*:pytest.PytestUnraisableExceptionWarning", +] markers = [ "slow: runs that take >=10s (regardless of dependency); CI default excludes these. Run with `pytest -m slow`.", "live_llm: requires real LLM / embedder / reranker credentials from .env; burns tokens. CI default excludes. Run with `pytest -m live_llm`.", diff --git a/scripts/check_consistency.py b/scripts/check_consistency.py index 3b8bdae..80923a3 100755 --- a/scripts/check_consistency.py +++ b/scripts/check_consistency.py @@ -159,7 +159,7 @@ def _print_monotonicity(reports: list[MonotonicityReport]) -> int: async def run_lifespan_mode(corpus: Path) -> int: """Full strict check via app lifespan; covers every kind in KIND_REGISTRY.""" - os.environ["EVEROS_MEMORY__ROOT"] = str(corpus) + os.environ["EVEROS_ROOT"] = str(corpus) from everos.config import load_settings load_settings.cache_clear() diff --git a/scripts/check_deprecated_names.py b/scripts/check_deprecated_names.py index 07d21a9..f8ba8e6 100644 --- a/scripts/check_deprecated_names.py +++ b/scripts/check_deprecated_names.py @@ -58,6 +58,8 @@ def _tracked_paths() -> list[Path]: def _tracked_text_files() -> Iterable[tuple[str, str]]: for path in _tracked_paths(): + if not path.exists(): + continue if path.suffix.lower() in SKIP_SUFFIXES: continue try: diff --git a/scripts/e2e_memorize/README.md b/scripts/e2e_memorize/README.md index 97ba45e..7cd9373 100644 --- a/scripts/e2e_memorize/README.md +++ b/scripts/e2e_memorize/README.md @@ -16,9 +16,9 @@ batching by 6 messages per `/add` call and then `/flush` at the end. 1. **LLM client configured** in `.env`: - `EVEROS_LLM__API_KEY=...` - `EVEROS_LLM__BASE_URL=...` (OpenAI-compatible) - - `EVEROS_LLM__MODEL=...` (defaults to `gpt-4o-mini`) + - `EVEROS_LLM__MODEL=...` (defaults to `gpt-4.1-mini`) - Without these, the boundary stage logs `memorize_no_llm_client` and skips the run. -2. **Memory root**: defaults to `~/.everos`; override with `EVEROS_MEMORY__ROOT=...`. +2. **Memory root**: defaults to `~/.everos`; override with `EVEROS_ROOT=...`. 3. **Mode** is read from `settings.memorize.mode` (toml/env) before the first `memorize()` call. ## Run diff --git a/src/everos/component/embedding/__init__.py b/src/everos/component/embedding/__init__.py index 98f0e1e..f1ec987 100644 --- a/src/everos/component/embedding/__init__.py +++ b/src/everos/component/embedding/__init__.py @@ -4,7 +4,8 @@ Public surface: - :class:`EmbeddingProvider` — Protocol every provider satisfies. -- :class:`EmbeddingError` — provider-side failure. +- :class:`EmbeddingServiceError` — provider-side failure. +- :class:`EmbeddingError` — backward-compat alias for ``EmbeddingServiceError``. - :class:`OpenAIEmbeddingProvider` — concrete provider for any OpenAI-protocol embeddings endpoint (DeepInfra, vLLM, OpenAI, …). - :func:`build_embedding_provider` — settings-driven factory. @@ -16,6 +17,8 @@ External usage:: vec = await provider.embed("hello") """ +from everos.core.errors import EmbeddingServiceError as EmbeddingServiceError + from .accessor import EmbeddingNotConfiguredError as EmbeddingNotConfiguredError from .accessor import get_embedder as get_embedder from .factory import build_embedding_provider as build_embedding_provider @@ -27,6 +30,7 @@ __all__ = [ "EmbeddingError", "EmbeddingNotConfiguredError", "EmbeddingProvider", + "EmbeddingServiceError", "OpenAIEmbeddingProvider", "build_embedding_provider", "get_embedder", diff --git a/src/everos/component/embedding/openai_provider.py b/src/everos/component/embedding/openai_provider.py index 836a5eb..f756127 100644 --- a/src/everos/component/embedding/openai_provider.py +++ b/src/everos/component/embedding/openai_provider.py @@ -23,7 +23,7 @@ from collections.abc import Sequence import openai -from .protocol import EmbeddingError +from .protocol import EmbeddingServiceError class OpenAIEmbeddingProvider: @@ -93,6 +93,6 @@ class OpenAIEmbeddingProvider: input=chunk, ) except openai.OpenAIError as exc: - raise EmbeddingError(str(exc)) from exc + raise EmbeddingServiceError(str(exc)) from exc # OpenAI returns ``data`` indexed by request order; truncate to ``dim``. return [list(item.embedding[: self.dim]) for item in response.data] diff --git a/src/everos/component/embedding/protocol.py b/src/everos/component/embedding/protocol.py index 5f6fa03..24bc0a2 100644 --- a/src/everos/component/embedding/protocol.py +++ b/src/everos/component/embedding/protocol.py @@ -13,13 +13,10 @@ from __future__ import annotations from collections.abc import Sequence from typing import Protocol, runtime_checkable +from everos.core.errors import EmbeddingServiceError as EmbeddingServiceError -class EmbeddingError(Exception): - """Raised on any provider-side embedding failure. - - Wraps the upstream SDK exception via ``__cause__`` (PEP 3134) so - diagnostic loggers preserve the original error chain. - """ +# Backward compat — old name still importable from this module. +EmbeddingError = EmbeddingServiceError @runtime_checkable @@ -42,7 +39,7 @@ class EmbeddingProvider(Protocol): Implementations chunk by ``batch_size`` and bound in-flight requests by ``max_concurrent`` (both from settings). On failure, - raises :class:`EmbeddingError` — the worker treats it as a + raises :class:`EmbeddingServiceError` — the worker treats it as a retryable / unrecoverable case per HTTP-status mapping. """ ... diff --git a/src/everos/component/parser/__init__.py b/src/everos/component/parser/__init__.py new file mode 100644 index 0000000..09cc205 --- /dev/null +++ b/src/everos/component/parser/__init__.py @@ -0,0 +1,17 @@ +"""component.parser — shared multimodal file parsing via everalgo-parser. + +External usage: + from everos.component.parser import aparse_file, parser_available +""" + +from __future__ import annotations + +from ._core import aparse_file as aparse_file +from ._core import parser_available as parser_available +from ._core import require_parser as require_parser + +__all__ = [ + "aparse_file", + "parser_available", + "require_parser", +] diff --git a/src/everos/component/parser/_core.py b/src/everos/component/parser/_core.py new file mode 100644 index 0000000..a93e57d --- /dev/null +++ b/src/everos/component/parser/_core.py @@ -0,0 +1,67 @@ +"""Core parse dispatch — wraps everalgo-parser with LLM injection and error mapping. + +``everalgo-parser`` is an optional dependency (``everos[multimodal]``). +All imports are deferred so this module is safe to import without the extra. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from everos.core.errors import MultimodalNotEnabledError, UnsupportedModalityError + +if TYPE_CHECKING: + from everalgo.types import ParsedContent, RawFile + + +def parser_available() -> bool: + """Whether ``everalgo.parser`` is importable.""" + try: + import everalgo.parser # noqa: F401 + except ImportError: + return False + return True + + +def require_parser() -> None: + """Raise when the parser extra is not installed. + + Raises: + MultimodalNotEnabledError: When ``everalgo.parser`` cannot be imported. + """ + if not parser_available(): + raise MultimodalNotEnabledError( + "Multimodal parsing requires the parser extra. " + "Install with: pip install 'everos[multimodal]'" + ) + + +async def aparse_file(raw_file: RawFile) -> ParsedContent: + """Parse a file via everalgo-parser with the multimodal LLM client. + + Args: + raw_file: Hydrated ``RawFile`` with ``content`` bytes or ``uri``. + + Returns: + Parsed text content with modality metadata. + + Raises: + MultimodalNotEnabledError: Parser not installed or system dep missing. + UnsupportedModalityError: File format not supported by the parser. + """ + from everalgo.llm import LLMError + from everalgo.parser import aparse # Deferred: optional dep + + from everos.component.llm import get_multimodal_llm_client + from everos.core.errors import LLMServiceError + + try: + return await aparse(raw_file, llm=get_multimodal_llm_client()) + except NotImplementedError as exc: + raise UnsupportedModalityError(f"modality not supported: {exc}") from exc + except LLMError as exc: + raise LLMServiceError(str(exc)) from exc + except ValueError as exc: + raise UnsupportedModalityError(str(exc)) from exc + except RuntimeError as exc: + raise MultimodalNotEnabledError(str(exc)) from exc diff --git a/src/everos/component/rerank/__init__.py b/src/everos/component/rerank/__init__.py index ebea251..b36f306 100644 --- a/src/everos/component/rerank/__init__.py +++ b/src/everos/component/rerank/__init__.py @@ -3,12 +3,12 @@ Public surface: - :class:`RerankProvider` — Protocol every provider satisfies. -- :class:`RerankResult` / :class:`RerankError` — value type + error. +- :class:`RerankResult` / :class:`RerankServiceError` — value type + error. +- :class:`RerankError` — backward-compat alias for :class:`RerankServiceError`. - :class:`DeepInfraRerankProvider` — DeepInfra inference-API rerank. +- :class:`DashScopeRerankProvider` — Aliyun Bailian / DashScope rerank. - :class:`VllmRerankProvider` — OpenAI-compat ``/v1/rerank`` (vLLM, self-hosted, other compatible servers). -- :class:`DashScopeRerankProvider` — Aliyun Bailian / DashScope - ``gte-rerank-v2`` native ``text-rerank`` endpoint. - :func:`build_rerank_provider` — settings-driven factory that picks the concrete provider via ``settings.rerank.provider``. @@ -19,6 +19,8 @@ External usage:: scored = await provider.rerank("how to file a claim", documents) """ +from everos.core.errors import RerankServiceError as RerankServiceError + from .dashscope_provider import DashScopeRerankProvider as DashScopeRerankProvider from .deepinfra_provider import DeepInfraRerankProvider as DeepInfraRerankProvider from .factory import build_rerank_provider as build_rerank_provider @@ -33,6 +35,7 @@ __all__ = [ "RerankError", "RerankProvider", "RerankResult", + "RerankServiceError", "VllmRerankProvider", "build_rerank_provider", ] diff --git a/src/everos/component/rerank/_errors.py b/src/everos/component/rerank/_errors.py new file mode 100644 index 0000000..65480b8 --- /dev/null +++ b/src/everos/component/rerank/_errors.py @@ -0,0 +1,45 @@ +"""Shared error construction for HTTP-based rerank providers.""" + +from __future__ import annotations + +import httpx + +from everos.core.observability.logging import get_logger + +from .protocol import RerankServiceError + +logger = get_logger(__name__) + + +def upstream_http_error(provider: str, response: httpx.Response) -> RerankServiceError: + """Log the upstream response body and return a client-safe error. + + The body can carry provider-internal detail, so it is logged rather than + surfaced in the returned message (which exposes only the status code). + + Args: + provider: Human-readable provider label (e.g. ``"vLLM"``). + response: The non-success HTTP response from the rerank backend. + + Returns: + A :class:`RerankServiceError` naming the provider and status code. + """ + logger.warning( + "rerank_http_error", + provider=provider, + status=response.status_code, + body=response.text[:200], + ) + return RerankServiceError( + f"{provider} rerank upstream error (HTTP {response.status_code})." + ) + + +def transport_error(provider: str, exc: Exception) -> RerankServiceError: + """Build the error for a transport-level failure (connect / timeout).""" + return RerankServiceError(f"{provider} rerank transport failure: {exc}") + + +def retries_exhausted_error(provider: str, max_retries: int) -> RerankServiceError: + """Build the error for a retry loop that fell through without a result.""" + return RerankServiceError(f"{provider} rerank exhausted retries ({max_retries}).") diff --git a/src/everos/component/rerank/deepinfra_provider.py b/src/everos/component/rerank/deepinfra_provider.py index 82cbc61..8ef5b85 100644 --- a/src/everos/component/rerank/deepinfra_provider.py +++ b/src/everos/component/rerank/deepinfra_provider.py @@ -35,11 +35,12 @@ from typing import Any import httpx -from .protocol import RerankError, RerankResult +from ._errors import retries_exhausted_error, transport_error, upstream_http_error +from .protocol import RerankResult, RerankServiceError # Qwen3-Reranker chat template. The DeepInfra inference API treats the reranker # as a yes/no generator, so the prompt scaffolding must be supplied client-side -# (verbatim mirror of the benchmark reranker client). Without it the +# (verbatim mirror of the EverAlgo benchmark's reranker client). Without it the # model scores raw text off-template and returns uncalibrated relevance. _QWEN3_PREFIX = ( "<|im_start|>system\n" @@ -149,9 +150,7 @@ class DeepInfraRerankProvider: ) except httpx.HTTPError as exc: if attempt == self._max_retries: - raise RerankError( - f"DeepInfra rerank transport failure: {exc}" - ) from exc + raise transport_error("DeepInfra", exc) from exc continue if response.status_code == 200: @@ -160,19 +159,11 @@ class DeepInfraRerankProvider: # Retry on 5xx / 429 only; surface 4xx immediately. if response.status_code >= 500 or response.status_code == 429: if attempt == self._max_retries: - raise RerankError( - f"DeepInfra rerank HTTP {response.status_code}: " - f"{response.text[:200]}" - ) + raise upstream_http_error("DeepInfra", response) continue - raise RerankError( - f"DeepInfra rerank HTTP {response.status_code}: " - f"{response.text[:200]}" - ) + raise upstream_http_error("DeepInfra", response) - raise RerankError( - f"DeepInfra rerank exhausted retries ({self._max_retries})" - ) + raise retries_exhausted_error("DeepInfra", self._max_retries) def _extract_scores(body: dict[str, Any], expected_len: int) -> list[float]: @@ -187,10 +178,10 @@ def _extract_scores(body: dict[str, Any], expected_len: int) -> list[float]: """ raw = body.get("scores") if not isinstance(raw, list): - raise RerankError(f"DeepInfra rerank response missing scores: {body!r}") + raise RerankServiceError(f"DeepInfra rerank response missing scores: {body!r}") row = raw[0] if raw and isinstance(raw[0], list) else raw if len(row) != expected_len: - raise RerankError( + raise RerankServiceError( f"DeepInfra rerank returned {len(row)} scores, expected {expected_len}" ) return [float(s) for s in row] diff --git a/src/everos/component/rerank/factory.py b/src/everos/component/rerank/factory.py index 760d341..1161a35 100644 --- a/src/everos/component/rerank/factory.py +++ b/src/everos/component/rerank/factory.py @@ -26,7 +26,7 @@ from .vllm_provider import VllmRerankProvider logger = get_logger(__name__) -# host substring → provider. Ordered most-specific first; matched against +# host substring -> provider. Ordered most-specific first; matched against # the ``base_url`` host so a Bailian / DeepInfra URL routes to the right # request-shape without the operator also having to set ``provider``. _PROVIDER_HOST_HINTS: tuple[tuple[str, str], ...] = ( diff --git a/src/everos/component/rerank/protocol.py b/src/everos/component/rerank/protocol.py index 341fa84..5de595a 100644 --- a/src/everos/component/rerank/protocol.py +++ b/src/everos/component/rerank/protocol.py @@ -14,9 +14,10 @@ from __future__ import annotations from collections.abc import Sequence from typing import NamedTuple, Protocol, runtime_checkable +from everos.core.errors import RerankServiceError as RerankServiceError -class RerankError(Exception): - """Raised on any provider-side rerank failure.""" +# Backward compat alias. +RerankError = RerankServiceError class RerankResult(NamedTuple): diff --git a/src/everos/component/rerank/vllm_provider.py b/src/everos/component/rerank/vllm_provider.py index 7dbd836..cfedcfb 100644 --- a/src/everos/component/rerank/vllm_provider.py +++ b/src/everos/component/rerank/vllm_provider.py @@ -41,7 +41,8 @@ from typing import Any import httpx -from .protocol import RerankError, RerankResult +from ._errors import retries_exhausted_error, transport_error, upstream_http_error +from .protocol import RerankResult, RerankServiceError class VllmRerankProvider: @@ -133,9 +134,7 @@ class VllmRerankProvider: ) except httpx.HTTPError as exc: if attempt == self._max_retries: - raise RerankError( - f"vLLM rerank transport failure: {exc}" - ) from exc + raise transport_error("vLLM", exc) from exc continue if response.status_code == 200: @@ -143,22 +142,17 @@ class VllmRerankProvider: if response.status_code >= 500 or response.status_code == 429: if attempt == self._max_retries: - raise RerankError( - f"vLLM rerank HTTP {response.status_code}: " - f"{response.text[:200]}" - ) + raise upstream_http_error("vLLM", response) continue - raise RerankError( - f"vLLM rerank HTTP {response.status_code}: {response.text[:200]}" - ) + raise upstream_http_error("vLLM", response) - raise RerankError(f"vLLM rerank exhausted retries ({self._max_retries})") + raise retries_exhausted_error("vLLM", self._max_retries) def _parse_rerank_results(body: dict[str, Any]) -> list[RerankResult]: items = body.get("results") if not isinstance(items, list): - raise RerankError(f"vLLM rerank response missing results: {body!r}") + raise RerankServiceError(f"vLLM rerank response missing results: {body!r}") parsed: list[RerankResult] = [] for item in items: try: @@ -169,5 +163,7 @@ def _parse_rerank_results(body: dict[str, Any]) -> list[RerankResult]: ) ) except (KeyError, TypeError, ValueError) as exc: - raise RerankError(f"malformed rerank result entry: {item!r}") from exc + raise RerankServiceError( + f"malformed rerank result entry: {item!r}" + ) from exc return parsed diff --git a/src/everos/component/tokenizer/__init__.py b/src/everos/component/tokenizer/__init__.py index fec08f3..7bee0dd 100644 --- a/src/everos/component/tokenizer/__init__.py +++ b/src/everos/component/tokenizer/__init__.py @@ -14,11 +14,9 @@ External usage:: """ from .factory import build_tokenizer as build_tokenizer -from .jieba_provider import JiebaTokenizer as JiebaTokenizer from .protocol import Tokenizer as Tokenizer __all__ = [ - "JiebaTokenizer", "Tokenizer", "build_tokenizer", ] diff --git a/src/everos/component/tokenizer/factory.py b/src/everos/component/tokenizer/factory.py index 9b88f75..29fa488 100644 --- a/src/everos/component/tokenizer/factory.py +++ b/src/everos/component/tokenizer/factory.py @@ -8,10 +8,14 @@ change — see ``17_lancedb_tables_design.md`` §2.4.1. from __future__ import annotations -from .jieba_provider import JiebaTokenizer from .protocol import Tokenizer def build_tokenizer() -> Tokenizer: """Build the default tokenizer (``JiebaTokenizer``).""" + # Deferred: jieba contains invalid escape sequences that raise + # SyntaxError on Python 3.12+; defer so the cost is paid only when + # tokenization is actually needed (not at import time). + from .jieba_provider import JiebaTokenizer + return JiebaTokenizer() diff --git a/src/everos/config/__init__.py b/src/everos/config/__init__.py index c7fe834..c295902 100644 --- a/src/everos/config/__init__.py +++ b/src/everos/config/__init__.py @@ -5,7 +5,7 @@ Public API: Settings, MemorySettings, SqliteSettings, LanceDBSettings, LLMSettings, EmbeddingSettings, RerankSettings, BoundaryDetectionSettings, - load_settings, + load_settings, resolve_root, ) Distinct from ``everos.component.config`` (which is a *capability* — @@ -22,6 +22,7 @@ from .settings import RerankSettings as RerankSettings from .settings import Settings as Settings from .settings import SqliteSettings as SqliteSettings from .settings import load_settings as load_settings +from .settings import resolve_root as resolve_root __all__ = [ "BoundaryDetectionSettings", @@ -34,4 +35,5 @@ __all__ = [ "Settings", "SqliteSettings", "load_settings", + "resolve_root", ] diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index 08d3ab3..d96a439 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -2,19 +2,15 @@ # # Lookup order (later overrides earlier): # 1. This file (shipped defaults; lowest priority) -# 2. ~/.everos/config.toml — user-level overrides (optional; -# path is overridable via EVEROS_CONFIG_FILE) -# 3. .env file in the working directory -# 4. Environment variables — EVEROS_
__ +# 2. /everos.toml — user config (optional; root resolved by +# resolve_root(): EVEROS_ROOT env > ~/.everos) +# 3. Environment variables — EVEROS_
__ # e.g. EVEROS_SQLITE__BUSY_TIMEOUT_MS=10000 -# 5. Programmatic init args (highest priority) +# 4. Programmatic init args (highest priority) # # `null` (omitted in TOML) means "use the Pydantic default declared in code". [memory] -# memory-root is the single directory holding all persisted memory. -# `~` is expanded; the path is resolved when MemoryRoot is constructed. -root = "~/.everos" # Effective timezone for date buckets and timestamps. Drives # component.utils.datetime; this is the SOLE source — OS `TZ` is not # read. Override via `EVEROS_MEMORY__TIMEZONE` env var if needed. @@ -57,8 +53,8 @@ cache_size_kb = 2048 [llm] # Provider-agnostic OpenAI-protocol client config. Override via env: # EVEROS_LLM__MODEL, EVEROS_LLM__API_KEY, EVEROS_LLM__BASE_URL -# Or via a ``.env`` file next to the project root (auto-loaded). -model = "gpt-4o-mini" +# Or set the field directly in this file (/everos.toml). +model = "gpt-4.1-mini" # api_key = "" # base_url = "" @@ -135,6 +131,22 @@ vector_strategy = "maxsim_atomic" # requires a restart. Override via EVEROS_MEMORIZE__MODE. mode = "agent" +[knowledge] +# Max bytes for an uploaded knowledge document (default 50 MiB). Oversized +# uploads are rejected with HTTP 422 before parsing/extraction. Note: the +# multipart body is still buffered by the server first, so set a reverse-proxy +# / gateway body-size limit for hard ingress protection. +# Override via EVEROS_KNOWLEDGE__MAX_UPLOAD_BYTES. +max_upload_bytes = 52_428_800 # 50 MiB + +[knowledge.search] +recall_n = 200 +rerank_n = 50 +# "lambda" is a Python keyword — aliased as "lam" in Settings +lambda = 0.1 +mass_top_m = 50 +top_k_cap = 100 + # Maximum wall-clock for one memorize() invocation while holding the # per-session lock. On timeout the outer asyncio.timeout cancels the call # and the lock auto-releases so subsequent concurrent /add on the same @@ -142,3 +154,11 @@ mode = "agent" # synchronous portion of pipeline dispatch. # Override via EVEROS_MEMORIZE__SESSION_LOCK_TIMEOUT_SECONDS. session_lock_timeout_seconds = 360.0 + +[clustering] +# Geometry-clustering: cosine similarity threshold and time window. +# Episodes older than ``time_window_days`` from the newest cluster +# member are excluded from merge consideration. +# Override via EVEROS_CLUSTERING__THRESHOLD, EVEROS_CLUSTERING__TIME_WINDOW_DAYS. +threshold = 0.65 +time_window_days = 7.0 diff --git a/src/everos/config/default_ome.toml b/src/everos/config/default_ome.toml index 5094d64..864d724 100644 --- a/src/everos/config/default_ome.toml +++ b/src/everos/config/default_ome.toml @@ -1,11 +1,12 @@ -# everos OME (Offline Memory Engine) — per-strategy overrides. +# everos OME (Offline Memory Engine) — strategy configuration. # -# This file is materialised at ``/ome.toml`` by -# ``MemoryRoot.ensure()`` on first server start. Edit it to toggle -# individual strategies or tweak their gate / retry / cron without -# restarting the server; the engine watches this file and hot-reloads -# changes within ~2 seconds. Re-running ``ensure()`` will NOT overwrite -# your edits — the file is only materialised when absent. +# THIS IS THE SINGLE ENTRY POINT for all strategy settings (enabled, +# cron, max_retries, gate). Edit this file to toggle strategies on/off +# or tune their scheduling — changes are hot-reloaded within ~2 seconds, +# no server restart needed. +# +# Created at ``~/.everos/ome.toml`` on first server start. Editing the +# file after creation is safe — it will not be overwritten. # # Overrides are partial: only the keys you set replace the in-code # defaults; omitted keys keep each strategy's coded value. Unknown @@ -46,6 +47,17 @@ # [strategies.extract_user_profile] # enabled = false +# ── Reflection ────────────────────────────────────────────────────────── + +# Offline memory consolidation. Disabled by default — uncomment to enable. +# Once enabled, runs on the default schedule: weekly, Monday 02:00. +# Run Reflection sparingly — at most once a week. Triggering it too +# often degrades quality, since each run is a lossy LLM re-merge. +# The cron expression can be overridden without restarting the server. +# [strategies.reflect_episodes] +# enabled = true +# cron = "0 2 * * 1" + # ── Agent-memory pipeline ─────────────────────────────────────────────── # Agent case extraction (runs per agent memcell). One per tool call cycle. diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index fd57a7c..bdde544 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -3,14 +3,13 @@ Loaded by :func:`load_settings`. Source priority (later wins): 1. ``config/default.toml`` (shipped values; lowest priority) - 2. ``~/.everos/config.toml`` (user-level overrides; optional) - 3. ``.env`` file in the working directory (secrets / machine-specific) - 4. ``EVEROS_
__`` environment variables - 5. Init args passed programmatically (highest priority) + 2. ``/everos.toml`` (user config; optional; ```` resolved by + :func:`resolve_root`) + 3. ``EVEROS_
__`` environment variables + 4. Init args passed programmatically (highest priority) -The user-level toml path defaults to ``~/.everos/config.toml``. Override -with the ``EVEROS_CONFIG_FILE`` environment variable. The file is -optional — if it does not exist, the source is silently skipped. +The memory root is resolved by :func:`resolve_root`: +``explicit arg > EVEROS_ROOT env > ~/.everos``. The settings tree mirrors the TOML structure: ``settings.sqlite.busy_timeout_ms`` maps to ``[sqlite].busy_timeout_ms`` and to ``EVEROS_SQLITE__BUSY_TIMEOUT_MS``. @@ -29,7 +28,7 @@ from pathlib import Path from typing import Literal from zoneinfo import ZoneInfo, ZoneInfoNotFoundError -from pydantic import BaseModel, Field, SecretStr, field_validator +from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator from pydantic_settings import ( BaseSettings, PydanticBaseSettingsSource, @@ -38,24 +37,31 @@ from pydantic_settings import ( ) _DEFAULT_TOML_PATH = Path(__file__).parent / "default.toml" -_USER_TOML_ENV_VAR = "EVEROS_CONFIG_FILE" -_DEFAULT_USER_TOML_PATH = Path("~/.everos/config.toml").expanduser() +_DEFAULT_ROOT = Path("~/.everos") -def _resolve_user_toml_path() -> Path: - """Resolve the user-level ``config.toml`` path. +def resolve_root(explicit: str | None = None) -> Path: + """Resolve the memory-root path. - Defaults to ``~/.everos/config.toml``; override with the - ``EVEROS_CONFIG_FILE`` environment variable. + Priority: explicit arg > EVEROS_ROOT env > ~/.everos default. + + Args: + explicit: Caller-supplied path string (e.g. from ``--root`` CLI flag). + + Returns: + Absolute resolved path to the memory root. """ - override = os.environ.get(_USER_TOML_ENV_VAR) - return Path(override).expanduser() if override else _DEFAULT_USER_TOML_PATH + if explicit: + return Path(explicit).expanduser().resolve() + from_env = os.environ.get("EVEROS_ROOT") + if from_env: + return Path(from_env).expanduser().resolve() + return _DEFAULT_ROOT.expanduser().resolve() class MemorySettings(BaseModel): - """memory-root configuration.""" + """Memory configuration.""" - root: Path = Path("~/.everos") timezone: str = "UTC" """Effective timezone for date buckets and timestamps. @@ -123,7 +129,7 @@ class LLMSettings(BaseModel): EVEROS_LLM__BASE_URL """ - model: str = "gpt-4o-mini" + model: str = "gpt-4.1-mini" api_key: SecretStr | None = None base_url: str | None = None @@ -258,6 +264,18 @@ class MemorizeSettings(BaseModel): session_lock_timeout_seconds: float = Field(default=360.0, gt=0) +class ClusteringSettings(BaseModel): + """Geometry-clustering tunables. + + Env binding: + EVEROS_CLUSTERING__THRESHOLD + EVEROS_CLUSTERING__TIME_WINDOW_DAYS + """ + + threshold: float = Field(default=0.65, gt=0, le=1) + time_window_days: float = Field(default=7.0, gt=0) + + class SearchSettings(BaseModel): """Search-pipeline policy knobs. @@ -343,6 +361,25 @@ class LanceDBSettings(BaseModel): index_cache_size_bytes: int = 16 * 1024 * 1024 +class KnowledgeSearchSettings(BaseModel): + """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" + + recall_n: int = 200 + rerank_n: int = 50 + mass_top_m: int = 50 + lam: float = Field(0.1, alias="lambda") + top_k_cap: int = 100 + + model_config = ConfigDict(populate_by_name=True) + + +class KnowledgeSettings(BaseModel): + """``[knowledge]`` — knowledge module configuration.""" + + max_upload_bytes: int = 52_428_800 # 50 MiB + search: KnowledgeSearchSettings = KnowledgeSearchSettings() + + class Settings(BaseSettings): """Top-level application settings.""" @@ -355,18 +392,44 @@ class Settings(BaseSettings): rerank: RerankSettings = RerankSettings() boundary_detection: BoundaryDetectionSettings = BoundaryDetectionSettings() memorize: MemorizeSettings = MemorizeSettings() + clustering: ClusteringSettings = ClusteringSettings() search: SearchSettings = SearchSettings() multimodal: MultimodalSettings = MultimodalSettings() + knowledge: KnowledgeSettings = KnowledgeSettings() model_config = SettingsConfigDict( env_prefix="EVEROS_", env_nested_delimiter="__", - env_file=".env", - env_file_encoding="utf-8", toml_file=_DEFAULT_TOML_PATH, extra="ignore", ) + def __init__(self, *, _everos_root: Path | None = None, **kwargs: object) -> None: + """Initialise settings, optionally pinning the memory-root for testing. + + Args: + _everos_root: Override the memory root used to locate + ``everos.toml``. Intended for tests only; pass ``None`` + (the default) in production to use :func:`resolve_root`. + **kwargs: Forwarded verbatim to :class:`pydantic_settings.BaseSettings`. + """ + if _everos_root is not None: + # Temporarily inject EVEROS_ROOT so that settings_customise_sources + # (a classmethod that cannot access instance state) picks it up via + # resolve_root(). We restore the original value after super().__init__ + # returns to avoid leaking the override into the process environment. + _prev = os.environ.get("EVEROS_ROOT") + os.environ["EVEROS_ROOT"] = str(_everos_root) + try: + super().__init__(**kwargs) + finally: + if _prev is None: + os.environ.pop("EVEROS_ROOT", None) + else: + os.environ["EVEROS_ROOT"] = _prev + else: + super().__init__(**kwargs) + @classmethod def settings_customise_sources( cls, @@ -376,26 +439,18 @@ class Settings(BaseSettings): dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, ) -> tuple[PydanticBaseSettingsSource, ...]: - """Layer TOML sources between env / dotenv and the secret store. - - Order (earlier wins in pydantic-settings): - init_args > env > .env > user_toml > default_toml > secrets - - The user-level toml (default ``~/.everos/config.toml``) is only - registered when the file exists, so the source list stays tight. - """ + """Source order: init_args > env_vars > everos.toml > default.toml.""" sources: list[PydanticBaseSettingsSource] = [ init_settings, env_settings, - dotenv_settings, ] - user_toml_path = _resolve_user_toml_path() - if user_toml_path.is_file(): + # Attempt to load /everos.toml if it exists. + everos_toml = resolve_root() / "everos.toml" + if everos_toml.is_file(): sources.append( - TomlConfigSettingsSource(settings_cls, toml_file=user_toml_path) + TomlConfigSettingsSource(settings_cls, toml_file=everos_toml) ) - sources.append(TomlConfigSettingsSource(settings_cls)) - sources.append(file_secret_settings) + sources.append(TomlConfigSettingsSource(settings_cls)) # default.toml return tuple(sources) diff --git a/src/everos/core/errors.py b/src/everos/core/errors.py index 650da2f..ba34c7e 100644 --- a/src/everos/core/errors.py +++ b/src/everos/core/errors.py @@ -1,43 +1,184 @@ -"""Cross-cutting domain errors surfaced to API callers. +"""Cross-cutting exception hierarchy for EverOS. -These live in ``core`` so the ``memory`` layer can raise them and the -``entrypoints`` layer can catch them without crossing the layered import -boundary — ``any -> core`` is the only edge both share (entrypoints must -not import ``memory`` directly). +All application exceptions derive from ``AppError``, split into four branches: + +- ``DomainError`` — business-rule violations (not-found, conflict, invalid + input, path traversal, unsupported format). +- ``InfrastructureError`` — transient storage and external-service failures + (retryable). +- ``CapabilityError`` — permanent server-side capability gaps (not retryable). +- ``ConfigurationError`` — misconfiguration detected at runtime. + +Any layer may raise ``AppError`` subclasses; the entrypoints layer catches +them and maps them to aligned HTTP responses. """ from __future__ import annotations +from enum import StrEnum -class PathTraversalError(Exception): + +class ErrorCode(StrEnum): + """Machine-readable error codes returned in the API error envelope. + + Each code maps to exactly one HTTP status code. Clients can switch on + this value to decide retry / display / routing behaviour without parsing + the human-readable ``message`` field. + """ + + NOT_FOUND = "NOT_FOUND" + CONFLICT = "CONFLICT" + INVALID_INPUT = "INVALID_INPUT" + EXTRACTION_EMPTY = "EXTRACTION_EMPTY" + BAD_REQUEST = "BAD_REQUEST" + UNSUPPORTED_FORMAT = "UNSUPPORTED_FORMAT" + EXTERNAL_SERVICE_UNAVAILABLE = "EXTERNAL_SERVICE_UNAVAILABLE" + CAPABILITY_UNAVAILABLE = "CAPABILITY_UNAVAILABLE" + CONFIGURATION_ERROR = "CONFIGURATION_ERROR" + INTERNAL_ERROR = "INTERNAL_ERROR" + + +# --------------------------------------------------------------------------- +# Root +# --------------------------------------------------------------------------- + + +class AppError(Exception): + """Root for all EverOS application exceptions.""" + + +# --------------------------------------------------------------------------- +# Domain branch — client-side / business-rule errors +# --------------------------------------------------------------------------- + + +class DomainError(AppError): + """Business-rule violation originating in the domain or service layer.""" + + +class NotFoundError(DomainError): + """A requested resource does not exist.""" + + +class DocumentNotFoundError(NotFoundError): + """A document with the given identifier was not found.""" + + +class TopicNotFoundError(NotFoundError): + """A knowledge topic with the given identifier was not found.""" + + +class ConflictError(DomainError): + """An operation conflicts with existing state (e.g. duplicate resource).""" + + +class DuplicateDocumentError(ConflictError): + """A document with the same identifier already exists.""" + + +class InvalidInputError(DomainError): + """Input does not meet domain rules.""" + + +class ExtractionEmptyError(InvalidInputError): + """An extraction pipeline produced no output when output was required.""" + + +class FilterError(InvalidInputError): + """A caller-supplied filter expression is invalid or malformed.""" + + +class PathTraversalError(DomainError): """A write target resolved outside the configured memory root. Raised by the markdown writer as a defense-in-depth backstop: any - caller-supplied identifier that becomes a path segment is validated at - the DTO layer, but this containment check does not depend on every such - id being sanitized upstream. The API layer maps it to HTTP 400. + caller-supplied identifier that becomes a path segment (``app_id`` / + ``project_id`` / ``sender_id`` -> ``owner_id``) is validated at the DTO + layer, but this containment check does not depend on every such id being + sanitised upstream. The API layer maps it to HTTP 400. """ -class MultimodalError(Exception): - """Base for multimodal-parsing errors meant to reach the caller. +class UnsupportedModalityError(DomainError): + """The uploaded file format is not supported (e.g. video, unknown type). - The API layer maps any ``MultimodalError`` to an aligned - ``{error: {code, message}}`` envelope (HTTP 415). + Wraps everalgo's ``NotImplementedError`` / dispatch ``ValueError`` so + the caller gets a stable 415 instead of a raw 500. """ -class UnsupportedModalityError(MultimodalError): - """everalgo cannot handle this modality (e.g. video stub, unknown type). +# --------------------------------------------------------------------------- +# Infrastructure branch — transient failures (retryable) +# --------------------------------------------------------------------------- - Wraps everalgo's ``NotImplementedError`` / dispatch ``ValueError`` so the - caller gets a stable, aligned error instead of a raw 500. + +class InfrastructureError(AppError): + """Transient failure in a storage adapter or external service.""" + + +class StorageError(InfrastructureError): + """A markdown or SQLite persistence operation failed.""" + + +class VectorStoreError(InfrastructureError): + """A LanceDB vector-store operation failed.""" + + +class ExternalServiceError(InfrastructureError): + """An external service (LLM, embedding, rerank) returned an error or timed out.""" + + +class LLMServiceError(ExternalServiceError): + """The configured LLM provider returned an error or timed out.""" + + +class EmbeddingServiceError(ExternalServiceError): + """The configured embedding provider returned an error or timed out.""" + + +class RerankServiceError(ExternalServiceError): + """The configured rerank provider returned an error or timed out.""" + + +# --------------------------------------------------------------------------- +# Capability branch — permanent server-side gaps (not retryable) +# --------------------------------------------------------------------------- + + +class CapabilityError(AppError): + """A required server-side capability is not available. + + Unlike ``InfrastructureError`` (transient — retry may help), + ``CapabilityError`` signals a permanent gap that requires admin + action (install a dependency, enable a feature). """ -class MultimodalNotEnabledError(MultimodalError): - """Multimodal capability is not ready. +class MultimodalNotEnabledError(CapabilityError): + """Multimodal parsing capability is not available. - Raised when the ``everos[multimodal]`` extra is not installed, or when a - required system dependency (LibreOffice for Office documents) is absent. + Raised when the ``everos[multimodal]`` extra is not installed, or when + a required system dependency (LibreOffice for Office documents) is absent. """ + + +# --------------------------------------------------------------------------- +# Configuration branch — misconfiguration detected at runtime +# --------------------------------------------------------------------------- + + +class ConfigurationError(AppError): + """A required configuration is missing or invalid. + + Raised when a mandatory setting (e.g. embedding model, rerank provider) + is not configured but the code path requires it. + """ + + +# --------------------------------------------------------------------------- +# Backward compatibility aliases +# --------------------------------------------------------------------------- + +# Renamed in v0.2 — old names kept for external consumers. +DocumentAlreadyExistsError = DuplicateDocumentError +ValidationError = InvalidInputError diff --git a/src/everos/core/middleware/__init__.py b/src/everos/core/middleware/__init__.py index a773ee6..c6949c3 100644 --- a/src/everos/core/middleware/__init__.py +++ b/src/everos/core/middleware/__init__.py @@ -1,6 +1,7 @@ """Cross-cutting HTTP middleware components. -External usage: +External usage:: + from everos.core.middleware import ( DEFAULT_CORS_ALLOW_CREDENTIALS, DEFAULT_CORS_ALLOW_HEADERS, @@ -8,7 +9,6 @@ External usage: DEFAULT_CORS_ORIGINS, ProfileMiddleware, PrometheusMiddleware, - global_exception_handler, ) """ @@ -16,7 +16,6 @@ from .cors import DEFAULT_CORS_ALLOW_CREDENTIALS as DEFAULT_CORS_ALLOW_CREDENTIA from .cors import DEFAULT_CORS_ALLOW_HEADERS as DEFAULT_CORS_ALLOW_HEADERS from .cors import DEFAULT_CORS_ALLOW_METHODS as DEFAULT_CORS_ALLOW_METHODS from .cors import DEFAULT_CORS_ORIGINS as DEFAULT_CORS_ORIGINS -from .global_exception import global_exception_handler as global_exception_handler from .profile import ProfileMiddleware as ProfileMiddleware from .prometheus import PrometheusMiddleware as PrometheusMiddleware @@ -27,5 +26,4 @@ __all__ = [ "DEFAULT_CORS_ORIGINS", "ProfileMiddleware", "PrometheusMiddleware", - "global_exception_handler", ] diff --git a/src/everos/core/middleware/global_exception.py b/src/everos/core/middleware/global_exception.py deleted file mode 100644 index e6ef4d2..0000000 --- a/src/everos/core/middleware/global_exception.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Global exception handler — uniform error envelope per v1 API brief §1. - -Envelope shape (matches the v1 API brief §1 — ``request_id`` at the top -level alongside ``error``; the ``error`` object carries ``code`` / -``message`` plus ops-friendly ``timestamp`` / ``path`` for debugging):: - - { - "request_id": "<32 lowercase hex chars — W3C trace_id format>", - "error": { - "code": "HTTP_ERROR" | "SYSTEM_ERROR", - "message": "", - "timestamp": "", - "path": "" - } - } - -Rules: -- 4xx (DTO / business validation / HTTPException) → ``code="HTTP_ERROR"`` - with the human-readable reason in ``message``. -- 5xx (unhandled exception) → ``code="SYSTEM_ERROR"`` with a fixed - ``message="Internal server error"`` — internal exception details are - logged but never leak to the client. -- ``request_id`` is sourced from ``request.state.request_id`` (set by - upstream middleware); falls back to a freshly minted id when absent. -""" - -from __future__ import annotations - -from fastapi import HTTPException, Request -from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse -from starlette.status import ( - HTTP_422_UNPROCESSABLE_ENTITY, - HTTP_500_INTERNAL_SERVER_ERROR, -) - -from everos.component.utils.datetime import ( - get_now_with_timezone, - to_iso_format, -) -from everos.core.observability.logging import get_logger -from everos.core.observability.tracing import gen_request_id - -logger = get_logger(__name__) - -_INTERNAL_ERROR_MESSAGE = "Internal server error" - - -def _request_id(request: Request) -> str: - """Return the request_id set by middleware, or mint a fresh fallback.""" - rid = getattr(request.state, "request_id", None) - if rid: - return str(rid) - return gen_request_id() - - -def _envelope( - *, - code: str, - message: str, - request: Request, -) -> dict[str, object]: - """Build the canonical error envelope (wiki §1 shape — nested ``error``). - - ``request_id`` at the top level, ``error`` object carries the - contract fields (``code`` / ``message``) plus ops-friendly - ``timestamp`` / ``path``. - """ - return { - "request_id": _request_id(request), - "error": { - "code": code, - "message": message, - "timestamp": to_iso_format(get_now_with_timezone()), - "path": str(request.url.path), - }, - } - - -async def global_exception_handler(request: Request, exc: Exception) -> JSONResponse: - """Convert any exception into a uniform JSON error response.""" - path = str(request.url.path) - method = request.method - - if isinstance(exc, RequestValidationError): - errors = exc.errors() - if errors: - first = errors[0] - loc = ".".join(str(p) for p in first.get("loc", []) if p != "body") - msg = first.get("msg", "Validation error") - message = f"{msg}: {loc}" if loc else msg - else: - message = "Request validation error" - - logger.warning("validation_error", method=method, path=path, message=message) - return JSONResponse( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, - content=_envelope(code="HTTP_ERROR", message=message, request=request), - ) - - if isinstance(exc, HTTPException): - logger.warning( - "http_exception", - method=method, - path=path, - status_code=exc.status_code, - detail=exc.detail, - ) - # 5xx routed through HTTPException is rare but valid; still honour - # the SYSTEM_ERROR code so the envelope is consistent. - if exc.status_code >= 500: - return JSONResponse( - status_code=exc.status_code, - content=_envelope( - code="SYSTEM_ERROR", - message=_INTERNAL_ERROR_MESSAGE, - request=request, - ), - ) - return JSONResponse( - status_code=exc.status_code, - content=_envelope( - code="HTTP_ERROR", - message=str(exc.detail), - request=request, - ), - ) - - logger.error( - "unhandled_exception", - method=method, - path=path, - exception_type=type(exc).__name__, - exc_info=True, - ) - return JSONResponse( - status_code=HTTP_500_INTERNAL_SERVER_ERROR, - content=_envelope( - code="SYSTEM_ERROR", - message=_INTERNAL_ERROR_MESSAGE, - request=request, - ), - ) diff --git a/src/everos/core/persistence/lancedb/repository.py b/src/everos/core/persistence/lancedb/repository.py index 3057b97..0c3a9de 100644 --- a/src/everos/core/persistence/lancedb/repository.py +++ b/src/everos/core/persistence/lancedb/repository.py @@ -78,7 +78,7 @@ class LanceRepoBase[T: BaseLanceTable]: reads stay unlocked so search QPS is not impacted by writers. Locks live in a class-level dict keyed by table name and are never - evicted (mirrors :mod:`everos.memory.strategies._partition_locks` + evicted (mirrors :mod:`everos.memory._partition_locks` on bpo-28427 — a lock with pending waiters must outlive any dict entry that points to it). """ @@ -111,7 +111,7 @@ class LanceRepoBase[T: BaseLanceTable]: a module-level lock surviving across tests fails with "bound to a different event loop". The production cascade worker runs on one loop forever and does not need this hook. Mirrors - :func:`everos.memory.strategies._partition_locks._reset_for_tests`. + :func:`everos.memory._partition_locks._reset_for_tests`. """ cls._table_locks.clear() @@ -427,6 +427,29 @@ class LanceRepoBase[T: BaseLanceTable]: q = q.where(where) return await q.limit(limit).to_list() + # ── Update ───────────────────────────────────────────────────────────── + + async def update( + self, + updates: dict[str, Any], + *, + where: str, + ) -> None: + """Partial column update for rows matching ``where``. + + Wraps ``AsyncTable.update`` — sets specific column values without + rewriting the full row. Useful for lightweight metadata patches + (e.g. setting ``deprecated_by``) where a full embed+upsert cycle + is unnecessary. + + Args: + updates: Column-name to new-value mapping. + where: SQL-like predicate scoping the update. + """ + table = await self._table() + async with self._write_lock(self.table_name): + await table.update(updates, where=where) + # ── Delete ───────────────────────────────────────────────────────────── async def delete(self, predicate: str) -> None: diff --git a/src/everos/core/persistence/markdown/__init__.py b/src/everos/core/persistence/markdown/__init__.py index 2aed616..54a9b0e 100644 --- a/src/everos/core/persistence/markdown/__init__.py +++ b/src/everos/core/persistence/markdown/__init__.py @@ -18,6 +18,8 @@ External usage (frontmatter schema chassis): from everos.core.persistence.markdown import ( BaseFrontmatter, UserScopedFrontmatter, AgentScopedFrontmatter, DailyLogPathMixin, SkillPathMixin, ProfilePathMixin, + KnowledgeScopedMixin, KnowledgeDocumentPathMixin, + KnowledgeTopicPathMixin, ) """ @@ -31,6 +33,9 @@ from .entries import split_entries as split_entries from .frontmatter import AgentScopedFrontmatter as AgentScopedFrontmatter from .frontmatter import BaseFrontmatter as BaseFrontmatter from .frontmatter import DailyLogPathMixin as DailyLogPathMixin +from .frontmatter import KnowledgeDocumentPathMixin as KnowledgeDocumentPathMixin +from .frontmatter import KnowledgeScopedMixin as KnowledgeScopedMixin +from .frontmatter import KnowledgeTopicPathMixin as KnowledgeTopicPathMixin from .frontmatter import ProfilePathMixin as ProfilePathMixin from .frontmatter import SkillPathMixin as SkillPathMixin from .frontmatter import UserScopedFrontmatter as UserScopedFrontmatter @@ -46,6 +51,9 @@ __all__ = [ "DailyLogPathMixin", "Entry", "EntryId", + "KnowledgeDocumentPathMixin", + "KnowledgeScopedMixin", + "KnowledgeTopicPathMixin", "MarkdownReader", "MarkdownWriter", "ParsedMarkdown", diff --git a/src/everos/core/persistence/markdown/frontmatter.py b/src/everos/core/persistence/markdown/frontmatter.py index 26ca436..78760aa 100644 --- a/src/everos/core/persistence/markdown/frontmatter.py +++ b/src/everos/core/persistence/markdown/frontmatter.py @@ -298,3 +298,47 @@ class AgentScopedFrontmatter(BaseFrontmatter): agent_id: str track: Literal["agent"] = "agent" + + +class KnowledgeScopedMixin: + """Records in the knowledge scope (no owner_id). + + Unlike user/agent scopes, knowledge records are not owned by an + individual entity. The scope directory is ``knowledge/``. + """ + + SCOPE_DIR: ClassVar[str] = "knowledge" + + +class KnowledgeDocumentPathMixin: + """Path strategy for ``knowledge/{category}/{doc_title}/index.md``. + + Place this mixin first so MRO resolves ``path_glob()`` here:: + + class KnowledgeDocumentFrontmatter( + KnowledgeDocumentPathMixin, KnowledgeScopedMixin, BaseFrontmatter + ): ... + """ + + SCOPE_DIR: ClassVar[str] + + @classmethod + def path_glob(cls) -> str: + return f"*/*/{cls.SCOPE_DIR}/*/*/index.md" + + +class KnowledgeTopicPathMixin: + """Path strategy for ``knowledge/{category}/{doc_title}/_.md``. + + Place this mixin first so MRO resolves ``path_glob()`` here:: + + class KnowledgeTopicFrontmatter( + KnowledgeTopicPathMixin, KnowledgeScopedMixin, BaseFrontmatter + ): ... + """ + + SCOPE_DIR: ClassVar[str] + + @classmethod + def path_glob(cls) -> str: + return f"*/*/{cls.SCOPE_DIR}/*/*/[0-9]*.md" diff --git a/src/everos/core/persistence/markdown/writer.py b/src/everos/core/persistence/markdown/writer.py index 066772b..f6af817 100644 --- a/src/everos/core/persistence/markdown/writer.py +++ b/src/everos/core/persistence/markdown/writer.py @@ -52,7 +52,7 @@ from everos.core.errors import PathTraversalError from ..memory_root import MemoryRoot from .entries import EntryId -from .frontmatter import dump_frontmatter +from .frontmatter import dump_frontmatter, parse_frontmatter from .reader import MarkdownReader @@ -60,9 +60,11 @@ class MarkdownWriter: """Atomic writer for markdown files inside a memory-root. The ``memory_root`` reference anchors a containment check: every write - target must resolve inside ``memory_root.root``. This is defense-in-depth - against path traversal via caller-supplied identifiers that become path - segments. + target must resolve inside ``memory_root.root`` (see + :meth:`_ensure_within_root`). This is defense-in-depth against path + traversal via any caller-supplied identifier that becomes a path + segment — the DTO layer also rejects ``.``/``..`` in such ids, but this + check does not depend on every id being sanitised upstream. """ def __init__(self, memory_root: MemoryRoot) -> None: @@ -100,8 +102,30 @@ class MarkdownWriter: return lock def _ensure_within_root(self, target: Path) -> Path: - """Reject a write target that resolves outside the memory root.""" - root = self._memory_root.root + """Reject a write target that resolves outside the memory root. + + Defense-in-depth against path traversal: a caller-supplied id that + becomes a path segment (e.g. ``sender_id`` -> ``owner_id``) could + otherwise smuggle ``..`` segments and walk the write out of the + configured root. ``resolve()`` collapses ``..`` and symlinks + lexically/physically before the comparison, so the check holds even + though ``target`` does not exist yet. + + Must run *before* any filesystem touch — the ``mkdir`` on the write + path and the read-modify-write read on the append path both call this + first, so an escaping path never creates parent directories nor opens + an out-of-root file. + + Args: + target: The intended write path. + + Returns: + The resolved, root-contained absolute path. + + Raises: + PathTraversalError: If the resolved path is not within the root. + """ + root = self._memory_root.root # already absolute + resolved resolved = target.resolve() if not resolved.is_relative_to(root): raise PathTraversalError( @@ -121,6 +145,9 @@ class MarkdownWriter: Returns: ``path`` (resolved as written). + + Raises: + PathTraversalError: If ``path`` resolves outside the memory root. """ target = Path(path) self._ensure_within_root(target) @@ -146,6 +173,43 @@ class MarkdownWriter: head = dump_frontmatter(frontmatter or {}) return await self.write(path, head + body) + async def patch_frontmatter(self, path: Path, updates: Mapping[str, Any]) -> None: + """Update frontmatter fields on an existing md file in-place. + + Reads the file, merges ``updates`` into frontmatter, writes back. + Only the frontmatter portion is rewritten; entries are untouched. + Uses the same per-path lock as ``append_entries`` for concurrency + safety. + + For dict-type fields (e.g. ``deprecated_entries``) the merge is + additive: existing keys are preserved, new keys are added or + overwritten. Scalar fields are replaced wholesale. + + Args: + path: Target markdown file (must exist). + updates: Mapping of frontmatter keys to merge. + + Raises: + FileNotFoundError: If ``path`` does not exist. + """ + target = Path(path) + async with self.lock_for(target): + # 1. Read raw text. + raw = await anyio.Path(target).read_text(encoding="utf-8") + + # 2. Split into frontmatter + remainder (entries body). + existing_fm, remainder = parse_frontmatter(raw) + + # 3. Deep-merge dict fields; replace scalars. + for key, value in updates.items(): + if isinstance(value, dict) and isinstance(existing_fm.get(key), dict): + existing_fm[key].update(value) + else: + existing_fm[key] = value + + # 4. Atomic write with merged frontmatter + original body. + await self.write(target, dump_frontmatter(existing_fm) + remainder) + async def append_entry( self, path: Path, @@ -239,6 +303,9 @@ class MarkdownWriter: breaks the safety contract. """ target = Path(path) + # Guard the read too, not just the final write: an escaping path must + # not even reach MarkdownReader (otherwise an out-of-root file would be + # opened and parsed before the write-side check rejected it). self._ensure_within_root(target) # 1. Load existing markdown (or initialise empty). diff --git a/src/everos/core/persistence/memory_root.py b/src/everos/core/persistence/memory_root.py index ab83c08..7a96bd2 100644 --- a/src/everos/core/persistence/memory_root.py +++ b/src/everos/core/persistence/memory_root.py @@ -49,12 +49,6 @@ _DEFAULT_SCOPE_ID = "default" _DEFAULT_APP_DIR = "default_app" _DEFAULT_PROJECT_DIR = "default_project" -# Path to the shipped OME override template; copied to ``/ome.toml`` on -# first ``ensure()`` so users have a real file to edit instead of having to -# create one from scratch. ``parents[2]`` is the ``src/everos/`` package root -# (memory_root.py sits at ``core/persistence/memory_root.py``). -_OME_TEMPLATE_PATH = Path(__file__).parents[2] / "config" / "default_ome.toml" - def app_dir_name(app_id: str) -> str: """Map an ``app_id`` to its on-disk directory name.""" @@ -98,16 +92,18 @@ class MemoryRoot: object.__setattr__(self, "root", resolved) @classmethod - def default(cls) -> MemoryRoot: - """Return the memory-root from :class:`everos.config.Settings`. + def default(cls, *, explicit_root: str | None = None) -> MemoryRoot: + """Return the memory-root resolved from CLI / env / default. - The effective default lives in ``config/default.toml`` (``[memory] - root``); environment variable ``EVEROS_MEMORY__ROOT`` overrides it. + Resolution: ``explicit_root`` > ``EVEROS_ROOT`` env > ``~/.everos``. + + Args: + explicit_root: Caller-supplied path (e.g. from ``--root`` CLI flag). """ # Lazy import to keep this module dependency-free at import time. - from everos.config import load_settings + from everos.config.settings import resolve_root - return cls(load_settings().memory.root) + return cls(resolve_root(explicit_root)) # ── User-visible (partitioned by app / project) ────────────────────────── # @@ -184,19 +180,17 @@ class MemoryRoot: @property def ome_config(self) -> Path: - """``/ome.toml`` — user-editable OME strategy overrides. + """``/ome.toml`` — single entry point for strategy configuration. - Drop a file here to toggle strategies on/off or tweak per-strategy - knobs (max_retries, gate, cron …) without restarting the server. - The engine watches this file and hot-reloads changes within ~2 s. + All strategy settings (enabled, cron, max_retries, gate) are managed + here. The engine watches this file and hot-reloads changes within ~2 s. + No server restart needed. - Example to disable foresight and user-profile extraction:: + Example — enable Reflection with a custom cron:: - [strategies.extract_foresight] - enabled = false - - [strategies.extract_user_profile] - enabled = false + [strategies.reflect_episodes] + enabled = true + cron = "0 3 * * *" """ return self.root / "ome.toml" @@ -237,7 +231,3 @@ class MemoryRoot: self.sqlite_dir.mkdir(parents=True, exist_ok=True) self.lancedb_dir.mkdir(parents=True, exist_ok=True) self.tmp_dir.mkdir(parents=True, exist_ok=True) - # Materialize the OME override template on first run; existence-only - # check preserves any edits the user has already made. - if not self.ome_config.exists(): - self.ome_config.write_bytes(_OME_TEMPLATE_PATH.read_bytes()) diff --git a/src/everos/entrypoints/api/app.py b/src/everos/entrypoints/api/app.py index 20c0484..337fdc5 100644 --- a/src/everos/entrypoints/api/app.py +++ b/src/everos/entrypoints/api/app.py @@ -8,8 +8,7 @@ from __future__ import annotations import os -from fastapi import FastAPI, HTTPException -from fastapi.exceptions import RequestValidationError +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from everos import __version__ @@ -25,10 +24,10 @@ from everos.core.middleware import ( DEFAULT_CORS_ORIGINS, ProfileMiddleware, PrometheusMiddleware, - global_exception_handler, ) from everos.core.observability.logging import get_logger +from .exception_handlers import register_handlers from .lifespans import ( CascadeLifespanProvider, LanceDBLifespanProvider, @@ -39,8 +38,10 @@ from .lifespans import ( from .routes import ( get, health, + knowledge, memorize, metrics, + ome, search, ) @@ -97,10 +98,8 @@ def create_app( openapi_url="/openapi.json" if enable_docs else None, ) - # Exception handlers: HTTPException, validation errors, plus a fallback. - app.add_exception_handler(HTTPException, global_exception_handler) - app.add_exception_handler(RequestValidationError, global_exception_handler) - app.add_exception_handler(Exception, global_exception_handler) + # Exception handlers + register_handlers(app) # Middleware order: earlier `add_middleware` calls become inner, later ones outer. # CORS innermost (matches base_app.py legacy pattern). @@ -120,6 +119,8 @@ def create_app( app.include_router(memorize.router) app.include_router(search.router) app.include_router(get.router) + app.include_router(ome.router) + app.include_router(knowledge.router) logger.info("app_created", docs_enabled=enable_docs) return app diff --git a/src/everos/entrypoints/api/exception_handlers.py b/src/everos/entrypoints/api/exception_handlers.py new file mode 100644 index 0000000..da7675a --- /dev/null +++ b/src/everos/entrypoints/api/exception_handlers.py @@ -0,0 +1,362 @@ +"""Per-type exception handlers for the EverOS FastAPI application. + +Each handler converts a specific exception class (or hierarchy root) into +the canonical error envelope:: + + { + "request_id": "<32 lowercase hex chars>", + "error": { + "code": "", + "message": "", + "timestamp": "", + "path": "" + } + } + +Register all handlers at once with ``register_handlers(app)``. +""" + +from __future__ import annotations + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import BaseModel +from starlette.status import ( + HTTP_400_BAD_REQUEST, + HTTP_404_NOT_FOUND, + HTTP_409_CONFLICT, + HTTP_415_UNSUPPORTED_MEDIA_TYPE, + HTTP_422_UNPROCESSABLE_CONTENT, + HTTP_500_INTERNAL_SERVER_ERROR, + HTTP_503_SERVICE_UNAVAILABLE, +) + +from everos.component.utils.datetime import get_now_with_timezone, to_iso_format +from everos.core.errors import ( + CapabilityError, + ConfigurationError, + ConflictError, + ErrorCode, + ExtractionEmptyError, + InfrastructureError, + InvalidInputError, + NotFoundError, + PathTraversalError, + UnsupportedModalityError, +) +from everos.core.observability.logging import get_logger + +from .utils import extract_request_id + +logger = get_logger(__name__) + +_INTERNAL_ERROR_MESSAGE = "Internal server error" + + +# --------------------------------------------------------------------------- +# Response model (visible in OpenAPI docs) +# --------------------------------------------------------------------------- + + +class ErrorDetail(BaseModel): + """Inner ``error`` object in the canonical error envelope.""" + + code: ErrorCode + message: str + timestamp: str + path: str + + +class ErrorResponse(BaseModel): + """Canonical error envelope returned by all error handlers.""" + + request_id: str + error: ErrorDetail + + +# --------------------------------------------------------------------------- +# Envelope builder +# --------------------------------------------------------------------------- + + +def _error_response( + request: Request, + status_code: int, + code: ErrorCode, + message: str, +) -> JSONResponse: + """Build a JSONResponse with the canonical error envelope.""" + body = ErrorResponse( + request_id=extract_request_id(request), + error=ErrorDetail( + code=code, + message=message, + timestamp=to_iso_format(get_now_with_timezone()), + path=str(request.url.path), + ), + ) + return JSONResponse( + status_code=status_code, + content=body.model_dump(), + ) + + +# --------------------------------------------------------------------------- +# Handlers +# --------------------------------------------------------------------------- + + +async def not_found_handler( + request: Request, + exc: NotFoundError, +) -> JSONResponse: + """NotFoundError (and subclasses) -> 404.""" + return _error_response( + request, + HTTP_404_NOT_FOUND, + ErrorCode.NOT_FOUND, + str(exc), + ) + + +async def conflict_handler( + request: Request, + exc: ConflictError, +) -> JSONResponse: + """ConflictError (and subclasses) -> 409.""" + return _error_response( + request, + HTTP_409_CONFLICT, + ErrorCode.CONFLICT, + str(exc), + ) + + +async def extraction_empty_handler( + request: Request, + exc: ExtractionEmptyError, +) -> JSONResponse: + """ExtractionEmptyError -> 422 with dedicated code.""" + return _error_response( + request, + HTTP_422_UNPROCESSABLE_CONTENT, + ErrorCode.EXTRACTION_EMPTY, + str(exc), + ) + + +async def invalid_input_handler( + request: Request, + exc: InvalidInputError, +) -> JSONResponse: + """InvalidInputError (and subclasses) -> 422.""" + return _error_response( + request, + HTTP_422_UNPROCESSABLE_CONTENT, + ErrorCode.INVALID_INPUT, + str(exc), + ) + + +async def path_traversal_handler( + request: Request, + exc: PathTraversalError, +) -> JSONResponse: + """PathTraversalError -> 400.""" + return _error_response( + request, + HTTP_400_BAD_REQUEST, + ErrorCode.BAD_REQUEST, + "Invalid input: path contains illegal characters.", + ) + + +async def unsupported_modality_handler( + request: Request, + exc: UnsupportedModalityError, +) -> JSONResponse: + """UnsupportedModalityError -> 415.""" + return _error_response( + request, + HTTP_415_UNSUPPORTED_MEDIA_TYPE, + ErrorCode.UNSUPPORTED_FORMAT, + str(exc), + ) + + +async def infrastructure_handler( + request: Request, + exc: InfrastructureError, +) -> JSONResponse: + """InfrastructureError (and subclasses) -> 503.""" + logger.warning( + "infrastructure_error", + path=str(request.url.path), + exception_type=type(exc).__name__, + message=str(exc), + ) + return _error_response( + request, + HTTP_503_SERVICE_UNAVAILABLE, + ErrorCode.EXTERNAL_SERVICE_UNAVAILABLE, + str(exc), + ) + + +async def capability_handler( + request: Request, + exc: CapabilityError, +) -> JSONResponse: + """CapabilityError (and subclasses) -> 503 (not retryable).""" + logger.warning( + "capability_error", + path=str(request.url.path), + exception_type=type(exc).__name__, + message=str(exc), + ) + return _error_response( + request, + HTTP_503_SERVICE_UNAVAILABLE, + ErrorCode.CAPABILITY_UNAVAILABLE, + str(exc), + ) + + +async def configuration_handler( + request: Request, + exc: ConfigurationError, +) -> JSONResponse: + """ConfigurationError -> 500.""" + logger.error( + "configuration_error", + path=str(request.url.path), + message=str(exc), + ) + return _error_response( + request, + HTTP_500_INTERNAL_SERVER_ERROR, + ErrorCode.CONFIGURATION_ERROR, + str(exc), + ) + + +# --------------------------------------------------------------------------- +# Pydantic / FastAPI built-in exceptions +# --------------------------------------------------------------------------- + +_FIELD_HINTS: dict[str, str] = { + "doc_id": ( + "Invalid doc_id format. Use GET /documents to look up valid doc_id values." + ), + "topic_id": ( + "Invalid topic_id format. " + "Use GET /documents/{doc_id} to look up valid topic_id values." + ), + "query": "Search query cannot be empty.", + "title": "Invalid title. Must contain at least one letter or digit.", +} + + +async def request_validation_handler( + request: Request, + exc: RequestValidationError, +) -> JSONResponse: + """FastAPI RequestValidationError -> 422.""" + errors = exc.errors() + if errors: + first = errors[0] + loc_parts = [str(p) for p in first.get("loc", []) if p != "body"] + field = loc_parts[-1] if loc_parts else "" + if field in _FIELD_HINTS: + message = _FIELD_HINTS[field] + else: + loc = ".".join(loc_parts) + msg = first.get("msg", "Validation error") + message = f"{msg}: {loc}" if loc else msg + else: + message = "Request validation error" + return _error_response( + request, + HTTP_422_UNPROCESSABLE_CONTENT, + ErrorCode.INVALID_INPUT, + message, + ) + + +async def http_exception_handler( + request: Request, + exc: HTTPException, +) -> JSONResponse: + """FastAPI HTTPException -> envelope with original status code.""" + if exc.status_code >= 500: + logger.error( + "http_exception_5xx", + path=str(request.url.path), + status_code=exc.status_code, + ) + return _error_response( + request, + exc.status_code, + ErrorCode.INTERNAL_ERROR, + _INTERNAL_ERROR_MESSAGE, + ) + return _error_response( + request, + exc.status_code, + ErrorCode.BAD_REQUEST, + str(exc.detail), + ) + + +async def unexpected_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + """Catch-all for unhandled exceptions -> 500 (no detail leak).""" + logger.error( + "unhandled_exception", + path=str(request.url.path), + exception_type=type(exc).__name__, + exc_info=True, + ) + return _error_response( + request, + HTTP_500_INTERNAL_SERVER_ERROR, + ErrorCode.INTERNAL_ERROR, + _INTERNAL_ERROR_MESSAGE, + ) + + +# --------------------------------------------------------------------------- +# Registration +# --------------------------------------------------------------------------- + + +def register_handlers(app: FastAPI) -> None: + """Register all per-type exception handlers on ``app``. + + Starlette walks the exception MRO and picks the first matching + handler, so more-specific types are registered before their parents. + """ + # Domain errors (specific before parent) + app.add_exception_handler(PathTraversalError, path_traversal_handler) + app.add_exception_handler(UnsupportedModalityError, unsupported_modality_handler) + app.add_exception_handler(NotFoundError, not_found_handler) + app.add_exception_handler(ConflictError, conflict_handler) + app.add_exception_handler(ExtractionEmptyError, extraction_empty_handler) + app.add_exception_handler(InvalidInputError, invalid_input_handler) + # Infrastructure errors (transient, retryable) + app.add_exception_handler(InfrastructureError, infrastructure_handler) + # Capability errors (permanent, not retryable) + app.add_exception_handler(CapabilityError, capability_handler) + # Configuration errors + app.add_exception_handler(ConfigurationError, configuration_handler) + # FastAPI built-in exceptions + app.add_exception_handler(HTTPException, http_exception_handler) + app.add_exception_handler( + RequestValidationError, + request_validation_handler, + ) + # Catch-all + app.add_exception_handler(Exception, unexpected_handler) diff --git a/src/everos/entrypoints/api/routes/get.py b/src/everos/entrypoints/api/routes/get.py index 9fc14c8..cbd8562 100644 --- a/src/everos/entrypoints/api/routes/get.py +++ b/src/everos/entrypoints/api/routes/get.py @@ -7,10 +7,9 @@ return the envelope verbatim. ``request_id`` is generated inside the from __future__ import annotations -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter from everos.memory.get import GetRequest, GetResponse -from everos.memory.search import FilterError from everos.service import get as get_service router = APIRouter(prefix="/api/v1/memory", tags=["memory"]) @@ -19,8 +18,4 @@ router = APIRouter(prefix="/api/v1/memory", tags=["memory"]) @router.post("/get", response_model=GetResponse) async def post_get(req: GetRequest) -> GetResponse: """Paginated listing over the requested ``memory_type``.""" - try: - return await get_service(req) - except FilterError as exc: - # Filter-DSL violations surface as 422 with the compile message. - raise HTTPException(status_code=422, detail=str(exc)) from exc + return await get_service(req) diff --git a/src/everos/entrypoints/api/routes/knowledge.py b/src/everos/entrypoints/api/routes/knowledge.py new file mode 100644 index 0000000..1b853ac --- /dev/null +++ b/src/everos/entrypoints/api/routes/knowledge.py @@ -0,0 +1,643 @@ +"""Knowledge document CRUD + search HTTP endpoints. + +Routes follow the thin-adapter pattern: validate the DTO, delegate to +the service layer, format the response envelope. No business logic +lives here. + +Endpoint overview (design spec sections 6.2-6.6): + POST /documents — upload a new document (multipart) + PUT /documents/{doc_id} — replace an existing document (multipart) + PATCH /documents/{doc_id} — update mutable metadata + DELETE /documents/{doc_id} — remove a document + GET /documents — paginated document listing + GET /documents/{doc_id} — single document detail + GET /topics/{topic_id} — single topic detail + POST /search — knowledge retrieval + GET /categories — taxonomy listing +""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING, Annotated, Literal + +if TYPE_CHECKING: + from everos.service.knowledge import KnowledgeExtractor + +from everalgo.types import ParsedContent +from fastapi import APIRouter, Path, Query, Request, Response, UploadFile +from fastapi.params import Form +from pydantic import BaseModel, Field + +from everos.component.llm import get_llm_client +from everos.component.utils.datetime import to_display_tz +from everos.config import load_settings +from everos.core.errors import ( + InvalidInputError, + UnsupportedModalityError, +) +from everos.core.persistence import MemoryRoot +from everos.entrypoints.api.utils import extract_request_id +from everos.service import ( + CreateDocumentResult, + DocumentDetail, + DocumentListResult, + SearchKnowledgeResult, + TopicDetail, + create_document, + delete_document, + get_document, + get_topic, + list_categories, + list_documents, + patch_document, + replace_document, + search_knowledge, +) + +# PathSafeId and SuccessEnvelope are imported from memorize routes; +# a shared module would be cleaner but is out of scope for this PR. +from .memorize import PathSafeId, SuccessEnvelope + +router = APIRouter(prefix="/api/v1/knowledge", tags=["knowledge"]) + + +# ── Annotated param types (satisfies B008) ────────────────────────────────── + +_FormTitle = Annotated[str, Form(min_length=1, pattern=r"\w")] +_FormOptStr = Annotated[str | None, Form()] +_FormPathSafe = Annotated[PathSafeId, Form()] +_QueryPathSafe = Annotated[PathSafeId, Query()] +_QueryOptStr = Annotated[str | None, Query()] +_QueryPage = Annotated[int, Query(ge=1)] +_QueryPageSize = Annotated[int, Query(ge=1, le=100)] +_QuerySortBy = Annotated[Literal["created_at", "updated_at", "title"], Query()] +_QuerySortOrder = Annotated[Literal["asc", "desc"], Query()] + +# Upper bound on a search query string — guards the embedding call against +# pathologically long input. +_MAX_QUERY_LENGTH = 2000 + +# doc_id: system-generated "d_"; topic_id: "{doc_id}_{index}". +_DOC_ID_PATTERN = r"^d_[a-f0-9]{12,32}$" +_TOPIC_ID_PATTERN = r"^d_[a-f0-9]{12,32}_\d+$" +_PathDocId = Annotated[str, Path(pattern=_DOC_ID_PATTERN)] +_PathTopicId = Annotated[str, Path(pattern=_TOPIC_ID_PATTERN)] + + +# ── Response DTOs ──────────────────────────────────────────────────────────── + + +class DocumentCreateResponse(BaseModel): + """Response for POST/PUT /documents.""" + + doc_id: str + category_id: str + topic_count: int + source_name: str | None + md_path: str + original_file_path: str | None + + +class DocumentDeleteResponse(BaseModel): + """Response for DELETE /documents/{doc_id}.""" + + doc_id: str + deleted_topics: int + + +class TopicOverviewDTO(BaseModel): + """Minimal topic summary inside a document detail.""" + + topic_id: str + topic_name: str + topic_path: str + depth: int + summary: str + + +class DocumentDetailResponse(BaseModel): + """Response for GET /documents/{doc_id}.""" + + doc_id: str + category_id: str + title: str + summary: str + source_name: str | None + source_type: str | None + original_file_path: str | None + topics: list[TopicOverviewDTO] + created_at: datetime + updated_at: datetime + + +class DocumentOverviewItemDTO(BaseModel): + """One row in the paginated document list.""" + + doc_id: str + category_id: str + title: str + topic_count: int + created_at: datetime + + +class DocumentListResponse(BaseModel): + """Response for GET /documents.""" + + documents: list[DocumentOverviewItemDTO] + total: int + page: int + page_size: int + + +class TopicDetailResponse(BaseModel): + """Response for GET /topics/{topic_id}.""" + + topic_id: str + doc_id: str + category_id: str + topic_name: str + topic_path: str + depth: int + summary: str + content: str + content_labels: list[str] + parent_topic_id: str | None + children_topic_ids: list[str] + created_at: datetime + updated_at: datetime + + +class DocumentContextDTO(BaseModel): + """L1 document metadata attached to every search hit.""" + + doc_id: str + title: str + summary: str + + +class SearchHitDTO(BaseModel): + """One ranked result from knowledge search.""" + + topic_id: str + category_id: str + topic_name: str + topic_path: str + depth: int + summary: str + content: str | None + score: float + retrieval_method: str + source: str | None + document: DocumentContextDTO + + +class KnowledgeSearchResponse(BaseModel): + """Response for POST /search.""" + + hits: list[SearchHitDTO] + total: int + took_ms: float + + +class CategoryDTO(BaseModel): + """One taxonomy category.""" + + category_id: str + description: str + document_count: int + + +class CategoryListResponse(BaseModel): + """Response for GET /categories.""" + + categories: list[CategoryDTO] + + +class DocumentPatchResponse(BaseModel): + """Response for PATCH /documents/{doc_id}.""" + + doc_id: str + updated_fields: list[str] + updated_at: datetime + + +# ── Request DTOs ───────────────────────────────────────────────────────────── + + +class KnowledgeSearchRequest(BaseModel): + """Request body for POST /search.""" + + query: str = Field(..., min_length=1, max_length=_MAX_QUERY_LENGTH) + method: Literal["keyword", "vector", "hybrid"] = "hybrid" + top_k: int = Field(default=10, ge=1, le=100) + score_threshold: float | None = None + include_content: bool = False + app_id: PathSafeId = "default" + project_id: PathSafeId = "default" + + +class DocumentPatchRequest(BaseModel): + """Request body for PATCH /documents/{doc_id}.""" + + title: str | None = Field(default=None, min_length=1, pattern=r"\w") + category_id: str | None = Field(default=None, min_length=1) + app_id: PathSafeId = "default" + project_id: PathSafeId = "default" + + +# ── Extractor builder ─────────────────────────────────────────────────────── + + +def _build_extractor() -> KnowledgeExtractor: + """Lazily import and build the knowledge extractor from ``everalgo``. + + Returns an object satisfying the ``KnowledgeExtractor`` protocol + defined in ``service.knowledge``. + """ + # Deferred: heavy everalgo import; only needed on document creation. + from everalgo.knowledge import KnowledgeExtractor as AlgoKnowledgeExtractor + + return AlgoKnowledgeExtractor(llm=get_llm_client()) + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def _reject_oversized_upload(file: UploadFile) -> None: + """Reject an upload whose declared size exceeds the configured limit. + + Raises: + InvalidInputError: When ``file.size`` exceeds ``knowledge.max_upload_bytes``. + """ + max_bytes = load_settings().knowledge.max_upload_bytes + if file.size is not None and file.size > max_bytes: + limit_mib = max_bytes / (1024 * 1024) + raise InvalidInputError(f"Uploaded file exceeds the {limit_mib:.1f} MiB limit.") + + +async def _parse_upload( + file: UploadFile, *, raw_bytes: bytes | None = None +) -> ParsedContent: + """Parse an uploaded file via component.parser, or fall back to UTF-8. + + Args: + file: FastAPI upload file handle. + raw_bytes: Pre-read bytes; when provided, skips ``file.read()``. + + Raises: + UnsupportedModalityError: When the file cannot be parsed. + InvalidInputError: When the parsed content is empty. + """ + from everos.component.parser import parser_available # Deferred: optional dep + + if raw_bytes is None: + raw_bytes = await file.read() + + if parser_available(): + from everalgo.types import RawFile # Deferred: optional dep + + from everos.component.parser import aparse_file # Deferred: optional dep + + extension = "" + if file.filename and "." in file.filename: + extension = file.filename.rsplit(".", 1)[-1].lower() + parsed = await aparse_file( + RawFile( + content=raw_bytes, + mime=file.content_type or "", + extension=extension, + ) + ) + if not parsed.text or not parsed.text.strip(): + raise InvalidInputError("Uploaded file has no valid content.") + return parsed + + try: + text = raw_bytes.decode("utf-8") + except UnicodeDecodeError as exc: + raise UnsupportedModalityError( + "File is not UTF-8 text. " + "Install everos[multimodal] for PDF/HTML/DOCX support." + ) from exc + parsed = ParsedContent(text=text) + if not parsed.text or not parsed.text.strip(): + raise InvalidInputError("Uploaded file has no valid content.") + return parsed + + +def _map_create_result(result: CreateDocumentResult) -> DocumentCreateResponse: + """Map CreateDocumentResult to response DTO.""" + return DocumentCreateResponse( + doc_id=result.doc_id, + category_id=result.category_id, + topic_count=result.topic_count, + source_name=result.source_name, + md_path=result.md_path, + original_file_path=result.original_file_path, + ) + + +def _map_document_detail(detail: DocumentDetail) -> DocumentDetailResponse: + """Map DocumentDetail to response DTO.""" + return DocumentDetailResponse( + doc_id=detail.doc_id, + category_id=detail.category_id, + title=detail.title, + summary=detail.summary, + source_name=detail.source_name, + source_type=detail.source_type, + original_file_path=detail.original_file_path, + topics=[ + TopicOverviewDTO( + topic_id=t.topic_id, + topic_name=t.topic_name, + topic_path=t.topic_path, + depth=t.depth, + summary=t.summary, + ) + for t in detail.topics + ], + created_at=to_display_tz(detail.created_at), + updated_at=to_display_tz(detail.updated_at), + ) + + +def _map_list_result(result: DocumentListResult) -> DocumentListResponse: + """Map DocumentListResult to response DTO.""" + return DocumentListResponse( + documents=[ + DocumentOverviewItemDTO( + doc_id=d.doc_id, + category_id=d.category_id, + title=d.title, + topic_count=d.topic_count, + created_at=to_display_tz(d.created_at), + ) + for d in result.documents + ], + total=result.total, + page=result.page, + page_size=result.page_size, + ) + + +def _map_topic_detail(detail: TopicDetail) -> TopicDetailResponse: + """Map TopicDetail to response DTO.""" + return TopicDetailResponse( + topic_id=detail.topic_id, + doc_id=detail.doc_id, + category_id=detail.category_id, + topic_name=detail.topic_name, + topic_path=detail.topic_path, + depth=detail.depth, + summary=detail.summary, + content=detail.content, + content_labels=detail.content_labels, + parent_topic_id=detail.parent_topic_id, + children_topic_ids=detail.children_topic_ids, + created_at=to_display_tz(detail.created_at), + updated_at=to_display_tz(detail.updated_at), + ) + + +def _map_search_result(result: SearchKnowledgeResult) -> KnowledgeSearchResponse: + """Map SearchKnowledgeResult to response DTO.""" + return KnowledgeSearchResponse( + hits=[ + SearchHitDTO( + topic_id=h.topic_id, + category_id=h.category_id, + topic_name=h.topic_name, + topic_path=h.topic_path, + depth=h.depth, + summary=h.summary, + content=h.content, + score=h.score, + retrieval_method=h.retrieval_method, + source=h.source, + document=DocumentContextDTO( + doc_id=h.document.doc_id, + title=h.document.title, + summary=h.document.summary, + ), + ) + for h in result.hits + ], + total=result.total, + took_ms=result.took_ms, + ) + + +# ── Routes ─────────────────────────────────────────────────────────────────── + + +@router.post("/documents", status_code=201) +# FastAPI requires flat Form/Query params — ≤5 positional rule exempted. +async def create_document_route( + request: Request, + file: UploadFile, + title: _FormTitle, + source_type: _FormOptStr = None, + category_id: _FormOptStr = None, + app_id: _FormPathSafe = "default", + project_id: _FormPathSafe = "default", +) -> SuccessEnvelope[DocumentCreateResponse]: + """Upload a new knowledge document.""" + rid = extract_request_id(request) + _reject_oversized_upload(file) + file_content = await file.read() + parsed = await _parse_upload(file, raw_bytes=file_content) + source_name = file.filename + + knowledge_dir = MemoryRoot.default().knowledge_dir(app_id, project_id) + extractor = _build_extractor() + + result = await create_document( + extractor=extractor, + parsed=parsed, + title=title, + knowledge_dir=knowledge_dir, + source_name=source_name, + source_type=source_type, + category_id=category_id, + file_content=file_content, + ) + return SuccessEnvelope(request_id=rid, data=_map_create_result(result)) + + +@router.put("/documents/{doc_id}", status_code=200) +# FastAPI requires flat Form/Query params — ≤5 positional rule exempted. +async def replace_document_route( + request: Request, + doc_id: _PathDocId, + file: UploadFile, + title: _FormTitle, + source_type: _FormOptStr = None, + category_id: _FormOptStr = None, + app_id: _FormPathSafe = "default", + project_id: _FormPathSafe = "default", +) -> SuccessEnvelope[DocumentCreateResponse]: + """Replace an existing knowledge document (atomic backup/restore on failure).""" + rid = extract_request_id(request) + _reject_oversized_upload(file) + file_content = await file.read() + parsed = await _parse_upload(file, raw_bytes=file_content) + + knowledge_dir = MemoryRoot.default().knowledge_dir(app_id, project_id) + extractor = _build_extractor() + + result = await replace_document( + extractor=extractor, + parsed=parsed, + title=title, + doc_id=doc_id, + knowledge_dir=knowledge_dir, + source_name=file.filename, + source_type=source_type, + category_id=category_id, + file_content=file_content, + ) + return SuccessEnvelope(request_id=rid, data=_map_create_result(result)) + + +@router.delete("/documents/{doc_id}", response_model=None) +async def delete_document_route( + request: Request, + doc_id: _PathDocId, + app_id: _QueryPathSafe = "default", + project_id: _QueryPathSafe = "default", +) -> SuccessEnvelope[DocumentDeleteResponse] | Response: + """Remove a knowledge document.""" + rid = extract_request_id(request) + result = await delete_document(doc_id, app_id, project_id) + + if result.deleted_topics == 0: + return Response(status_code=204) + + return SuccessEnvelope( + request_id=rid, + data=DocumentDeleteResponse( + doc_id=result.doc_id, + deleted_topics=result.deleted_topics, + ), + ) + + +@router.get("/documents") +# FastAPI requires flat Form/Query params — ≤5 positional rule exempted. +async def list_documents_route( + request: Request, + app_id: _QueryPathSafe = "default", + project_id: _QueryPathSafe = "default", + category_id: _QueryOptStr = None, + page: _QueryPage = 1, + page_size: _QueryPageSize = 20, + sort_by: _QuerySortBy = "created_at", + sort_order: _QuerySortOrder = "desc", +) -> SuccessEnvelope[DocumentListResponse]: + """Paginated document listing.""" + rid = extract_request_id(request) + result = await list_documents( + app_id, + project_id, + category_id=category_id, + page=page, + page_size=page_size, + sort_by=sort_by, + sort_order=sort_order, + ) + return SuccessEnvelope(request_id=rid, data=_map_list_result(result)) + + +@router.get("/documents/{doc_id}") +async def get_document_route( + request: Request, + doc_id: _PathDocId, + app_id: _QueryPathSafe = "default", + project_id: _QueryPathSafe = "default", +) -> SuccessEnvelope[DocumentDetailResponse]: + """Fetch a single document with its topic list.""" + rid = extract_request_id(request) + detail = await get_document(doc_id, app_id, project_id) + return SuccessEnvelope(request_id=rid, data=_map_document_detail(detail)) + + +@router.get("/topics/{topic_id}") +async def get_topic_route( + request: Request, + topic_id: _PathTopicId, + app_id: _QueryPathSafe = "default", + project_id: _QueryPathSafe = "default", +) -> SuccessEnvelope[TopicDetailResponse]: + """Fetch a single topic with full content.""" + rid = extract_request_id(request) + detail = await get_topic(topic_id, app_id, project_id) + return SuccessEnvelope(request_id=rid, data=_map_topic_detail(detail)) + + +@router.post("/search") +async def search_knowledge_route( + request: Request, + req: KnowledgeSearchRequest, +) -> SuccessEnvelope[KnowledgeSearchResponse]: + """Knowledge retrieval (keyword / vector / hybrid).""" + rid = extract_request_id(request) + result = await search_knowledge( + query=req.query, + method=req.method, + top_k=req.top_k, + score_threshold=req.score_threshold, + include_content=req.include_content, + app_id=req.app_id, + project_id=req.project_id, + ) + return SuccessEnvelope(request_id=rid, data=_map_search_result(result)) + + +@router.get("/categories") +async def list_categories_route( + request: Request, + app_id: _QueryPathSafe = "default", + project_id: _QueryPathSafe = "default", +) -> SuccessEnvelope[CategoryListResponse]: + """List taxonomy categories from ``.taxonomy.md``.""" + rid = extract_request_id(request) + overviews = await list_categories(app_id, project_id) + categories = [ + CategoryDTO( + category_id=c.category_id, + description=c.description, + document_count=c.document_count, + ) + for c in overviews + ] + return SuccessEnvelope( + request_id=rid, + data=CategoryListResponse(categories=categories), + ) + + +@router.patch("/documents/{doc_id}") +async def patch_document_route( + request: Request, + doc_id: _PathDocId, + req: DocumentPatchRequest, +) -> SuccessEnvelope[DocumentPatchResponse]: + """Update mutable document metadata fields.""" + rid = extract_request_id(request) + result = await patch_document( + doc_id, + req.app_id, + req.project_id, + title=req.title, + category_id=req.category_id, + ) + return SuccessEnvelope( + request_id=rid, + data=DocumentPatchResponse( + doc_id=result.doc_id, + updated_fields=result.updated_fields, + updated_at=to_display_tz(result.updated_at), + ), + ) diff --git a/src/everos/entrypoints/api/routes/memorize.py b/src/everos/entrypoints/api/routes/memorize.py index b3681de..af901ae 100644 --- a/src/everos/entrypoints/api/routes/memorize.py +++ b/src/everos/entrypoints/api/routes/memorize.py @@ -10,13 +10,13 @@ server-side and does not expose this endpoint). from __future__ import annotations +import re from typing import Annotated, Any, Literal -from fastapi import APIRouter, HTTPException, Request +from fastapi import APIRouter, Request from pydantic import AfterValidator, BaseModel, ConfigDict, Field -from everos.core.errors import MultimodalError, PathTraversalError -from everos.core.observability.tracing import gen_request_id +from everos.entrypoints.api.utils import extract_request_id from everos.service import memorize router = APIRouter(prefix="/api/v1/memory", tags=["memory"]) @@ -24,22 +24,34 @@ router = APIRouter(prefix="/api/v1/memory", tags=["memory"]) # ── Path-safe identifier ──────────────────────────────────────────────────── # ``app_id`` / ``project_id`` / ``sender_id`` all become directory segments -# under the memory root (``sender_id`` flows through to ``owner_id``), so they -# must reject ``.`` and ``..`` (path traversal). The basic character whitelist -# is enforced via ``pattern`` (pydantic_core uses the Rust regex engine, which -# does NOT support lookaround), and the two reserved tokens are filtered out -# with a follow-up ``AfterValidator``. +# under the memory root (``sender_id`` flows through to ``owner_id`` and is +# joined into the daily-log write path), so they must reject ``.`` and ``..`` +# (path traversal). The basic character whitelist is enforced via ``pattern`` +# (pydantic_core uses the Rust regex engine, which does NOT support +# lookaround), and the two reserved tokens are filtered out with a follow-up +# ``AfterValidator``. # -# ``@`` and ``+`` are admitted so real-world ids survive (email-style ids and -# plus-addressing). Path separators and NUL stay out of the whitelist, while -# the markdown writer's root-containment check is the final backstop. +# ``@`` and ``+`` are admitted so real-world ids survive (email-style +# ``user@example.com``, plus-addressing ``user+tag``); both are legal, +# non-separator filename chars on every target filesystem (incl. NTFS, whose +# reserved set is ``< > : " / \ | ? *``). The genuinely path-dangerous chars +# (``/`` ``\`` NUL) stay out of the whitelist, and ``.``/``..`` stay blocked +# by the token filter; the markdown writer's ``_ensure_within_root`` is the +# final backstop regardless. _PATH_SAFE_CHARSET = r"^[a-zA-Z0-9_.@+-]+$" _PATH_TRAVERSAL_TOKENS = frozenset({".", ".."}) +_PATH_SAFE_RE = re.compile(_PATH_SAFE_CHARSET) + + def _reject_path_traversal(value: str) -> str: if value in _PATH_TRAVERSAL_TOKENS: raise ValueError("'.' and '..' are reserved (path traversal)") + if not _PATH_SAFE_RE.match(value): + raise ValueError( + "Only alphanumerics, underscore, dot, hyphen, @, and + are allowed" + ) return value @@ -75,6 +87,9 @@ class ContentItemDTO(BaseModel): class MessageItemDTO(BaseModel): + # ``sender_id`` becomes ``owner_id`` and then a directory segment on the + # episode write path, so it carries the same path-safety guard as + # ``app_id`` / ``project_id`` (charset whitelist + ``.``/``..`` rejection). sender_id: PathSafeId = Field( ..., min_length=1, @@ -155,13 +170,8 @@ async def add_memory( request: Request, ) -> SuccessEnvelope[AddResponseData]: """Add messages into the user-memory + agent-memory pipelines.""" - request_id = getattr(request.state, "request_id", None) or _gen_request_id() - try: - result = await memorize(req.model_dump()) - except MultimodalError as exc: - raise HTTPException(status_code=415, detail=str(exc)) from exc - except PathTraversalError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc + request_id = extract_request_id(request) + result = await memorize(req.model_dump()) return SuccessEnvelope( request_id=request_id, data=AddResponseData( @@ -181,7 +191,7 @@ async def flush_memory( [OSS-only] — cloud edition decides boundary timing server-side and does not expose this endpoint. """ - request_id = getattr(request.state, "request_id", None) or _gen_request_id() + request_id = extract_request_id(request) result = await memorize( { "session_id": req.session_id, @@ -200,8 +210,3 @@ async def flush_memory( request_id=request_id, data=FlushResponseData(status=status), ) - - -def _gen_request_id() -> str: - """Fallback request id when no middleware set one.""" - return gen_request_id() diff --git a/src/everos/entrypoints/api/routes/ome.py b/src/everos/entrypoints/api/routes/ome.py new file mode 100644 index 0000000..baf3de6 --- /dev/null +++ b/src/everos/entrypoints/api/routes/ome.py @@ -0,0 +1,47 @@ +"""OME trigger route — manually invoke a registered strategy.""" + +from __future__ import annotations + +from fastapi import APIRouter +from pydantic import BaseModel + +from everos.core.errors import NotFoundError +from everos.core.observability.logging import get_logger + +router = APIRouter(prefix="/api/v1/ome", tags=["ome"]) + +logger = get_logger(__name__) + + +class TriggerRequest(BaseModel): + """Request body for ``POST /api/v1/ome/trigger``.""" + + name: str + timeout: float = 120.0 + force: bool = False + + +class TriggerResponse(BaseModel): + """Response body for ``POST /api/v1/ome/trigger``.""" + + status: str + name: str + + +@router.post("/trigger", response_model=TriggerResponse) +async def trigger(req: TriggerRequest) -> TriggerResponse: + """Manually trigger a registered OME strategy and wait for completion.""" + # Deferred: avoid importing heavy OME engine at module level. + from everos.service.memorize import _get_engine # noqa: SLF001 + + engine = _get_engine() + try: + await engine.trigger_manual(req.name, force=req.force) + except KeyError: + raise NotFoundError(f"strategy '{req.name}' not found") from None + logger.info("ome_trigger_manual", strategy=req.name) + idle = await engine.wait_idle(timeout=req.timeout) + if not idle: + logger.warning("ome_trigger_timeout", strategy=req.name, timeout=req.timeout) + return TriggerResponse(status="timeout", name=req.name) + return TriggerResponse(status="ok", name=req.name) diff --git a/src/everos/entrypoints/api/routes/search.py b/src/everos/entrypoints/api/routes/search.py index f8b1c5f..26be838 100644 --- a/src/everos/entrypoints/api/routes/search.py +++ b/src/everos/entrypoints/api/routes/search.py @@ -8,9 +8,9 @@ on the way out. from __future__ import annotations -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter -from everos.memory.search import FilterError, SearchRequest, SearchResponse +from everos.memory.search import SearchRequest, SearchResponse from everos.service import search router = APIRouter(prefix="/api/v1/memory", tags=["memory"]) @@ -19,9 +19,4 @@ router = APIRouter(prefix="/api/v1/memory", tags=["memory"]) @router.post("/search", response_model=SearchResponse) async def post_search(req: SearchRequest) -> SearchResponse: """Hybrid retrieval across the configured memory backends.""" - try: - return await search(req) - except FilterError as exc: - # Filter-DSL violations surface as 422 with the compile message - # (mirrors /get's contract). - raise HTTPException(status_code=422, detail=str(exc)) from exc + return await search(req) diff --git a/src/everos/entrypoints/api/utils.py b/src/everos/entrypoints/api/utils.py new file mode 100644 index 0000000..d09238f --- /dev/null +++ b/src/everos/entrypoints/api/utils.py @@ -0,0 +1,13 @@ +"""Shared helpers for the API layer (routes + exception handlers).""" + +from __future__ import annotations + +from fastapi import Request + +from everos.core.observability.tracing import gen_request_id + + +def extract_request_id(request: Request) -> str: + """Return the request_id set by middleware, or mint a fresh fallback.""" + rid = getattr(request.state, "request_id", None) + return str(rid) if rid else gen_request_id() diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index 1926676..8d27e8a 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -21,6 +21,7 @@ scanner background task is started. from __future__ import annotations import asyncio +import os from contextlib import asynccontextmanager from pathlib import Path from typing import Annotated @@ -53,6 +54,19 @@ app = typer.Typer( ) +@app.callback() +def _cascade_callback( + root: str | None = typer.Option( + None, + "--root", + help="Memory root directory (env: EVEROS_ROOT, default: ~/.everos)", + ), +) -> None: + """Set memory root before any cascade subcommand runs.""" + if root: + os.environ["EVEROS_ROOT"] = root + + # ── shared runtime context ─────────────────────────────────────────────── diff --git a/src/everos/entrypoints/cli/commands/config_cmd.py b/src/everos/entrypoints/cli/commands/config_cmd.py new file mode 100644 index 0000000..3850a8f --- /dev/null +++ b/src/everos/entrypoints/cli/commands/config_cmd.py @@ -0,0 +1,85 @@ +"""``everos config show`` — display effective configuration.""" + +from __future__ import annotations + +import os + +import typer + +from everos.config.settings import resolve_root + +app = typer.Typer( + name="config", + help="Configuration management", + no_args_is_help=True, +) + +_SECRET_FIELDS = {"api_key"} + +_SECTION_NAMES = ( + "memory", + "api", + "sqlite", + "lancedb", + "llm", + "multimodal", + "embedding", + "rerank", + "boundary_detection", + "memorize", + "clustering", + "search", + "knowledge", +) + + +def _mask(value: str) -> str: + """Mask a secret value, keeping first/last 4 chars if long enough.""" + if len(value) <= 8: + return "****" + return value[:4] + "****" + value[-4:] + + +@app.command("show") +def show( + root: str | None = typer.Option( + None, + "--root", + help="Memory root directory", + ), +) -> None: + """Print the effective configuration.""" + if root: + os.environ["EVEROS_ROOT"] = root + + resolved = resolve_root(root) + typer.echo(f"Root: {resolved}") + typer.echo() + + everos_toml = resolved / "everos.toml" + if everos_toml.is_file(): + typer.echo(f"Config: {everos_toml}") + else: + typer.echo("Config: (no everos.toml found, using defaults)") + + ome_toml = resolved / "ome.toml" + if ome_toml.is_file(): + typer.echo(f"Strategy: {ome_toml}") + typer.echo() + + from everos.config import load_settings + + load_settings.cache_clear() + settings = load_settings() + + for section_name in _SECTION_NAMES: + section = getattr(settings, section_name, None) + if section is None: + continue + typer.secho(f"[{section_name}]", bold=True) + for field_name, value in section.model_dump().items(): + display = str(value) + if field_name in _SECRET_FIELDS and value: + display = _mask(str(value)) + typer.echo(f" {field_name} = {display}") + typer.echo() diff --git a/src/everos/entrypoints/cli/commands/init_cmd.py b/src/everos/entrypoints/cli/commands/init_cmd.py index 9f25c69..092d13e 100644 --- a/src/everos/entrypoints/cli/commands/init_cmd.py +++ b/src/everos/entrypoints/cli/commands/init_cmd.py @@ -1,10 +1,9 @@ -"""``everos init`` — generate a starter ``.env`` from the packaged template. +"""``everos init`` — generate starter config files in the memory root. -The ``env.template`` ships inside the wheel as package data at -``everos/templates/env.template``. ``init`` reads it via -:mod:`importlib.resources`, so the command works identically for pip- -installed users and source-tree users (the file is the single source -of truth). +Copies the shipped ``default.toml`` and ``default_ome.toml`` templates +into the resolved memory root as ``everos.toml`` and ``ome.toml`` +respectively. Users then edit these files to fill in API keys and tune +strategy schedules. Subcommand mounted as ``everos init`` (top-level leaf command — not a Typer group), to match the idiomatic ``alembic init`` / ``django-admin @@ -13,74 +12,14 @@ startproject`` shape. from __future__ import annotations -import contextlib -import logging -import os -import sys -import tempfile -from importlib import resources from pathlib import Path import typer -_TEMPLATE_PACKAGE = "everos.templates" -_TEMPLATE_NAME = "env.template" +from everos.config.settings import resolve_root -_log = logging.getLogger("everos.cli.init") - - -def _read_template() -> str: - """Read the packaged ``env.template`` from wheel resources. - - Returns the file contents as a UTF-8 string. Raises ``RuntimeError`` - on missing-file — if this fires it means the wheel was built from a - source tree where ``src/everos/templates/env.template`` was missing - (canonical location; auto-included via ``packages=["src/everos"]`` - in ``pyproject.toml``). - """ - try: - return ( - resources.files(_TEMPLATE_PACKAGE) - .joinpath(_TEMPLATE_NAME) - .read_text(encoding="utf-8") - ) - except (FileNotFoundError, ModuleNotFoundError) as exc: - raise RuntimeError( - f"packaged template {_TEMPLATE_NAME!r} not found under " - f"{_TEMPLATE_PACKAGE!r}; the wheel is missing its " - "force-include entry (see pyproject.toml " - "[tool.hatch.build.targets.wheel.force-include])." - ) from exc - - -def _xdg_default_path() -> Path: - """``$XDG_CONFIG_HOME/everos/.env`` (default ``~/.config/everos/.env``).""" - xdg = os.environ.get("XDG_CONFIG_HOME") or "~/.config" - return Path(xdg).expanduser() / "everos" / ".env" - - -def _atomic_write(target: Path, content: str, mode: int = 0o600) -> None: - """Write ``content`` to ``target`` atomically with ``mode`` permission. - - Writes to a tempfile in the same directory then ``os.replace``s it - onto the target — guarantees either the full new file is visible or - the original (if any) is untouched. Permission bits applied before - the rename so the file is never readable by other users. - """ - target.parent.mkdir(parents=True, exist_ok=True) - fd, tmp_path = tempfile.mkstemp( - prefix=target.name + ".", - dir=target.parent, - ) - try: - with os.fdopen(fd, "w", encoding="utf-8") as f: - f.write(content) - os.chmod(tmp_path, mode) - os.replace(tmp_path, target) - except Exception: - with contextlib.suppress(OSError): - os.unlink(tmp_path) - raise +_EVEROS_TEMPLATE = Path(__file__).resolve().parents[3] / "config" / "default.toml" +_OME_TEMPLATE = Path(__file__).resolve().parents[3] / "config" / "default_ome.toml" def register(parent: typer.Typer) -> None: @@ -88,98 +27,65 @@ def register(parent: typer.Typer) -> None: @parent.command("init") def init( - to: str | None = typer.Option( + root: str | None = typer.Option( None, - "--to", - help=( - "Target path for the .env file (default: ./.env). " - "Parent directories are created if needed." - ), + "--root", + help="Memory root directory (default: ~/.everos)", ), force: bool = typer.Option( False, "--force", - help="Overwrite an existing file at the target path.", + help="Overwrite existing files", ), print_: bool = typer.Option( False, "--print", - help="Print the template to stdout instead of writing to disk.", - ), - xdg: bool = typer.Option( - False, - "--xdg", - help=( - "Shortcut for --to=${XDG_CONFIG_HOME:-~/.config}/everos/.env " - "(mutually exclusive with --to)." - ), + help="Print the everos.toml template to stdout instead of writing to disk.", ), ) -> None: - """Generate a starter ``.env`` from the packaged template. + """Generate starter configuration files. Common flows:: - everos init # writes ./.env - everos init --xdg # writes ~/.config/everos/.env - everos init --to /etc/foo.env --force - everos init --print > custom.env + everos init # writes to ~/.everos/ + everos init --root /data/everos # writes to /data/everos/ + everos init --force # overwrites existing files + everos init --print # prints everos.toml to stdout Exit codes: - - 0 — written successfully (or printed to stdout). - - 1 — target file already exists and ``--force`` was not given. - - 2 — packaged template missing (wheel build problem). - - 3 — write failed (permissions / disk full / parent unwritable). + - 0 — files created successfully (or printed to stdout). + - 1 — files already exist and ``--force`` was not given. """ - if xdg and to is not None: - typer.secho( - "error: --xdg and --to are mutually exclusive", - fg=typer.colors.RED, - err=True, - ) - raise typer.Exit(code=2) - - try: - template = _read_template() - except RuntimeError as exc: - typer.secho(f"error: {exc}", fg=typer.colors.RED, err=True) - raise typer.Exit(code=2) from exc - if print_: - sys.stdout.write(template) + import sys + + sys.stdout.write(_EVEROS_TEMPLATE.read_text(encoding="utf-8")) return - if xdg: - target = _xdg_default_path() - elif to is not None: - target = Path(to).expanduser().resolve() - else: - target = Path.cwd() / ".env" + resolved = resolve_root(root) + resolved.mkdir(parents=True, exist_ok=True) - if target.exists() and not force: - typer.secho( - f"error: {target} already exists; pass --force to overwrite", - fg=typer.colors.RED, - err=True, - ) + everos_toml = resolved / "everos.toml" + ome_toml = resolved / "ome.toml" + + created: list[Path] = [] + for target, template in [ + (everos_toml, _EVEROS_TEMPLATE), + (ome_toml, _OME_TEMPLATE), + ]: + if target.exists() and not force: + typer.echo(f" exists: {target} (skipped)") + continue + target.write_bytes(template.read_bytes()) + created.append(target) + typer.secho(f" created: {target}", fg=typer.colors.GREEN) + + if not created: + typer.echo("Nothing to create (use --force to overwrite).") raise typer.Exit(code=1) - try: - _atomic_write(target, template) - except OSError as exc: - typer.secho( - f"error: failed to write {target}: {exc}", - fg=typer.colors.RED, - err=True, - ) - raise typer.Exit(code=3) from exc - - # Friendly next-step block (stdout — quiet enough for piping). - size_kb = target.stat().st_size / 1024 - typer.secho(f"✓ wrote {target} ({size_kb:.1f} KB)", fg=typer.colors.GREEN) - typer.echo("Next steps:") - typer.echo(" 1. Edit the file and fill in the API keys (see comments inside).") - typer.echo(" 2. Run `everos server start`.") - typer.echo( - "Docs: https://github.com/EverMind-AI/EverOS/blob/main/QUICKSTART.md" - ) + typer.echo("\nNext steps:") + typer.echo(f" 1. Edit {everos_toml} — fill in API keys (see comments inside)") + root_flag = f" --root {resolved}" if root else "" + typer.echo(f" 2. Run: everos server start{root_flag}") diff --git a/src/everos/entrypoints/cli/commands/server.py b/src/everos/entrypoints/cli/commands/server.py index 31f48f3..c3f5c0b 100644 --- a/src/everos/entrypoints/cli/commands/server.py +++ b/src/everos/entrypoints/cli/commands/server.py @@ -11,11 +11,12 @@ from __future__ import annotations import logging import os import sys -from pathlib import Path import typer import uvicorn +from everos.config.settings import resolve_root + app = typer.Typer( name="server", help="Run / manage the HTTP API server", @@ -23,58 +24,6 @@ app = typer.Typer( ) -def _resolve_env_file(explicit: str | None) -> Path | None: - """Find the first existing ``.env`` along the four-layer search path. - - Search order (highest-wins): - - 1. ``explicit`` — when the caller passed ``--env-file ``. - 2. ``./.env`` — the current working directory (project-local convention). - 3. ``${XDG_CONFIG_HOME:-~/.config}/everos/.env`` — XDG-standard user config. - 4. ``~/.everos/.env`` — the project's default memory-root location. - - Returns ``None`` if none of the layers exist (caller may then fall back - to inherited process env / CI secrets). - """ - candidates: list[Path] = [] - if explicit: - candidates.append(Path(explicit).expanduser()) - candidates.append(Path.cwd() / ".env") - xdg = os.environ.get("XDG_CONFIG_HOME") or "~/.config" - candidates.append(Path(xdg).expanduser() / "everos" / ".env") - candidates.append(Path("~/.everos/.env").expanduser()) - for p in candidates: - try: - if p.is_file(): - return p - except OSError: - # Path traversal / permission denied on a fallback candidate - # must not crash the search — skip and keep going. - continue - return None - - -def _load_env_file(path: str | None) -> Path | None: - """Load environment variables from the resolved ``.env`` file. - - Returns the path that was loaded, or ``None`` when no ``.env`` was - found anywhere along the search path. Existence of a ``.env`` is - optional — the user may rely entirely on inherited process env - (e.g. container / CI secret injection). - """ - resolved = _resolve_env_file(path) - if resolved is None: - return None - try: - from dotenv import load_dotenv - - load_dotenv(resolved, override=False) - except ImportError: - # python-dotenv is in our deps; tolerate its absence anyway. - pass - return resolved - - @app.command("start") def start( host: str | None = typer.Option( @@ -87,14 +36,10 @@ def start( "--port", help="Bind port (env: EVEROS_API__PORT, default: 8000)", ), - env_file: str | None = typer.Option( + root: str | None = typer.Option( None, - "--env-file", - help=( - "Path to a dotenv file (highest priority). When omitted, " - "the server searches: ./.env → ${XDG_CONFIG_HOME:-~/.config}" - "/everos/.env → ~/.everos/.env. Run `everos init` to create one." - ), + "--root", + help="Memory root directory (env: EVEROS_ROOT, default: ~/.everos)", ), reload: bool = typer.Option( False, @@ -108,10 +53,20 @@ def start( ), ) -> None: """Start the HTTP API server.""" - loaded_env = _load_env_file(env_file) + if root: + os.environ["EVEROS_ROOT"] = root + + resolved_root = resolve_root(root) + everos_toml = resolved_root / "everos.toml" + if not everos_toml.is_file(): + typer.secho( + f"Error: {everos_toml} not found.\n" + f"Run `everos init` first to create configuration files.", + fg=typer.colors.RED, + err=True, + ) + raise typer.Exit(code=1) - # Load settings AFTER .env is in place so EVEROS_API__HOST and - # EVEROS_API__PORT (and any other env override) are honored. from everos.config import load_settings settings = load_settings() @@ -125,13 +80,6 @@ def start( configure_logging(level=log_level_resolved) bootstrap_logger = logging.getLogger("everos.cli.server") - if loaded_env is not None: - bootstrap_logger.info("loaded env file: %s", loaded_env) - else: - bootstrap_logger.info( - "no .env found along the search path; relying on inherited env vars " - "(run `everos init` to generate one)" - ) bootstrap_logger.info("starting everos on %s:%d", host_resolved, port_resolved) if host_resolved == "0.0.0.0": bootstrap_logger.warning( @@ -147,11 +95,6 @@ def start( reload=reload, factory=True, log_level=log_level_resolved.lower(), - # ``configure_logging()`` above already installed the root - # handler + structlog ProcessorFormatter. ``log_config=None`` - # stops uvicorn from running its own ``dictConfig`` over - # ours; otherwise uvicorn / fastapi messages revert to the - # ``INFO:`` no-structlog format on every restart. log_config=None, ) except KeyboardInterrupt: diff --git a/src/everos/entrypoints/cli/main.py b/src/everos/entrypoints/cli/main.py index e030678..b7338bd 100644 --- a/src/everos/entrypoints/cli/main.py +++ b/src/everos/entrypoints/cli/main.py @@ -13,7 +13,7 @@ from __future__ import annotations import typer -from .commands import cascade, demo, init_cmd, server +from .commands import cascade, config_cmd, demo, init_cmd, server app = typer.Typer( name="everos", @@ -24,6 +24,7 @@ app = typer.Typer( app.add_typer(server.app, name="server") app.add_typer(cascade.app, name="cascade") +app.add_typer(config_cmd.app, name="config") # ``init`` is a top-level leaf command (not a Typer group) — match the # idiomatic ``alembic init`` / ``django-admin startproject`` shape. diff --git a/src/everos/infra/ome/_background/config_reloader.py b/src/everos/infra/ome/_background/config_reloader.py index 722f406..ca6a65b 100644 --- a/src/everos/infra/ome/_background/config_reloader.py +++ b/src/everos/infra/ome/_background/config_reloader.py @@ -207,6 +207,11 @@ class ConfigReloader: """Fire-and-forget the watch loop. Idempotent: raises on double-start.""" if self._path is None: return + if not self._path.exists(): + raise FileNotFoundError( + f"{self._path} not found. " + "Run `everos init` to create configuration files." + ) if self._task is not None and not self._task.done(): raise RuntimeError("ConfigReloader already started") self._task = asyncio.create_task(self._loop()) diff --git a/src/everos/infra/ome/_dispatch/registry.py b/src/everos/infra/ome/_dispatch/registry.py index ad0b49b..b022ccc 100644 --- a/src/everos/infra/ome/_dispatch/registry.py +++ b/src/everos/infra/ome/_dispatch/registry.py @@ -9,10 +9,8 @@ Kahn-style topological pass on the event-flow DAG implied by from __future__ import annotations from collections import defaultdict, deque -from collections.abc import Callable -from typing import Any -from everos.infra.ome.decorator import StrategyMeta +from everos.infra.ome.decorator import Strategy, StrategyMeta from everos.infra.ome.events import BaseEvent, CronTick, IdleTick from everos.infra.ome.exceptions import StartupValidationError from everos.infra.ome.triggers import Cron, Idle, Immediate, Trigger @@ -24,18 +22,19 @@ class StrategyRegistry: def __init__(self) -> None: self._strategies: dict[str, StrategyMeta] = {} - def register(self, func: Callable[..., Any]) -> None: - """Register a strategy function (reads ``_ome_strategy_meta``). + def register(self, strategy: Strategy) -> None: + """Register a :class:`Strategy` returned by ``@offline_strategy``. - Raises ``StartupValidationError`` if ``func`` is not decorated - with ``@offline_strategy`` or if its name is already registered. + Raises: + StartupValidationError: If ``strategy`` is not a Strategy + instance or its name is already registered. """ - meta = getattr(func, "_ome_strategy_meta", None) - if not isinstance(meta, StrategyMeta): - fn_name = getattr(func, "__name__", repr(func)) + if not isinstance(strategy, Strategy): + label = getattr(strategy, "__name__", repr(strategy)) raise StartupValidationError( - f"register: {fn_name} is not decorated with @offline_strategy" + f"register: {label} is not decorated with @offline_strategy" ) + meta = strategy.meta if meta.name in self._strategies: raise StartupValidationError( f"register: duplicate strategy name {meta.name!r}" diff --git a/src/everos/infra/ome/_dispatch/runner.py b/src/everos/infra/ome/_dispatch/runner.py index 4372623..6b87619 100644 --- a/src/everos/infra/ome/_dispatch/runner.py +++ b/src/everos/infra/ome/_dispatch/runner.py @@ -22,6 +22,7 @@ from __future__ import annotations import asyncio import traceback from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING from uuid import uuid4 from structlog.contextvars import bound_contextvars @@ -35,14 +36,18 @@ from everos.infra.ome.events import BaseEvent from everos.infra.ome.exceptions import EmitNotDeclaredError, StrategyContractError from everos.infra.ome.records import RunRecord +if TYPE_CHECKING: + from everos.infra.ome.engine import OfflineEngine + logger = get_logger(__name__) class _RunCtx: - """Per-invocation context handed to ``meta.func(event, ctx)``. + """Implements :class:`~everos.infra.ome.context.StrategyContext` Protocol. - Carries ``run_id``, a strategy-scoped logger, and the ``emit`` - callback that enforces the declared ``emits=[...]`` contract. + Carries ``run_id``, a strategy-scoped logger, the ``emit`` + callback that enforces the declared ``emits=[...]`` contract, + and engine-delegated helpers for event/run queries. """ def __init__( @@ -52,12 +57,14 @@ class _RunCtx: strategy_name: str, emit_hook: Callable[[BaseEvent], Awaitable[None]], declared_emits: frozenset[type[BaseEvent]], + engine: OfflineEngine, ) -> None: self.run_id = run_id self.logger = get_logger("ome.strategy") self._emit_hook = emit_hook self._declared = declared_emits self._strategy_name = strategy_name + self._engine = engine async def emit(self, event: BaseEvent) -> None: if type(event) not in self._declared: @@ -67,6 +74,19 @@ class _RunCtx: ) await self._emit_hook(event) + async def wait_for_event( + self, + event_id: str, + *, + timeout: float = 120.0, # noqa: ASYNC109 + ) -> list[RunRecord]: + """Poll until all runs for ``event_id`` reach a terminal status.""" + return await self._engine.wait_for_event(event_id, timeout=timeout) + + async def list_runs_by_event_id(self, event_id: str) -> list[RunRecord]: + """Return all run records triggered by ``event_id``.""" + return await self._engine.list_runs_by_event_id(event_id) + class Runner: """Drive one strategy invocation through retries to a terminal state.""" @@ -78,11 +98,13 @@ class Runner: engine_sem: asyncio.Semaphore, emit_hook: Callable[[BaseEvent], Awaitable[None]], on_dead_letter: Callable[[RunRecord], None] | None = None, + engine: OfflineEngine, ) -> None: self._rec = run_record_store self._sem = engine_sem self._emit_hook = emit_hook self._on_dead_letter = on_dead_letter + self._engine = engine async def run( self, @@ -143,6 +165,7 @@ class Runner: strategy_name=meta.name, emit_hook=self._emit_hook, declared_emits=meta.emits, + engine=self._engine, ) with bound_contextvars( # type: ignore[arg-type] # structlog typed as Generator; @contextmanager wraps at runtime (structlog/contextvars.py:170) strategy_name=meta.name, @@ -156,6 +179,7 @@ class Runner: event_topic=event_topic, event_payload=event_payload, max_retries_snapshot=max_retries_snapshot, + event_id=event.event_id, ): return True # mark_running failed; abort run, no DB row exists try: @@ -194,6 +218,7 @@ class Runner: event_topic: str, event_payload: str, max_retries_snapshot: int, + event_id: str, ) -> bool: """Persist this attempt as RUNNING; return ``False`` on write failure. @@ -210,6 +235,7 @@ class Runner: event_topic=event_topic, event_payload=event_payload, max_retries_snapshot=max_retries_snapshot, + event_id=event_id, ) except Exception: # noqa: BLE001 logger.exception( diff --git a/src/everos/infra/ome/_stores/run_record.py b/src/everos/infra/ome/_stores/run_record.py index 7d66ea6..deaf5d1 100644 --- a/src/everos/infra/ome/_stores/run_record.py +++ b/src/everos/infra/ome/_stores/run_record.py @@ -40,14 +40,15 @@ class RunRecordStore: event_topic: str, event_payload: str, max_retries_snapshot: int, + event_id: str, ) -> None: """Insert a new RUNNING row and trim the strategy's ring buffer atomically.""" async with self._storage.transaction() as conn: await conn.execute( "INSERT INTO run_record " "(run_id, strategy_name, status, attempt, started_at, " - " event_topic, event_payload, max_retries_snapshot) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + " event_topic, event_payload, max_retries_snapshot, event_id) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", ( run_id, strategy_name, @@ -57,6 +58,7 @@ class RunRecordStore: event_topic, event_payload, max_retries_snapshot, + event_id, ), ) await conn.execute( @@ -145,10 +147,20 @@ class RunRecordStore: rows = await cur.fetchall() return [_row_to_record(r) for r in rows] + async def list_by_event_id(self, event_id: str) -> list[RunRecord]: + """Return all runs triggered by the given ``event_id``.""" + async with self._storage.connect() as conn: + cur = await conn.execute( + _SELECT_COLUMNS + " WHERE event_id = ? ORDER BY started_at DESC", + (event_id,), + ) + rows = await cur.fetchall() + return [_row_to_record(r) for r in rows] + _SELECT_COLUMNS = ( "SELECT run_id, strategy_name, status, attempt, started_at, finished_at, " - " error, event_topic, event_payload, max_retries_snapshot " + " error, event_topic, event_payload, max_retries_snapshot, event_id " "FROM run_record" ) @@ -165,4 +177,5 @@ def _row_to_record(row: tuple) -> RunRecord: event_topic=row[7], event_payload=row[8], max_retries_snapshot=row[9], + event_id=row[10], ) diff --git a/src/everos/infra/ome/_stores/storage.py b/src/everos/infra/ome/_stores/storage.py index ac8fd38..ced3f71 100644 --- a/src/everos/infra/ome/_stores/storage.py +++ b/src/everos/infra/ome/_stores/storage.py @@ -55,12 +55,15 @@ CREATE TABLE IF NOT EXISTS run_record ( error TEXT, event_topic TEXT NOT NULL, event_payload TEXT NOT NULL, - max_retries_snapshot INTEGER NOT NULL + max_retries_snapshot INTEGER NOT NULL, + event_id TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_run_strategy_started ON run_record (strategy_name, started_at DESC); CREATE INDEX IF NOT EXISTS idx_run_status_started ON run_record (status, started_at DESC); +CREATE INDEX IF NOT EXISTS idx_run_event_id + ON run_record (event_id); """ _INIT_PRAGMAS = ("PRAGMA journal_mode=WAL",) @@ -78,14 +81,31 @@ class OMEStorage: self.db_path = db_path async def init(self) -> None: - """Create parent dirs + apply file-level pragmas + create schema.""" + """Create parent dirs + apply file-level pragmas + create schema. + + Runs forward-only migrations for existing databases (e.g. adding + the ``event_id`` column introduced in P3). + """ self.db_path.parent.mkdir(parents=True, exist_ok=True) async with aiosqlite.connect(self.db_path) as conn: for pragma in _INIT_PRAGMAS: await conn.execute(pragma) + await self._migrate(conn) await conn.executescript(_SCHEMA) await conn.commit() + @staticmethod + async def _migrate(conn: aiosqlite.Connection) -> None: + """Forward-only column migrations for existing databases.""" + cur = await conn.execute("PRAGMA table_info(run_record)") + columns = {row[1] for row in await cur.fetchall()} + if not columns: + return + if "event_id" not in columns: + await conn.execute( + "ALTER TABLE run_record ADD COLUMN event_id TEXT NOT NULL DEFAULT ''" + ) + @asynccontextmanager async def connect(self) -> AsyncIterator[aiosqlite.Connection]: """Yield an aiosqlite connection with per-connection pragmas applied.""" diff --git a/src/everos/infra/ome/context.py b/src/everos/infra/ome/context.py index 4038dd5..e583785 100644 --- a/src/everos/infra/ome/context.py +++ b/src/everos/infra/ome/context.py @@ -13,21 +13,34 @@ from typing import Protocol from structlog.types import FilteringBoundLogger from everos.infra.ome.events import BaseEvent +from everos.infra.ome.records import RunRecord class StrategyContext(Protocol): """Per-run context handed to a strategy function. - - run_id: the current RunRecord id (string). - - logger: structlog logger; ``strategy_name`` / ``run_id`` / - ``attempt`` are auto-injected into every log record in this call - — strategies don't have to use this specific logger to get those - fields. - - emit(event): chain-emit a follow-up event (must be in decorator's - ``emits=[...]``, else EmitNotDeclaredError). + Attributes: + run_id: The current RunRecord id. + logger: Structlog logger with ``strategy_name`` / ``run_id`` / + ``attempt`` auto-bound. + emit: Chain-emit a follow-up event (must be in decorator's + ``emits=[...]``, else EmitNotDeclaredError). + wait_for_event: Poll until all runs triggered by an event_id + reach a terminal status. + list_runs_by_event_id: Return all run records triggered by an + event_id. """ run_id: str logger: FilteringBoundLogger async def emit(self, event: BaseEvent) -> None: ... + + async def wait_for_event( + self, + event_id: str, + *, + timeout: float = 120.0, # noqa: ASYNC109 + ) -> list[RunRecord]: ... + + async def list_runs_by_event_id(self, event_id: str) -> list[RunRecord]: ... diff --git a/src/everos/infra/ome/decorator.py b/src/everos/infra/ome/decorator.py index 6c0d5ee..30b72e1 100644 --- a/src/everos/infra/ome/decorator.py +++ b/src/everos/infra/ome/decorator.py @@ -1,7 +1,7 @@ -"""@offline_strategy decorator — attaches StrategyMeta to the function. +"""@offline_strategy decorator — attaches StrategyMeta to a Strategy wrapper. Decorator is side-effect-free; engine collects via explicit -`engine.register(func)`. +``engine.register(strategy)``. """ from __future__ import annotations @@ -9,15 +9,22 @@ from __future__ import annotations import inspect from collections.abc import Awaitable, Callable from dataclasses import dataclass +from typing import Any, TypeVar, overload from everos.infra.ome.context import StrategyContext -from everos.infra.ome.events import BaseEvent +from everos.infra.ome.events import BaseEvent, CronTick, IdleTick from everos.infra.ome.gates import Counter -from everos.infra.ome.triggers import Trigger +from everos.infra.ome.triggers import Cron, Idle, Immediate, Trigger type AppliesTo = str | Callable[[BaseEvent], bool] | None type StrategyFn = Callable[[BaseEvent, StrategyContext], Awaitable[None]] +_E = TypeVar("_E", bound=BaseEvent) + +_CronStrategyFn = Callable[[CronTick, StrategyContext], Awaitable[None]] +_IdleStrategyFn = Callable[[IdleTick, StrategyContext], Awaitable[None]] +_EventStrategyFn = Callable[[_E, StrategyContext], Awaitable[None]] + @dataclass(frozen=True) class StrategyMeta: @@ -33,6 +40,68 @@ class StrategyMeta: func: StrategyFn +class Strategy: + """Wrapper returned by :func:`offline_strategy`. + + Carries typed :attr:`meta` and delegates ``__call__`` to the + original async function — so ``await my_strategy(event, ctx)`` + works transparently in both production and tests. + + Args: + meta: Frozen strategy metadata captured at decoration time. + """ + + __slots__ = ("meta",) + + def __init__(self, meta: StrategyMeta) -> None: + self.meta = meta + + async def __call__(self, event: BaseEvent, ctx: StrategyContext) -> None: + await self.meta.func(event, ctx) + + def __repr__(self) -> str: + return f"Strategy({self.meta.name!r})" + + +@overload +def offline_strategy( + *, + name: str, + trigger: Cron, + emits: list[type[BaseEvent]], + applies_to: AppliesTo = ..., + gate: Counter | None = ..., + max_retries: int | None = ..., + enabled: bool = ..., +) -> Callable[[_CronStrategyFn], Strategy]: ... + + +@overload +def offline_strategy( + *, + name: str, + trigger: Idle, + emits: list[type[BaseEvent]], + applies_to: AppliesTo = ..., + gate: Counter | None = ..., + max_retries: int | None = ..., + enabled: bool = ..., +) -> Callable[[_IdleStrategyFn], Strategy]: ... + + +@overload +def offline_strategy( + *, + name: str, + trigger: Immediate, + emits: list[type[BaseEvent]], + applies_to: AppliesTo = ..., + gate: Counter | None = ..., + max_retries: int | None = ..., + enabled: bool = ..., +) -> Callable[[_EventStrategyFn[_E]], Strategy]: ... + + def offline_strategy( *, name: str, @@ -42,13 +111,30 @@ def offline_strategy( gate: Counter | None = None, max_retries: int | None = None, enabled: bool = True, -) -> Callable[[StrategyFn], StrategyFn]: - """Mark an async function as an OME strategy.""" +) -> Any: # overloads above provide call-site precision + """Mark an async function as an OME strategy. + + Args: + name: Unique strategy name (used for logging, run records, config). + trigger: When to fire — ``Cron``, ``Idle``, or ``Immediate``. + emits: Event types this strategy may emit via ``ctx.emit()``. + applies_to: Optional gate predicate (None = all events). + gate: Optional counter-based rate limiter. + max_retries: Override engine default; ``None`` uses engine config. + enabled: ``False`` disables without unregistering. + + Returns: + Decorator that wraps the function in a :class:`Strategy` instance. + + Raises: + ValueError: If ``name`` is empty or whitespace-only. + TypeError: If the decorated function is not async. + """ if not name or not name.strip(): raise ValueError("offline_strategy: name must be a non-empty string") - def wrap(func: StrategyFn) -> StrategyFn: + def wrap(func: StrategyFn) -> Strategy: if not inspect.iscoroutinefunction(func): raise TypeError( f"offline_strategy: {func.__name__} must be async (coroutine function)" @@ -63,7 +149,6 @@ def offline_strategy( enabled=enabled, func=func, ) - func._ome_strategy_meta = meta # type: ignore[attr-defined] - return func + return Strategy(meta) return wrap diff --git a/src/everos/infra/ome/engine.py b/src/everos/infra/ome/engine.py index e1dd403..ac9bf62 100644 --- a/src/everos/infra/ome/engine.py +++ b/src/everos/infra/ome/engine.py @@ -36,7 +36,7 @@ from everos.infra.ome._stores.idle import IdleStore from everos.infra.ome._stores.run_record import RunRecordStore from everos.infra.ome._stores.storage import OMEStorage from everos.infra.ome.config import OMEConfig -from everos.infra.ome.decorator import StrategyMeta +from everos.infra.ome.decorator import Strategy, StrategyMeta from everos.infra.ome.events import BaseEvent, CronTick, ManualTick, resolve_topic from everos.infra.ome.exceptions import ( EngineCallFromStrategyError, @@ -120,7 +120,12 @@ async def _cron_entry(engine_id: str, strategy_name: str) -> None: Looks the engine up by id and emits ``CronTick`` so the event flows back through the standard dispatch pipeline. + + Clears ``_CURRENT_STRATEGY`` because APScheduler may inherit the + context from a running strategy task, which would trip the + ``_guard_no_strategy_call`` check on ``engine.emit()``. """ + _CURRENT_STRATEGY.set(None) engine = _ENGINES.get(engine_id) if engine is None: logger.error( @@ -136,8 +141,10 @@ async def _idle_entry(engine_id: str, strategy_name: str) -> None: """Module-level APS jobstore callback for Idle IntervalTriggers. Looks the engine up by id and hands off to - :meth:`OfflineEngine.run_idle_scan`. + :meth:`OfflineEngine.run_idle_scan`. Clears ``_CURRENT_STRATEGY`` + for the same reason as :func:`_cron_entry`. """ + _CURRENT_STRATEGY.set(None) engine = _ENGINES.get(engine_id) if engine is None: logger.error( @@ -200,16 +207,22 @@ class OfflineEngine: self._active_runs = 0 self._idle_event: asyncio.Event | None = None - def register(self, func: Callable[..., Any]) -> None: - """Register a strategy decorated with :func:`offline_strategy`. + def register(self, strategy: Strategy) -> None: + """Register a :class:`Strategy` returned by ``@offline_strategy``. Must be called before :meth:`start`; registering after start raises :class:`OMEError` because the scheduler has already snapshotted the strategy set for Cron / Idle job creation. + + Args: + strategy: A Strategy instance produced by the decorator. + + Raises: + OMEError: If the engine has already started. """ if self._started: raise OMEError("register: cannot register after start()") - self._registry.register(func) + self._registry.register(strategy) @_refuse_inside_strategy def reschedule_cron_job(self, name: str, expr: str) -> None: @@ -301,6 +314,7 @@ class OfflineEngine: engine_sem=self._engine_sem, emit_hook=self._dispatch_event, on_dead_letter=self._on_dead_letter, + engine=self, ) self._idle_store = IdleStore(storage=self._storage) @@ -519,14 +533,15 @@ class OfflineEngine: def _acquire_lock(self) -> None: lock_path = Path(str(self._config.jobstore_path) + ".lock") lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = open(lock_path, "a+") # noqa: SIM115 try: - handle = open(lock_path, "a+") # noqa: SIM115 portalocker.lock(handle, portalocker.LOCK_EX | portalocker.LOCK_NB) - self._lock_handle = handle except portalocker.LockException as e: + handle.close() raise EngineLockHeldError( f"another OfflineEngine instance already holds {lock_path}" ) from e + self._lock_handle = handle def _release_lock(self) -> None: if self._lock_handle is not None: @@ -795,3 +810,75 @@ class OfflineEngine: if not self._started: raise OMEError("get_run_status: engine not started") return await self._run_record_store.get(run_id) + + async def list_runs_by_event_id(self, event_id: str) -> list[RunRecord]: + """Return all run records triggered by ``event_id``. + + Not guarded by ``_refuse_inside_strategy`` — designed to be + called from inside a strategy (e.g. Reflection waiting for + downstream extraction runs). + + Args: + event_id: The ``BaseEvent.event_id`` of the triggering event. + + Returns: + All ``RunRecord`` instances whose ``event_id`` matches, + newest first. + + Raises: + OMEError: Engine has not been started. + """ + if not self._started: + raise OMEError("list_runs_by_event_id: engine not started") + return await self._run_record_store.list_by_event_id(event_id) + + _TERMINAL_STATUSES = frozenset( + { + RunStatus.SUCCESS, + RunStatus.FAILED, + RunStatus.DEAD_LETTER, + RunStatus.CRASHED, + } + ) + + async def wait_for_event( + self, + event_id: str, + *, + timeout: float = 120.0, # noqa: ASYNC109 + poll_interval: float = 0.5, + ) -> list[RunRecord]: + """Poll until all runs triggered by ``event_id`` reach a terminal status. + + Not guarded by ``_refuse_inside_strategy`` — designed to be + called from inside a strategy (e.g. Reflection waiting for + downstream extraction runs to complete before deprecating). + + Args: + event_id: The ``BaseEvent.event_id`` of the triggering event. + timeout: Maximum seconds to wait before raising ``TimeoutError``. + poll_interval: Seconds between polls. + + Returns: + All ``RunRecord`` instances for ``event_id`` once every run + has reached a terminal status (SUCCESS / FAILED / DEAD_LETTER + / CRASHED). + + Raises: + TimeoutError: If any run is still non-terminal after ``timeout``. + OMEError: Engine has not been started. + """ + if not self._started: + raise OMEError("wait_for_event: engine not started") + deadline = asyncio.get_event_loop().time() + timeout + while True: + runs = await self._run_record_store.list_by_event_id(event_id) + if runs and all(r.status in self._TERMINAL_STATUSES for r in runs): + return runs + remaining = deadline - asyncio.get_event_loop().time() + if remaining <= 0: + raise TimeoutError( + f"wait_for_event: timed out after {timeout}s " + f"for event_id={event_id!r}" + ) + await asyncio.sleep(min(poll_interval, remaining)) diff --git a/src/everos/infra/ome/records.py b/src/everos/infra/ome/records.py index fc74c36..30dca69 100644 --- a/src/everos/infra/ome/records.py +++ b/src/everos/infra/ome/records.py @@ -56,6 +56,7 @@ class RunRecord(BaseModel): ), ] max_retries_snapshot: Annotated[int, Field(ge=0)] + event_id: str = "" @model_validator(mode="after") def _check_status_invariants(self) -> Self: diff --git a/src/everos/infra/ome/testing/fakes.py b/src/everos/infra/ome/testing/fakes.py index 5446bba..6347e66 100644 --- a/src/everos/infra/ome/testing/fakes.py +++ b/src/everos/infra/ome/testing/fakes.py @@ -8,23 +8,22 @@ from __future__ import annotations from everos.core.observability.logging import get_logger from everos.infra.ome.events import BaseEvent +from everos.infra.ome.records import RunRecord class FakeStrategyContext: """Implements StrategyContext Protocol; collects emit() calls in a list. + Args: + run_id: Run identifier, defaults to ``"fake_run"``. + Attributes: - run_id: Unique identifier for this run (default: "fake_run"). + run_id: Unique identifier for this run (default: ``"fake_run"``). logger: A structlog BoundLogger for test logging. emitted: List of BaseEvent objects passed to emit(). """ def __init__(self, *, run_id: str = "fake_run") -> None: - """Initialize a FakeStrategyContext. - - Args: - run_id: Run identifier, defaults to "fake_run". - """ self.run_id = run_id self.logger = get_logger("ome.fake_ctx") self.emitted: list[BaseEvent] = [] @@ -36,3 +35,31 @@ class FakeStrategyContext: event: The BaseEvent to emit. """ self.emitted.append(event) + + async def wait_for_event( + self, + event_id: str, + *, + timeout: float = 120.0, # noqa: ASYNC109 + ) -> list[RunRecord]: + """No-op stub; returns an empty list. + + Args: + event_id: The event identifier to wait for. + timeout: Maximum seconds to wait (unused in fake). + + Returns: + Empty list (no runs in test doubles). + """ + return [] + + async def list_runs_by_event_id(self, event_id: str) -> list[RunRecord]: + """No-op stub; returns an empty list. + + Args: + event_id: The event identifier to query. + + Returns: + Empty list (no runs in test doubles). + """ + return [] diff --git a/src/everos/infra/ome/testing/harness.py b/src/everos/infra/ome/testing/harness.py index da87804..32e61d0 100644 --- a/src/everos/infra/ome/testing/harness.py +++ b/src/everos/infra/ome/testing/harness.py @@ -12,6 +12,7 @@ from tempfile import mkdtemp from typing import Any from everos.infra.ome.config import OMEConfig +from everos.infra.ome.decorator import Strategy from everos.infra.ome.engine import OfflineEngine from everos.infra.ome.events import BaseEvent from everos.infra.ome.records import RunRecord, RunStatus @@ -55,13 +56,13 @@ class StrategyTestHarness: finally: shutil.rmtree(self._tmpdir, ignore_errors=True) # noqa: SLF001 - def register(self, func: Any) -> None: - """Register a strategy function. + def register(self, strategy: Strategy) -> None: + """Register a :class:`Strategy` returned by ``@offline_strategy``. Args: - func: A function decorated with @offline_strategy. + strategy: A Strategy instance produced by the decorator. """ - self._engine.register(func) + self._engine.register(strategy) async def start(self) -> None: """Start the OfflineEngine.""" diff --git a/src/everos/infra/persistence/lancedb/__init__.py b/src/everos/infra/persistence/lancedb/__init__.py index 04edb5b..01fe6e6 100644 --- a/src/everos/infra/persistence/lancedb/__init__.py +++ b/src/everos/infra/persistence/lancedb/__init__.py @@ -13,8 +13,10 @@ External usage:: from everos.infra.persistence.lancedb import ( get_connection, get_table, dispose_connection, Episode, AtomicFact, Foresight, AgentCase, AgentSkill, UserProfile, + KnowledgeTopic, episode_repo, atomic_fact_repo, foresight_repo, agent_case_repo, agent_skill_repo, user_profile_repo, + knowledge_topic_repo, ) Three index kinds: scalar / BM25 / vector. Tables are created lazily on @@ -33,12 +35,14 @@ from .repos import agent_skill_repo as agent_skill_repo from .repos import atomic_fact_repo as atomic_fact_repo from .repos import episode_repo as episode_repo from .repos import foresight_repo as foresight_repo +from .repos import knowledge_topic_repo as knowledge_topic_repo from .repos import user_profile_repo as user_profile_repo from .tables import AgentCase as AgentCase from .tables import AgentSkill as AgentSkill from .tables import AtomicFact as AtomicFact from .tables import Episode as Episode from .tables import Foresight as Foresight +from .tables import KnowledgeTopic as KnowledgeTopic from .tables import ParentType as ParentType from .tables import UserProfile as UserProfile @@ -49,6 +53,7 @@ _BUSINESS_SCHEMAS = ( AgentCase, AgentSkill, UserProfile, + KnowledgeTopic, ) @@ -115,6 +120,7 @@ __all__ = [ "AtomicFact", "Episode", "Foresight", + "KnowledgeTopic", "LanceDBSchemaMismatchError", "ParentType", "UserProfile", @@ -127,6 +133,7 @@ __all__ = [ "foresight_repo", "get_connection", "get_table", + "knowledge_topic_repo", "user_profile_repo", "verify_business_schemas", ] diff --git a/src/everos/infra/persistence/lancedb/repos/__init__.py b/src/everos/infra/persistence/lancedb/repos/__init__.py index ec01524..186f87b 100644 --- a/src/everos/infra/persistence/lancedb/repos/__init__.py +++ b/src/everos/infra/persistence/lancedb/repos/__init__.py @@ -15,6 +15,7 @@ External usage:: agent_case_repo, agent_skill_repo, user_profile_repo, + knowledge_topic_repo, ) await episode_repo.add([Episode(...)]) @@ -25,6 +26,7 @@ from .agent_skill import agent_skill_repo as agent_skill_repo from .atomic_fact import atomic_fact_repo as atomic_fact_repo from .episode import episode_repo as episode_repo from .foresight import foresight_repo as foresight_repo +from .knowledge_topic import knowledge_topic_repo as knowledge_topic_repo from .user_profile import user_profile_repo as user_profile_repo __all__ = [ @@ -33,5 +35,6 @@ __all__ = [ "atomic_fact_repo", "episode_repo", "foresight_repo", + "knowledge_topic_repo", "user_profile_repo", ] diff --git a/src/everos/infra/persistence/lancedb/repos/knowledge_topic.py b/src/everos/infra/persistence/lancedb/repos/knowledge_topic.py new file mode 100644 index 0000000..9ada8c4 --- /dev/null +++ b/src/everos/infra/persistence/lancedb/repos/knowledge_topic.py @@ -0,0 +1,22 @@ +"""LanceDB repo singleton for the ``knowledge_topic`` table.""" + +from __future__ import annotations + +from lancedb import AsyncTable + +from everos.core.persistence.lancedb import LanceRepoBase + +from ..lancedb_manager import get_table +from ..tables.knowledge_topic import KnowledgeTopic + + +class _KnowledgeTopicRepo(LanceRepoBase[KnowledgeTopic]): + """LanceDB repository for the ``knowledge_topic`` table.""" + + schema = KnowledgeTopic + + async def _table_lookup(self) -> AsyncTable: + return await get_table(self.schema.TABLE_NAME, self.schema) + + +knowledge_topic_repo = _KnowledgeTopicRepo() diff --git a/src/everos/infra/persistence/lancedb/tables/__init__.py b/src/everos/infra/persistence/lancedb/tables/__init__.py index db6a9f3..79bdb43 100644 --- a/src/everos/infra/persistence/lancedb/tables/__init__.py +++ b/src/everos/infra/persistence/lancedb/tables/__init__.py @@ -12,6 +12,7 @@ External usage:: AgentCase, AgentSkill, UserProfile, + KnowledgeTopic, ParentType, ) """ @@ -22,6 +23,7 @@ from .agent_skill import AgentSkill as AgentSkill from .atomic_fact import AtomicFact as AtomicFact from .episode import Episode as Episode from .foresight import Foresight as Foresight +from .knowledge_topic import KnowledgeTopic as KnowledgeTopic from .user_profile import UserProfile as UserProfile __all__ = [ @@ -30,6 +32,7 @@ __all__ = [ "AtomicFact", "Episode", "Foresight", + "KnowledgeTopic", "ParentType", "UserProfile", ] diff --git a/src/everos/infra/persistence/lancedb/tables/_parent_type.py b/src/everos/infra/persistence/lancedb/tables/_parent_type.py index 9112a96..49506ca 100644 --- a/src/everos/infra/persistence/lancedb/tables/_parent_type.py +++ b/src/everos/infra/persistence/lancedb/tables/_parent_type.py @@ -1,16 +1,19 @@ """``ParentType`` — provenance label for memory records linked back to a source. -Currently the only value is :attr:`ParentType.MEMCELL`: every business row -(episode / foresight / atomic_fact / agent_case) points back to a source -MemCell. The earlier opensource design enumerated ``"episode"`` as an -alternative parent but the production path never wrote that value, so the -new framework collapses the enum to its single in-use member. +Three values cover the current provenance graph: -Kept as an :class:`enum.Enum` (rather than a bare string constant) so that -adding a future parent kind stays a non-breaking enum extension. LanceDB's -pydantic-to-arrow conversion does not accept ``Enum`` field annotations, -so table schemas declare ``parent_type: str = ParentType.MEMCELL.value`` -and reference the enum only at the default-value level. +* :attr:`ParentType.MEMCELL` — the original ingestion unit; every + business row (episode / foresight / atomic_fact / agent_case) points + back to a source MemCell by default. +* :attr:`ParentType.EPISODE` — used by atomic facts that are extracted + from an episode rather than directly from a MemCell. +* :attr:`ParentType.CLUSTER` — used by merged episodes produced by the + Reflection consolidation mechanism. + +LanceDB's pydantic-to-arrow conversion does not accept ``Enum`` field +annotations, so table schemas declare +``parent_type: str = ParentType.MEMCELL.value`` and reference the enum +only at the default-value level. """ from __future__ import annotations @@ -22,3 +25,5 @@ class ParentType(StrEnum): """Provenance label of a memory record's parent.""" MEMCELL = "memcell" + EPISODE = "episode" + CLUSTER = "cluster" diff --git a/src/everos/infra/persistence/lancedb/tables/atomic_fact.py b/src/everos/infra/persistence/lancedb/tables/atomic_fact.py index 63f8448..488ff86 100644 --- a/src/everos/infra/persistence/lancedb/tables/atomic_fact.py +++ b/src/everos/infra/persistence/lancedb/tables/atomic_fact.py @@ -34,7 +34,7 @@ class AtomicFact(BaseLanceTable): app_id: str = "default" project_id: str = "default" """App / project scope (default ``"default"``); cascade fills from md path.""" - session_id: str + session_id: str | None = None timestamp: _dt.datetime parent_type: str = ParentType.MEMCELL.value @@ -59,4 +59,9 @@ class AtomicFact(BaseLanceTable): (owner_id / session_id / timestamp / parent_id / sender_ids) are NOT in the hash.""" + deprecated_by: str | None = None + """Soft-delete marker set by Reflection when this fact is + consolidated. Value is the cluster entry_id that supersedes this + row. ``NULL`` means the row is still active.""" + vector: Vector(_DIM) # type: ignore[valid-type] diff --git a/src/everos/infra/persistence/lancedb/tables/episode.py b/src/everos/infra/persistence/lancedb/tables/episode.py index b69e35d..d4e2f4e 100644 --- a/src/everos/infra/persistence/lancedb/tables/episode.py +++ b/src/everos/infra/persistence/lancedb/tables/episode.py @@ -36,7 +36,7 @@ class Episode(BaseLanceTable): app_id: str = "default" project_id: str = "default" """App / project scope (default ``"default"``); cascade fills from md path.""" - session_id: str + session_id: str | None = None timestamp: _dt.datetime parent_type: str = ParentType.MEMCELL.value @@ -75,4 +75,9 @@ class Episode(BaseLanceTable): NOT in the hash so editing them doesn't waste an embedding call. See ``16_cascade_impl_design.md`` §3.3.""" + deprecated_by: str | None = None + """Soft-delete marker set by Reflection when this episode is + consolidated into a cluster. Value is the cluster entry_id that + supersedes this row. ``NULL`` means the row is still active.""" + vector: Vector(_DIM) # type: ignore[valid-type] diff --git a/src/everos/infra/persistence/lancedb/tables/foresight.py b/src/everos/infra/persistence/lancedb/tables/foresight.py index b9783a2..8070d24 100644 --- a/src/everos/infra/persistence/lancedb/tables/foresight.py +++ b/src/everos/infra/persistence/lancedb/tables/foresight.py @@ -35,7 +35,7 @@ class Foresight(BaseLanceTable): app_id: str = "default" project_id: str = "default" """App / project scope (default ``"default"``); cascade fills from md path.""" - session_id: str + session_id: str | None = None timestamp: _dt.datetime """Foresight generation time.""" diff --git a/src/everos/infra/persistence/lancedb/tables/knowledge_topic.py b/src/everos/infra/persistence/lancedb/tables/knowledge_topic.py new file mode 100644 index 0000000..31e12ad --- /dev/null +++ b/src/everos/infra/persistence/lancedb/tables/knowledge_topic.py @@ -0,0 +1,36 @@ +"""LanceDB table schema for L2 knowledge topic nodes.""" + +from __future__ import annotations + +import datetime as dt +from typing import ClassVar + +from everos.core.persistence.lancedb import BaseLanceTable, Vector + +_DIM = 1024 + + +class KnowledgeTopic(BaseLanceTable): + """L2 topic node — dense + dual-column BM25 retrieval.""" + + TABLE_NAME: ClassVar[str] = "knowledge_topic" + BM25_FIELDS: ClassVar[list[str]] = ["summary_tokens", "content_tokens"] + + id: str + doc_id: str + category_id: str + app_id: str + project_id: str + topic_name: str + topic_path: str + depth: int + parent_node_id: str = "" + summary: str + summary_tokens: str + content_tokens: str + content_labels: list[str] = [] + md_path: str + content_sha256: str + vector: Vector(_DIM) # type: ignore[valid-type] -- Vector() is runtime-constructed; static analyzers cannot verify + created_at: dt.datetime + updated_at: dt.datetime diff --git a/src/everos/infra/persistence/markdown/__init__.py b/src/everos/infra/persistence/markdown/__init__.py index eb52d2b..af4f39d 100644 --- a/src/everos/infra/persistence/markdown/__init__.py +++ b/src/everos/infra/persistence/markdown/__init__.py @@ -33,6 +33,8 @@ from .mds import AgentSkillFrontmatter as AgentSkillFrontmatter from .mds import AtomicFactDailyFrontmatter as AtomicFactDailyFrontmatter from .mds import EpisodeDailyFrontmatter as EpisodeDailyFrontmatter from .mds import ForesightDailyFrontmatter as ForesightDailyFrontmatter +from .mds import KnowledgeDocumentFrontmatter as KnowledgeDocumentFrontmatter +from .mds import KnowledgeTopicFrontmatter as KnowledgeTopicFrontmatter from .mds import UserProfileFrontmatter as UserProfileFrontmatter from .readers import AgentCaseReader as AgentCaseReader from .readers import AgentSkillReader as AgentSkillReader @@ -41,12 +43,15 @@ from .readers import BaseDailyReader as BaseDailyReader from .readers import EpisodeReader as EpisodeReader from .readers import ForesightReader as ForesightReader from .readers import ProfileReader as ProfileReader +from .readers import ensure_taxonomy as ensure_taxonomy +from .readers import parse_taxonomy as parse_taxonomy from .writers import AgentCaseWriter as AgentCaseWriter from .writers import AgentSkillWriter as AgentSkillWriter from .writers import AtomicFactWriter as AtomicFactWriter from .writers import BaseDailyWriter as BaseDailyWriter from .writers import EpisodeWriter as EpisodeWriter from .writers import ForesightWriter as ForesightWriter +from .writers import KnowledgeWriter as KnowledgeWriter from .writers import ProfileWriter as ProfileWriter __all__ = [ @@ -67,7 +72,12 @@ __all__ = [ "ForesightDailyFrontmatter", "ForesightReader", "ForesightWriter", + "KnowledgeDocumentFrontmatter", + "KnowledgeTopicFrontmatter", + "KnowledgeWriter", "ProfileReader", "ProfileWriter", "UserProfileFrontmatter", + "ensure_taxonomy", + "parse_taxonomy", ] diff --git a/src/everos/infra/persistence/markdown/mds/__init__.py b/src/everos/infra/persistence/markdown/mds/__init__.py index b743676..978e3db 100644 --- a/src/everos/infra/persistence/markdown/mds/__init__.py +++ b/src/everos/infra/persistence/markdown/mds/__init__.py @@ -28,6 +28,10 @@ from .agent_skill import AgentSkillFrontmatter as AgentSkillFrontmatter from .atomic_fact import AtomicFactDailyFrontmatter as AtomicFactDailyFrontmatter from .episode import EpisodeDailyFrontmatter as EpisodeDailyFrontmatter from .foresight import ForesightDailyFrontmatter as ForesightDailyFrontmatter +from .knowledge_document import ( + KnowledgeDocumentFrontmatter as KnowledgeDocumentFrontmatter, +) +from .knowledge_topic import KnowledgeTopicFrontmatter as KnowledgeTopicFrontmatter from .profile import UserProfileFrontmatter as UserProfileFrontmatter __all__ = [ @@ -36,5 +40,7 @@ __all__ = [ "AtomicFactDailyFrontmatter", "EpisodeDailyFrontmatter", "ForesightDailyFrontmatter", + "KnowledgeDocumentFrontmatter", + "KnowledgeTopicFrontmatter", "UserProfileFrontmatter", ] diff --git a/src/everos/infra/persistence/markdown/mds/atomic_fact.py b/src/everos/infra/persistence/markdown/mds/atomic_fact.py index 4630b59..89d7c99 100644 --- a/src/everos/infra/persistence/markdown/mds/atomic_fact.py +++ b/src/everos/infra/persistence/markdown/mds/atomic_fact.py @@ -17,6 +17,8 @@ from __future__ import annotations import datetime as _dt from typing import ClassVar, Literal +from pydantic import Field + from everos.core.persistence.markdown import ( DailyLogPathMixin, UserScopedFrontmatter, @@ -36,3 +38,4 @@ class AtomicFactDailyFrontmatter(DailyLogPathMixin, UserScopedFrontmatter): entry_count: int = 0 created_at: _dt.datetime | None = None last_appended_at: _dt.datetime | None = None + deprecated_entries: dict[str, str] = Field(default_factory=dict) diff --git a/src/everos/infra/persistence/markdown/mds/episode.py b/src/everos/infra/persistence/markdown/mds/episode.py index 802de81..7bd715b 100644 --- a/src/everos/infra/persistence/markdown/mds/episode.py +++ b/src/everos/infra/persistence/markdown/mds/episode.py @@ -12,6 +12,8 @@ from __future__ import annotations import datetime as _dt from typing import ClassVar, Literal +from pydantic import Field + from everos.core.persistence.markdown import ( DailyLogPathMixin, UserScopedFrontmatter, @@ -31,3 +33,4 @@ class EpisodeDailyFrontmatter(DailyLogPathMixin, UserScopedFrontmatter): entry_count: int = 0 created_at: _dt.datetime | None = None last_appended_at: _dt.datetime | None = None + deprecated_entries: dict[str, str] = Field(default_factory=dict) diff --git a/src/everos/infra/persistence/markdown/mds/knowledge_document.py b/src/everos/infra/persistence/markdown/mds/knowledge_document.py new file mode 100644 index 0000000..7b85fbd --- /dev/null +++ b/src/everos/infra/persistence/markdown/mds/knowledge_document.py @@ -0,0 +1,26 @@ +"""Frontmatter schema for ``knowledge/{category}/{doc_title}/index.md``.""" + +from __future__ import annotations + +from typing import Literal + +from everos.core.persistence.markdown import ( + BaseFrontmatter, + KnowledgeDocumentPathMixin, + KnowledgeScopedMixin, +) + + +class KnowledgeDocumentFrontmatter( + KnowledgeDocumentPathMixin, + KnowledgeScopedMixin, + BaseFrontmatter, +): + """L1 document-level frontmatter (index.md). Body = doc summary.""" + + type: Literal["knowledge_document"] = "knowledge_document" + doc_id: str + category_id: str + title: str + source_name: str | None = None + source_type: str | None = None diff --git a/src/everos/infra/persistence/markdown/mds/knowledge_topic.py b/src/everos/infra/persistence/markdown/mds/knowledge_topic.py new file mode 100644 index 0000000..bf744a8 --- /dev/null +++ b/src/everos/infra/persistence/markdown/mds/knowledge_topic.py @@ -0,0 +1,32 @@ +"""Frontmatter schema for ``knowledge/{category}/{doc_title}/_.md``.""" + +from __future__ import annotations + +from typing import Literal + +from everos.core.persistence.markdown import ( + BaseFrontmatter, + KnowledgeScopedMixin, + KnowledgeTopicPathMixin, +) + + +class KnowledgeTopicFrontmatter( + KnowledgeTopicPathMixin, + KnowledgeScopedMixin, + BaseFrontmatter, +): + """L2 topic-node frontmatter. Body = topic content full text.""" + + type: Literal["knowledge_topic"] = "knowledge_topic" + node_id: str + doc_id: str + category_id: str + topic_index: int + topic_name: str + topic_path: str + summary: str + depth: int + parent_node_id: str | None = None + children_node_ids: list[str] = [] + content_labels: list[str] = [] diff --git a/src/everos/infra/persistence/markdown/readers/__init__.py b/src/everos/infra/persistence/markdown/readers/__init__.py index 2d70516..342bb50 100644 --- a/src/everos/infra/persistence/markdown/readers/__init__.py +++ b/src/everos/infra/persistence/markdown/readers/__init__.py @@ -37,6 +37,8 @@ from .base import BaseDailyReader as BaseDailyReader from .episode_reader import EpisodeReader as EpisodeReader from .foresight_reader import ForesightReader as ForesightReader from .profile_reader import ProfileReader as ProfileReader +from .taxonomy_reader import ensure_taxonomy as ensure_taxonomy +from .taxonomy_reader import parse_taxonomy as parse_taxonomy __all__ = [ "AgentCaseReader", @@ -46,4 +48,6 @@ __all__ = [ "EpisodeReader", "ForesightReader", "ProfileReader", + "ensure_taxonomy", + "parse_taxonomy", ] diff --git a/src/everos/infra/persistence/markdown/readers/taxonomy_reader.py b/src/everos/infra/persistence/markdown/readers/taxonomy_reader.py new file mode 100644 index 0000000..7847afd --- /dev/null +++ b/src/everos/infra/persistence/markdown/readers/taxonomy_reader.py @@ -0,0 +1,217 @@ +"""Read and auto-generate ``.taxonomy.md`` — the knowledge category taxonomy. + +The taxonomy file lives at ``knowledge_dir/.taxonomy.md``. It uses YAML +frontmatter with a ``categories`` list, each entry having ``id`` and +``description`` — matching ``everalgo.types.CategorySpec`` exactly. + +``ensure_taxonomy(knowledge_dir)`` creates the file from +``DEFAULT_TAXONOMY`` if it does not exist. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import anyio +import yaml +from everalgo.types import CategorySpec + +TAXONOMY_FILENAME = ".taxonomy.md" + + +DEFAULT_TAXONOMY: list[dict[str, str]] = [ + { + "id": "Technology", + "description": ( + "Computer science, software engineering, AI/ML, semiconductors, " + "cloud computing, cybersecurity, and general IT infrastructure." + ), + }, + { + "id": "Science", + "description": ( + "Natural sciences including physics, chemistry, biology, earth sciences, " + "astronomy, and interdisciplinary research." + ), + }, + { + "id": "Medical", + "description": ( + "Clinical medicine, disease diagnosis, drug specifications, " + "medical devices, public health, nursing, and healthcare administration." + ), + }, + { + "id": "Finance", + "description": ( + "Securities, banking, insurance, accounting, taxation, corporate finance, " + "macroeconomics, and fintech." + ), + }, + { + "id": "Legal", + "description": ( + "Laws, regulations, contracts, compliance, intellectual property, " + "litigation, and legal procedures." + ), + }, + { + "id": "Education", + "description": ( + "Teaching methods, curriculum design, academic research, " + "student assessment, e-learning, and educational policy." + ), + }, + { + "id": "Business", + "description": ( + "Corporate strategy, marketing, sales, operations management, " + "supply chain, HR, and entrepreneurship." + ), + }, + { + "id": "Engineering", + "description": ( + "Mechanical, civil, electrical, chemical, and industrial engineering — " + "design, manufacturing, and construction." + ), + }, + { + "id": "Arts", + "description": ( + "Visual arts, music, literature, film, theater, design, " + "and cultural studies." + ), + }, + { + "id": "Sports", + "description": ( + "Athletic training, sports events, fitness, sports science, " + "and sports industry management." + ), + }, + { + "id": "Travel", + "description": ( + "Tourism, hospitality, travel guides, destination reviews, " + "and transportation logistics." + ), + }, + { + "id": "Food", + "description": ( + "Culinary arts, nutrition science, food safety, restaurant management, " + "and food industry." + ), + }, + { + "id": "Environment", + "description": ( + "Climate change, ecology, pollution control, renewable energy, " + "conservation, and sustainability." + ), + }, + { + "id": "Politics", + "description": ( + "Government policy, international relations, elections, " + "public administration, and geopolitics." + ), + }, + { + "id": "History", + "description": ( + "Historical events, civilizations, archaeology, historical analysis, " + "and historiography." + ), + }, + { + "id": "Psychology", + "description": ( + "Cognitive science, behavioral psychology, clinical psychology, " + "mental health, and neuroscience." + ), + }, + { + "id": "Agriculture", + "description": ( + "Farming techniques, crop science, animal husbandry, agribusiness, " + "and food production systems." + ), + }, + { + "id": "RealEstate", + "description": ( + "Property development, real estate investment, urban planning, " + "architecture, and housing policy." + ), + }, + { + "id": "Media", + "description": ( + "Journalism, broadcasting, social media, public relations, " + "advertising, and communications." + ), + }, + { + "id": "Others", + "description": ( + "Documents that do not clearly fit any of the above categories." + ), + }, +] + + +async def parse_taxonomy(path: anyio.Path | Path) -> list[CategorySpec]: + """Parse ``.taxonomy.md`` and return the category list. + + Args: + path: Path to the ``.taxonomy.md`` file. + + Returns: + Parsed category list, or an empty list when the file does not + exist or has no categories. + """ + apath = anyio.Path(path) if not isinstance(path, anyio.Path) else path + if not await apath.exists(): + return [] + text = await apath.read_text(encoding="utf-8") + parts = text.split("---", 2) + if len(parts) < 3: + return [] + data: dict[str, Any] = yaml.safe_load(parts[1]) or {} + raw_categories = data.get("categories") or [] + return [ + CategorySpec(id=c["id"], description=c.get("description", "")) + for c in raw_categories + if isinstance(c, dict) and "id" in c + ] + + +async def ensure_taxonomy(knowledge_dir: anyio.Path | Path) -> Path: + """Create ``.taxonomy.md`` from defaults if it does not exist. + + Args: + knowledge_dir: Knowledge directory where ``.taxonomy.md`` lives. + + Returns: + Path to the taxonomy file (whether created or pre-existing). + """ + adir = ( + knowledge_dir + if isinstance(knowledge_dir, anyio.Path) + else anyio.Path(knowledge_dir) + ) + p = adir / TAXONOMY_FILENAME + if await p.exists(): + return Path(p) + await adir.mkdir(parents=True, exist_ok=True) + frontmatter = yaml.dump( + {"kind": "knowledge_taxonomy", "categories": DEFAULT_TAXONOMY}, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + ) + await p.write_text(f"---\n{frontmatter}---\n", encoding="utf-8") + return Path(p) diff --git a/src/everos/infra/persistence/markdown/writers/__init__.py b/src/everos/infra/persistence/markdown/writers/__init__.py index 323251a..377f7f6 100644 --- a/src/everos/infra/persistence/markdown/writers/__init__.py +++ b/src/everos/infra/persistence/markdown/writers/__init__.py @@ -30,6 +30,7 @@ from .atomic_fact_writer import AtomicFactWriter as AtomicFactWriter from .base import BaseDailyWriter as BaseDailyWriter from .episode_writer import EpisodeWriter as EpisodeWriter from .foresight_writer import ForesightWriter as ForesightWriter +from .knowledge_writer import KnowledgeWriter as KnowledgeWriter from .profile_writer import ProfileWriter as ProfileWriter __all__ = [ @@ -39,5 +40,6 @@ __all__ = [ "BaseDailyWriter", "EpisodeWriter", "ForesightWriter", + "KnowledgeWriter", "ProfileWriter", ] diff --git a/src/everos/infra/persistence/markdown/writers/base.py b/src/everos/infra/persistence/markdown/writers/base.py index a24f41d..dbdc5ca 100644 --- a/src/everos/infra/persistence/markdown/writers/base.py +++ b/src/everos/infra/persistence/markdown/writers/base.py @@ -234,6 +234,18 @@ class BaseDailyWriter: ) return eid + async def patch_frontmatter(self, path: Path, updates: Mapping[str, Any]) -> None: + """Merge ``updates`` into the frontmatter of an existing daily-log file. + + Delegates to the underlying :class:`MarkdownWriter` so that callers + do not need to reach through the private ``_writer`` attribute. + + Args: + path: Target markdown file (must exist). + updates: Mapping of frontmatter keys to merge. + """ + await self._writer.patch_frontmatter(path, updates) + # ── Hooks (subclass override) ───────────────────────────────────────── async def _current_count(self, path: Path) -> int: diff --git a/src/everos/infra/persistence/markdown/writers/knowledge_writer.py b/src/everos/infra/persistence/markdown/writers/knowledge_writer.py new file mode 100644 index 0000000..f6fd5a7 --- /dev/null +++ b/src/everos/infra/persistence/markdown/writers/knowledge_writer.py @@ -0,0 +1,246 @@ +"""KnowledgeWriter — write knowledge document + topic markdown files. + +Knowledge storage uses a **directory per document** layout:: + + knowledge/{category_id}/{title_dirname}/index.md ← document root + knowledge/{category_id}/{title_dirname}/1_topic_slug.md ← topic node + knowledge/{category_id}/{title_dirname}/2_topic_slug.md ← topic node + +Each call to :meth:`write` replaces the entire document directory +(delete-then-recreate) so that stale topic files from a prior extraction +do not linger. + +The writer is intentionally **static** — it takes a ``knowledge_dir`` +path rather than binding to :class:`MemoryRoot`. The service layer +resolves ``knowledge_dir`` from ``MemoryRoot.knowledge_dir(app, project)`` +and passes it in. This keeps the writer decoupled from the root-resolution +logic and easier to test. +""" + +from __future__ import annotations + +import re +import shutil +from pathlib import Path + +import anyio +import yaml +from everalgo.types import KnowledgeMemory + +from everos.core.observability.logging import get_logger + +logger = get_logger(__name__) + +_MAX_DIRNAME_LEN = 50 +_SAFE_CHARS = re.compile(r"[^\w\-.]", re.UNICODE) + + +# ── Writer ──────────────────────────────────────────────────────────────── + + +class KnowledgeWriter: + """Convert ``KnowledgeMemory`` list to markdown files.""" + + @staticmethod + async def write( + memories: list[KnowledgeMemory], + knowledge_dir: Path, + *, + source_name: str | None = None, + source_type: str | None = None, + ) -> Path: + """Write md files and return the document directory path. + + Args: + memories: Flat list of nodes produced by everalgo extraction. + Must contain exactly one root node (``topic_index=0``). + knowledge_dir: Base ``knowledge/`` directory (from + ``MemoryRoot.knowledge_dir``). + source_name: Optional provenance label (e.g. URL, filename). + source_type: Optional provenance type (e.g. ``"url"``, + ``"file"``). + + Returns: + Absolute path of the written document directory. + + Raises: + ValueError: If *memories* is empty or has no root node. + """ + root_node, topic_nodes = _split_root_and_topics(memories) + doc_dir = _resolve_doc_dir(knowledge_dir, root_node) + + # Overwrite: remove existing directory, then recreate. + await _remove_dir(doc_dir) + await anyio.Path(doc_dir).mkdir(parents=True, exist_ok=True) + + await _write_index(doc_dir, root_node, source_name, source_type) + for node in topic_nodes: + await _write_topic(doc_dir, node, root_node.doc_id) + + logger.info( + "knowledge document written", + doc_id=root_node.doc_id, + category_id=root_node.category_id or "Others", + topic_count=len(topic_nodes), + ) + return doc_dir + + +# ── Internals ───────────────────────────────────────────────────────────── + + +def _split_root_and_topics( + memories: list[KnowledgeMemory], +) -> tuple[KnowledgeMemory, list[KnowledgeMemory]]: + """Separate the root node from topic nodes. + + Raises: + ValueError: If *memories* is empty or contains no root node. + """ + if not memories: + raise ValueError("memories must not be empty") + + root: KnowledgeMemory | None = None + topics: list[KnowledgeMemory] = [] + for m in memories: + if m.topic_index == 0: + root = m + else: + topics.append(m) + + if root is None: + raise ValueError("memories must contain a root node (topic_index=0)") + return root, topics + + +def _sanitize_dirname(raw: str, fallback: str) -> str: + """Produce a safe directory/file name segment. + + * Replace spaces with underscores. + * Strip characters outside ``[a-zA-Z0-9_\\-.]``. + * Truncate to 50 characters. + * Fall back to *fallback* if the result is empty. + """ + slug = raw.replace(" ", "_") + slug = _SAFE_CHARS.sub("", slug) + slug = slug[:_MAX_DIRNAME_LEN] + return slug if slug else fallback + + +def _resolve_doc_dir(knowledge_dir: Path, root: KnowledgeMemory) -> Path: + """Build the document directory path from category, title, and doc_id.""" + category = _sanitize_dirname( + root.category_id if root.category_id else "Others", "Others" + ) + title_slug = _sanitize_dirname(root.topic, "doc") + dir_name = f"{title_slug}_{root.doc_id}" + return knowledge_dir / category / dir_name + + +def _build_index_frontmatter( + root: KnowledgeMemory, + source_name: str | None, + source_type: str | None, +) -> dict[str, object]: + """Build YAML frontmatter dict for the document index file.""" + category = root.category_id if root.category_id else "Others" + fm: dict[str, object] = { + "type": "knowledge_document", + "id": root.doc_id, + "doc_id": root.doc_id, + "category_id": category, + "title": root.topic, + "schema_version": 1, + } + if source_name is not None: + fm["source_name"] = source_name + if source_type is not None: + fm["source_type"] = source_type + return fm + + +def _build_topic_frontmatter( + node: KnowledgeMemory, + doc_id: str, +) -> dict[str, object]: + """Build YAML frontmatter dict for a single topic file.""" + category = node.category_id if node.category_id else "Others" + node_id = f"{doc_id}_{node.topic_index}" + + parent_node_id = None if node.depth <= 1 else f"{doc_id}_{node.parent_index}" + + children_node_ids = [f"{doc_id}_{ci}" for ci in node.children_index] + + return { + "type": "knowledge_topic", + "id": node_id, + "node_id": node_id, + "doc_id": doc_id, + "category_id": category, + "topic_index": node.topic_index, + "topic_name": node.topic, + "topic_path": node.topic_path, + "summary": node.summary, + "depth": node.depth, + "parent_node_id": parent_node_id, + "children_node_ids": children_node_ids, + "content_labels": node.content_labels, + "schema_version": 1, + } + + +def _dump_yaml_frontmatter(meta: dict[str, object]) -> str: + """Render a YAML frontmatter block with ``---`` delimiters.""" + yaml_block = yaml.safe_dump( + meta, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + ) + return f"---\n{yaml_block}---\n" + + +def _ensure_trailing_newline(text: str) -> str: + """Append a newline if *text* does not already end with one.""" + if not text: + return "" + return text if text.endswith("\n") else text + "\n" + + +async def _write_file(path: Path, content: str) -> None: + """Write content to *path* via anyio (async).""" + await anyio.Path(path.parent).mkdir(parents=True, exist_ok=True) + await anyio.Path(path).write_text(content, encoding="utf-8") + + +async def _write_index( + doc_dir: Path, + root: KnowledgeMemory, + source_name: str | None, + source_type: str | None, +) -> None: + """Write index.md with frontmatter and summary body.""" + fm = _build_index_frontmatter(root, source_name, source_type) + body = _ensure_trailing_newline(root.summary) + content = _dump_yaml_frontmatter(fm) + body + await _write_file(doc_dir / "index.md", content) + + +async def _write_topic( + doc_dir: Path, + node: KnowledgeMemory, + doc_id: str, +) -> None: + """Write a numbered topic md file with frontmatter and content body.""" + slug = _sanitize_dirname(node.topic, f"topic_{node.topic_index}") + filename = f"{node.topic_index}_{slug}.md" + fm = _build_topic_frontmatter(node, doc_id) + body = _ensure_trailing_newline(node.content) + content = _dump_yaml_frontmatter(fm) + body + await _write_file(doc_dir / filename, content) + + +async def _remove_dir(path: Path) -> None: + """Remove directory tree if it exists (sync shutil offloaded).""" + if await anyio.Path(path).is_dir(): + await anyio.to_thread.run_sync(shutil.rmtree, path) diff --git a/src/everos/infra/persistence/sqlite/__init__.py b/src/everos/infra/persistence/sqlite/__init__.py index ec2281f..d7896a2 100644 --- a/src/everos/infra/persistence/sqlite/__init__.py +++ b/src/everos/infra/persistence/sqlite/__init__.py @@ -17,7 +17,9 @@ External usage:: # ``infra.persistence.sqlite.**`` (sub-packages) are forbidden # to ``service`` / ``memory`` / ``entrypoints`` by import-linter. UnprocessedBuffer, Memcell, ConversationStatus, + KnowledgeDocumentRow, KnowledgeTopicRow, unprocessed_buffer_repo, memcell_repo, conversation_status_repo, + knowledge_document_repo, knowledge_topic_sqlite_repo, ) The :class:`SqliteLifespanProvider` runs ``SQLModel.metadata.create_all`` @@ -29,12 +31,18 @@ not need to manage either. # ``SQLModel.metadata`` so ``SqliteLifespanProvider.startup`` can # ``create_all`` without callers having to import each model module. from . import tables as tables # noqa: F401 +from .repos import DocumentListPage as DocumentListPage +from .repos import DocumentUpsertPayload as DocumentUpsertPayload from .repos import QueueSummary as QueueSummary +from .repos import TopicUpsertPayload as TopicUpsertPayload from .repos import cluster_repo as cluster_repo from .repos import conversation_status_repo as conversation_status_repo +from .repos import knowledge_document_repo as knowledge_document_repo +from .repos import knowledge_topic_sqlite_repo as knowledge_topic_sqlite_repo from .repos import md_change_state_repo as md_change_state_repo from .repos import memcell_repo as memcell_repo from .repos import mint_cluster_id as mint_cluster_id +from .repos import reflection_report_repo as reflection_report_repo from .repos import unprocessed_buffer_repo as unprocessed_buffer_repo from .sqlite_manager import dispose_engine as dispose_engine from .sqlite_manager import get_engine as get_engine @@ -42,25 +50,37 @@ from .sqlite_manager import get_session_factory as get_session_factory from .tables import Cluster as Cluster from .tables import ClusterMember as ClusterMember from .tables import ConversationStatus as ConversationStatus +from .tables import KnowledgeDocumentRow as KnowledgeDocumentRow +from .tables import KnowledgeTopicRow as KnowledgeTopicRow from .tables import MdChangeState as MdChangeState from .tables import Memcell as Memcell +from .tables import ReflectionReport as ReflectionReport from .tables import UnprocessedBuffer as UnprocessedBuffer __all__ = [ "Cluster", "ClusterMember", "ConversationStatus", + "DocumentListPage", + "DocumentUpsertPayload", + "KnowledgeDocumentRow", + "KnowledgeTopicRow", "MdChangeState", "Memcell", "QueueSummary", + "ReflectionReport", + "TopicUpsertPayload", "UnprocessedBuffer", "cluster_repo", "conversation_status_repo", "dispose_engine", "get_engine", "get_session_factory", + "knowledge_document_repo", + "knowledge_topic_sqlite_repo", "md_change_state_repo", "memcell_repo", "mint_cluster_id", + "reflection_report_repo", "unprocessed_buffer_repo", ] diff --git a/src/everos/infra/persistence/sqlite/repos/__init__.py b/src/everos/infra/persistence/sqlite/repos/__init__.py index c11de55..c4f485b 100644 --- a/src/everos/infra/persistence/sqlite/repos/__init__.py +++ b/src/everos/infra/persistence/sqlite/repos/__init__.py @@ -7,17 +7,29 @@ engine singleton. from .cluster import cluster_repo as cluster_repo from .cluster import mint_cluster_id as mint_cluster_id from .conversation_status import conversation_status_repo as conversation_status_repo +from .knowledge import DocumentListPage as DocumentListPage +from .knowledge import DocumentUpsertPayload as DocumentUpsertPayload +from .knowledge import TopicUpsertPayload as TopicUpsertPayload +from .knowledge import knowledge_document_repo as knowledge_document_repo +from .knowledge import knowledge_topic_sqlite_repo as knowledge_topic_sqlite_repo from .md_change_state import QueueSummary as QueueSummary from .md_change_state import md_change_state_repo as md_change_state_repo from .memcell import memcell_repo as memcell_repo +from .reflection_report import reflection_report_repo as reflection_report_repo from .unprocessed_buffer import unprocessed_buffer_repo as unprocessed_buffer_repo __all__ = [ + "DocumentListPage", + "DocumentUpsertPayload", "QueueSummary", + "TopicUpsertPayload", "cluster_repo", "conversation_status_repo", + "knowledge_document_repo", + "knowledge_topic_sqlite_repo", "md_change_state_repo", "memcell_repo", "mint_cluster_id", + "reflection_report_repo", "unprocessed_buffer_repo", ] diff --git a/src/everos/infra/persistence/sqlite/repos/cluster.py b/src/everos/infra/persistence/sqlite/repos/cluster.py index 35987b8..be367d0 100644 --- a/src/everos/infra/persistence/sqlite/repos/cluster.py +++ b/src/everos/infra/persistence/sqlite/repos/cluster.py @@ -23,7 +23,7 @@ import uuid import numpy as np from everalgo.clustering import Cluster as AlgoCluster -from sqlalchemy import select +from sqlalchemy import delete, func, select, update from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -42,6 +42,13 @@ def mint_cluster_id() -> str: class _ClusterRepo(RepoBase[Cluster]): + """CRUD repository for the ``cluster`` + ``cluster_member`` table pair. + + Bridges between the SQLite row shape and the algo-side + :class:`everalgo.clustering.Cluster` value object, handling centroid + bytes round-trip, preview JSON serialisation, and membership joins. + """ + model = Cluster def _factory_lookup(self) -> async_sessionmaker[AsyncSession]: @@ -121,6 +128,144 @@ class _ClusterRepo(RepoBase[Cluster]): ) return (await s.execute(stmt)).scalar_one_or_none() + # ── Member-level CRUD ─────────────────────────────────────────────────── + + async def remove_members(self, cluster_id: str, member_ids: set[str]) -> None: + """Hard-delete specific member rows from cluster_member. + + Args: + cluster_id: Target cluster identifier. + member_ids: Set of member ids to remove; no-op when empty. + """ + if not member_ids: + return + async with session_scope(self._factory) as s: + await s.execute( + delete(ClusterMember) + .where(ClusterMember.cluster_id == cluster_id) + .where(ClusterMember.member_id.in_(member_ids)) + ) + await s.commit() + + async def add_member( + self, cluster_id: str, member_id: str, member_type: str + ) -> None: + """Add a single member to an existing cluster. + + Args: + cluster_id: Target cluster identifier. + member_id: Unique id of the member entity. + member_type: Kind discriminator (e.g. ``"episode"``). + """ + async with session_scope(self._factory) as s: + s.add( + ClusterMember( + cluster_id=cluster_id, + member_id=member_id, + member_type=member_type, + added_ts=get_utc_now(), + ) + ) + await s.commit() + + async def update_metadata( + self, + cluster_id: str, + *, + centroid_blob: bytes, + count: int, + last_ts_ms: int, + preview_json: str, + ) -> None: + """Update cluster metadata after member changes. + + Args: + cluster_id: Target cluster identifier. + centroid_blob: Serialised float32 centroid vector bytes. + count: Updated member count. + last_ts_ms: Latest member timestamp in epoch milliseconds. + preview_json: JSON-encoded preview text list. + """ + async with session_scope(self._factory) as s: + await s.execute( + update(Cluster) + .where(Cluster.cluster_id == cluster_id) + .values( + centroid_blob=centroid_blob, + count=count, + last_ts_ms=last_ts_ms, + preview_json=preview_json, + ) + ) + await s.commit() + + # ── Lightweight queries ─────────────────────────────────────────────── + + async def list_ids_and_member_counts( + self, + owner_id: str, + kind: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[tuple[str, int]]: + """Return ``(cluster_id, member_count)`` from actual member rows. + + Args: + owner_id: Scope owner identifier. + kind: Cluster kind discriminator. + app_id: Application scope (default ``"default"``). + project_id: Project scope (default ``"default"``). + """ + async with session_scope(self._factory) as s: + stmt = ( + select( + Cluster.cluster_id, + func.count(ClusterMember.member_id), + ) + .join( + ClusterMember, + Cluster.cluster_id == ClusterMember.cluster_id, + ) + .where(Cluster.owner_id == owner_id) + .where(Cluster.kind == kind) + .where(Cluster.app_id == app_id) + .where(Cluster.project_id == project_id) + .group_by(Cluster.cluster_id) + ) + return list((await s.execute(stmt)).all()) + + async def get_members_with_type(self, cluster_id: str) -> list[tuple[str, str]]: + """Return ``(member_id, member_type)`` pairs for a cluster. + + Args: + cluster_id: Target cluster identifier. + """ + async with session_scope(self._factory) as s: + stmt = ( + select(ClusterMember.member_id, ClusterMember.member_type) + .where(ClusterMember.cluster_id == cluster_id) + .order_by(ClusterMember.added_ts) + ) + return list((await s.execute(stmt)).all()) + + async def list_distinct_owners( + self, + ) -> list[tuple[str, str, str, str]]: + """Return distinct ``(owner_id, owner_type, app_id, project_id)`` tuples. + + Used by the Reflection cron strategy to enumerate all scope + combinations that may have clusters to reflect. + """ + async with session_scope(self._factory) as s: + stmt = select( + Cluster.owner_id, + Cluster.owner_type, + Cluster.app_id, + Cluster.project_id, + ).distinct() + return list((await s.execute(stmt)).all()) + # ── Write ────────────────────────────────────────────────────────────── async def upsert_with_members( diff --git a/src/everos/infra/persistence/sqlite/repos/knowledge.py b/src/everos/infra/persistence/sqlite/repos/knowledge.py new file mode 100644 index 0000000..26cfc7d --- /dev/null +++ b/src/everos/infra/persistence/sqlite/repos/knowledge.py @@ -0,0 +1,342 @@ +"""Repositories for ``knowledge_documents`` and ``knowledge_topics`` tables. + +Two singleton repos — one per table — wired to the process-wide SQLite engine. +Callers construct rows and pass them in; these repos handle persistence only. +""" + +from __future__ import annotations + +import dataclasses + +from sqlalchemy import asc, delete, desc, func, select +from sqlalchemy.dialects.sqlite import insert +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from everos.component.utils.datetime import get_utc_now +from everos.core.persistence.sqlite import RepoBase, session_scope + +from ..sqlite_manager import get_session_factory +from ..tables.knowledge import KnowledgeDocumentRow, KnowledgeTopicRow + +_SORTABLE_COLUMNS: dict[str, object] = { + "created_at": KnowledgeDocumentRow.created_at, + "updated_at": KnowledgeDocumentRow.updated_at, + "title": KnowledgeDocumentRow.title, +} + + +@dataclasses.dataclass(frozen=True) +class DocumentListPage: + """Result of a paginated document list query.""" + + rows: list[KnowledgeDocumentRow] + total: int + + +@dataclasses.dataclass(frozen=True) +class DocumentUpsertPayload: + """Payload for document cascade upsert.""" + + doc_id: str + app_id: str + project_id: str + category_id: str + title: str + summary: str + source_name: str | None + source_type: str | None + md_path: str + + +@dataclasses.dataclass(frozen=True) +class TopicUpsertPayload: + """Payload for topic cascade upsert.""" + + node_id: str + doc_id: str + app_id: str + project_id: str + category_id: str + topic_index: int + topic_name: str + topic_path: str + depth: int + parent_node_id: str | None + children_node_ids: str | None + summary: str + content: str + content_labels: str | None + md_path: str + + +def _apply_document_fields( + row: KnowledgeDocumentRow, payload: DocumentUpsertPayload +) -> None: + """Copy payload fields onto an existing SQLAlchemy row.""" + row.app_id = payload.app_id + row.project_id = payload.project_id + row.category_id = payload.category_id + row.title = payload.title + row.summary = payload.summary + row.source_name = payload.source_name + row.source_type = payload.source_type + row.md_path = payload.md_path + + +class _KnowledgeDocumentRepo(RepoBase[KnowledgeDocumentRow]): + """SQLite repository for ``knowledge_documents`` table.""" + + model = KnowledgeDocumentRow + + def _factory_lookup(self) -> async_sessionmaker[AsyncSession]: + return get_session_factory() + + async def get_by_doc_id(self, doc_id: str) -> KnowledgeDocumentRow | None: + """Return the document row for ``doc_id``, or ``None`` if absent.""" + async with session_scope(self._factory) as s: + return await s.get(KnowledgeDocumentRow, doc_id) + + async def get_documents_by_ids( + self, doc_ids: set[str] + ) -> list[KnowledgeDocumentRow]: + """Batch-fetch document rows by primary key set.""" + if not doc_ids: + return [] + async with session_scope(self._factory) as s: + stmt = select(KnowledgeDocumentRow).where( + KnowledgeDocumentRow.doc_id.in_(doc_ids) + ) + return list((await s.execute(stmt)).scalars().all()) + + async def delete_by_md_path(self, md_path: str) -> int: + """Delete all document rows for ``md_path``; return the deleted count.""" + async with session_scope(self._factory) as s: + result = await s.execute( + delete(KnowledgeDocumentRow).where( + KnowledgeDocumentRow.md_path == md_path + ) + ) + await s.commit() + return int(result.rowcount or 0) + + async def doc_id_exists(self, doc_id: str) -> bool: + """Return ``True`` if a row with ``doc_id`` exists.""" + async with session_scope(self._factory) as s: + row = await s.get(KnowledgeDocumentRow, doc_id) + return row is not None + + async def list_documents( + self, + *, + app_id: str, + project_id: str, + category_id: str | None, + page: int, + page_size: int, + sort_by: str, + sort_order: str, + ) -> DocumentListPage: + """Return a paginated, optionally filtered slice of documents. + + Args: + app_id: Tenant application identifier. + project_id: Tenant project identifier. + category_id: When provided, restricts results to this category. + page: 1-based page number. + page_size: Maximum rows per page. + sort_by: Column name — one of ``created_at``, ``updated_at``, ``title``. + sort_order: ``"asc"`` or ``"desc"``. + + Returns: + DocumentListPage with matched rows and total count. + """ + col = _SORTABLE_COLUMNS.get(sort_by, KnowledgeDocumentRow.created_at) + order_fn = asc if sort_order.lower() == "asc" else desc + offset = (page - 1) * page_size + + base_filter = [ + KnowledgeDocumentRow.app_id == app_id, + KnowledgeDocumentRow.project_id == project_id, + ] + if category_id is not None: + base_filter.append(KnowledgeDocumentRow.category_id == category_id) + + async with session_scope(self._factory) as s: + count_stmt = ( + select(func.count()) + .select_from(KnowledgeDocumentRow) + .where(*base_filter) + ) + total = (await s.execute(count_stmt)).scalar_one() + + rows_stmt = ( + select(KnowledgeDocumentRow) + .where(*base_filter) + .order_by(order_fn(col)) # type: ignore[arg-type] -- col is SA column via dict lookup; static type is object + .offset(offset) + .limit(page_size) + ) + rows = list((await s.execute(rows_stmt)).scalars().all()) + + return DocumentListPage(rows=rows, total=int(total)) + + async def count_by_category(self, app_id: str, project_id: str) -> dict[str, int]: + """Return ``{category_id: document_count}`` for all categories with docs.""" + async with session_scope(self._factory) as s: + stmt = ( + select( + KnowledgeDocumentRow.category_id, + func.count().label("cnt"), + ) + .where( + KnowledgeDocumentRow.app_id == app_id, + KnowledgeDocumentRow.project_id == project_id, + ) + .group_by(KnowledgeDocumentRow.category_id) + ) + rows = (await s.execute(stmt)).all() + return {r.category_id: r.cnt for r in rows} + + async def upsert_from_handler(self, payload: DocumentUpsertPayload) -> None: + """Insert or update a document row from the cascade handler. + + Use SQLite's atomic ``ON CONFLICT`` form rather than a + load-mutate-commit cycle. Category moves can produce concurrent + cascade events for the old and new ``index.md`` paths; an ORM row + loaded before a delete can otherwise raise ``StaleDataError`` on + flush. + """ + async with session_scope(self._factory) as s: + now = get_utc_now() + values = { + **dataclasses.asdict(payload), + "created_at": now, + "updated_at": now, + } + stmt = insert(KnowledgeDocumentRow).values(**values) + update_fields = { + key: values[key] + for key in ( + "app_id", + "project_id", + "category_id", + "title", + "summary", + "source_name", + "source_type", + "md_path", + "updated_at", + ) + } + await s.execute( + stmt.on_conflict_do_update( + index_elements=["doc_id"], + set_=update_fields, + ) + ) + await s.commit() + + +def _apply_topic_fields(row: KnowledgeTopicRow, payload: TopicUpsertPayload) -> None: + """Copy payload fields onto an existing SQLAlchemy row.""" + row.doc_id = payload.doc_id + row.app_id = payload.app_id + row.project_id = payload.project_id + row.category_id = payload.category_id + row.topic_index = payload.topic_index + row.topic_name = payload.topic_name + row.topic_path = payload.topic_path + row.depth = payload.depth + row.parent_node_id = payload.parent_node_id + row.children_node_ids = payload.children_node_ids + row.summary = payload.summary + row.content = payload.content + row.content_labels = payload.content_labels + row.md_path = payload.md_path + + +class _KnowledgeTopicRepo(RepoBase[KnowledgeTopicRow]): + """SQLite repository for ``knowledge_topics`` table.""" + + model = KnowledgeTopicRow + + def _factory_lookup(self) -> async_sessionmaker[AsyncSession]: + return get_session_factory() + + async def get_topics_by_ids(self, node_ids: list[str]) -> list[KnowledgeTopicRow]: + """Batch-fetch topic rows by node id list — preserves caller order.""" + if not node_ids: + return [] + async with session_scope(self._factory) as s: + stmt = select(KnowledgeTopicRow).where( + KnowledgeTopicRow.node_id.in_(node_ids) + ) + rows = list((await s.execute(stmt)).scalars().all()) + by_id = {r.node_id: r for r in rows} + return [by_id[nid] for nid in node_ids if nid in by_id] + + async def get_topics_by_doc_id(self, doc_id: str) -> list[KnowledgeTopicRow]: + """Return all topic rows for ``doc_id``, ordered by ``topic_index``.""" + async with session_scope(self._factory) as s: + stmt = ( + select(KnowledgeTopicRow) + .where(KnowledgeTopicRow.doc_id == doc_id) + .order_by(KnowledgeTopicRow.topic_index) + ) + return list((await s.execute(stmt)).scalars().all()) + + async def count_by_doc_id(self, doc_id: str) -> int: + """Return the number of topic rows for ``doc_id``.""" + async with session_scope(self._factory) as s: + stmt = ( + select(func.count()) + .select_from(KnowledgeTopicRow) + .where(KnowledgeTopicRow.doc_id == doc_id) + ) + return int((await s.execute(stmt)).scalar_one()) + + async def delete_by_md_path(self, md_path: str) -> int: + """Delete all topic rows for ``md_path``; return the deleted count.""" + async with session_scope(self._factory) as s: + result = await s.execute( + delete(KnowledgeTopicRow).where(KnowledgeTopicRow.md_path == md_path) + ) + await s.commit() + return int(result.rowcount or 0) + + async def delete_by_doc_id(self, doc_id: str) -> int: + """Delete all topic rows for ``doc_id``; return the deleted count.""" + async with session_scope(self._factory) as s: + result = await s.execute( + delete(KnowledgeTopicRow).where(KnowledgeTopicRow.doc_id == doc_id) + ) + await s.commit() + return int(result.rowcount or 0) + + async def upsert_from_handler(self, payload: TopicUpsertPayload) -> None: + """Insert or update a topic row from the cascade handler. + + Checks by ``node_id`` (PK). If the row exists, all mutable + columns are overwritten and ``updated_at`` bumps. If not, a + fresh row is inserted with both ``created_at`` and + ``updated_at`` set to now. + """ + async with session_scope(self._factory) as s: + existing = await s.get(KnowledgeTopicRow, payload.node_id) + now = get_utc_now() + if existing is not None: + _apply_topic_fields(existing, payload) + existing.updated_at = now + s.add(existing) + else: + row = KnowledgeTopicRow( + **dataclasses.asdict(payload), + created_at=now, + updated_at=now, + ) + s.add(row) + await s.commit() + + +knowledge_document_repo = _KnowledgeDocumentRepo() +knowledge_topic_sqlite_repo = _KnowledgeTopicRepo() diff --git a/src/everos/infra/persistence/sqlite/repos/reflection_report.py b/src/everos/infra/persistence/sqlite/repos/reflection_report.py new file mode 100644 index 0000000..3ac233b --- /dev/null +++ b/src/everos/infra/persistence/sqlite/repos/reflection_report.py @@ -0,0 +1,78 @@ +"""Repository for the ``reflection_report`` table.""" + +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from everos.core.persistence.sqlite import RepoBase, session_scope + +from ..sqlite_manager import get_session_factory +from ..tables import ReflectionReport + + +class _ReflectionReportRepo(RepoBase[ReflectionReport]): + """CRUD repository for the ``reflection_report`` audit table. + + Provides creation, latest-by-cluster lookup, and reflected-cluster + enumeration used by the Reflection orchestrator and cron strategy. + """ + + model = ReflectionReport + + def _factory_lookup(self) -> async_sessionmaker[AsyncSession]: + return get_session_factory() + + async def create(self, report: ReflectionReport) -> None: + """Persist a new reflection report row. + + Args: + report: Fully populated ReflectionReport instance. + """ + async with session_scope(self._factory) as s: + s.add(report) + await s.commit() + + async def get_latest_for_cluster(self, cluster_id: str) -> ReflectionReport | None: + """Most recent completed report for a cluster, or ``None``. + + Args: + cluster_id: Cluster identifier to look up. + """ + async with session_scope(self._factory) as s: + stmt = ( + select(ReflectionReport) + .where(ReflectionReport.cluster_id == cluster_id) + .where(ReflectionReport.status == "completed") + .order_by(ReflectionReport.created_at.desc()) + .limit(1) + ) + return (await s.execute(stmt)).scalar_one_or_none() + + async def list_reflected_cluster_ids( + self, + owner_id: str, + app_id: str = "default", + project_id: str = "default", + ) -> set[str]: + """Distinct cluster ids that have at least one completed report. + + Args: + owner_id: Scope owner identifier. + app_id: Application scope (default ``"default"``). + project_id: Project scope (default ``"default"``). + """ + async with session_scope(self._factory) as s: + stmt = ( + select(ReflectionReport.cluster_id) + .where(ReflectionReport.owner_id == owner_id) + .where(ReflectionReport.app_id == app_id) + .where(ReflectionReport.project_id == project_id) + .where(ReflectionReport.status == "completed") + .distinct() + ) + rows = (await s.execute(stmt)).scalars().all() + return set(rows) + + +reflection_report_repo = _ReflectionReportRepo() diff --git a/src/everos/infra/persistence/sqlite/tables/__init__.py b/src/everos/infra/persistence/sqlite/tables/__init__.py index c4c734e..39d7bd3 100644 --- a/src/everos/infra/persistence/sqlite/tables/__init__.py +++ b/src/everos/infra/persistence/sqlite/tables/__init__.py @@ -10,15 +10,21 @@ every registered table. from .cluster import Cluster as Cluster from .cluster import ClusterMember as ClusterMember from .conversation_status import ConversationStatus as ConversationStatus +from .knowledge import KnowledgeDocumentRow as KnowledgeDocumentRow +from .knowledge import KnowledgeTopicRow as KnowledgeTopicRow from .md_change_state import MdChangeState as MdChangeState from .memcell import Memcell as Memcell +from .reflection_report import ReflectionReport as ReflectionReport from .unprocessed_buffer import UnprocessedBuffer as UnprocessedBuffer __all__ = [ "Cluster", "ClusterMember", "ConversationStatus", + "KnowledgeDocumentRow", + "KnowledgeTopicRow", "MdChangeState", "Memcell", + "ReflectionReport", "UnprocessedBuffer", ] diff --git a/src/everos/infra/persistence/sqlite/tables/cluster.py b/src/everos/infra/persistence/sqlite/tables/cluster.py index d56e73e..9e340a0 100644 --- a/src/everos/infra/persistence/sqlite/tables/cluster.py +++ b/src/everos/infra/persistence/sqlite/tables/cluster.py @@ -88,12 +88,12 @@ class ClusterMember(BaseTable, table=True): """Parent cluster id.""" member_id: str = Field(primary_key=True) - """``memcell_id`` (member_type=``memcell``) or md entry_id - (member_type=``case``) — the entity grouped into this cluster.""" + """Opaque entity id grouped into this cluster. Semantics depend on + ``member_type`` (e.g. episode entry_id, case entry_id).""" member_type: str - """``"memcell"`` or ``"case"``. Echoes the parent cluster's ``kind`` - domain but kept on the row so the reverse index is self-contained.""" + """Caller-defined entity kind (e.g. ``"episode"``, ``"case"``). Kept + on the row so the reverse index is self-contained.""" added_ts: UtcDatetime = Field(sa_type=UtcDateTimeColumn) """When this entity was first attached to the cluster.""" diff --git a/src/everos/infra/persistence/sqlite/tables/knowledge.py b/src/everos/infra/persistence/sqlite/tables/knowledge.py new file mode 100644 index 0000000..2a0cb27 --- /dev/null +++ b/src/everos/infra/persistence/sqlite/tables/knowledge.py @@ -0,0 +1,69 @@ +"""``knowledge_documents`` + ``knowledge_topics`` — L1/L2 knowledge metadata. + +``KnowledgeDocumentRow`` holds per-document metadata (category, title, +summary, source, md path). ``KnowledgeTopicRow`` holds per-topic content +and tree structure (parent / children, depth, path) derived from the parsed +document outline. + +Both tables use ``BaseTable`` so they inherit ``created_at`` / ``updated_at`` +with automatic UTC enforcement and ``onupdate`` refresh. +""" + +from __future__ import annotations + +from sqlalchemy import Index + +from everos.core.persistence.sqlite import BaseTable, Field + + +class KnowledgeDocumentRow(BaseTable, table=True): + """One row per knowledge document. PK ``doc_id``.""" + + __tablename__ = "knowledge_documents" # type: ignore[assignment] -- SQLModel tablename typing limitation + __table_args__ = ( + Index( + "ix_knowledge_documents_category", + "app_id", + "project_id", + "category_id", + ), + Index("ix_knowledge_documents_md_path", "md_path"), + ) + + doc_id: str = Field(primary_key=True) + app_id: str = Field(default="default") + project_id: str = Field(default="default") + category_id: str + title: str + summary: str + source_name: str | None = Field(default=None) + source_type: str | None = Field(default=None) + md_path: str + + +class KnowledgeTopicRow(BaseTable, table=True): + """One row per topic node within a knowledge document. PK ``node_id``.""" + + __tablename__ = "knowledge_topics" # type: ignore[assignment] -- SQLModel tablename typing limitation + __table_args__ = ( + Index("ix_knowledge_topics_doc", "doc_id"), + Index("ix_knowledge_topics_app_proj", "app_id", "project_id"), + ) + + node_id: str = Field(primary_key=True) + doc_id: str + app_id: str = Field(default="default") + project_id: str = Field(default="default") + category_id: str + topic_index: int + topic_name: str + topic_path: str + depth: int + parent_node_id: str | None = Field(default=None) + children_node_ids: str | None = Field(default=None) + """JSON-encoded list[str] of child node ids.""" + summary: str + content: str + content_labels: str | None = Field(default=None) + """JSON-encoded list[str] of content labels / tags.""" + md_path: str diff --git a/src/everos/infra/persistence/sqlite/tables/reflection_report.py b/src/everos/infra/persistence/sqlite/tables/reflection_report.py new file mode 100644 index 0000000..c238bda --- /dev/null +++ b/src/everos/infra/persistence/sqlite/tables/reflection_report.py @@ -0,0 +1,42 @@ +"""ReflectionReport — audit record for each Reflection operation.""" + +from __future__ import annotations + +from sqlmodel import Field + +from everos.component.utils.datetime import UtcDatetime, get_utc_now +from everos.core.persistence.sqlite import BaseTable + + +class ReflectionReport(BaseTable, table=True): + """One row per completed Reflection merge operation. + + Attributes: + id: Primary key for this report. + cluster_id: Cluster that was reflected. + owner_id: Scope owner (user or agent id). + app_id: Application scope, default ``"default"``. + project_id: Project scope, default ``"default"``. + mode: Reflection strategy mode (``"init"`` or ``"update"``). + source_members: JSON-encoded list of member ids consumed. + source_count: Number of source members consumed. + merged_entry_id: Entry id of the merged output written to storage. + deprecated_fact_count: Number of facts deprecated during merge. + status: Completion status, default ``"completed"``. + created_at: UTC timestamp when the report was created. + """ + + __tablename__ = "reflection_report" + + id: str = Field(primary_key=True) + cluster_id: str = Field(index=True) + owner_id: str = Field(index=True) + app_id: str = Field(default="default") + project_id: str = Field(default="default") + mode: str + source_members: str + source_count: int + merged_entry_id: str + deprecated_fact_count: int + status: str = Field(default="completed") + created_at: UtcDatetime = Field(default_factory=get_utc_now) diff --git a/src/everos/memory/_partition_locks.py b/src/everos/memory/_partition_locks.py new file mode 100644 index 0000000..a4e3b8d --- /dev/null +++ b/src/everos/memory/_partition_locks.py @@ -0,0 +1,66 @@ +"""Per-strategy partition locks for serialising RMW critical sections. + +The OME engine intentionally does NOT serialise concurrent runs of the +same strategy +(``local/specs/2026-04-27-ome-tech-design.md`` §4.5.2: the business logic +must guard itself, deciding inside the strategy body via +``async with lock`` bucketed by a business key). Offline strategies whose +body is a read → +modify → write on shared state (cluster rows, user.md, SKILL.md) +serialise on a business key (``owner_id`` / ``agent_id``) here. + +Mirrors :mod:`everos.service._session_lock` (and +:class:`everos.core.persistence.markdown.writer.MarkdownWriter`'s +per-path lock pool): one ``asyncio.Lock`` per +``(strategy_name, partition_key)`` pair, **never evicted** — a lock +with pending waiters must outlive any dict entry that points to it, +otherwise GC racing waiters can drop the lock mid-flight (CPython +bpo-28427). The pool grows with the live partition-key set, which in +practice is bounded by the agent / user / cluster counts a single +everos process owns. + +No acquire timeout: an OME strategy run has no upstream client +waiting on it, so timing out a queued caller would only convert +"slow" into a permanent ``dead_letter`` data-loss (`max_retries` +exhaustion). The LLM client owns the per-request timeout +(`component.llm.openai_provider`, default 60s) — that is the layer +that breaks a stuck LLM call, not this one. If a genuinely hung +strategy holds the lock indefinitely it surfaces as a stuck queue +under process-level monitoring; the recovery is a process restart, +not a silent data drop. + +Cross-process safety is out of scope: everos is single-process by +design (see ``CLAUDE.md`` deployment notes); the enterprise edition +layers a distributed coordinator on top. +""" + +from __future__ import annotations + +import asyncio + +_pools: dict[str, dict[str, asyncio.Lock]] = {} + + +def get_partition_lock(strategy_name: str, partition_key: str) -> asyncio.Lock: + """Return the lock for ``(strategy_name, partition_key)``; create on first use. + + ``dict.setdefault`` is atomic under single-threaded asyncio — no + ``await`` runs between the nested ``setdefault`` calls, so the + "check then insert" pair is indivisible. No meta-lock is needed. + + Callers acquire the lock with ``async with``; the lock object is + cached forever (see module docstring on bpo-28427), and the inner + asyncio queue gives FIFO fairness across waiters on the same key. + """ + return _pools.setdefault(strategy_name, {}).setdefault( + partition_key, asyncio.Lock() + ) + + +def _reset_for_tests() -> None: + """Test-only: drop every registered lock pool. + + Used by test fixtures that need a clean lock registry between + cases (no inherited holders, no inherited waiters). + """ + _pools.clear() diff --git a/src/everos/memory/cascade/handlers/__init__.py b/src/everos/memory/cascade/handlers/__init__.py index f75e3d5..b9306d7 100644 --- a/src/everos/memory/cascade/handlers/__init__.py +++ b/src/everos/memory/cascade/handlers/__init__.py @@ -3,10 +3,10 @@ Four daily-log handlers (episode / atomic_fact / foresight / agent_case) inherit :class:`BaseDailyLogHandler` for the shared read / diff / upsert / delete loop; the per-kind subclass only -declares its repo binding and ``_build_row`` mapping. ``agent_skill`` -and ``user_profile`` stand alone — they're single-file kinds (no -entries, no per-entry diff), so they implement :class:`Handler` -directly and own their reconcile loop. +declares its repo binding and ``_build_row`` mapping. ``agent_skill``, +``user_profile``, and ``knowledge_topic`` stand alone — they're +single-file kinds (no entries, no per-entry diff), so they implement +:class:`Handler` directly and own their reconcile loop. """ from .agent_case import AgentCaseHandler as AgentCaseHandler @@ -16,6 +16,8 @@ from .base import Handler as Handler from .base import HandlerDeps as HandlerDeps from .episode import EpisodeHandler as EpisodeHandler from .foresight import ForesightHandler as ForesightHandler +from .knowledge_document import KnowledgeDocumentHandler as KnowledgeDocumentHandler +from .knowledge_topic import KnowledgeTopicHandler as KnowledgeTopicHandler from .user_profile import UserProfileHandler as UserProfileHandler __all__ = [ @@ -26,5 +28,7 @@ __all__ = [ "ForesightHandler", "Handler", "HandlerDeps", + "KnowledgeDocumentHandler", + "KnowledgeTopicHandler", "UserProfileHandler", ] diff --git a/src/everos/memory/cascade/handlers/_daily_log_base.py b/src/everos/memory/cascade/handlers/_daily_log_base.py index ba005d3..8791a41 100644 --- a/src/everos/memory/cascade/handlers/_daily_log_base.py +++ b/src/everos/memory/cascade/handlers/_daily_log_base.py @@ -28,6 +28,7 @@ import asyncio import dataclasses from typing import Any, ClassVar +from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, StructuredEntry from ..types import HandlerOutcome @@ -35,6 +36,8 @@ from ._common import content_sha256 as compute_content_sha256 from ._common import resolve_owner, resolve_scope from .base import Handler +logger = get_logger(__name__) + @dataclasses.dataclass(frozen=True) class ParsedEntry: @@ -103,17 +106,50 @@ class BaseDailyLogHandler(Handler): ) for entry in parsed.entries ] - new_by_id = {e.entry_id: e for e in new_entries} existing = await self.lance_repo.find_where( f"md_path = '{_q(md_path)}'", limit=10_000, ) - existing_by_entry = {row.entry_id: row for row in existing} - owner_id, owner_type = resolve_owner(parsed.frontmatter, md_path) app_id, project_id = resolve_scope(md_path) + to_build, skipped = self._diff_entries(new_entries, existing) + to_upsert = await self._embed_entries( + to_build, + owner_id, + owner_type, + app_id, + project_id, + md_path, + ) + new_by_id = {e.entry_id for e in new_entries} + to_delete_ids = [ + row.entry_id for row in existing if row.entry_id not in new_by_id + ] + + await self._apply_lance_changes(to_upsert, to_delete_ids, md_path) + await self._propagate_deprecations( + parsed.frontmatter, + owner_id, + app_id, + project_id, + ) + return HandlerOutcome( + md_path=md_path, + kind=self.kind, + upserted=len(to_upsert), + deleted=len(to_delete_ids), + skipped=skipped, + ) + + @staticmethod + def _diff_entries( + new_entries: list[ParsedEntry], + existing: list[Any], + ) -> tuple[list[ParsedEntry], int]: + """Compare new entries against existing rows, return changed + skip count.""" + existing_by_entry = {row.entry_id: row for row in existing} to_build: list[ParsedEntry] = [] skipped = 0 for entry in new_entries: @@ -122,35 +158,43 @@ class BaseDailyLogHandler(Handler): skipped += 1 continue to_build.append(entry) + return to_build, skipped - # Build rows concurrently; ``_build_row`` calls ``embedder.embed`` - # which is already capped by a process-global ``asyncio.Semaphore`` - # at ``max_concurrent`` (see OpenAIEmbeddingProvider). This unblocks - # per-md-path embedding pipelining without uncapping embed-API rate. - to_upsert: list[Any] = ( - list( - await asyncio.gather( - *( - self._build_row( - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - md_path=md_path, - entry=entry, - ) - for entry in to_build + async def _embed_entries( + self, + to_build: list[ParsedEntry], + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + md_path: str, + ) -> list[Any]: + """Build LanceDB rows for changed entries (embed concurrently).""" + if not to_build: + return [] + return list( + await asyncio.gather( + *( + self._build_row( + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + md_path=md_path, + entry=entry, ) + for entry in to_build ) ) - if to_build - else [] ) - to_delete_ids = [ - row.entry_id for row in existing if row.entry_id not in new_by_id - ] - + async def _apply_lance_changes( + self, + to_upsert: list[Any], + to_delete_ids: list[str], + md_path: str, + ) -> None: + """Flush upserts and deletes to LanceDB.""" if to_upsert: await self.lance_repo.upsert(to_upsert) if to_delete_ids: @@ -159,14 +203,6 @@ class BaseDailyLogHandler(Handler): f"md_path = '{_q(md_path)}' AND entry_id IN ({in_list})" ) - return HandlerOutcome( - md_path=md_path, - kind=self.kind, - upserted=len(to_upsert), - deleted=len(to_delete_ids), - skipped=skipped, - ) - async def handle_deleted(self, md_path: str) -> HandlerOutcome: deleted = await self.lance_repo.delete_by_md_path(md_path) return HandlerOutcome( @@ -177,6 +213,65 @@ class BaseDailyLogHandler(Handler): skipped=0, ) + async def _propagate_deprecations( + self, + frontmatter: Any, + owner_id: str, + app_id: str, + project_id: str, + ) -> None: + """Propagate deprecated_entries from frontmatter to LanceDB. + + The md file is the source of truth; cascade reconstructs the + ``deprecated_by`` column on every sync/rebuild. + """ + deprecated = getattr(frontmatter, "deprecated_entries", None) + if not deprecated and isinstance(frontmatter, dict): + deprecated = frontmatter.get("deprecated_entries") + if not deprecated: + return + scope = (app_id, project_id) + await asyncio.gather( + *( + self._mark_deprecated(owner_id, entry_id, deprecated_by_val, scope) + for entry_id, deprecated_by_val in deprecated.items() + ) + ) + + async def _mark_deprecated( + self, + owner_id: str, + entry_id: str, + deprecated_by: str, + scope: tuple[str, str], + ) -> None: + """Set ``deprecated_by`` on a LanceDB row matching ``entry_id``. + + Scoped to ``(app_id, project_id, owner_id, entry_id)`` to avoid + cross-space collisions. A missing row is silently ignored — the + entry may have been deleted or not yet indexed. + """ + app_id, project_id = scope + predicate = ( + f"owner_id = '{_q(owner_id)}' " + f"AND entry_id = '{_q(entry_id)}' " + f"AND app_id = '{_q(app_id)}' " + f"AND project_id = '{_q(project_id)}'" + ) + try: + await self.lance_repo.update( + {"deprecated_by": deprecated_by}, + where=predicate, + ) + except Exception: + logger.warning( + "failed to mark entry deprecated", + entry_id=entry_id, + deprecated_by=deprecated_by, + kind=self.kind, + exc_info=True, + ) + @abc.abstractmethod async def _build_row( self, diff --git a/src/everos/memory/cascade/handlers/atomic_fact.py b/src/everos/memory/cascade/handlers/atomic_fact.py index 5331e16..a33e94e 100644 --- a/src/everos/memory/cascade/handlers/atomic_fact.py +++ b/src/everos/memory/cascade/handlers/atomic_fact.py @@ -55,7 +55,7 @@ class AtomicFactHandler(BaseDailyLogHandler): owner_type=owner_type, app_id=app_id, project_id=project_id, - session_id=s.inline.get("session_id", ""), + session_id=s.inline.get("session_id"), timestamp=require_iso_timestamp(s.inline.get("timestamp")), parent_type=s.inline.get("parent_type") or ParentType.MEMCELL.value, parent_id=s.inline.get("parent_id", ""), diff --git a/src/everos/memory/cascade/handlers/episode.py b/src/everos/memory/cascade/handlers/episode.py index 37fc955..590c7f6 100644 --- a/src/everos/memory/cascade/handlers/episode.py +++ b/src/everos/memory/cascade/handlers/episode.py @@ -75,7 +75,7 @@ class EpisodeHandler(BaseDailyLogHandler): owner_type=owner_type, app_id=app_id, project_id=project_id, - session_id=s.inline.get("session_id", ""), + session_id=s.inline.get("session_id"), timestamp=require_iso_timestamp(s.inline.get("timestamp")), parent_type=s.inline.get("parent_type") or ParentType.MEMCELL.value, parent_id=s.inline.get("parent_id", ""), diff --git a/src/everos/memory/cascade/handlers/foresight.py b/src/everos/memory/cascade/handlers/foresight.py index ad6f0e9..50bae77 100644 --- a/src/everos/memory/cascade/handlers/foresight.py +++ b/src/everos/memory/cascade/handlers/foresight.py @@ -79,7 +79,7 @@ class ForesightHandler(BaseDailyLogHandler): owner_type=owner_type, app_id=app_id, project_id=project_id, - session_id=s.inline.get("session_id", ""), + session_id=s.inline.get("session_id"), timestamp=require_iso_timestamp(s.inline.get("timestamp")), start_time=optional_iso_timestamp(s.inline.get("start_time")), end_time=optional_iso_timestamp(s.inline.get("end_time")), diff --git a/src/everos/memory/cascade/handlers/knowledge_document.py b/src/everos/memory/cascade/handlers/knowledge_document.py new file mode 100644 index 0000000..69d8593 --- /dev/null +++ b/src/everos/memory/cascade/handlers/knowledge_document.py @@ -0,0 +1,94 @@ +"""KnowledgeDocument cascade handler — md → SQLite only. + +Handles ``index.md`` files inside knowledge document directories +(``knowledge/{category}/{doc_title}/index.md``). Unlike +:class:`KnowledgeTopicHandler`, this handler writes to **SQLite only** +— there is no LanceDB write, no embedding, and no tokenization. + +The document-level index carries title, category, and a short summary; +the summary is the body of the file (plain text, no entries). Writes +are always upserted — the SQLite upsert is cheap and document metadata +changes infrequently. + +md contract: + +- ``knowledge/{category}/{doc_title}/index.md`` frontmatter: + ``type: knowledge_document``, ``id`` (== ``doc_id``), + ``category_id``, ``title``, ``source_name`` (optional), + ``source_type`` (optional). +- Body: document summary (plain text). +""" + +from __future__ import annotations + +from typing import Any + +from everos.core.persistence import MarkdownReader, ParsedMarkdown +from everos.infra.persistence.sqlite import ( + DocumentUpsertPayload, + knowledge_document_repo, +) + +from ..types import HandlerOutcome +from ._common import resolve_scope +from .base import Handler + + +class KnowledgeDocumentHandler(Handler): + """Cascade handler for ``knowledge/{category}/{doc_title}/index.md``.""" + + kind = "knowledge_document" + + async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: + absolute = self._deps.memory_root.root / md_path + parsed = await MarkdownReader.read(absolute) + + if parsed.frontmatter.get("type") != "knowledge_document": + return HandlerOutcome( + md_path=md_path, + kind=self.kind, + upserted=0, + deleted=0, + skipped=1, + ) + + payload = self._build_payload(parsed, md_path) + await knowledge_document_repo.upsert_from_handler(payload) + + return HandlerOutcome( + md_path=md_path, + kind=self.kind, + upserted=1, + deleted=0, + skipped=0, + ) + + def _build_payload( + self, parsed: ParsedMarkdown, md_path: str + ) -> DocumentUpsertPayload: + """Construct the SQLite upsert payload from parsed frontmatter and body.""" + fm: dict[str, Any] = parsed.frontmatter + app_id, project_id = resolve_scope(md_path) + source_name = fm.get("source_name") + source_type = fm.get("source_type") + return DocumentUpsertPayload( + doc_id=str(fm["id"]), + app_id=app_id, + project_id=project_id, + category_id=str(fm.get("category_id", "")), + title=str(fm.get("title", "")), + summary=parsed.body.strip(), + source_name=source_name if isinstance(source_name, str) else None, + source_type=source_type if isinstance(source_type, str) else None, + md_path=md_path, + ) + + async def handle_deleted(self, md_path: str) -> HandlerOutcome: + deleted = await knowledge_document_repo.delete_by_md_path(md_path) + return HandlerOutcome( + md_path=md_path, + kind=self.kind, + upserted=0, + deleted=deleted, + skipped=0, + ) diff --git a/src/everos/memory/cascade/handlers/knowledge_topic.py b/src/everos/memory/cascade/handlers/knowledge_topic.py new file mode 100644 index 0000000..80e2cdf --- /dev/null +++ b/src/everos/memory/cascade/handlers/knowledge_topic.py @@ -0,0 +1,227 @@ +"""KnowledgeTopic cascade handler — md → LanceDB + SQLite. + +Unlike every other handler which writes to LanceDB only, the knowledge +topic handler writes to **both** LanceDB (summary embedding + BM25 +tokens) and SQLite (content full text + tree structure). This is the +cross-storage pattern described in design spec §5.6. + +Cross-storage failure is handled by cascade worker retry: if either +write fails, the handler raises, the worker marks the md_path as +failed and retries later. Both writes are idempotent (upsert), so +retries are safe. + +md contract: + +- ``knowledge/{category}/{doc_title}/_.md`` frontmatter: + ``type: knowledge_topic``, ``id`` / ``node_id`` / ``doc_id`` / + ``category_id`` / ``topic_index`` / ``topic_name`` / ``topic_path`` + / ``summary`` / ``depth`` / ``parent_node_id`` / ``children_node_ids`` + / ``content_labels``. +- Body: topic content full text. + +Diff strategy: SHA-256 over the **content-bearing fields** (summary, +topic_name, topic_path, category_id, depth, body). Audit / tree +structure fields are excluded — they change on re-parse without +semantic drift. + +Embedding source: ``summary`` (mirrors the search recaller's anchor). +""" + +from __future__ import annotations + +import json +from typing import Any, ClassVar + +from everos.component.utils.datetime import get_utc_now +from everos.core.persistence import MarkdownReader, ParsedMarkdown +from everos.infra.persistence.lancedb import KnowledgeTopic, knowledge_topic_repo +from everos.infra.persistence.sqlite import ( + TopicUpsertPayload, + knowledge_topic_sqlite_repo, +) + +from ..types import HandlerOutcome +from ._common import content_sha256 as compute_content_sha256 +from ._common import resolve_scope +from .base import Handler + + +class KnowledgeTopicHandler(Handler): + """Cascade handler for + ``knowledge/{category}/{doc_title}/_.md``.""" + + kind = "knowledge_topic" + lance_repo: ClassVar[Any] = knowledge_topic_repo + + content_change_keys: ClassVar[tuple[str, ...]] = ( + "frontmatter:summary", + "frontmatter:topic_name", + "frontmatter:topic_path", + "frontmatter:category_id", + "frontmatter:depth", + "body", + ) + + # ------------------------------------------------------------------ + # Public entry point + # ------------------------------------------------------------------ + + async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: + absolute = self._deps.memory_root.root / md_path + parsed = await MarkdownReader.read(absolute) + + fields = self._parse_topic_fields(parsed, md_path) + if fields is None: + return HandlerOutcome( + md_path=md_path, + kind=self.kind, + upserted=0, + deleted=0, + skipped=1, + ) + + digest = compute_content_sha256( + { + "frontmatter:summary": fields["summary"], + "frontmatter:topic_name": fields["topic_name"], + "frontmatter:topic_path": fields["topic_path"], + "frontmatter:category_id": fields["category_id"], + "frontmatter:depth": str(fields["depth"]), + "body": fields["content"].rstrip(), + } + ) + + prior = await knowledge_topic_repo.get_by_id(fields["node_id"]) + if prior is not None and prior.content_sha256 == digest: + return HandlerOutcome( + md_path=md_path, + kind=self.kind, + upserted=0, + deleted=0, + skipped=1, + ) + + row = await self._build_lance_row(fields, digest, md_path) + await knowledge_topic_repo.upsert([row]) + + topic_payload = self._build_sqlite_payload(fields, md_path) + await knowledge_topic_sqlite_repo.upsert_from_handler(topic_payload) + + return HandlerOutcome( + md_path=md_path, + kind=self.kind, + upserted=1, + deleted=0, + skipped=0, + ) + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _parse_topic_fields( + self, + parsed: ParsedMarkdown, + md_path: str, + ) -> dict[str, Any] | None: + """Extract frontmatter fields and body; ``None`` if type mismatch.""" + fm = parsed.frontmatter + if fm.get("type") != "knowledge_topic": + return None + app_id, project_id = resolve_scope(md_path) + return { + "node_id": str(fm["id"]), + "doc_id": str(fm["doc_id"]), + "summary": str(fm.get("summary", "")), + "content": parsed.body, + "app_id": app_id, + "project_id": project_id, + "topic_name": str(fm.get("topic_name", "")), + "topic_path": str(fm.get("topic_path", "")), + "category_id": str(fm.get("category_id", "")), + "depth": int(fm.get("depth", 0)), + "parent_node_id": fm.get("parent_node_id"), + "children_node_ids": fm.get("children_node_ids", []), + "content_labels": fm.get("content_labels", []), + "topic_index": int(fm.get("topic_index", 0)), + } + + def _build_sqlite_payload( + self, + fields: dict[str, Any], + md_path: str, + ) -> TopicUpsertPayload: + """Construct the SQLite upsert payload from parsed topic fields.""" + return TopicUpsertPayload( + node_id=fields["node_id"], + doc_id=fields["doc_id"], + app_id=fields["app_id"], + project_id=fields["project_id"], + category_id=fields["category_id"], + topic_index=fields["topic_index"], + topic_name=fields["topic_name"], + topic_path=fields["topic_path"], + depth=fields["depth"], + parent_node_id=fields["parent_node_id"], + children_node_ids=( + json.dumps(fields["children_node_ids"]) + if fields["children_node_ids"] + else None + ), + summary=fields["summary"], + content=fields["content"], + content_labels=( + json.dumps(fields["content_labels"]) + if fields["content_labels"] + else None + ), + md_path=md_path, + ) + + async def _build_lance_row( + self, + fields: dict[str, Any], + digest: str, + md_path: str, + ) -> KnowledgeTopic: + """Tokenize, embed, and construct the LanceDB row.""" + summary_tokens = " ".join( + self._deps.tokenizer.tokenize(fields["summary"]), + ) + content_tokens = " ".join( + self._deps.tokenizer.tokenize(fields["content"]), + ) + vector = await self._deps.embedder.embed(fields["summary"]) + now = get_utc_now() + return KnowledgeTopic( + id=fields["node_id"], + doc_id=fields["doc_id"], + category_id=fields["category_id"], + app_id=fields["app_id"], + project_id=fields["project_id"], + topic_name=fields["topic_name"], + topic_path=fields["topic_path"], + depth=fields["depth"], + parent_node_id=fields["parent_node_id"] or "", + summary=fields["summary"], + summary_tokens=summary_tokens, + content_tokens=content_tokens, + content_labels=list(fields["content_labels"]), + md_path=md_path, + content_sha256=digest, + vector=vector, + created_at=now, + updated_at=now, + ) + + async def handle_deleted(self, md_path: str) -> HandlerOutcome: + lance_deleted = await knowledge_topic_repo.delete_by_md_path(md_path) + sqlite_deleted = await knowledge_topic_sqlite_repo.delete_by_md_path(md_path) + deleted = max(lance_deleted, sqlite_deleted) + return HandlerOutcome( + md_path=md_path, + kind=self.kind, + upserted=0, + deleted=deleted, + skipped=0, + ) diff --git a/src/everos/memory/cascade/registry.py b/src/everos/memory/cascade/registry.py index 201e8e4..34de209 100644 --- a/src/everos/memory/cascade/registry.py +++ b/src/everos/memory/cascade/registry.py @@ -25,12 +25,14 @@ from everos.infra.persistence.lancedb import ( AtomicFact, Episode, Foresight, + KnowledgeTopic, UserProfile, agent_case_repo, agent_skill_repo, atomic_fact_repo, episode_repo, foresight_repo, + knowledge_topic_repo, user_profile_repo, ) from everos.infra.persistence.markdown import ( @@ -39,6 +41,8 @@ from everos.infra.persistence.markdown import ( AtomicFactDailyFrontmatter, EpisodeDailyFrontmatter, ForesightDailyFrontmatter, + KnowledgeDocumentFrontmatter, + KnowledgeTopicFrontmatter, UserProfileFrontmatter, ) @@ -50,6 +54,8 @@ from .handlers import ( ForesightHandler, Handler, HandlerDeps, + KnowledgeDocumentHandler, + KnowledgeTopicHandler, UserProfileHandler, ) @@ -68,9 +74,9 @@ class KindSpec: name: str frontmatter_schema: type[BaseFrontmatter] - lance_schema: type - lance_repo: object handler_factory: type[Handler] + lance_schema: type | None = None + lance_repo: object | None = None def path_glob(self) -> str: """Glob (relative to memory root) for every md this kind covers.""" @@ -130,6 +136,20 @@ KIND_REGISTRY: tuple[KindSpec, ...] = ( lance_repo=user_profile_repo, handler_factory=UserProfileHandler, ), + KindSpec( + name="knowledge_document", + frontmatter_schema=KnowledgeDocumentFrontmatter, + handler_factory=KnowledgeDocumentHandler, + lance_schema=None, + lance_repo=None, + ), + KindSpec( + name="knowledge_topic", + frontmatter_schema=KnowledgeTopicFrontmatter, + handler_factory=KnowledgeTopicHandler, + lance_schema=KnowledgeTopic, + lance_repo=knowledge_topic_repo, + ), ) """Every cascade kind, evaluated in declaration order by :func:`match_kind`.""" diff --git a/src/everos/memory/events.py b/src/everos/memory/events.py index c211987..cce4702 100644 --- a/src/everos/memory/events.py +++ b/src/everos/memory/events.py @@ -56,8 +56,10 @@ class EpisodeExtracted(BaseEvent): episode_text: str episode_timestamp_ms: int owner_id: str + session_id: str | None = None app_id: str = "default" project_id: str = "default" + source: str = "pipeline" class AgentCaseExtracted(BaseEvent): diff --git a/src/everos/memory/extract/ingest/service.py b/src/everos/memory/extract/ingest/service.py index c3f66a1..955070e 100644 --- a/src/everos/memory/extract/ingest/service.py +++ b/src/everos/memory/extract/ingest/service.py @@ -26,7 +26,6 @@ from __future__ import annotations from typing import Any -from everos.component.llm import get_multimodal_llm_client from everos.component.utils.datetime import from_timestamp from everos.config import load_settings from everos.memory import CanonicalMessage, IngestResult, ToolCall @@ -59,7 +58,6 @@ async def process(payload: dict[str, Any]) -> IngestResult: require_multimodal() await enrich_content_items( content_items, - llm=get_multimodal_llm_client(), max_concurrency=load_settings().multimodal.max_concurrency, ) text, non_text = derive_text(content_items) @@ -96,6 +94,7 @@ async def process(payload: dict[str, Any]) -> IngestResult: def _coerce_tool_calls( raw: list[dict[str, Any]] | list[Any] | None, ) -> list[ToolCall] | None: + """Convert raw tool-call dicts to ToolCall models.""" if not raw: return None out: list[ToolCall] = [] diff --git a/src/everos/memory/extract/parser/availability.py b/src/everos/memory/extract/parser/availability.py index 544ed69..037869f 100644 --- a/src/everos/memory/extract/parser/availability.py +++ b/src/everos/memory/extract/parser/availability.py @@ -27,11 +27,9 @@ def has_unparsed_multimodal(items: list[dict[str, Any]]) -> bool: def multimodal_available() -> bool: """Whether the ``everalgo.parser`` extra is importable.""" - try: - import everalgo.parser # noqa: F401 - except ImportError: - return False - return True + from everos.component.parser import parser_available # Deferred: optional dep probe + + return parser_available() def require_multimodal() -> None: diff --git a/src/everos/memory/extract/parser/enrich.py b/src/everos/memory/extract/parser/enrich.py index 708e9c0..d257c64 100644 --- a/src/everos/memory/extract/parser/enrich.py +++ b/src/everos/memory/extract/parser/enrich.py @@ -1,9 +1,8 @@ -"""Parse non-text content items via everalgo.parser, backfilling in place. +"""Parse non-text content items via component.parser, backfilling in place. -The ``everalgo.parser`` import is deferred to call time so importing this -module never requires the optional ``everos[multimodal]`` extra. The ingest -stage calls :func:`require_multimodal` first, so a missing extra surfaces the -guided install error before this runs. +Delegates actual parsing to :func:`everos.component.parser.aparse_file` which +handles LLM injection and error mapping. This module owns the batch +concurrency and per-item degradation logic specific to the add/ingest path. """ from __future__ import annotations @@ -12,9 +11,8 @@ import asyncio from typing import Any from everalgo.llm import LLMError -from everalgo.llm.protocols import LLMClient -from everos.core.errors import MultimodalNotEnabledError, UnsupportedModalityError +from everos.core.errors import UnsupportedModalityError from everos.core.observability.logging import get_logger from .mapping import build_raw_file @@ -23,22 +21,20 @@ logger = get_logger(__name__) async def enrich_content_items( - items: list[dict[str, Any]], *, llm: LLMClient, max_concurrency: int = 4 + items: list[dict[str, Any]], *, max_concurrency: int = 4 ) -> None: """Parse each non-text item and backfill ``parsed_content`` in place. Synchronous to the request; items parse concurrently under a bounded semaphore. Deterministic failures (unsupported modality, missing system - dependency) raise a :class:`~everos.core.errors.MultimodalError` subclass - and abort the batch; transient failures (LLM errors) degrade per item - (``parse_status="failed"``) without dropping the rest. + dependency) propagate and abort the batch; transient LLM failures degrade + per item (``parse_status="failed"``) without dropping the rest. Args: items: ContentItem dicts (mutated in place). - llm: Multimodal LLM client passed to ``everalgo.parser.aparse``. max_concurrency: Upper bound on concurrent parse calls. """ - from everalgo.parser import aparse # optional dependency, imported lazily + from everos.component.parser import aparse_file # Deferred: optional dep targets = [ item @@ -53,26 +49,19 @@ async def enrich_content_items( async def _parse_one(item: dict[str, Any]) -> None: async with semaphore: try: - parsed = await aparse(await build_raw_file(item), llm=llm) - except NotImplementedError as exc: - raise UnsupportedModalityError( - f"modality not supported: {item.get('type')!r}" - ) from exc - except LLMError as exc: - # Transient: degrade this item, keep the rest of the batch. + raw = await build_raw_file(item) + except ValueError as exc: + raise UnsupportedModalityError(str(exc)) from exc + try: + parsed = await aparse_file(raw) + except LLMError: item["parse_status"] = "failed" - item["parse_error"] = type(exc).__name__ + item["parse_error"] = "LLMError" logger.warning( "multimodal_parse_failed", extra={"content_type": item.get("type")}, ) return - except ValueError as exc: - # everalgo dispatch / mapping rejected the input. - raise UnsupportedModalityError(str(exc)) from exc - except RuntimeError as exc: - # e.g. LibreOffice missing for Office documents. - raise MultimodalNotEnabledError(str(exc)) from exc item["parsed_content"] = parsed.text item["parse_status"] = "success" diff --git a/src/everos/memory/extract/pipeline/user_memory.py b/src/everos/memory/extract/pipeline/user_memory.py index 417f115..394fa0c 100644 --- a/src/everos/memory/extract/pipeline/user_memory.py +++ b/src/everos/memory/extract/pipeline/user_memory.py @@ -135,8 +135,10 @@ class UserMemoryPipeline: episode_text=ep.episode, episode_timestamp_ms=ep.timestamp, owner_id=ep.owner_id, + session_id=ingested.session_id, app_id=ingested.app_id, project_id=ingested.project_id, + source="pipeline", ) ) diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 407a06b..2533aeb 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -25,6 +25,7 @@ def compile_filters_for_get( owner_type: str, app_id: str = "default", project_id: str = "default", + exclude_deprecated: bool = True, ) -> str: """Compile ``/get`` filters via the shared ``compile_filters`` path. @@ -37,4 +38,5 @@ def compile_filters_for_get( owner_type=owner_type, app_id=app_id, project_id=project_id, + exclude_deprecated=exclude_deprecated, ) diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index 1000cbd..dd299a3 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -76,6 +76,7 @@ class GetManager: owner_type=req.owner_type, app_id=req.app_id, project_id=req.project_id, + exclude_deprecated=req.memory_type == GetMemoryType.EPISODE, ) match req.memory_type: diff --git a/src/everos/memory/models.py b/src/everos/memory/models.py index d8afb1a..b83f20b 100644 --- a/src/everos/memory/models.py +++ b/src/everos/memory/models.py @@ -113,7 +113,7 @@ class Episode(BaseModel): timestamp: int # everos engineering metadata. - session_id: str + session_id: str | None = None sender_ids: list[str] = Field(default_factory=list) parent_id: str @@ -125,7 +125,7 @@ class Episode(BaseModel): algo_episode: AlgoEpisode, *, owner_id: str, - session_id: str, + session_id: str | None, sender_ids: list[str], parent_id: str, ) -> Episode: @@ -159,15 +159,15 @@ class AtomicFact(BaseModel): :class:`Episode`: everos keeps the *semantic* fields algo emits (``owner_id`` / ``fact`` / ``timestamp``) and adds engineering context (``session_id`` / ``parent_id``) so md writer + cascade can audit-link - back to the source memcell. + back to the source episode. No ``sender_ids``: an atomic fact is a statement about its ``owner_id``; the surrounding participants are not part of the fact itself. (Episode keeps ``sender_ids`` because the narrative is *about* the conversation as a whole.) - ``parent_id`` is the source memcell id, supplied by the caller because - the new everalgo types no longer carry it on AtomicFact. + ``parent_id`` is the source episode entry_id, supplied by the caller + because the new everalgo types no longer carry it on AtomicFact. """ owner_id: str @@ -175,7 +175,7 @@ class AtomicFact(BaseModel): timestamp: int # everos engineering metadata. - session_id: str + session_id: str | None = None parent_id: str model_config = ConfigDict(extra="allow") @@ -186,18 +186,18 @@ class AtomicFact(BaseModel): algo_fact: AlgoAtomicFact, *, owner_id: str, - session_id: str, + session_id: str | None, parent_id: str, ) -> AtomicFact: """Build a domain AtomicFact from an algo AtomicFact plus context. ``owner_id`` is supplied by the caller (not read from ``algo_fact``) because atomic_fact extraction uses a subject-agnostic prompt — one - LLM call produces a template that fans out to multiple owners. The - algo-side ``owner_id`` is therefore a placeholder; the caller knows - the real one. Same rationale for ``parent_id``: algo no longer - carries the source memcell id; caller injects the authoritative - value (any ``extra='allow'`` smuggled values are dropped). + LLM call produces facts per owner. The algo-side ``owner_id`` is + therefore a placeholder; the caller knows the real one. Same + rationale for ``parent_id``: algo no longer carries the source + episode entry_id; caller injects the authoritative value (any + ``extra='allow'`` smuggled values are dropped). The algo type exposes the fact sentence as ``content``; everos's domain field is ``fact``. This boundary is where that rename is @@ -241,7 +241,7 @@ class Foresight(BaseModel): duration_days: int | None = None # everos engineering metadata. - session_id: str + session_id: str | None = None parent_id: str model_config = ConfigDict(extra="allow") @@ -251,7 +251,7 @@ class Foresight(BaseModel): cls, algo_foresight: AlgoForesight, *, - session_id: str, + session_id: str | None, parent_id: str, ) -> Foresight: """Build a domain Foresight from an algo Foresight plus context. diff --git a/src/everos/memory/reflection/__init__.py b/src/everos/memory/reflection/__init__.py new file mode 100644 index 0000000..aa46fa1 --- /dev/null +++ b/src/everos/memory/reflection/__init__.py @@ -0,0 +1,14 @@ +"""Reflection — offline memory consolidation. + +Merges fragmented cluster members into higher-quality episodes, re- +extracts atomic facts, and deprecates the originals. + +External usage: + from everos.memory.reflection import ReflectionOrchestrator +""" + +from __future__ import annotations + +from .orchestrator import ReflectionOrchestrator as ReflectionOrchestrator + +__all__ = ["ReflectionOrchestrator"] diff --git a/src/everos/memory/reflection/orchestrator.py b/src/everos/memory/reflection/orchestrator.py new file mode 100644 index 0000000..1c36d1a --- /dev/null +++ b/src/everos/memory/reflection/orchestrator.py @@ -0,0 +1,1093 @@ +"""ReflectionOrchestrator — Select -> Merge -> Re-extract -> Deprecate. + +Consolidates fragmented cluster members (memcell-derived episodes) into +a single high-quality merged episode per cluster. The merged episode is +written to md, re-extracted for atomic facts via ``EpisodeExtracted``, +and the originals are deprecated in both md frontmatter and LanceDB. + +See ``local/2026-06-14-reflection-everos-design.md`` for the full design. +""" + +from __future__ import annotations + +import asyncio +import datetime as _dt +import json +import uuid +from collections import defaultdict +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from everalgo.types import Episode as AlgoEpisode + from everalgo.user_memory import EpisodeReflector + + from everos.component.embedding import EmbeddingProvider + from everos.infra.persistence.markdown import EpisodeWriter + +import numpy as np + +from everos.component.utils.datetime import from_timestamp, to_iso_format +from everos.core.errors import AppError +from everos.core.observability.logging import get_logger +from everos.core.persistence import MemoryRoot +from everos.infra.ome.context import StrategyContext +from everos.memory._partition_locks import get_partition_lock +from everos.memory.events import EpisodeExtracted + +logger = get_logger(__name__) + +_MAX_CLUSTERS_PER_RUN = 10 +_WAIT_TIMEOUT_SECONDS = 120.0 + + +def _escape_sql(value: str) -> str: + """Escape single quotes for LanceDB SQL-like ``where`` predicates. + + LanceDB has no parameterised query API; doubling the quote + (``'`` -> ``''``) is the SQL-standard escape. + + Args: + value: Raw string to escape. + + Returns: + Escaped string safe for interpolation into a WHERE clause. + """ + return value.replace("'", "''") + + +class ReflectionOrchestrator: + """Run one Reflection cycle for a single owner scope. + + Consolidates fragmented cluster members into a single merged episode + per cluster via Select -> Merge -> Re-extract -> Deprecate. + + Args: + cluster_repo: SQLite cluster repository (member CRUD + queries). + episode_store: LanceDB episode repository (read + update). + atomic_fact_store: LanceDB atomic fact repository (update). + episode_writer: Markdown daily-log writer for episodes. + report_repo: SQLite reflection report repository. + reflector: Algorithm-side EpisodeReflector (areflect). + embedder: Embedding provider for centroid recomputation. + """ + + def __init__( + self, + *, + cluster_repo: Any, + episode_store: Any, + atomic_fact_store: Any, + episode_writer: EpisodeWriter, + report_repo: Any, + reflector: EpisodeReflector, + embedder: EmbeddingProvider, + ) -> None: + self._cluster_repo = cluster_repo + self._episode_store = episode_store + self._atomic_fact_store = atomic_fact_store + self._episode_writer = episode_writer + self._report_repo = report_repo + self._reflector = reflector + self._embedder = embedder + + async def run( + self, + *, + ctx: StrategyContext, + owner_id: str, + owner_type: str = "user", + kind: str = "user_memory", + app_id: str = "default", + project_id: str = "default", + ) -> list[object]: + """Run one Reflection cycle for a single owner scope. + + Args: + ctx: Runtime context (event bus + wait). + owner_id: Target owner identifier. + owner_type: Owner type discriminator. + kind: Memory kind for cluster lookup. + app_id: Application scope. + project_id: Project scope. + + Returns: + List of successful ReflectionReport rows (typed as object + because the table class lives in infra). + """ + candidates = await self._select_candidates( + owner_id=owner_id, + kind=kind, + app_id=app_id, + project_id=project_id, + ) + logger.info( + "reflection_candidates_selected", + owner_id=owner_id, + candidate_count=len(candidates), + ) + if not candidates: + return [] + + reports: list[object] = [] + skip_count = 0 + for cluster_id in candidates: + report = await self._process_cluster_safely( + ctx=ctx, + cluster_id=cluster_id, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + ) + if report is not None: + reports.append(report) + else: + skip_count += 1 + + logger.info( + "reflection_cycle_completed", + owner_id=owner_id, + success_count=len(reports), + skip_count=skip_count, + ) + return reports + + async def _process_cluster_safely( + self, + *, + ctx: StrategyContext, + cluster_id: str, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + ) -> object | None: + """Process one cluster, catching errors to allow the cycle to continue. + + Args: + ctx: Runtime context (event bus + wait). + cluster_id: Target cluster identifier. + owner_id: Target owner identifier. + owner_type: Owner type discriminator. + app_id: Application scope. + project_id: Project scope. + + Returns: + A ReflectionReport on success, ``None`` on skip or error. + """ + try: + return await self._process_cluster( + ctx=ctx, + cluster_id=cluster_id, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + ) + except AppError: + logger.warning( + "reflection_cluster_skipped", + cluster_id=cluster_id, + exc_info=True, + ) + return None + except Exception: + logger.error( + "reflection_cluster_unexpected_error", + cluster_id=cluster_id, + exc_info=True, + ) + return None + + # ── SELECT ──────────────────────────────────────────────────────────── + + async def _select_candidates( + self, + *, + owner_id: str, + kind: str, + app_id: str, + project_id: str, + ) -> list[str]: + """Two-step DB-agnostic candidate selection. + + Args: + owner_id: Target owner identifier. + kind: Memory kind for cluster lookup. + app_id: Application scope. + project_id: Project scope. + + Returns: + Cluster IDs sorted by member count descending, limited + to ``_MAX_CLUSTERS_PER_RUN``. + """ + reflected = await self._report_repo.list_reflected_cluster_ids( + owner_id, app_id, project_id + ) + clusters = await self._cluster_repo.list_ids_and_member_counts( + owner_id, kind, app_id=app_id, project_id=project_id + ) + count_map = dict(clusters) + candidates = [ + cid + for cid, count in clusters + if (cid not in reflected and count >= 2) or (cid in reflected and count > 1) + ] + candidates.sort(key=lambda cid: count_map[cid], reverse=True) + return candidates[:_MAX_CLUSTERS_PER_RUN] + + # ── Per-cluster processing ──────────────────────────────────────────── + + async def _process_cluster( + self, + *, + ctx: StrategyContext, + cluster_id: str, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + ) -> object | None: + """Full flow for one cluster: merge, write, re-extract, deprecate. + + Args: + ctx: Runtime context (event bus + wait). + cluster_id: Target cluster identifier. + owner_id: Target owner identifier. + owner_type: Owner type discriminator. + app_id: Application scope. + project_id: Project scope. + + Returns: + A ReflectionReport on success, ``None`` on skip. + """ + await self._detect_orphans(cluster_id, owner_id, app_id, project_id) + + scope = dict(owner_id=owner_id, app_id=app_id, project_id=project_id) + members, episodes = await self._load_cluster_episodes( + cluster_id=cluster_id, **scope + ) + if not members or not episodes: + return None + + mode, algo_result = await self._reflect_cluster( + episodes=episodes, + owner_id=owner_id, + ) + if algo_result is None: + return None + + merged_entry_id = await self._write_and_reextract( + ctx=ctx, + cluster_id=cluster_id, + **scope, + algo_result=algo_result, + episodes=episodes, + mode=mode, + members=members, + ) + if merged_entry_id is None: + return None + + return await self._deprecate( + ctx=ctx, + cluster_id=cluster_id, + owner_type=owner_type, + **scope, + original_members=members, + merged_entry_id=merged_entry_id, + algo_result=algo_result, + mode=mode, + episodes=episodes, + ) + + async def _reflect_cluster( + self, + *, + episodes: list[Any], + owner_id: str, + ) -> tuple[str, AlgoEpisode | None]: + """Determine reflection mode and call the algo reflector. + + Args: + episodes: Source episode rows from LanceDB. + owner_id: Owner for logging on failure. + + Returns: + ``(mode, algo_result)`` where mode is ``"init"`` or ``"update"`` + and algo_result is ``None`` on failure. + """ + merged_entry_ids = [e.entry_id for e in episodes if e.parent_type == "cluster"] + is_update = bool(merged_entry_ids) + mode = "update" if is_update else "init" + algo_result = await self._call_reflector( + episodes=episodes, + merged_entry_ids=merged_entry_ids, + is_update=is_update, + owner_id=owner_id, + ) + return mode, algo_result + + async def _load_cluster_episodes( + self, + *, + cluster_id: str, + owner_id: str, + app_id: str, + project_id: str, + ) -> tuple[list[tuple[str, str]], list[Any]]: + """Read cluster members and fetch their episode rows from LanceDB. + + Args: + cluster_id: Target cluster identifier. + owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. + + Returns: + ``(members, episodes)`` tuple; either may be empty on skip. + """ + members = await self._cluster_repo.get_members_with_type(cluster_id) + if not members: + return [], [] + + member_ids = [mid for mid, _ in members] + episodes = await self._fetch_episodes( + entry_ids=member_ids, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + ) + return members, episodes + + async def _write_and_reextract( + self, + *, + ctx: StrategyContext, + cluster_id: str, + owner_id: str, + app_id: str, + project_id: str, + algo_result: AlgoEpisode, + episodes: list[Any], + mode: str, + members: list[tuple[str, str]], + ) -> str | None: + """Write merged episode to md and emit re-extraction event. + + Args: + ctx: Runtime context (event bus + wait). + cluster_id: Target cluster identifier. + owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. + algo_result: Algo reflector output with ``.episode`` / ``.subject``. + episodes: Source episode rows (for timestamp derivation). + mode: ``"init"`` or ``"update"``. + members: Original cluster members ``(member_id, member_type)``. + + Returns: + The ``merged_entry_id`` on success, ``None`` on extraction timeout. + """ + last_ts = max(ep.timestamp for ep in episodes) + merged_entry_id = await self._write_merged_episode( + cluster_id=cluster_id, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + algo_result=algo_result, + last_ts=last_ts, + ) + logger.info( + "reflection_merged", + cluster_id=cluster_id, + mode=mode, + source_count=len(members), + merged_entry_id=merged_entry_id, + ) + return await self._emit_and_wait_extraction( + ctx=ctx, + cluster_id=cluster_id, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + algo_result=algo_result, + merged_entry_id=merged_entry_id, + last_ts=last_ts, + ) + + async def _write_merged_episode( + self, + *, + cluster_id: str, + owner_id: str, + app_id: str, + project_id: str, + algo_result: AlgoEpisode, + last_ts: object, + ) -> str: + """Write the merged episode entry to markdown. + + Args: + cluster_id: Parent cluster identifier. + owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. + algo_result: Algo reflector output with ``.episode`` / ``.subject``. + last_ts: Latest source episode timestamp (datetime or int). + + Returns: + The formatted ``merged_entry_id``. + """ + last_ts_iso = to_iso_format(from_timestamp(_ts_to_ms(last_ts))) + if last_ts_iso is None: + raise ValueError("to_iso_format returned None for valid timestamp") + inline, sections = _merged_episode_to_entry_body( + algo_result, cluster_id, owner_id, last_ts_iso + ) + entry_ids = await self._episode_writer.append_entries( + owner_id, + [(inline, sections)], + app_id=app_id, + project_id=project_id, + ) + return entry_ids[0].format() + + async def _emit_and_wait_extraction( + self, + *, + ctx: StrategyContext, + cluster_id: str, + owner_id: str, + app_id: str, + project_id: str, + algo_result: AlgoEpisode, + merged_entry_id: str, + last_ts: object, + ) -> str | None: + """Emit ``EpisodeExtracted`` and wait for cascade to process it. + + Args: + ctx: Runtime context (event bus + wait). + cluster_id: Target cluster identifier (for error logging). + owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. + algo_result: Algo reflector output (episode text). + merged_entry_id: Entry ID of the written merged episode. + last_ts: Latest source episode timestamp (datetime or int). + + Returns: + The ``merged_entry_id`` on success, ``None`` on timeout. + """ + event = EpisodeExtracted( + memcell_id=merged_entry_id, + episode_entry_id=merged_entry_id, + episode_text=algo_result.episode, + episode_timestamp_ms=_ts_to_ms(last_ts), + owner_id=owner_id, + session_id=None, + app_id=app_id, + project_id=project_id, + source="reflection", + ) + await ctx.emit(event) + try: + await ctx.wait_for_event(event.event_id, timeout=_WAIT_TIMEOUT_SECONDS) + except TimeoutError: + logger.error( + "reflection_extraction_timeout", + cluster_id=cluster_id, + event_id=event.event_id, + merged_entry_id=merged_entry_id, + ) + return None + return merged_entry_id + + # ── Helpers ─────────────────────────────────────────────────────────── + + async def _detect_orphans( + self, + cluster_id: str, + owner_id: str, + app_id: str, + project_id: str, + ) -> None: + """Log warning if orphan merged episodes exist for this cluster. + + Args: + cluster_id: Target cluster identifier. + owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. + """ + where = ( + f"parent_type = 'cluster' AND parent_id = '{_escape_sql(cluster_id)}' " + f"AND deprecated_by IS NULL " + f"AND owner_id = '{_escape_sql(owner_id)}' " + f"AND app_id = '{_escape_sql(app_id)}' " + f"AND project_id = '{_escape_sql(project_id)}'" + ) + orphans = await self._episode_store.find_where(where, limit=10) + if orphans: + logger.warning( + "reflection_orphan_detected", + cluster_id=cluster_id, + orphan_entry_ids=[o.entry_id for o in orphans], + ) + + async def _fetch_episodes( + self, + *, + entry_ids: list[str], + owner_id: str, + app_id: str, + project_id: str, + ) -> list[Any]: + """Fetch episodes by entry_id. + + Returns: + Episode list sorted by timestamp ascending. + """ + rows = await self._episode_store.find_by_owner_entries( + owner_id, + entry_ids, + app_id=app_id, + project_id=project_id, + ) + rows.sort(key=lambda e: e.timestamp) + return rows + + async def _call_reflector( + self, + *, + episodes: list[Any], + merged_entry_ids: list[str], + is_update: bool, + owner_id: str, + ) -> AlgoEpisode | None: + """Call the algo reflector (INIT or UPDATE mode). + + Args: + episodes: Source episode rows from LanceDB. + merged_entry_ids: Entry IDs of previously merged episodes + (parent_type=cluster). Empty for INIT. + is_update: Whether this is an UPDATE (vs INIT) reflection. + owner_id: Owner for logging on failure. + + Returns: + An algo Episode result, or ``None`` on failure. + """ + algo_episodes = _to_algo_episodes(episodes) + try: + if is_update: + return await self._reflect_update( + algo_episodes=algo_episodes, + episodes=episodes, + merged_entry_ids=merged_entry_ids, + ) + return await self._reflector.areflect(algo_episodes) + except AppError: + logger.warning( + "reflection_reflector_failed", + owner_id=owner_id, + exc_info=True, + ) + return None + except Exception: + logger.error( + "reflection_reflector_unexpected_error", + owner_id=owner_id, + exc_info=True, + ) + return None + + async def _reflect_update( + self, + *, + algo_episodes: list[AlgoEpisode], + episodes: list[Any], + merged_entry_ids: list[str], + ) -> AlgoEpisode | None: + """Run UPDATE-mode reflection by splitting old/new episodes. + + Args: + algo_episodes: Converted algo Episode objects (parallel to ``episodes``). + episodes: Source episode rows from LanceDB. + merged_entry_ids: Entry IDs of previously merged episodes. + + Returns: + An algo Episode result, or ``None`` when no old episodes remain. + """ + merged_set = set(merged_entry_ids) + old_algo_eps = [ + ae + for ae, e in zip(algo_episodes, episodes, strict=True) + if e.entry_id in merged_set + ] + new_algo_eps = [ + ae + for ae, e in zip(algo_episodes, episodes, strict=True) + if e.entry_id not in merged_set + ] + if not old_algo_eps: + return None + return await self._reflector.areflect(new_algo_eps, old_episode=old_algo_eps[0]) + + # ── Deprecate (orchestrator + sub-steps) ───────────────────────────── + + async def _deprecate( + self, + *, + ctx: StrategyContext, + cluster_id: str, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + original_members: list[tuple[str, str]], + merged_entry_id: str, + algo_result: AlgoEpisode, + mode: str, + episodes: list[Any], + ) -> object | None: + """Deprecate originals and update cluster membership. + + Runs under a partition lock for concurrency safety. + + Args: + ctx: Runtime context (unused here, kept for signature compat). + cluster_id: Target cluster identifier. + owner_id: Target owner identifier. + owner_type: Owner type discriminator. + app_id: Application scope. + project_id: Project scope. + original_members: Snapshot ``(member_id, member_type)`` from selection. + merged_entry_id: Entry ID of the newly written merged episode. + algo_result: Algo reflector output (for centroid + report). + mode: ``"init"`` or ``"update"``. + episodes: Source episode rows (for md patching + timestamp). + + Returns: + A ReflectionReport on success, ``None`` on failure or empty diff. + """ + partition = f"{app_id}:{project_id}:{cluster_id}" + try: + async with get_partition_lock("reflection_deprecate", partition): + return await self._execute_deprecation( + cluster_id=cluster_id, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + original_members=original_members, + merged_entry_id=merged_entry_id, + algo_result=algo_result, + mode=mode, + episodes=episodes, + ) + except AppError: + logger.warning( + "reflection_deprecate_failed", + cluster_id=cluster_id, + exc_info=True, + ) + return None + except Exception: + logger.error( + "reflection_deprecate_unexpected_error", + cluster_id=cluster_id, + exc_info=True, + ) + return None + + async def _execute_deprecation( + self, + *, + cluster_id: str, + owner_id: str, + app_id: str, + project_id: str, + original_members: list[tuple[str, str]], + merged_entry_id: str, + algo_result: AlgoEpisode, + mode: str, + episodes: list[Any], + ) -> object | None: + """Run the deprecation steps inside the partition lock. + + Args: + cluster_id: Target cluster identifier. + owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. + original_members: Snapshot ``(member_id, member_type)`` from selection. + merged_entry_id: Entry ID of the newly written merged episode. + algo_result: Algo reflector output (for centroid + report). + mode: ``"init"`` or ``"update"``. + episodes: Source episode rows (for md patching + timestamp). + + Returns: + A ReflectionReport on success, ``None`` when no members to deprecate. + """ + to_deprecate = await self._resolve_deprecation_targets( + cluster_id=cluster_id, + original_members=original_members, + ) + if not to_deprecate: + return None + + dep_ep, dep_fact = await self._apply_deprecation_writes( + episodes=episodes, + to_deprecate=to_deprecate, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + merged_entry_id=merged_entry_id, + ) + await self._update_cluster_after_merge( + cluster_id=cluster_id, + to_deprecate=to_deprecate, + merged_entry_id=merged_entry_id, + algo_result=algo_result, + episodes=episodes, + ) + report = await self._create_reflection_report( + cluster_id=cluster_id, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + mode=mode, + original_members=original_members, + to_deprecate=to_deprecate, + merged_entry_id=merged_entry_id, + deprecated_fact_count=dep_fact, + ) + logger.info( + "reflection_deprecated", + cluster_id=cluster_id, + deprecated_episode_count=dep_ep, + deprecated_fact_count=dep_fact, + ) + return report + + async def _apply_deprecation_writes( + self, + *, + episodes: list[Any], + to_deprecate: set[str], + owner_id: str, + app_id: str, + project_id: str, + merged_entry_id: str, + ) -> tuple[int, int]: + """Patch md frontmatter and mark episodes/facts deprecated in LanceDB. + + Args: + episodes: Source episode rows (for md patching). + to_deprecate: Set of member IDs being deprecated. + owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. + merged_entry_id: Entry ID of the replacement merged episode. + + Returns: + ``(deprecated_episode_count, deprecated_fact_count)``. + """ + await self._patch_md_frontmatter( + episodes=episodes, + to_deprecate=to_deprecate, + merged_entry_id=merged_entry_id, + ) + deprecated_ep_count = await self._deprecate_lance_episodes( + entry_ids=to_deprecate, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + merged_entry_id=merged_entry_id, + ) + deprecated_fact_count = await self._deprecate_lance_facts( + parent_ids=to_deprecate, + owner_id=owner_id, + merged_entry_id=merged_entry_id, + ) + return deprecated_ep_count, deprecated_fact_count + + async def _resolve_deprecation_targets( + self, + *, + cluster_id: str, + original_members: list[tuple[str, str]], + ) -> set[str]: + """Re-read cluster members and intersect with the original snapshot. + + Args: + cluster_id: Target cluster identifier. + original_members: Snapshot ``(member_id, member_type)`` from selection. + + Returns: + Set of member IDs safe to deprecate (present in both snapshots). + """ + current_members = await self._cluster_repo.get_members_with_type(cluster_id) + current_ids = {mid for mid, _ in current_members} + original_ids = {mid for mid, _ in original_members} + return original_ids & current_ids + + async def _deprecate_lance_episodes( + self, + *, + entry_ids: set[str], + owner_id: str, + app_id: str, + project_id: str, + merged_entry_id: str, + ) -> int: + """Mark deprecated episodes in LanceDB by entry_id. + + Returns: + Number of LanceDB update calls issued. + """ + coros: list[Any] = [ + self._episode_store.update( + {"deprecated_by": merged_entry_id}, + where=( + f"entry_id = '{_escape_sql(eid)}' " + f"AND owner_id = '{_escape_sql(owner_id)}' " + f"AND app_id = '{_escape_sql(app_id)}' " + f"AND project_id = '{_escape_sql(project_id)}'" + ), + ) + for eid in entry_ids + ] + if coros: + await asyncio.gather(*coros) + return len(coros) + + async def _deprecate_lance_facts( + self, + *, + parent_ids: set[str], + owner_id: str, + merged_entry_id: str, + ) -> int: + """Mark deprecated atomic facts in LanceDB. + + Args: + parent_ids: Parent IDs (memcell or episode) whose facts to deprecate. + owner_id: Target owner identifier. + merged_entry_id: Entry ID of the replacement merged episode. + + Returns: + Total number of LanceDB update calls issued. + """ + if not parent_ids: + return 0 + + coros = [ + self._atomic_fact_store.update( + {"deprecated_by": merged_entry_id}, + where=( + f"parent_id = '{_escape_sql(pid)}' " + f"AND owner_id = '{_escape_sql(owner_id)}' " + f"AND deprecated_by IS NULL" + ), + ) + for pid in parent_ids + ] + await asyncio.gather(*coros) + return len(coros) + + async def _update_cluster_after_merge( + self, + *, + cluster_id: str, + to_deprecate: set[str], + merged_entry_id: str, + algo_result: AlgoEpisode, + episodes: list[Any], + ) -> None: + """Remove old members, add merged, and recompute centroid. + + Args: + cluster_id: Target cluster identifier. + to_deprecate: Member IDs to remove from the cluster. + merged_entry_id: Entry ID of the newly merged episode. + algo_result: Algo reflector output (episode text for centroid). + episodes: Source episode rows (for last timestamp). + """ + await self._cluster_repo.remove_members(cluster_id, to_deprecate) + await self._cluster_repo.add_member(cluster_id, merged_entry_id, "episode") + + centroid = await self._embedder.embed(algo_result.episode) + centroid_blob = np.asarray(centroid, dtype=np.float32).tobytes() + last_ts_ms = _ts_to_ms(max(ep.timestamp for ep in episodes)) + await self._cluster_repo.update_metadata( + cluster_id, + centroid_blob=centroid_blob, + count=1, + last_ts_ms=last_ts_ms, + preview_json=json.dumps([algo_result.episode[:200]], ensure_ascii=False), + ) + + async def _create_reflection_report( + self, + *, + cluster_id: str, + owner_id: str, + app_id: str, + project_id: str, + mode: str, + original_members: list[tuple[str, str]], + to_deprecate: set[str], + merged_entry_id: str, + deprecated_fact_count: int, + ) -> object: + """Build and persist a ReflectionReport row. + + Args: + cluster_id: Target cluster identifier. + owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. + mode: ``"init"`` or ``"update"``. + original_members: Full member snapshot ``(member_id, member_type)``. + to_deprecate: Subset of members that were actually deprecated. + merged_entry_id: Entry ID of the replacement merged episode. + deprecated_fact_count: Number of atomic fact deprecation calls. + + Returns: + The persisted ReflectionReport row. + """ + # Deferred: avoid pulling heavy SQLModel table at module import. + from everos.infra.persistence.sqlite import ReflectionReport + + source_members_json = json.dumps( + [ + {"member_id": mid, "member_type": mtype} + for mid, mtype in original_members + if mid in to_deprecate + ], + ensure_ascii=False, + ) + report = ReflectionReport( + id=uuid.uuid4().hex, + cluster_id=cluster_id, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + mode=mode, + source_members=source_members_json, + source_count=len(to_deprecate), + merged_entry_id=merged_entry_id, + deprecated_fact_count=deprecated_fact_count, + ) + await self._report_repo.create(report) + return report + + async def _patch_md_frontmatter( + self, + *, + episodes: list[Any], + to_deprecate: set[str], + merged_entry_id: str, + ) -> None: + """Patch ``deprecated_entries`` in md frontmatter for affected files. + + Groups deprecated episodes by md_path and issues one + ``patch_frontmatter`` call per file. + + Args: + episodes: Source episode rows (must have ``md_path``). + to_deprecate: Set of member IDs being deprecated. + merged_entry_id: Entry ID of the replacement merged episode. + """ + path_to_entries: dict[str, dict[str, str]] = defaultdict(dict) + for ep in episodes: + is_deprecated = ep.parent_id in to_deprecate or ep.entry_id in to_deprecate + if is_deprecated and ep.md_path: + path_to_entries[ep.md_path][ep.entry_id] = merged_entry_id + + root = MemoryRoot.default().root + for md_path, deprecated_map in path_to_entries.items(): + await self._episode_writer.patch_frontmatter( + root / md_path, + {"deprecated_entries": deprecated_map}, + ) + + +def _to_algo_episodes(episodes: list[Any]) -> list[AlgoEpisode]: + """Convert LanceDB episode rows to algo Episode objects. + + Args: + episodes: Source episode rows from LanceDB. + + Returns: + Parallel list of algo Episode objects. + """ + # Deferred: avoid pulling LLM libs at module import time. + from everalgo.types import Episode as AlgoEpisode + + return [ + AlgoEpisode( + owner_id=e.owner_id, + episode=e.episode, + subject=e.subject or "", + timestamp=_ts_to_ms(e.timestamp), + ) + for e in episodes + ] + + +def _merged_episode_to_entry_body( + algo_result: AlgoEpisode, + cluster_id: str, + owner_id: str, + timestamp_iso: str, +) -> tuple[dict[str, object], dict[str, str]]: + """Build ``(inline, sections)`` for a merged episode md entry. + + ``session_id`` is intentionally omitted (aggregation product has no + session); the cascade handler defaults to ``None``. + + Args: + algo_result: Algo reflector output with ``.subject`` / ``.episode``. + cluster_id: Parent cluster identifier. + owner_id: Target owner identifier. + timestamp_iso: ISO-formatted timestamp for the entry. + + Returns: + ``(inline, sections)`` tuple ready for ``append_entries``. + """ + inline: dict[str, object] = { + "owner_id": owner_id, + "timestamp": timestamp_iso, + "parent_type": "cluster", + "parent_id": cluster_id, + } + sections: dict[str, str] = { + "Subject": algo_result.subject or "", + "Content": algo_result.episode, + } + return inline, sections + + +def _ts_to_ms(ts: object) -> int: + """Coerce a timestamp to milliseconds. + + LanceDB episode rows store ``timestamp`` as a ``datetime`` object; + the algo Episode type uses ``int`` (milliseconds). This helper + handles both. + + Args: + ts: A ``datetime``, ``int``, or ``float`` timestamp. + + Returns: + Timestamp in milliseconds. + + Raises: + TypeError: When ``ts`` is not a recognised type. + """ + if isinstance(ts, _dt.datetime): + return int(ts.timestamp() * 1000) + if isinstance(ts, (int, float)): + return int(ts) + raise TypeError(f"unexpected timestamp type: {type(ts)}") diff --git a/src/everos/memory/search/agentic.py b/src/everos/memory/search/agentic.py index d57fe56..48a54d7 100644 --- a/src/everos/memory/search/agentic.py +++ b/src/everos/memory/search/agentic.py @@ -11,9 +11,11 @@ Implements the cluster main path from ``benchmarks/common/stages/search.py`` Hyperparameters match benchmark ``config.py`` defaults and are frozen as module-level constants — no env/TOML knobs at this layer. -id contract: candidates flowing through the pipeline carry ``id=memcell_id`` -(fact.parent_id chain). The final shaping step remaps to ``id=episode_id`` -via ``metadata["episode_id"]`` before calling ``shape_episode_from_candidate``. +id contract: candidates flowing through the pipeline carry +``id=memcell_id`` (regular episodes, parent_type=memcell) or +``id=entry_id`` (merged episodes, parent_type=cluster). The final +shaping step remaps to ``id=episode_id`` via ``metadata["episode_id"]`` +before calling ``shape_episode_from_candidate``. """ from __future__ import annotations @@ -47,7 +49,7 @@ if TYPE_CHECKING: _DENSE_CANDIDATES: int = 50 _SPARSE_CANDIDATES: int = 50 _HYBRID_RRF_K: int = 40 -_CLUSTER_BASE_CANDIDATES: int = 100 +_CLUSTER_BASE_CANDIDATES: int | None = None _CLUSTER_TOP_K: int = 10 _ROUND1_TOP_N: int = 50 _ROUND1_RERANK_TOP_N: int = 10 @@ -55,9 +57,10 @@ _ROUND2_CAP: int = 40 _MULTI_QUERY_COUNT: int = 3 _REFINEMENT_STRATEGY: str = "multi_query" -# Child-pool sizing — mirrors SearchManager._MAXSIM_FACT_MULTIPLIER / _CAP. -_FACT_CHILD_MULTIPLIER: int = 20 -_FACT_CHILD_CAP: int = 2000 +# Child-pool sizing for amaxsim_retrieve. The benchmark passes +# len(full_fact_corpus); EverOS doesn't know the corpus size upfront, +# so we pass a large sentinel and let the LanceDB limit clamp naturally. +_FACT_CHILD_CANDIDATES: int = 100_000 # Qwen3-Reranker task instruction for the search scene (benchmark # ``config.reranker_instruction``). Steers the cross-encoder toward fact / @@ -103,26 +106,24 @@ async def search_episodes_agentic( vec = await embed_query_fn(q) if not vec: return [] - child_limit = min(k * _FACT_CHILD_MULTIPLIER, _FACT_CHILD_CAP) - return await atomic_fact_recaller.dense_recall(vec, where, limit=child_limit) + return await atomic_fact_recaller.dense_recall(vec, where, limit=k) async def _fact_sparse(q: str, k: int) -> list[Candidate]: - child_limit = min(k * _FACT_CHILD_MULTIPLIER, _FACT_CHILD_CAP) - return await atomic_fact_recaller.sparse_recall(q, where, limit=child_limit) + return await atomic_fact_recaller.sparse_recall(q, where, limit=k) - # 2. parent_fetch: maps memcell_ids -> Candidate(id=memcell_id) for the amaxsim - # score lookup. Stores the real LanceDB episode id in metadata["episode_id"] - # for final shaping. - async def _parent_fetch(memcell_ids: list[str]) -> list[Candidate]: - ep_cands = await episode_recaller.fetch_by_parent_ids(memcell_ids, where) + # 2. parent_fetch: maps entry_ids (from atomic_fact.parent_id) to episodes. + # Atomic facts always point to episodes via entry_id regardless of + # whether the episode is memcell-based or cluster-merged. + async def _parent_fetch(parent_ids: list[str]) -> list[Candidate]: + episodes = await episode_recaller.fetch_by_entry_ids(parent_ids, where) result: list[Candidate] = [] - for c in ep_cands: - mc_id = c.metadata.get("parent_id") - if not isinstance(mc_id, str): + for c in episodes: + entry_id = c.metadata.get("entry_id") + if not isinstance(entry_id, str): continue result.append( Candidate( - id=mc_id, + id=entry_id, score=0.0, source=c.source, metadata=_to_everalgo_doc_metadata( @@ -139,7 +140,7 @@ async def search_episodes_agentic( child_retrieve=_fact_dense, parent_fetch=_parent_fetch, top_n=k, - child_candidates=min(k * _FACT_CHILD_MULTIPLIER, _FACT_CHILD_CAP), + child_candidates=_FACT_CHILD_CANDIDATES, ) async def _sparse(q: str, k: int) -> list[Candidate]: @@ -148,7 +149,7 @@ async def search_episodes_agentic( child_retrieve=_fact_sparse, parent_fetch=_parent_fetch, top_n=k, - child_candidates=min(k * _FACT_CHILD_MULTIPLIER, _FACT_CHILD_CAP), + child_candidates=_FACT_CHILD_CANDIDATES, ) # 4. hybrid_full: RRF fusion of dense + sparse MaxSim. @@ -215,18 +216,24 @@ def _to_everalgo_doc_metadata(metadata: dict[str, Any]) -> dict[str, Any]: ``aagentic_retrieve`` renders Round-1 candidates into the sufficiency / multi-query LLM prompt via ``everalgo.rank.agentic._format_docs``, which - reads the doc body from ``metadata["content"] | metadata["text"] | id`` and - the date from a ms-epoch ``metadata["timestamp"]``. everos episode rows - carry the body in ``episode`` (str) and the time in ``timestamp`` (datetime); - without this bridge the prompt degrades to the memcell id as the body and a - "N/A" date. ``episode`` is left untouched so the reranker and shaper -- both - expecting a plain string -- keep working. ``_restore_shaper_metadata`` - reverts the timestamp before DTO shaping. + reads ``metadata["episode"]`` as a dict with ``subject`` + ``content`` + keys and the date from a ms-epoch ``metadata["timestamp"]``. everos + episode rows carry the body in ``episode`` (str) and the time in + ``timestamp`` (datetime); without this bridge ``_format_docs`` raises + ``TypeError``. + + The flat ``episode`` string is also kept as ``text`` for the reranker + (which reads a plain string). ``_restore_shaper_metadata`` reverts + the restructured metadata before DTO shaping. """ bridged = dict(metadata) episode = metadata.get("episode") if isinstance(episode, str): bridged["text"] = episode + bridged["episode"] = { + "subject": metadata.get("subject", ""), + "content": episode, + } timestamp = metadata.get("timestamp") if isinstance(timestamp, _dt.datetime): bridged["timestamp"] = to_timestamp_ms(timestamp) @@ -234,17 +241,20 @@ def _to_everalgo_doc_metadata(metadata: dict[str, Any]) -> dict[str, Any]: def _restore_shaper_metadata(metadata: dict[str, Any]) -> dict[str, Any]: - """Revert the ms-epoch ``timestamp`` injected for everalgo back to datetime. + """Revert bridged metadata fields before DTO shaping. - ``shape_episode_from_candidate`` requires a ``datetime`` timestamp and drops - the row otherwise; the agentic pipeline carried it as ms-epoch for the LLM - prompt. The extra ``text`` key is ignored by the shaper and left in place. + Undoes two transforms from ``_to_everalgo_doc_metadata``: + 1. ``timestamp``: ms-epoch (int) → ``datetime`` (shaper requires it). + 2. ``episode``: dict ``{"subject", "content"}`` → flat str (shaper + reads ``metadata["episode"]`` as a plain string). """ - timestamp = metadata.get("timestamp") - if not isinstance(timestamp, (int, float)): - return metadata reverted = dict(metadata) - reverted["timestamp"] = from_timestamp(timestamp) + timestamp = metadata.get("timestamp") + if isinstance(timestamp, (int, float)): + reverted["timestamp"] = from_timestamp(timestamp) + episode = metadata.get("episode") + if isinstance(episode, dict): + reverted["episode"] = episode.get("content", "") return reverted diff --git a/src/everos/memory/search/dto.py b/src/everos/memory/search/dto.py index ec4fd16..6f7b1c7 100644 --- a/src/everos/memory/search/dto.py +++ b/src/everos/memory/search/dto.py @@ -4,7 +4,7 @@ Contract per the final design: * ``owner_type`` is a hard partition. ``user`` returns ``episodes`` (and optionally ``profiles``); ``agent`` returns ``agent_cases`` + - ``agent_skills``. The five ``data.*`` arrays always exist; routes not + ``agent_skills``. The four ``data.*`` arrays always exist; routes not applicable to the current ``owner_type`` stay as ``[]``. * ``atomic_facts`` are **nested** inside :class:`SearchEpisodeItem`, never returned as a top-level array. @@ -80,6 +80,14 @@ class SearchRequest(BaseModel): method: SearchMethod = SearchMethod.HYBRID top_k: int = -1 radius: float | None = Field(default=None, ge=0.0, le=1.0) + min_score: float | None = Field(default=None, ge=0.0, le=1.0) + """Post-fusion relevance floor for the episode HYBRID (hierarchy) path. + + Applied after Layer 4 against the LR-calibrated score in ``[0, 1]``: + items scoring below this value are dropped. Independent of ``radius`` + (which gates raw cosine at recall time); ``None`` disables the floor. + Only the episode hierarchy path consumes it — other methods ignore it. + """ include_profile: bool = False enable_llm_rerank: bool = Field( default=False, @@ -148,7 +156,8 @@ class SearchEpisodeItem(BaseModel): """Owning user (``None`` only on malformed cascade rows).""" app_id: str = "default" project_id: str = "default" - session_id: str + session_id: str | None = None + """``None`` for merged episodes (Reflection aggregation products).""" timestamp: _dt.datetime sender_ids: list[str] = Field(default_factory=list) summary: str diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 6318562..e80acfa 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -35,14 +35,10 @@ import datetime as _dt from typing import Any, Final from everos.component.utils.datetime import from_timestamp, to_iso_format +from everos.core.errors import FilterError as FilterError # noqa: F401 from .dto import FilterNode - -class FilterError(ValueError): - """Raised when the DSL contains a disallowed field, operator, or value.""" - - # ── Allow-lists ────────────────────────────────────────────────────────── _OP_MAP: Final[dict[str, str]] = { @@ -96,6 +92,7 @@ def compile_filters( owner_type: str, app_id: str = "default", project_id: str = "default", + exclude_deprecated: bool = True, ) -> str: """Compile a request's filters into a single LanceDB ``where`` string. @@ -112,6 +109,8 @@ def compile_filters( f"app_id = '{_escape_str(app_id)}'", f"project_id = '{_escape_str(project_id)}'", ] + if exclude_deprecated: + base.append("deprecated_by IS NULL") if node is None: return " AND ".join(base) compiled = _compile_node(node.model_dump(exclude_none=True)) diff --git a/src/everos/memory/search/hierarchy.py b/src/everos/memory/search/hierarchy.py index 6f1a7bd..3b8b4f8 100644 --- a/src/everos/memory/search/hierarchy.py +++ b/src/everos/memory/search/hierarchy.py @@ -2,8 +2,9 @@ Episode HYBRID search path: combines episode-level hybrid recall (Layer 1) with fact-driven MaxSim re-scoring (Layer 2), merges via RRF (Layer 3), then -runs a single-pass eviction where a fact that outscores its parent episode -enters top-N in place of the episode (Layer 4). +runs a hierarchical fact eviction where parent episode and its facts compete on a +single LR-calibrated scale and the best fact replaces the episode when it +wins (Layer 4). Uses everalgo operators as pure algorithm primitives; all I/O is injected via recaller callbacks. No changes to the everalgo library are required. @@ -14,7 +15,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from everalgo.rank import amaxsim_retrieve -from everalgo.rank.fusion import rrf +from everalgo.rank.fusion import cosine_to_lr_score, rrf from everalgo.types import Candidate, FactCandidate, ScoredItem from everos.core.observability.logging import get_logger @@ -30,6 +31,9 @@ if TYPE_CHECKING: logger = get_logger(__name__) +_HIERARCHY_ALPHA = 1.0 +_HIERARCHY_FACTS_PER_EPISODE = 3 + async def hierarchy_retrieve_episodes( query: str, @@ -42,6 +46,8 @@ async def hierarchy_retrieve_episodes( where: str, top_k: int, fact_child_candidates: int = 200, + alpha: float = _HIERARCHY_ALPHA, + min_score: float | None = None, ) -> list[SearchEpisodeItem]: """Run the four-layer hierarchical episode retrieval pipeline. @@ -49,8 +55,9 @@ async def hierarchy_retrieve_episodes( Layer 2: MaxSim re-score via atomic-fact child retrieval (fact cosine ANN → group by parent memcell → episode re-score by best fact). Layer 3: RRF merge of Layer-1 and Layer-2 results, sliced to top_k. - Layer 4: Pre-fetch facts for merged episodes, then single-pass eviction - (fact outscoring its parent episode enters top-N instead). + Layer 4: Pre-fetch facts for merged episodes, then hierarchical eviction — + parent and facts compete on one LR-calibrated scale; the best + fact replaces its episode when it wins. Args: query: Raw query string passed to amaxsim_retrieve. @@ -65,10 +72,14 @@ async def hierarchy_retrieve_episodes( top_k: Maximum number of items in the final merged slice before eviction. fact_child_candidates: How many atomic-fact ANN candidates to pull in Layer 2. Default 200. + alpha: Child (fact) weight in the Layer-4 LR-scale blend. Default + ``_HIERARCHY_ALPHA``. + min_score: Optional post-Layer-4 relevance floor on the LR-calibrated + score in ``[0, 1]``; items below it are dropped. ``None`` disables. Returns: Shaped SearchEpisodeItem list (episodes with nested atomic_facts), - sorted by score descending. + sorted by score descending, each carrying an LR-calibrated score. """ # Layer 1 — episode RRF fusion layer1_episodes = rrf(sparse, dense) @@ -91,57 +102,99 @@ async def hierarchy_retrieve_episodes( return [] # Layer 4a — pre-fetch facts for merged episodes - ep_to_memcell = _build_ep_to_memcell(merged) + ep_to_parents = _build_ep_to_fact_parents(merged) episode_to_facts = await fact_recaller.facts_for_episodes( - ep_to_memcell, + ep_to_parents, where, per_episode=max(top_k * 2, 20), query_vector=query_vector, ) - # Layer 4b — single-pass eviction - scored_items = _hierarchy_eviction_pass(merged, episode_to_facts) + ep_cosine: dict[str, float] = {} + for c in (*dense, *layer2_episodes): + if c.id: + ep_cosine[c.id] = max(ep_cosine.get(c.id, 0.0), c.score) + ep_bm25 = {c.id: c.score for c in sparse if c.id} + + scored_items = _hierarchy_eviction_pass( + merged, + episode_to_facts, + ep_cosine=ep_cosine, + ep_bm25=ep_bm25, + alpha=alpha, + ) # Build episode pool for orphan fact parent lookup. # Include layer2_episodes so episodes surfaced only via MaxSim path # (not in the original sparse/dense recall) can still serve as parent. episode_pool = {c.id: c for c in (*sparse, *dense, *layer2_episodes)} - return reshape_hybrid_output(scored_items, episode_pool=episode_pool) + shaped = reshape_hybrid_output(scored_items, episode_pool=episode_pool) + + # Post-Layer-4 relevance floor on the LR-calibrated score. + if min_score is not None: + shaped = [item for item in shaped if item.score >= min_score] + return shaped def _hierarchy_eviction_pass( merged: list[Candidate], episode_to_facts: dict[str, list[FactCandidate]], + *, + ep_cosine: dict[str, float], + ep_bm25: dict[str, float], + alpha: float = _HIERARCHY_ALPHA, + facts_per_episode: int = _HIERARCHY_FACTS_PER_EPISODE, ) -> list[ScoredItem]: - """Single-pass eviction: fact outscoring its parent episode enters top-N. + """Hierarchical fact eviction: parent and facts compete on one LR-calibrated scale. - For each episode in merged order: if its best matching atomic fact scores - higher than the episode itself, emit the fact as a ScoredItem - (item_type='atomic_fact') and mark the episode as an orphan parent. - Otherwise emit the episode directly as item_type='episode'. + For each merged episode the parent and its candidate facts are calibrated + to an LR probability via ``cosine_to_lr_score`` so a raw fact cosine and an + episode's recall relevance become directly comparable (replacing the prior + cosine-vs-RRF comparison, which mixed scales). Each fact's blended score is + ``alpha * child_lr + (1 - alpha) * parent_lr``; the single best-scoring + fact replaces the episode (eviction) when it beats the parent's own LR + score, otherwise the episode is emitted at ``parent_lr``. Args: merged: RRF-merged episode candidates, ordered by descending score. - episode_to_facts: Map from episode_id to its pre-fetched FactCandidates, + Their ``.score`` (RRF) is used only for ordering, not for scoring. + episode_to_facts: Map from episode id to its pre-fetched FactCandidates, sorted by cosine similarity descending. + ep_cosine: Per-episode best cosine relevance (dense / MaxSim routes). + ep_bm25: Per-episode BM25 score (sparse route); ``0.0`` when absent. + alpha: Child (fact) weight in the blend; ``1.0`` lets the fact's own + calibrated relevance fully drive the blended score. + facts_per_episode: Max facts per episode entered into the competition. Returns: - Mixed list of ScoredItem instances (episodes and atomic_facts) ready - for reshape_hybrid_output. + Mixed list of ScoredItem instances (episodes and atomic_facts), each + carrying an LR-calibrated ``score`` in ``[0, 1]``, ready for + reshape_hybrid_output. """ out: list[ScoredItem] = [] for episode in merged: - facts = episode_to_facts.get(episode.id, []) - best_fact = facts[0] if facts else None + parent_bm25 = ep_bm25.get(episode.id, 0.0) + parent_cosine = ep_cosine.get(episode.id, 0.0) + parent_lr = cosine_to_lr_score(parent_cosine, parent_bm25) - if best_fact is not None and best_fact.score > episode.score: - # Fact wins: emit fact; episode becomes orphan parent + # A fact must strictly beat the parent's LR score to evict it. + best_fact: FactCandidate | None = None + best_blended = parent_lr + for fact in episode_to_facts.get(episode.id, [])[:facts_per_episode]: + child_lr = cosine_to_lr_score(fact.score, parent_bm25) + blended = alpha * child_lr + (1.0 - alpha) * parent_lr + if blended > best_blended: + best_blended = blended + best_fact = fact + + if best_fact is not None: + # Fact wins: emit fact at its blended score; episode becomes orphan parent. out.append( ScoredItem( id=best_fact.id, - score=best_fact.score, + score=best_blended, item_type="atomic_fact", metadata=best_fact.metadata, parent_episode_id=episode.id, @@ -151,15 +204,15 @@ def _hierarchy_eviction_pass( "hierarchy_eviction_fact_wins", episode_id=episode.id, fact_id=best_fact.id, - fact_score=best_fact.score, - episode_score=episode.score, + fact_score=best_blended, + episode_score=parent_lr, ) else: - # Episode wins: emit episode with its metadata intact + # Episode wins: emit episode at its LR-calibrated parent score. out.append( ScoredItem( id=episode.id, - score=episode.score, + score=parent_lr, item_type="episode", metadata=dict(episode.metadata), parent_episode_id=None, @@ -184,8 +237,8 @@ async def _maxsim_episode_rescore( """Run amaxsim_retrieve to produce MaxSim-rescored episode candidates. Atomic facts serve as child documents (their metadata["parent_id"] is - the memcell_id). Episodes are fetched as parents via - episode_recaller.fetch_by_parent_ids. + the episode entry_id). Episodes are fetched as parents via + episode_recaller.fetch_by_entry_ids. ``amaxsim_retrieve`` calls ``child_retrieve`` exactly once with the original query string. We reuse the pre-computed ``query_vector`` to @@ -209,8 +262,8 @@ async def _maxsim_episode_rescore( # Reuse the pre-computed query_vector instead of re-embedding. return await fact_recaller.dense_recall(query_vector, where, limit=n) - async def parent_fetch(memcell_ids: list[str]) -> list[Candidate]: - return await episode_recaller.fetch_by_parent_ids(memcell_ids, where) + async def parent_fetch(entry_ids: list[str]) -> list[Candidate]: + return await episode_recaller.fetch_by_entry_ids(entry_ids, where) return await amaxsim_retrieve( query, @@ -221,22 +274,32 @@ async def _maxsim_episode_rescore( ) -def _build_ep_to_memcell(episodes: list[Candidate]) -> dict[str, str]: - """Extract episode_id → memcell_id mapping from episode candidates. +def _build_ep_to_fact_parents(episodes: list[Candidate]) -> dict[str, list[str]]: + """Map episode candidate id to all possible fact parent_id values. - Episodes store their source memcell id in metadata["parent_id"]. - Entries missing or having a non-string parent_id are silently skipped - (they will receive no facts during Layer 4). + New facts (post-1.5): parent_id = episode entry_id. + Old facts (pre-1.5): parent_id = memcell_id (episode.parent_id). + Both are collected so the IN query covers both eras without backfill. + + Invariant: entry_id (ep_*) and memcell_id (mc_*) namespaces never + overlap, so mixing them in one IN clause is safe. Args: episodes: Merged episode candidate list. Returns: - Dict mapping episode LanceDB id to memcell id. + Dict mapping episode LanceDB id to a list of candidate parent_ids + (entry_id and/or memcell_id). """ - result: dict[str, str] = {} + result: dict[str, list[str]] = {} for ep in episodes: - mc_id = ep.metadata.get("parent_id") - if isinstance(mc_id, str) and mc_id: - result[ep.id] = mc_id + parents: list[str] = [] + entry_id = ep.metadata.get("entry_id") + if isinstance(entry_id, str) and entry_id: + parents.append(entry_id) + parent_id = ep.metadata.get("parent_id") + if isinstance(parent_id, str) and parent_id and parent_id != entry_id: + parents.append(parent_id) + if parents: + result[ep.id] = parents return result diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index b7dcae5..4fd03d7 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -161,6 +161,7 @@ class SearchManager: owner_type=req.owner_type, app_id=req.app_id, project_id=req.project_id, + exclude_deprecated=req.owner_type == "user", ) self._validate_components(req) @@ -302,6 +303,7 @@ class SearchManager: episode_recaller=self._ep, where=where, top_k=top_k, + min_score=req.min_score, ) # rrf / lr: standard everalgo fusion path (fallback). @@ -518,35 +520,31 @@ class SearchManager: ``atomic_fact`` table) for finer-grained semantic match — long episodes whose single mean-pooled vector dilutes a specific topic recover via the matching atomic fact's own embedding. Mirrors - the EverOS MaxSim retrieval pattern. + EverOS/EverAlgo's MaxSim retrieval pattern. """ vector = await self._embed_query(req.query) if not vector: return [] fact_limit = min(top_k * _MAXSIM_FACT_MULTIPLIER, _MAXSIM_FACT_POOL_CAP) fact_cands = await self._fact.dense_recall(vector, where, limit=fact_limit) - # Max-pool fact scores by their parent memcell. ``atomic_fact`` - # rows always carry ``parent_id = memcell_id`` (cascade contract). - mc_score: dict[str, float] = {} + # Max-pool fact scores by parent episode entry_id. + ep_score: dict[str, float] = {} for fc in fact_cands: - mc = fc.metadata.get("parent_id") - if not isinstance(mc, str) or not mc: + pid = fc.metadata.get("parent_id") + if not isinstance(pid, str) or not pid: continue - if fc.score > mc_score.get(mc, -1.0): - mc_score[mc] = fc.score - if not mc_score: + if fc.score > ep_score.get(pid, -1.0): + ep_score[pid] = fc.score + if not ep_score: return [] - ranked = sorted(mc_score.items(), key=lambda kv: kv[1], reverse=True)[:top_k] - top_mc_ids = [mc for mc, _ in ranked] - score_by_mc = dict(ranked) - # One LanceDB scan: ``WHERE parent_id IN (...)``. The episode - # ``where`` re-applies the partition filter so episodes whose - # owner partition no longer matches the request are dropped. - ep_cands = await self._ep.fetch_by_parent_ids(top_mc_ids, where) + ranked = sorted(ep_score.items(), key=lambda kv: kv[1], reverse=True)[:top_k] + top_entry_ids = [eid for eid, _ in ranked] + score_by_entry = dict(ranked) + ep_cands = await self._ep.fetch_by_entry_ids(top_entry_ids, where) rescored: list[Candidate] = [] for c in ep_cands: - mc = c.metadata.get("parent_id") - s = score_by_mc.get(mc, 0.0) if isinstance(mc, str) else 0.0 + eid = c.metadata.get("entry_id") + s = score_by_entry.get(eid, 0.0) if isinstance(eid, str) else 0.0 rescored.append( Candidate(id=c.id, score=s, source="vector", metadata=c.metadata) ) diff --git a/src/everos/memory/search/recall/__init__.py b/src/everos/memory/search/recall/__init__.py index bcb8b1b..522512e 100644 --- a/src/everos/memory/search/recall/__init__.py +++ b/src/everos/memory/search/recall/__init__.py @@ -10,6 +10,7 @@ External usage:: AgentCaseRecaller, AgentSkillRecaller, ProfileRecaller, + KnowledgeTopicRecaller, ) """ @@ -21,6 +22,7 @@ from .base import RecallerDeps as RecallerDeps from .base import cosine_score_from_distance as cosine_score_from_distance from .base import row_to_candidate as row_to_candidate from .episode import EpisodeRecaller as EpisodeRecaller +from .knowledge_topic import KnowledgeTopicRecaller as KnowledgeTopicRecaller from .profile import ProfileRecaller as ProfileRecaller __all__ = [ @@ -29,6 +31,7 @@ __all__ = [ "AtomicFactRecaller", "EpisodeRecaller", "KindRecaller", + "KnowledgeTopicRecaller", "ProfileRecaller", "RecallerDeps", "cosine_score_from_distance", diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 0e1a4da..69eba22 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -5,19 +5,19 @@ Beyond the standard sparse / dense pair the recaller exposes atomic facts to their parent episodes (``episode_to_facts`` fed into the fact eviction pass). -Episode-fact linkage is **indirect through the shared memcell parent**: -both kinds are written with ``parent_id = memcell_id`` by the cascade. -The caller hands in an ``episode_id → memcell_id`` map; we query facts -by ``parent_id IN (memcell_ids)`` and regroup by episode using the -inverse map, so one fact bucket-shows under every episode that shares -the source memcell. +Episode-fact linkage uses a **dual parent_id strategy**: +- New facts (post-1.5): ``parent_id = episode_entry_id``. +- Old facts (pre-1.5): ``parent_id = memcell_id``. +The caller hands in an ``episode_id → [parent_id, ...]`` map; we query +facts by ``parent_id IN (all_parent_ids)`` and regroup by episode using +the inverse map, so both old and new facts are surfaced without backfill. """ from __future__ import annotations from collections import defaultdict from collections.abc import Mapping, Sequence -from typing import ClassVar +from typing import Any, ClassVar from everalgo.types import Candidate, FactCandidate @@ -66,6 +66,16 @@ class AtomicFactRecaller: async def dense_recall( self, vector: Sequence[float], where: str, *, limit: int ) -> list[Candidate]: + """Cosine ANN recall over the atomic_fact table. + + Args: + vector: Query embedding vector; empty returns no results. + where: LanceDB SQL filter clause scoping the search. + limit: Maximum number of candidates to return. + + Returns: + Candidates ranked by cosine similarity (descending). + """ if not vector: return [] table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) @@ -88,7 +98,7 @@ class AtomicFactRecaller: async def facts_for_episodes( self, - ep_to_memcell: Mapping[str, str], + ep_to_parents: Mapping[str, Sequence[str]], where: str, *, per_episode: int, @@ -96,12 +106,12 @@ class AtomicFactRecaller: ) -> dict[str, list[FactCandidate]]: """Pull facts for a set of episodes, bucketed by episode id. - ``ep_to_memcell`` maps the candidate episode's LanceDB id to the - source memcell id (read off ``episode.parent_id`` by the - caller). Facts are queried by their own ``parent_id`` against - the deduped memcell set, then re-bucketed under every episode - that shares each memcell — two episodes pulled from the same - memcell each get a copy of that memcell's facts. + ``ep_to_parents`` maps the candidate episode's LanceDB id to a + list of possible fact parent_id values (entry_id for post-1.5 + facts, memcell_id for pre-1.5 facts). Facts are queried by + ``parent_id IN (all_unique_parent_ids)`` and re-bucketed under + every episode that claims each parent_id — two episodes sharing + a parent_id each get a copy of that parent's facts. When ``query_vector`` is provided, the LanceDB query layers cosine ANN on top of the ``parent_id IN (...)`` filter, so each @@ -110,37 +120,23 @@ class AtomicFactRecaller: case every fact ships with ``score=0.0`` — the caller is responsible for not consuming the score in that mode. """ - if not ep_to_memcell: + if not ep_to_parents: return {} - memcell_to_eps: dict[str, list[str]] = defaultdict(list) - for ep_id, mc_id in ep_to_memcell.items(): - if mc_id: - memcell_to_eps[mc_id].append(ep_id) - if not memcell_to_eps: + parent_to_eps = _build_parent_to_episode_map(ep_to_parents) + if not parent_to_eps: return {} - quoted = ", ".join(f"'{_q(mc_id)}'" for mc_id in memcell_to_eps) - clause = f"parent_id IN ({quoted})" - full_where = f"({where}) AND ({clause})" - limit = per_episode * max(len(memcell_to_eps), 1) - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - if query_vector: - rows = ( - await table.query() - .nearest_to(list(query_vector)) - .distance_type("cosine") - .where(full_where) - .limit(limit) - .to_list() - ) - else: - rows = await table.query().where(full_where).limit(limit).to_list() + rows = await self._query_facts_for_parents( + parent_to_eps, where, per_episode=per_episode, query_vector=query_vector + ) + + # Bucket rows by episode and cap each bucket. buckets: dict[str, list[FactCandidate]] = defaultdict(list) for r in rows: - mc_id = r.get("parent_id") + fact_parent_id = r.get("parent_id") fid = r.get("id") - if not isinstance(mc_id, str) or not isinstance(fid, str): + if not isinstance(fact_parent_id, str) or not isinstance(fid, str): continue metadata = { k: v for k, v in r.items() if k not in _NOISE_COLUMNS and k != "id" @@ -148,7 +144,7 @@ class AtomicFactRecaller: score = ( cosine_score_from_distance(r.get("_distance")) if query_vector else 0.0 ) - for ep_id in memcell_to_eps.get(mc_id, ()): + for ep_id in parent_to_eps.get(fact_parent_id, ()): buckets[ep_id].append( FactCandidate( id=fid, @@ -157,11 +153,47 @@ class AtomicFactRecaller: metadata=metadata, ) ) - # Per-bucket cap; with query_vector the rows arrive sorted by - # cosine ascending (closest first) so slicing keeps the most - # relevant facts per episode. + # With query_vector the rows arrive sorted by cosine ascending + # (closest first) so slicing keeps the most relevant facts. return {ep_id: bucket[:per_episode] for ep_id, bucket in buckets.items()} + async def _query_facts_for_parents( + self, + parent_to_eps: dict[str, list[str]], + where: str, + *, + per_episode: int, + query_vector: Sequence[float] | None, + ) -> list[dict[str, Any]]: + """Construct and execute the LanceDB query for parent_id IN (...).""" + quoted = ", ".join(f"'{_q(pid)}'" for pid in parent_to_eps) + clause = f"parent_id IN ({quoted})" + full_where = f"({where}) AND ({clause})" + limit = per_episode * max(len(parent_to_eps), 1) + table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) + if query_vector: + return await ( + table.query() + .nearest_to(list(query_vector)) + .distance_type("cosine") + .where(full_where) + .limit(limit) + .to_list() + ) + return await table.query().where(full_where).limit(limit).to_list() + + +def _build_parent_to_episode_map( + ep_to_parents: Mapping[str, Sequence[str]], +) -> dict[str, list[str]]: + """Invert ep-to-parents map to a parent-to-episodes map.""" + parent_to_eps: dict[str, list[str]] = defaultdict(list) + for ep_id, parent_ids in ep_to_parents.items(): + for pid in parent_ids: + if pid: + parent_to_eps[pid].append(ep_id) + return parent_to_eps + def _q(value: str) -> str: return value.replace("'", "''") diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index 69d6581..6dd3293 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -77,53 +77,42 @@ class EpisodeRecaller: for r in rows ] - async def fetch_by_parent_ids( - self, parent_ids: Sequence[str], where: str - ) -> list[Candidate]: - """Batch-fetch episodes whose ``parent_id`` (memcell id) is in the set. - - One LanceDB scan per call (``WHERE parent_id IN (...)``) — used by - the MaxSim-style vector strategy that first ranks memcells via - ``atomic_fact`` cosine and then reverse-resolves the episode. - ``score`` on the returned candidates is left at ``0.0``; the - caller re-attaches the upstream max-pool score before sorting. - """ - if not parent_ids: - return [] - table = await get_table(Episode.TABLE_NAME, Episode) - quoted = ", ".join(f"'{_q(p)}'" for p in parent_ids) - full_where = f"({where}) AND (parent_id IN ({quoted}))" - rows = await table.query().where(full_where).limit(len(parent_ids)).to_list() - return [row_to_candidate(r, source="vector", score=0.0) for r in rows] - async def fetch_all_for_owner(self, where: str) -> list[Candidate]: - """Flat scan — all episodes for this owner, keyed by memcell id. + """Flat scan — all episodes for this owner, keyed by entry_id. - Returns every episode row as a ``Candidate`` with ``id = parent_id`` - (the memcell id) so ``acluster_retrieve`` membership matching against - ``cluster.members`` (also memcell ids) works without extra mapping. - The real LanceDB episode id travels in ``metadata["episode_id"]`` so - the agentic orchestrator can restore canonical episode identity after - ``aagentic_retrieve`` returns. + Cluster membership matching in ``acluster_retrieve`` compares + ``Candidate.id`` against ``Cluster.members``. Both are now + episode entry_ids regardless of parent_type. - No ``limit`` is applied — the full owner partition is required for - cluster membership matching (``acluster_retrieve`` needs ``all_docs`` - to cover every member of every cluster). + No ``limit`` — the full owner partition is required for cluster + membership matching. """ table = await get_table(Episode.TABLE_NAME, Episode) rows = await table.query().where(where).to_list() result: list[Candidate] = [] for r in rows: - mc_id = r.get("parent_id") - if not isinstance(mc_id, str) or not mc_id: + entry_id = r.get("entry_id") + if not isinstance(entry_id, str) or not entry_id: continue base = row_to_candidate(r, source="vector", score=0.0) result.append( Candidate( - id=mc_id, + id=entry_id, score=0.0, source="vector", metadata={**base.metadata, "episode_id": base.id}, ) ) return result + + async def fetch_by_entry_ids( + self, entry_ids: list[str], where: str + ) -> list[Candidate]: + """Fetch episodes by entry_id (for facts whose parent_id is an entry_id).""" + if not entry_ids: + return [] + table = await get_table(Episode.TABLE_NAME, Episode) + quoted = ", ".join(f"'{_q(eid)}'" for eid in entry_ids) + full_where = f"({where}) AND (entry_id IN ({quoted}))" + rows = await table.query().where(full_where).limit(len(entry_ids)).to_list() + return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/knowledge_topic.py b/src/everos/memory/search/recall/knowledge_topic.py new file mode 100644 index 0000000..ea484e1 --- /dev/null +++ b/src/everos/memory/search/recall/knowledge_topic.py @@ -0,0 +1,128 @@ +"""KnowledgeTopic recaller — dual-column BM25 + cosine ANN. + +The schema declares two BM25 columns (``summary_tokens`` — primary anchor — +and ``content_tokens`` — secondary detail match). LanceDB's +``nearest_to_text`` searches one column at a time, so we run the BM25 query +twice in parallel and merge by row id keeping the max score across columns. +Vector recall is single-shot over the ``summary`` embedding. + +Mirrors :class:`AgentCaseRecaller` structurally — both kinds share the +multi-BM25-column pattern. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence +from typing import ClassVar + +from everalgo.types import Candidate + +from everos.infra.persistence.lancedb import KnowledgeTopic, get_table + +from .base import ( + RecallerDeps, + build_or_query_multi_column, + cosine_score_from_distance, + row_to_candidate, +) + + +def _merge_bm25_results( + per_column: tuple[list[dict], ...], + *, + limit: int, +) -> list[dict]: + """Merge multi-column BM25 results by id, keeping max score.""" + best: dict[str, dict] = {} + for rows in per_column: + for r in rows: + rid = r.get("id") + if not isinstance(rid, str): + continue + score = float(r.get("_score", 0.0)) + existing = best.get(rid) + if existing is None or score > float(existing.get("_score", 0.0)): + merged = dict(r) + merged["_score"] = score + best[rid] = merged + return sorted( + best.values(), + key=lambda r: float(r.get("_score", 0.0)), + reverse=True, + )[:limit] + + +class KnowledgeTopicRecaller: + """BM25 (dual-column) + vector recall over the LanceDB ``knowledge_topic`` table. + + Args: + deps: Shared recaller dependencies (tokenizer, embedding provider). + """ + + kind: ClassVar[str] = "knowledge_topic" + everalgo_memory_type: ClassVar[str] = "knowledge" + text_field: ClassVar[str] = "summary" + + def __init__(self, deps: RecallerDeps) -> None: + self._deps = deps + + async def sparse_recall( + self, query: str, where: str, *, limit: int + ) -> list[Candidate]: + """Dual-column BM25 recall via OR-mode BooleanQuery per column. + + Queries ``summary_tokens`` (primary) and ``content_tokens`` + (secondary) in parallel. Results merge by id, keeping the max + BM25 score across the two columns. This ensures that a topic + matching the query in either its summary or its content body is + surfaced without double-counting. + """ + column_queries = build_or_query_multi_column( + self._deps.tokenizer, query, KnowledgeTopic.BM25_FIELDS + ) + if column_queries is None: + return [] + table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) + + async def _query_one(column: str) -> list[dict]: + return ( + await table.query() + .nearest_to_text(column_queries[column]) + .where(where) + .limit(limit) + .to_list() + ) + + per_column = await asyncio.gather( + *(_query_one(col) for col in KnowledgeTopic.BM25_FIELDS), + ) + merged_rows = _merge_bm25_results(per_column, limit=limit) + return [ + row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) + for r in merged_rows + ] + + async def dense_recall( + self, vector: Sequence[float], where: str, *, limit: int + ) -> list[Candidate]: + """Cosine ANN over the ``summary`` vector (1024-d).""" + if not vector: + return [] + table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) + rows = ( + await table.query() + .nearest_to(list(vector)) + .distance_type("cosine") + .where(where) + .limit(limit) + .to_list() + ) + return [ + row_to_candidate( + r, + source="vector", + score=cosine_score_from_distance(r.get("_distance")), + ) + for r in rows + ] diff --git a/src/everos/memory/search/shaper.py b/src/everos/memory/search/shaper.py index 6652bc0..14bfce6 100644 --- a/src/everos/memory/search/shaper.py +++ b/src/everos/memory/search/shaper.py @@ -66,7 +66,7 @@ def shape_episode_from_candidate( return None session_id = md.get("session_id") episode = md.get("episode") - if not isinstance(session_id, str) or not isinstance(episode, str): + if not isinstance(episode, str): logger.warning("shape_episode_missing_required_field", id=candidate.id) return None return SearchEpisodeItem( diff --git a/src/everos/memory/strategies/__init__.py b/src/everos/memory/strategies/__init__.py index e4b6ea1..1509bdb 100644 --- a/src/everos/memory/strategies/__init__.py +++ b/src/everos/memory/strategies/__init__.py @@ -7,6 +7,7 @@ External usage: extract_atomic_facts, extract_foresight, extract_user_profile, + reflect_episodes, trigger_profile_clustering, trigger_skill_clustering, ) @@ -17,6 +18,7 @@ from .extract_agent_skill import extract_agent_skill as extract_agent_skill from .extract_atomic_facts import extract_atomic_facts as extract_atomic_facts from .extract_foresight import extract_foresight as extract_foresight from .extract_user_profile import extract_user_profile as extract_user_profile +from .reflect_episodes import reflect_episodes as reflect_episodes from .trigger_profile_clustering import ( trigger_profile_clustering as trigger_profile_clustering, ) @@ -30,6 +32,7 @@ __all__ = [ "extract_atomic_facts", "extract_foresight", "extract_user_profile", + "reflect_episodes", "trigger_profile_clustering", "trigger_skill_clustering", ] diff --git a/src/everos/memory/strategies/extract_agent_skill.py b/src/everos/memory/strategies/extract_agent_skill.py index 07b3b5f..ceccf5d 100644 --- a/src/everos/memory/strategies/extract_agent_skill.py +++ b/src/everos/memory/strategies/extract_agent_skill.py @@ -39,8 +39,8 @@ from everalgo.types import AgentCase as AlgoAgentCase from everalgo.types import AgentSkill as AlgoAgentSkill from everos.component.embedding import ( - EmbeddingError, EmbeddingNotConfiguredError, + EmbeddingServiceError, get_embedder, ) from everos.component.llm import get_llm_client @@ -64,8 +64,8 @@ from everos.infra.persistence.markdown import ( AgentSkillWriter, ) from everos.infra.persistence.sqlite import cluster_repo +from everos.memory._partition_locks import get_partition_lock from everos.memory.events import SkillClusterUpdated -from everos.memory.strategies._partition_locks import get_partition_lock logger = get_logger(__name__) @@ -272,7 +272,7 @@ async def _resolve_query_vector(target: LanceAgentCase) -> list[float]: try: embedder = get_embedder() return list(await embedder.embed(target.task_intent)) - except (EmbeddingNotConfiguredError, EmbeddingError) as exc: + except (EmbeddingNotConfiguredError, EmbeddingServiceError) as exc: logger.warning( "agent_skill_query_embed_failed", case_entry_id=target.entry_id, diff --git a/src/everos/memory/strategies/extract_atomic_facts.py b/src/everos/memory/strategies/extract_atomic_facts.py index a1f7458..b00ae44 100644 --- a/src/everos/memory/strategies/extract_atomic_facts.py +++ b/src/everos/memory/strategies/extract_atomic_facts.py @@ -1,32 +1,14 @@ -"""extract_atomic_facts strategy — derive AtomicFacts from a fresh MemCell. +"""extract_atomic_facts strategy — derive AtomicFacts from an Episode. -One LLM call per memcell, then md-level fan-out to every user sender. -Mirrors :class:`UserMemoryPipeline`'s Episode handling: the algo -prompt is subject-agnostic (``INPUT_TEXT`` + ``TIME`` only, no -``sender_id`` placeholder — see -``everalgo.user_memory.atomic_fact.AtomicFactExtractor.aextract``), so -calling it once per sender would waste LLM tokens and let non- -determinism drift the per-sender md files apart. Instead, run the -extractor once with ``sender_id=None`` (algo's "generic owner" -signal) and rebroadcast the same fact list under each user sender. - -Per-owner batching: each sender's full fact list is appended in one -batched ``append_entries`` call rather than ``len(algo_facts)`` single -appends, dropping the per-cell IO complexity from ``O(N²)`` to -``O(N)`` (one read + one write per owner instead of N of each) and -narrowing the per-path lock window from N read-modify-write cycles to -one. - -Note ``extract_foresight`` does run per-sender because its prompt -template *does* condition on the target sender; do not collapse that -strategy in the same way without re-checking the prompt. +Triggered per :class:`EpisodeExtracted` event (one per episode per +sender). Uses :meth:`AtomicFactExtractor.aextract_from_text` to extract +facts from the episode narrative. Each event carries a single +``owner_id``; all facts are written under that owner in one batched +:meth:`append_entries` call. """ from __future__ import annotations -from collections import defaultdict -from collections.abc import Mapping - from everalgo.user_memory import AtomicFactExtractor from everos.component.llm import get_llm_client @@ -37,7 +19,7 @@ from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate from everos.infra.persistence.markdown import AtomicFactWriter -from everos.memory.events import UserPipelineStarted +from everos.memory.events import EpisodeExtracted from everos.memory.models import AtomicFact logger = get_logger(__name__) @@ -46,6 +28,7 @@ _writer: AtomicFactWriter | None = None def _get_writer() -> AtomicFactWriter: + """Return the lazily-initialised AtomicFactWriter singleton.""" global _writer if _writer is None: _writer = AtomicFactWriter(root=MemoryRoot.default()) @@ -54,63 +37,51 @@ def _get_writer() -> AtomicFactWriter: @offline_strategy( name="extract_atomic_facts", - trigger=Immediate(on=[UserPipelineStarted]), + trigger=Immediate(on=[EpisodeExtracted]), emits=[], max_retries=2, ) -async def extract_atomic_facts( - event: UserPipelineStarted, ctx: StrategyContext -) -> None: - # 1. List the user senders in this memcell; bail early if there are none. - memcell = event.memcell - sender_ids = sorted({m.sender_id for m in memcell.items if m.role == "user"}) - if not sender_ids: +async def extract_atomic_facts(event: EpisodeExtracted, ctx: StrategyContext) -> None: + """Extract atomic facts from an episode and persist as markdown entries.""" + # 1. Run LLM extractor on episode text. + extractor = AtomicFactExtractor(llm=get_llm_client()) + algo_facts = await extractor.aextract_from_text( + event.episode_text, timestamp=event.episode_timestamp_ms + ) + if not algo_facts: logger.info( "atomic_facts_extracted", memcell_id=event.memcell_id, session_id=event.session_id, count=0, - owner_ids=[], + owner_id=event.owner_id, ) return - # 2. Run the LLM extractor once (algo prompt is subject-agnostic). - extractor = AtomicFactExtractor(llm=get_llm_client()) - algo_facts = await extractor.aextract(memcell, sender_id=None) - - # 3. Fan the fact list out to one domain AtomicFact per (sender, algo_fact). + # 2. Build domain AtomicFacts (single owner from event). facts: list[AtomicFact] = [ AtomicFact.from_algo( - algo_fact, - owner_id=sid, + af, + owner_id=event.owner_id, session_id=event.session_id, - parent_id=event.memcell_id, + parent_id=event.episode_entry_id, ) - for sid in sender_ids - for algo_fact in algo_facts + for af in algo_facts ] - # 4. Group facts by owner so each sender's full list lands in one - # batched write. - by_owner: dict[str, list[tuple[Mapping[str, object], Mapping[str, str]]]] = ( - defaultdict(list) - ) - for fact in facts: - by_owner[fact.owner_id].append(_atomic_fact_to_entry_body(fact)) - - # 5. Write each owner's full list with one batched append_entries. + # 3. Write all facts in one batched append. writer = _get_writer() - for owner_id, items in by_owner.items(): - await writer.append_entries( - owner_id, items, app_id=event.app_id, project_id=event.project_id - ) + items = [_atomic_fact_to_entry_body(f) for f in facts] + await writer.append_entries( + event.owner_id, items, app_id=event.app_id, project_id=event.project_id + ) logger.info( "atomic_facts_extracted", memcell_id=event.memcell_id, session_id=event.session_id, count=len(facts), - owner_ids=sender_ids, + owner_id=event.owner_id, ) @@ -126,10 +97,11 @@ def _atomic_fact_to_entry_body( """ inline: dict[str, object] = { "owner_id": fact.owner_id, - "session_id": fact.session_id, "timestamp": to_iso_format(from_timestamp(fact.timestamp)), - "parent_type": "memcell", + "parent_type": "episode", "parent_id": fact.parent_id, } + if fact.session_id is not None: + inline["session_id"] = fact.session_id sections = {"Fact": fact.fact} return inline, sections diff --git a/src/everos/memory/strategies/extract_user_profile.py b/src/everos/memory/strategies/extract_user_profile.py index 9903a60..c4c9a6e 100644 --- a/src/everos/memory/strategies/extract_user_profile.py +++ b/src/everos/memory/strategies/extract_user_profile.py @@ -37,14 +37,15 @@ from everos.core.persistence import MemoryRoot from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate +from everos.infra.persistence.lancedb import episode_repo from everos.infra.persistence.markdown import ( ProfileReader, ProfileWriter, UserProfileFrontmatter, ) from everos.infra.persistence.sqlite import cluster_repo, memcell_repo +from everos.memory._partition_locks import get_partition_lock from everos.memory.events import ProfileClusterUpdated -from everos.memory.strategies._partition_locks import get_partition_lock logger = get_logger(__name__) @@ -127,20 +128,34 @@ async def extract_user_profile( if not target_clusters: return - # 3. Bail if the candidate set is too thin to be worth an LLM call. - member_ids = [m for c in target_clusters for m in c.members] - if len(member_ids) < PROFILE_MIN_MEMCELLS: + # 3. Resolve cluster members (episode entry_ids) → memcell_ids. + # Cluster members store episode entry_ids. To reach the memcell + # payloads we look up each episode's parent_id (= memcell_id) + # in LanceDB, skipping merged episodes (parent_type=cluster) + # whose source memcells were already processed before Reflection. + entry_ids = [m for c in target_clusters for m in c.members] + episodes = await episode_repo.find_by_owner_entries( + event.owner_id, + entry_ids, + app_id=event.app_id, + project_id=event.project_id, + ) + memcell_ids = [ + ep.parent_id + for ep in episodes + if ep.parent_type == "memcell" and ep.parent_id + ] + if len(memcell_ids) < PROFILE_MIN_MEMCELLS: logger.info( "profile_extraction_below_min_memcells", owner_id=event.owner_id, - memcell_count=len(member_ids), + memcell_count=len(memcell_ids), threshold=PROFILE_MIN_MEMCELLS, ) return - # 4. Pull memcell payloads from SQLite, rehydrate to algo types, - # time-sort. - memcell_rows = await memcell_repo.find_by_ids(member_ids) + # 4. Pull memcell payloads from SQLite, rehydrate to algo types. + memcell_rows = await memcell_repo.find_by_ids(memcell_ids) algo_memcells = sorted( (AlgoMemCell.model_validate_json(r.payload_json) for r in memcell_rows), key=lambda mc: mc.timestamp, @@ -166,6 +181,7 @@ async def extract_user_profile( "user_profile_extracted", owner_id=event.owner_id, cluster_count=len(target_clusters), + episode_count=len(episodes), memcell_count=len(algo_memcells), mode="UPDATE" if old_profile is not None else "INIT", ) diff --git a/src/everos/memory/strategies/reflect_episodes.py b/src/everos/memory/strategies/reflect_episodes.py new file mode 100644 index 0000000..8f72f2a --- /dev/null +++ b/src/everos/memory/strategies/reflect_episodes.py @@ -0,0 +1,89 @@ +"""reflect_episodes Cron strategy — nightly Reflection consolidation. + +Triggered by a cron schedule (default: ``0 2 * * 1``). Enumerates all +distinct owner scopes from the cluster table and runs the +:class:`ReflectionOrchestrator` for each. Configuration lives in +``[reflection]`` of ``config/default.toml``. + +The strategy is a thin entry point: it constructs the orchestrator with +production singletons and iterates over owners. All business logic +lives in :mod:`everos.memory.reflection.orchestrator`. +""" + +from __future__ import annotations + +import asyncio + +from everos.component.embedding import get_embedder +from everos.component.llm import get_llm_client +from everos.core.observability.logging import get_logger +from everos.core.persistence import MemoryRoot +from everos.infra.ome.context import StrategyContext +from everos.infra.ome.decorator import offline_strategy +from everos.infra.ome.events import CronTick +from everos.infra.ome.triggers import Cron +from everos.infra.persistence.lancedb import ( + atomic_fact_repo, + episode_repo, +) +from everos.infra.persistence.markdown import EpisodeWriter +from everos.infra.persistence.sqlite import ( + cluster_repo, + reflection_report_repo, +) +from everos.memory.events import EpisodeExtracted +from everos.memory.reflection import ReflectionOrchestrator + +logger = get_logger(__name__) + +_episode_writer: EpisodeWriter | None = None + + +def _get_episode_writer() -> EpisodeWriter: + """Return the lazily-initialised EpisodeWriter singleton.""" + global _episode_writer + if _episode_writer is None: + _episode_writer = EpisodeWriter(root=MemoryRoot.default()) + return _episode_writer + + +@offline_strategy( + name="reflect_episodes", + trigger=Cron(expr="0 2 * * 1"), + emits=[EpisodeExtracted], + enabled=False, + max_retries=1, +) +async def reflect_episodes(event: CronTick, ctx: StrategyContext) -> None: + """Run Reflection for all owner scopes. + + Args: + event: Cron tick event (unused; triggers the scheduled run). + ctx: OME strategy context for emit and logging. + """ + # Deferred: avoid pulling LLM libs at module import time. + from everalgo.user_memory import EpisodeReflector + + orchestrator = ReflectionOrchestrator( + cluster_repo=cluster_repo, + episode_store=episode_repo, + atomic_fact_store=atomic_fact_repo, + episode_writer=_get_episode_writer(), + report_repo=reflection_report_repo, + reflector=EpisodeReflector(llm=get_llm_client()), + embedder=get_embedder(), + ) + + owners = await cluster_repo.list_distinct_owners() + await asyncio.gather( + *( + orchestrator.run( + ctx=ctx, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + ) + for owner_id, owner_type, app_id, project_id in owners + ) + ) diff --git a/src/everos/memory/strategies/trigger_profile_clustering.py b/src/everos/memory/strategies/trigger_profile_clustering.py index 28c06af..83b8f00 100644 --- a/src/everos/memory/strategies/trigger_profile_clustering.py +++ b/src/everos/memory/strategies/trigger_profile_clustering.py @@ -1,18 +1,11 @@ -"""trigger_profile_clustering strategy — group user memcells by episode topic. +"""trigger_profile_clustering strategy — group user episodes by topic. Listens to :class:`EpisodeExtracted` (emitted per-episode after the user pipeline writes its md), embeds the ``episode_text``, and merges the resulting size-1 :class:`everalgo.clustering.Cluster` into the user's existing user-memory cluster set. -Profile-track parity with opensource: uses :func:`cluster_by_geometry` -(rather than the LLM-refined variant) — opensource routes -``has_case=False`` (user-memory) memcells through the embedding-only -path. The members on the merged cluster are ``memcell_id`` rather than -``episode_entry_id`` because the downstream profile-extraction step -needs to feed full memcells (chat messages) back into -:class:`everalgo.user_memory.ProfileExtractor`, not the per-sender -episode summaries. +Uses :func:`cluster_by_geometry` (embedding-only cosine + time-window). """ from __future__ import annotations @@ -22,13 +15,14 @@ from everalgo.clustering import Cluster as AlgoCluster from everalgo.clustering import cluster_by_geometry from everos.component.embedding import get_embedder +from everos.config import load_settings from everos.core.observability.logging import get_logger from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate from everos.infra.persistence.sqlite import cluster_repo, mint_cluster_id +from everos.memory._partition_locks import get_partition_lock from everos.memory.events import EpisodeExtracted, ProfileClusterUpdated -from everos.memory.strategies._partition_locks import get_partition_lock logger = get_logger(__name__) @@ -37,6 +31,7 @@ logger = get_logger(__name__) name="trigger_profile_clustering", trigger=Immediate(on=[EpisodeExtracted]), emits=[ProfileClusterUpdated], + applies_to=lambda e: e.source == "pipeline", max_retries=2, ) async def trigger_profile_clustering( @@ -62,14 +57,14 @@ async def trigger_profile_clustering( project_id=event.project_id, ) - # 3. Build a size-1 cluster for the fresh memcell (id minted upfront). + # 3. Build a size-1 cluster for the new episode. new_cluster = AlgoCluster( id=mint_cluster_id(), centroid=vector, count=1, last_ts=event.episode_timestamp_ms, preview=[event.episode_text], - members=[event.memcell_id], + members=[event.episode_entry_id], ) # 4. Geometry-merge it into an existing cluster (or keep as-is). @@ -77,7 +72,13 @@ async def trigger_profile_clustering( # time-window math, no I/O) returning ``Cluster | None`` directly, so # it must not be awaited (``await None`` raises when there is no # existing cluster to merge into). - merged = cluster_by_geometry(new_cluster, existing) + settings = load_settings() + merged = cluster_by_geometry( + new_cluster, + existing, + threshold=settings.clustering.threshold, + time_window_days=settings.clustering.time_window_days, + ) to_save = merged if merged is not None else new_cluster # 5. Persist the (possibly-merged) cluster back to SQLite. @@ -86,7 +87,7 @@ async def trigger_profile_clustering( owner_id=event.owner_id, owner_type="user", kind="user_memory", - member_type="memcell", + member_type="episode", app_id=event.app_id, project_id=event.project_id, ) diff --git a/src/everos/memory/strategies/trigger_skill_clustering.py b/src/everos/memory/strategies/trigger_skill_clustering.py index 01683ee..151d43c 100644 --- a/src/everos/memory/strategies/trigger_skill_clustering.py +++ b/src/everos/memory/strategies/trigger_skill_clustering.py @@ -30,8 +30,8 @@ from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate from everos.infra.persistence.sqlite import cluster_repo, mint_cluster_id +from everos.memory._partition_locks import get_partition_lock from everos.memory.events import AgentCaseExtracted, SkillClusterUpdated -from everos.memory.strategies._partition_locks import get_partition_lock logger = get_logger(__name__) diff --git a/src/everos/service/__init__.py b/src/everos/service/__init__.py index 6bb059d..2f801cf 100644 --- a/src/everos/service/__init__.py +++ b/src/everos/service/__init__.py @@ -5,16 +5,79 @@ command or API endpoint maps to one service method. External usage: from everos.service import MemorizeResult, get, memorize, search + from everos.service import ( + CategoryOverview, CreateDocumentResult, DuplicateDocumentError, + ExtractionEmptyError, + create_document, DocumentContext, DocumentDetail, DocumentListResult, + DocumentNotFoundError, DocumentOverviewItem, DeleteResult, + PatchResult, SearchHit, SearchKnowledgeResult, + TopicDetail, TopicOverview, TopicNotFoundError, + delete_document, get_document, get_topic, list_categories, + list_documents, patch_document, replace_document, search_knowledge, + ) """ +from everos.core.errors import DocumentNotFoundError as DocumentNotFoundError +from everos.core.errors import DuplicateDocumentError as DuplicateDocumentError +from everos.core.errors import ExtractionEmptyError as ExtractionEmptyError +from everos.core.errors import TopicNotFoundError as TopicNotFoundError + from .get import get as get +from .knowledge import CategoryOverview as CategoryOverview +from .knowledge import CreateDocumentResult as CreateDocumentResult +from .knowledge import DeleteResult as DeleteResult +from .knowledge import DocumentContext as DocumentContext +from .knowledge import DocumentDetail as DocumentDetail +from .knowledge import DocumentListResult as DocumentListResult +from .knowledge import DocumentOverviewItem as DocumentOverviewItem +from .knowledge import PatchResult as PatchResult +from .knowledge import SearchHit as SearchHit +from .knowledge import SearchKnowledgeResult as SearchKnowledgeResult +from .knowledge import TopicDetail as TopicDetail +from .knowledge import TopicOverview as TopicOverview +from .knowledge import compile_knowledge_where as compile_knowledge_where +from .knowledge import create_document as create_document +from .knowledge import delete_document as delete_document +from .knowledge import get_document as get_document +from .knowledge import get_topic as get_topic +from .knowledge import list_categories as list_categories +from .knowledge import list_documents as list_documents +from .knowledge import patch_document as patch_document +from .knowledge import replace_document as replace_document +from .knowledge import search_knowledge as search_knowledge from .memorize import MemorizeResult as MemorizeResult from .memorize import memorize as memorize from .search import search as search __all__ = [ + "CategoryOverview", + "CreateDocumentResult", + "DeleteResult", + "DuplicateDocumentError", + "DocumentContext", + "DocumentDetail", + "DocumentListResult", + "DocumentNotFoundError", + "DocumentOverviewItem", + "ExtractionEmptyError", "MemorizeResult", + "PatchResult", + "SearchHit", + "SearchKnowledgeResult", + "TopicDetail", + "TopicNotFoundError", + "TopicOverview", + "compile_knowledge_where", + "create_document", + "delete_document", "get", + "get_document", + "get_topic", + "list_categories", + "list_documents", "memorize", + "patch_document", + "replace_document", "search", + "search_knowledge", ] diff --git a/src/everos/service/knowledge.py b/src/everos/service/knowledge.py new file mode 100644 index 0000000..a73f9df --- /dev/null +++ b/src/everos/service/knowledge.py @@ -0,0 +1,1329 @@ +"""Knowledge document CRUD + search use cases. + +Functions: + create_document — full upload-document pipeline. + get_document — fetch document detail with topic list. + list_documents — paginated document listing. + get_topic — fetch a single topic with content. + delete_document — remove document directory (cascade handles SQLite/LanceDB). + replace_document — atomic replace (backup + restore on failure). + patch_document — update mutable document metadata fields. + search_knowledge — knowledge retrieval pipeline (keyword / vector / hybrid). +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +import re +import shutil +import time +from collections.abc import Sequence +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING, Protocol +from uuid import uuid4 + +import anyio +from everalgo.rank.fusion import rrf +from everalgo.types import Candidate, CategorySpec, KnowledgeMemory, ParsedContent + +from everos.component.utils.datetime import get_utc_now +from everos.core.errors import ( + ConfigurationError, + DocumentNotFoundError, + DuplicateDocumentError, + ExtractionEmptyError, + TopicNotFoundError, +) +from everos.core.observability.logging import get_logger +from everos.core.persistence import MemoryRoot +from everos.core.persistence.markdown import dump_frontmatter, parse_frontmatter +from everos.infra.persistence.markdown import ( + KnowledgeWriter, + ensure_taxonomy, + parse_taxonomy, +) +from everos.infra.persistence.sqlite import ( + DocumentUpsertPayload, + knowledge_document_repo, + knowledge_topic_sqlite_repo, +) + +if TYPE_CHECKING: + from everos.component.embedding import EmbeddingProvider + from everos.component.rerank import RerankProvider + from everos.config.settings import KnowledgeSearchSettings + from everos.memory.search.recall import KnowledgeTopicRecaller + +logger = get_logger(__name__) + +_FALLBACK_CATEGORY = "Others" +_DOC_ID_PREFIX = "d_" +_DOC_ID_HEX_LEN = 12 +_MAX_MINT_RETRIES = 5 +_SCOPE_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_.\-@+]+$") +_ORIGINAL_DIR_NAME = "_original" + + +class KnowledgeExtractor(Protocol): + """Structural type for the algo extractor. + + Matches ``everalgo.knowledge.KnowledgeExtractor.aextract`` so + callers can pass any compatible implementation (including test + doubles). + """ + + async def aextract( + self, + parsed: ParsedContent, + *, + doc_id: str, + title: str, + categories: list[CategorySpec] | None = None, + category_id: str = "", + ) -> list[KnowledgeMemory]: ... + + +@dataclasses.dataclass(frozen=True) +class CreateDocumentResult: + """Returned by :func:`create_document` on success.""" + + doc_id: str + category_id: str + topic_count: int + source_name: str | None + md_path: str + original_file_path: str | None = None + + +async def _extract_memories( + extractor: KnowledgeExtractor, + parsed: ParsedContent, + doc_id: str, + title: str, + *, + categories: list[CategorySpec], + category_id: str | None, +) -> list[KnowledgeMemory]: + """Run the algo extractor and return non-empty memories. + + Args: + extractor: Algo extractor injected by the caller. + parsed: Parsed document content to extract from. + doc_id: Document identifier passed to the extractor. + title: Human-readable document title. + categories: Taxonomy categories for LLM classification. + category_id: Pre-known category; empty string lets the LLM classify. + + Returns: + Non-empty list of KnowledgeMemory with category fallback applied. + + Raises: + ExtractionEmptyError: When the extractor produces no memories. + """ + memories: list[KnowledgeMemory] = await extractor.aextract( + parsed, + doc_id=doc_id, + title=title, + categories=categories, + category_id=category_id or "", + ) + if not memories: + raise ExtractionEmptyError( + f"Extractor returned no memories for doc_id={doc_id!r}" + ) + return _apply_category_fallback(memories) + + +async def create_document( + *, + extractor: KnowledgeExtractor, + parsed: ParsedContent, + title: str, + knowledge_dir: Path, + source_name: str | None = None, + source_type: str | None = None, + doc_id: str | None = None, + category_id: str | None = None, + file_content: bytes | None = None, +) -> CreateDocumentResult: + """Create a knowledge document from parsed content. + + Args: + extractor: Algo extractor injected by the caller. + parsed: Parsed document content (from parser or raw text). + title: Human-readable document title. + knowledge_dir: Absolute path from ``MemoryRoot.knowledge_dir``. + source_name: Optional provenance label (URL, filename, ...). + source_type: Optional provenance type (``"url"``, ``"file"``, ...). + doc_id: Caller-provided doc_id; ``None`` mints a new one. + category_id: Pre-known category; ``None`` lets the LLM classify. + file_content: Raw uploaded file bytes to persist in ``_original/``. + + Returns: + Result containing doc_id, category, topic count, and md path. + + Raises: + DuplicateDocumentError: When ``doc_id`` already exists. + ExtractionEmptyError: When the extractor produces no memories. + """ + doc_id = doc_id or await _mint_doc_id() + if await knowledge_document_repo.doc_id_exists(doc_id): + raise DuplicateDocumentError( + f"Document {doc_id!r} already exists; use PUT to replace" + ) + return await _write_document( + extractor=extractor, + parsed=parsed, + title=title, + knowledge_dir=knowledge_dir, + source_name=source_name, + source_type=source_type, + doc_id=doc_id, + category_id=category_id, + file_content=file_content, + ) + + +async def _write_document( + *, + extractor: KnowledgeExtractor, + parsed: ParsedContent, + title: str, + knowledge_dir: Path, + source_name: str | None, + source_type: str | None, + doc_id: str, + category_id: str | None, + file_content: bytes | None = None, +) -> CreateDocumentResult: + """Extract topics and write markdown — shared by create and replace.""" + await ensure_taxonomy(knowledge_dir) + categories = await parse_taxonomy(knowledge_dir / ".taxonomy.md") + + memories = await _extract_memories( + extractor, + parsed, + doc_id, + title, + categories=categories, + category_id=category_id, + ) + resolved_category = memories[0].category_id + + md_path = await KnowledgeWriter.write( + memories, + knowledge_dir, + source_name=source_name, + source_type=source_type, + ) + + original_file_path: str | None = None + if file_content and source_name: + written = await _write_original_file(md_path, source_name, file_content) + original_file_path = str(written) + + topic_count = sum(1 for m in memories if m.topic_index != 0) + + logger.info( + "document created", + doc_id=doc_id, + category_id=resolved_category, + topic_count=topic_count, + ) + + return CreateDocumentResult( + doc_id=doc_id, + category_id=resolved_category, + topic_count=topic_count, + source_name=source_name, + md_path=str(md_path), + original_file_path=original_file_path, + ) + + +# ── CRUD result types ───────────────────────────────────────────────────────── + + +@dataclasses.dataclass(frozen=True) +class TopicOverview: + """Minimal topic summary embedded in :class:`DocumentDetail`.""" + + topic_id: str + topic_name: str + topic_path: str + depth: int + summary: str + + +@dataclasses.dataclass(frozen=True) +class DocumentDetail: + """Full document record returned by :func:`get_document`.""" + + doc_id: str + category_id: str + title: str + summary: str + source_name: str | None + source_type: str | None + original_file_path: str | None + topics: list[TopicOverview] + created_at: datetime + updated_at: datetime + + +@dataclasses.dataclass(frozen=True) +class DocumentOverviewItem: + """One row in a paginated document list.""" + + doc_id: str + category_id: str + title: str + topic_count: int + created_at: datetime + + +@dataclasses.dataclass(frozen=True) +class DocumentListResult: + """Paginated document list returned by :func:`list_documents`.""" + + documents: list[DocumentOverviewItem] + total: int + page: int + page_size: int + + +@dataclasses.dataclass(frozen=True) +class TopicDetail: + """Full topic record returned by :func:`get_topic`.""" + + topic_id: str + doc_id: str + category_id: str + topic_name: str + topic_path: str + depth: int + summary: str + content: str + content_labels: list[str] + parent_topic_id: str | None + children_topic_ids: list[str] + created_at: datetime + updated_at: datetime + + +@dataclasses.dataclass(frozen=True) +class CategoryOverview: + """One taxonomy category with its document count.""" + + category_id: str + description: str + document_count: int + + +@dataclasses.dataclass(frozen=True) +class DeleteResult: + """Result of :func:`delete_document`.""" + + doc_id: str + deleted_topics: int + + +@dataclasses.dataclass(frozen=True) +class PatchResult: + """Result of :func:`patch_document`.""" + + doc_id: str + updated_fields: list[str] + updated_at: datetime + + +# ── CRUD functions ──────────────────────────────────────────────────────────── + + +async def get_document( + doc_id: str, + app_id: str, + project_id: str, +) -> DocumentDetail: + """Fetch a document with its topic list. + + Args: + doc_id: Document primary key. + app_id: Tenant application identifier. + project_id: Tenant project identifier. + + Returns: + DocumentDetail with all topics. + + Raises: + DocumentNotFoundError: When no row exists for ``doc_id``. + """ + row = await knowledge_document_repo.get_by_doc_id(doc_id) + if row is None: + raise DocumentNotFoundError(f"Document {doc_id!r} not found") + + topic_rows = await knowledge_topic_sqlite_repo.get_topics_by_doc_id(doc_id) + topics = [ + TopicOverview( + topic_id=t.node_id, + topic_name=t.topic_name, + topic_path=t.topic_path, + depth=t.depth, + summary=t.summary, + ) + for t in topic_rows + ] + + original_file_path = await _resolve_original_file_path(row.md_path, row.source_name) + + return DocumentDetail( + doc_id=row.doc_id, + category_id=row.category_id, + title=row.title, + summary=row.summary, + source_name=row.source_name, + source_type=row.source_type, + original_file_path=original_file_path, + topics=topics, + created_at=row.created_at, + updated_at=row.updated_at, + ) + + +async def list_documents( + app_id: str, + project_id: str, + category_id: str | None = None, + *, + page: int = 1, + page_size: int = 20, + sort_by: str = "created_at", + sort_order: str = "desc", +) -> DocumentListResult: + """Return a paginated document list with per-document topic counts. + + Args: + app_id: Tenant application identifier. + project_id: Tenant project identifier. + category_id: Optional category filter. + page: 1-based page number. + page_size: Rows per page. + sort_by: Sort column — ``created_at``, ``updated_at``, or ``title``. + sort_order: ``"asc"`` or ``"desc"``. + + Returns: + DocumentListResult with items, total, page, and page_size. + """ + page_result = await knowledge_document_repo.list_documents( + app_id=app_id, + project_id=project_id, + category_id=category_id, + page=page, + page_size=page_size, + sort_by=sort_by, + sort_order=sort_order, + ) + + doc_ids = [r.doc_id for r in page_result.rows] + counts = await asyncio.gather( + *[knowledge_topic_sqlite_repo.count_by_doc_id(did) for did in doc_ids] + ) + topic_counts = dict(zip(doc_ids, counts, strict=True)) + + items = [ + DocumentOverviewItem( + doc_id=r.doc_id, + category_id=r.category_id, + title=r.title, + topic_count=topic_counts.get(r.doc_id, 0), + created_at=r.created_at, + ) + for r in page_result.rows + ] + + return DocumentListResult( + documents=items, + total=page_result.total, + page=page, + page_size=page_size, + ) + + +async def get_topic( + topic_id: str, + app_id: str, + project_id: str, +) -> TopicDetail: + """Fetch a single topic with full content. + + Args: + topic_id: Topic node_id primary key. + app_id: Tenant application identifier (unused but part of scoped API). + project_id: Tenant project identifier (unused but part of scoped API). + + Returns: + TopicDetail with parsed JSON list fields. + + Raises: + TopicNotFoundError: When no row exists for ``topic_id``. + """ + rows = await knowledge_topic_sqlite_repo.get_topics_by_ids([topic_id]) + if not rows: + raise TopicNotFoundError(f"Topic {topic_id!r} not found") + + t = rows[0] + children = json.loads(t.children_node_ids) if t.children_node_ids else [] + labels = json.loads(t.content_labels) if t.content_labels else [] + + return TopicDetail( + topic_id=t.node_id, + doc_id=t.doc_id, + category_id=t.category_id, + topic_name=t.topic_name, + topic_path=t.topic_path, + depth=t.depth, + summary=t.summary, + content=t.content, + content_labels=labels, + parent_topic_id=t.parent_node_id, + children_topic_ids=children, + created_at=t.created_at, + updated_at=t.updated_at, + ) + + +async def delete_document( + doc_id: str, + app_id: str, + project_id: str, +) -> DeleteResult: + """Remove a document directory; cascade handles SQLite/LanceDB cleanup. + + Idempotent: returns ``deleted_topics=0`` when the document does not exist. + + Args: + doc_id: Document primary key. + app_id: Tenant application identifier. + project_id: Tenant project identifier. + + Returns: + DeleteResult with the topic count that was present before deletion. + """ + row = await knowledge_document_repo.get_by_doc_id(doc_id) + if row is None: + return DeleteResult(doc_id=doc_id, deleted_topics=0) + + topic_count = await knowledge_topic_sqlite_repo.count_by_doc_id(doc_id) + + memory_root = MemoryRoot.default() + doc_dir = memory_root.root / Path(row.md_path).parent + if await anyio.Path(doc_dir).is_dir(): + await anyio.to_thread.run_sync(shutil.rmtree, doc_dir) + + logger.info( + "document deleted", + doc_id=doc_id, + topic_count=topic_count, + ) + return DeleteResult(doc_id=doc_id, deleted_topics=topic_count) + + +async def replace_document( + *, + extractor: KnowledgeExtractor, + parsed: ParsedContent, + title: str, + doc_id: str, + knowledge_dir: Path, + source_name: str | None = None, + source_type: str | None = None, + category_id: str | None = None, + file_content: bytes | None = None, +) -> CreateDocumentResult: + """Replace a document atomically — old data preserved on failure. + + The replacement is performed in-place: the md directory is backed up, + then ``create_document`` overwrites both md files and SQLite rows via + upsert. No explicit SQLite delete happens before the write, so a + failure during extraction leaves the database intact and the backup + restore brings the md directory back to its original state. + + Args: + extractor: Algo extractor injected by the caller. + parsed: Parsed document content. + title: Human-readable document title. + doc_id: Existing document id to replace. + knowledge_dir: Absolute path from ``MemoryRoot.knowledge_dir``. + source_name: Optional provenance label. + source_type: Optional provenance type. + category_id: Pre-known category or ``None`` for LLM classification. + file_content: Raw uploaded file bytes to persist in ``_original/``. + + Returns: + CreateDocumentResult for the new document. + + Raises: + DocumentNotFoundError: When *doc_id* does not exist. + ExtractionEmptyError: When extraction produces no memories. + The original document is restored in this case. + Exception: All extraction-pipeline errors propagate after the + backup directory is restored. + """ + row = await knowledge_document_repo.get_by_doc_id(doc_id) + if row is None: + raise DocumentNotFoundError(f"Document {doc_id!r} not found") + + backup = await _backup_doc_dir(doc_id) + try: + result = await _write_document( + extractor=extractor, + parsed=parsed, + title=title, + knowledge_dir=knowledge_dir, + source_name=source_name, + source_type=source_type, + doc_id=doc_id, + category_id=category_id, + file_content=file_content, + ) + except Exception: + await _restore_backup(backup, doc_id) + raise + + await _drop_backup(backup) + return result + + +async def _backup_doc_dir(doc_id: str) -> tuple[Path, Path] | None: + """Move the existing document directory to a hidden backup path. + + Returns ``(backup_path, original_path)`` so callers can restore + without reverse-engineering the name, or ``None`` when no directory + exists. + """ + memory_root = MemoryRoot.default() + row = await knowledge_document_repo.get_by_doc_id(doc_id) + if row is None: + return None + + original_dir = memory_root.root / Path(row.md_path).parent + if not await anyio.Path(original_dir).is_dir(): + return None + + backup_dir = original_dir.with_name(f".{original_dir.name}.backup") + await anyio.to_thread.run_sync(shutil.move, str(original_dir), str(backup_dir)) + return backup_dir, original_dir + + +async def _restore_backup( + backup: tuple[Path, Path] | None, + doc_id: str, +) -> None: + """Restore a backup created by :func:`_backup_doc_dir`. + + No-op when ``backup`` is ``None`` or the backup no longer exists. + """ + if backup is None: + return + backup_dir, original_dir = backup + if not await anyio.Path(backup_dir).is_dir(): + return + + await anyio.to_thread.run_sync(shutil.move, str(backup_dir), str(original_dir)) + logger.info("document_backup_restored", doc_id=doc_id) + + +async def _drop_backup(backup: tuple[Path, Path] | None) -> None: + """Remove the backup directory after a successful replacement. + + No-op when ``backup`` is ``None`` or already absent. + """ + if backup is None: + return + backup_dir, _ = backup + if await anyio.Path(backup_dir).is_dir(): + await anyio.to_thread.run_sync(shutil.rmtree, str(backup_dir)) + + +async def _locate_index_md( + knowledge_dir: Path, + doc_id: str, +) -> Path | None: + """Scan knowledge_dir to find the index.md whose frontmatter doc_id matches. + + Used when the SQLite row is absent (cascade hasn't processed the file yet). + Returns the absolute index.md path, or None if not found. + """ + adir = anyio.Path(knowledge_dir) + if not await adir.exists(): + return None + async for index_md in adir.rglob("index.md"): + text = await index_md.read_text(encoding="utf-8") + fm, _ = parse_frontmatter(text) + if fm.get("doc_id") == doc_id: + # anyio.Path → stdlib Path for callers that need synchronous ops. + return Path(str(index_md)) + return None + + +async def _update_index_frontmatter( + index_path: Path, + title: str, + category_id: str, +) -> None: + """Rewrite index.md frontmatter with updated title/category, preserving body.""" + apath = anyio.Path(index_path) + text = await apath.read_text(encoding="utf-8") + fm, body = parse_frontmatter(text) + fm["title"] = title + fm["category_id"] = category_id + await apath.write_text(dump_frontmatter(fm) + body, encoding="utf-8") + + +_DIR_SAFE = re.compile(r"[^\w\-.]", re.UNICODE) + + +def _safe_category(raw: str) -> str: + """Sanitize category_id for use as a directory name component.""" + slug = raw.replace(" ", "_") + slug = _DIR_SAFE.sub("", slug)[:50] + return slug or "Others" + + +async def _move_doc_directory( + memory_root: MemoryRoot, + old_md_path: str, + new_category: str, +) -> str: + """Move document directory to new category folder, return new md_path.""" + old_index = memory_root.root / old_md_path + old_dir = old_index.parent + new_dir = old_dir.parent.parent / _safe_category(new_category) / old_dir.name + await anyio.Path(new_dir.parent).mkdir(parents=True, exist_ok=True) + await anyio.to_thread.run_sync(shutil.move, str(old_dir), str(new_dir)) + new_index = new_dir / "index.md" + return str(new_index.relative_to(memory_root.root)) + + +async def _update_topics_category(doc_dir: Path, new_category: str) -> None: + """Rewrite category_id in all topic md files within doc_dir.""" + entries = await anyio.to_thread.run_sync(lambda: sorted(doc_dir.iterdir())) + topic_files = [f for f in entries if f.suffix == ".md" and f.name != "index.md"] + + async def _rewrite(path: Path) -> None: + apath = anyio.Path(path) + text = await apath.read_text(encoding="utf-8") + fm, body = parse_frontmatter(text) + if fm.get("category_id") == new_category: + return + fm["category_id"] = new_category + await apath.write_text(dump_frontmatter(fm) + body, encoding="utf-8") + + await asyncio.gather(*[_rewrite(f) for f in topic_files]) + + +@dataclasses.dataclass(frozen=True) +class _ResolvedDoc: + """Snapshot of current document state used by :func:`patch_document`.""" + + title: str + category_id: str + md_path: str + app_id: str + project_id: str + summary: str + source_name: str | None + source_type: str | None + + +async def _resolve_current_doc( + doc_id: str, + app_id: str, + project_id: str, + memory_root: MemoryRoot, +) -> _ResolvedDoc: + """Resolve the authoritative document state for patching. + + Raises: + DocumentNotFoundError: When neither SQLite nor md contain ``doc_id``. + """ + row = await knowledge_document_repo.get_by_doc_id(doc_id) + if row is not None: + return _ResolvedDoc( + title=row.title, + category_id=row.category_id, + md_path=row.md_path, + app_id=row.app_id, + project_id=row.project_id, + summary=row.summary, + source_name=row.source_name, + source_type=row.source_type, + ) + + # Cascade may not have synced yet — scan md as fallback. + knowledge_dir = memory_root.knowledge_dir(app_id, project_id) + index_md = await _locate_index_md(knowledge_dir, doc_id) + if index_md is None: + raise DocumentNotFoundError(f"Document {doc_id!r} not found") + raw = await anyio.Path(index_md).read_text(encoding="utf-8") + fm, _ = parse_frontmatter(raw) + return _ResolvedDoc( + title=fm.get("title", ""), + category_id=fm.get("category_id", ""), + md_path=str(index_md.relative_to(memory_root.root)), + app_id=app_id, + project_id=project_id, + summary="", + source_name=None, + source_type=None, + ) + + +async def _apply_patch_writes( + doc_id: str, + current: _ResolvedDoc, + new_title: str, + *, + new_category: str, + new_md_path: str, + memory_root: MemoryRoot, +) -> str: + """Write md frontmatter, move directory if needed, upsert SQLite. + + Returns the (possibly updated) md_path after a category move. + """ + index_path = memory_root.root / current.md_path + await _update_index_frontmatter(index_path, new_title, new_category) + + if new_category != current.category_id: + new_md_path = await _move_doc_directory( + memory_root, current.md_path, new_category + ) + new_doc_dir = memory_root.root / Path(new_md_path).parent + await _update_topics_category(new_doc_dir, new_category) + + await knowledge_document_repo.upsert_from_handler( + DocumentUpsertPayload( + doc_id=doc_id, + app_id=current.app_id, + project_id=current.project_id, + category_id=new_category, + title=new_title, + summary=current.summary, + source_name=current.source_name, + source_type=current.source_type, + md_path=new_md_path, + ) + ) + return new_md_path + + +async def patch_document( + doc_id: str, + app_id: str, + project_id: str, + title: str | None = None, + category_id: str | None = None, +) -> PatchResult: + """Update mutable document metadata in md (truth) and SQLite (immediate). + + Args: + doc_id: Document primary key. + app_id: Tenant application identifier. + project_id: Tenant project identifier. + title: New title, or ``None`` to leave unchanged. + category_id: New category, or ``None`` to leave unchanged. + + Returns: + PatchResult listing updated fields. + + Raises: + DocumentNotFoundError: When neither SQLite nor md files contain ``doc_id``. + """ + memory_root = MemoryRoot.default() + current = await _resolve_current_doc(doc_id, app_id, project_id, memory_root) + + new_title = title if title is not None else current.title + new_category = category_id if category_id is not None else current.category_id + + updated_fields: list[str] = [] + if new_title != current.title: + updated_fields.append("title") + if new_category != current.category_id: + updated_fields.append("category_id") + + if not updated_fields: + return PatchResult(doc_id=doc_id, updated_fields=[], updated_at=get_utc_now()) + + await _apply_patch_writes( + doc_id, + current, + new_title, + new_category=new_category, + new_md_path=current.md_path, + memory_root=memory_root, + ) + now = get_utc_now() + return PatchResult(doc_id=doc_id, updated_fields=updated_fields, updated_at=now) + + +async def list_categories(app_id: str, project_id: str) -> list[CategoryOverview]: + """List taxonomy categories with per-category document counts. + + Creates the taxonomy file if it does not yet exist. + + Args: + app_id: Tenant application identifier. + project_id: Tenant project identifier. + + Returns: + List of CategoryOverview with document counts from SQLite. + """ + knowledge_dir = MemoryRoot.default().knowledge_dir(app_id, project_id) + await ensure_taxonomy(knowledge_dir) + specs = await parse_taxonomy(knowledge_dir / ".taxonomy.md") + counts = await knowledge_document_repo.count_by_category(app_id, project_id) + return [ + CategoryOverview( + category_id=s.id, + description=s.description, + document_count=counts.get(s.id, 0), + ) + for s in specs + ] + + +async def _mint_doc_id() -> str: + """Generate a unique ``d_`` document id. + + Retries up to ``_MAX_MINT_RETRIES`` times if the id already exists + in SQLite (astronomically unlikely with 48-bit hex, but defensive). + """ + for _ in range(_MAX_MINT_RETRIES): + candidate = f"{_DOC_ID_PREFIX}{uuid4().hex[:_DOC_ID_HEX_LEN]}" + if not await knowledge_document_repo.doc_id_exists(candidate): + return candidate + # Last-resort: use full uuid hex to avoid infinite loops. + return f"{_DOC_ID_PREFIX}{uuid4().hex}" + + +def _apply_category_fallback( + memories: list[KnowledgeMemory], +) -> list[KnowledgeMemory]: + """Replace empty ``category_id`` with the fallback category.""" + patched: list[KnowledgeMemory] = [] + for m in memories: + if not m.category_id: + m = m.model_copy(update={"category_id": _FALLBACK_CATEGORY}) + patched.append(m) + return patched + + +async def _resolve_original_file_path( + md_path: str, source_name: str | None +) -> str | None: + """Derive the original file path from md_path and source_name. + + Returns the absolute path string if the file exists on disk, + ``None`` otherwise (legacy documents or missing source_name). + """ + if not source_name: + return None + memory_root = MemoryRoot.default() + doc_dir = memory_root.root / Path(md_path).parent + candidate = doc_dir / _ORIGINAL_DIR_NAME / source_name + if await anyio.Path(candidate).is_file(): + return str(candidate) + return None + + +async def _write_original_file( + doc_dir: Path, source_name: str, file_content: bytes +) -> Path: + """Write the uploaded binary to ``_original/`` and return its path.""" + original_dir = doc_dir / _ORIGINAL_DIR_NAME + await anyio.Path(original_dir).mkdir(parents=True, exist_ok=True) + target = original_dir / source_name + await anyio.Path(target).write_bytes(file_content) + return target + + +# ── Knowledge search ───────────────────────────────────────────────────────── + +# Lazy singleton — mirrors the pattern in service/search.py. +_embedding: EmbeddingProvider | None = None +_embedding_resolved = False + + +def _get_embedding() -> EmbeddingProvider | None: + """Build the embedding client on first call. ``None`` when not configured.""" + global _embedding, _embedding_resolved # noqa: PLW0603 + if _embedding_resolved: + return _embedding + + from everos.component.embedding import ( # Deferred: singleton + build_embedding_provider, + ) + from everos.config import load_settings # Deferred: singleton + + cfg = load_settings().embedding + if not cfg.model or cfg.api_key is None: + logger.warning( + "knowledge_embedding_not_configured", + hint="set [embedding] model / api_key to enable vector / hybrid search", + ) + _embedding = None + else: + _embedding = build_embedding_provider(cfg) + logger.info("knowledge_embedding_built", model=cfg.model) + _embedding_resolved = True + return _embedding + + +# Lazy singleton — mirrors the pattern for _embedding above. +_recaller: KnowledgeTopicRecaller | None = None +_recaller_resolved = False + + +def _build_recaller() -> KnowledgeTopicRecaller: + """Return the shared :class:`KnowledgeTopicRecaller`, building it on first call.""" + global _recaller, _recaller_resolved # noqa: PLW0603 + if _recaller_resolved: + return _recaller # type: ignore[return-value] -- guarded by _recaller_resolved + + from everos.component.tokenizer import ( # Deferred: singleton + build_tokenizer, + ) + from everos.memory.search.recall import ( # Deferred: singleton + KnowledgeTopicRecaller, + RecallerDeps, + ) + + _recaller = KnowledgeTopicRecaller(RecallerDeps(tokenizer=build_tokenizer())) + _recaller_resolved = True + return _recaller + + +# Lazy singleton — mirrors the pattern for _embedding above. +_reranker: RerankProvider | None = None +_reranker_resolved = False + + +def _get_reranker() -> RerankProvider | None: + """Build the rerank client on first call. ``None`` when not configured.""" + global _reranker, _reranker_resolved # noqa: PLW0603 + if _reranker_resolved: + return _reranker + + from everos.component.rerank import ( # Deferred: singleton + build_rerank_provider, + ) + from everos.config import load_settings # Deferred: singleton + + cfg = load_settings().rerank + if not cfg.model or not cfg.base_url: + _reranker = None + else: + _reranker = build_rerank_provider(cfg) + logger.info("knowledge_reranker_built", model=cfg.model, provider=cfg.provider) + _reranker_resolved = True + return _reranker + + +# ── Search result types ────────────────────────────────────────────────────── + + +@dataclasses.dataclass(frozen=True) +class DocumentContext: + """L1 document metadata attached to every :class:`SearchHit`.""" + + doc_id: str + title: str + summary: str + + +@dataclasses.dataclass(frozen=True) +class SearchHit: + """One ranked result from :func:`search_knowledge`.""" + + topic_id: str + category_id: str + topic_name: str + topic_path: str + depth: int + summary: str + content: str | None + score: float + retrieval_method: str + source: str | None + document: DocumentContext + + +@dataclasses.dataclass(frozen=True) +class SearchKnowledgeResult: + """Envelope returned by :func:`search_knowledge`.""" + + hits: list[SearchHit] + total: int + took_ms: float + + +# ── Where-clause builder ───────────────────────────────────────────────────── + + +def _validate_scope_id(value: str, name: str) -> None: + """Reject scope ids with characters that could break LanceDB SQL. + + Args: + value: The identifier value to validate. + name: The parameter name (for error messages). + + Raises: + ValueError: If the value is empty or contains invalid characters. + """ + if not value or not _SCOPE_ID_PATTERN.match(value): + raise ValueError(f"{name} contains invalid characters: {value!r}") + + +def compile_knowledge_where(app_id: str, project_id: str) -> str: + """Build a LanceDB ``where`` clause scoped to the given tenant. + + Args: + app_id: Tenant application identifier. + project_id: Tenant project identifier. + + Returns: + SQL-style predicate string safe for use in LanceDB ``where`` parameter. + + Raises: + ValueError: If either id contains invalid characters. + """ + _validate_scope_id(app_id, "app_id") + _validate_scope_id(project_id, "project_id") + + def _esc(v: str) -> str: + return v.replace("'", "''") + + return f"app_id = '{_esc(app_id)}' AND project_id = '{_esc(project_id)}'" + + +# ── Recall helpers ─────────────────────────────────────────────────────────── + + +async def _base_retrieve( + recaller: KnowledgeTopicRecaller, + where: str, + *, + method: str, + query: str, + vector: list[float], + limit: int, +) -> list[Candidate]: + """Run the appropriate recall path and return ranked candidates.""" + if method == "keyword": + return await recaller.sparse_recall(query, where, limit=limit) + if method == "vector": + return await recaller.dense_recall(vector, where, limit=limit) + # hybrid: parallel sparse + dense, fuse with RRF + sparse, dense = await asyncio.gather( + recaller.sparse_recall(query, where, limit=limit), + recaller.dense_recall(vector, where, limit=limit), + ) + return rrf(sparse, dense)[:limit] + + +async def _enrich_with_content( + candidates: list[Candidate], +) -> list[Candidate]: + """Batch-fetch SQLite content and attach to candidate metadata. + + Acts as a no-reranker path: returns candidates in their original + order with ``content`` added to metadata for downstream hit + conversion. + """ + if not candidates: + return candidates + + topic_ids = [c.id for c in candidates] + topics = await knowledge_topic_sqlite_repo.get_topics_by_ids(topic_ids) + content_map = {t.node_id: t.content for t in topics} + + return [ + c.model_copy( + update={ + "metadata": {**c.metadata, "content": content_map.get(c.id, "")}, + } + ) + for c in candidates + ] + + +async def _to_search_hits( + candidates: list[Candidate], + include_content: bool, + method: str, +) -> list[SearchHit]: + """Convert ranked ``Candidate`` list into ``SearchHit`` DTOs.""" + if not candidates: + return [] + + doc_ids = {c.metadata.get("doc_id", "") for c in candidates} - {""} + docs = await knowledge_document_repo.get_documents_by_ids(doc_ids) + doc_map = {d.doc_id: d for d in docs} + + hits: list[SearchHit] = [] + for c in candidates: + doc = doc_map.get(c.metadata.get("doc_id", "")) + content = c.metadata.get("content", "") if include_content else None + source = c.source if c.source != "other" else None + hits.append( + SearchHit( + topic_id=c.id, + category_id=c.metadata.get("category_id", ""), + topic_name=c.metadata.get("topic_name", ""), + topic_path=c.metadata.get("topic_path", ""), + depth=int(c.metadata.get("depth", 0)), + summary=c.metadata.get("summary", ""), + content=content, + score=c.score, + retrieval_method=method, + source=source, + document=DocumentContext( + doc_id=doc.doc_id if doc else "", + title=doc.title if doc else "", + summary=doc.summary if doc else "", + ), + ) + ) + return hits + + +# ── Public search entry point ──────────────────────────────────────────────── + + +def _require_search_providers() -> tuple[EmbeddingProvider, RerankProvider]: + """Return embedding + reranker providers, raising if not configured. + + Raises: + ConfigurationError: When the embedding or rerank provider is not + configured (a required setting is missing). + """ + embedder = _get_embedding() + if embedder is None: + raise ConfigurationError( + "Embedding provider not configured. " + "Set EVEROS_EMBEDDING__MODEL and EVEROS_EMBEDDING__API_KEY." + ) + reranker = _get_reranker() + if reranker is None: + raise ConfigurationError( + "Rerank provider not configured. " + "Set EVEROS_RERANK__MODEL and EVEROS_RERANK__BASE_URL." + ) + return embedder, reranker + + +async def _run_category_pipeline( + query: str, + where: str, + *, + method: str, + vector: list[float], + reranker: RerankProvider, + config: KnowledgeSearchSettings, + top_k: int, +) -> list[Candidate]: + """Execute the full acategory_retrieve pipeline.""" + from everalgo.rank import acategory_retrieve # Deferred: heavy dep + + from everos.memory.search.callbacks import ( # Deferred: heavy dep + build_rerank_fn, + ) + + recaller = _build_recaller() + + async def _retrieve(q: str, k: int) -> list[Candidate]: + return await _base_retrieve( + recaller, where, method=method, query=q, vector=vector, limit=k + ) + + raw_rerank = build_rerank_fn(reranker, text_field="content") + + async def _rerank_with_enrich( + q: str, candidates: Sequence[Candidate] + ) -> list[Candidate]: + enriched = await _enrich_with_content(list(candidates)) + return await raw_rerank(q, enriched) + + effective_k = min(top_k, config.top_k_cap) + return await acategory_retrieve( + query, + base_retrieve=_retrieve, + rerank_fn=_rerank_with_enrich, + recall_n=config.recall_n, + rerank_n=config.rerank_n, + mass_top_m=config.mass_top_m, + lam=config.lam, + top_n=effective_k, + ) + + +async def search_knowledge( + *, + query: str, + method: str = "hybrid", + top_k: int = 10, + score_threshold: float | None = None, + include_content: bool = False, + app_id: str = "default", + project_id: str = "default", +) -> SearchKnowledgeResult: + """Search knowledge topics by keyword, vector, or hybrid retrieval. + + Args: + query: User search query string. + method: Retrieval mode — ``"keyword"``, ``"vector"``, or ``"hybrid"``. + top_k: Maximum hits to return. + score_threshold: Drop candidates scoring below this value. + include_content: When ``True``, populate ``SearchHit.content``. + app_id: Tenant application identifier. + project_id: Tenant project identifier. + + Returns: + SearchKnowledgeResult with ranked hits and timing. + """ + from everos.config import load_settings # Deferred: singleton + + t0 = time.monotonic() + config = load_settings().knowledge.search + where = compile_knowledge_where(app_id, project_id) + + embedder, reranker = _require_search_providers() + vector = await embedder.embed(query) + + ranked = await _run_category_pipeline( + query, + where, + method=method, + vector=vector, + reranker=reranker, + config=config, + top_k=top_k, + ) + + if score_threshold is not None: + ranked = [c for c in ranked if c.score >= score_threshold] + hits = await _to_search_hits(ranked, include_content, method) + + took_ms = (time.monotonic() - t0) * 1000 + logger.info( + "knowledge_search_complete", + method=method, + query_len=len(query), + hits=len(hits), + took_ms=round(took_ms, 1), + ) + return SearchKnowledgeResult(hits=hits, total=len(hits), took_ms=took_ms) diff --git a/src/everos/service/memorize.py b/src/everos/service/memorize.py index c7df2be..a92a04b 100644 --- a/src/everos/service/memorize.py +++ b/src/everos/service/memorize.py @@ -47,6 +47,7 @@ from everos.memory.strategies import ( extract_atomic_facts, extract_foresight, extract_user_profile, + reflect_episodes, trigger_profile_clustering, trigger_skill_clustering, ) @@ -135,6 +136,7 @@ def _get_engine() -> OfflineEngine: engine.register(extract_agent_skill) engine.register(trigger_profile_clustering) engine.register(extract_user_profile) + engine.register(reflect_episodes) _ome_engine = engine return _ome_engine diff --git a/src/everos/templates/env.template b/src/everos/templates/env.template index fe392d2..11ada68 100755 --- a/src/everos/templates/env.template +++ b/src/everos/templates/env.template @@ -4,23 +4,19 @@ # ===================================================== # # Setup: -# 1. Create .env with `everos init` or `cp .env.example .env` +# 1. cp env.template .env # 2. Edit .env with your values # 3. .env is gitignored (never commit) # # Override priority (low → high): # src/everos/config/default.toml (shipped baseline) # ↓ -# ~/.everos/config.toml (user-level overrides; optional) +# /everos.toml (user config; optional; root resolved +# by EVEROS_ROOT env > ~/.everos) # ↓ -# .env (this file; gitignored) -# ↓ -# EVEROS_
__ process envs +# EVEROS_
__ process envs (this file sources these) # ↓ # programmatic init args / CLI flags -# -# The user-level toml path defaults to ~/.everos/config.toml; override -# with EVEROS_CONFIG_FILE=/path/to/your.toml. Missing file is skipped. # ===================================================== @@ -65,7 +61,7 @@ EVEROS_MULTIMODAL__BASE_URL=https://openrouter.ai/api/v1 # ─── Embedding (OpenAI-protocol /embeddings) ───────── # Any OpenAI-compatible embedding endpoint plugs in via base_url. # model / api_key / base_url have no shipped default — set them here -# or in ~/.everos/config.toml before the embedding capability is used. +# or in /everos.toml before the embedding capability is used. EVEROS_EMBEDDING__MODEL=Qwen/Qwen3-Embedding-4B EVEROS_EMBEDDING__API_KEY= @@ -109,10 +105,10 @@ EVEROS_RERANK__BASE_URL=https://api.deepinfra.com/v1/inference # ─── Storage paths ─────────────────────────────────── # memory-root holds md files + .index/ (LanceDB) + .system.db (SQLite) + ... -# Override the default with EVEROS_MEMORY__ROOT (note the double-underscore -# for nested config keys); see config/default.toml for all tunables. +# Override the default (~/.everos) with EVEROS_ROOT; also controls which +# everos.toml is loaded. See config/default.toml for all other tunables. -# EVEROS_MEMORY__ROOT=~/.everos +# EVEROS_ROOT=~/.everos # ─── HTTP API ──────────────────────────────────────── diff --git a/tests/_consistency_assertions.py b/tests/_consistency_assertions.py index fc3693f..5890cba 100644 --- a/tests/_consistency_assertions.py +++ b/tests/_consistency_assertions.py @@ -97,7 +97,7 @@ async def assert_md_lance_strict_consistent( Args: memory_root: Absolute path to the memory root directory - (e.g. the value of ``EVEROS_MEMORY__ROOT`` / + (e.g. the value of ``EVEROS_ROOT`` / ``MemoryRoot.root``). expect_at_least: Optional ``{kind_name: min_md_files}`` map. Raises ``AssertionError`` if a listed kind has fewer md diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 3adf228..f9359c3 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -22,7 +22,7 @@ The ``long_conversation`` fixture (LoCoMo conv_0) lives in Conventions: - ``.env`` is loaded at import time (before any everos module reads - settings) — overrides for ``EVEROS_MEMORY__ROOT`` happen per-test. + settings) — overrides for ``EVEROS_ROOT`` happen per-test. - This file does **not** define ``cascade_runtime`` — that name belongs to ``tests/integration/test_cascade_integration.py``'s local fixture. The pipeline test uses ``core_pipeline_runtime`` to avoid name @@ -35,6 +35,7 @@ import asyncio import importlib import json from collections.abc import AsyncIterator, Awaitable, Callable +from importlib import resources from pathlib import Path import httpx @@ -76,7 +77,7 @@ _STRATEGY_SINGLETONS: tuple[tuple[str, tuple[str, ...]], ...] = ( def _reset_strategy_singletons(monkeypatch: pytest.MonkeyPatch) -> None: """Null every strategy ``_writer`` / ``_reader`` so the next test rebuilds against its own ``MemoryRoot.default()`` (driven by the - fresh ``EVEROS_MEMORY__ROOT`` env var set by the calling fixture). + fresh ``EVEROS_ROOT`` env var set by the calling fixture). """ for mod_name, attrs in _STRATEGY_SINGLETONS: mod = importlib.import_module(mod_name) @@ -122,7 +123,10 @@ async def core_pipeline_runtime( Keeps real LLM / embedding settings from ``.env`` (do NOT overwrite ``EVEROS_LLM__*`` or ``EVEROS_EMBEDDING__*``). """ - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + + default_ome = resources.files("everos.config").joinpath("default_ome.toml") + (tmp_path / "ome.toml").write_text(default_ome.read_text(encoding="utf-8")) from everos.config import load_settings diff --git a/tests/e2e/test_get_endpoint_e2e.py b/tests/e2e/test_get_endpoint_e2e.py index 3a3d9a8..55af098 100644 --- a/tests/e2e/test_get_endpoint_e2e.py +++ b/tests/e2e/test_get_endpoint_e2e.py @@ -1,7 +1,7 @@ """End-to-end integration tests for ``POST /api/v1/memory/get``. These tests spin up the FastAPI app with **no lifespan providers** -against a tmp ``EVEROS_MEMORY__ROOT``, populate a real LanceDB +against a tmp ``EVEROS_ROOT``, populate a real LanceDB ``episode`` table directly via the repo singleton, and exercise the HTTP route. They cover the wiring that unit tests cannot: pydantic 422s from the route, JSON envelope shape, and the full @@ -130,7 +130,7 @@ async def client( monkeypatch: pytest.MonkeyPatch, ) -> AsyncIterator[AsyncClient]: """Build the FastAPI app against a tmp memory root with no lifespan.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) load_settings.cache_clear() # Reset every module-level singleton the get-path touches. diff --git a/tests/e2e/test_knowledge_e2e.py b/tests/e2e/test_knowledge_e2e.py new file mode 100644 index 0000000..19dc8a5 --- /dev/null +++ b/tests/e2e/test_knowledge_e2e.py @@ -0,0 +1,283 @@ +"""Knowledge HTTP e2e — all 9 endpoints via real LLM + full HTTP stack. + +Drives every knowledge API endpoint end-to-end with a **real LLM**: + + GET /categories → taxonomy auto-generation + POST /documents → real LLM extraction → md → cascade + GET /documents/{doc_id} → document detail with topics + GET /topics/{topic_id} → topic content + POST /search (keyword) → BM25 retrieval + POST /search (vector) → ANN retrieval + POST /search (hybrid) → RRF fusion + POST /search (include_content) → content enrichment + PATCH /documents/{doc_id} → metadata update + GET /documents → paginated listing + PUT /documents/{doc_id} → replace (delete + recreate) + DELETE /documents/{doc_id} → cleanup + +Uses ``httpx.AsyncClient`` against ``create_app()`` with full lifespan +(SQLite + LanceDB + Cascade + OME). + +Marked ``live_llm`` + ``slow`` — requires ``EVEROS_LLM__*`` + +``EVEROS_EMBEDDING__*`` credentials in ``.env``. +""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +# --------------------------------------------------------------------------- +# Test document — 3 clear sections for predictable topic extraction. +# --------------------------------------------------------------------------- + +_TEST_DOCUMENT = """\ +# 2028 Los Angeles Olympics Budget Plan + +## Venue Construction +The venue construction program covers 12 new facilities and 8 renovated \ +sites across greater Los Angeles. Total venue construction budget is \ +estimated at $5.3 billion, with the Intuit Dome as the centerpiece for \ +basketball events. Temporary overlay structures account for $800 million. + +## Transportation Infrastructure +A comprehensive transportation plan connects all venue clusters via \ +dedicated Olympic lanes. The LAX-to-Downtown express shuttle operates \ +24 hours during Games time. Budget allocation for transportation is \ +$1.2 billion including temporary bus fleet leases. + +## Security Operations +Multi-agency security coordination involves LAPD, FBI, and DHS. \ +Cybersecurity operations center monitors digital threats 24/7. \ +Drone detection perimeter extends 30 miles around the Olympic Village. \ +Security budget is $2.1 billion. +""" + +_TEST_TITLE = "2028 LA Olympics Budget Plan" +_PREFIX = "/api/v1/knowledge" + + +# --------------------------------------------------------------------------- +# Fixtures — reuse the shared e2e conftest (async_client + lifespan) +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.live_llm +@pytest.mark.slow +async def test_knowledge_full_http_lifecycle( + async_client: httpx.AsyncClient, + cascade_done_poll, +) -> None: + """Full HTTP lifecycle: create → get → search → list → delete. + + Validates every HTTP endpoint with real LLM extraction output, + checking response shapes, status codes, and semantic correctness + against the known input document. + """ + client = async_client + + # ── 1. GET /categories — taxonomy auto-generated ────────────── + resp = await client.get(f"{_PREFIX}/categories") + assert resp.status_code == 200 + cats = resp.json()["data"]["categories"] + assert len(cats) >= 10, f"Expected >=10 default categories, got {len(cats)}" + cat_ids = {c["id"] for c in cats} + assert "Sports" in cat_ids + assert "Others" in cat_ids + + # ── 2. POST /documents — create with real LLM extraction ───── + resp = await client.post( + f"{_PREFIX}/documents", + data={"title": _TEST_TITLE, "source_type": "file"}, + files={"file": ("budget.md", _TEST_DOCUMENT.encode(), "text/plain")}, + ) + assert resp.status_code == 201, f"Create failed: {resp.text}" + create_data = resp.json()["data"] + doc_id = create_data["doc_id"] + assert doc_id.startswith("d_") + assert create_data["topic_count"] >= 2, ( + f"3-section doc should produce >= 2 topics, got {create_data['topic_count']}" + ) + assert create_data["category_id"], "LLM should assign a category" + assert create_data["source_name"] == "budget.md" + + # ── 3. Wait for cascade to index all topics ───────────────── + # The watcher needs time to detect new files, then the worker + # processes them. Poll the GET endpoint until topics appear. + expected_topics = create_data["topic_count"] + async with asyncio.timeout(60.0): + while True: + resp = await client.get(f"{_PREFIX}/documents/{doc_id}") + if resp.status_code == 200: + detail = resp.json()["data"] + if len(detail.get("topics", [])) >= expected_topics: + break + await asyncio.sleep(1.0) + # Extra settle for LanceDB FTS index. + await asyncio.sleep(1.0) + + # ── 4. GET /documents/{doc_id} — document detail ───────────── + assert detail["doc_id"] == doc_id + assert detail["title"] == _TEST_TITLE + assert len(detail["summary"]) > 30, "Document summary should be meaningful" + assert len(detail["topics"]) >= 2 + + # Verify topic names relate to the input document sections. + topic_names = [t["topic_name"].lower() for t in detail["topics"]] + found_sections = { + "venue": any("venue" in n for n in topic_names), + "transport": any("transport" in n for n in topic_names), + "security": any("security" in n for n in topic_names), + } + assert sum(found_sections.values()) >= 2, ( + f"Expected >= 2 of 3 sections in topic names, got: {topic_names}" + ) + + # All topics have topic_path and summary. + for t in detail["topics"]: + assert t["topic_path"], f"topic_path empty for {t['topic_id']}" + assert len(t["summary"]) > 10, f"summary too short for {t['topic_id']}" + + # ── 5. GET /topics/{topic_id} — topic detail with content ──── + first_topic = detail["topics"][0] + topic_id = first_topic["topic_id"] + resp = await client.get(f"{_PREFIX}/topics/{topic_id}") + assert resp.status_code == 200 + topic_detail = resp.json()["data"] + assert topic_detail["topic_id"] == topic_id + assert topic_detail["doc_id"] == doc_id + assert len(topic_detail["content"]) > 20, "Topic should have content body" + assert topic_detail["category_id"] == create_data["category_id"] + + # ── 6. POST /search — keyword search ───────────────────────── + resp = await client.post( + f"{_PREFIX}/search", + json={"query": "budget", "method": "keyword", "top_k": 10}, + ) + assert resp.status_code == 200 + search_data = resp.json()["data"] + assert search_data["total"] >= 1, "Keyword search 'budget' should find hits, got 0" + assert search_data["took_ms"] > 0 + + hit = search_data["hits"][0] + assert hit["score"] > 0 + assert hit["topic_id"] + assert hit["document"]["doc_id"] == doc_id + assert hit["document"]["title"] == _TEST_TITLE + + # ── 7. POST /search — include_content=true ─────────────────── + resp = await client.post( + f"{_PREFIX}/search", + json={ + "query": "venue construction", + "method": "keyword", + "top_k": 5, + "include_content": True, + }, + ) + assert resp.status_code == 200 + hits_with_content = resp.json()["data"]["hits"] + if hits_with_content: + assert hits_with_content[0]["content"], ( + "include_content=true should populate content field" + ) + + # ── 8. POST /search — vector search ───────────────────────── + resp = await client.post( + f"{_PREFIX}/search", + json={"query": "Olympic venue stadium", "method": "vector", "top_k": 5}, + ) + assert resp.status_code == 200 + vector_data = resp.json()["data"] + assert vector_data["total"] >= 1, "Vector search should find hits" + assert vector_data["hits"][0]["retrieval_method"] == "vector" + + # ── 9. POST /search — hybrid search ────────────────────────── + resp = await client.post( + f"{_PREFIX}/search", + json={"query": "security operations", "method": "hybrid", "top_k": 5}, + ) + assert resp.status_code == 200 + hybrid_data = resp.json()["data"] + assert hybrid_data["total"] >= 1, "Hybrid search should find hits" + + # ── 10. PATCH /documents/{doc_id} — update metadata ────────── + new_title = "Updated Olympics Budget 2028" + resp = await client.patch( + f"{_PREFIX}/documents/{doc_id}", + json={"title": new_title}, + ) + assert resp.status_code == 200 + patch_data = resp.json()["data"] + assert patch_data["doc_id"] == doc_id + assert "title" in patch_data["updated_fields"] + + # Verify title change persisted via GET. + resp = await client.get(f"{_PREFIX}/documents/{doc_id}") + assert resp.json()["data"]["title"] == new_title + + # ── 11. GET /documents — paginated listing ─────────────────── + resp = await client.get(f"{_PREFIX}/documents") + assert resp.status_code == 200 + list_data = resp.json()["data"] + assert list_data["total"] >= 1 + assert any(d["doc_id"] == doc_id for d in list_data["documents"]) + our_doc = next(d for d in list_data["documents"] if d["doc_id"] == doc_id) + assert our_doc["title"] == new_title + assert our_doc["topic_count"] >= 2 + + # ── 12. PUT /documents/{doc_id} — replace with new content ─── + replacement_doc = ( + "# Revised 2028 LA Olympics Plan\n\n" + "## Athlete Village\n" + "The athlete village will house 15,000 athletes in UCLA campus " + "facilities. Total village budget is $1.8 billion.\n" + ) + resp = await client.put( + f"{_PREFIX}/documents/{doc_id}", + data={"title": "Revised Olympics Plan"}, + files={"file": ("revised.md", replacement_doc.encode(), "text/plain")}, + ) + assert resp.status_code == 200, f"Replace failed: {resp.text}" + replace_data = resp.json()["data"] + assert replace_data["doc_id"] == doc_id, "PUT should preserve doc_id" + assert replace_data["topic_count"] >= 1 + + # Wait for cascade to fully cycle: old topics deleted + new topics indexed. + # Poll until GET returns new title AND topic names from the replacement doc. + async with asyncio.timeout(60.0): + while True: + resp = await client.get(f"{_PREFIX}/documents/{doc_id}") + if resp.status_code == 200: + d = resp.json()["data"] + names = [t["topic_name"].lower() for t in d.get("topics", [])] + has_new = any("village" in n or "athlete" in n for n in names) + has_old = any("venue" in n or "security" in n for n in names) + if d["title"] == "Revised Olympics Plan" and has_new and not has_old: + break + await asyncio.sleep(1.0) + + # Verify replaced content. + resp = await client.get(f"{_PREFIX}/documents/{doc_id}") + replaced = resp.json()["data"] + assert replaced["title"] == "Revised Olympics Plan" + + # ── 13. DELETE /documents/{doc_id} ──────────────────────────── + resp = await client.delete(f"{_PREFIX}/documents/{doc_id}") + assert resp.status_code == 200 + del_data = resp.json()["data"] + assert del_data["doc_id"] == doc_id + assert del_data["deleted_topics"] >= 1 + + # Note: cascade cleanup of SQLite/LanceDB rows is eventually + # consistent (scanner interval + FK constraint retry). The service + # DELETE removes md files immediately; the cascade handler catches + # up on subsequent scan passes. Integration tests verify cascade + # cleanup; this e2e test verifies the HTTP contract only. diff --git a/tests/e2e/test_search_endpoint_e2e.py b/tests/e2e/test_search_endpoint_e2e.py index 78eb3e5..508d35a 100644 --- a/tests/e2e/test_search_endpoint_e2e.py +++ b/tests/e2e/test_search_endpoint_e2e.py @@ -17,7 +17,7 @@ Coverage matrix (see 21_test_taxonomy_debate.md context): - include_profile (true / false) - filter DSL: session_id eq / timestamp range / sender_id in / parent_id (= memcell bridge) / top-level OR / nested AND-OR -- MRAG fact embedding: hybrid method embeds atomic_facts that share +- Hierarchical fact eviction: hybrid method embeds atomic_facts that share the matched episode's memcell parent Methods other than ``keyword`` require ``EMBEDDING_*`` creds in .env — @@ -76,7 +76,7 @@ async def client( from everos.core.persistence.sqlite import SQLModel as _SQLModel from everos.infra.persistence.sqlite import sqlite_manager - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) load_settings.cache_clear() # Lance: reset connection + cached table handles. @@ -370,7 +370,7 @@ async def test_vector_search_returns_episode_hits( for ep in data["episodes"]: assert ep["user_id"] == "caroline" assert ep["score"] > 0 # cosine similarity in [0, 1] - # vector path doesn't run MRAG, so no nested facts. + # vector path doesn't run hierarchical fact eviction, so no nested facts. assert ep["atomic_facts"] == [] @@ -419,15 +419,17 @@ async def test_agentic_search_returns_episode_hits( # ── Agent owner_type dispatch (separate path: agent_case + agent_skill) ─ -async def _seed_one_agent_corpus(owner: str = "a1") -> None: +async def _seed_one_agent_corpus( + owner: str = "a1", *, use_real_embeddings: bool = False +) -> None: """Single seed used by the parametrized agent dispatch test. One case + one skill sharing surface tokens with the test query ("refactor authentication") so BM25 deterministically hits both - tables; dense / agentic methods exercise the same rows. Both rows - are embedded with the real embedder so LanceDB's ``nearest_to`` - can rank them (zero vectors are undefined under cosine distance — - the dense path returns 0 hits for them). + tables. Dense / agentic methods exercise the same rows and opt into + real embeddings so LanceDB's ``nearest_to`` can rank them (zero + vectors are undefined under cosine distance — the dense path returns + 0 hits for them). """ from everos.service.search import _get_embedding @@ -436,16 +438,16 @@ async def _seed_one_agent_corpus(owner: str = "a1") -> None: skill_desc = "refactor authentication middleware reliably" skill_body = "step-by-step approach for auth refactors" - embedder = _get_embedding() + embedder = _get_embedding() if use_real_embeddings else None if embedder is not None: case_vec, skill_vec = await embedder.embed_batch( [f"{case_intent}\n{case_approach}", f"{skill_desc}\n{skill_body}"] ) else: - # No embedder credentials → leave zeros; only keyword assertions - # will pass, vector/hybrid/agentic methods are skipped anyway. - case_vec = [0.0] * 1024 - skill_vec = [0.0] * 1024 + # Keyword-only default runs offline in CI; live dense variants + # pass real embeddings via ``use_real_embeddings=True``. + case_vec = [1.0, *([0.0] * 1023)] + skill_vec = [1.0, *([0.0] * 1023)] await _seed_agent_cases( [ @@ -509,7 +511,7 @@ async def test_search_agent_dispatch_per_method( All methods must enforce the owner_type hard partition: ``episodes`` / ``profiles`` stay empty. """ - await _seed_one_agent_corpus() + await _seed_one_agent_corpus(use_real_embeddings=method != "keyword") resp = await _post( client, @@ -543,7 +545,7 @@ async def test_hybrid_with_llm_rerank_returns_hits( ) -> None: """``method=hybrid`` + ``enable_llm_rerank=true`` runs the phase-5 LLM pass. - Default hybrid stops after MRAG / LR fusion; opting in adds one + Default hybrid stops after hierarchical eviction / LR fusion; opting in adds one ``chat`` call that re-ranks the top-K. The route must accept the flag and still return well-formed episodes. """ @@ -1045,16 +1047,16 @@ async def test_search_filter_no_match_returns_empty( # ═══════════════════════════════════════════════════════════════════════ -# 7. MRAG fact embedding — the memcell-bridge contract +# 7. Hierarchical fact eviction — the memcell-bridge contract # ═══════════════════════════════════════════════════════════════════════ @pytest.mark.slow @pytest.mark.live_llm -async def test_search_hybrid_mrag_path_runs_with_memcell_facts( +async def test_search_hybrid_hierarchical_eviction_with_memcell_facts( client: AsyncClient, search_seed: dict ) -> None: - """HYBRID + MRAG path executes end-to-end with shared-memcell facts seeded. + """HYBRID + hierarchical eviction end-to-end with memcell facts. Verifies the wiring: - hybrid recall over episodes returns hits @@ -1065,7 +1067,7 @@ async def test_search_hybrid_mrag_path_runs_with_memcell_facts( asserted because ``atomic_fact_recaller.facts_for_episodes`` currently emits ``FactCandidate(score=0.0)`` for every prefetched fact (it's a parent_id lookup, not a query-aware recall). The - MRAG ``_expand_heap`` skips facts with non-positive scores, so + Hierarchical eviction ``_expand_heap`` skips facts with non-positive scores, so they never promote into the top-N. Once facts get a real query-aware relevance score (e.g. by running a separate dense recall on atomic_fact too), tighten this assertion to verify @@ -1099,14 +1101,14 @@ async def test_search_hybrid_mrag_path_runs_with_memcell_facts( @pytest.mark.slow @pytest.mark.live_llm -async def test_hybrid_mrag_injects_facts_with_alpha_zero( +async def test_hybrid_hierarchical_eviction_injects_facts_with_alpha_zero( client: AsyncClient, search_seed: dict, monkeypatch: pytest.MonkeyPatch, ) -> None: - """MRAG end-to-end fact injection, exercised with ``alpha=0``. + """Hierarchical eviction end-to-end fact injection, exercised with ``alpha=0``. - Companion to :func:`test_search_hybrid_mrag_path_runs_with_memcell_facts`. + Companion to :func:`test_search_hybrid_hierarchical_eviction_with_memcell_facts`. The sibling asserts the contract under prod defaults (``alpha=1`` × ``fact.score=0`` → final ≤ 0 → fact never enters the top-N). This test patches ``RankConfig.alpha=0`` so facts inherit @@ -1139,7 +1141,9 @@ async def test_hybrid_mrag_injects_facts_with_alpha_zero( assert data["episodes"], "hybrid should return at least one episode" facts_attached = sum(len(ep["atomic_facts"]) for ep in data["episodes"]) - assert facts_attached >= 1, "alpha=0 should let MRAG promote ≥1 fact into the top-N" + assert facts_attached >= 1, ( + "alpha=0 should let hierarchical eviction promote >=1 fact" + ) # Memcell-bridge invariant — every attached fact's parent_id must # match its host episode's parent_id. @@ -1286,7 +1290,7 @@ async def test_search_filter_error_returns_422( # FastAPI's default ``{"detail": ...}``). The FilterError text # lands in ``error.message``. body = resp.json() - assert body["error"]["code"] == "HTTP_ERROR" + assert body["error"]["code"] == "INVALID_INPUT" assert "this_field_does_not_exist" in body["error"]["message"] diff --git a/tests/fixtures/_dump_search_seed.py b/tests/fixtures/_dump_search_seed.py index b5762eb..24437b3 100644 --- a/tests/fixtures/_dump_search_seed.py +++ b/tests/fixtures/_dump_search_seed.py @@ -13,7 +13,7 @@ Sampling rules: bridge-consistent. - **atomic_fact**: every row whose ``parent_id`` is in the episode- parent set above, capped at 50 to keep the seed compact. This - guarantees MRAG-fusion testing can verify "facts sharing a + guarantees hierarchical-eviction testing can verify "facts sharing a memcell with the matched episode get embedded". - **foresight**: 5 per owner. Archived for future use; current ``/search`` does not query foresight, so the seed only exists so @@ -85,7 +85,7 @@ def main() -> None: parent_memcells.add(r["parent_id"]) # 2) atomic_facts — every fact whose parent_id is in the episode - # parent set, capped to keep the seed compact (and so MRAG + # parent set, capped to keep the seed compact (and so hierarchical # ``facts_for_episodes`` has a useful but bounded pool to # bucket back into episodes). afs_all = _read(db, "atomic_fact") @@ -93,7 +93,7 @@ def main() -> None: # mentions two users gets two rows, one for each owner) — sampling # naively can leave one owner with zero facts. Take per-owner caps # so both caroline and melanie have facts whose parent_id matches - # their own episodes' parent_id (MRAG bridge). + # their own episodes' parent_id (memcell bridge). afs: list[dict[str, Any]] = [] for owner in ALL_OWNERS: afs.extend( diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 0000000..589dd84 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1 @@ +"""Shared test helpers — reusable across unit / integration / e2e.""" diff --git a/tests/helpers/knowledge_md.py b/tests/helpers/knowledge_md.py new file mode 100644 index 0000000..4621d8d --- /dev/null +++ b/tests/helpers/knowledge_md.py @@ -0,0 +1,68 @@ +"""Helpers for reading knowledge md files (truth layer) in tests. + +Provides functions to parse index.md and topic md files so tests can +assert against the truth layer rather than only the derived SQLite/API +responses. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + + +def read_document_md(doc_dir: Path) -> dict[str, Any]: + """Read ``index.md`` from *doc_dir*, return frontmatter + body. + + Returns: + ``{"frontmatter": dict, "body": str}`` + + Raises: + FileNotFoundError: If ``index.md`` does not exist. + """ + index = doc_dir / "index.md" + text = index.read_text(encoding="utf-8") + return _parse_md(text) + + +def read_topic_mds(doc_dir: Path) -> list[dict[str, Any]]: + """Read all ``N_*.md`` topic files, return list of frontmatter + body. + + Sorted by filename (i.e. by topic_index prefix). + """ + results = [] + for f in sorted(doc_dir.iterdir()): + if f.name == "index.md" or f.suffix != ".md": + continue + text = f.read_text(encoding="utf-8") + parsed = _parse_md(text) + parsed["filename"] = f.name + results.append(parsed) + return results + + +def find_doc_dir(knowledge_dir: Path, doc_id: str) -> Path | None: + """Scan *knowledge_dir* tree to find the directory containing *doc_id*'s index.md. + + Returns the directory Path, or ``None`` if not found. + """ + if not knowledge_dir.exists(): + return None + for index_md in knowledge_dir.rglob("index.md"): + text = index_md.read_text(encoding="utf-8") + parsed = _parse_md(text) + if parsed["frontmatter"].get("doc_id") == doc_id: + return index_md.parent + return None + + +def _parse_md(text: str) -> dict[str, Any]: + """Split YAML frontmatter from body.""" + parts = text.split("---", 2) + if len(parts) < 3: + return {"frontmatter": {}, "body": text} + fm: dict[str, Any] = yaml.safe_load(parts[1]) or {} + body = parts[2].strip() + return {"frontmatter": fm, "body": body} diff --git a/tests/integration/search/_helpers.py b/tests/integration/search/_helpers.py index c475b46..5629c47 100644 --- a/tests/integration/search/_helpers.py +++ b/tests/integration/search/_helpers.py @@ -12,9 +12,8 @@ the keyword / vector / hybrid recall tests so the assertion logic is in one place. -* :func:`flatten_hits` — collapses ``SearchData``'s four scored result - arrays into one ``(owner_id, score, text)`` tuple list for relevance - checks. +* :func:`flatten_hits` — collapses ``SearchData``'s four arrays into + one ``(owner_id, score, text)`` tuple list for relevance checks. The helpers do **not** hardcode topical keywords ("hiking" / "work") — they are derived from what the pipeline produced. This keeps the @@ -145,7 +144,7 @@ def _extract_fact_sections(md: Path) -> list[str]: def flatten_hits(data: dict[str, Any]) -> list[tuple[str | None, float, str]]: - """Collapse the four scored arrays into ``(owner_id, score, text)``. + """Collapse ``SearchData``'s four arrays into ``(owner_id, score, text)``. Stable shape across track-kinds so the recall / partition tests don't have to branch. Episodes / profiles carry ``user_id`` on the @@ -201,7 +200,7 @@ async def assert_recall( """Hit ``/search`` and lock the four standard recall invariants. 1. **Status** 200 — the route compiled. - 2. **Existence** — ``total >= 1`` across the four scored arrays. + 2. **Existence** — ``total >= 1`` across the four arrays. 3. **Owner partition** — every non-``None`` ``owner_id`` matches the queried owner. Profile hits may carry ``None`` so they're skipped from the check. diff --git a/tests/integration/search/_rerun_probes.py b/tests/integration/search/_rerun_probes.py index f1a02f3..62b139c 100644 --- a/tests/integration/search/_rerun_probes.py +++ b/tests/integration/search/_rerun_probes.py @@ -33,7 +33,7 @@ from _run_full_report import ( # noqa: E402 async def main() -> None: if not (CORPUS_ROOT / "users").is_dir(): raise SystemExit(f"{CORPUS_ROOT} not populated — run _run_full_report.py first") - os.environ["EVEROS_MEMORY__ROOT"] = str(CORPUS_ROOT) + os.environ["EVEROS_ROOT"] = str(CORPUS_ROOT) from everos.config import load_settings load_settings.cache_clear() diff --git a/tests/integration/search/_run_full_report.py b/tests/integration/search/_run_full_report.py index 2166e7f..d7367e9 100644 --- a/tests/integration/search/_run_full_report.py +++ b/tests/integration/search/_run_full_report.py @@ -609,7 +609,7 @@ async def main() -> None: if CORPUS_ROOT.exists(): shutil.rmtree(CORPUS_ROOT) CORPUS_ROOT.mkdir(parents=True) - os.environ["EVEROS_MEMORY__ROOT"] = str(CORPUS_ROOT) + os.environ["EVEROS_ROOT"] = str(CORPUS_ROOT) # Reset cached singletons so they pick up the new env. from everos.config import load_settings diff --git a/tests/integration/search/conftest.py b/tests/integration/search/conftest.py index 4764006..5caffdc 100644 --- a/tests/integration/search/conftest.py +++ b/tests/integration/search/conftest.py @@ -17,7 +17,7 @@ Layout:: search_client (function-scoped) └── per-test ``httpx.AsyncClient`` wired to a freshly built - FastAPI app, ``EVEROS_MEMORY__ROOT`` pointed at the + FastAPI app, ``EVEROS_ROOT`` pointed at the session corpus. Singletons are reset so each test starts with cold caches and the lifespan is the only thing constructing them. @@ -47,7 +47,7 @@ from sqlalchemy import text # Set ``EVEROS_REUSE_CORPUS=`` to skip ingest and point the # session fixture at an existing memory_root (md + lancedb already # populated). Search is a read-only path, so no copy is needed — the -# fixture just sets ``EVEROS_MEMORY__ROOT`` to that directory. +# fixture just sets ``EVEROS_ROOT`` to that directory. _REUSE_ENV = "EVEROS_REUSE_CORPUS" # Memorize-service module-level lazy singletons; reset between phases so @@ -85,7 +85,7 @@ def _session_monkeypatch() -> Generator[pytest.MonkeyPatch, None, None]: def _reset_memorize_singletons(mp: pytest.MonkeyPatch) -> None: """Null out memorize/strategy/LLM-client lazy singletons. - Called once before ingest (so the freshly-set ``EVEROS_MEMORY__ROOT`` + Called once before ingest (so the freshly-set ``EVEROS_ROOT`` actually wins) and once per test (so the session corpus's lifespan sees clean caches). """ @@ -138,8 +138,9 @@ def _ingested_memory_root( else: memory_root = tmp_path_factory.mktemp("search_corpus") - _session_monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(memory_root)) + _session_monkeypatch.setenv("EVEROS_ROOT", str(memory_root)) _reset_memorize_singletons(_session_monkeypatch) + (memory_root / "ome.toml").write_text("# test\n") if reuse: # Search is read-only; the corpus is consumed in place, no copy. @@ -217,7 +218,7 @@ async def search_client( manager builds a fresh embedding / rerank / LLM client per test — we don't want cross-test client state to mask a regression. """ - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(_ingested_memory_root)) + monkeypatch.setenv("EVEROS_ROOT", str(_ingested_memory_root)) _reset_memorize_singletons(monkeypatch) # The search service has its own module-level singletons; reset diff --git a/tests/integration/search/test_search_e2e.py b/tests/integration/search/test_search_e2e.py index 10566a3..f32b688 100644 --- a/tests/integration/search/test_search_e2e.py +++ b/tests/integration/search/test_search_e2e.py @@ -192,7 +192,7 @@ async def test_partition_respects_owner_id( async def test_unknown_owner_returns_empty_200( search_client: httpx.AsyncClient, ) -> None: - """An owner that the corpus never saw → 200 with empty result arrays.""" + """An owner that the corpus never saw → 200 with four empty arrays.""" resp = await search_client.post( "/api/v1/memory/search", json={ @@ -209,7 +209,6 @@ async def test_unknown_owner_returns_empty_200( assert data["profiles"] == [] assert data["agent_cases"] == [] assert data["agent_skills"] == [] - assert data["unprocessed_messages"] == [] # ── 6. Filter DSL ────────────────────────────────────────────────────── diff --git a/tests/integration/test_cascade_all_kinds_consistency.py b/tests/integration/test_cascade_all_kinds_consistency.py index 966b257..2499a6e 100644 --- a/tests/integration/test_cascade_all_kinds_consistency.py +++ b/tests/integration/test_cascade_all_kinds_consistency.py @@ -83,7 +83,7 @@ class _StubEmbedder(EmbeddingProvider): async def cascade_runtime( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> AsyncIterator[MemoryRoot]: - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model") monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1") monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key") @@ -93,6 +93,7 @@ async def cascade_runtime( async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) await ensure_business_indexes() + (tmp_path / "ome.toml").write_text("# test\n") yield MemoryRoot.default() await dispose_connection() await dispose_engine() diff --git a/tests/integration/test_cascade_cli_integration.py b/tests/integration/test_cascade_cli_integration.py index 613eabd..fc57a64 100644 --- a/tests/integration/test_cascade_cli_integration.py +++ b/tests/integration/test_cascade_cli_integration.py @@ -39,21 +39,15 @@ class _StubEmbedder(EmbeddingProvider): return [[0.0] * self.dim for _ in texts] -_ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") - - -def _strip_ansi(text: str) -> str: - return _ANSI_RE.sub("", text) - - @pytest.fixture def cli_runtime(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: """Tmp memory root + clean singletons; CLI bootstraps the schema itself.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model") monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1") monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key") load_settings.cache_clear() + (tmp_path / "ome.toml").write_text("# test\n") # Strip any singleton state from a neighbouring test. asyncio.run(_dispose_all()) @@ -127,12 +121,12 @@ def test_sync_with_path_outside_root_errors( other.write_text("# unrelated\n") result = CliRunner().invoke(cascade_mod.app, ["sync", str(other)]) assert result.exit_code != 0 - # Typer.BadParameter surfaces in stderr / mixed output. The Rich - # error box may wrap the message, pad each line with box characters, - # and inject ANSI control codes on CI. Strip ANSI first, then allow - # non-word separators between the split message fragments. + # Typer.BadParameter surfaces in stderr / mixed output. Rich may wrap + # the error box at different terminal widths, so assert the stable + # semantic fragments instead of their exact adjacency. output = result.stdout + (result.stderr or "") - assert re.search(r"not under[^\w]+memory root", _strip_ansi(output)), output + assert re.search(r"\bnot under\b", output), output + assert re.search(r"\bmemory root\b", output), output def test_sync_with_unmatched_path( diff --git a/tests/integration/test_cascade_fsevents_repro.py b/tests/integration/test_cascade_fsevents_repro.py index f7e36cc..7f95ec8 100644 --- a/tests/integration/test_cascade_fsevents_repro.py +++ b/tests/integration/test_cascade_fsevents_repro.py @@ -68,7 +68,7 @@ class _StubEmbedder(EmbeddingProvider): async def cascade_runtime( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> AsyncIterator[MemoryRoot]: - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model") monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1") monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key") @@ -80,6 +80,7 @@ async def cascade_runtime( async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) await ensure_business_indexes() + (tmp_path / "ome.toml").write_text("# test\n") yield MemoryRoot.default() diff --git a/tests/integration/test_cascade_integration.py b/tests/integration/test_cascade_integration.py index 9e077ba..5f25efb 100644 --- a/tests/integration/test_cascade_integration.py +++ b/tests/integration/test_cascade_integration.py @@ -67,7 +67,7 @@ async def cascade_runtime( guarantee no state leaks in from neighbouring tests, then dispose on the way out so the next test sees a clean slate. """ - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) # Embedding settings are required for the lifespan factory; the # stub bypasses real network, but the orchestrator still expects # the env to be valid-looking. @@ -82,6 +82,7 @@ async def cascade_runtime( async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) await ensure_business_indexes() + (tmp_path / "ome.toml").write_text("# test\n") yield MemoryRoot.default() diff --git a/tests/integration/test_cascade_scenarios.py b/tests/integration/test_cascade_scenarios.py index 7ef03d9..70cba27 100644 --- a/tests/integration/test_cascade_scenarios.py +++ b/tests/integration/test_cascade_scenarios.py @@ -79,7 +79,7 @@ class _StubEmbedder(EmbeddingProvider): async def cascade_runtime( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> AsyncIterator[MemoryRoot]: - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model") monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1") monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key") @@ -91,6 +91,7 @@ async def cascade_runtime( async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) await ensure_business_indexes() + (tmp_path / "ome.toml").write_text("# test\n") yield MemoryRoot.default() diff --git a/tests/integration/test_knowledge_integration.py b/tests/integration/test_knowledge_integration.py new file mode 100644 index 0000000..3dbcebf --- /dev/null +++ b/tests/integration/test_knowledge_integration.py @@ -0,0 +1,1132 @@ +"""End-to-end integration tests for the knowledge module. + +Drives the full pipeline with real components except the embedding +provider (stubbed) and the knowledge extractor (mocked): + + create_document -> KnowledgeWriter -> md files on disk + watchdog FSEvents -> CascadeWatcher -> md_change_state + CascadeWorker -> KnowledgeDocumentHandler + KnowledgeTopicHandler + -> SQLite rows + LanceDB rows + search_knowledge -> BM25 / vector recall -> SearchKnowledgeResult + +Validates that the cascade pipeline correctly indexes knowledge +documents for both document-level and topic-level storage, and that +search retrieval works against the indexed data. +""" + +from __future__ import annotations + +import asyncio +import shutil +from collections.abc import AsyncIterator +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from everalgo.types import KnowledgeMemory, ParsedContent +from sqlmodel import SQLModel + +from everos.component.embedding import EmbeddingProvider +from everos.component.rerank import RerankResult +from everos.component.tokenizer import build_tokenizer +from everos.core.persistence import MemoryRoot +from everos.infra.persistence.lancedb import ( + KnowledgeTopic, + dispose_connection, + ensure_business_indexes, +) +from everos.infra.persistence.lancedb.lancedb_manager import get_table +from everos.infra.persistence.sqlite import ( + DocumentUpsertPayload, + dispose_engine, + get_engine, + knowledge_document_repo, + knowledge_topic_sqlite_repo, + md_change_state_repo, +) +from everos.memory.cascade import CascadeConfig, CascadeOrchestrator +from everos.service.knowledge import ( + ExtractionEmptyError, + create_document, + delete_document, + patch_document, + replace_document, + search_knowledge, +) +from tests.helpers.knowledge_md import find_doc_dir, read_document_md, read_topic_mds + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class _StubEmbedder(EmbeddingProvider): + """1024-dim deterministic vector; counts calls.""" + + dim = 1024 + + def __init__(self) -> None: + self.calls = 0 + + async def embed(self, text: str) -> list[float]: + self.calls += 1 + return [float(i % 7) / 7.0 for i in range(self.dim)] + + async def embed_batch(self, texts: list[str]) -> list[list[float]]: + return [await self.embed(t) for t in texts] + + +class _StubReranker: + """Deterministic reranker — returns candidates in original order.""" + + async def rerank( + self, + query: str, + documents: list[str], + instruction: str | None = None, + ) -> list[RerankResult]: + return [ + RerankResult(index=i, score=1.0 - i * 0.01) for i in range(len(documents)) + ] + + +def _build_mock_extractor( + memories: list[KnowledgeMemory], +) -> AsyncMock: + """Return a mock ``KnowledgeExtractor`` whose ``aextract`` returns *memories*.""" + extractor = AsyncMock() + extractor.aextract.return_value = memories + return extractor + + +def _make_memories( + doc_id: str, + category_id: str = "Sports", +) -> list[KnowledgeMemory]: + """Build a 3-node knowledge tree: root + 2 topic nodes.""" + return [ + KnowledgeMemory( + doc_id=doc_id, + topic_index=0, + topic="Olympics Plan", + summary="Overview of the 2028 Olympics plan.", + content="", + depth=0, + category_id=category_id, + topic_path="Olympics Plan", + ), + KnowledgeMemory( + doc_id=doc_id, + topic_index=1, + topic="Budget", + summary="Budget overview for the Games.", + content="Total budget is $50B allocated across venues and operations.", + depth=1, + parent_index=0, + children_index=[], + topic_path="Olympics Plan > Budget", + content_labels=["finance", "planning"], + category_id=category_id, + ), + KnowledgeMemory( + doc_id=doc_id, + topic_index=2, + topic="Venue", + summary="Venue plans for the Games.", + content="Three new stadiums will be constructed in downtown LA.", + depth=1, + parent_index=0, + children_index=[], + topic_path="Olympics Plan > Venue", + content_labels=["infrastructure"], + category_id=category_id, + ), + ] + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _reset_lancedb_write_locks() -> None: + """Drop per-table asyncio.Lock objects between tests.""" + from everos.core.persistence.lancedb.repository import LanceRepoBase + + LanceRepoBase._reset_locks_for_tests() + + +@pytest.fixture(autouse=True) +def _reset_knowledge_embedding_singleton() -> None: + """Reset the lazy embedding and reranker singletons in service.knowledge.""" + import everos.service.knowledge as _kmod + + for attr in ("_embedding", "_reranker"): + setattr(_kmod, attr, None) + for attr in ("_embedding_resolved", "_reranker_resolved"): + setattr(_kmod, attr, False) + + +@pytest.fixture +async def cascade_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> AsyncIterator[MemoryRoot]: + """Boot sqlite + lancedb against a tmp memory_root; dispose at teardown.""" + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model") + monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1") + monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key") + + await dispose_connection() + await dispose_engine() + + engine = get_engine() + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + await ensure_business_indexes() + (tmp_path / "ome.toml").write_text("# test\n") + + yield MemoryRoot.default() + + await dispose_connection() + await dispose_engine() + + +def _build_orchestrator( + memory_root: MemoryRoot, + embedder: _StubEmbedder, + *, + scan_interval: float = 60.0, +) -> CascadeOrchestrator: + """Factory for a tight-polling cascade orchestrator.""" + return CascadeOrchestrator( + memory_root=memory_root, + embedder=embedder, + tokenizer=build_tokenizer(), + config=CascadeConfig( + scan_interval_seconds=scan_interval, + worker_batch_size=20, + worker_max_retry=2, + worker_poll_interval_seconds=0.05, + worker_retry_backoff_seconds=0.0, + ), + ) + + +async def _wait_drain(*, deadline: float = 20.0) -> None: + """Poll until the cascade queue has no pending items.""" + async with asyncio.timeout(deadline): + while True: + summary = await md_change_state_repo.queue_summary() + if summary.pending == 0: + return + await asyncio.sleep(0.05) + + +async def _wait_lance_rows( + doc_id: str, + expected: int, + *, + deadline: float = 20.0, +) -> None: + """Poll until LanceDB has exactly *expected* rows for *doc_id*.""" + table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) + async with asyncio.timeout(deadline): + while True: + count = await table.count_rows( + filter=f"doc_id = '{doc_id}'", + ) + if count == expected: + return + await asyncio.sleep(0.05) + + +async def _create_test_document( + memory_root: MemoryRoot, + *, + doc_id: str = "d_test12345678", + category_id: str = "Sports", + app_id: str = "default", + project_id: str = "default", +): + """Convenience: create a document using the standard 3-node fixture.""" + memories = _make_memories(doc_id, category_id) + extractor = _build_mock_extractor(memories) + knowledge_dir = memory_root.knowledge_dir(app_id, project_id) + + result = await create_document( + extractor=extractor, + parsed=ParsedContent(text="Full document text about the Olympics."), + title="Olympics Plan", + knowledge_dir=knowledge_dir, + doc_id=doc_id, + category_id=category_id, + ) + return result + + +# --------------------------------------------------------------------------- +# A. Document Creation +# --------------------------------------------------------------------------- + + +async def test_create_document_end_to_end( + cascade_runtime: MemoryRoot, +) -> None: + """Full pipeline: create -> md -> cascade -> SQLite + LanceDB.""" + memory_root = cascade_runtime + embedder = _StubEmbedder() + orchestrator = _build_orchestrator(memory_root, embedder) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_test12345678" + result = await _create_test_document(memory_root, doc_id=doc_id) + assert result.doc_id == doc_id + assert result.category_id == "Sports" + assert result.topic_count == 2 + + # Wait for cascade to process all files. + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + # -- Assert md files -- + knowledge_dir = memory_root.knowledge_dir() + doc_dir = knowledge_dir / "Sports" / f"Olympics_Plan_{doc_id}" + assert doc_dir.is_dir() + assert (doc_dir / "index.md").is_file() + topic_files = sorted(f.name for f in doc_dir.iterdir() if f.name != "index.md") + assert len(topic_files) == 2 + assert any("Budget" in f for f in topic_files) + assert any("Venue" in f for f in topic_files) + + # -- Assert SQLite: knowledge_documents -- + doc_row = await knowledge_document_repo.get_by_doc_id(doc_id) + assert doc_row is not None + assert doc_row.title == "Olympics Plan" + assert doc_row.category_id == "Sports" + + # -- Assert SQLite: knowledge_topics -- + topic_rows = await knowledge_topic_sqlite_repo.get_topics_by_doc_id( + doc_id, + ) + assert len(topic_rows) == 2 + topic_names = {r.topic_name for r in topic_rows} + assert "Budget" in topic_names + assert "Venue" in topic_names + + # -- Assert LanceDB: knowledge_topic -- + table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) + lance_count = await table.count_rows( + filter=f"doc_id = '{doc_id}'", + ) + assert lance_count == 2 + + # Verify vector dimension and token fields. + lance_rows = ( + await table.query().where(f"doc_id = '{doc_id}'").limit(10).to_list() + ) + for row in lance_rows: + assert len(row["vector"]) == 1024 + assert row["summary_tokens"] + assert row["content_tokens"] + + assert embedder.calls >= 2 + + # ── Truth layer (md files) verification ── + doc_dir = find_doc_dir(memory_root.knowledge_dir(), result.doc_id) + assert doc_dir is not None, "Document directory should exist" + + index = read_document_md(doc_dir) + assert index["frontmatter"]["type"] == "knowledge_document" + assert index["frontmatter"]["doc_id"] == result.doc_id + assert index["frontmatter"]["title"] == "Olympics Plan" + assert len(index["body"]) > 10, "Document summary should be in body" + + topics = read_topic_mds(doc_dir) + assert len(topics) == 2, "Should have 2 topic md files (Budget + Venue)" + for t in topics: + fm = t["frontmatter"] + assert fm["type"] == "knowledge_topic" + assert fm["doc_id"] == result.doc_id + assert fm["node_id"].startswith(result.doc_id + "_") + assert len(t["body"]) > 10, "Topic should have content body" + + finally: + await orchestrator.stop() + + +# --------------------------------------------------------------------------- +# B. Search +# --------------------------------------------------------------------------- + + +async def test_search_finds_ingested_topic( + cascade_runtime: MemoryRoot, +) -> None: + """Keyword search finds topics after cascade indexing.""" + import everos.service.knowledge as _kmod + + memory_root = cascade_runtime + embedder = _StubEmbedder() + _kmod._embedding = embedder + _kmod._embedding_resolved = True + _kmod._reranker = _StubReranker() + _kmod._reranker_resolved = True + orchestrator = _build_orchestrator(memory_root, embedder) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_search001" + await _create_test_document(memory_root, doc_id=doc_id) + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + result = await search_knowledge( + query="budget", + method="keyword", + top_k=10, + ) + assert result.hits, "Expected at least one search hit" + budget_hits = [h for h in result.hits if "Budget" in h.topic_name] + assert budget_hits, "Expected a hit with topic_name containing Budget" + assert budget_hits[0].score > 0 + assert budget_hits[0].document.title == "Olympics Plan" + assert result.took_ms > 0 + + finally: + await orchestrator.stop() + + +async def test_search_include_content( + cascade_runtime: MemoryRoot, +) -> None: + """include_content flag controls whether content is populated.""" + import everos.service.knowledge as _kmod + + memory_root = cascade_runtime + embedder = _StubEmbedder() + _kmod._embedding = embedder + _kmod._embedding_resolved = True + _kmod._reranker = _StubReranker() + _kmod._reranker_resolved = True + orchestrator = _build_orchestrator(memory_root, embedder) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_content01" + await _create_test_document(memory_root, doc_id=doc_id) + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + # Without content. + r_no = await search_knowledge( + query="budget", + method="keyword", + top_k=10, + include_content=False, + ) + assert r_no.hits + assert r_no.hits[0].content is None + + # With content. + r_yes = await search_knowledge( + query="budget", + method="keyword", + top_k=10, + include_content=True, + ) + assert r_yes.hits + assert r_yes.hits[0].content + assert len(r_yes.hits[0].content) > 0 + + finally: + await orchestrator.stop() + + +async def test_search_score_threshold_filters( + cascade_runtime: MemoryRoot, +) -> None: + """score_threshold filters out low-scoring results.""" + import everos.service.knowledge as _kmod + + memory_root = cascade_runtime + embedder = _StubEmbedder() + _kmod._embedding = embedder + _kmod._embedding_resolved = True + _kmod._reranker = _StubReranker() + _kmod._reranker_resolved = True + orchestrator = _build_orchestrator(memory_root, embedder) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_thresh01" + await _create_test_document(memory_root, doc_id=doc_id) + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + r_none = await search_knowledge( + query="budget", + method="keyword", + top_k=10, + score_threshold=None, + ) + assert r_none.hits + + r_high = await search_knowledge( + query="budget", + method="keyword", + top_k=10, + score_threshold=0.99, + ) + assert len(r_high.hits) <= len(r_none.hits) + + finally: + await orchestrator.stop() + + +async def test_search_app_project_isolation( + cascade_runtime: MemoryRoot, +) -> None: + """Documents in different app/project scopes are isolated in search.""" + import everos.service.knowledge as _kmod + + memory_root = cascade_runtime + embedder = _StubEmbedder() + _kmod._embedding = embedder + _kmod._embedding_resolved = True + _kmod._reranker = _StubReranker() + _kmod._reranker_resolved = True + orchestrator = _build_orchestrator(memory_root, embedder) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_a = "d_iso_a00001" + doc_b = "d_iso_b00001" + + # Create doc A in (app1, proj1). + memories_a = _make_memories(doc_a) + ext_a = _build_mock_extractor(memories_a) + await create_document( + extractor=ext_a, + parsed=ParsedContent(text="Doc A about Olympics."), + title="Olympics Plan", + knowledge_dir=memory_root.knowledge_dir("app1", "proj1"), + doc_id=doc_a, + category_id="Sports", + ) + + # Create doc B in (app2, proj2). + memories_b = [ + KnowledgeMemory( + doc_id=doc_b, + topic_index=0, + topic="Quantum Computing", + summary="Overview of quantum computing.", + content="", + depth=0, + category_id="Technology", + topic_path="Quantum Computing", + ), + KnowledgeMemory( + doc_id=doc_b, + topic_index=1, + topic="Qubits", + summary="Qubit fundamentals.", + content="A qubit is the basic unit of quantum information.", + depth=1, + parent_index=0, + children_index=[], + topic_path="Quantum Computing > Qubits", + category_id="Technology", + ), + ] + ext_b = _build_mock_extractor(memories_b) + await create_document( + extractor=ext_b, + parsed=ParsedContent(text="Doc B about quantum computing."), + title="Quantum Computing", + knowledge_dir=memory_root.knowledge_dir("app2", "proj2"), + doc_id=doc_b, + category_id="Technology", + ) + + await _wait_lance_rows(doc_a, expected=2, deadline=20.0) + await _wait_lance_rows(doc_b, expected=1, deadline=20.0) + await _wait_drain(deadline=20.0) + + # Search in (app1, proj1) scope. + r1 = await search_knowledge( + query="budget stadium qubit", + method="keyword", + top_k=10, + app_id="app1", + project_id="proj1", + ) + r1_doc_ids = {h.document.doc_id for h in r1.hits} + if r1.hits: + assert doc_b not in r1_doc_ids, "app1/proj1 must not see doc B" + + # Search in (app2, proj2) scope. + r2 = await search_knowledge( + query="budget stadium qubit", + method="keyword", + top_k=10, + app_id="app2", + project_id="proj2", + ) + r2_doc_ids = {h.document.doc_id for h in r2.hits} + if r2.hits: + assert doc_a not in r2_doc_ids, "app2/proj2 must not see doc A" + + finally: + await orchestrator.stop() + + +# --------------------------------------------------------------------------- +# C. Delete +# --------------------------------------------------------------------------- + + +async def test_delete_document_end_to_end( + cascade_runtime: MemoryRoot, +) -> None: + """Manual directory removal + scanner detects deletion -> cleanup. + + ``delete_document`` reports the correct topic count. We then + manually remove the md directory and let the scanner (short + interval) detect the missing files and cascade-delete SQLite + + LanceDB rows. + """ + memory_root = cascade_runtime + embedder = _StubEmbedder() + orchestrator = _build_orchestrator( + memory_root, + embedder, + scan_interval=2.0, + ) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_del0000001" + await _create_test_document(memory_root, doc_id=doc_id) + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + # Confirm pre-delete state. + assert await knowledge_document_repo.get_by_doc_id(doc_id) is not None + topic_count_before = await knowledge_topic_sqlite_repo.count_by_doc_id( + doc_id, + ) + assert topic_count_before == 2 + + # Service-level delete reports correct counts. + del_result = await delete_document( + doc_id=doc_id, + app_id="default", + project_id="default", + ) + assert del_result.doc_id == doc_id + assert del_result.deleted_topics == 2 + + # Manually remove the md directory to trigger cascade cleanup. + knowledge_dir = memory_root.knowledge_dir() + doc_dir = knowledge_dir / "Sports" / f"Olympics_Plan_{doc_id}" + if doc_dir.exists(): + shutil.rmtree(doc_dir) + assert not doc_dir.exists() + + # Wait for cascade scanner to detect + process deletions. + # LanceDB topic rows should be cleared first. + await _wait_lance_rows(doc_id, expected=0, deadline=20.0) + await _wait_drain(deadline=20.0) + + # LanceDB: no topic rows. + table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) + lance_count = await table.count_rows( + filter=f"doc_id = '{doc_id}'", + ) + assert lance_count == 0 + + # SQLite: topic rows gone. + topic_count_after = await knowledge_topic_sqlite_repo.count_by_doc_id( + doc_id, + ) + assert topic_count_after == 0 + + # The document row may need a second scanner pass if the FK + # constraint prevented deletion on the first attempt (index.md + # processed before topic files). Wait for the retry. + async with asyncio.timeout(15.0): + while True: + row = await knowledge_document_repo.get_by_doc_id(doc_id) + if row is None: + break + await asyncio.sleep(0.2) + assert await knowledge_document_repo.get_by_doc_id(doc_id) is None + + finally: + await orchestrator.stop() + + +async def test_delete_idempotent( + cascade_runtime: MemoryRoot, +) -> None: + """Deleting an already-deleted (or nonexistent) document is a no-op.""" + memory_root = cascade_runtime + embedder = _StubEmbedder() + orchestrator = _build_orchestrator( + memory_root, + embedder, + scan_interval=2.0, + ) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_idemp00001" + await _create_test_document(memory_root, doc_id=doc_id) + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + # First delete (service) + manual rmtree + cascade cleanup. + await delete_document( + doc_id=doc_id, + app_id="default", + project_id="default", + ) + knowledge_dir = memory_root.knowledge_dir() + doc_dir = knowledge_dir / "Sports" / f"Olympics_Plan_{doc_id}" + if doc_dir.exists(): + shutil.rmtree(doc_dir) + await _wait_lance_rows(doc_id, expected=0, deadline=20.0) + await _wait_drain(deadline=20.0) + + # Second delete: must not raise, reports 0. + result = await delete_document( + doc_id=doc_id, + app_id="default", + project_id="default", + ) + assert result.deleted_topics == 0 + + finally: + await orchestrator.stop() + + +# --------------------------------------------------------------------------- +# D. Replace (PUT) +# --------------------------------------------------------------------------- + + +async def test_replace_document_end_to_end( + cascade_runtime: MemoryRoot, +) -> None: + """Replace = delete old + create new with same doc_id.""" + memory_root = cascade_runtime + embedder = _StubEmbedder() + orchestrator = _build_orchestrator( + memory_root, + embedder, + scan_interval=2.0, + ) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_repl000001" + knowledge_dir = memory_root.knowledge_dir() + + # V1: 2 topics. + await _create_test_document(memory_root, doc_id=doc_id) + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + # Delete V1 (service + manual rmtree + cascade cleanup). + await delete_document( + doc_id=doc_id, + app_id="default", + project_id="default", + ) + doc_dir_v1 = knowledge_dir / "Sports" / f"Olympics_Plan_{doc_id}" + if doc_dir_v1.exists(): + shutil.rmtree(doc_dir_v1) + await _wait_lance_rows(doc_id, expected=0, deadline=20.0) + await _wait_drain(deadline=20.0) + + # V2: 3 topics (same doc_id). + memories_v2 = [ + KnowledgeMemory( + doc_id=doc_id, + topic_index=0, + topic="Olympics Plan V2", + summary="Updated overview.", + content="", + depth=0, + category_id="Sports", + topic_path="Olympics Plan V2", + ), + KnowledgeMemory( + doc_id=doc_id, + topic_index=1, + topic="Budget V2", + summary="Updated budget overview.", + content="Revised budget is $60B.", + depth=1, + parent_index=0, + children_index=[], + topic_path="Olympics Plan V2 > Budget V2", + category_id="Sports", + ), + KnowledgeMemory( + doc_id=doc_id, + topic_index=2, + topic="Venue V2", + summary="Updated venue plans.", + content="Four stadiums now planned.", + depth=1, + parent_index=0, + children_index=[], + topic_path="Olympics Plan V2 > Venue V2", + category_id="Sports", + ), + KnowledgeMemory( + doc_id=doc_id, + topic_index=3, + topic="Transport", + summary="Transport infrastructure.", + content="New metro line connecting all venues.", + depth=1, + parent_index=0, + children_index=[], + topic_path="Olympics Plan V2 > Transport", + category_id="Sports", + ), + ] + ext_v2 = _build_mock_extractor(memories_v2) + await create_document( + extractor=ext_v2, + parsed=ParsedContent(text="Updated Olympic document."), + title="Olympics Plan V2", + knowledge_dir=knowledge_dir, + doc_id=doc_id, + category_id="Sports", + ) + await _wait_lance_rows(doc_id, expected=3, deadline=20.0) + await _wait_drain(deadline=20.0) + + # SQLite: 1 document row, title changed. + doc_row = await knowledge_document_repo.get_by_doc_id(doc_id) + assert doc_row is not None + assert doc_row.title == "Olympics Plan V2" + + # SQLite: 3 topic rows. + topic_rows = await knowledge_topic_sqlite_repo.get_topics_by_doc_id( + doc_id, + ) + assert len(topic_rows) == 3 + + # LanceDB: 3 rows. + table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) + lance_count = await table.count_rows( + filter=f"doc_id = '{doc_id}'", + ) + assert lance_count == 3 + + finally: + await orchestrator.stop() + + +# --------------------------------------------------------------------------- +# E. Patch +# --------------------------------------------------------------------------- + + +async def test_patch_title_updates_metadata( + cascade_runtime: MemoryRoot, +) -> None: + """patch_document updates the title in SQLite without touching topics.""" + memory_root = cascade_runtime + embedder = _StubEmbedder() + orchestrator = _build_orchestrator(memory_root, embedder) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_patch00001" + await _create_test_document(memory_root, doc_id=doc_id) + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + # Patch title. + p_result = await patch_document( + doc_id=doc_id, + app_id="default", + project_id="default", + title="New Title", + ) + assert "title" in p_result.updated_fields + + # SQLite: title updated. + doc_row = await knowledge_document_repo.get_by_doc_id(doc_id) + assert doc_row is not None + assert doc_row.title == "New Title" + + # SQLite: topic count unchanged. + topic_count = await knowledge_topic_sqlite_repo.count_by_doc_id(doc_id) + assert topic_count == 2 + + finally: + await orchestrator.stop() + + +async def test_patch_category_updates_sqlite( + cascade_runtime: MemoryRoot, +) -> None: + """patch_document with category_id updates the document row (SQLite-only MVP).""" + memory_root = cascade_runtime + embedder = _StubEmbedder() + orchestrator = _build_orchestrator(memory_root, embedder) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + doc_id = "d_patchcat01" + await _create_test_document(memory_root, doc_id=doc_id) + await _wait_lance_rows(doc_id, expected=2, deadline=20.0) + await _wait_drain(deadline=20.0) + + p_result = await patch_document( + doc_id=doc_id, + app_id="default", + project_id="default", + category_id="Technology", + ) + assert "category_id" in p_result.updated_fields + + doc_row = await knowledge_document_repo.get_by_doc_id(doc_id) + assert doc_row is not None + assert doc_row.category_id == "Technology" + + finally: + await orchestrator.stop() + + +# --------------------------------------------------------------------------- +# F. API Errors (service-level) +# --------------------------------------------------------------------------- + + +async def test_get_nonexistent_document_raises( + cascade_runtime: MemoryRoot, +) -> None: + """get_document for a missing doc_id raises DocumentNotFoundError.""" + from everos.service.knowledge import DocumentNotFoundError, get_document + + with pytest.raises(DocumentNotFoundError): + await get_document( + doc_id="d_nonexistent", + app_id="default", + project_id="default", + ) + + +async def test_search_empty_query_returns_empty( + cascade_runtime: MemoryRoot, +) -> None: + """An empty query string returns empty results (no crash).""" + import everos.service.knowledge as _kmod + + _kmod._embedding = _StubEmbedder() + _kmod._embedding_resolved = True + _kmod._reranker = _StubReranker() + _kmod._reranker_resolved = True + + result = await search_knowledge( + query="", + method="keyword", + top_k=10, + ) + assert result.total == 0 + assert result.hits == [] + + +async def test_patch_nonexistent_raises( + cascade_runtime: MemoryRoot, +) -> None: + """Patching a non-existent document raises DocumentNotFoundError.""" + from everos.service.knowledge import DocumentNotFoundError + + with pytest.raises(DocumentNotFoundError): + await patch_document( + doc_id="d_nonexistent", + app_id="default", + project_id="default", + title="Nope", + ) + + +# --------------------------------------------------------------------------- +# G. Edge Cases +# --------------------------------------------------------------------------- + + +async def test_create_document_empty_result( + cascade_runtime: MemoryRoot, +) -> None: + """Extractor returning [] raises ExtractionEmptyError; no files created.""" + memory_root = cascade_runtime + knowledge_dir = memory_root.knowledge_dir() + + extractor = _build_mock_extractor([]) + + with pytest.raises(ExtractionEmptyError): + await create_document( + extractor=extractor, + parsed=ParsedContent(text="Some content"), + title="Empty Doc", + knowledge_dir=knowledge_dir, + doc_id="d_empty00001", + category_id="Sports", + ) + + # No document directory should have been created for this doc. + doc_dir = knowledge_dir / "Sports" / "Empty_Doc_d_empty00001" + assert not doc_dir.exists() + + +# --------------------------------------------------------------------------- +# H. Truth-layer bug-exposing tests (xfail) +# --------------------------------------------------------------------------- + + +async def test_patch_title_updates_md(cascade_runtime: MemoryRoot) -> None: + """PATCH title must update index.md frontmatter (truth layer).""" + memory_root = cascade_runtime + await _create_test_document(memory_root, doc_id="d_patch_title1") + + doc_dir = find_doc_dir(memory_root.knowledge_dir(), "d_patch_title1") + assert doc_dir is not None + + old_fm = read_document_md(doc_dir)["frontmatter"] + assert old_fm["title"] == "Olympics Plan" + + await patch_document("d_patch_title1", "default", "default", title="New Title") + + new_fm = read_document_md(doc_dir)["frontmatter"] + assert new_fm["title"] == "New Title" + + +async def test_patch_category_moves_directory(cascade_runtime: MemoryRoot) -> None: + """PATCH category_id must move the document directory to the new category.""" + memory_root = cascade_runtime + await _create_test_document( + memory_root, doc_id="d_patch_cat01", category_id="Sports" + ) + + knowledge_dir = memory_root.knowledge_dir() + old_dir = find_doc_dir(knowledge_dir, "d_patch_cat01") + assert old_dir is not None + assert "Sports" in str(old_dir) + + await patch_document("d_patch_cat01", "default", "default", category_id="Finance") + + new_dir = find_doc_dir(knowledge_dir, "d_patch_cat01") + assert new_dir is not None, ( + "Document directory should still exist after category change" + ) + assert "Finance" in str(new_dir), ( + f"Directory should be under Finance/, got {new_dir}" + ) + assert not old_dir.exists(), "Old Sports/ directory should be gone" + + +async def test_replace_failure_preserves_old_document( + cascade_runtime: MemoryRoot, +) -> None: + """replace_document restores the original md directory when extraction fails. + + White-box surfaces: md directory on disk, SQLite knowledge_documents row. + """ + memory_root = cascade_runtime + doc_id = "d_replace_fail" + result = await _create_test_document(memory_root, doc_id=doc_id) + + knowledge_dir = memory_root.knowledge_dir() + doc_dir = find_doc_dir(knowledge_dir, doc_id) + assert doc_dir is not None, "Setup: original document directory must exist" + + # replace_document checks SQLite before proceeding; simulate cascade sync. + await knowledge_document_repo.upsert_from_handler( + DocumentUpsertPayload( + doc_id=doc_id, + app_id="default", + project_id="default", + category_id=result.category_id, + title="Olympics Plan", + summary="test", + source_name=None, + source_type=None, + md_path=result.md_path, + ) + ) + + failing_extractor = AsyncMock() + failing_extractor.aextract.return_value = [] + + with pytest.raises(ExtractionEmptyError): + await replace_document( + extractor=failing_extractor, + parsed=ParsedContent(text=""), + title="Should Fail", + doc_id=doc_id, + knowledge_dir=knowledge_dir, + ) + + # After failure, the original md directory must be restored. + assert doc_dir.exists(), "Original md must survive a failed PUT replacement" + + +async def test_dirname_collision_different_docs(cascade_runtime: MemoryRoot) -> None: + """Two docs with titles that sanitize to the same dirname must not collide.""" + memory_root = cascade_runtime + knowledge_dir = memory_root.knowledge_dir() + + # Both titles sanitize to "Hello_World" after stripping punctuation. + memories1 = _make_memories("d_collision01", "Sports") + memories1[0] = memories1[0].model_copy(update={"topic": "Hello World!"}) + ext1 = _build_mock_extractor(memories1) + await create_document( + extractor=ext1, + parsed=ParsedContent(text="doc1"), + title="Hello World!", + knowledge_dir=knowledge_dir, + doc_id="d_collision01", + category_id="Sports", + ) + + memories2 = _make_memories("d_collision02", "Sports") + memories2[0] = memories2[0].model_copy(update={"topic": "Hello World?"}) + ext2 = _build_mock_extractor(memories2) + await create_document( + extractor=ext2, + parsed=ParsedContent(text="doc2"), + title="Hello World?", + knowledge_dir=knowledge_dir, + doc_id="d_collision02", + category_id="Sports", + ) + + dir1 = find_doc_dir(knowledge_dir, "d_collision01") + dir2 = find_doc_dir(knowledge_dir, "d_collision02") + assert dir1 is not None, "First doc directory should exist" + assert dir2 is not None, "Second doc directory should exist" + assert dir1 != dir2, "Different docs must have different directories" diff --git a/tests/integration/test_memorize_agent_mode.py b/tests/integration/test_memorize_agent_mode.py index b67db59..366b78a 100644 --- a/tests/integration/test_memorize_agent_mode.py +++ b/tests/integration/test_memorize_agent_mode.py @@ -106,6 +106,7 @@ async def memorize_env( MemoryRoot, "default", classmethod(lambda cls: MemoryRoot(root=tmp_path)) ) (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") svc = importlib.import_module("everos.service.memorize") af_mod = importlib.import_module("everos.memory.strategies.extract_atomic_facts") diff --git a/tests/integration/test_memorize_concurrent_session_lock.py b/tests/integration/test_memorize_concurrent_session_lock.py index 78e9ef0..e7dd23f 100644 --- a/tests/integration/test_memorize_concurrent_session_lock.py +++ b/tests/integration/test_memorize_concurrent_session_lock.py @@ -95,6 +95,7 @@ async def memorize_env_locked( MemoryRoot, "default", classmethod(lambda cls: MemoryRoot(root=tmp_path)) ) (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") svc = importlib.import_module("everos.service.memorize") af_mod = importlib.import_module("everos.memory.strategies.extract_atomic_facts") diff --git a/tests/integration/test_memorize_integration.py b/tests/integration/test_memorize_integration.py index f8a82a9..54277ba 100644 --- a/tests/integration/test_memorize_integration.py +++ b/tests/integration/test_memorize_integration.py @@ -110,6 +110,7 @@ async def memorize_env( MemoryRoot, "default", classmethod(lambda cls: MemoryRoot(root=tmp_path)) ) (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") svc = importlib.import_module("everos.service.memorize") af_mod = importlib.import_module("everos.memory.strategies.extract_atomic_facts") diff --git a/tests/integration/test_memorize_window_segmentation.py b/tests/integration/test_memorize_window_segmentation.py index 8b151c5..611af6f 100644 --- a/tests/integration/test_memorize_window_segmentation.py +++ b/tests/integration/test_memorize_window_segmentation.py @@ -98,6 +98,7 @@ async def memorize_env_scripted( MemoryRoot, "default", classmethod(lambda cls: MemoryRoot(root=tmp_path)) ) (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") svc = importlib.import_module("everos.service.memorize") af_mod = importlib.import_module("everos.memory.strategies.extract_atomic_facts") diff --git a/tests/integration/test_ome_strategies_integration.py b/tests/integration/test_ome_strategies_integration.py index 7ab5812..111534c 100644 --- a/tests/integration/test_ome_strategies_integration.py +++ b/tests/integration/test_ome_strategies_integration.py @@ -136,16 +136,18 @@ async def test_emit_dispatches_both_strategies_to_success( ), capture_logs() as logs, ): - mock_af.return_value.aextract = AsyncMock(return_value=[fake_fact]) + mock_af.return_value.aextract_from_text = AsyncMock(return_value=[fake_fact]) mock_fs.return_value.aextract = AsyncMock(return_value=[fake_foresight]) # Ensure the sqlite dir exists before the engine creates ome.db. (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") await _setup_system_db_schema(monkeypatch) engine = svc._get_engine() await engine.start() try: + # Foresight still subscribes to UserPipelineStarted. await engine.emit( UserPipelineStarted( memcell_id="mc_a", @@ -153,6 +155,17 @@ async def test_emit_dispatches_both_strategies_to_success( memcell=_sample_memcell(), ) ) + # Atomic facts now subscribes to EpisodeExtracted. + await engine.emit( + EpisodeExtracted( + memcell_id="mc_a", + episode_entry_id="ep_20260517_0001", + episode_text="alice likes hiking", + episode_timestamp_ms=1_700_000_000_000, + owner_id="u_alice", + session_id="s1", + ) + ) # Poll until both strategies reach SUCCESS (max 5 s). af_rows: list = [] @@ -180,11 +193,9 @@ async def test_emit_dispatches_both_strategies_to_success( fs_logs = [r for r in logs if r.get("event") == "foresights_extracted"] assert af_logs, "expected atomic_facts_extracted log line" assert fs_logs, "expected foresights_extracted log line" - # The sample MemCell has 2 user senders (u_alice, u_bob), so each - # strategy gathers one result per sender and flattens them: - # extract_atomic_facts: 2 senders × 1 fake_fact each = 2 - # extract_foresight: 2 senders × 1 fake_foresight each = 2 - assert af_logs[0]["count"] == 2 + # extract_atomic_facts: 1 EpisodeExtracted → 1 fact for u_alice + # extract_foresight: 2 senders × 1 foresight each = 2 + assert af_logs[0]["count"] == 1 assert fs_logs[0]["count"] == 2 @@ -297,6 +308,7 @@ async def test_emit_dispatches_agent_case_strategy_to_success( mock_ac.return_value.aextract = AsyncMock(return_value=[fake_case]) (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") await _setup_system_db_schema(monkeypatch) engine = svc._get_engine() @@ -433,6 +445,7 @@ async def test_skill_chain_e2e( mock_writer_cls.return_value.write_main = AsyncMock(return_value=None) (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") await _setup_system_db_schema(monkeypatch) engine = svc._get_engine() @@ -543,11 +556,19 @@ async def test_profile_chain_e2e( } ) + fake_episode_row = MagicMock() + fake_episode_row.parent_type = "memcell" + fake_episode_row.parent_id = "mc_aaaaaaaaaaa1" + fake_episode_row.entry_id = "ep_20260517_0001" + with ( patch( "everos.memory.strategies.trigger_profile_clustering.get_embedder", return_value=embedder, ), + patch( + "everos.memory.strategies.extract_user_profile.episode_repo" + ) as mock_episode_repo, patch( "everos.memory.strategies.extract_user_profile.memcell_repo" ) as mock_memcell_repo, @@ -566,12 +587,16 @@ async def test_profile_chain_e2e( ), capture_logs() as logs, ): + mock_episode_repo.find_by_owner_entries = AsyncMock( + return_value=[fake_episode_row] + ) mock_memcell_repo.find_by_ids = AsyncMock(return_value=[fake_memcell_row]) mock_reader_cls.return_value.read = AsyncMock(return_value=None) mock_writer_cls.return_value.write = AsyncMock(return_value=None) mock_extractor_cls.return_value.aextract = AsyncMock(return_value=new_profile) (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") await _setup_system_db_schema(monkeypatch) engine = svc._get_engine() @@ -584,6 +609,7 @@ async def test_profile_chain_e2e( episode_text="alice likes hiking", episode_timestamp_ms=1_700_000_001_000, owner_id="u_alice", + session_id="s_integration", ) ) diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py new file mode 100644 index 0000000..afa019a --- /dev/null +++ b/tests/integration/test_reflection_integration.py @@ -0,0 +1,826 @@ +"""End-to-end Reflection INIT cycle integration test. + +Drives ``ReflectionOrchestrator.run()`` with real SQLite (cluster + +report repos), real LanceDB (episode + atomic_fact stores), real +EpisodeWriter (md files), a ``FakeLLMClient``-backed +``EpisodeReflector``, and a stub embedder. Verifies the full flow: + + select candidates → merge episodes (FakeLLM) → write merged md → + emit EpisodeExtracted + wait (no-op via FakeStrategyContext) → + deprecate originals → update cluster membership → write report + +White-box surfaces: sqlite cluster_member / reflection_report tables, +LanceDB episode.deprecated_by column, md file existence + frontmatter. +""" + +from __future__ import annotations + +import datetime as _dt +import hashlib +import json +from pathlib import Path + +import numpy as np +import pytest +from everalgo.clustering import Cluster as AlgoCluster +from everalgo.testing.fake_llm import FakeLLMClient +from everalgo.user_memory.reflect import EpisodeReflector +from sqlmodel import SQLModel + +from everos.config import LanceDBSettings, load_settings +from everos.core.persistence import ( + MemoryRoot, + open_lancedb_connection, +) +from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase +from everos.infra.ome.testing import FakeStrategyContext +from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact +from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode +from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter +from everos.infra.persistence.sqlite import cluster_repo, reflection_report_repo +from everos.memory._partition_locks import _reset_for_tests +from everos.memory.reflection.orchestrator import ReflectionOrchestrator + +# --------------------------------------------------------------------------- +# Stub embedder +# --------------------------------------------------------------------------- + + +class _StubEmbedder: + """Return deterministic 1024-dim vectors seeded by input text.""" + + dim: int = 1024 + + async def embed(self, text: str) -> list[float]: + digest = hashlib.sha256(text.encode("utf-8")).digest() + seed = int.from_bytes(digest[:8], "little") + rng = np.random.default_rng(seed) + vec = rng.standard_normal(self.dim).astype(np.float32) + norm = float(np.linalg.norm(vec)) or 1.0 + vec /= norm + return vec.tolist() + + +# --------------------------------------------------------------------------- +# LanceDB repo wrappers (inject table directly) +# --------------------------------------------------------------------------- + + +class _EpisodeRepo(LanceDailyLogRepoBase[LanceEpisode]): + schema = LanceEpisode + + +class _AtomicFactRepo(LanceDailyLogRepoBase[AtomicFact]): + schema = AtomicFact + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _reset_locks() -> None: + """Drop per-table write locks + partition locks between tests.""" + LanceRepoBase._reset_locks_for_tests() + _reset_for_tests() + + +@pytest.fixture +def memory_root(tmp_path: Path) -> MemoryRoot: + mr = MemoryRoot(tmp_path) + mr.ensure() + (tmp_path / ".index" / "sqlite").mkdir(parents=True, exist_ok=True) + (tmp_path / "ome.toml").write_text("# test\n") + return mr + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_lance_episode( + *, + entry_id: str, + owner_id: str, + episode: str, + timestamp: _dt.datetime, + parent_type: str = "memcell", + parent_id: str, + session_id: str | None = "s_test", + md_path: str = "", +) -> LanceEpisode: + """Build a LanceDB Episode row with required fields.""" + digest = hashlib.sha256(episode.encode("utf-8")).digest() + seed = int.from_bytes(digest[:8], "little") + rng = np.random.default_rng(seed) + vec = rng.standard_normal(1024).astype(np.float32) + vec /= float(np.linalg.norm(vec)) or 1.0 + + return LanceEpisode( + id=f"{owner_id}_{entry_id}", + entry_id=entry_id, + owner_id=owner_id, + owner_type="user", + app_id="default", + project_id="default", + session_id=session_id, + timestamp=timestamp, + parent_type=parent_type, + parent_id=parent_id, + sender_ids=[owner_id], + subject="test", + summary=None, + episode=episode, + episode_tokens=episode.lower(), + md_path=md_path, + content_sha256=hashlib.sha256(episode.encode()).hexdigest(), + deprecated_by=None, + vector=vec.tolist(), + ) + + +def _make_lance_fact( + *, + entry_id: str, + owner_id: str, + fact: str, + parent_id: str, + parent_type: str = "memcell", + timestamp: _dt.datetime, +) -> AtomicFact: + """Build a LanceDB AtomicFact row.""" + digest = hashlib.sha256(fact.encode("utf-8")).digest() + seed = int.from_bytes(digest[:8], "little") + rng = np.random.default_rng(seed) + vec = rng.standard_normal(1024).astype(np.float32) + vec /= float(np.linalg.norm(vec)) or 1.0 + + return AtomicFact( + id=f"{owner_id}_{entry_id}", + entry_id=entry_id, + owner_id=owner_id, + owner_type="user", + app_id="default", + project_id="default", + session_id="s_test", + timestamp=timestamp, + parent_type=parent_type, + parent_id=parent_id, + sender_ids=[owner_id], + fact=fact, + fact_tokens=fact.lower(), + md_path="", + content_sha256=hashlib.sha256(fact.encode()).hexdigest(), + deprecated_by=None, + vector=vec.tolist(), + ) + + +async def _setup_sqlite(monkeypatch: pytest.MonkeyPatch) -> None: + """Reset the sqlite_manager singleton and create_all tables.""" + from everos.infra.persistence.sqlite import sqlite_manager + + if sqlite_manager._engine is not None: # noqa: SLF001 + await sqlite_manager.dispose_engine() + monkeypatch.setattr(sqlite_manager, "_engine", None, raising=False) + monkeypatch.setattr(sqlite_manager, "_session_factory", None, raising=False) + engine = sqlite_manager.get_engine() + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + + +async def _teardown_sqlite() -> None: + from everos.infra.persistence.sqlite import sqlite_manager + + if sqlite_manager._engine is not None: # noqa: SLF001 + await sqlite_manager.dispose_engine() + + +# --------------------------------------------------------------------------- +# Test +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_reflection_init_merges_cluster_episodes( + tmp_path: Path, + memory_root: MemoryRoot, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Full INIT Reflection cycle: 3 episodes in a cluster -> merge -> verify. + + Verifies: + - ReflectionReport created with mode="init", source_count=3 + - Source episodes deprecated in LanceDB (deprecated_by is set) + - Merged episode written to md with parent_type="cluster" + - Cluster members updated: old 3 removed, 1 new episode member + - Report.merged_entry_id matches the new cluster member's member_id + - Atomic facts for source episodes are deprecated + """ + # -- Redirect MemoryRoot.default() to tmp_path. + monkeypatch.setattr( + MemoryRoot, + "default", + classmethod(lambda cls: MemoryRoot(root=tmp_path)), + ) + monkeypatch.setenv("EVEROS_LLM__API_KEY", "fake-key") + monkeypatch.setenv("EVEROS_LLM__BASE_URL", "https://fake.example.com") + + load_settings.cache_clear() + + await _setup_sqlite(monkeypatch) + + try: + # -- LanceDB setup: create tables + insert test episodes. + conn = await open_lancedb_connection(memory_root.lancedb_dir, LanceDBSettings()) + ep_table = await conn.create_table("episode", schema=LanceEpisode) + af_table = await conn.create_table("atomic_fact", schema=AtomicFact) + + ep_repo = _EpisodeRepo(table=ep_table) + af_repo = _AtomicFactRepo(table=af_table) + + owner_id = "u_test" + cluster_id = "cl_test_reflect" + + ts1 = _dt.datetime(2026, 6, 10, 10, 0, 0, tzinfo=_dt.UTC) + ts2 = _dt.datetime(2026, 6, 10, 11, 0, 0, tzinfo=_dt.UTC) + ts3 = _dt.datetime(2026, 6, 10, 12, 0, 0, tzinfo=_dt.UTC) + + ep1 = _make_lance_episode( + entry_id="ep_20260610_0001", + owner_id=owner_id, + episode="Andrew has no pets", + timestamp=ts1, + parent_id="mc_001", + ) + ep2 = _make_lance_episode( + entry_id="ep_20260610_0002", + owner_id=owner_id, + episode="Andrew adopted Toby", + timestamp=ts2, + parent_id="mc_002", + ) + ep3 = _make_lance_episode( + entry_id="ep_20260610_0003", + owner_id=owner_id, + episode="Andrew adopted Buddy", + timestamp=ts3, + parent_id="mc_003", + ) + + await ep_repo.add([ep1, ep2, ep3]) + + # Insert atomic facts linked to the source memcells. + fact1 = _make_lance_fact( + entry_id="af_20260610_0001", + owner_id=owner_id, + fact="Andrew has no pets", + parent_id="ep_20260610_0001", + timestamp=ts1, + ) + fact2 = _make_lance_fact( + entry_id="af_20260610_0002", + owner_id=owner_id, + fact="Andrew adopted Toby", + parent_id="ep_20260610_0002", + timestamp=ts2, + ) + fact3 = _make_lance_fact( + entry_id="af_20260610_0003", + owner_id=owner_id, + fact="Andrew adopted Buddy", + parent_id="ep_20260610_0003", + timestamp=ts3, + ) + await af_repo.add([fact1, fact2, fact3]) + + # -- SQLite: create cluster + cluster_members. + centroid = np.zeros(1024, dtype=np.float32) + algo_cluster = AlgoCluster( + id=cluster_id, + centroid=centroid, + count=3, + last_ts=int(ts3.timestamp() * 1000), + preview=["Andrew has no pets", "Andrew adopted Toby"], + members=["ep_20260610_0001", "ep_20260610_0002", "ep_20260610_0003"], + ) + await cluster_repo.upsert_with_members( + algo_cluster, + owner_id=owner_id, + owner_type="user", + kind="user_memory", + member_type="episode", + ) + + # Verify cluster members are created. + members_before = await cluster_repo.get_members_with_type(cluster_id) + assert len(members_before) == 3 + + # -- Build the EpisodeReflector with FakeLLM. + merged_content = ( + "Andrew initially had no pets. He later adopted a dog named Toby, " + "and then adopted another dog named Buddy." + ) + merged_title = "Andrew's pet adoption journey" + reflect_response = json.dumps( + {"content": merged_content, "title": merged_title} + ) + fake_llm = FakeLLMClient(responses=[reflect_response]) + reflector = EpisodeReflector(llm=fake_llm) + + # -- Build the EpisodeWriter. + episode_writer = EpisodeWriter(memory_root) + + # -- Build the orchestrator with real repos. + orchestrator = ReflectionOrchestrator( + cluster_repo=cluster_repo, + episode_store=ep_repo, + atomic_fact_store=af_repo, + episode_writer=episode_writer, + report_repo=reflection_report_repo, + reflector=reflector, + embedder=_StubEmbedder(), + ) + + # -- Run the orchestrator. + fake_ctx = FakeStrategyContext() + reports = await orchestrator.run(ctx=fake_ctx, owner_id=owner_id) + + # -- Verify: exactly one report created. + assert len(reports) == 1 + report = reports[0] + assert report.mode == "init" + assert report.source_count == 3 + assert report.status == "completed" + assert report.cluster_id == cluster_id + + merged_entry_id = report.merged_entry_id + + # -- Verify: source episodes deprecated in LanceDB. + for ep in [ep1, ep2, ep3]: + rows = await ep_repo.find_where( + f"entry_id = '{ep.entry_id}' AND owner_id = '{owner_id}'" + ) + assert len(rows) == 1, f"expected 1 row for {ep.entry_id}" + assert rows[0].deprecated_by == merged_entry_id, ( + f"{ep.entry_id} should be deprecated by {merged_entry_id}" + ) + + # -- Verify: atomic facts deprecated in LanceDB. + for fact in [fact1, fact2, fact3]: + rows = await af_repo.find_where( + f"entry_id = '{fact.entry_id}' AND owner_id = '{owner_id}'" + ) + assert len(rows) == 1, f"expected 1 row for {fact.entry_id}" + assert rows[0].deprecated_by == merged_entry_id, ( + f"{fact.entry_id} should be deprecated by {merged_entry_id}" + ) + + # -- Verify: cluster membership updated. + members_after = await cluster_repo.get_members_with_type(cluster_id) + assert len(members_after) == 1, ( + f"expected 1 member after merge, got {len(members_after)}" + ) + new_member_id, new_member_type = members_after[0] + assert new_member_type == "episode" + assert new_member_id == merged_entry_id + + # -- Verify: merged episode written to md. + users_dir = memory_root.users_dir("default", "default") + episode_files = sorted( + (users_dir / owner_id / "episodes").rglob("episode-*.md") + ) + assert len(episode_files) == 1 + md_text = episode_files[0].read_text() + assert merged_content in md_text + assert "parent_type" in md_text + assert "cluster" in md_text + + # -- Verify: FakeStrategyContext received an EpisodeExtracted event. + assert len(fake_ctx.emitted) == 1 + emitted_event = fake_ctx.emitted[0] + assert emitted_event.episode_entry_id == merged_entry_id + assert emitted_event.owner_id == owner_id + assert emitted_event.source == "reflection" + + # -- Verify: report in sqlite matches. + db_report = await reflection_report_repo.get_latest_for_cluster(cluster_id) + assert db_report is not None + assert db_report.merged_entry_id == merged_entry_id + assert db_report.mode == "init" + + finally: + conn.close() + await _teardown_sqlite() + + +@pytest.mark.asyncio +async def test_reflection_update_merges_new_episodes_with_existing_merged( + tmp_path: Path, + memory_root: MemoryRoot, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """INIT -> add 4th episode -> UPDATE cycle: verify incremental merge. + + Verifies: + - Second report has mode="update", source_count=2 (old merged + mc_004) + - Old merged episode (v1) deprecated in LanceDB + - New merged episode (v2) exists with updated content + - mc_004's episode deprecated in LanceDB + - Cluster members: exactly 1, member_type="episode", member_id = v2 + - Old atomic facts from v1's sources remain deprecated + """ + monkeypatch.setattr( + MemoryRoot, + "default", + classmethod(lambda cls: MemoryRoot(root=tmp_path)), + ) + monkeypatch.setenv("EVEROS_LLM__API_KEY", "fake-key") + monkeypatch.setenv("EVEROS_LLM__BASE_URL", "https://fake.example.com") + + load_settings.cache_clear() + + await _setup_sqlite(monkeypatch) + + try: + # -- LanceDB setup. + conn = await open_lancedb_connection(memory_root.lancedb_dir, LanceDBSettings()) + ep_table = await conn.create_table("episode", schema=LanceEpisode) + af_table = await conn.create_table("atomic_fact", schema=AtomicFact) + + ep_repo = _EpisodeRepo(table=ep_table) + af_repo = _AtomicFactRepo(table=af_table) + + owner_id = "u_test" + cluster_id = "cl_test_update" + + ts1 = _dt.datetime(2026, 6, 10, 10, 0, 0, tzinfo=_dt.UTC) + ts2 = _dt.datetime(2026, 6, 10, 11, 0, 0, tzinfo=_dt.UTC) + ts3 = _dt.datetime(2026, 6, 10, 12, 0, 0, tzinfo=_dt.UTC) + + ep1 = _make_lance_episode( + entry_id="ep_20260610_0001", + owner_id=owner_id, + episode="Andrew has no pets", + timestamp=ts1, + parent_id="mc_001", + ) + ep2 = _make_lance_episode( + entry_id="ep_20260610_0002", + owner_id=owner_id, + episode="Andrew adopted Toby", + timestamp=ts2, + parent_id="mc_002", + ) + ep3 = _make_lance_episode( + entry_id="ep_20260610_0003", + owner_id=owner_id, + episode="Andrew adopted Buddy", + timestamp=ts3, + parent_id="mc_003", + ) + await ep_repo.add([ep1, ep2, ep3]) + + # Atomic facts for source episodes. + fact1 = _make_lance_fact( + entry_id="af_20260610_0001", + owner_id=owner_id, + fact="Andrew has no pets", + parent_id="ep_20260610_0001", + timestamp=ts1, + ) + fact2 = _make_lance_fact( + entry_id="af_20260610_0002", + owner_id=owner_id, + fact="Andrew adopted Toby", + parent_id="ep_20260610_0002", + timestamp=ts2, + ) + fact3 = _make_lance_fact( + entry_id="af_20260610_0003", + owner_id=owner_id, + fact="Andrew adopted Buddy", + parent_id="ep_20260610_0003", + timestamp=ts3, + ) + await af_repo.add([fact1, fact2, fact3]) + + # -- SQLite: cluster with 3 memcell members. + centroid = np.zeros(1024, dtype=np.float32) + algo_cluster = AlgoCluster( + id=cluster_id, + centroid=centroid, + count=3, + last_ts=int(ts3.timestamp() * 1000), + preview=["Andrew has no pets", "Andrew adopted Toby"], + members=["ep_20260610_0001", "ep_20260610_0002", "ep_20260610_0003"], + ) + await cluster_repo.upsert_with_members( + algo_cluster, + owner_id=owner_id, + owner_type="user", + kind="user_memory", + member_type="episode", + ) + + # -- Phase 1: INIT merge. + init_response = json.dumps( + {"content": "Andrew adopted Toby and Buddy.", "title": "Andrew pets v1"} + ) + update_response = json.dumps( + { + "content": "Andrew adopted Toby, Buddy, and Scout.", + "title": "Andrew pets v2", + } + ) + fake_llm = FakeLLMClient(responses=[init_response, update_response]) + reflector = EpisodeReflector(llm=fake_llm) + + episode_writer = EpisodeWriter(memory_root) + + orchestrator = ReflectionOrchestrator( + cluster_repo=cluster_repo, + episode_store=ep_repo, + atomic_fact_store=af_repo, + episode_writer=episode_writer, + report_repo=reflection_report_repo, + reflector=reflector, + embedder=_StubEmbedder(), + ) + + fake_ctx = FakeStrategyContext() + reports_init = await orchestrator.run(ctx=fake_ctx, owner_id=owner_id) + + assert len(reports_init) == 1 + report_init = reports_init[0] + assert report_init.mode == "init" + merged_v1_entry_id = report_init.merged_entry_id + + # FakeStrategyContext is a no-op, so the merged episode is not + # inserted into LanceDB by the extraction pipeline. Simulate the + # real pipeline by inserting the merged v1 episode manually. + merged_v1_ep = _make_lance_episode( + entry_id=merged_v1_entry_id, + owner_id=owner_id, + episode="Andrew adopted Toby and Buddy.", + timestamp=ts3, + parent_type="cluster", + parent_id=cluster_id, + session_id=None, + ) + await ep_repo.add([merged_v1_ep]) + + # -- Phase 2: add a 4th episode + cluster member, then run UPDATE. + ts4 = _dt.datetime(2026, 6, 10, 13, 0, 0, tzinfo=_dt.UTC) + ep4 = _make_lance_episode( + entry_id="ep_20260610_0004", + owner_id=owner_id, + episode="Andrew adopted Scout", + timestamp=ts4, + parent_id="mc_004", + ) + await ep_repo.add([ep4]) + + await cluster_repo.add_member(cluster_id, "ep_20260610_0004", "episode") + + # Fresh orchestrator, same FakeLLM (next pop = update_response). + orchestrator2 = ReflectionOrchestrator( + cluster_repo=cluster_repo, + episode_store=ep_repo, + atomic_fact_store=af_repo, + episode_writer=episode_writer, + report_repo=reflection_report_repo, + reflector=reflector, + embedder=_StubEmbedder(), + ) + fake_ctx2 = FakeStrategyContext() + reports_update = await orchestrator2.run(ctx=fake_ctx2, owner_id=owner_id) + + # -- Verify: second report is UPDATE with source_count=2. + assert len(reports_update) == 1 + report_update = reports_update[0] + assert report_update.mode == "update" + assert report_update.source_count == 2 + merged_v2_entry_id = report_update.merged_entry_id + + # -- Verify: old merged episode (v1) deprecated. + v1_rows = await ep_repo.find_where( + f"entry_id = '{merged_v1_entry_id}' AND owner_id = '{owner_id}'" + ) + assert len(v1_rows) == 1 + assert v1_rows[0].deprecated_by == merged_v2_entry_id + + # -- Verify: mc_004's episode deprecated. + mc4_rows = await ep_repo.find_where( + f"parent_type = 'memcell' AND parent_id = 'mc_004' " + f"AND owner_id = '{owner_id}'" + ) + assert len(mc4_rows) == 1 + assert mc4_rows[0].deprecated_by == merged_v2_entry_id + + # -- Verify: new merged episode (v2) written to markdown. + # (FakeStrategyContext does not run the extraction pipeline, so v2 + # is not yet in LanceDB — verify via the md file instead.) + users_dir = memory_root.users_dir("default", "default") + episode_files = sorted( + (users_dir / owner_id / "episodes").rglob("episode-*.md") + ) + assert len(episode_files) >= 1 + # Both v1 and v2 land in the same daily-log file; check full text. + all_md = "\n".join(f.read_text() for f in episode_files) + assert "Andrew adopted Toby, Buddy, and Scout." in all_md + assert "parent_type" in all_md and "cluster" in all_md + + # -- Verify: cluster members = exactly 1, type=episode, id=v2. + members_final = await cluster_repo.get_members_with_type(cluster_id) + assert len(members_final) == 1 + final_mid, final_mtype = members_final[0] + assert final_mtype == "episode" + assert final_mid == merged_v2_entry_id + + # -- Verify: original atomic facts still deprecated by v1 + # (they were deprecated in the INIT phase; UPDATE does not touch them). + for fact in [fact1, fact2, fact3]: + rows = await af_repo.find_where( + f"entry_id = '{fact.entry_id}' AND owner_id = '{owner_id}'" + ) + assert len(rows) == 1 + assert rows[0].deprecated_by == merged_v1_entry_id, ( + f"{fact.entry_id} should still be deprecated by v1" + ) + + finally: + conn.close() + await _teardown_sqlite() + + +@pytest.mark.asyncio +async def test_reflected_episodes_visible_in_search_deprecated_excluded( + tmp_path: Path, + memory_root: MemoryRoot, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """After INIT Reflection, search filters correctly include/exclude episodes. + + Verifies: + - ``deprecated_by IS NULL`` returns only the merged episode + - Merged episode has parent_type="cluster" and session_id=None + - Adding session_id filter excludes the merged episode (session_id IS NULL) + """ + monkeypatch.setattr( + MemoryRoot, + "default", + classmethod(lambda cls: MemoryRoot(root=tmp_path)), + ) + monkeypatch.setenv("EVEROS_LLM__API_KEY", "fake-key") + monkeypatch.setenv("EVEROS_LLM__BASE_URL", "https://fake.example.com") + + load_settings.cache_clear() + + await _setup_sqlite(monkeypatch) + + try: + # -- LanceDB setup. + conn = await open_lancedb_connection(memory_root.lancedb_dir, LanceDBSettings()) + ep_table = await conn.create_table("episode", schema=LanceEpisode) + af_table = await conn.create_table("atomic_fact", schema=AtomicFact) + + ep_repo = _EpisodeRepo(table=ep_table) + af_repo = _AtomicFactRepo(table=af_table) + + owner_id = "u_test" + cluster_id = "cl_test_search" + + ts1 = _dt.datetime(2026, 6, 10, 10, 0, 0, tzinfo=_dt.UTC) + ts2 = _dt.datetime(2026, 6, 10, 11, 0, 0, tzinfo=_dt.UTC) + ts3 = _dt.datetime(2026, 6, 10, 12, 0, 0, tzinfo=_dt.UTC) + + ep1 = _make_lance_episode( + entry_id="ep_20260610_0001", + owner_id=owner_id, + episode="Andrew has no pets", + timestamp=ts1, + parent_id="mc_001", + ) + ep2 = _make_lance_episode( + entry_id="ep_20260610_0002", + owner_id=owner_id, + episode="Andrew adopted Toby", + timestamp=ts2, + parent_id="mc_002", + ) + ep3 = _make_lance_episode( + entry_id="ep_20260610_0003", + owner_id=owner_id, + episode="Andrew adopted Buddy", + timestamp=ts3, + parent_id="mc_003", + ) + await ep_repo.add([ep1, ep2, ep3]) + + # Atomic facts (needed for the orchestrator to complete). + fact1 = _make_lance_fact( + entry_id="af_20260610_0001", + owner_id=owner_id, + fact="Andrew has no pets", + parent_id="ep_20260610_0001", + timestamp=ts1, + ) + fact2 = _make_lance_fact( + entry_id="af_20260610_0002", + owner_id=owner_id, + fact="Andrew adopted Toby", + parent_id="ep_20260610_0002", + timestamp=ts2, + ) + fact3 = _make_lance_fact( + entry_id="af_20260610_0003", + owner_id=owner_id, + fact="Andrew adopted Buddy", + parent_id="ep_20260610_0003", + timestamp=ts3, + ) + await af_repo.add([fact1, fact2, fact3]) + + # -- SQLite: cluster with 3 memcell members. + centroid = np.zeros(1024, dtype=np.float32) + algo_cluster = AlgoCluster( + id=cluster_id, + centroid=centroid, + count=3, + last_ts=int(ts3.timestamp() * 1000), + preview=["Andrew has no pets", "Andrew adopted Toby"], + members=["ep_20260610_0001", "ep_20260610_0002", "ep_20260610_0003"], + ) + await cluster_repo.upsert_with_members( + algo_cluster, + owner_id=owner_id, + owner_type="user", + kind="user_memory", + member_type="episode", + ) + + # -- Run INIT Reflection. + merged_content = ( + "Andrew initially had no pets. He later adopted a dog named Toby, " + "and then adopted another dog named Buddy." + ) + merged_title = "Andrew's pet adoption journey" + reflect_response = json.dumps( + {"content": merged_content, "title": merged_title} + ) + fake_llm = FakeLLMClient(responses=[reflect_response]) + reflector = EpisodeReflector(llm=fake_llm) + episode_writer = EpisodeWriter(memory_root) + + orchestrator = ReflectionOrchestrator( + cluster_repo=cluster_repo, + episode_store=ep_repo, + atomic_fact_store=af_repo, + episode_writer=episode_writer, + report_repo=reflection_report_repo, + reflector=reflector, + embedder=_StubEmbedder(), + ) + + fake_ctx = FakeStrategyContext() + reports = await orchestrator.run(ctx=fake_ctx, owner_id=owner_id) + assert len(reports) == 1 + merged_entry_id = reports[0].merged_entry_id + + # FakeStrategyContext is a no-op, so the merged episode is not + # inserted into LanceDB by the extraction pipeline. Simulate the + # real pipeline by inserting the merged episode manually. + merged_ep = _make_lance_episode( + entry_id=merged_entry_id, + owner_id=owner_id, + episode=merged_content, + timestamp=ts3, + parent_type="cluster", + parent_id=cluster_id, + session_id=None, + ) + await ep_repo.add([merged_ep]) + + # -- Verify: non-deprecated episodes = only the merged one. + active_rows = await ep_repo.find_where( + f"owner_id = '{owner_id}' AND deprecated_by IS NULL" + ) + assert len(active_rows) == 1 + merged_row = active_rows[0] + assert merged_row.entry_id == merged_entry_id + assert merged_row.parent_type == "cluster" + assert merged_row.session_id is None + + # -- Verify: session_id filter excludes the merged episode. + session_rows = await ep_repo.find_where( + f"owner_id = '{owner_id}' AND deprecated_by IS NULL " + f"AND session_id = 's_test'" + ) + assert len(session_rows) == 0 + + finally: + conn.close() + await _teardown_sqlite() diff --git a/tests/run_locomo_10x3.sh b/tests/run_locomo_10x3.sh index b104d0d..7f33f56 100755 --- a/tests/run_locomo_10x3.sh +++ b/tests/run_locomo_10x3.sh @@ -46,7 +46,7 @@ REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." &> /dev/null && pwd)" # ── Defaults ────────────────────────────────────────────────────────── BASE_URL="${BASE_URL:-http://localhost:8000}" DATA_PATH="${DATA_PATH:-data/locomo10.json}" -MEMORY_ROOT="${EVEROS_MEMORY__ROOT:-$HOME/.everos-report-corpus}" +MEMORY_ROOT="${EVEROS_ROOT:-$HOME/.everos-report-corpus}" MODE="skip-add" # default; toggle via --fresh-corpus TS="$(date +%Y%m%d_%H%M%S)" OUTPUT_ROOT="$REPO_ROOT/benchmark_results/run_${TS}_10x3" @@ -83,7 +83,7 @@ echo # 1. Server up? if ! curl -fsS -o /dev/null "$BASE_URL/health" 2>/dev/null; then echo "❌ server at $BASE_URL is not responding" - echo " start with: EVEROS_MEMORY__ROOT=$MEMORY_ROOT PYTHONPATH=src \\" + echo " start with: EVEROS_ROOT=$MEMORY_ROOT PYTHONPATH=src \\" echo " python -m everos.entrypoints.cli.main server start --port 8000" exit 1 fi @@ -127,7 +127,7 @@ if [[ "$MODE" == "fresh" ]]; then echo " starting fresh server..." ( cd "$REPO_ROOT" - EVEROS_MEMORY__ROOT="$MEMORY_ROOT" \ + EVEROS_ROOT="$MEMORY_ROOT" \ PYTHONPATH=src \ nohup python -m everos.entrypoints.cli.main server start --port 8000 \ > /tmp/everos-server-${TS}.log 2>&1 & diff --git a/tests/run_locomo_batch.sh b/tests/run_locomo_batch.sh index 8f5a4eb..955ef5d 100755 --- a/tests/run_locomo_batch.sh +++ b/tests/run_locomo_batch.sh @@ -36,9 +36,9 @@ OUTPUT_ROOT="" CONCURRENCY="${CONCURRENCY:-1}" # Default to polling cascade pending==0 (not fixed sleep). Falls back to # ~/.everos to match the server's default data root; override via env or -# EVEROS_MEMORY__ROOT (which the server consumes). post-flush-wait becomes +# EVEROS_ROOT (which the server consumes). post-flush-wait becomes # the MAX wait when corpus-path is set. -CORPUS_PATH="${CORPUS_PATH:-${EVEROS_MEMORY__ROOT:-$HOME/.everos}}" +CORPUS_PATH="${CORPUS_PATH:-${EVEROS_ROOT:-$HOME/.everos}}" POST_FLUSH_WAIT="${POST_FLUSH_WAIT:-600}" EXTRA_ARGS=() diff --git a/tests/run_locomo_full.sh b/tests/run_locomo_full.sh new file mode 100644 index 0000000..b32dd6f --- /dev/null +++ b/tests/run_locomo_full.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────── +# Full LoCoMo Benchmark Runner +# +# Usage: +# bash tests/run_locomo_full.sh # default: 5 runs × hybrid,agentic +# bash tests/run_locomo_full.sh --runs 3 # 3 runs +# bash tests/run_locomo_full.sh --methods hybrid # hybrid only +# bash tests/run_locomo_full.sh --skip-add # reuse existing data +# ────────────────────────────────────────────────────────────────────── +set -euo pipefail + +RUNS="${RUNS:-5}" +METHODS="${METHODS:-hybrid,agentic}" +SKIP_ADD="" +DATA_PATH="data/locomo10.json" +CONVS=10 +POST_FLUSH_WAIT=180 +JUDGE_MODEL="gpt-4o-mini" +JUDGE_RUNS=5 +TOP_K=10 +OUTPUT_DIR="benchmark_results" +SEARCH_CONCURRENCY=1 + +# Parse args +while [[ $# -gt 0 ]]; do + case "$1" in + --runs) RUNS="$2"; shift 2 ;; + --methods) METHODS="$2"; shift 2 ;; + --skip-add) SKIP_ADD="--skip-add"; shift ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + --post-flush-wait) POST_FLUSH_WAIT="$2"; shift 2 ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac +done + +TS=$(date +%Y%m%d_%H%M%S) +RUN_DIR="${OUTPUT_DIR}/run_${TS}" +mkdir -p "$RUN_DIR" + +echo "════════════════════════════════════════════════════════════════" +echo " LoCoMo Full Benchmark" +echo " Runs: $RUNS | Methods: $METHODS | Convs: $CONVS" +echo " Judge: $JUDGE_MODEL (${JUDGE_RUNS} runs/question)" +echo " Output: $RUN_DIR" +echo "════════════════════════════════════════════════════════════════" + +# Phase 1: Add all 10 conversations (once) +if [[ -z "$SKIP_ADD" ]]; then + echo "" + echo "──── Phase 1: Loading all $CONVS conversations ────" + for conv in $(seq 0 $((CONVS - 1))); do + echo " Loading conv $conv..." + uv run python tests/test_locomo.py \ + --conv-index "$conv" \ + --methods hybrid \ + --data-path "$DATA_PATH" \ + --post-flush-wait "$POST_FLUSH_WAIT" \ + --judge-model "$JUDGE_MODEL" \ + --judge-runs 1 \ + --top-k 1 \ + --quiet \ + --search-concurrency 1 \ + --checkpoint-dir "$RUN_DIR/load_conv${conv}" \ + 2>&1 | tail -5 + echo " conv $conv loaded." + done + echo " All conversations loaded." + SKIP_ADD="--skip-add" +fi + +# Phase 2: Run benchmark (search + answer + judge) +IFS=',' read -ra METHOD_LIST <<< "$METHODS" + +for run_idx in $(seq 1 "$RUNS"); do + for method in "${METHOD_LIST[@]}"; do + echo "" + echo "══════════════════════════════════════════════════════════" + echo " Run $run_idx/$RUNS — method=$method" + echo "══════════════════════════════════════════════════════════" + + run_out="$RUN_DIR/${method}_run${run_idx}" + mkdir -p "$run_out" + summary_file="$run_out/summary.json" + + all_correct=0 + all_total=0 + + for conv in $(seq 0 $((CONVS - 1))); do + conv_out="$run_out/conv${conv}" + result_file="$conv_out/${method}_results.json" + + echo " conv $conv ($method, run $run_idx)..." + uv run python tests/test_locomo.py \ + --conv-index "$conv" \ + --methods "$method" \ + --data-path "$DATA_PATH" \ + --skip-add \ + --judge-model "$JUDGE_MODEL" \ + --judge-runs "$JUDGE_RUNS" \ + --top-k "$TOP_K" \ + --quiet \ + --search-concurrency "$SEARCH_CONCURRENCY" \ + --checkpoint-dir "$conv_out" \ + --output "$result_file" \ + 2>&1 | grep -E "Overall:|Done:" | head -3 + + # Extract accuracy from result JSON + if [[ -f "$result_file" ]]; then + conv_correct=$(python3 -c " +import json, sys +d = json.load(open('$result_file')) +s = d.get('methods', {}).get('$method', {}).get('summary', {}) +print(s.get('correct', 0)) +" 2>/dev/null || echo 0) + conv_total=$(python3 -c " +import json, sys +d = json.load(open('$result_file')) +s = d.get('methods', {}).get('$method', {}).get('summary', {}) +print(s.get('total', 0)) +" 2>/dev/null || echo 0) + all_correct=$((all_correct + conv_correct)) + all_total=$((all_total + conv_total)) + fi + done + + # Write run summary + if [[ $all_total -gt 0 ]]; then + accuracy=$(python3 -c "print(f'{$all_correct / $all_total * 100:.1f}')") + else + accuracy="0.0" + fi + echo "{\"method\": \"$method\", \"run\": $run_idx, \"correct\": $all_correct, \"total\": $all_total, \"accuracy\": $accuracy}" > "$summary_file" + echo " ── Run $run_idx $method: $all_correct / $all_total ($accuracy%) ──" + done +done + +# Final summary +echo "" +echo "════════════════════════════════════════════════════════════════" +echo " Final Results" +echo "════════════════════════════════════════════════════════════════" +for method in "${METHOD_LIST[@]}"; do + echo " $method:" + for run_idx in $(seq 1 "$RUNS"); do + summary="$RUN_DIR/${method}_run${run_idx}/summary.json" + if [[ -f "$summary" ]]; then + python3 -c " +import json +d = json.load(open('$summary')) +print(f\" Run {d['run']}: {d['correct']}/{d['total']} ({d['accuracy']}%)\") +" + fi + done +done +echo "════════════════════════════════════════════════════════════════" diff --git a/tests/test_locomo.py b/tests/test_locomo.py index a27e515..98d9654 100644 --- a/tests/test_locomo.py +++ b/tests/test_locomo.py @@ -81,14 +81,13 @@ It is CRITICAL that you move beyond simple fact extraction and perform logical i 2. ALWAYS include exact numbers, amounts, prices, percentages, dates, times 3. PRESERVE frequencies exactly - "every Tuesday and Thursday" not "twice a week" 4. MAINTAIN all proper nouns and entities as they appear +5. EXPLICITLY state confidence levels for inferences (High/Medium/Low) # RESPONSE FORMAT (You MUST follow this structure): ## STEP 1: RELEVANT MEMORIES EXTRACTION [List each memory that relates to the question, with its timestamp] -- Memory 1: [timestamp] - [content] -- Memory 2: [timestamp] - [content] -... +- Memory [ID]: [timestamp] - [content snippet] ## STEP 2: KEY INFORMATION IDENTIFICATION [Extract ALL specific details from the memories] @@ -98,35 +97,33 @@ It is CRITICAL that you move beyond simple fact extraction and perform logical i - Frequencies: [list any recurring patterns] - Other entities: [list brands, products, etc.] -## STEP 3: CROSS-MEMORY LINKING +## STEP 3: CROSS-MEMORY LINKING & INFERENCE [Identify entities that appear in multiple memories and link related information. Make reasonable inferences when entities are strongly connected.] - Shared entities: [list people, places, events mentioned across different memories] -- Connections found: [e.g., "Memory 1 mentions A moved from hometown → Memory 2 mentions A's hometown is LA → Therefore A moved from LA"] -- Inferred facts: [list any facts that require combining information from multiple memories] +- Connections found: [e.g., "Memory 1 mentions A moved from hometown -> Memory 2 mentions A's hometown is LA -> Therefore A moved from LA"] +- Inferences: [Connect the dots. Label confidence: (Confidence: High/Medium/Low)] ## STEP 4: TIME REFERENCE CALCULATION -[If applicable, convert relative time references] +[If applicable, convert relative time references using the timestamps] - Original reference: [e.g., "last year" from May 2022] -- Calculated actual time: [e.g., "2021"] +- Calculation: [Show logic] +- Actual time: [e.g., "2021"] -## STEP 5: CONTRADICTION CHECK -[If multiple memories contain different information] -- Conflicting information: [describe] -- Resolution: [explain which is most recent/reliable] +## STEP 5: CONTRADICTION & GAP ANALYSIS +[Check for conflicts and missing details] +- Conflicting information: [describe conflicts and resolution strategy] +- Missing information: [explicitly state what details are requested but missing from context] ## STEP 6: DETAIL VERIFICATION CHECKLIST -- [ ] All person names included: [list them] -- [ ] All locations included: [list them] -- [ ] All numbers exact: [list them] -- [ ] All frequencies specific: [list them] -- [ ] All dates/times precise: [list them] -- [ ] All proper nouns preserved: [list them] +- [ ] All person names included? +- [ ] All locations included? +- [ ] All numbers exact? +- [ ] All frequencies specific? +- [ ] All dates/times precise? +- [ ] All proper nouns preserved? -## STEP 7: ANSWER FORMULATION -[Explain how you're combining the information to answer the question] - -## FINAL ANSWER: -[Provide the concise answer with ALL specific details preserved] +## STEP 7: FINAL ANSWER +[Provide the concise answer with ALL specific details preserved. Do not include the internal checklist in this section, just the final synthesized answer.] --- @@ -730,41 +727,56 @@ def run_search_phase( # ============================================================================= +_CONTEXT_TEMPLATE = """Episodes memories for conversation between {speaker_a} and {speaker_b}: + + {episodes} +""" + + def _build_context( episodes: list[dict], profiles: list[dict], speaker_a: str, speaker_b: str ) -> str: - """Build context string from search results.""" - lines = [ - f"Episodes memories for conversation between {speaker_a} and {speaker_b}:\n" + """Build context string from search results. + + Matches the benchmark's context format: each episode renders as + ``{subject}: {episode_text}\\n---`` with double-newline separators. + Profile memories are intentionally omitted (benchmark doesn't use them). + """ + episode_lines = [ + f"{ep.get('subject', 'N/A')}: " + f"{ep.get('episode') or ep.get('summary') or ep.get('content') or 'N/A'}\n---" + for ep in episodes ] - for idx, ep in enumerate(episodes, 1): - subject = ep.get("subject", "") - body = ep.get("episode") or ep.get("summary") or ep.get("content") or "" - prefix = f"{subject}: " if subject else "" - lines.append(f"{idx}. {prefix}{body}") - - if profiles: - lines.append("\nProfile memories:") - for idx, p in enumerate(profiles, 1): - content = p.get("content") or p.get("summary") or "" - lines.append(f" {idx}. {content}") - - return "\n".join(lines) + return _CONTEXT_TEMPLATE.format( + speaker_a=speaker_a, + speaker_b=speaker_b, + episodes="\n\n".join(episode_lines), + ) def _extract_final_answer(text: str) -> str: - """Extract text after 'FINAL ANSWER:' marker.""" - marker = "FINAL ANSWER:" - idx = text.upper().rfind(marker.upper()) - if idx != -1: - answer = text[idx + len(marker) :].strip() - answer = re.sub(r"^#+\s*", "", answer).strip() - return answer - for line in reversed(text.strip().splitlines()): - line = line.strip() - if line: - return line - return text.strip() + """Extract the final answer using a 3-marker priority chain. + + Matches the benchmark's extraction logic (``answer.py:_extract_final_answer``): + 1. ``## STEP 7: FINAL ANSWER`` (prompt STEP 7 section header) + 2. ``FINAL ANSWER:`` (colon-suffixed) + 3. ``FINAL ANSWER`` (bare — leading colon stripped if present) + + Each marker uses ``rsplit`` to take the LAST occurrence (handles marker + appearing in reasoning prose before the actual answer). + """ + result = text.strip() + for marker in ("## STEP 7: FINAL ANSWER", "FINAL ANSWER:", "FINAL ANSWER"): + if marker in result: + answer = result.rsplit(marker, 1)[1].strip() + # Bare "FINAL ANSWER" may have a leading ":" — strip it + if marker == "FINAL ANSWER" and answer.startswith(":"): + answer = answer[1:].strip() + return answer + return result + + +_ANSWER_MAX_RETRIES = 5 def _answer_one( @@ -778,11 +790,9 @@ def _answer_one( ) -> dict: """Generate an answer for a single search result; safe to run in a thread. - Retry up to 3× with rising temperature when the response parses to an - empty FINAL ANSWER:. gpt-4.1-mini occasionally finishes the STEP 7 - reasoning, emits the marker, then stops without the body — at temperature=0 - the truncation is deterministic, so retries bump temperature to break the - same sampling path. + Retry up to 5x with temperature=0.0 on every attempt (matching benchmark's + ``_retry_llm_answer``). max_tokens=32768 is set explicitly to match the + benchmark. The openai ``timeout=300`` kwarg is a per-request socket deadline passed directly to the underlying HTTP client, which is safe to use from a thread @@ -799,24 +809,29 @@ def _answer_one( generated_answer = "" last_error: str | None = None attempts_used = 0 - for attempt, temp in enumerate((0.0, 0.3, 0.6)): + for attempt in range(_ANSWER_MAX_RETRIES): attempts_used = attempt + 1 try: r = llm_client.chat.completions.create( model=llm_model, messages=[{"role": "user", "content": prompt}], - temperature=temp, + temperature=0.0, + max_tokens=32768, timeout=300, ) raw_answer = r.choices[0].message.content or "" except Exception as e: last_error = f"[ERROR: {e}]" raw_answer = last_error + if attempt < _ANSWER_MAX_RETRIES - 1: + time.sleep(1.0 * (2**attempt)) continue generated_answer = _extract_final_answer(raw_answer) if generated_answer.strip(): break + if attempt < _ANSWER_MAX_RETRIES - 1: + time.sleep(1.0 * (2**attempt)) if not generated_answer.strip() and last_error: generated_answer = last_error @@ -894,6 +909,9 @@ def _extract_json(content: str) -> str | None: return content.strip() +_JUDGE_MAX_RETRIES = 5 + + def _judge_single( llm_client: LLMClientPool, llm_model: str, @@ -903,6 +921,11 @@ def _judge_single( ) -> bool: """Judge a single answer. Returns True if CORRECT. + Retries up to ``_JUDGE_MAX_RETRIES`` times on any error (API failures, + JSON parse errors, missing label) with exponential backoff, matching the + benchmark's ``llm_retry(max_attempts=config.llm_max_retries)`` pattern. + Defaults to WRONG only after all retries are exhausted. + Uses ``timeout=300`` passed directly to the openai HTTP client so this function is safe to call from a thread pool without further nesting. """ @@ -911,25 +934,33 @@ def _judge_single( golden_answer=golden_answer, generated_answer=generated_answer, ) - try: - r = llm_client.chat.completions.create( - model=llm_model, - messages=[ - {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, - {"role": "user", "content": user_prompt}, - ], - temperature=0, - timeout=300, - ) - content = r.choices[0].message.content or "" - json_str = _extract_json(content) - if not json_str: + for attempt in range(_JUDGE_MAX_RETRIES): + try: + r = llm_client.chat.completions.create( + model=llm_model, + messages=[ + {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ], + temperature=0, + timeout=300, + ) + content = r.choices[0].message.content or "" + json_str = _extract_json(content) + if not json_str: + raise ValueError("Empty JSON from judge response") + result = json.loads(json_str) + label = result.get("label", "").strip().upper() + if label not in ("CORRECT", "WRONG"): + raise ValueError(f"Unknown judge label: {label!r}") + return label == "CORRECT" + except Exception as e: # noqa: BLE001 + if attempt < _JUDGE_MAX_RETRIES - 1: + time.sleep(0.5 * (2**attempt)) + continue + print(f" Judge error after {_JUDGE_MAX_RETRIES} retries: {e}") return False - result = json.loads(json_str) - return result.get("label", "").strip().upper() == "CORRECT" - except Exception as e: - print(f" Judge error: {e}") - return False + return False # unreachable, but satisfies type checker def _evaluate_one( @@ -1337,6 +1368,29 @@ def parse_args() -> argparse.Namespace: choices=["speaker_a", "speaker_b"], help="Which speaker's memory partition to query (Plan C: single-owner eval)", ) + p.add_argument( + "--smoke", + action="store_true", + help="Smoke mode: 2 sessions x 5 QA. Quick sanity check, not a scored run.", + ) + p.add_argument( + "--smoke-session-limit", + type=int, + default=2, + help="Max sessions to load in smoke mode (default: 2)", + ) + p.add_argument( + "--smoke-msg-limit", + type=int, + default=50, + help="Max messages per session in smoke mode (default: 50)", + ) + p.add_argument( + "--smoke-qa-limit", + type=int, + default=5, + help="Max QA pairs in smoke mode (default: 5)", + ) p.add_argument( "--skip-add", action="store_true", help="Skip add phase (reuse existing data)" ) @@ -1425,26 +1479,27 @@ def main(): load_dotenv() - # Resolution: CLI flag > ANSWER_*/JUDGE_* env > LLM_* env > default. + # Resolution: CLI flag > ANSWER_*/JUDGE_* env > LLM_* env > EVEROS_LLM__* env > default. # Empty strings from getenv fall through via `or`. answer_model = ( args.answer_model or os.getenv("ANSWER_MODEL") or os.getenv("LLM_MODEL") - or "gpt-4o-mini" + or os.getenv("EVEROS_LLM__MODEL") + or "gpt-4.1-mini" ) answer_base_url = ( args.answer_base_url or os.getenv("ANSWER_BASE_URL") or os.getenv("LLM_BASE_URL") + or os.getenv("EVEROS_LLM__BASE_URL") or "https://api.openai.com/v1" ) - # API keys are comma-separated lists; the LLMClientPool round-robins across - # them and fails over to the next on RateLimitError. answer_api_keys = _split_keys( args.answer_api_key or os.getenv("ANSWER_API_KEY") or os.getenv("LLM_API_KEY") + or os.getenv("EVEROS_LLM__API_KEY") or "" ) @@ -1452,18 +1507,21 @@ def main(): args.judge_model or os.getenv("JUDGE_MODEL") or os.getenv("LLM_MODEL") - or "gpt-4o-mini" + or os.getenv("EVEROS_LLM__MODEL") + or "gpt-4.1-mini" ) judge_base_url = ( args.judge_base_url or os.getenv("JUDGE_BASE_URL") or os.getenv("LLM_BASE_URL") + or os.getenv("EVEROS_LLM__BASE_URL") or "https://api.openai.com/v1" ) judge_api_keys = _split_keys( args.judge_api_key or os.getenv("JUDGE_API_KEY") or os.getenv("LLM_API_KEY") + or os.getenv("EVEROS_LLM__API_KEY") or "" ) @@ -1505,10 +1563,18 @@ def main(): # 1. Load data (preserve LoCoMo session boundaries) print_section("Loading Data") sessions, qa_list, spk_a, spk_b = load_conversation(args.data_path, args.conv_index) + + if args.smoke: + sessions = sessions[: args.smoke_session_limit] + for s in sessions: + s["messages"] = s["messages"][: args.smoke_msg_limit] + qa_list = qa_list[: args.smoke_qa_limit] + conv_label = f"conv_{args.conv_index} ({spk_a} & {spk_b})" total_msgs = sum(len(s["messages"]) for s in sessions) + mode_tag = " [SMOKE]" if args.smoke else "" print( - f" Conversation: {conv_label}\n" + f" Conversation: {conv_label}{mode_tag}\n" f" LoCoMo sessions: {len(sessions)} | Messages: {total_msgs} | " f"QA pairs: {len(qa_list)} (excl. category 5)" ) @@ -1529,7 +1595,13 @@ def main(): "benchmark_checkpoints", f"run_{ts}_conv{args.conv_index}" ) - # 4. Add phase + # 4. Smoke-mode defaults: less wait, single judge run. + if args.smoke: + if args.post_flush_wait == 180: + args.post_flush_wait = 60 + if args.judge_runs == 3: + args.judge_runs = 1 + add_result = None if not args.skip_add: add_result = run_add_phase( diff --git a/tests/test_reflection_e2e.py b/tests/test_reflection_e2e.py new file mode 100644 index 0000000..bb26a54 --- /dev/null +++ b/tests/test_reflection_e2e.py @@ -0,0 +1,810 @@ +"""Reflection E2E test -- validates the full Reflection pipeline with real +LLM, real embedder, and LoCoMo conversation data. + +Usage: + python tests/test_reflection_e2e.py # run all TCs + python tests/test_reflection_e2e.py --tc 1,2,14 # run selected TCs + python tests/test_reflection_e2e.py --verbose # verbose output +""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys +import time +from pathlib import Path +from typing import Any + +# Path setup for sibling-module import (test_locomo lives in the same dir). +# load_dotenv() is deferred to main() to avoid module-level side effects. +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from test_locomo import ( # noqa: E402 + ANSWER_PROMPT, + JUDGE_SYSTEM_PROMPT, + JUDGE_USER_PROMPT, + EverosClient, + LLMClientPool, + _build_context, + _extract_final_answer, + _extract_json, + _parse_session_timestamp, + print_section, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Constants — session indices per storyline and golden data +# --------------------------------------------------------------------------- + +DATA_PATH = Path(__file__).resolve().parent.parent / "data" / "locomo10.json" + +ADOPTION_INIT_SESSIONS = [2, 8, 13, 17] +ADOPTION_UPDATE_SESSIONS = [19] +LGBTQ_INIT_SESSIONS = [1, 3, 5, 12] +LGBTQ_UPDATE_SESSIONS = [14] +PET_INIT_SESSIONS = [1, 5, 12, 24] +PET_UPDATE_SESSIONS = [27, 28] +HEALTH_INIT_SESSIONS = [2, 4, 8, 10, 13, 14] +HEALTH_UPDATE_SESSIONS = [16, 20] + +QUERIES = { + "adoption": "What steps has Caroline taken toward adoption?", + "lgbtq": "How has Caroline dealt with discrimination?", + "pet": "How many pets does Andrew have and what are their names?", + "health": "How has Sam's diet and health journey been going?", +} + +GOLDEN_FACTS = { + "adoption": [ + "research", + "adoption council", + "applied", + "mentor", + "interview", + ], + "lgbtq": [ + "support group", + "school", + "pride", + "discriminat", + "apolog", + ], + "pet": [ + "no pet", + "toby", + "buddy", + "scout", + ], + "health": [ + "doctor", + "diet", + "before and after", + "snack", + "gastritis", + "weight watchers", + "struggl", + ], +} + +# --------------------------------------------------------------------------- +# Data parsing +# --------------------------------------------------------------------------- + + +def load_locomo() -> list[dict[str, Any]]: + """Load the LoCoMo dataset from the project data directory.""" + with open(DATA_PATH) as f: + return json.load(f) + + +def parse_sessions( + conv: dict[str, Any], + session_indices: list[int], + conv_index: int, +) -> list[dict[str, Any]]: + """Parse LoCoMo sessions into the everos /add message format. + + Returns a list of dicts, each with ``session_idx``, ``session_id``, + and ``messages`` (ready for the ``/api/v1/memory/add`` payload). + """ + raw = conv["conversation"] + results: list[dict[str, Any]] = [] + for idx in session_indices: + key = f"session_{idx}" + if key not in raw: + raise ValueError(f"session {key} not found in conv {conv_index}") + date_key = f"{key}_date_time" + base_ts = _parse_session_timestamp(raw.get(date_key, "")) + session_id = f"refl_conv{conv_index}_s{idx}" + messages: list[dict[str, Any]] = [] + for i, dia in enumerate(raw[key]): + messages.append( + { + "sender_id": f"{dia['speaker'].lower()}_conv{conv_index}", + "sender_name": dia["speaker"], + "role": "user", + "timestamp": base_ts + i * 30, + "content": [{"type": "text", "text": dia["text"]}], + } + ) + results.append( + { + "session_idx": idx, + "session_id": session_id, + "messages": messages, + } + ) + return results + + +# --------------------------------------------------------------------------- +# Infrastructure helpers +# --------------------------------------------------------------------------- + + +_SYSTEM_DB = DATA_PATH.parent.parent / ".everos" / ".index" / "sqlite" / "system.db" + + +def print_episode_locations( + owner_id: str, + episodes: list[dict[str, Any]], +) -> None: + """Print md paths for human review of merged vs source episodes.""" + merged = [e for e in episodes if e.get("session_id") is None] + original = [e for e in episodes if e.get("session_id") is not None] + print(f"\n episode locations ({owner_id}):") + if merged: + for ep in merged: + print(f" [MERGED] {ep.get('id', '?')}") + if original: + for ep in original[:3]: + print(f" [source] {ep.get('id', '?')} session={ep.get('session_id')}") + if len(original) > 3: + print(f" ... and {len(original) - 3} more sources") + root = str(DATA_PATH.parent.parent / ".everos") + print(f" md root: {root}") + + +def _owner_id(speaker: str, conv_index: int) -> str: + """Build the canonical owner_id for a speaker in a conversation.""" + return f"{speaker.lower()}_conv{conv_index}" + + +def count_reflection_reports(owner_id: str) -> int: + """Query SQLite directly to count reflection reports for an owner.""" + import sqlite3 + + conn = sqlite3.connect(str(_SYSTEM_DB)) + try: + cur = conn.execute( + "SELECT count(*) FROM reflection_report WHERE owner_id = ?", + (owner_id,), + ) + return cur.fetchone()[0] + finally: + conn.close() + + +def count_deprecated_episodes(owner_id: str) -> int: + """Query LanceDB via search with a special filter is not possible from + outside the server. Instead check reflection_report source_count as proxy.""" + import sqlite3 + + conn = sqlite3.connect(str(_SYSTEM_DB)) + try: + cur = conn.execute( + "SELECT coalesce(sum(source_count), 0) " + "FROM reflection_report WHERE owner_id = ?", + (owner_id,), + ) + return cur.fetchone()[0] + finally: + conn.close() + + +def add_and_flush( + client: EverosClient, + sessions: list[dict[str, Any]], + *, + quiet: bool = True, +) -> None: + """Ingest sessions: /add all messages first, then /flush each session.""" + for sess in sessions: + payload = {"session_id": sess["session_id"], "messages": sess["messages"]} + status, _ = client.post("/api/v1/memory/add", payload, quiet=quiet) + assert status == 200, f"add failed for {sess['session_id']}: {status}" + + for sess in sessions: + status, _ = client.post( + "/api/v1/memory/flush", + {"session_id": sess["session_id"]}, + quiet=quiet, + ) + assert status == 200, f"flush failed for {sess['session_id']}: {status}" + + +def wait_pipeline(seconds: int = 180) -> None: + """Wait for cascade + OME pipeline to settle after flush.""" + print(f" waiting {seconds}s for pipeline to settle...") + time.sleep(seconds) # tz-noqa — wall-clock delay, not a datetime + print(" pipeline wait done") + + +def trigger_reflection( + client: EverosClient, + *, + timeout: float = 120.0, +) -> None: + """Trigger Reflection via HTTP endpoint on the running server.""" + print(" triggering reflection via HTTP...") + status, resp = client.post( + "/api/v1/ome/trigger", + {"name": "reflect_episodes", "timeout": timeout, "force": True}, + quiet=True, + ) + result_status = resp.get("status", "unknown") if isinstance(resp, dict) else "error" + print(f" trigger response: status={result_status}") + if status != 200 or result_status != "ok": + raise RuntimeError(f"reflection trigger failed: HTTP {status}, {resp}") + + +def search_episodes( + client: EverosClient, + query: str, + owner_id: str, + *, + method: str = "hybrid", + top_k: int = 10, +) -> dict[str, Any]: + """Run a memory search and return the ``data`` payload.""" + payload = { + "query": query, + "method": method, + "top_k": top_k, + "user_id": owner_id, + } + status, resp = client.post("/api/v1/memory/search", payload, quiet=True) + assert status == 200, f"search failed: {status}" + return resp.get("data", {}) + + +def answer_and_judge( + query: str, + search_data: dict[str, Any], + golden_answer: str, + *, + speaker_a: str, + speaker_b: str, + llm_client: LLMClientPool, + llm_model: str, +) -> dict[str, Any]: + """Generate an answer from search results and judge correctness. + + Returns a dict with ``answer``, ``judge_score`` (0 or 1), and + ``episodes_count``. + """ + context = _build_context( + search_data.get("episodes", []), + search_data.get("profiles", []), + speaker_a, + speaker_b, + ) + prompt = ANSWER_PROMPT.format(context=context, question=query) + try: + resp = llm_client.chat.completions.create( + model=llm_model, + messages=[{"role": "user", "content": prompt}], + temperature=0.0, + ) + answer = _extract_final_answer(resp.choices[0].message.content or "") + except Exception as e: + answer = f"[error: {e}]" + + try: + judge_resp = llm_client.chat.completions.create( + model=llm_model, + messages=[ + {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, + { + "role": "user", + "content": JUDGE_USER_PROMPT.format( + question=query, + golden_answer=golden_answer, + generated_answer=answer, + ), + }, + ], + temperature=0.0, + ) + judge_text = judge_resp.choices[0].message.content or "" + raw_json = _extract_json(judge_text) + if raw_json: + parsed = json.loads(raw_json) + is_correct = parsed.get("label", "").upper() == "CORRECT" + else: + is_correct = False + except Exception: + logger.warning("judge evaluation failed", exc_info=True) + is_correct = False + + return { + "answer": answer, + "judge_score": 1 if is_correct else 0, + "episodes_count": len(search_data.get("episodes", [])), + } + + +def compute_fact_coverage(text: str, facts: list[str]) -> float: + """Compute fraction of golden facts found (case-insensitive substring).""" + text_lower = text.lower() + hits = sum(1 for f in facts if f.lower() in text_lower) + return hits / len(facts) if facts else 0.0 + + +# --------------------------------------------------------------------------- +# TCResult — lightweight per-test-case assertion tracker +# --------------------------------------------------------------------------- + + +class TCResult: + """Accumulate pass/fail checks for a single test case.""" + + def __init__(self, name: str) -> None: + self.name = name + self.passed: list[str] = [] + self.failed: list[str] = [] + + def check(self, condition: bool, description: str) -> None: + (self.passed if condition else self.failed).append(description) + + @property + def ok(self) -> bool: + return len(self.failed) == 0 + + def print_summary(self) -> None: + status = "PASS" if self.ok else "FAIL" + print(f"\n {self.name}: {status}") + for p in self.passed: + print(f" [ok] {p}") + for f in self.failed: + print(f" [FAIL] {f}") + + +# --------------------------------------------------------------------------- +# Test cases (TC1-TC8) — INIT + UPDATE per storyline +# --------------------------------------------------------------------------- + + +def tc1_adoption_init(client: EverosClient) -> TCResult: + tc = TCResult("TC1: Adoption INIT") + print_section("TC1: Adoption INIT (conv0, sessions 2,8,13,17)") + owner = _owner_id("caroline", 0) + data = load_locomo() + sessions = parse_sessions(data[0], ADOPTION_INIT_SESSIONS, 0) + add_and_flush(client, sessions) + wait_pipeline() + trigger_reflection(client) + # Positive: reflection report was written (deprecation completed) + reports = count_reflection_reports(owner) + tc.check(reports >= 1, f"reflection report created ({reports} found)") + dep_count = count_deprecated_episodes(owner) + tc.check(dep_count >= 1, f"source episodes deprecated ({dep_count} source_count)") + # Search: merged episode visible, deprecated filtered out + result = search_episodes(client, QUERIES["adoption"], owner) + episodes = result.get("episodes", []) + tc.check(len(episodes) > 0, "search returns episodes") + merged = [e for e in episodes if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists (session_id=None)") + print_episode_locations(owner, episodes) + tc.print_summary() + return tc + + +def tc2_adoption_update(client: EverosClient) -> TCResult: + tc = TCResult("TC2: Adoption UPDATE") + print_section("TC2: Adoption UPDATE (conv0, session 19)") + owner = _owner_id("caroline", 0) + reports_before = count_reflection_reports(owner) + data = load_locomo() + sessions = parse_sessions(data[0], ADOPTION_UPDATE_SESSIONS, 0) + add_and_flush(client, sessions) + wait_pipeline() + trigger_reflection(client) + reports_after = count_reflection_reports(owner) + tc.check( + reports_after > reports_before, + f"report count up ({reports_before}->{reports_after})", + ) + result = search_episodes(client, QUERIES["adoption"], owner) + merged = [e for e in result.get("episodes", []) if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists after update") + print_episode_locations(owner, result.get("episodes", [])) + tc.print_summary() + return tc + + +def tc3_lgbtq_init(client: EverosClient) -> TCResult: + tc = TCResult("TC3: LGBTQ+Conflict INIT") + print_section("TC3: LGBTQ+Conflict INIT (conv0, sessions 1,3,5,12)") + owner = _owner_id("caroline", 0) + data = load_locomo() + sessions = parse_sessions(data[0], LGBTQ_INIT_SESSIONS, 0) + add_and_flush(client, sessions) + wait_pipeline() + trigger_reflection(client) + reports = count_reflection_reports(owner) + tc.check(reports >= 1, f"reflection report(s) exist ({reports})") + result = search_episodes(client, QUERIES["lgbtq"], owner) + merged = [e for e in result.get("episodes", []) if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists") + print_episode_locations(owner, result.get("episodes", [])) + tc.print_summary() + return tc + + +def tc4_lgbtq_update(client: EverosClient) -> TCResult: + tc = TCResult("TC4: LGBTQ+Conflict UPDATE") + print_section("TC4: LGBTQ+Conflict UPDATE (conv0, session 14)") + owner = _owner_id("caroline", 0) + reports_before = count_reflection_reports(owner) + data = load_locomo() + sessions = parse_sessions(data[0], LGBTQ_UPDATE_SESSIONS, 0) + add_and_flush(client, sessions) + wait_pipeline() + trigger_reflection(client) + reports_after = count_reflection_reports(owner) + tc.check( + reports_after > reports_before, + f"report count up ({reports_before}->{reports_after})", + ) + result = search_episodes(client, QUERIES["lgbtq"], owner) + merged = [e for e in result.get("episodes", []) if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists after update") + print_episode_locations(owner, result.get("episodes", [])) + tc.print_summary() + return tc + + +def tc5_pet_init(client: EverosClient) -> TCResult: + tc = TCResult("TC5: Pet Count INIT") + print_section("TC5: Pet Count INIT (conv5, sessions 1,5,12,24)") + owner = _owner_id("andrew", 5) + data = load_locomo() + sessions = parse_sessions(data[5], PET_INIT_SESSIONS, 5) + add_and_flush(client, sessions) + wait_pipeline() + trigger_reflection(client) + reports = count_reflection_reports(owner) + tc.check(reports >= 1, f"reflection report created ({reports})") + result = search_episodes(client, QUERIES["pet"], owner) + merged = [e for e in result.get("episodes", []) if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists") + print_episode_locations(owner, result.get("episodes", [])) + tc.print_summary() + return tc + + +def tc6_pet_update(client: EverosClient) -> TCResult: + tc = TCResult("TC6: Pet Count UPDATE") + print_section("TC6: Pet Count UPDATE (conv5, sessions 27,28)") + owner = _owner_id("andrew", 5) + reports_before = count_reflection_reports(owner) + data = load_locomo() + sessions = parse_sessions(data[5], PET_UPDATE_SESSIONS, 5) + add_and_flush(client, sessions) + wait_pipeline() + trigger_reflection(client) + reports_after = count_reflection_reports(owner) + tc.check( + reports_after > reports_before, + f"report count up ({reports_before}->{reports_after})", + ) + result = search_episodes(client, QUERIES["pet"], owner) + merged = [e for e in result.get("episodes", []) if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists after update") + print_episode_locations(owner, result.get("episodes", [])) + tc.print_summary() + return tc + + +def tc7_health_init(client: EverosClient) -> TCResult: + tc = TCResult("TC7: Health Relapse INIT") + print_section("TC7: Health INIT (conv8, sessions 2,4,8,10,13,14)") + owner = _owner_id("sam", 8) + data = load_locomo() + sessions = parse_sessions(data[8], HEALTH_INIT_SESSIONS, 8) + add_and_flush(client, sessions) + wait_pipeline() + trigger_reflection(client) + reports = count_reflection_reports(owner) + tc.check(reports >= 1, f"reflection report created ({reports})") + result = search_episodes(client, QUERIES["health"], owner) + merged = [e for e in result.get("episodes", []) if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists") + print_episode_locations(owner, result.get("episodes", [])) + tc.print_summary() + return tc + + +def tc8_health_update(client: EverosClient) -> TCResult: + tc = TCResult("TC8: Health Relapse UPDATE") + print_section("TC8: Health UPDATE (conv8, sessions 16,20)") + owner = _owner_id("sam", 8) + reports_before = count_reflection_reports(owner) + data = load_locomo() + sessions = parse_sessions(data[8], HEALTH_UPDATE_SESSIONS, 8) + add_and_flush(client, sessions) + wait_pipeline() + trigger_reflection(client) + reports_after = count_reflection_reports(owner) + tc.check( + reports_after > reports_before, + f"report count up ({reports_before}->{reports_after})", + ) + result = search_episodes(client, QUERIES["health"], owner) + merged = [e for e in result.get("episodes", []) if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists after update") + print_episode_locations(owner, result.get("episodes", [])) + tc.print_summary() + return tc + + +# --------------------------------------------------------------------------- +# Test cases (TC9, TC11-TC14) — cross-cutting validation +# --------------------------------------------------------------------------- + + +def tc9_search_visibility(client: EverosClient) -> TCResult: + tc = TCResult("TC9: Search Visibility") + print_section("TC9: Search Visibility") + checks = [ + (QUERIES["adoption"], _owner_id("caroline", 0)), + (QUERIES["lgbtq"], _owner_id("caroline", 0)), + (QUERIES["pet"], _owner_id("andrew", 5)), + (QUERIES["health"], _owner_id("sam", 8)), + ] + for query, owner in checks: + # Positive: deprecation actually happened + reports = count_reflection_reports(owner) + tc.check(reports >= 1, f"reports exist for {owner} ({reports})") + # Search: merged visible, deprecated filtered + data = search_episodes(client, query, owner) + episodes = data.get("episodes", []) + merged = [e for e in episodes if e.get("session_id") is None] + tc.check(len(merged) >= 1, f"merged present for '{query[:40]}...'") + tc.print_summary() + return tc + + +def tc11_idempotency(client: EverosClient) -> TCResult: + tc = TCResult("TC11: Idempotency") + print_section("TC11: Idempotency") + owner = _owner_id("caroline", 0) + before = search_episodes(client, QUERIES["adoption"], owner) + merged_before = [ + e for e in before.get("episodes", []) if e.get("session_id") is None + ] + count_before = len(merged_before) + trigger_reflection(client) + after = search_episodes(client, QUERIES["adoption"], owner) + merged_after = [e for e in after.get("episodes", []) if e.get("session_id") is None] + tc.check( + len(merged_after) == count_before, + f"merged count unchanged ({count_before} -> {len(merged_after)})", + ) + if merged_before and merged_after: + tc.check( + merged_before[0].get("id") == merged_after[0].get("id"), + "same merged episode ID (no duplicate)", + ) + tc.print_summary() + return tc + + +def tc12_atomic_facts(client: EverosClient) -> TCResult: + tc = TCResult("TC12: Atomic Facts Re-extraction") + print_section("TC12: Atomic Facts Re-extraction") + owner = _owner_id("caroline", 0) + result = search_episodes(client, QUERIES["adoption"], owner) + merged = [e for e in result.get("episodes", []) if e.get("session_id") is None] + tc.check(len(merged) >= 1, "merged episode exists") + if merged: + facts = merged[0].get("atomic_facts", []) + tc.check(len(facts) > 0, f"merged has atomic facts ({len(facts)} found)") + tc.print_summary() + return tc + + +def tc13_topic_isolation(client: EverosClient) -> TCResult: + tc = TCResult("TC13: Cross-topic Isolation") + print_section("TC13: Cross-topic Isolation") + owner = _owner_id("caroline", 0) + adoption = search_episodes(client, QUERIES["adoption"], owner) + lgbtq = search_episodes(client, QUERIES["lgbtq"], owner) + a_merged = [e for e in adoption.get("episodes", []) if e.get("session_id") is None] + l_merged = [e for e in lgbtq.get("episodes", []) if e.get("session_id") is None] + tc.check(len(a_merged) >= 1, "adoption has merged episode") + tc.check(len(l_merged) >= 1, "lgbtq has merged episode") + if a_merged and l_merged: + tc.check( + a_merged[0].get("id") != l_merged[0].get("id"), + "different merged episode IDs", + ) + a_text = a_merged[0].get("episode", "").lower() + l_text = l_merged[0].get("episode", "").lower() + tc.check( + "discriminat" not in a_text and "hike" not in a_text, + "adoption text has no discrimination content", + ) + tc.check( + "agenc" not in l_text and "adoption council" not in l_text, + "lgbtq text has no adoption process content", + ) + tc.print_summary() + return tc + + +def tc14_answer_judge( + client: EverosClient, + llm_client: LLMClientPool, + llm_model: str, +) -> TCResult: + tc = TCResult("TC14: Answer+Judge Quality") + print_section("TC14: Answer+Judge Quality Comparison") + + golden_answers = { + "adoption": ( + "Caroline researched adoption agencies, attended an adoption council " + "meeting, applied to multiple agencies, contacted her mentor for advice, " + "and passed the adoption agency interviews." + ), + "lgbtq": ( + "Caroline dealt with discrimination by attending LGBTQ support groups, " + "speaking at her school, participating in a Pride parade. When she " + "encountered discrimination on a hike from religious conservatives, " + "she later wrote an apology letter to reconcile." + ), + "pet": ( + "Andrew has three dogs: Toby, Buddy, and Scout. He initially had no " + "pets, then adopted Toby, followed by Buddy from a shelter, and most " + "recently Scout." + ), + "health": ( + "Sam's journey has been non-linear. After a doctor warned about his " + "weight, he started dieting with good results. But he relapsed by " + "buying unhealthy snacks, then had a gastritis emergency. He recovered " + "to become a Weight Watchers coach, but later struggled again." + ), + } + owner_map = { + "adoption": (_owner_id("caroline", 0), "Caroline", "Melanie"), + "lgbtq": (_owner_id("caroline", 0), "Caroline", "Melanie"), + "pet": (_owner_id("andrew", 5), "Audrey", "Andrew"), + "health": (_owner_id("sam", 8), "Evan", "Sam"), + } + + total_score = 0 + for topic, query in QUERIES.items(): + owner, speaker_a, speaker_b = owner_map[topic] + data = search_episodes(client, query, owner) + result = answer_and_judge( + query, + data, + golden_answers[topic], + speaker_a=speaker_a, + speaker_b=speaker_b, + llm_client=llm_client, + llm_model=llm_model, + ) + score = result["judge_score"] + total_score += score + merged = [e for e in data.get("episodes", []) if e.get("session_id") is None] + merged_text = merged[0].get("episode", "") if merged else "" + coverage = compute_fact_coverage(merged_text, GOLDEN_FACTS[topic]) + status = "CORRECT" if score else "WRONG" + print(f" {topic}: {status} | fact_coverage={coverage:.0%}") + print(f" answer: {result['answer'][:120]}...") + tc.check(score == 1, f"{topic} answered correctly") + + print(f"\n Overall: {total_score}/{len(QUERIES)}") + tc.print_summary() + return tc + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +TC_REGISTRY: dict[int, tuple[str, Any]] = { + 1: ("Adoption INIT", lambda c, **kw: tc1_adoption_init(c)), + 2: ("Adoption UPDATE", lambda c, **kw: tc2_adoption_update(c)), + 3: ("LGBTQ INIT", lambda c, **kw: tc3_lgbtq_init(c)), + 4: ("LGBTQ UPDATE", lambda c, **kw: tc4_lgbtq_update(c)), + 5: ("Pet INIT", lambda c, **kw: tc5_pet_init(c)), + 6: ("Pet UPDATE", lambda c, **kw: tc6_pet_update(c)), + 7: ("Health INIT", lambda c, **kw: tc7_health_init(c)), + 8: ("Health UPDATE", lambda c, **kw: tc8_health_update(c)), + 9: ("Search Visibility", lambda c, **kw: tc9_search_visibility(c)), + # TC10 removed: was a duplicate of TC9 visibility checks. + 11: ("Idempotency", lambda c, **kw: tc11_idempotency(c)), + 12: ("Atomic Facts", lambda c, **kw: tc12_atomic_facts(c)), + 13: ("Topic Isolation", lambda c, **kw: tc13_topic_isolation(c)), + 14: ( + "Answer+Judge", + lambda c, **kw: tc14_answer_judge(c, kw["llm_client"], kw["llm_model"]), + ), +} + + +def main() -> None: + import os + + from dotenv import load_dotenv + + load_dotenv() + + parser = argparse.ArgumentParser(description="Reflection E2E Test") + parser.add_argument( + "--tc", + type=str, + default=None, + help="Comma-separated TC numbers (e.g. '1,2,14'). Default: all.", + ) + parser.add_argument("--base-url", default="http://localhost:8000") + parser.add_argument("--llm-model", default=None) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.WARNING, + format="%(levelname)s %(name)s: %(message)s", + ) + + tc_ids = ( + [int(x.strip()) for x in args.tc.split(",")] + if args.tc + else sorted(TC_REGISTRY.keys()) + ) + + client = EverosClient(base_url=args.base_url) + llm_model = args.llm_model or os.getenv("EVEROS_LLM__MODEL", "openai/gpt-4.1-mini") + api_key = os.getenv("EVEROS_LLM__API_KEY", "") + base_url = os.getenv("EVEROS_LLM__BASE_URL", "https://openrouter.ai/api/v1") + llm_client = LLMClientPool(api_keys=[api_key], base_url=base_url) + + print_section("Reflection E2E Test") + print(f" TCs: {tc_ids}") + print(f" Server: {args.base_url}") + print(f" LLM: {llm_model}") + + results: list[TCResult] = [] + for tc_id in tc_ids: + if tc_id not in TC_REGISTRY: + print(f" WARNING: TC{tc_id} not found, skipping") + continue + name, func = TC_REGISTRY[tc_id] + try: + r = func(client, llm_client=llm_client, llm_model=llm_model) + results.append(r) + except Exception as e: + print(f"\n TC{tc_id} ({name}) CRASHED: {e}") + tc = TCResult(f"TC{tc_id}: {name}") + tc.check(False, f"crashed: {e}") + results.append(tc) + + print_section("SUMMARY") + passed = sum(1 for r in results if r.ok) + for r in results: + status = "PASS" if r.ok else "FAIL" + checks = f"{len(r.passed)}/{len(r.passed) + len(r.failed)}" + print(f" {status} {r.name} ({checks} checks)") + print(f"\n Total: {passed}/{len(results)} TCs passed") + sys.exit(0 if passed == len(results) else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_component/test_llm/test_client.py b/tests/unit/test_component/test_llm/test_client.py index dd9eff2..eb7427e 100644 --- a/tests/unit/test_component/test_llm/test_client.py +++ b/tests/unit/test_component/test_llm/test_client.py @@ -27,7 +27,7 @@ def _patch_settings( """Stub the ``load_settings`` reference bound inside the client module.""" cfg = Settings( llm=LLMSettings( - model="gpt-4o-mini", + model="gpt-4.1-mini", api_key=SecretStr(api_key) if api_key is not None else None, base_url=base_url, ) diff --git a/tests/unit/test_component/test_rerank/test_deepinfra_provider.py b/tests/unit/test_component/test_rerank/test_deepinfra_provider.py index 6ef016b..dacfe23 100644 --- a/tests/unit/test_component/test_rerank/test_deepinfra_provider.py +++ b/tests/unit/test_component/test_rerank/test_deepinfra_provider.py @@ -13,7 +13,10 @@ from collections.abc import Callable import httpx import pytest -from everos.component.rerank import DeepInfraRerankProvider, RerankError +from everos.component.rerank import ( + DeepInfraRerankProvider, + RerankServiceError, +) def _patch_httpx( @@ -115,7 +118,7 @@ async def test_4xx_raises_immediately(monkeypatch: pytest.MonkeyPatch) -> None: p = DeepInfraRerankProvider( model="m", api_key="k", base_url="https://api/v1", max_retries=3 ) - with pytest.raises(RerankError, match="HTTP 400"): + with pytest.raises(RerankServiceError, match="HTTP 400"): await p.rerank("q", ["a"]) assert calls == 1 # no retry on 4xx @@ -146,7 +149,7 @@ async def test_5xx_exhausts_retries(monkeypatch: pytest.MonkeyPatch) -> None: p = DeepInfraRerankProvider( model="m", api_key="k", base_url="https://api/v1", max_retries=1 ) - with pytest.raises(RerankError, match="HTTP 500"): + with pytest.raises(RerankServiceError, match="HTTP 500"): await p.rerank("q", ["a"]) @@ -178,7 +181,7 @@ async def test_transport_error_retries_then_fails( p = DeepInfraRerankProvider( model="m", api_key="k", base_url="https://api/v1", max_retries=1 ) - with pytest.raises(RerankError, match="transport failure"): + with pytest.raises(RerankServiceError, match="transport failure"): await p.rerank("q", ["a"]) @@ -188,7 +191,7 @@ async def test_malformed_scores_raises(monkeypatch: pytest.MonkeyPatch) -> None: _patch_httpx(monkeypatch, handler) p = DeepInfraRerankProvider(model="m", api_key="k", base_url="https://api/v1") - with pytest.raises(RerankError, match="missing scores"): + with pytest.raises(RerankServiceError, match="missing scores"): await p.rerank("q", ["a"]) @@ -200,7 +203,7 @@ async def test_score_length_mismatch_raises(monkeypatch: pytest.MonkeyPatch) -> p = DeepInfraRerankProvider( model="m", api_key="k", base_url="https://api/v1", batch_size=10 ) - with pytest.raises(RerankError, match="returned 2 scores, expected 3"): + with pytest.raises(RerankServiceError, match="returned 2 scores, expected 3"): await p.rerank("q", ["a", "b", "c"]) diff --git a/tests/unit/test_component/test_rerank/test_vllm_provider.py b/tests/unit/test_component/test_rerank/test_vllm_provider.py index 91534c6..d9adbf2 100644 --- a/tests/unit/test_component/test_rerank/test_vllm_provider.py +++ b/tests/unit/test_component/test_rerank/test_vllm_provider.py @@ -7,7 +7,7 @@ from collections.abc import Callable import httpx import pytest -from everos.component.rerank import RerankError, VllmRerankProvider +from everos.component.rerank import RerankServiceError, VllmRerankProvider def _patch_httpx( @@ -126,7 +126,7 @@ async def test_4xx_raises_immediately(monkeypatch: pytest.MonkeyPatch) -> None: p = VllmRerankProvider( model="m", api_key="bad", base_url="http://x/v1", max_retries=3 ) - with pytest.raises(RerankError, match="HTTP 401"): + with pytest.raises(RerankServiceError, match="HTTP 401"): await p.rerank("q", ["a"]) assert state["calls"] == 1 @@ -153,7 +153,7 @@ async def test_5xx_exhausts_retries(monkeypatch: pytest.MonkeyPatch) -> None: _patch_httpx(monkeypatch, handler) p = VllmRerankProvider(model="m", api_key="", base_url="http://x/v1", max_retries=1) - with pytest.raises(RerankError, match="HTTP 500"): + with pytest.raises(RerankServiceError, match="HTTP 500"): await p.rerank("q", ["a"]) @@ -163,7 +163,7 @@ async def test_transport_error_exhausts(monkeypatch: pytest.MonkeyPatch) -> None _patch_httpx(monkeypatch, handler) p = VllmRerankProvider(model="m", api_key="", base_url="http://x/v1", max_retries=1) - with pytest.raises(RerankError, match="transport failure"): + with pytest.raises(RerankServiceError, match="transport failure"): await p.rerank("q", ["a"]) @@ -173,7 +173,7 @@ async def test_malformed_results_missing_key(monkeypatch: pytest.MonkeyPatch) -> _patch_httpx(monkeypatch, handler) p = VllmRerankProvider(model="m", api_key="", base_url="http://x/v1") - with pytest.raises(RerankError, match="missing results"): + with pytest.raises(RerankServiceError, match="missing results"): await p.rerank("q", ["a"]) @@ -183,5 +183,5 @@ async def test_malformed_result_entry(monkeypatch: pytest.MonkeyPatch) -> None: _patch_httpx(monkeypatch, handler) p = VllmRerankProvider(model="m", api_key="", base_url="http://x/v1") - with pytest.raises(RerankError, match="malformed rerank result"): + with pytest.raises(RerankServiceError, match="malformed rerank result"): await p.rerank("q", ["a"]) diff --git a/tests/unit/test_component/test_tokenizer/test_jieba.py b/tests/unit/test_component/test_tokenizer/test_jieba.py index d153412..40f91ab 100644 --- a/tests/unit/test_component/test_tokenizer/test_jieba.py +++ b/tests/unit/test_component/test_tokenizer/test_jieba.py @@ -10,31 +10,59 @@ Verify the contract that callers downstream depend on: The tokenizer is symmetric — cascade write side and search query side both go through this code path, so changes here change BM25 recall on both ends. + +``JiebaTokenizer`` is imported inside each test (not at module level) +because ``jieba==0.42.1`` contains invalid escape sequences that +Python 3.12 treats as DeprecationWarning; our strict +``filterwarnings=["error"]`` converts those to errors during pytest +collection. Deferring the import to test-run time lets the per-module +``ignore`` filter take effect. """ from __future__ import annotations -from everos.component.tokenizer import JiebaTokenizer, build_tokenizer +import warnings + +import pytest + +from everos.component.tokenizer import build_tokenizer + + +def _make_tokenizer(**kwargs): + """Import JiebaTokenizer at call time, suppressing jieba's warnings.""" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "invalid escape sequence", DeprecationWarning) + from everos.component.tokenizer.jieba_provider import JiebaTokenizer + + return JiebaTokenizer(**kwargs) + + +@pytest.fixture(autouse=True) +def _suppress_jieba_warnings(): + """Suppress jieba's invalid-escape DeprecationWarnings for all tests.""" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "invalid escape sequence", DeprecationWarning) + yield def test_tokenize_returns_list_for_english() -> None: - tokens = JiebaTokenizer().tokenize("hello world") + tokens = _make_tokenizer().tokenize("hello world") assert tokens == ["hello", "world"] def test_tokenize_drops_pure_whitespace() -> None: """Whitespace-only tokens never reach the BM25 column.""" - tokens = JiebaTokenizer().tokenize("foo bar") + tokens = _make_tokenizer().tokenize("foo bar") assert all(t.strip() for t in tokens) def test_tokenize_empty_input() -> None: - assert JiebaTokenizer().tokenize("") == [] + assert _make_tokenizer().tokenize("") == [] def test_tokenize_cjk_keeps_multichar_words() -> None: """``cut_for_search`` keeps multi-character compounds usable by BM25.""" - tokens = JiebaTokenizer().tokenize("我爱北京天安门") + tokens = _make_tokenizer().tokenize("我爱北京天安门") # Single-char tokens (我 / 爱) are filtered by min_length=2 (and 我 # is also in the default stopword set). Multi-char compounds survive. assert "我" not in tokens @@ -44,7 +72,7 @@ def test_tokenize_cjk_keeps_multichar_words() -> None: def test_tokenize_drops_default_english_stopwords() -> None: - tokens = JiebaTokenizer().tokenize("the quick brown fox") + tokens = _make_tokenizer().tokenize("the quick brown fox") assert "the" not in tokens assert "quick" in tokens assert "brown" in tokens @@ -53,7 +81,7 @@ def test_tokenize_drops_default_english_stopwords() -> None: def test_tokenize_drops_short_tokens_below_min_length() -> None: """Single-char ASCII tokens are dropped by the default ``min_length=2``.""" - tokens = JiebaTokenizer().tokenize("a quick b run") + tokens = _make_tokenizer().tokenize("a quick b run") assert "a" not in tokens assert "b" not in tokens assert "quick" in tokens @@ -62,12 +90,12 @@ def test_tokenize_drops_short_tokens_below_min_length() -> None: def test_tokenize_is_case_insensitive() -> None: """Lowercasing is part of the symmetric contract.""" - tokens = JiebaTokenizer().tokenize("HELLO World") + tokens = _make_tokenizer().tokenize("HELLO World") assert tokens == ["hello", "world"] def test_extra_stopwords_extend_defaults() -> None: - tk = JiebaTokenizer(extra_stopwords=frozenset({"hello"})) + tk = _make_tokenizer(extra_stopwords=frozenset({"hello"})) tokens = tk.tokenize("hello world") assert "hello" not in tokens assert "world" in tokens @@ -79,7 +107,7 @@ def test_custom_min_token_length_relaxes_filter() -> None: Stopword filter still applies — even at ``min_length=1`` the English article ``"a"`` stays filtered because it's in the default stopwords. """ - tokens = JiebaTokenizer(min_token_length=1).tokenize("a quick b") + tokens = _make_tokenizer(min_token_length=1).tokenize("a quick b") # 'a' is in the default English stopword set even at min_length=1. assert "a" not in tokens assert "b" in tokens @@ -87,7 +115,7 @@ def test_custom_min_token_length_relaxes_filter() -> None: def test_tokenize_batch_preserves_order() -> None: - tk = JiebaTokenizer() + tk = _make_tokenizer() out = tk.tokenize_batch(["foo bar", "baz", ""]) assert len(out) == 3 assert out[2] == [] @@ -95,4 +123,8 @@ def test_tokenize_batch_preserves_order() -> None: def test_build_tokenizer_returns_jieba_default() -> None: """Factory exposes the same JiebaTokenizer the cascade handler uses.""" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "invalid escape sequence", DeprecationWarning) + from everos.component.tokenizer.jieba_provider import JiebaTokenizer + assert isinstance(build_tokenizer(), JiebaTokenizer) diff --git a/tests/unit/test_component/test_utils/test_datetime.py b/tests/unit/test_component/test_utils/test_datetime.py index 7123a3e..edb707c 100644 --- a/tests/unit/test_component/test_utils/test_datetime.py +++ b/tests/unit/test_component/test_utils/test_datetime.py @@ -424,7 +424,7 @@ def test_sqlite_round_trip_under_shanghai_display_tz( import asyncio import json as _json - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_MEMORY__TIMEZONE", "Asia/Shanghai") load_settings.cache_clear() dt_module._display_tz.cache_clear() @@ -591,7 +591,7 @@ def test_reverse_tz_switch_utc_to_shanghai_no_drift( import asyncio import json as _json - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_MEMORY__TIMEZONE", "UTC") load_settings.cache_clear() dt_module._display_tz.cache_clear() @@ -681,7 +681,7 @@ def test_sqlite_before_insert_event_normalises_aware_non_utc_to_utc( import asyncio import json as _json - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_MEMORY__TIMEZONE", "Asia/Shanghai") load_settings.cache_clear() dt_module._display_tz.cache_clear() @@ -795,7 +795,7 @@ def test_sqlite_load_hook_attaches_utc_on_all_base_table_subclasses( import asyncio import json as _json - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_MEMORY__TIMEZONE", "Asia/Shanghai") load_settings.cache_clear() dt_module._display_tz.cache_clear() @@ -920,7 +920,7 @@ def test_sqlite_load_hook_attaches_utc_on_all_base_table_subclasses( def _build_engine_for_test(monkeypatch, tmp_path, tz: str = "Asia/Shanghai"): """Common setup: tmp memory root + tz + fresh engine.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_MEMORY__TIMEZONE", tz) load_settings.cache_clear() dt_module._display_tz.cache_clear() diff --git a/tests/unit/test_config/test_knowledge_settings.py b/tests/unit/test_config/test_knowledge_settings.py new file mode 100644 index 0000000..11d636e --- /dev/null +++ b/tests/unit/test_config/test_knowledge_settings.py @@ -0,0 +1,17 @@ +"""Knowledge settings load from TOML.""" + +from __future__ import annotations + +from everos.config import load_settings + + +class TestKnowledgeSettings: + def test_defaults(self) -> None: + load_settings.cache_clear() + s = load_settings() + ks = s.knowledge + assert ks.search.recall_n == 200 + assert ks.search.rerank_n == 50 + assert ks.search.mass_top_m == 50 + assert ks.search.lam == 0.1 + assert ks.search.top_k_cap == 100 diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index de6d849..1f71289 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -1,36 +1,68 @@ -"""Unit tests for Settings loading.""" +"""Unit tests for Settings loading (everos.toml-based).""" from __future__ import annotations +import os from pathlib import Path import pytest from everos.config import Settings, load_settings +from everos.config.settings import resolve_root @pytest.fixture(autouse=True) -def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None: - """Strip any EVEROS_* env vars from the host so tests are deterministic.""" - for key in list(__import__("os").environ): +def _isolate_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Strip EVEROS_* env vars and move CWD away from any config file.""" + for key in list(os.environ): if key.startswith("EVEROS_"): monkeypatch.delenv(key, raising=False) + monkeypatch.chdir(tmp_path) load_settings.cache_clear() -def test_load_settings_defaults_from_toml() -> None: +def test_load_settings_defaults_from_shipped_toml() -> None: s = load_settings() - # Values straight out of config/default.toml - assert s.memory.root == Path("~/.everos") assert s.memory.timezone == "UTC" assert s.sqlite.journal_mode == "WAL" assert s.sqlite.synchronous == "NORMAL" - assert s.sqlite.foreign_keys is True - assert s.sqlite.temp_store == "MEMORY" assert s.sqlite.busy_timeout_ms == 5000 - assert s.sqlite.journal_size_limit_bytes == 64 * 1024 * 1024 - assert s.sqlite.cache_size_kb == 2048 - assert s.lancedb.read_consistency_seconds is None + assert s.api.host == "127.0.0.1" + assert s.api.port == 8000 + + +def test_everos_toml_overrides_defaults(tmp_path: Path) -> None: + """/everos.toml overrides shipped default.toml values.""" + root = tmp_path / "myroot" + root.mkdir() + (root / "everos.toml").write_text( + '[sqlite]\nbusy_timeout_ms = 7777\n[memory]\ntimezone = "Asia/Tokyo"\n', + encoding="utf-8", + ) + s = Settings(_everos_root=root) + assert s.sqlite.busy_timeout_ms == 7777 + assert s.memory.timezone == "Asia/Tokyo" + assert s.sqlite.journal_mode == "WAL" # untouched → default + + +def test_env_var_overrides_everos_toml( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """EVEROS_* env vars beat everos.toml.""" + root = tmp_path / "myroot" + root.mkdir() + (root / "everos.toml").write_text( + "[sqlite]\nbusy_timeout_ms = 7777\n", encoding="utf-8" + ) + monkeypatch.setenv("EVEROS_SQLITE__BUSY_TIMEOUT_MS", "9999") + s = Settings(_everos_root=root) + assert s.sqlite.busy_timeout_ms == 9999 + + +def test_no_everos_toml_uses_defaults_only(tmp_path: Path) -> None: + """Missing everos.toml is not an error — falls back to defaults.""" + s = Settings(_everos_root=tmp_path) + assert s.sqlite.busy_timeout_ms == 5000 def test_env_overrides_toml(monkeypatch: pytest.MonkeyPatch) -> None: @@ -39,7 +71,6 @@ def test_env_overrides_toml(monkeypatch: pytest.MonkeyPatch) -> None: s = Settings() assert s.sqlite.busy_timeout_ms == 10000 assert s.sqlite.journal_mode == "DELETE" - # Untouched values stay at TOML defaults. assert s.sqlite.synchronous == "NORMAL" @@ -48,7 +79,7 @@ def test_init_args_override_env(monkeypatch: pytest.MonkeyPatch) -> None: from everos.config.settings import SqliteSettings s = Settings(sqlite=SqliteSettings(busy_timeout_ms=99999)) - assert s.sqlite.busy_timeout_ms == 99999 # init beats env + assert s.sqlite.busy_timeout_ms == 99999 def test_invalid_journal_mode_rejected() -> None: @@ -65,28 +96,7 @@ def test_negative_busy_timeout_rejected() -> None: Settings.model_validate({"sqlite": {"busy_timeout_ms": -1}}) -def test_lancedb_read_consistency_optional_float() -> None: - s = Settings.model_validate({"lancedb": {"read_consistency_seconds": 5.0}}) - assert s.lancedb.read_consistency_seconds == 5.0 - s2 = Settings.model_validate({"lancedb": {"read_consistency_seconds": None}}) - assert s2.lancedb.read_consistency_seconds is None - - -def test_memory_timezone_overridable_via_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("EVEROS_MEMORY__TIMEZONE", "Asia/Shanghai") - s = Settings() - assert s.memory.timezone == "Asia/Shanghai" - - -def test_memory_timezone_invalid_rejected() -> None: - from pydantic import ValidationError - - with pytest.raises(ValidationError, match="invalid timezone"): - Settings.model_validate({"memory": {"timezone": "Not/A/Real_Zone"}}) - - def test_load_settings_is_cached() -> None: - """Repeated calls return the same Settings object until cache_clear.""" a = load_settings() b = load_settings() assert a is b @@ -96,42 +106,13 @@ def test_load_settings_is_cached() -> None: def test_embedding_rerank_defaults() -> None: - """Embedding / rerank ship with runtime knobs but no model credentials.""" - # ``_isolate_env`` already strips shell env; ``_env_file=None`` further - # prevents a developer's ``.env`` (which typically sets MODEL / API_KEY / - # BASE_URL for live runs) from leaking into this default-state check. - s = Settings(_env_file=None) # type: ignore[call-arg] - # Credentials must be set explicitly (no default). + s = Settings() assert s.embedding.model is None assert s.embedding.api_key is None - assert s.embedding.base_url is None - # Runtime knobs come from default.toml. assert s.embedding.timeout_seconds == 30.0 - assert s.embedding.max_retries == 3 - assert s.embedding.batch_size == 10 - assert s.embedding.max_concurrent == 5 - # Rerank mirrors the shape. assert s.rerank.model is None + assert s.rerank.provider is None assert s.rerank.timeout_seconds == 30.0 - assert s.rerank.batch_size == 10 - - -def test_embedding_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "intfloat/e5-large-v2") - monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://localhost:8000/v1") - monkeypatch.setenv("EVEROS_EMBEDDING__BATCH_SIZE", "32") - s = Settings() - assert s.embedding.model == "intfloat/e5-large-v2" - assert s.embedding.base_url == "http://localhost:8000/v1" - assert s.embedding.batch_size == 32 - - -def test_rerank_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("EVEROS_RERANK__MODEL", "BAAI/bge-reranker-v2-m3") - monkeypatch.setenv("EVEROS_RERANK__MAX_CONCURRENT", "8") - s = Settings() - assert s.rerank.model == "BAAI/bge-reranker-v2-m3" - assert s.rerank.max_concurrent == 8 def test_dashscope_one_key_can_configure_llm_embedding_and_rerank( @@ -147,7 +128,6 @@ def test_dashscope_one_key_can_configure_llm_embedding_and_rerank( monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "text-embedding-v4") monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", key) monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", compatible_base_url) - monkeypatch.setenv("EVEROS_RERANK__PROVIDER", "dashscope") monkeypatch.setenv("EVEROS_RERANK__MODEL", "gte-rerank-v2") monkeypatch.setenv("EVEROS_RERANK__API_KEY", key) monkeypatch.setenv("EVEROS_RERANK__BASE_URL", "https://dashscope.aliyuncs.com") @@ -162,58 +142,19 @@ def test_dashscope_one_key_can_configure_llm_embedding_and_rerank( assert s.rerank.api_key.get_secret_value() == key assert s.llm.base_url == compatible_base_url assert s.embedding.base_url == compatible_base_url - assert s.rerank.provider == "dashscope" - - from everos.component.embedding import ( - OpenAIEmbeddingProvider, - build_embedding_provider, - ) - from everos.component.llm import build_llm_provider - from everos.component.llm.openai_provider import OpenAIProvider - from everos.component.rerank import ( - DashScopeRerankProvider, - build_rerank_provider, - ) - - assert isinstance(build_llm_provider(s.llm), OpenAIProvider) - assert isinstance(build_embedding_provider(s.embedding), OpenAIEmbeddingProvider) - assert isinstance(build_rerank_provider(s.rerank), DashScopeRerankProvider) + assert s.rerank.provider is None + assert s.rerank.base_url == "https://dashscope.aliyuncs.com" -def test_user_toml_override_via_env_path( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """``EVEROS_CONFIG_FILE`` points pydantic-settings at a user toml.""" - user_toml = tmp_path / "config.toml" - user_toml.write_text( - '[sqlite]\nbusy_timeout_ms = 7777\n[memory]\ntimezone = "Asia/Tokyo"\n', - encoding="utf-8", - ) - monkeypatch.setenv("EVEROS_CONFIG_FILE", str(user_toml)) - s = Settings() - assert s.sqlite.busy_timeout_ms == 7777 - assert s.memory.timezone == "Asia/Tokyo" - # Values not touched by the user toml still come from the shipped default. - assert s.sqlite.journal_mode == "WAL" +def test_resolve_root_default() -> None: + """No --root, no EVEROS_ROOT → ~/.everos.""" + assert resolve_root() == Path("~/.everos").expanduser().resolve() -def test_user_toml_loses_to_env( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """env vars beat the user-level toml.""" - user_toml = tmp_path / "config.toml" - user_toml.write_text("[sqlite]\nbusy_timeout_ms = 7777\n", encoding="utf-8") - monkeypatch.setenv("EVEROS_CONFIG_FILE", str(user_toml)) - monkeypatch.setenv("EVEROS_SQLITE__BUSY_TIMEOUT_MS", "9999") - s = Settings() - assert s.sqlite.busy_timeout_ms == 9999 +def test_resolve_root_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EVEROS_ROOT", "/data/everos") + assert resolve_root() == Path("/data/everos").resolve() -def test_user_toml_missing_file_is_skipped( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """A non-existent user toml path is silently skipped, not an error.""" - monkeypatch.setenv("EVEROS_CONFIG_FILE", str(tmp_path / "nope.toml")) - s = Settings() - # Falls back to shipped defaults. - assert s.sqlite.busy_timeout_ms == 5000 +def test_resolve_root_explicit() -> None: + assert resolve_root("/custom/root") == Path("/custom/root").resolve() diff --git a/tests/unit/test_core/test_errors.py b/tests/unit/test_core/test_errors.py new file mode 100644 index 0000000..dbae082 --- /dev/null +++ b/tests/unit/test_core/test_errors.py @@ -0,0 +1,110 @@ +"""Tests for the DDD-aligned exception hierarchy.""" + +from __future__ import annotations + +from everos.core.errors import ( + AppError, + CapabilityError, + ConfigurationError, + ConflictError, + DocumentNotFoundError, + DomainError, + DuplicateDocumentError, + EmbeddingServiceError, + ExternalServiceError, + ExtractionEmptyError, + FilterError, + InfrastructureError, + InvalidInputError, + LLMServiceError, + MultimodalNotEnabledError, + NotFoundError, + PathTraversalError, + RerankServiceError, + StorageError, + TopicNotFoundError, + UnsupportedModalityError, + VectorStoreError, +) + + +class TestDomainBranch: + def test_domain_errors_are_app_errors(self) -> None: + assert issubclass(DomainError, AppError) + assert issubclass(NotFoundError, DomainError) + assert issubclass(ConflictError, DomainError) + assert issubclass(InvalidInputError, DomainError) + + def test_not_found_subtypes(self) -> None: + assert issubclass(DocumentNotFoundError, NotFoundError) + assert issubclass(TopicNotFoundError, NotFoundError) + + def test_conflict_subtypes(self) -> None: + assert issubclass(DuplicateDocumentError, ConflictError) + + def test_invalid_input_subtypes(self) -> None: + assert issubclass(ExtractionEmptyError, InvalidInputError) + assert issubclass(FilterError, InvalidInputError) + + def test_path_traversal_is_domain_not_invalid_input(self) -> None: + assert issubclass(PathTraversalError, DomainError) + assert not issubclass(PathTraversalError, InvalidInputError) + + def test_unsupported_modality_is_domain(self) -> None: + assert issubclass(UnsupportedModalityError, DomainError) + assert not issubclass(UnsupportedModalityError, InfrastructureError) + + +class TestInfrastructureBranch: + def test_infrastructure_errors_are_app_errors(self) -> None: + assert issubclass(InfrastructureError, AppError) + assert issubclass(StorageError, InfrastructureError) + assert issubclass(VectorStoreError, InfrastructureError) + assert issubclass(ExternalServiceError, InfrastructureError) + + def test_external_service_subtypes(self) -> None: + assert issubclass(LLMServiceError, ExternalServiceError) + assert issubclass(EmbeddingServiceError, ExternalServiceError) + assert issubclass(RerankServiceError, ExternalServiceError) + + +class TestCapabilityBranch: + def test_capability_is_app_error(self) -> None: + assert issubclass(CapabilityError, AppError) + + def test_multimodal_not_enabled(self) -> None: + assert issubclass(MultimodalNotEnabledError, CapabilityError) + assert not issubclass(MultimodalNotEnabledError, InfrastructureError) + assert not issubclass(MultimodalNotEnabledError, DomainError) + + +class TestConfigurationBranch: + def test_configuration_is_app_error(self) -> None: + assert issubclass(ConfigurationError, AppError) + assert not issubclass(ConfigurationError, DomainError) + assert not issubclass(ConfigurationError, InfrastructureError) + + +class TestBackwardCompat: + def test_old_document_already_exists_alias(self) -> None: + from everos.core.errors import DocumentAlreadyExistsError + + assert DocumentAlreadyExistsError is DuplicateDocumentError + + def test_old_validation_error_alias(self) -> None: + from everos.core.errors import ValidationError + + assert ValidationError is InvalidInputError + + +class TestMRODispatch: + def test_infrastructure_catches_embedding(self) -> None: + exc = EmbeddingServiceError("provider down") + assert isinstance(exc, InfrastructureError) + assert isinstance(exc, AppError) + + def test_instantiation_with_message(self) -> None: + exc = DocumentNotFoundError("d_abc123") + assert str(exc) == "d_abc123" + exc2 = LLMServiceError("timeout after 30s") + assert str(exc2) == "timeout after 30s" diff --git a/tests/unit/test_core/test_middleware/test_global_exception.py b/tests/unit/test_core/test_middleware/test_global_exception.py deleted file mode 100644 index 1c48f7d..0000000 --- a/tests/unit/test_core/test_middleware/test_global_exception.py +++ /dev/null @@ -1,106 +0,0 @@ -"""``global_exception_handler`` — uniform error envelope per v1 API §1. - -We mount the handler on a minimal FastAPI app with three error-emitting -routes (HTTPException 4xx / 5xx, RequestValidationError, raw exception) -and assert the envelope shape + status code each route produces. -""" - -from __future__ import annotations - -from collections.abc import AsyncIterator - -import pytest -from fastapi import FastAPI, HTTPException -from fastapi.exceptions import RequestValidationError -from httpx import ASGITransport, AsyncClient -from pydantic import BaseModel - -from everos.core.middleware.global_exception import global_exception_handler - - -class _Body(BaseModel): - name: str - - -def _build_app() -> FastAPI: - app = FastAPI() - app.add_exception_handler(HTTPException, global_exception_handler) - app.add_exception_handler(RequestValidationError, global_exception_handler) - app.add_exception_handler(Exception, global_exception_handler) - - @app.get("/raise-400") - async def raise_400() -> None: - raise HTTPException(status_code=400, detail="bad input") - - @app.get("/raise-500-http") - async def raise_500_http() -> None: - raise HTTPException(status_code=503, detail="upstream dead") - - @app.get("/boom") - async def boom() -> None: - raise RuntimeError("hidden internals") - - @app.post("/validate") - async def validate(_body: _Body) -> dict[str, str]: - return {"ok": "yes"} - - return app - - -@pytest.fixture -async def client() -> AsyncIterator[AsyncClient]: - app = _build_app() - # raise_app_exceptions=False — let the registered handler convert the - # RuntimeError into a 500 response instead of re-raising into the test. - transport = ASGITransport(app=app, raise_app_exceptions=False) - async with AsyncClient(transport=transport, base_url="http://test") as c: - yield c - - -def _assert_envelope(body: dict[str, object], *, code: str, path: str) -> None: - """Wiki §1 envelope: ``{request_id, error: {code, message, timestamp, path}}``.""" - assert isinstance(body["request_id"], str) and body["request_id"] - error = body["error"] - assert isinstance(error, dict) - assert error["code"] == code - assert isinstance(error["message"], str) and error["message"] - assert isinstance(error["timestamp"], str) and "T" in error["timestamp"] - assert error["path"] == path - - -async def test_http_exception_4xx(client: AsyncClient) -> None: - resp = await client.get("/raise-400") - assert resp.status_code == 400 - body = resp.json() - _assert_envelope(body, code="HTTP_ERROR", path="/raise-400") - assert body["error"]["message"] == "bad input" - - -async def test_http_exception_5xx_uses_system_error(client: AsyncClient) -> None: - """5xx routed through HTTPException still produces SYSTEM_ERROR + generic msg.""" - resp = await client.get("/raise-500-http") - assert resp.status_code == 503 - body = resp.json() - _assert_envelope(body, code="SYSTEM_ERROR", path="/raise-500-http") - # Internal detail "upstream dead" is suppressed in 5xx envelopes. - assert body["error"]["message"] == "Internal server error" - - -async def test_unhandled_exception_5xx(client: AsyncClient) -> None: - """RuntimeError → 500 with generic ``SYSTEM_ERROR`` envelope; details hidden.""" - resp = await client.get("/boom") - assert resp.status_code == 500 - body = resp.json() - _assert_envelope(body, code="SYSTEM_ERROR", path="/boom") - assert body["error"]["message"] == "Internal server error" - # Must not leak the internal exception message. - assert "hidden internals" not in resp.text - - -async def test_validation_error_returns_422(client: AsyncClient) -> None: - resp = await client.post("/validate", json={}) # missing ``name`` - assert resp.status_code == 422 - body = resp.json() - _assert_envelope(body, code="HTTP_ERROR", path="/validate") - # First-error message includes the offending field somewhere. - assert "name" in body["error"]["message"].lower() diff --git a/tests/unit/test_core/test_persistence/test_frontmatter_knowledge.py b/tests/unit/test_core/test_persistence/test_frontmatter_knowledge.py new file mode 100644 index 0000000..650cb87 --- /dev/null +++ b/tests/unit/test_core/test_persistence/test_frontmatter_knowledge.py @@ -0,0 +1,56 @@ +"""Tests for knowledge-scoped frontmatter mixins.""" + +from __future__ import annotations + +from everos.core.persistence.markdown.frontmatter import ( + BaseFrontmatter, + KnowledgeDocumentPathMixin, + KnowledgeScopedMixin, + KnowledgeTopicPathMixin, +) + + +class _DocFm(KnowledgeDocumentPathMixin, KnowledgeScopedMixin, BaseFrontmatter): + type: str = "knowledge_document" + id: str = "d_abc" + schema_version: int = 1 + + +class _TopicFm(KnowledgeTopicPathMixin, KnowledgeScopedMixin, BaseFrontmatter): + type: str = "knowledge_topic" + id: str = "d_abc_1" + schema_version: int = 1 + + +class TestKnowledgeScopedMixin: + def test_scope_dir_is_knowledge(self) -> None: + assert _DocFm.SCOPE_DIR == "knowledge" + + def test_doc_path_glob_matches_index_md(self) -> None: + glob = _DocFm.path_glob() + assert glob == "*/*/knowledge/*/*/index.md" + + def test_topic_path_glob_matches_numbered_md(self) -> None: + glob = _TopicFm.path_glob() + assert glob == "*/*/knowledge/*/*/[0-9]*.md" + + def test_doc_glob_has_app_project_prefix(self) -> None: + from pathlib import PurePosixPath + + glob = _DocFm.path_glob() + path = "default_app/default_project/knowledge/Sports/Olympics/index.md" + assert PurePosixPath(path).match(glob) + + def test_topic_glob_matches_numbered_topic(self) -> None: + from pathlib import PurePosixPath + + glob = _TopicFm.path_glob() + path = "default_app/default_project/knowledge/Sports/Olympics/1_Budget.md" + assert PurePosixPath(path).match(glob) + + def test_topic_glob_does_not_match_index(self) -> None: + from pathlib import PurePosixPath + + glob = _TopicFm.path_glob() + path = "default_app/default_project/knowledge/Sports/Olympics/index.md" + assert not PurePosixPath(path).match(glob) diff --git a/tests/unit/test_core/test_persistence/test_locking.py b/tests/unit/test_core/test_persistence/test_locking.py index 05e0618..62a2ff0 100644 --- a/tests/unit/test_core/test_persistence/test_locking.py +++ b/tests/unit/test_core/test_persistence/test_locking.py @@ -2,8 +2,7 @@ from __future__ import annotations -import subprocess -import sys +import multiprocessing import time from pathlib import Path @@ -12,88 +11,6 @@ import pytest from everos.core.persistence import LockError, MemoryRoot, memory_root_lock -_LOCK_HOLDER_SCRIPT = """ -import fcntl -import os -import sys -import time - -from pathlib import Path - -lock_path, ready_path, release_path = sys.argv[1:] -Path(lock_path).parent.mkdir(parents=True, exist_ok=True) -fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o644) -try: - fcntl.flock(fd, fcntl.LOCK_EX) - Path(ready_path).write_text("ready") - while not Path(release_path).exists(): - time.sleep(0.05) -finally: - try: - fcntl.flock(fd, fcntl.LOCK_UN) - finally: - os.close(fd) -""" -LOCK_HOLDER_READY_TIMEOUT = 5.0 - - -async def _assert_subprocess_ready( - ready_path: Path, - proc: subprocess.Popen[str], -) -> None: - deadline = time.monotonic() + LOCK_HOLDER_READY_TIMEOUT - while time.monotonic() < deadline: - if await anyio.to_thread.run_sync(ready_path.exists): - return - if proc.poll() is not None: - stdout, stderr = proc.communicate() - raise AssertionError( - "subprocess exited before acquiring lock " - f"(exitcode={proc.returncode}, stdout={stdout!r}, stderr={stderr!r})" - ) - await anyio.sleep(0.05) - - proc.terminate() - stdout, stderr = proc.communicate(timeout=1) - raise AssertionError( - "subprocess failed to acquire lock " - f"(exitcode={proc.returncode}, stdout={stdout!r}, stderr={stderr!r})" - ) - - -def _spawn_lock_holder(mr: MemoryRoot) -> tuple[subprocess.Popen[str], Path, Path]: - ready_path = mr.root / ".test-lock-ready" - release_path = mr.root / ".test-lock-release" - proc = subprocess.Popen( - [ - sys.executable, - "-c", - _LOCK_HOLDER_SCRIPT, - str(mr.lock_file), - str(ready_path), - str(release_path), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - return proc, ready_path, release_path - - -async def _start_lock_holder(mr: MemoryRoot) -> tuple[subprocess.Popen[str], Path]: - proc, ready_path, release_path = _spawn_lock_holder(mr) - await _assert_subprocess_ready(ready_path, proc) - return proc, release_path - - -def _stop_lock_holder(proc: subprocess.Popen[str], release_path: Path) -> None: - release_path.write_text("release") - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.terminate() - proc.wait(timeout=5) - async def test_lock_creates_anchor_file(tmp_path: Path) -> None: mr = MemoryRoot(tmp_path) @@ -110,30 +27,60 @@ async def test_lock_acquire_release_acquire(tmp_path: Path) -> None: pass +def _hold_lock(memory_root_path: str, ready: object, release: object) -> None: + """Subprocess helper: acquire blocking lock, signal, wait, release. + + The subprocess runs its own event loop via :func:`anyio.run` since + :func:`memory_root_lock` is now async. + """ + + async def _run() -> None: + mr = MemoryRoot(memory_root_path) + async with memory_root_lock(mr, blocking=True): + ready.set() + # Use a thread-offloaded wait so we don't block the event loop. + await anyio.to_thread.run_sync(release.wait, 5) + + anyio.run(_run) + + async def test_nonblocking_raises_when_held_by_other_process(tmp_path: Path) -> None: """Different process holding the lock → blocking=False raises LockError.""" mr = MemoryRoot(tmp_path) - proc, release_path = await _start_lock_holder(mr) + ctx = multiprocessing.get_context("spawn") + ready = ctx.Event() + release = ctx.Event() + proc = ctx.Process(target=_hold_lock, args=(str(mr.root), ready, release)) + proc.start() try: + assert ready.wait(timeout=5), "subprocess failed to acquire lock" with pytest.raises(LockError): async with memory_root_lock(mr, blocking=False): pass finally: - _stop_lock_holder(proc, release_path) + release.set() + proc.join(timeout=5) + if proc.is_alive(): + proc.terminate() async def test_blocking_waits_for_release(tmp_path: Path) -> None: """Different process holding lock + main process blocking=True waits.""" mr = MemoryRoot(tmp_path) - proc, release_path = await _start_lock_holder(mr) + ctx = multiprocessing.get_context("spawn") + ready = ctx.Event() + release = ctx.Event() + proc = ctx.Process(target=_hold_lock, args=(str(mr.root), ready, release)) + proc.start() try: + assert ready.wait(timeout=5) # Schedule the subprocess to release shortly; main process should # acquire the lock after that. release_started = time.monotonic() def release_after_short_delay() -> None: time.sleep(0.2) - release_path.write_text("release") + release.set() import threading @@ -143,4 +90,7 @@ async def test_blocking_waits_for_release(tmp_path: Path) -> None: # Should have waited at least roughly the delay. assert elapsed >= 0.1 finally: - _stop_lock_holder(proc, release_path) + release.set() + proc.join(timeout=5) + if proc.is_alive(): + proc.terminate() diff --git a/tests/unit/test_core/test_persistence/test_markdown/test_writer.py b/tests/unit/test_core/test_persistence/test_markdown/test_writer.py index cd2365b..59de3aa 100644 --- a/tests/unit/test_core/test_persistence/test_markdown/test_writer.py +++ b/tests/unit/test_core/test_persistence/test_markdown/test_writer.py @@ -234,6 +234,8 @@ async def test_append_entry_round_trip_with_reader(tmp_path: Path) -> None: async def test_write_rejects_target_escaping_root(tmp_path: Path) -> None: + # A path-segment id carrying ``..`` (e.g. an unsanitised owner_id) would + # otherwise walk the write out of the configured root. root = tmp_path / "memory_root" root.mkdir() writer = MarkdownWriter(MemoryRoot(root)) @@ -242,6 +244,7 @@ async def test_write_rejects_target_escaping_root(tmp_path: Path) -> None: with pytest.raises(PathTraversalError): await writer.write(escaping, "x") + # The escaping path must not even have its parent directories created. assert not (tmp_path / "ESCAPED").exists() @@ -272,6 +275,9 @@ async def test_append_entry_rejects_escaping_target(tmp_path: Path) -> None: async def test_append_entry_does_not_read_out_of_root_file(tmp_path: Path) -> None: + # The containment guard must fire BEFORE the read-modify-write read, so an + # escaping target can never open + parse an existing out-of-root file (an + # arbitrary-file-read precursor) even though its write would be rejected. root = tmp_path / "memory_root" root.mkdir() secret = tmp_path / "secret.md" @@ -289,10 +295,12 @@ async def test_append_entry_does_not_read_out_of_root_file(tmp_path: Path) -> No entry_id=EntryId(prefix="umc", date=dt.date(2026, 4, 22), seq=1), ) spy.assert_not_called() + # The out-of-root file is left untouched. assert secret.read_text(encoding="utf-8") == "---\ntop: secret\n---\nbody\n" async def test_write_allows_target_inside_root(tmp_path: Path) -> None: + # Containment guard must not reject legitimate in-root writes. root = tmp_path / "memory_root" root.mkdir() writer = MarkdownWriter(MemoryRoot(root)) diff --git a/tests/unit/test_core/test_persistence/test_markdown/test_writer_patch_frontmatter.py b/tests/unit/test_core/test_persistence/test_markdown/test_writer_patch_frontmatter.py new file mode 100644 index 0000000..de22fe0 --- /dev/null +++ b/tests/unit/test_core/test_persistence/test_markdown/test_writer_patch_frontmatter.py @@ -0,0 +1,102 @@ +"""Unit tests for MarkdownWriter.patch_frontmatter.""" + +from __future__ import annotations + +import datetime as dt +from pathlib import Path + +from everos.core.persistence import ( + EntryId, + MarkdownReader, + MarkdownWriter, + MemoryRoot, +) + + +def _make_writer(tmp_path: Path) -> MarkdownWriter: + return MarkdownWriter(MemoryRoot(tmp_path)) + + +async def test_patch_frontmatter_adds_new_field(tmp_path: Path) -> None: + """Patching a field that does not exist in frontmatter adds it.""" + writer = _make_writer(tmp_path) + target = tmp_path / "doc.md" + + await writer.write_markdown( + target, + frontmatter={"type": "episode_daily", "entry_count": 0}, + body="", + ) + await writer.patch_frontmatter(target, {"new_key": "new_val"}) + + parsed = await MarkdownReader.read(target) + assert parsed.frontmatter["new_key"] == "new_val" + assert parsed.frontmatter["type"] == "episode_daily" + assert parsed.frontmatter["entry_count"] == 0 + + +async def test_patch_frontmatter_updates_existing_field(tmp_path: Path) -> None: + """Patching an existing scalar field overwrites it.""" + writer = _make_writer(tmp_path) + target = tmp_path / "doc.md" + + await writer.write_markdown( + target, + frontmatter={"type": "episode_daily", "entry_count": 3}, + body="", + ) + await writer.patch_frontmatter(target, {"entry_count": 5}) + + parsed = await MarkdownReader.read(target) + assert parsed.frontmatter["entry_count"] == 5 + assert parsed.frontmatter["type"] == "episode_daily" + + +async def test_patch_frontmatter_merges_dict_field(tmp_path: Path) -> None: + """Dict fields are merged additively, not replaced wholesale.""" + writer = _make_writer(tmp_path) + target = tmp_path / "doc.md" + + await writer.write_markdown( + target, + frontmatter={ + "type": "episode_daily", + "deprecated_entries": {"ep_001": "ep_100"}, + }, + body="", + ) + # Merge a second entry — the first must survive. + await writer.patch_frontmatter(target, {"deprecated_entries": {"ep_002": "ep_101"}}) + + parsed = await MarkdownReader.read(target) + dep = parsed.frontmatter["deprecated_entries"] + assert dep == {"ep_001": "ep_100", "ep_002": "ep_101"} + + +async def test_patch_frontmatter_preserves_entries(tmp_path: Path) -> None: + """Entry blocks in the body must be byte-identical after a patch.""" + writer = _make_writer(tmp_path) + target = tmp_path / "doc.md" + + eid1 = EntryId(prefix="ep", date=dt.date(2026, 5, 1), seq=1) + eid2 = EntryId(prefix="ep", date=dt.date(2026, 5, 1), seq=2) + await writer.append_entries( + target, + [("first entry body", eid1), ("second entry body", eid2)], + frontmatter_updates={"type": "episode_daily", "entry_count": 2}, + ) + + # Snapshot entries before patch. + pre = await MarkdownReader.read(target) + pre_ids = [e.id for e in pre.entries] + pre_bodies = [e.body for e in pre.entries] + + # Patch frontmatter only. + await writer.patch_frontmatter(target, {"deprecated_entries": {"ep_001": "ep_999"}}) + + # Entries must survive unchanged. + post = await MarkdownReader.read(target) + assert [e.id for e in post.entries] == pre_ids + assert [e.body for e in post.entries] == pre_bodies + assert post.frontmatter["deprecated_entries"] == {"ep_001": "ep_999"} + assert post.frontmatter["entry_count"] == 2 diff --git a/tests/unit/test_core/test_persistence/test_memory_root.py b/tests/unit/test_core/test_persistence/test_memory_root.py index 584d09e..8dd6dad 100644 --- a/tests/unit/test_core/test_persistence/test_memory_root.py +++ b/tests/unit/test_core/test_persistence/test_memory_root.py @@ -9,16 +9,31 @@ import pytest from everos.core.persistence import MemoryRoot -def test_default_returns_home_everos(monkeypatch: pytest.MonkeyPatch) -> None: - # Isolate from any ambient EVEROS_MEMORY__ROOT (e.g. the session-scoped - # search-corpus fixture sets it for the whole run); the autouse - # _reset_settings_cache fixture clears the load_settings cache, so the - # delenv takes effect for this assertion of the hard-coded default. - monkeypatch.delenv("EVEROS_MEMORY__ROOT", raising=False) +def test_default_returns_home_everos( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.delenv("EVEROS_ROOT", raising=False) + monkeypatch.chdir(tmp_path) + from everos.config import load_settings + + load_settings.cache_clear() mr = MemoryRoot.default() assert mr.root == (Path.home() / ".everos").resolve() +def test_default_from_everos_root_env( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path / "custom")) + mr = MemoryRoot.default() + assert mr.root == (tmp_path / "custom").resolve() + + +def test_default_explicit_root(tmp_path: Path) -> None: + mr = MemoryRoot.default(explicit_root=str(tmp_path / "explicit")) + assert mr.root == (tmp_path / "explicit").resolve() + + def test_accepts_str_path(tmp_path: Path) -> None: mr = MemoryRoot(str(tmp_path)) assert mr.root == tmp_path.resolve() @@ -79,37 +94,11 @@ def test_ensure_is_idempotent(tmp_path: Path) -> None: assert mr.tmp_dir.is_dir() -def test_ensure_materializes_ome_config_template(tmp_path: Path) -> None: - """First ensure() drops a real ``ome.toml`` users can edit. - - Without this, ``pip install everos && everos server start`` produced - a warning (``config_reload_failed: No such file``) because the OME - config reloader had no file to point at. The template ships under - ``src/everos/config/default_ome.toml`` and is byte-copied on first run. - """ - mr = MemoryRoot(tmp_path) +def test_ensure_does_not_create_ome_toml(tmp_path: Path) -> None: + """ome.toml creation moved to ``everos init``; ensure() only makes dirs.""" + mr = MemoryRoot(tmp_path / "fresh") mr.ensure() - assert mr.ome_config.is_file() - # Content is the shipped template verbatim — protects against a future - # diff that silently changes what users see on first run. - template = Path(__file__).resolve().parents[4] / ( - "src/everos/config/default_ome.toml" - ) - assert mr.ome_config.read_bytes() == template.read_bytes() - - -def test_ensure_preserves_user_edited_ome_config(tmp_path: Path) -> None: - """Second ensure() must not overwrite user edits. - - The template materialisation is an existence check, not a content - sync — once the user has tweaked their overrides the file is theirs. - """ - mr = MemoryRoot(tmp_path) - mr.ensure() - custom = b"# user-edited\n[strategies.extract_foresight]\nenabled = false\n" - mr.ome_config.write_bytes(custom) - mr.ensure() - assert mr.ome_config.read_bytes() == custom + assert not mr.ome_config.exists() def test_frozen_dataclass_hashable(tmp_path: Path) -> None: diff --git a/tests/unit/test_entrypoints/test_api/test_exception_handlers.py b/tests/unit/test_entrypoints/test_api/test_exception_handlers.py new file mode 100644 index 0000000..e395167 --- /dev/null +++ b/tests/unit/test_entrypoints/test_api/test_exception_handlers.py @@ -0,0 +1,288 @@ +"""Unit tests for per-type exception handlers registered via register_handlers(). + +Tests use a minimal FastAPI app with synthetic routes that raise specific +exceptions. The full handler suite is wired in via ``register_handlers()``. + +White-box surfaces: none — assertions are purely on HTTP response shape. +""" + +from __future__ import annotations + +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from everos.core.errors import ( + DocumentNotFoundError, + DuplicateDocumentError, + EmbeddingServiceError, + ExtractionEmptyError, + FilterError, + LLMServiceError, + MultimodalNotEnabledError, + PathTraversalError, + StorageError, + UnsupportedModalityError, +) +from everos.entrypoints.api.exception_handlers import register_handlers + +# --------------------------------------------------------------------------- +# Fixture: minimal app with one route per exception type +# --------------------------------------------------------------------------- + + +def _make_app() -> FastAPI: + """Build a minimal FastAPI app with synthetic routes that raise each error.""" + app = FastAPI() + register_handlers(app) + + @app.get("/raise/not-found") + async def _not_found() -> None: + raise DocumentNotFoundError("doc_abc123") + + @app.get("/raise/conflict") + async def _conflict() -> None: + raise DuplicateDocumentError("doc_abc123 already exists") + + @app.get("/raise/extraction-empty") + async def _extraction_empty() -> None: + raise ExtractionEmptyError("extraction produced no output") + + @app.get("/raise/filter-error") + async def _filter_error() -> None: + raise FilterError("unknown field: foo") + + @app.get("/raise/path-traversal") + async def _path_traversal() -> None: + raise PathTraversalError("../etc/passwd") + + @app.get("/raise/unsupported-modality") + async def _unsupported_modality() -> None: + raise UnsupportedModalityError("video not supported") + + @app.get("/raise/multimodal-not-enabled") + async def _multimodal_not_enabled() -> None: + raise MultimodalNotEnabledError("multimodal extra not installed") + + @app.get("/raise/storage") + async def _storage() -> None: + raise StorageError("disk full") + + @app.get("/raise/embedding") + async def _embedding() -> None: + raise EmbeddingServiceError("embedding provider timeout") + + @app.get("/raise/llm") + async def _llm() -> None: + raise LLMServiceError("LLM rate limit exceeded") + + @app.get("/raise/runtime") + async def _runtime() -> None: + raise RuntimeError("oops") + + @app.post("/raise/request-validation") + async def _request_validation(body: dict) -> None: + # Starlette raises RequestValidationError automatically when + # a JSON body is expected but not provided. + pass # pragma: no cover + + return app + + +@pytest.fixture +async def client() -> AsyncClient: + """Async test client against the minimal exception-handler app. + + ``raise_app_exceptions=False`` lets ServerErrorMiddleware send the 500 + response before re-raising, so the test sees the JSON body rather than + the raw exception. In production an ASGI server (uvicorn) absorbs the + re-raise for its own logging; the client already received the response. + """ + app = _make_app() + transport = ASGITransport(app=app, raise_app_exceptions=False) + async with AsyncClient(transport=transport, base_url="http://test") as c: + yield c + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _assert_envelope(data: dict, *, path: str) -> None: + """Assert the canonical envelope shape is present.""" + assert "request_id" in data, "envelope must have request_id" + assert len(data["request_id"]) == 32, "request_id must be 32 hex chars" + error = data.get("error", {}) + assert "code" in error + assert "message" in error + assert "timestamp" in error + assert "path" in error + assert error["path"] == path + + +# --------------------------------------------------------------------------- +# Status code + error.code tests +# --------------------------------------------------------------------------- + + +async def test_not_found_returns_404(client: AsyncClient) -> None: + """DocumentNotFoundError → HTTP 404, code=NOT_FOUND.""" + resp = await client.get("/raise/not-found") + assert resp.status_code == 404 + data = resp.json() + assert data["error"]["code"] == "NOT_FOUND" + _assert_envelope(data, path="/raise/not-found") + + +async def test_conflict_returns_409(client: AsyncClient) -> None: + """DuplicateDocumentError → HTTP 409, code=CONFLICT.""" + resp = await client.get("/raise/conflict") + assert resp.status_code == 409 + data = resp.json() + assert data["error"]["code"] == "CONFLICT" + _assert_envelope(data, path="/raise/conflict") + + +async def test_extraction_empty_returns_422(client: AsyncClient) -> None: + """ExtractionEmptyError → HTTP 422, code=EXTRACTION_EMPTY.""" + resp = await client.get("/raise/extraction-empty") + assert resp.status_code == 422 + data = resp.json() + assert data["error"]["code"] == "EXTRACTION_EMPTY" + _assert_envelope(data, path="/raise/extraction-empty") + + +async def test_filter_error_returns_422(client: AsyncClient) -> None: + """FilterError (subclass of InvalidInputError) → HTTP 422.""" + resp = await client.get("/raise/filter-error") + assert resp.status_code == 422 + data = resp.json() + assert data["error"]["code"] == "INVALID_INPUT" + _assert_envelope(data, path="/raise/filter-error") + + +async def test_path_traversal_returns_400(client: AsyncClient) -> None: + """PathTraversalError → HTTP 400, code=BAD_REQUEST.""" + resp = await client.get("/raise/path-traversal") + assert resp.status_code == 400 + data = resp.json() + assert data["error"]["code"] == "BAD_REQUEST" + _assert_envelope(data, path="/raise/path-traversal") + + +async def test_unsupported_modality_returns_415(client: AsyncClient) -> None: + """UnsupportedModalityError → HTTP 415, code=UNSUPPORTED_MEDIA_TYPE.""" + resp = await client.get("/raise/unsupported-modality") + assert resp.status_code == 415 + data = resp.json() + assert data["error"]["code"] == "UNSUPPORTED_FORMAT" + _assert_envelope(data, path="/raise/unsupported-modality") + + +async def test_multimodal_not_enabled_returns_503(client: AsyncClient) -> None: + """MultimodalNotEnabledError (CapabilityError) → HTTP 503.""" + resp = await client.get("/raise/multimodal-not-enabled") + assert resp.status_code == 503 + data = resp.json() + assert data["error"]["code"] == "CAPABILITY_UNAVAILABLE" + _assert_envelope(data, path="/raise/multimodal-not-enabled") + + +async def test_storage_error_returns_503(client: AsyncClient) -> None: + """StorageError → HTTP 503, code=SERVICE_UNAVAILABLE.""" + resp = await client.get("/raise/storage") + assert resp.status_code == 503 + data = resp.json() + assert data["error"]["code"] == "EXTERNAL_SERVICE_UNAVAILABLE" + _assert_envelope(data, path="/raise/storage") + + +# --------------------------------------------------------------------------- +# MRO dispatch tests +# --------------------------------------------------------------------------- + + +async def test_embedding_service_error_routes_to_503(client: AsyncClient) -> None: + """EmbeddingServiceError (InfrastructureError subclass) → 503 via MRO.""" + resp = await client.get("/raise/embedding") + assert resp.status_code == 503 + data = resp.json() + assert data["error"]["code"] == "EXTERNAL_SERVICE_UNAVAILABLE" + _assert_envelope(data, path="/raise/embedding") + + +async def test_llm_service_error_routes_to_503(client: AsyncClient) -> None: + """LLMServiceError (InfrastructureError subclass) → 503 via MRO.""" + resp = await client.get("/raise/llm") + assert resp.status_code == 503 + data = resp.json() + assert data["error"]["code"] == "EXTERNAL_SERVICE_UNAVAILABLE" + _assert_envelope(data, path="/raise/llm") + + +# --------------------------------------------------------------------------- +# Unexpected exception test +# --------------------------------------------------------------------------- + + +async def test_unexpected_exception_returns_500(client: AsyncClient) -> None: + """RuntimeError → HTTP 500, INTERNAL_ERROR, generic message (no leak).""" + resp = await client.get("/raise/runtime") + assert resp.status_code == 500 + data = resp.json() + assert data["error"]["code"] == "INTERNAL_ERROR" + assert data["error"]["message"] == "Internal server error" + _assert_envelope(data, path="/raise/runtime") + + +# --------------------------------------------------------------------------- +# RequestValidationError test +# --------------------------------------------------------------------------- + + +async def test_request_validation_error_returns_422(client: AsyncClient) -> None: + """RequestValidationError (FastAPI body parse failure) → 422 VALIDATION_ERROR.""" + # POST with invalid JSON triggers RequestValidationError automatically. + resp = await client.post( + "/raise/request-validation", + content=b"not-json", + headers={"Content-Type": "application/json"}, + ) + assert resp.status_code == 422 + data = resp.json() + assert data["error"]["code"] == "INVALID_INPUT" + _assert_envelope(data, path="/raise/request-validation") + + +# --------------------------------------------------------------------------- +# Envelope shape tests (request_id, timestamp, path) +# --------------------------------------------------------------------------- + + +async def test_envelope_has_request_id_32_hex(client: AsyncClient) -> None: + """request_id in envelope must be exactly 32 lowercase hex characters.""" + resp = await client.get("/raise/not-found") + data = resp.json() + rid = data["request_id"] + assert len(rid) == 32 + assert rid == rid.lower() + assert all(c in "0123456789abcdef" for c in rid) + + +async def test_envelope_has_iso_timestamp(client: AsyncClient) -> None: + """error.timestamp must be a non-empty ISO 8601 string.""" + resp = await client.get("/raise/storage") + data = resp.json() + ts = data["error"]["timestamp"] + assert isinstance(ts, str) + assert len(ts) > 0 + # ISO 8601 basic sanity: contains 'T' separator + assert "T" in ts + + +async def test_envelope_path_matches_request(client: AsyncClient) -> None: + """error.path must match the actual request path.""" + resp = await client.get("/raise/conflict") + data = resp.json() + assert data["error"]["path"] == "/raise/conflict" diff --git a/tests/unit/test_entrypoints/test_api/test_lifespans/test_cascade.py b/tests/unit/test_entrypoints/test_api/test_lifespans/test_cascade.py index 9506461..0e9103b 100644 --- a/tests/unit/test_entrypoints/test_api/test_lifespans/test_cascade.py +++ b/tests/unit/test_entrypoints/test_api/test_lifespans/test_cascade.py @@ -30,7 +30,7 @@ def test_provider_metadata() -> None: async def test_startup_constructs_and_starts_orchestrator( tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model") monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1") monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key") @@ -59,7 +59,7 @@ async def test_shutdown_without_startup_is_noop() -> None: async def test_shutdown_stops_orchestrator_and_clears_reference( tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model") monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1") monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key") diff --git a/tests/unit/test_entrypoints/test_api/test_lifespans/test_ome.py b/tests/unit/test_entrypoints/test_api/test_lifespans/test_ome.py index 2e26021..45447e4 100644 --- a/tests/unit/test_entrypoints/test_api/test_lifespans/test_ome.py +++ b/tests/unit/test_entrypoints/test_api/test_lifespans/test_ome.py @@ -21,6 +21,7 @@ async def test_lifespan_starts_and_stops_engine( monkeypatch.setattr( MemoryRoot, "default", classmethod(lambda cls: MemoryRoot(root=tmp_path)) ) + (tmp_path / "ome.toml").write_text("# test\n") monkeypatch.setattr(svc, "_ome_engine", None, raising=False) provider = OmeLifespanProvider() diff --git a/tests/unit/test_entrypoints/test_api/test_lifespans/test_storage.py b/tests/unit/test_entrypoints/test_api/test_lifespans/test_storage.py index def53e2..312bbfe 100644 --- a/tests/unit/test_entrypoints/test_api/test_lifespans/test_storage.py +++ b/tests/unit/test_entrypoints/test_api/test_lifespans/test_storage.py @@ -18,7 +18,7 @@ from everos.infra.persistence.sqlite import sqlite_manager @pytest.fixture(autouse=True) async def _reset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """Redirect both managers at an isolated memory-root.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) sqlite_manager._engine = None sqlite_manager._session_factory = None lancedb_manager._conn = None diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_get_route_validation.py b/tests/unit/test_entrypoints/test_api/test_routes/test_get_route_validation.py index 8bf588b..11f0587 100644 --- a/tests/unit/test_entrypoints/test_api/test_routes/test_get_route_validation.py +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_get_route_validation.py @@ -37,7 +37,7 @@ async def client( monkeypatch: pytest.MonkeyPatch, ) -> AsyncIterator[AsyncClient]: """FastAPI app with no lifespan; resets get-path singletons per test.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) load_settings.cache_clear() lancedb_manager._conn = None diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_knowledge_api.py b/tests/unit/test_entrypoints/test_api/test_routes/test_knowledge_api.py new file mode 100644 index 0000000..40c8ff7 --- /dev/null +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_knowledge_api.py @@ -0,0 +1,541 @@ +"""Validation paths for knowledge HTTP API routes. + +Tests exercise DTO validation, error mapping, and route-level behavior +without external services (no LLM / no LanceDB / no embedder). Service +functions are mocked to isolate the presentation layer. + +White-box surfaces: none — all assertions are on HTTP responses. +""" + +from __future__ import annotations + +import sys +from collections.abc import AsyncIterator +from datetime import UTC, datetime +from importlib import import_module +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from httpx import ASGITransport, AsyncClient + +from everos.config import load_settings +from everos.entrypoints.api.app import create_app +from everos.infra.persistence.lancedb import lancedb_manager +from everos.service import ( + DocumentDetail, + DocumentListResult, + DocumentNotFoundError, + TopicDetail, + TopicNotFoundError, + TopicOverview, +) + +knowledge_service_mod = import_module("everos.service.knowledge") + +# The route module binds service functions at import time, so patches +# must target the name in the route module's namespace. +_ROUTE_MOD = "everos.entrypoints.api.routes.knowledge" + + +@pytest.fixture +async def client( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> AsyncIterator[AsyncClient]: + """FastAPI app with no lifespan; resets knowledge singletons per test.""" + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + load_settings.cache_clear() + + lancedb_manager._conn = None + lancedb_manager._tables.clear() + for attr in ("_embedding", "_reranker"): + setattr(knowledge_service_mod, attr, None) + for attr in ("_embedding_resolved", "_reranker_resolved"): + setattr(knowledge_service_mod, attr, False) + + app = create_app(lifespan_providers=[]) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as c: + yield c + + await lancedb_manager.dispose_connection() + load_settings.cache_clear() + + +# ── Fixtures ───────────────────────────────────────────────────────────────── + +_NOW = datetime(2026, 1, 1, tzinfo=UTC) + + +def _make_document_detail(doc_id: str = "d_aabbccddee01123abc123") -> DocumentDetail: + return DocumentDetail( + doc_id=doc_id, + category_id="Technology", + title="Test Doc", + summary="A test document.", + source_name="test.txt", + source_type="file", + original_file_path=None, + topics=[ + TopicOverview( + topic_id="d_abc123abc123_1", + topic_name="Intro", + topic_path="Intro", + depth=1, + summary="Introduction section.", + ), + ], + created_at=_NOW, + updated_at=_NOW, + ) + + +def _make_topic_detail(topic_id: str = "d_abc123abc123_1") -> TopicDetail: + return TopicDetail( + topic_id=topic_id, + doc_id="d_aabbccddee01123abc123", + category_id="Technology", + topic_name="Intro", + topic_path="Intro", + depth=1, + summary="Introduction section.", + content="Some content here.", + content_labels=["intro"], + parent_topic_id=None, + children_topic_ids=["d_abc123abc123_2"], + created_at=_NOW, + updated_at=_NOW, + ) + + +# ── GET /documents/{doc_id} ───────────────────────────────────────────────── + + +async def test_get_document_success(client: AsyncClient) -> None: + """Mocked service returns detail; route maps to 200 envelope.""" + detail = _make_document_detail() + with patch( + f"{_ROUTE_MOD}.get_document", + new_callable=AsyncMock, + return_value=detail, + ): + resp = await client.get("/api/v1/knowledge/documents/d_aabbccddee01123abc123") + + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["doc_id"] == "d_aabbccddee01123abc123" + assert body["data"]["topics"][0]["topic_id"] == "d_abc123abc123_1" + assert "request_id" in body + + +async def test_get_document_404(client: AsyncClient) -> None: + """Nonexistent doc_id returns 404.""" + with patch( + f"{_ROUTE_MOD}.get_document", + new_callable=AsyncMock, + side_effect=DocumentNotFoundError("d_000000000000"), + ): + resp = await client.get("/api/v1/knowledge/documents/d_000000000000") + + assert resp.status_code == 404 + + +# ── GET /topics/{topic_id} ────────────────────────────────────────────────── + + +async def test_get_topic_success(client: AsyncClient) -> None: + """Mocked service returns topic detail; route maps to 200 envelope.""" + detail = _make_topic_detail() + with patch( + f"{_ROUTE_MOD}.get_topic", + new_callable=AsyncMock, + return_value=detail, + ): + resp = await client.get("/api/v1/knowledge/topics/d_abc123abc123_1") + + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["topic_id"] == "d_abc123abc123_1" + assert body["data"]["children_topic_ids"] == ["d_abc123abc123_2"] + + +async def test_get_topic_404(client: AsyncClient) -> None: + """Nonexistent topic_id returns 404.""" + with patch( + f"{_ROUTE_MOD}.get_topic", + new_callable=AsyncMock, + side_effect=TopicNotFoundError("d_000000000000_999"), + ): + resp = await client.get("/api/v1/knowledge/topics/d_000000000000_999") + + assert resp.status_code == 404 + + +# ── POST /search ───────────────────────────────────────────────────────────── + + +async def test_search_422_empty_query(client: AsyncClient) -> None: + """Empty query string violates min_length=1.""" + resp = await client.post( + "/api/v1/knowledge/search", + json={"query": ""}, + ) + assert resp.status_code == 422 + + +async def test_search_422_invalid_method(client: AsyncClient) -> None: + """Invalid method value returns 422.""" + resp = await client.post( + "/api/v1/knowledge/search", + json={"query": "hello", "method": "bm42"}, + ) + assert resp.status_code == 422 + + +async def test_search_422_top_k_out_of_range(client: AsyncClient) -> None: + """top_k=0 or >100 returns 422.""" + resp = await client.post( + "/api/v1/knowledge/search", + json={"query": "hello", "top_k": 0}, + ) + assert resp.status_code == 422 + + resp = await client.post( + "/api/v1/knowledge/search", + json={"query": "hello", "top_k": 101}, + ) + assert resp.status_code == 422 + + +async def test_search_422_query_too_long(client: AsyncClient) -> None: + """A query beyond max_length is rejected before the embedding call.""" + resp = await client.post( + "/api/v1/knowledge/search", + json={"query": "x" * 2001}, + ) + assert resp.status_code == 422 + + +# ── GET /categories ────────────────────────────────────────────────────────── + + +async def test_get_categories_returns_taxonomy( + client: AsyncClient, + tmp_path: Path, +) -> None: + """Returns category list with document counts.""" + from everos.service import CategoryOverview + + overviews = [ + CategoryOverview( + category_id="Tech", description="Technology topics", document_count=3 + ), + CategoryOverview( + category_id="Science", description="Science topics", document_count=0 + ), + ] + with patch( + f"{_ROUTE_MOD}.list_categories", + new=AsyncMock(return_value=overviews), + ): + resp = await client.get("/api/v1/knowledge/categories") + + assert resp.status_code == 200 + body = resp.json() + cats = body["data"]["categories"] + assert len(cats) == 2 + assert cats[0]["category_id"] == "Tech" + assert cats[0]["document_count"] == 3 + assert cats[1]["description"] == "Science topics" + assert cats[1]["document_count"] == 0 + + +# ── GET /documents (list) ─────────────────────────────────────────────────── + + +async def test_list_documents_empty(client: AsyncClient) -> None: + """Empty result returns paginated envelope with zero items.""" + result = DocumentListResult(documents=[], total=0, page=1, page_size=20) + with patch( + f"{_ROUTE_MOD}.list_documents", + new_callable=AsyncMock, + return_value=result, + ): + resp = await client.get("/api/v1/knowledge/documents") + + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["documents"] == [] + assert body["data"]["total"] == 0 + + +async def test_list_documents_pagination_params(client: AsyncClient) -> None: + """Pagination params are forwarded to the service.""" + result = DocumentListResult(documents=[], total=0, page=2, page_size=5) + mock = AsyncMock(return_value=result) + with patch(f"{_ROUTE_MOD}.list_documents", mock): + resp = await client.get( + "/api/v1/knowledge/documents", + params={"page": 2, "page_size": 5, "sort_by": "title", "sort_order": "asc"}, + ) + + assert resp.status_code == 200 + mock.assert_called_once_with( + "default", + "default", + category_id=None, + page=2, + page_size=5, + sort_by="title", + sort_order="asc", + ) + + +async def test_list_documents_invalid_page_size(client: AsyncClient) -> None: + """page_size > 100 returns 422.""" + resp = await client.get( + "/api/v1/knowledge/documents", + params={"page_size": 200}, + ) + assert resp.status_code == 422 + + +async def test_list_documents_sort_by_updated_at(client: AsyncClient) -> None: + """sort_by=updated_at is accepted and forwarded (repo supports it).""" + result = DocumentListResult(documents=[], total=0, page=1, page_size=20) + mock = AsyncMock(return_value=result) + with patch(f"{_ROUTE_MOD}.list_documents", mock): + resp = await client.get( + "/api/v1/knowledge/documents", + params={"sort_by": "updated_at"}, + ) + + assert resp.status_code == 200 + assert mock.call_args.kwargs["sort_by"] == "updated_at" + + +def test_reject_oversized_upload() -> None: + """Uploads above max_upload_bytes are rejected; smaller/unknown pass.""" + from types import SimpleNamespace + + from everos.core.errors import InvalidInputError + from everos.entrypoints.api.routes.knowledge import _reject_oversized_upload + + over_limit = SimpleNamespace(size=load_settings().knowledge.max_upload_bytes + 1) + with pytest.raises(InvalidInputError, match="exceeds"): + _reject_oversized_upload(over_limit) # type: ignore[arg-type] # duck-typed stub + + _reject_oversized_upload(SimpleNamespace(size=1024)) # type: ignore[arg-type] + _reject_oversized_upload(SimpleNamespace(size=None)) # type: ignore[arg-type] + + +# ── DELETE /documents/{doc_id} ────────────────────────────────────────────── + + +async def test_delete_document_returns_204_when_not_found( + client: AsyncClient, +) -> None: + """Idempotent delete: nonexistent doc returns 204.""" + from everos.service import DeleteResult + + result = DeleteResult(doc_id="d_111111111111", deleted_topics=0) + with patch( + f"{_ROUTE_MOD}.delete_document", + new_callable=AsyncMock, + return_value=result, + ): + resp = await client.delete("/api/v1/knowledge/documents/d_111111111111") + + assert resp.status_code == 204 + + +async def test_delete_document_returns_envelope(client: AsyncClient) -> None: + """Successful delete with topics returns envelope.""" + from everos.service import DeleteResult + + result = DeleteResult(doc_id="d_aabbccddee01", deleted_topics=3) + with patch( + f"{_ROUTE_MOD}.delete_document", + new_callable=AsyncMock, + return_value=result, + ): + resp = await client.delete("/api/v1/knowledge/documents/d_aabbccddee01") + + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["deleted_topics"] == 3 + + +# ── PUT /documents/{doc_id} ─────────────────────────────────────────────── + + +async def test_put_document_404_when_not_found(client: AsyncClient) -> None: + """PUT on nonexistent doc_id returns 404 (strict replace, not upsert).""" + from everalgo.types import ParsedContent + + with ( + patch( + f"{_ROUTE_MOD}._parse_upload", + new_callable=AsyncMock, + return_value=ParsedContent(text="# Hello"), + ), + patch( + f"{_ROUTE_MOD}._build_extractor", + return_value=AsyncMock(), + ), + patch( + f"{_ROUTE_MOD}.replace_document", + new_callable=AsyncMock, + side_effect=DocumentNotFoundError("d_000000000000"), + ), + ): + resp = await client.put( + "/api/v1/knowledge/documents/d_000000000000", + files={"file": ("test.md", b"# Hello", "text/markdown")}, + data={"title": "Test"}, + ) + + assert resp.status_code == 404 + + +# ── PATCH /documents/{doc_id} ────────────────────────────────────────────── + + +async def test_patch_document_success(client: AsyncClient) -> None: + """Successful patch returns updated fields.""" + from everos.service import PatchResult + + result = PatchResult( + doc_id="d_aabbccddee01", updated_fields=["title"], updated_at=_NOW + ) + with patch( + f"{_ROUTE_MOD}.patch_document", + new_callable=AsyncMock, + return_value=result, + ): + resp = await client.patch( + "/api/v1/knowledge/documents/d_aabbccddee01", + json={"title": "New Title"}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["updated_fields"] == ["title"] + + +async def test_patch_document_404(client: AsyncClient) -> None: + """Patching nonexistent doc returns 404.""" + with patch( + f"{_ROUTE_MOD}.patch_document", + new_callable=AsyncMock, + side_effect=DocumentNotFoundError("d_000000000000"), + ): + resp = await client.patch( + "/api/v1/knowledge/documents/d_000000000000", + json={"title": "New Title"}, + ) + + assert resp.status_code == 404 + + +# ── PathSafeId validation ─────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/knowledge/documents/bad_format", + "/api/v1/knowledge/documents/'; DROP TABLE--", + "/api/v1/knowledge/documents/d_ZZZZ00000000", + ], +) +async def test_invalid_doc_id_format_returns_422( + client: AsyncClient, + path: str, +) -> None: + """doc_id path param must match ``d_[a-f0-9]{12,32}``.""" + resp = await client.get(path) + assert resp.status_code == 422 + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/knowledge/topics/bad_format", + "/api/v1/knowledge/topics/d_abc123abc123", + "/api/v1/knowledge/topics/not_valid_at_all", + ], +) +async def test_invalid_topic_id_format_returns_422( + client: AsyncClient, + path: str, +) -> None: + """topic_id path param must match ``d_[a-f0-9]{12,32}_\\d+``.""" + resp = await client.get(path) + assert resp.status_code == 422 + + +async def test_pathsafe_rejects_traversal_in_query(client: AsyncClient) -> None: + """app_id with '..' in query param is rejected.""" + result = DocumentListResult(documents=[], total=0, page=1, page_size=20) + with patch( + f"{_ROUTE_MOD}.list_documents", + new_callable=AsyncMock, + return_value=result, + ): + resp = await client.get( + "/api/v1/knowledge/documents", + params={"app_id": ".."}, + ) + + assert resp.status_code == 422 + + +async def test_pathsafe_rejects_traversal_in_body(client: AsyncClient) -> None: + """app_id with '..' in JSON body is rejected.""" + resp = await client.post( + "/api/v1/knowledge/search", + json={"query": "hello", "app_id": ".."}, + ) + assert resp.status_code == 422 + + +# ── _parse_upload: binary file rejection ──────────────────────────────────── + + +def _hide_everalgo_parser() -> MagicMock: + """Return a sys.modules patch that makes ``everalgo.parser`` unimportable.""" + fake_modules = {k: v for k, v in sys.modules.items()} + fake_modules["everalgo.parser"] = None # type: ignore[assignment] + return fake_modules + + +async def test_post_binary_file_without_parser_returns_415( + client: AsyncClient, +) -> None: + """Non-UTF-8 binary file without parser → UnsupportedModalityError → 415.""" + with patch.dict(sys.modules, {"everalgo.parser": None}): # type: ignore[dict-item] + resp = await client.post( + "/api/v1/knowledge/documents", + files={ + "file": ("test.bin", b"\x80\x81\x82\x83", "application/octet-stream") + }, + data={"title": "Binary Test"}, + ) + assert resp.status_code == 415 + + +async def test_put_binary_file_without_parser_returns_415( + client: AsyncClient, +) -> None: + """Non-UTF-8 binary file on PUT → UnsupportedModalityError → 415.""" + with patch.dict(sys.modules, {"everalgo.parser": None}): # type: ignore[dict-item] + resp = await client.put( + "/api/v1/knowledge/documents/d_aabbccddee01", + files={ + "file": ("test.bin", b"\x80\x81\x82\x83", "application/octet-stream") + }, + data={"title": "Binary Test"}, + ) + assert resp.status_code == 415 diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py b/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py index 234c49a..045bae2 100644 --- a/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py @@ -1,4 +1,12 @@ -"""DTO-layer path-safety validation for ``POST /api/v1/memory/add``.""" +"""DTO-layer path-safety validation for ``POST /api/v1/memory/add``. + +``sender_id`` flows through to ``owner_id`` and is joined into the episode +write path as a directory segment, so it must carry the same path-traversal +guard as ``app_id`` / ``project_id`` (charset whitelist + ``.``/``..`` +rejection). These tests pin that guard at the DTO layer; the writer-level +containment backstop is covered in +``tests/unit/test_core/test_persistence/test_markdown/test_writer.py``. +""" from __future__ import annotations @@ -23,13 +31,13 @@ def _message(sender_id: str) -> MessageItemDTO: @pytest.mark.parametrize( "bad_sender_id", [ - "../../../../etc", - "..", - ".", - "a/b", - "a/../b", - "with space", - "", + "../../../../etc", # classic traversal + "..", # reserved parent token + ".", # reserved current-dir token + "a/b", # embedded path separator + "a/../b", # separator + traversal mid-string + "with space", # outside the charset whitelist + "", # empty (min_length) ], ) def test_message_item_rejects_unsafe_sender_id(bad_sender_id: str) -> None: @@ -45,9 +53,9 @@ def test_message_item_rejects_unsafe_sender_id(bad_sender_id: str) -> None: "user-123", "a.b_c-1", "default", - "user@example.com", - "user+tag", - "user+tag@example.com", + "user@example.com", # email-style id (``@`` + dotted domain) + "user+tag", # plus-addressing + "user+tag@example.com", # both, combined ], ) def test_message_item_accepts_path_safe_sender_id(good_sender_id: str) -> None: @@ -55,6 +63,7 @@ def test_message_item_accepts_path_safe_sender_id(good_sender_id: str) -> None: def test_add_request_rejects_traversal_sender_id_in_messages() -> None: + # The guard fires through the nested message list, not just on a bare DTO. with pytest.raises(ValidationError): MemorizeAddRequest( session_id="s1", diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_metrics_route.py b/tests/unit/test_entrypoints/test_api/test_routes/test_metrics_route.py index 94c86b1..abeff4b 100644 --- a/tests/unit/test_entrypoints/test_api/test_routes/test_metrics_route.py +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_metrics_route.py @@ -37,7 +37,7 @@ async def client( monkeypatch: pytest.MonkeyPatch, ) -> AsyncIterator[AsyncClient]: """FastAPI app with no lifespan; middleware stack is wired by ``create_app``.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) load_settings.cache_clear() app = create_app(lifespan_providers=[]) diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_search_route_validation.py b/tests/unit/test_entrypoints/test_api/test_routes/test_search_route_validation.py index 4c7e609..27a16da 100644 --- a/tests/unit/test_entrypoints/test_api/test_routes/test_search_route_validation.py +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_search_route_validation.py @@ -28,7 +28,7 @@ async def client( monkeypatch: pytest.MonkeyPatch, ) -> AsyncIterator[AsyncClient]: """FastAPI app with no lifespan; resets search singletons per test.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) load_settings.cache_clear() lancedb_manager._conn = None diff --git a/tests/unit/test_entrypoints/test_cli/test_cascade_command.py b/tests/unit/test_entrypoints/test_cli/test_cascade_command.py index 5434d7c..d27a1c4 100644 --- a/tests/unit/test_entrypoints/test_cli/test_cascade_command.py +++ b/tests/unit/test_entrypoints/test_cli/test_cascade_command.py @@ -37,7 +37,7 @@ def test_help_exits_zero() -> None: def test_resolve_relative_under_root( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) from everos.config import load_settings load_settings.cache_clear() @@ -49,7 +49,7 @@ def test_resolve_relative_under_root( def test_resolve_relative_outside_root_raises( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path / "memory")) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path / "memory")) from everos.config import load_settings load_settings.cache_clear() diff --git a/tests/unit/test_entrypoints/test_cli/test_init_command.py b/tests/unit/test_entrypoints/test_cli/test_init_command.py index 79f0e45..f38be73 100644 --- a/tests/unit/test_entrypoints/test_cli/test_init_command.py +++ b/tests/unit/test_entrypoints/test_cli/test_init_command.py @@ -2,18 +2,15 @@ Covers: -- default ``./.env`` path, written with 0600 permissions -- ``--to `` creates parent dirs -- ``--force`` overwrites; without it the command refuses with exit 1 -- ``--print`` writes to stdout, NOT to disk -- ``--xdg`` and ``--to`` are mutually exclusive (exit 2) -- ``--xdg`` honors ``XDG_CONFIG_HOME`` +- default ``~/.everos/`` root with ``everos.toml`` + ``ome.toml`` +- ``--root `` creates target dir and both files +- ``--force`` overwrites; without it the command exits 1 +- ``--print`` writes everos.toml template to stdout, NOT to disk """ from __future__ import annotations import os -import stat from pathlib import Path import pytest @@ -27,213 +24,92 @@ def runner() -> CliRunner: return CliRunner() -@pytest.fixture -def in_tmp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Run from a fresh tmp cwd so default ``./.env`` lands in tmp_path.""" +@pytest.fixture(autouse=True) +def _isolate_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Strip EVEROS_* env vars and move CWD away from any config file.""" + for key in list(os.environ): + if key.startswith("EVEROS_"): + monkeypatch.delenv(key, raising=False) monkeypatch.chdir(tmp_path) - return tmp_path -def test_default_writes_dotenv_in_cwd(runner: CliRunner, in_tmp: Path) -> None: - result = runner.invoke(app, ["init"]) +def test_root_creates_both_toml_files(runner: CliRunner, tmp_path: Path) -> None: + target = tmp_path / "myroot" + result = runner.invoke(app, ["init", "--root", str(target)]) assert result.exit_code == 0, result.output - written = in_tmp / ".env" - assert written.exists() - assert written.stat().st_size > 0 - assert "EVEROS_LLM__API_KEY" in written.read_text() - assert "https://github.com/EverMind-AI/EverOS/blob/main/QUICKSTART.md" in ( - result.output + assert (target / "everos.toml").is_file() + assert (target / "ome.toml").is_file() + + +def test_created_everos_toml_matches_shipped_template( + runner: CliRunner, tmp_path: Path +) -> None: + target = tmp_path / "myroot" + runner.invoke(app, ["init", "--root", str(target)]) + template = Path(__file__).resolve().parents[4] / "src/everos/config/default.toml" + assert (target / "everos.toml").read_bytes() == template.read_bytes() + + +def test_created_ome_toml_matches_shipped_template( + runner: CliRunner, tmp_path: Path +) -> None: + target = tmp_path / "myroot" + runner.invoke(app, ["init", "--root", str(target)]) + template = ( + Path(__file__).resolve().parents[4] / "src/everos/config/default_ome.toml" ) + assert (target / "ome.toml").read_bytes() == template.read_bytes() -def test_default_file_permissions_are_0600(runner: CliRunner, in_tmp: Path) -> None: - """The generated .env holds API keys — must not be world-readable.""" - result = runner.invoke(app, ["init"]) - assert result.exit_code == 0 - mode = stat.S_IMODE((in_tmp / ".env").stat().st_mode) - assert mode == 0o600, f"expected 0o600, got {oct(mode)}" - - -def test_refuses_overwrite_without_force(runner: CliRunner, in_tmp: Path) -> None: - (in_tmp / ".env").write_text("PREEXISTING=1\n") - result = runner.invoke(app, ["init"]) +def test_refuses_overwrite_without_force(runner: CliRunner, tmp_path: Path) -> None: + target = tmp_path / "myroot" + target.mkdir() + (target / "everos.toml").write_text("# user-edited\n") + (target / "ome.toml").write_text("# user-edited\n") + result = runner.invoke(app, ["init", "--root", str(target)]) assert result.exit_code == 1 - assert "already exists" in (result.output + (result.stderr or "")) # Original content must be preserved. - assert (in_tmp / ".env").read_text() == "PREEXISTING=1\n" + assert (target / "everos.toml").read_text() == "# user-edited\n" -def test_force_overwrites(runner: CliRunner, in_tmp: Path) -> None: - (in_tmp / ".env").write_text("PREEXISTING=1\n") - result = runner.invoke(app, ["init", "--force"]) +def test_force_overwrites(runner: CliRunner, tmp_path: Path) -> None: + target = tmp_path / "myroot" + target.mkdir() + (target / "everos.toml").write_text("# user-edited\n") + (target / "ome.toml").write_text("# user-edited\n") + result = runner.invoke(app, ["init", "--root", str(target), "--force"]) assert result.exit_code == 0 - body = (in_tmp / ".env").read_text() - assert "PREEXISTING=1" not in body - assert "EVEROS_LLM__API_KEY" in body + # Content is now the shipped template, not the user edit. + assert (target / "everos.toml").read_text() != "# user-edited\n" -def test_to_creates_parent_dirs(runner: CliRunner, in_tmp: Path) -> None: - target = in_tmp / "nested" / "subdir" / ".env" - result = runner.invoke(app, ["init", "--to", str(target)]) - assert result.exit_code == 0 - assert target.exists() - assert "EVEROS_LLM__API_KEY" in target.read_text() - - -def test_print_writes_stdout_not_disk(runner: CliRunner, in_tmp: Path) -> None: +def test_print_writes_stdout_not_disk(runner: CliRunner, tmp_path: Path) -> None: result = runner.invoke(app, ["init", "--print"]) assert result.exit_code == 0 - assert "EVEROS_LLM__API_KEY" in result.output - # No disk side-effect. - assert not (in_tmp / ".env").exists() + # Output contains shipped default.toml content. + assert "[sqlite]" in result.output + assert "[api]" in result.output + # No disk side-effect in tmp cwd. + assert not (tmp_path / "everos.toml").exists() -def test_xdg_writes_to_xdg_config_home( - runner: CliRunner, in_tmp: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - xdg_root = in_tmp / "xdg" - monkeypatch.setenv("XDG_CONFIG_HOME", str(xdg_root)) - result = runner.invoke(app, ["init", "--xdg"]) +def test_partial_overwrite_skips_existing(runner: CliRunner, tmp_path: Path) -> None: + """When only one file exists, only the missing file is created.""" + target = tmp_path / "myroot" + target.mkdir() + (target / "everos.toml").write_text("# user-edited\n") + result = runner.invoke(app, ["init", "--root", str(target)]) assert result.exit_code == 0 - target = xdg_root / "everos" / ".env" - assert target.exists() + # everos.toml preserved, ome.toml created. + assert (target / "everos.toml").read_text() == "# user-edited\n" + assert (target / "ome.toml").is_file() -def test_xdg_falls_back_to_dot_config( - runner: CliRunner, in_tmp: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """No ``XDG_CONFIG_HOME`` → default ``~/.config``. - - We sandbox ``$HOME`` to ``in_tmp`` so the test does not touch a real - user's ``~/.config``. - """ - monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) - monkeypatch.setenv("HOME", str(in_tmp)) - result = runner.invoke(app, ["init", "--xdg"]) - assert result.exit_code == 0 - target = in_tmp / ".config" / "everos" / ".env" - assert target.exists() - - -def test_xdg_and_to_are_mutually_exclusive(runner: CliRunner, in_tmp: Path) -> None: - result = runner.invoke(app, ["init", "--xdg", "--to", str(in_tmp / "other.env")]) - assert result.exit_code == 2 - assert "mutually exclusive" in (result.output + (result.stderr or "")) - - -def test_template_resource_is_packaged_under_everos_templates() -> None: - """The packaged resource must remain at the canonical location. - - Guards the wheel/sdist layout: ``init_cmd`` reads - ``everos.templates.env.template`` via ``importlib.resources``; if - someone moves the file without updating ``_TEMPLATE_PACKAGE``, this - test fails immediately. - """ - from importlib import resources - - res = resources.files("everos.templates").joinpath("env.template") - assert res.is_file() - body = res.read_text(encoding="utf-8") - assert "EVEROS_LLM__API_KEY" in body - assert "DASHSCOPE_API_KEY" in body - assert "text-embedding-v4" in body - assert "gte-rerank-v2" in body - - -def test_template_has_single_active_value_per_env_key() -> None: - """Alternative provider examples must stay commented to avoid overrides.""" - from collections import Counter - from importlib import resources - - body = ( - resources.files("everos.templates") - .joinpath("env.template") - .read_text(encoding="utf-8") - ) - active_assignments = [ - line.strip() - for line in body.splitlines() - if line.strip() and not line.lstrip().startswith("#") and "=" in line - ] - assert all("<" not in line for line in active_assignments) - - keys = [line.split("=", 1)[0] for line in active_assignments] - duplicates = [key for key, count in Counter(keys).items() if count > 1] - assert duplicates == [] - - -# ── 4-layer .env resolution for ``server start`` ──────────────────────── - - -def test_resolve_env_file_explicit_wins(in_tmp: Path) -> None: - """``--env-file `` beats cwd / XDG / ~/.everos fallbacks.""" - from everos.entrypoints.cli.commands.server import _resolve_env_file - - explicit = in_tmp / "explicit.env" - explicit.write_text("X=1\n") - # Also seed cwd .env so we can prove the explicit wins. - (in_tmp / ".env").write_text("CWD=1\n") - resolved = _resolve_env_file(str(explicit)) - assert resolved == explicit - - -def test_resolve_env_file_cwd_wins_over_xdg( - in_tmp: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from everos.entrypoints.cli.commands.server import _resolve_env_file - - xdg_root = in_tmp / "xdg" - (xdg_root / "everos").mkdir(parents=True) - (xdg_root / "everos" / ".env").write_text("XDG=1\n") - monkeypatch.setenv("XDG_CONFIG_HOME", str(xdg_root)) - cwd_env = in_tmp / ".env" - cwd_env.write_text("CWD=1\n") - resolved = _resolve_env_file(None) - assert resolved == cwd_env - - -def test_resolve_env_file_xdg_when_no_cwd( - in_tmp: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from everos.entrypoints.cli.commands.server import _resolve_env_file - - xdg_root = in_tmp / "xdg" - (xdg_root / "everos").mkdir(parents=True) - target = xdg_root / "everos" / ".env" - target.write_text("XDG=1\n") - monkeypatch.setenv("XDG_CONFIG_HOME", str(xdg_root)) - # No cwd/.env. - resolved = _resolve_env_file(None) - assert resolved == target - - -def test_resolve_env_file_everos_home_fallback( - in_tmp: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """``~/.everos/.env`` is the last fallback when nothing else exists.""" - from everos.entrypoints.cli.commands.server import _resolve_env_file - - monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) - monkeypatch.setenv("HOME", str(in_tmp)) - target = in_tmp / ".everos" / ".env" - target.parent.mkdir(parents=True) - target.write_text("EVEROS_ROOT=1\n") - resolved = _resolve_env_file(None) - assert resolved == target - - -def test_resolve_env_file_none_when_no_layer_matches( - in_tmp: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """All four layers absent → ``None`` (the server then falls back to - inherited process env, which is the documented CI/container path).""" - from everos.entrypoints.cli.commands.server import _resolve_env_file - - monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) - monkeypatch.setenv("HOME", str(in_tmp)) - # Nothing in cwd, no XDG path, no ~/.everos/. - assert not (in_tmp / ".env").exists() - assert _resolve_env_file(None) is None +def test_output_shows_next_steps(runner: CliRunner, tmp_path: Path) -> None: + target = tmp_path / "myroot" + result = runner.invoke(app, ["init", "--root", str(target)]) + assert "Next steps" in result.output + assert "everos server start" in result.output # ``os`` imported above just to keep ruff from complaining; remove if Ruff diff --git a/tests/unit/test_entrypoints/test_cli/test_main.py b/tests/unit/test_entrypoints/test_cli/test_main.py index 3a4214f..093697c 100644 --- a/tests/unit/test_entrypoints/test_cli/test_main.py +++ b/tests/unit/test_entrypoints/test_cli/test_main.py @@ -14,6 +14,7 @@ def test_help_exits_zero() -> None: assert "server" in result.stdout assert "cascade" in result.stdout assert "demo" in result.stdout + assert "config" in result.stdout def test_no_args_shows_help_and_exits_nonzero() -> None: diff --git a/tests/unit/test_entrypoints/test_cli/test_server_command.py b/tests/unit/test_entrypoints/test_cli/test_server_command.py index bac93c8..025e15a 100644 --- a/tests/unit/test_entrypoints/test_cli/test_server_command.py +++ b/tests/unit/test_entrypoints/test_cli/test_server_command.py @@ -7,6 +7,8 @@ KeyboardInterrupt / OSError exit paths. from __future__ import annotations +import os + import pytest from typer.testing import CliRunner @@ -14,6 +16,20 @@ from everos.entrypoints.cli.commands import server as server_mod from everos.entrypoints.cli.main import app as root_app +@pytest.fixture(autouse=True) +def _isolate_env(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: # type: ignore[no-untyped-def] + """Strip EVEROS_* env vars so default resolution is deterministic.""" + for k in list(os.environ): + if k.startswith("EVEROS_"): + monkeypatch.delenv(k, raising=False) + monkeypatch.delenv("EVEROS_LOG_LEVEL", raising=False) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + (tmp_path / "everos.toml").write_text("# test\n") + from everos.config import load_settings + + load_settings.cache_clear() + + @pytest.fixture def captured(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: """Mock ``uvicorn.run`` and return the kwargs it was called with.""" @@ -24,20 +40,11 @@ def captured(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: captured["kwargs"] = kwargs monkeypatch.setattr(server_mod.uvicorn, "run", fake_run) - # Strip env so default resolution path is deterministic. - for k in ("EVEROS_HOST", "EVEROS_PORT", "EVEROS_LOG_LEVEL"): - monkeypatch.delenv(k, raising=False) return captured -# Typer lifts single-command sub-apps to root; we invoke via the real -# ``everos server start`` path through the assembled root app. - - def test_start_uses_default_host_port_log_level(captured: dict[str, object]) -> None: - result = CliRunner().invoke( - root_app, ["server", "start", "--env-file", "/nonexistent"] - ) + result = CliRunner().invoke(root_app, ["server", "start"]) assert result.exit_code == 0, result.stdout kwargs = captured["kwargs"] assert isinstance(kwargs, dict) @@ -55,13 +62,14 @@ def test_start_cli_flags_override_env( monkeypatch.setenv("EVEROS_API__HOST", "1.2.3.4") monkeypatch.setenv("EVEROS_API__PORT", "9000") monkeypatch.setenv("EVEROS_API__LOG_LEVEL", "debug") + from everos.config import load_settings + + load_settings.cache_clear() result = CliRunner().invoke( root_app, [ "server", "start", - "--env-file", - "/nonexistent", "--host", "127.0.0.1", "--port", @@ -83,9 +91,10 @@ def test_start_falls_back_to_env_when_flags_omitted( ) -> None: monkeypatch.setenv("EVEROS_API__HOST", "10.0.0.1") monkeypatch.setenv("EVEROS_API__PORT", "8765") - result = CliRunner().invoke( - root_app, ["server", "start", "--env-file", "/nonexistent"] - ) + from everos.config import load_settings + + load_settings.cache_clear() + result = CliRunner().invoke(root_app, ["server", "start"]) assert result.exit_code == 0, result.stdout kwargs = captured["kwargs"] assert isinstance(kwargs, dict) @@ -98,10 +107,7 @@ def test_start_swallows_keyboard_interrupt(monkeypatch: pytest.MonkeyPatch) -> N raise KeyboardInterrupt monkeypatch.setattr(server_mod.uvicorn, "run", boom) - result = CliRunner().invoke( - root_app, ["server", "start", "--env-file", "/nonexistent"] - ) - # KeyboardInterrupt path returns normally — exit 0. + result = CliRunner().invoke(root_app, ["server", "start"]) assert result.exit_code == 0 @@ -110,25 +116,19 @@ def test_start_exits_one_on_os_error(monkeypatch: pytest.MonkeyPatch) -> None: raise OSError("port in use") monkeypatch.setattr(server_mod.uvicorn, "run", boom) - result = CliRunner().invoke( - root_app, ["server", "start", "--env-file", "/nonexistent"] - ) + result = CliRunner().invoke(root_app, ["server", "start"]) assert result.exit_code == 1 -def test_load_env_file_missing_path_is_noop(tmp_path) -> None: # type: ignore[no-untyped-def] - # Function should not raise when the file does not exist. - server_mod._load_env_file(str(tmp_path / "does-not-exist.env")) +def test_start_with_root_option( + captured: dict[str, object], + monkeypatch: pytest.MonkeyPatch, + tmp_path, # type: ignore[no-untyped-def] +) -> None: + """``--root`` sets EVEROS_ROOT for settings resolution.""" + from everos.config import load_settings - -def test_load_env_file_reads_present_file( - tmp_path, monkeypatch: pytest.MonkeyPatch -) -> None: # type: ignore[no-untyped-def] - monkeypatch.delenv("EVEROS_TEST_DOTENV_VAR", raising=False) - env_file = tmp_path / ".env" - env_file.write_text("EVEROS_TEST_DOTENV_VAR=loaded\n") - server_mod._load_env_file(str(env_file)) - import os - - assert os.environ.get("EVEROS_TEST_DOTENV_VAR") == "loaded" - monkeypatch.delenv("EVEROS_TEST_DOTENV_VAR", raising=False) + load_settings.cache_clear() + result = CliRunner().invoke(root_app, ["server", "start", "--root", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + assert "kwargs" in captured diff --git a/tests/unit/test_infra/test_lancedb/test_lancedb_manager.py b/tests/unit/test_infra/test_lancedb/test_lancedb_manager.py index 82579f7..51dbdbb 100644 --- a/tests/unit/test_infra/test_lancedb/test_lancedb_manager.py +++ b/tests/unit/test_infra/test_lancedb/test_lancedb_manager.py @@ -25,7 +25,7 @@ class _DemoVec(BaseLanceTable): @pytest.fixture(autouse=True) async def _reset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """Point the singleton at an isolated memory-root and reset module state.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) lancedb_manager._conn = None lancedb_manager._tables.clear() yield diff --git a/tests/unit/test_infra/test_lancedb/test_repos/test_agent_skill.py b/tests/unit/test_infra/test_lancedb/test_repos/test_agent_skill.py index 7737030..97bfe27 100644 --- a/tests/unit/test_infra/test_lancedb/test_repos/test_agent_skill.py +++ b/tests/unit/test_infra/test_lancedb/test_repos/test_agent_skill.py @@ -52,7 +52,7 @@ def _skill_row( @pytest.fixture async def _real_lancedb(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """Spin up a clean LanceDB rooted under ``tmp_path`` for one test.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) lancedb_manager._conn = None lancedb_manager._tables.clear() yield diff --git a/tests/unit/test_infra/test_lancedb/test_tables/test_knowledge_topic_schema.py b/tests/unit/test_infra/test_lancedb/test_tables/test_knowledge_topic_schema.py new file mode 100644 index 0000000..9cff5b2 --- /dev/null +++ b/tests/unit/test_infra/test_lancedb/test_tables/test_knowledge_topic_schema.py @@ -0,0 +1,39 @@ +"""KnowledgeTopic LanceDB table schema validation.""" + +from __future__ import annotations + +from everos.infra.persistence.lancedb import KnowledgeTopic + + +class TestKnowledgeTopicSchema: + def test_table_name(self) -> None: + assert KnowledgeTopic.TABLE_NAME == "knowledge_topic" + + def test_bm25_fields_dual_column(self) -> None: + assert KnowledgeTopic.BM25_FIELDS == ["summary_tokens", "content_tokens"] + + def test_has_required_fields(self) -> None: + fields = set(KnowledgeTopic.model_fields.keys()) + required = { + "id", + "doc_id", + "category_id", + "app_id", + "project_id", + "topic_name", + "topic_path", + "depth", + "parent_node_id", + "summary", + "summary_tokens", + "content_tokens", + "content_labels", + "md_path", + "content_sha256", + "vector", + } + assert required.issubset(fields), f"Missing: {required - fields}" + + def test_arrow_schema_has_utc_timestamps(self) -> None: + schema = KnowledgeTopic.to_arrow_schema() + assert schema is not None diff --git a/tests/unit/test_infra/test_markdown/test_mds/test_knowledge_frontmatter.py b/tests/unit/test_infra/test_markdown/test_mds/test_knowledge_frontmatter.py new file mode 100644 index 0000000..5a1731d --- /dev/null +++ b/tests/unit/test_infra/test_markdown/test_mds/test_knowledge_frontmatter.py @@ -0,0 +1,80 @@ +"""Frontmatter parse/dump round-trip for knowledge document + topic.""" + +from __future__ import annotations + +from everos.infra.persistence.markdown import ( + KnowledgeDocumentFrontmatter, + KnowledgeTopicFrontmatter, +) + + +class TestKnowledgeDocumentFrontmatter: + def test_type_literal(self) -> None: + fm = KnowledgeDocumentFrontmatter( + type="knowledge_document", + id="d_abc123000000", + doc_id="d_abc123000000", + category_id="Sports", + title="Olympics Plan", + schema_version=1, + ) + assert fm.type == "knowledge_document" + assert fm.id == fm.doc_id + + def test_path_glob(self) -> None: + assert KnowledgeDocumentFrontmatter.path_glob() == "*/*/knowledge/*/*/index.md" + + def test_scope_dir(self) -> None: + assert KnowledgeDocumentFrontmatter.SCOPE_DIR == "knowledge" + + def test_optional_fields_default_none(self) -> None: + fm = KnowledgeDocumentFrontmatter( + type="knowledge_document", + id="d_abc", + doc_id="d_abc", + category_id="Sports", + title="X", + schema_version=1, + ) + assert fm.source_name is None + assert fm.source_type is None + + +class TestKnowledgeTopicFrontmatter: + def test_type_literal(self) -> None: + fm = KnowledgeTopicFrontmatter( + type="knowledge_topic", + id="d_abc_1", + node_id="d_abc_1", + doc_id="d_abc", + category_id="Sports", + topic_index=1, + topic_name="Budget", + topic_path="Olympics > Budget", + summary="Budget overview.", + depth=1, + schema_version=1, + ) + assert fm.type == "knowledge_topic" + assert fm.id == fm.node_id + + def test_path_glob(self) -> None: + assert KnowledgeTopicFrontmatter.path_glob() == "*/*/knowledge/*/*/[0-9]*.md" + + def test_defaults(self) -> None: + fm = KnowledgeTopicFrontmatter( + type="knowledge_topic", + id="d_abc_1", + node_id="d_abc_1", + doc_id="d_abc", + category_id="Sports", + topic_index=1, + topic_name="Budget", + topic_path="Olympics > Budget", + summary="Summary.", + depth=1, + schema_version=1, + ) + assert fm.parent_node_id is None + assert fm.children_node_ids == [] + assert fm.content_labels == [] diff --git a/tests/unit/test_infra/test_markdown/test_readers/test_taxonomy_reader.py b/tests/unit/test_infra/test_markdown/test_readers/test_taxonomy_reader.py new file mode 100644 index 0000000..788523f --- /dev/null +++ b/tests/unit/test_infra/test_markdown/test_readers/test_taxonomy_reader.py @@ -0,0 +1,56 @@ +"""Tests for .taxonomy.md parsing and auto-generation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from everos.infra.persistence.markdown import ensure_taxonomy, parse_taxonomy + + +@pytest.fixture() +def taxonomy_md(tmp_path: Path) -> Path: + p = tmp_path / ".taxonomy.md" + p.write_text( + "---\n" + "kind: knowledge_taxonomy\n" + "categories:\n" + " - id: Sports\n" + ' description: "Content about sports events."\n' + " - id: Others\n" + ' description: "Catch-all."\n' + "---\n" + ) + return p + + +class TestParseTaxonomy: + async def test_parses_categories(self, taxonomy_md: Path) -> None: + entries = await parse_taxonomy(taxonomy_md) + assert len(entries) == 2 + assert entries[0].id == "Sports" + assert entries[0].description == "Content about sports events." + + async def test_empty_categories_returns_empty(self, tmp_path: Path) -> None: + p = tmp_path / ".taxonomy.md" + p.write_text("---\nkind: knowledge_taxonomy\ncategories: []\n---\n") + assert await parse_taxonomy(p) == [] + + +class TestEnsureTaxonomy: + async def test_creates_default_when_missing(self, tmp_path: Path) -> None: + await ensure_taxonomy(tmp_path) + p = tmp_path / ".taxonomy.md" + assert p.exists() + entries = await parse_taxonomy(p) + assert len(entries) >= 20 + ids = [e.id for e in entries] + assert "Others" in ids + assert "Technology" in ids + assert "Medical" in ids + + async def test_does_not_overwrite_existing(self, taxonomy_md: Path) -> None: + await ensure_taxonomy(taxonomy_md.parent) + entries = await parse_taxonomy(taxonomy_md) + assert len(entries) == 2 # still the original 2, not overwritten diff --git a/tests/unit/test_infra/test_markdown/test_writers/test_knowledge_writer.py b/tests/unit/test_infra/test_markdown/test_writers/test_knowledge_writer.py new file mode 100644 index 0000000..f5ff901 --- /dev/null +++ b/tests/unit/test_infra/test_markdown/test_writers/test_knowledge_writer.py @@ -0,0 +1,306 @@ +"""Tests for :class:`KnowledgeWriter` — knowledge document directory layout.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from everos.infra.persistence.markdown.writers import KnowledgeWriter +from everos.infra.persistence.markdown.writers.knowledge_writer import ( + KnowledgeMemory, +) + +# ── Fixtures ────────────────────────────────────────────────────────────── + + +def _root_node( + doc_id: str = "d_abc123", + topic: str = "Olympics Plan", + summary: str = "Overview of the Olympic plan.", + category_id: str = "Sports", + **overrides: object, +) -> KnowledgeMemory: + defaults: dict[str, object] = { + "doc_id": doc_id, + "topic_index": 0, + "topic": topic, + "summary": summary, + "content": "", + "depth": 0, + "parent_index": None, + "children_index": [1, 2], + "topic_path": topic, + "content_labels": [], + "category_id": category_id, + } + defaults.update(overrides) + return KnowledgeMemory(**defaults) # type: ignore[arg-type] + + +def _topic_node( + doc_id: str = "d_abc123", + topic_index: int = 1, + topic: str = "Opening Ceremony", + summary: str = "Details on the opening ceremony.", + content: str = "The opening ceremony will feature...", + depth: int = 1, + parent_index: int | None = 0, + children_index: list[int] | None = None, + topic_path: str = "Olympics Plan > Opening Ceremony", + content_labels: list[str] | None = None, + category_id: str = "Sports", +) -> KnowledgeMemory: + return KnowledgeMemory( + doc_id=doc_id, + topic_index=topic_index, + topic=topic, + summary=summary, + content=content, + depth=depth, + parent_index=parent_index, + children_index=children_index or [], + topic_path=topic_path, + content_labels=content_labels or [], + category_id=category_id, + ) + + +def _parse_md(path: Path) -> tuple[dict[str, object], str]: + """Parse a markdown file into (frontmatter_dict, body).""" + text = path.read_text(encoding="utf-8") + parts = text.split("---", 2) + assert len(parts) >= 3, f"Expected YAML frontmatter in {path}" + fm = yaml.safe_load(parts[1]) or {} + body = parts[2] + if body.startswith("\n"): + body = body[1:] + return fm, body + + +# ── Tests ───────────────────────────────────────────────────────────────── + + +async def test_basic_write_creates_correct_structure(tmp_path: Path) -> None: + """Root + 2 topics -> index.md + 2 topic files.""" + memories = [ + _root_node(), + _topic_node(topic_index=1, topic="Opening Ceremony"), + _topic_node(topic_index=2, topic="Closing Ceremony"), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + assert doc_dir == tmp_path / "Sports" / "Olympics_Plan_d_abc123" + assert (doc_dir / "index.md").is_file() + assert (doc_dir / "1_Opening_Ceremony.md").is_file() + assert (doc_dir / "2_Closing_Ceremony.md").is_file() + + +async def test_index_frontmatter_has_knowledge_document_type( + tmp_path: Path, +) -> None: + memories = [_root_node()] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + fm, body = _parse_md(doc_dir / "index.md") + assert fm["type"] == "knowledge_document" + assert fm["id"] == "d_abc123" + assert fm["doc_id"] == "d_abc123" + assert fm["category_id"] == "Sports" + assert fm["title"] == "Olympics Plan" + assert fm["schema_version"] == 1 + assert body.rstrip("\n") == "Overview of the Olympic plan." + + +async def test_topic_frontmatter_has_knowledge_topic_type( + tmp_path: Path, +) -> None: + memories = [ + _root_node(), + _topic_node(topic_index=1, topic="Opening Ceremony"), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + fm, body = _parse_md(doc_dir / "1_Opening_Ceremony.md") + assert fm["type"] == "knowledge_topic" + assert fm["topic_name"] == "Opening Ceremony" + assert fm["topic_index"] == 1 + assert body.rstrip("\n") == "The opening ceremony will feature..." + + +async def test_node_id_format(tmp_path: Path) -> None: + """node_id and id follow ``{doc_id}_{topic_index}`` pattern.""" + memories = [ + _root_node(doc_id="d_xyz789"), + _topic_node(doc_id="d_xyz789", topic_index=3, topic="Venues"), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + fm, _ = _parse_md(doc_dir / "3_Venues.md") + assert fm["id"] == "d_xyz789_3" + assert fm["node_id"] == "d_xyz789_3" + assert fm["doc_id"] == "d_xyz789" + + +async def test_depth_1_parent_node_id_is_null(tmp_path: Path) -> None: + """Direct children of root (depth=1) have parent_node_id=null.""" + memories = [ + _root_node(), + _topic_node(topic_index=1, depth=1, parent_index=0), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + fm, _ = _parse_md(doc_dir / "1_Opening_Ceremony.md") + assert fm["parent_node_id"] is None + + +async def test_depth_gt1_parent_node_id_set(tmp_path: Path) -> None: + """Nested topics (depth>1) get ``{doc_id}_{parent_index}``.""" + memories = [ + _root_node(children_index=[1]), + _topic_node( + topic_index=1, + topic="Section A", + depth=1, + parent_index=0, + children_index=[2], + ), + _topic_node( + topic_index=2, + topic="Sub Section", + depth=2, + parent_index=1, + topic_path="Olympics Plan > Section A > Sub Section", + ), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + fm, _ = _parse_md(doc_dir / "2_Sub_Section.md") + assert fm["parent_node_id"] == "d_abc123_1" + + +async def test_children_node_ids_mapping(tmp_path: Path) -> None: + memories = [ + _root_node(children_index=[1, 2]), + _topic_node(topic_index=1, topic="A", children_index=[3]), + _topic_node(topic_index=2, topic="B"), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + fm, _ = _parse_md(doc_dir / "1_A.md") + assert fm["children_node_ids"] == ["d_abc123_3"] + + +async def test_empty_slug_fallback_for_topic(tmp_path: Path) -> None: + """Topic with all-special-chars name falls back to ``{idx}_topic_{idx}.md``.""" + memories = [ + _root_node(), + _topic_node(topic_index=5, topic="!!!@@@###"), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + assert (doc_dir / "5_topic_5.md").is_file() + + +async def test_empty_slug_fallback_for_title(tmp_path: Path) -> None: + """Document title with all-special-chars falls back to ``doc_{doc_id}``.""" + doc_id = "d_abcdef1234567890" + memories = [ + _root_node( + doc_id=doc_id, + topic="$$$%%%^^^", + ), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + assert doc_dir == tmp_path / "Sports" / f"doc_{doc_id}" + assert (doc_dir / "index.md").is_file() + + +async def test_empty_category_id_fallback_to_others(tmp_path: Path) -> None: + memories = [_root_node(category_id="")] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + + assert doc_dir.parent.name == "Others" + fm, _ = _parse_md(doc_dir / "index.md") + assert fm["category_id"] == "Others" + + +async def test_overwrite_replaces_existing_directory(tmp_path: Path) -> None: + """Second write deletes old files and creates new ones.""" + memories_v1 = [ + _root_node(), + _topic_node(topic_index=1, topic="Old Topic"), + ] + doc_dir = await KnowledgeWriter.write(memories_v1, tmp_path) + assert (doc_dir / "1_Old_Topic.md").is_file() + + memories_v2 = [ + _root_node(), + _topic_node(topic_index=1, topic="New Topic"), + ] + doc_dir = await KnowledgeWriter.write(memories_v2, tmp_path) + + assert (doc_dir / "1_New_Topic.md").is_file() + assert not (doc_dir / "1_Old_Topic.md").exists() + + +async def test_empty_memories_raises_value_error(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="must not be empty"): + await KnowledgeWriter.write([], tmp_path) + + +async def test_no_root_node_raises_value_error(tmp_path: Path) -> None: + memories = [_topic_node(topic_index=1)] + with pytest.raises(ValueError, match="root node"): + await KnowledgeWriter.write(memories, tmp_path) + + +async def test_source_name_and_type_in_index_frontmatter( + tmp_path: Path, +) -> None: + memories = [_root_node()] + doc_dir = await KnowledgeWriter.write( + memories, + tmp_path, + source_name="https://example.com/doc", + source_type="url", + ) + fm, _ = _parse_md(doc_dir / "index.md") + assert fm["source_name"] == "https://example.com/doc" + assert fm["source_type"] == "url" + + +async def test_source_fields_omitted_when_none(tmp_path: Path) -> None: + memories = [_root_node()] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + fm, _ = _parse_md(doc_dir / "index.md") + assert "source_name" not in fm + assert "source_type" not in fm + + +async def test_content_labels_preserved(tmp_path: Path) -> None: + memories = [ + _root_node(), + _topic_node( + topic_index=1, + content_labels=["sports", "ceremony"], + ), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + fm, _ = _parse_md(doc_dir / "1_Opening_Ceremony.md") + assert fm["content_labels"] == ["sports", "ceremony"] + + +async def test_topic_path_preserved(tmp_path: Path) -> None: + memories = [ + _root_node(), + _topic_node( + topic_index=1, + topic_path="Olympics Plan > Opening Ceremony", + ), + ] + doc_dir = await KnowledgeWriter.write(memories, tmp_path) + fm, _ = _parse_md(doc_dir / "1_Opening_Ceremony.md") + assert fm["topic_path"] == "Olympics Plan > Opening Ceremony" diff --git a/tests/unit/test_infra/test_ome/test_crash_recovery.py b/tests/unit/test_infra/test_ome/test_crash_recovery.py index 642d5b5..bbec18d 100644 --- a/tests/unit/test_infra/test_ome/test_crash_recovery.py +++ b/tests/unit/test_infra/test_ome/test_crash_recovery.py @@ -28,6 +28,7 @@ async def test_marks_old_running_as_crashed(rec_store: RunRecordStore) -> None: event_topic="x:E", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) async with rec_store._storage.connect() as conn: rewind = to_iso_format(get_now_with_timezone() - timedelta(hours=2)) @@ -68,6 +69,7 @@ async def test_recent_running_skipped(rec_store: RunRecordStore) -> None: event_topic="x:E", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) resumed: list = [] @@ -120,6 +122,7 @@ async def test_add_job_failure_does_not_abort_loop( event_topic="x:E", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) async with rec_store._storage.connect() as conn: rewind = to_iso_format(get_now_with_timezone() - timedelta(hours=2)) diff --git a/tests/unit/test_infra/test_ome/test_decorator.py b/tests/unit/test_infra/test_ome/test_decorator.py index e9b1afa..934501d 100644 --- a/tests/unit/test_infra/test_ome/test_decorator.py +++ b/tests/unit/test_infra/test_ome/test_decorator.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest from everos.infra.ome.context import StrategyContext -from everos.infra.ome.decorator import StrategyMeta, offline_strategy +from everos.infra.ome.decorator import Strategy, StrategyMeta, offline_strategy from everos.infra.ome.events import BaseEvent from everos.infra.ome.gates import Counter from everos.infra.ome.triggers import Immediate @@ -18,14 +18,15 @@ def test_decorator_attaches_metadata() -> None: async def s(event: _E, ctx: StrategyContext) -> None: return None - meta: StrategyMeta = s._ome_strategy_meta # type: ignore[attr-defined] + meta: StrategyMeta = s.meta assert meta.name == "x" assert meta.emits == frozenset({_E}) assert meta.gate is None assert meta.applies_to is None assert meta.max_retries is None assert meta.enabled is True - assert meta.func is s + assert isinstance(s, Strategy) + assert callable(meta.func) def test_decorator_with_full_params() -> None: @@ -41,7 +42,7 @@ def test_decorator_with_full_params() -> None: async def s(event: _E, ctx: StrategyContext) -> None: return None - meta = s._ome_strategy_meta # type: ignore[attr-defined] + meta = s.meta assert meta.applies_to == "user_id" assert meta.gate.threshold == 5 assert meta.max_retries == 3 @@ -61,7 +62,7 @@ def test_decorator_callable_applies_to() -> None: async def s(event: _E, ctx: StrategyContext) -> None: return None - meta = s._ome_strategy_meta # type: ignore[attr-defined] + meta = s.meta assert meta.applies_to is is_paid diff --git a/tests/unit/test_infra/test_ome/test_engine_event_id.py b/tests/unit/test_infra/test_ome/test_engine_event_id.py new file mode 100644 index 0000000..916e84d --- /dev/null +++ b/tests/unit/test_infra/test_ome/test_engine_event_id.py @@ -0,0 +1,82 @@ +"""Tests for OfflineEngine event_id tracking (P3).""" + +from __future__ import annotations + +import pytest + +from everos.infra.ome import BaseEvent, Immediate, RunStatus, offline_strategy +from everos.infra.ome.testing import StrategyTestHarness + + +class _Ping(BaseEvent): + """Test event.""" + + +@offline_strategy( + name="echo", + trigger=Immediate(on=[_Ping]), + emits=[], +) +async def _echo_strategy(event: BaseEvent, ctx: object) -> None: + pass + + +@pytest.mark.asyncio +async def test_list_runs_by_event_id() -> None: + async with StrategyTestHarness() as h: + h.register(_echo_strategy) + await h.start() + ping = _Ping() + await h.emit(ping) + await h.drain(timeout=5) + runs = await h._engine.list_runs_by_event_id(ping.event_id) + assert len(runs) == 1 + assert runs[0].event_id == ping.event_id + assert runs[0].status == RunStatus.SUCCESS + + +@pytest.mark.asyncio +async def test_wait_for_event_returns_on_success() -> None: + async with StrategyTestHarness() as h: + h.register(_echo_strategy) + await h.start() + ping = _Ping() + await h.emit(ping) + runs = await h._engine.wait_for_event(ping.event_id, timeout=5) + assert len(runs) == 1 + assert runs[0].status == RunStatus.SUCCESS + + +@pytest.mark.asyncio +async def test_wait_for_event_times_out_on_no_runs() -> None: + async with StrategyTestHarness() as h: + h.register(_echo_strategy) + await h.start() + with pytest.raises(TimeoutError): + await h._engine.wait_for_event("nonexistent_event", timeout=0.3) + + +class _Boom(BaseEvent): + """Event that triggers a failing strategy.""" + + +@offline_strategy( + name="fail_strategy", + trigger=Immediate(on=[_Boom]), + emits=[], + max_retries=0, +) +async def _fail_strategy(event: BaseEvent, ctx: object) -> None: + raise RuntimeError("intentional failure") + + +@pytest.mark.asyncio +async def test_wait_for_event_returns_on_terminal_failure() -> None: + async with StrategyTestHarness() as h: + h.register(_fail_strategy) + await h.start() + boom = _Boom() + await h.emit(boom) + runs = await h._engine.wait_for_event(boom.event_id, timeout=5) + assert len(runs) == 1 + assert runs[0].status == RunStatus.DEAD_LETTER diff --git a/tests/unit/test_infra/test_ome/test_records.py b/tests/unit/test_infra/test_ome/test_records.py index 198813d..2acd0e7 100644 --- a/tests/unit/test_infra/test_ome/test_records.py +++ b/tests/unit/test_infra/test_ome/test_records.py @@ -6,7 +6,7 @@ from typing import Any import pytest from pydantic import ValidationError -from everos.component.utils.datetime import get_now_with_timezone +from everos.component.utils.datetime import get_now_with_timezone, get_utc_now from everos.infra.ome.records import RunRecord, RunStatus, StrategyRouteInfo @@ -21,6 +21,7 @@ def _ok_kwargs(**overrides: Any) -> dict[str, Any]: "event_topic": "x:Y", "event_payload": "{}", "max_retries_snapshot": 1, + "event_id": "evt_test", } base.update(overrides) return base @@ -44,6 +45,7 @@ def test_run_record_minimal() -> None: event_topic="my_app.events:EpisodeSaved", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) assert rec.finished_at is None assert rec.error is None @@ -60,6 +62,7 @@ def test_run_record_round_trips_json() -> None: event_topic="x:Y", event_payload='{"a":1}', max_retries_snapshot=1, + event_id="evt_test", ) blob = rec.model_dump_json() restored = RunRecord.model_validate_json(blob) @@ -175,3 +178,24 @@ def test_strategy_route_info_rejects_empty_strategy_name() -> None: applies_to_pass=True, counter_pass=True, ) + + +def test_run_record_accepts_event_id() -> None: + rec = RunRecord( + run_id="r1", + strategy_name="s", + status=RunStatus.RUNNING, + attempt=0, + started_at=get_utc_now(), + event_topic="x:Y", + event_payload="{}", + max_retries_snapshot=1, + event_id="abc123", + ) + assert rec.event_id == "abc123" + + +def test_run_record_accepts_empty_event_id_for_migration_compat() -> None: + """Empty event_id is valid for pre-existing rows migrated from older schema.""" + rec = RunRecord(**_ok_kwargs(event_id="")) + assert rec.event_id == "" diff --git a/tests/unit/test_infra/test_ome/test_run_record_store.py b/tests/unit/test_infra/test_ome/test_run_record_store.py index 1330489..e2785bf 100644 --- a/tests/unit/test_infra/test_ome/test_run_record_store.py +++ b/tests/unit/test_infra/test_ome/test_run_record_store.py @@ -28,6 +28,7 @@ async def test_mark_running_inserts_row(store: RunRecordStore) -> None: event_topic="x:Y", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) rec = await store.get("r1") assert rec is not None @@ -43,6 +44,7 @@ async def test_mark_success_updates_row(store: RunRecordStore) -> None: event_topic="x:Y", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) await store.mark_success(run_id="r1", finished_at=get_now_with_timezone()) rec = await store.get("r1") @@ -59,6 +61,7 @@ async def test_mark_failed_records_error(store: RunRecordStore) -> None: event_topic="x:Y", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) await store.mark_failed( run_id="r1", finished_at=get_now_with_timezone(), error="boom" @@ -77,6 +80,7 @@ async def test_mark_dead_letter(store: RunRecordStore) -> None: event_topic="x:Y", event_payload="{}", max_retries_snapshot=2, + event_id="evt_test", ) await store.mark_dead_letter( run_id="r1", finished_at=get_now_with_timezone(), error="exhausted" @@ -98,6 +102,7 @@ async def test_ring_buffer_caps_strategy_records(store: RunRecordStore) -> None: event_topic="x:Y", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) listed = await store.list_runs(strategy_name="s") assert len(listed) <= 3 # never transiently above cap @@ -115,6 +120,7 @@ async def test_list_runs_filters_by_status(store: RunRecordStore) -> None: event_topic="x:Y", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) await store.mark_success(run_id="r1", finished_at=get_now_with_timezone()) await store.mark_running( @@ -124,6 +130,7 @@ async def test_list_runs_filters_by_status(store: RunRecordStore) -> None: event_topic="x:Y", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) success_runs = await store.list_runs(strategy_name="s", status=RunStatus.SUCCESS) assert [r.run_id for r in success_runs] == ["r1"] @@ -138,7 +145,65 @@ async def test_find_running_for_crash_recovery(store: RunRecordStore) -> None: event_topic="x:Y", event_payload="{}", max_retries_snapshot=1, + event_id="evt_test", ) running = await store.find_running() assert len(running) == 1 assert running[0].run_id == "r1" + + +@pytest.mark.asyncio +async def test_mark_running_persists_event_id(store: RunRecordStore) -> None: + await store.mark_running( + run_id="r1", + strategy_name="s", + attempt=0, + event_topic="x:Y", + event_payload="{}", + max_retries_snapshot=1, + event_id="evt_abc", + ) + rec = await store.get("r1") + assert rec is not None + assert rec.event_id == "evt_abc" + + +@pytest.mark.asyncio +async def test_list_by_event_id_returns_matching_runs(store: RunRecordStore) -> None: + await store.mark_running( + run_id="r1", + strategy_name="s1", + attempt=0, + event_topic="x:Y", + event_payload="{}", + max_retries_snapshot=1, + event_id="evt_1", + ) + await store.mark_running( + run_id="r2", + strategy_name="s2", + attempt=0, + event_topic="x:Y", + event_payload="{}", + max_retries_snapshot=1, + event_id="evt_1", + ) + await store.mark_running( + run_id="r3", + strategy_name="s3", + attempt=0, + event_topic="x:Y", + event_payload="{}", + max_retries_snapshot=1, + event_id="evt_other", + ) + results = await store.list_by_event_id("evt_1") + assert {r.run_id for r in results} == {"r1", "r2"} + + +@pytest.mark.asyncio +async def test_list_by_event_id_returns_empty_for_unknown( + store: RunRecordStore, +) -> None: + results = await store.list_by_event_id("nonexistent") + assert results == [] diff --git a/tests/unit/test_infra/test_ome/test_runner.py b/tests/unit/test_infra/test_ome/test_runner.py index cebad02..696de53 100644 --- a/tests/unit/test_infra/test_ome/test_runner.py +++ b/tests/unit/test_infra/test_ome/test_runner.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio from pathlib import Path +from unittest.mock import MagicMock import pytest @@ -40,9 +41,10 @@ async def test_runner_success_marks_record(setup) -> None: run_record_store=rec_store, engine_sem=sem, emit_hook=_no_emit, + engine=MagicMock(), ) await runner.run( - s._ome_strategy_meta, + s.meta, _E(), run_id="r1", max_retries_snapshot=1, @@ -72,9 +74,10 @@ async def test_runner_retries_on_failure(setup) -> None: run_record_store=rec_store, engine_sem=sem, emit_hook=_no_emit, + engine=MagicMock(), ) await runner.run( - s._ome_strategy_meta, + s.meta, _E(), run_id="r1", max_retries_snapshot=2, @@ -110,9 +113,10 @@ async def test_runner_dead_letter_after_exhaust(setup) -> None: engine_sem=sem, emit_hook=_no_emit, on_dead_letter=lambda r: dl_calls.append(r), + engine=MagicMock(), ) await runner.run( - s._ome_strategy_meta, + s.meta, _E(), run_id="r1", max_retries_snapshot=1, @@ -144,9 +148,10 @@ async def test_runner_emit_must_be_declared(setup) -> None: run_record_store=rec_store, engine_sem=sem, emit_hook=_no_emit, + engine=MagicMock(), ) await runner.run( - s._ome_strategy_meta, + s.meta, _E(), run_id="r1", max_retries_snapshot=0, @@ -172,10 +177,11 @@ async def test_runner_negative_max_retries_raises(setup) -> None: run_record_store=rec_store, engine_sem=sem, emit_hook=_no_emit, + engine=MagicMock(), ) with pytest.raises(ValueError, match=r"max_retries_snapshot must be >= 0"): await runner.run( - s._ome_strategy_meta, + s.meta, _E(), run_id="r1", max_retries_snapshot=-1, @@ -208,10 +214,11 @@ async def test_runner_aborts_silently_when_mark_running_fails( run_record_store=rec_store, engine_sem=sem, emit_hook=_no_emit, + engine=MagicMock(), ) # Must NOT raise; the framework swallows + logs. await runner.run( - s._ome_strategy_meta, + s.meta, _E(), run_id="r1", max_retries_snapshot=1, diff --git a/tests/unit/test_infra/test_ome/test_storage_migration.py b/tests/unit/test_infra/test_ome/test_storage_migration.py new file mode 100644 index 0000000..e7ea734 --- /dev/null +++ b/tests/unit/test_infra/test_ome/test_storage_migration.py @@ -0,0 +1,61 @@ +"""Tests for OME storage schema migration — event_id column.""" + +from __future__ import annotations + +from pathlib import Path + +import aiosqlite +import pytest + +from everos.infra.ome._stores.storage import OMEStorage + + +@pytest.mark.asyncio +async def test_fresh_db_has_event_id_column(tmp_path: Path) -> None: + """A brand-new database should have the event_id column.""" + storage = OMEStorage(db_path=tmp_path / "ome.db") + await storage.init() + async with aiosqlite.connect(tmp_path / "ome.db") as conn: + cur = await conn.execute("PRAGMA table_info(run_record)") + columns = {row[1] for row in await cur.fetchall()} + assert "event_id" in columns + + +@pytest.mark.asyncio +async def test_existing_db_without_event_id_gets_migrated(tmp_path: Path) -> None: + """An existing database created before P3 should gain the event_id + column after init() runs the migration. + """ + db_path = tmp_path / "ome.db" + async with aiosqlite.connect(db_path) as conn: + await conn.execute( + "CREATE TABLE run_record (" + " run_id TEXT PRIMARY KEY," + " strategy_name TEXT NOT NULL," + " status TEXT NOT NULL," + " attempt INTEGER NOT NULL DEFAULT 0," + " started_at TIMESTAMP NOT NULL," + " finished_at TIMESTAMP," + " error TEXT," + " event_topic TEXT NOT NULL," + " event_payload TEXT NOT NULL," + " max_retries_snapshot INTEGER NOT NULL" + ")" + ) + await conn.commit() + + storage = OMEStorage(db_path=db_path) + await storage.init() + + async with aiosqlite.connect(db_path) as conn: + cur = await conn.execute("PRAGMA table_info(run_record)") + columns = {row[1] for row in await cur.fetchall()} + assert "event_id" in columns + + +@pytest.mark.asyncio +async def test_migration_is_idempotent(tmp_path: Path) -> None: + """Calling init() twice on the same database must not fail.""" + storage = OMEStorage(db_path=tmp_path / "ome.db") + await storage.init() + await storage.init() diff --git a/tests/unit/test_infra/test_sqlite/test_knowledge_tables.py b/tests/unit/test_infra/test_sqlite/test_knowledge_tables.py new file mode 100644 index 0000000..66eab37 --- /dev/null +++ b/tests/unit/test_infra/test_sqlite/test_knowledge_tables.py @@ -0,0 +1,39 @@ +"""SQLite table model validation for knowledge_documents + knowledge_topics.""" + +from __future__ import annotations + +from everos.infra.persistence.sqlite import ( + KnowledgeDocumentRow, + KnowledgeTopicRow, +) + + +class TestKnowledgeDocumentRow: + def test_tablename(self) -> None: + assert KnowledgeDocumentRow.__tablename__ == "knowledge_documents" + + def test_primary_key(self) -> None: + pk_cols = [c.name for c in KnowledgeDocumentRow.__table__.primary_key.columns] + assert pk_cols == ["doc_id"] + + +class TestKnowledgeTopicRow: + def test_tablename(self) -> None: + assert KnowledgeTopicRow.__tablename__ == "knowledge_topics" + + def test_primary_key(self) -> None: + pk_cols = [c.name for c in KnowledgeTopicRow.__table__.primary_key.columns] + assert pk_cols == ["node_id"] + + def test_has_content_column(self) -> None: + cols = {c.name for c in KnowledgeTopicRow.__table__.columns} + assert "content" in cols + assert "summary" in cols + + def test_topic_doc_id_has_no_fk(self) -> None: + """doc_id has no FK — cascade handler ordering is not guaranteed.""" + from sqlmodel import SQLModel + + table = SQLModel.metadata.tables["knowledge_topics"] + fks = list(table.foreign_key_constraints) + assert len(fks) == 0 diff --git a/tests/unit/test_infra/test_sqlite/test_repos/test_cluster.py b/tests/unit/test_infra/test_sqlite/test_repos/test_cluster.py index 9c61d6f..15a16ba 100644 --- a/tests/unit/test_infra/test_sqlite/test_repos/test_cluster.py +++ b/tests/unit/test_infra/test_sqlite/test_repos/test_cluster.py @@ -296,3 +296,189 @@ async def test_find_cluster_id_for_member_reverse_lookup( assert await repo.find_cluster_id_for_member("case", "mc_one") is None assert await repo.find_cluster_id_for_member("memcell", "ac_20260517_0001") is None assert await repo.find_cluster_id_for_member("memcell", "mc_missing") is None + + +# ── remove_members ───────────────────────────────────────────────────── + + +async def test_remove_members_deletes_specified(repo: _ClusterRepo) -> None: + """Removing a subset of members leaves the rest intact.""" + cluster = _make_cluster( + cluster_id="cl_rm_000000001", + centroid_vals=[1.0, 0.0], + members=["mc_one", "mc_two", "mc_three"], + count=3, + ) + await repo.upsert_with_members( + cluster, + owner_id="u_alice", + owner_type="user", + kind="user_memory", + member_type="memcell", + ) + + await repo.remove_members("cl_rm_000000001", {"mc_one", "mc_three"}) + + members = await repo.get_members_with_type("cl_rm_000000001") + assert [mid for mid, _ in members] == ["mc_two"] + + +async def test_remove_members_empty_set_is_noop( + repo: _ClusterRepo, +) -> None: + """An empty member_ids set should not touch the database.""" + cluster = _make_cluster( + cluster_id="cl_noop0000001", + centroid_vals=[1.0, 0.0], + members=["mc_one"], + ) + await repo.upsert_with_members( + cluster, + owner_id="u_alice", + owner_type="user", + kind="user_memory", + member_type="memcell", + ) + + await repo.remove_members("cl_noop0000001", set()) + + members = await repo.get_members_with_type("cl_noop0000001") + assert len(members) == 1 + + +# ── add_member ───────────────────────────────────────────────────────── + + +async def test_add_member_with_episode_type(repo: _ClusterRepo) -> None: + """Add a single member and verify it appears in the membership list.""" + cluster = _make_cluster( + cluster_id="cl_add_00000001", + centroid_vals=[1.0, 0.0], + members=["mc_one"], + ) + await repo.upsert_with_members( + cluster, + owner_id="u_alice", + owner_type="user", + kind="user_memory", + member_type="memcell", + ) + + await repo.add_member("cl_add_00000001", "ep_new_001", "episode") + + members = await repo.get_members_with_type("cl_add_00000001") + member_ids = [mid for mid, _ in members] + assert "ep_new_001" in member_ids + # Verify type stored correctly + ep_row = [(mid, mt) for mid, mt in members if mid == "ep_new_001"] + assert ep_row[0][1] == "episode" + + +# ── update_metadata ──────────────────────────────────────────────────── + + +async def test_update_metadata_changes_cluster_row( + repo: _ClusterRepo, +) -> None: + """update_metadata overwrites centroid, count, last_ts_ms, preview.""" + cluster = _make_cluster( + cluster_id="cl_meta0000001", + centroid_vals=[1.0, 0.0], + members=["mc_one"], + count=1, + last_ts_ms=1_700_000_000_000, + preview=["old preview"], + ) + await repo.upsert_with_members( + cluster, + owner_id="u_alice", + owner_type="user", + kind="user_memory", + member_type="memcell", + ) + + new_centroid = np.array([0.5, 0.5], dtype=np.float32).tobytes() + await repo.update_metadata( + "cl_meta0000001", + centroid_blob=new_centroid, + count=3, + last_ts_ms=1_700_000_099_000, + preview_json='["new preview"]', + ) + + rows = await repo.list_for_owner("u_alice", "user_memory") + assert len(rows) == 1 + got = rows[0] + assert got.count == 3 + assert got.last_ts == 1_700_000_099_000 + assert got.preview == ["new preview"] + np.testing.assert_allclose( + np.asarray(got.centroid), + np.array([0.5, 0.5], dtype=np.float32), + ) + + +# ── list_ids_and_member_counts ───────────────────────────────────────── + + +async def test_list_ids_and_member_counts_returns_actual_member_count( + repo: _ClusterRepo, +) -> None: + """Count comes from cluster_member rows, not the Cluster.count field.""" + c1 = _make_cluster( + cluster_id="cl_cnt_00000001", + centroid_vals=[1.0, 0.0], + members=["mc_one", "mc_two"], + count=99, # deliberately wrong — repo counts actual rows + ) + c2 = _make_cluster( + cluster_id="cl_cnt_00000002", + centroid_vals=[0.0, 1.0], + members=["mc_three"], + count=99, + ) + await repo.upsert_with_members( + c1, + owner_id="u_alice", + owner_type="user", + kind="user_memory", + member_type="memcell", + ) + await repo.upsert_with_members( + c2, + owner_id="u_alice", + owner_type="user", + kind="user_memory", + member_type="memcell", + ) + + result = await repo.list_ids_and_member_counts("u_alice", "user_memory") + result_dict = dict(result) + assert result_dict["cl_cnt_00000001"] == 2 + assert result_dict["cl_cnt_00000002"] == 1 + + +# ── get_members_with_type ────────────────────────────────────────────── + + +async def test_get_members_with_type_returns_tuples( + repo: _ClusterRepo, +) -> None: + """Returns (member_id, member_type) tuples in insertion order.""" + cluster = _make_cluster( + cluster_id="cl_mtype000001", + centroid_vals=[1.0, 0.0], + members=["mc_one", "mc_two"], + ) + await repo.upsert_with_members( + cluster, + owner_id="u_alice", + owner_type="user", + kind="user_memory", + member_type="memcell", + ) + + members = await repo.get_members_with_type("cl_mtype000001") + assert len(members) == 2 + assert members[0] == ("mc_one", "memcell") + assert members[1] == ("mc_two", "memcell") diff --git a/tests/unit/test_infra/test_sqlite/test_repos/test_reflection_report.py b/tests/unit/test_infra/test_sqlite/test_repos/test_reflection_report.py new file mode 100644 index 0000000..493b6c4 --- /dev/null +++ b/tests/unit/test_infra/test_sqlite/test_repos/test_reflection_report.py @@ -0,0 +1,119 @@ +"""Tests for :class:`_ReflectionReportRepo` — reflection audit persistence. + +Verifies create, latest-for-cluster lookup, and reflected-cluster-id +listing including status filtering (rolled_back rows excluded). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from sqlmodel import SQLModel + +from everos.config import SqliteSettings +from everos.core.persistence import ( + MemoryRoot, + create_session_factory, + create_system_engine, +) +from everos.infra.persistence.sqlite.repos.reflection_report import ( + _ReflectionReportRepo, +) +from everos.infra.persistence.sqlite.tables import ReflectionReport + + +@pytest.fixture +async def repo(tmp_path: Path) -> _ReflectionReportRepo: + mr = MemoryRoot(tmp_path) + mr.ensure() + engine = create_system_engine(mr.system_db, SqliteSettings()) + factory = create_session_factory(engine) + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + return _ReflectionReportRepo(session_factory=factory) + + +def _make_report( + *, + report_id: str = "rr_001", + cluster_id: str = "cl_aaa000000001", + owner_id: str = "u_alice", + mode: str = "consolidation", + source_members: str = "mc_one,mc_two", + source_count: int = 2, + merged_entry_id: str = "ep_merged_001", + deprecated_fact_count: int = 1, + status: str = "completed", +) -> ReflectionReport: + return ReflectionReport( + id=report_id, + cluster_id=cluster_id, + owner_id=owner_id, + mode=mode, + source_members=source_members, + source_count=source_count, + merged_entry_id=merged_entry_id, + deprecated_fact_count=deprecated_fact_count, + status=status, + ) + + +async def test_create_and_get_latest(repo: _ReflectionReportRepo) -> None: + """Create a report then retrieve it as the latest for that cluster.""" + report = _make_report() + await repo.create(report) + + latest = await repo.get_latest_for_cluster("cl_aaa000000001") + assert latest is not None + assert latest.id == "rr_001" + assert latest.cluster_id == "cl_aaa000000001" + assert latest.owner_id == "u_alice" + assert latest.mode == "consolidation" + assert latest.source_count == 2 + assert latest.merged_entry_id == "ep_merged_001" + assert latest.deprecated_fact_count == 1 + assert latest.status == "completed" + + +async def test_get_latest_returns_none_when_empty( + repo: _ReflectionReportRepo, +) -> None: + """No reports exist for a cluster -> None.""" + result = await repo.get_latest_for_cluster("cl_nonexistent") + assert result is None + + +async def test_list_reflected_cluster_ids( + repo: _ReflectionReportRepo, +) -> None: + """Reports for two distinct clusters -> both cluster ids returned.""" + await repo.create(_make_report(report_id="rr_001", cluster_id="cl_aaa000000001")) + await repo.create(_make_report(report_id="rr_002", cluster_id="cl_bbb000000002")) + + ids = await repo.list_reflected_cluster_ids("u_alice") + assert ids == {"cl_aaa000000001", "cl_bbb000000002"} + + +async def test_list_reflected_excludes_rolled_back( + repo: _ReflectionReportRepo, +) -> None: + """A rolled_back report should not appear in the reflected set.""" + await repo.create( + _make_report( + report_id="rr_001", + cluster_id="cl_aaa000000001", + status="completed", + ) + ) + await repo.create( + _make_report( + report_id="rr_002", + cluster_id="cl_bbb000000002", + status="rolled_back", + ) + ) + + ids = await repo.list_reflected_cluster_ids("u_alice") + assert ids == {"cl_aaa000000001"} + assert "cl_bbb000000002" not in ids diff --git a/tests/unit/test_infra/test_sqlite/test_sqlite_manager.py b/tests/unit/test_infra/test_sqlite/test_sqlite_manager.py index 2794933..001236f 100644 --- a/tests/unit/test_infra/test_sqlite/test_sqlite_manager.py +++ b/tests/unit/test_infra/test_sqlite/test_sqlite_manager.py @@ -16,7 +16,7 @@ from everos.infra.persistence.sqlite import sqlite_manager @pytest.fixture(autouse=True) async def _reset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """Point the singleton at an isolated memory-root and reset module state.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) sqlite_manager._engine = None sqlite_manager._session_factory = None yield diff --git a/tests/unit/test_memory/test_cascade/test_handler_knowledge_document.py b/tests/unit/test_memory/test_cascade/test_handler_knowledge_document.py new file mode 100644 index 0000000..8fafbc9 --- /dev/null +++ b/tests/unit/test_memory/test_cascade/test_handler_knowledge_document.py @@ -0,0 +1,213 @@ +"""Tests for :class:`KnowledgeDocumentHandler` — SQLite-only cascade. + +KnowledgeDocumentHandler writes to **SQLite only** — no LanceDB, no +embedding, no tokenization. The handler reads ``index.md``, extracts +frontmatter + body (summary), and upserts to ``knowledge_documents``. + +Coverage: + +- ``handle_added_or_modified`` with valid frontmatter → upserts, + returns ``upserted=1`` +- ``handle_added_or_modified`` with wrong ``type`` → returns + ``skipped=1``, no upsert +- ``handle_deleted`` on an indexed path → calls + ``delete_by_md_path``, returns ``deleted=1`` +- ``handle_deleted`` on an unknown path → returns ``deleted=0`` +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest + +from everos.component.embedding import EmbeddingProvider +from everos.component.tokenizer import Tokenizer +from everos.core.persistence import MemoryRoot +from everos.infra.persistence.sqlite import DocumentUpsertPayload +from everos.memory.cascade.handlers import ( + HandlerDeps, + KnowledgeDocumentHandler, +) + +# ── Stubs ────────────────────────────────────────────────────────────── + + +class _StubTokenizer(Tokenizer): + def tokenize(self, text: str) -> list[str]: + return text.split() + + def tokenize_batch(self, texts): # type: ignore[no-untyped-def] + return [self.tokenize(t) for t in texts] + + +class _StubEmbedder(EmbeddingProvider): + dim = 1024 + + async def embed(self, text: str) -> list[float]: + return [0.0] * self.dim + + async def embed_batch(self, texts): # type: ignore[no-untyped-def] + return [await self.embed(t) for t in texts] + + +# ── Fake repo ────────────────────────────────────────────────────────── + + +class _FakeSqliteRepo: + """In-memory stand-in for ``knowledge_document_repo``.""" + + def __init__(self) -> None: + self.rows: dict[str, dict] = {} + self.upserts: list[dict] = [] + self.deletes: list[str] = [] + + async def upsert_from_handler(self, payload: DocumentUpsertPayload) -> None: + data = dataclasses.asdict(payload) + self.upserts.append(data) + self.rows[payload.doc_id] = data + + async def delete_by_md_path(self, md_path: str) -> int: + self.deletes.append(md_path) + before = len(self.rows) + self.rows = {k: v for k, v in self.rows.items() if v.get("md_path") != md_path} + return before - len(self.rows) + + +# ── Fixtures ─────────────────────────────────────────────────────────── + + +@pytest.fixture +def memory_root(tmp_path: Path) -> MemoryRoot: + mr = MemoryRoot(tmp_path) + mr.ensure() + return mr + + +@pytest.fixture +def fake_sqlite(monkeypatch: pytest.MonkeyPatch) -> _FakeSqliteRepo: + from everos.memory.cascade.handlers import knowledge_document as mod + + repo = _FakeSqliteRepo() + monkeypatch.setattr(mod, "knowledge_document_repo", repo) + return repo + + +# ── Helpers ──────────────────────────────────────────────────────────── + +_SAMPLE_FRONTMATTER = { + "type": "knowledge_document", + "id": "doc_budget", + "category_id": "finance", + "title": "Budget Planning Guide", + "source_name": "Internal Wiki", + "source_type": "wiki", + "schema_version": 1, +} + +_SAMPLE_BODY = "An overview of budget planning practices for Q4." + + +def _write_document_md( + memory_root: MemoryRoot, + *, + frontmatter: dict | None = None, + body: str = _SAMPLE_BODY, +) -> str: + """Write a knowledge document ``index.md`` on disk; return relative path.""" + fm = frontmatter or dict(_SAMPLE_FRONTMATTER) + lines = ["---"] + for key, value in fm.items(): + if value is None: + lines.append(f"{key}: null") + else: + lines.append(f"{key}: {value}") + lines.append("---") + lines.append(body) + content = "\n".join(lines) + + rel_dir = "default_app/default_project/knowledge/finance/Budget_Planning" + abs_dir = memory_root.root / rel_dir + abs_dir.mkdir(parents=True, exist_ok=True) + (abs_dir / "index.md").write_text(content, encoding="utf-8") + return f"{rel_dir}/index.md" + + +def _handler(memory_root: MemoryRoot) -> KnowledgeDocumentHandler: + return KnowledgeDocumentHandler( + HandlerDeps( + memory_root=memory_root, + embedder=_StubEmbedder(), + tokenizer=_StubTokenizer(), + ) + ) + + +# ── Tests ────────────────────────────────────────────────────────────── + + +async def test_handle_added_or_modified_upserts_to_sqlite( + memory_root: MemoryRoot, + fake_sqlite: _FakeSqliteRepo, +) -> None: + md_path = _write_document_md(memory_root) + outcome = await _handler(memory_root).handle_added_or_modified(md_path) + + assert outcome.upserted == 1 + assert outcome.deleted == 0 + assert outcome.skipped == 0 + + assert len(fake_sqlite.upserts) == 1 + row = fake_sqlite.upserts[0] + assert row["doc_id"] == "doc_budget" + assert row["category_id"] == "finance" + assert row["title"] == "Budget Planning Guide" + assert row["source_name"] == "Internal Wiki" + assert row["source_type"] == "wiki" + assert row["summary"] == _SAMPLE_BODY + assert row["app_id"] == "default" + assert row["project_id"] == "default" + assert row["md_path"] == md_path + + +async def test_handle_added_or_modified_wrong_type_skips( + memory_root: MemoryRoot, + fake_sqlite: _FakeSqliteRepo, +) -> None: + """A file whose ``type`` is not ``knowledge_document`` is skipped.""" + fm = dict(_SAMPLE_FRONTMATTER, type="knowledge_topic") + md_path = _write_document_md(memory_root, frontmatter=fm) + outcome = await _handler(memory_root).handle_added_or_modified(md_path) + + assert outcome.skipped == 1 + assert outcome.upserted == 0 + assert len(fake_sqlite.upserts) == 0 + + +async def test_handle_deleted_removes_row( + memory_root: MemoryRoot, + fake_sqlite: _FakeSqliteRepo, +) -> None: + """``handle_deleted`` calls ``delete_by_md_path`` and returns ``deleted=1``.""" + md_path = _write_document_md(memory_root) + handler = _handler(memory_root) + await handler.handle_added_or_modified(md_path) + + outcome = await handler.handle_deleted(md_path) + + assert outcome.deleted == 1 + assert outcome.upserted == 0 + assert md_path in fake_sqlite.deletes + + +async def test_handle_deleted_unknown_path_returns_zero( + memory_root: MemoryRoot, + fake_sqlite: _FakeSqliteRepo, +) -> None: + """Deleting a path that was never indexed returns ``deleted=0``.""" + handler = _handler(memory_root) + outcome = await handler.handle_deleted("nonexistent/index.md") + + assert outcome.deleted == 0 + assert outcome.upserted == 0 diff --git a/tests/unit/test_memory/test_cascade/test_handler_knowledge_topic.py b/tests/unit/test_memory/test_cascade/test_handler_knowledge_topic.py new file mode 100644 index 0000000..cef0f65 --- /dev/null +++ b/tests/unit/test_memory/test_cascade/test_handler_knowledge_topic.py @@ -0,0 +1,318 @@ +"""Tests for :class:`KnowledgeTopicHandler` — cross-storage cascade. + +KnowledgeTopic is the first handler to write to **both** LanceDB and +SQLite. The handler reads ``_.md``, extracts frontmatter, +computes a content digest, embeds the summary, and upserts to both +stores. These tests build the md file on disk and verify: + +- upsert to both stores on first pass +- skip when digest unchanged +- skip when ``type`` frontmatter is wrong +- delete from both stores on ``handle_deleted`` +""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + +import pytest + +from everos.component.embedding import EmbeddingProvider +from everos.component.tokenizer import Tokenizer +from everos.core.persistence import MemoryRoot +from everos.infra.persistence.lancedb import KnowledgeTopic +from everos.infra.persistence.sqlite import TopicUpsertPayload +from everos.memory.cascade.handlers import HandlerDeps, KnowledgeTopicHandler + +# ── Stubs ────────────────────────────────────────────────────────────── + + +class _StubTokenizer(Tokenizer): + def tokenize(self, text: str) -> list[str]: + return [tok for tok in text.split() if tok] + + def tokenize_batch(self, texts): # type: ignore[no-untyped-def] + return [self.tokenize(t) for t in texts] + + +class _StubEmbedder(EmbeddingProvider): + dim = 1024 + + async def embed(self, text: str) -> list[float]: + return [0.0] * self.dim + + async def embed_batch(self, texts): # type: ignore[no-untyped-def] + return [await self.embed(t) for t in texts] + + +# ── Fake repos ───────────────────────────────────────────────────────── + + +class _FakeLanceRepo: + """In-memory stand-in for the LanceDB knowledge_topic_repo.""" + + def __init__(self) -> None: + self.rows: dict[str, KnowledgeTopic] = {} + self.upserts: list[list[KnowledgeTopic]] = [] + self.deletes: list[str] = [] + + async def get_by_id(self, row_id: str) -> KnowledgeTopic | None: + return self.rows.get(row_id) + + async def upsert(self, rows: list[KnowledgeTopic]) -> None: + self.upserts.append(list(rows)) + for row in rows: + self.rows[row.id] = row + + async def delete_by_md_path(self, md_path: str) -> int: + self.deletes.append(md_path) + before = len(self.rows) + self.rows = {k: v for k, v in self.rows.items() if v.md_path != md_path} + return before - len(self.rows) + + +class _FakeSqliteRepo: + """In-memory stand-in for knowledge_topic_sqlite_repo.""" + + def __init__(self) -> None: + self.rows: dict[str, dict] = {} + self.upserts: list[dict] = [] + self.deletes: list[str] = [] + + async def upsert_from_handler(self, payload: TopicUpsertPayload) -> None: + data = dataclasses.asdict(payload) + self.upserts.append(data) + self.rows[payload.node_id] = data + + async def delete_by_md_path(self, md_path: str) -> int: + self.deletes.append(md_path) + before = len(self.rows) + self.rows = {k: v for k, v in self.rows.items() if v.get("md_path") != md_path} + return before - len(self.rows) + + +# ── Fixtures ─────────────────────────────────────────────────────────── + + +@pytest.fixture +def memory_root(tmp_path: Path) -> MemoryRoot: + mr = MemoryRoot(tmp_path) + mr.ensure() + return mr + + +@pytest.fixture +def fake_lance(monkeypatch: pytest.MonkeyPatch) -> _FakeLanceRepo: + from everos.memory.cascade.handlers import knowledge_topic as mod + + repo = _FakeLanceRepo() + monkeypatch.setattr(mod, "knowledge_topic_repo", repo) + return repo + + +@pytest.fixture +def fake_sqlite(monkeypatch: pytest.MonkeyPatch) -> _FakeSqliteRepo: + from everos.memory.cascade.handlers import knowledge_topic as mod + + repo = _FakeSqliteRepo() + monkeypatch.setattr(mod, "knowledge_topic_sqlite_repo", repo) + return repo + + +# ── Helpers ──────────────────────────────────────────────────────────── + +_SAMPLE_FRONTMATTER = { + "type": "knowledge_topic", + "id": "node_001", + "node_id": "node_001", + "doc_id": "doc_budget", + "category_id": "finance", + "topic_index": 1, + "topic_name": "Budget Planning", + "topic_path": "finance/Budget_Planning", + "summary": "Overview of budget planning practices.", + "depth": 0, + "parent_node_id": None, + "children_node_ids": ["node_002", "node_003"], + "content_labels": ["budget", "planning"], + "schema_version": 1, +} + + +def _write_topic_md( + memory_root: MemoryRoot, + *, + frontmatter: dict | None = None, + body: str = "Budget planning involves setting goals and tracking expenses.\n", +) -> str: + """Write a knowledge topic md file on disk, return relative md_path.""" + fm = frontmatter or dict(_SAMPLE_FRONTMATTER) + # Build the YAML frontmatter string. + lines = ["---"] + for key, value in fm.items(): + if isinstance(value, list): + lines.append(f"{key}:") + for item in value: + rendered = f" - {item!r}" if isinstance(item, str) else f" - {item}" + lines.append(rendered) + elif value is None: + lines.append(f"{key}: null") + else: + lines.append(f"{key}: {value}") + lines.append("---") + lines.append(body) + content = "\n".join(lines) + + rel_dir = "default_app/default_project/knowledge/finance/Budget_Planning" + abs_dir = memory_root.root / rel_dir + abs_dir.mkdir(parents=True, exist_ok=True) + filename = "1_Budget_Planning.md" + (abs_dir / filename).write_text(content, encoding="utf-8") + return f"{rel_dir}/{filename}" + + +def _handler(memory_root: MemoryRoot) -> KnowledgeTopicHandler: + return KnowledgeTopicHandler( + HandlerDeps( + memory_root=memory_root, + embedder=_StubEmbedder(), + tokenizer=_StubTokenizer(), + ) + ) + + +# ── Tests ────────────────────────────────────────────────────────────── + + +async def test_handle_added_or_modified_upserts_to_both_stores( + memory_root: MemoryRoot, + fake_lance: _FakeLanceRepo, + fake_sqlite: _FakeSqliteRepo, +) -> None: + md_path = _write_topic_md(memory_root) + outcome = await _handler(memory_root).handle_added_or_modified(md_path) + + assert outcome.upserted == 1 + assert outcome.deleted == 0 + assert outcome.skipped == 0 + + # LanceDB row assertions. + assert len(fake_lance.upserts) == 1 + row = fake_lance.upserts[0][0] + assert row.id == "node_001" + assert row.doc_id == "doc_budget" + assert row.category_id == "finance" + assert row.topic_name == "Budget Planning" + assert row.topic_path == "finance/Budget_Planning" + assert row.depth == 0 + assert row.parent_node_id == "" + assert row.summary == "Overview of budget planning practices." + assert "budget" in row.summary_tokens.lower() + assert "planning" in row.content_tokens.lower() + assert row.content_labels == ["budget", "planning"] + assert row.md_path == md_path + assert len(row.vector) == 1024 + assert row.content_sha256 # non-empty digest + + # SQLite row assertions. + assert len(fake_sqlite.upserts) == 1 + sq = fake_sqlite.upserts[0] + assert sq["node_id"] == "node_001" + assert sq["doc_id"] == "doc_budget" + assert sq["topic_index"] == 1 + assert sq["topic_name"] == "Budget Planning" + assert sq["summary"] == "Overview of budget planning practices." + assert "Budget planning involves" in sq["content"] + assert sq["children_node_ids"] == json.dumps(["node_002", "node_003"]) + assert sq["content_labels"] == json.dumps(["budget", "planning"]) + assert sq["md_path"] == md_path + + +async def test_same_digest_skips( + memory_root: MemoryRoot, + fake_lance: _FakeLanceRepo, + fake_sqlite: _FakeSqliteRepo, +) -> None: + """Second pass with identical content skips both stores.""" + md_path = _write_topic_md(memory_root) + handler = _handler(memory_root) + + first = await handler.handle_added_or_modified(md_path) + assert first.upserted == 1 + + second = await handler.handle_added_or_modified(md_path) + assert second.upserted == 0 + assert second.skipped == 1 + # Only one upsert batch total. + assert len(fake_lance.upserts) == 1 + assert len(fake_sqlite.upserts) == 1 + + +async def test_wrong_type_skips( + memory_root: MemoryRoot, + fake_lance: _FakeLanceRepo, + fake_sqlite: _FakeSqliteRepo, +) -> None: + """A file whose ``type`` is not ``knowledge_topic`` is skipped.""" + fm = dict(_SAMPLE_FRONTMATTER, type="knowledge_document") + md_path = _write_topic_md(memory_root, frontmatter=fm) + outcome = await _handler(memory_root).handle_added_or_modified(md_path) + + assert outcome.skipped == 1 + assert outcome.upserted == 0 + assert len(fake_lance.upserts) == 0 + assert len(fake_sqlite.upserts) == 0 + + +async def test_handle_deleted_removes_from_both_stores( + memory_root: MemoryRoot, + fake_lance: _FakeLanceRepo, + fake_sqlite: _FakeSqliteRepo, +) -> None: + """``handle_deleted`` calls ``delete_by_md_path`` on both repos.""" + # Seed a row so delete_by_md_path has something to find. + md_path = _write_topic_md(memory_root) + handler = _handler(memory_root) + await handler.handle_added_or_modified(md_path) + + outcome = await handler.handle_deleted(md_path) + + assert outcome.deleted == 1 + assert outcome.upserted == 0 + assert md_path in fake_lance.deletes + assert md_path in fake_sqlite.deletes + + +async def test_content_edit_triggers_upsert( + memory_root: MemoryRoot, + fake_lance: _FakeLanceRepo, + fake_sqlite: _FakeSqliteRepo, +) -> None: + """Editing the body changes the digest and triggers a re-upsert.""" + md_path = _write_topic_md(memory_root, body="Original content.\n") + handler = _handler(memory_root) + await handler.handle_added_or_modified(md_path) + + # Edit the body on disk. + abs_path = memory_root.root / md_path + text = abs_path.read_text(encoding="utf-8") + abs_path.write_text(text.replace("Original content.", "Revised content.")) + + outcome = await handler.handle_added_or_modified(md_path) + assert outcome.upserted == 1 + assert len(fake_lance.upserts) == 2 + assert len(fake_sqlite.upserts) == 2 + + +async def test_handle_deleted_on_unknown_path_returns_zero( + memory_root: MemoryRoot, + fake_lance: _FakeLanceRepo, + fake_sqlite: _FakeSqliteRepo, +) -> None: + """Deleting a path that was never indexed returns deleted=0.""" + handler = _handler(memory_root) + outcome = await handler.handle_deleted("nonexistent/path.md") + assert outcome.deleted == 0 + assert outcome.upserted == 0 diff --git a/tests/unit/test_memory/test_cascade/test_orchestrator.py b/tests/unit/test_memory/test_cascade/test_orchestrator.py index c9d01c1..d57eeba 100644 --- a/tests/unit/test_memory/test_cascade/test_orchestrator.py +++ b/tests/unit/test_memory/test_cascade/test_orchestrator.py @@ -34,7 +34,7 @@ async def runtime( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> AsyncIterator[MemoryRoot]: """Boot sqlite + lancedb against a tmp memory_root.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model") monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1") monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key") diff --git a/tests/unit/test_memory/test_cascade/test_registry.py b/tests/unit/test_memory/test_cascade/test_registry.py index f7400da..8db5b26 100644 --- a/tests/unit/test_memory/test_cascade/test_registry.py +++ b/tests/unit/test_memory/test_cascade/test_registry.py @@ -64,7 +64,7 @@ def test_match_kind_rejects_unregistered_paths(path: str) -> None: assert match_kind(path) is None -def test_registry_has_exactly_six_kinds() -> None: +def test_registry_has_exactly_eight_kinds() -> None: """The registry pins the cascade surface — no silent registration.""" names = [s.name for s in KIND_REGISTRY] assert names == [ @@ -74,6 +74,8 @@ def test_registry_has_exactly_six_kinds() -> None: "agent_case", "agent_skill", "user_profile", + "knowledge_document", + "knowledge_topic", ] diff --git a/tests/unit/test_memory/test_cascade/test_registry_knowledge.py b/tests/unit/test_memory/test_cascade/test_registry_knowledge.py new file mode 100644 index 0000000..3970250 --- /dev/null +++ b/tests/unit/test_memory/test_cascade/test_registry_knowledge.py @@ -0,0 +1,39 @@ +"""Knowledge kinds registered in CASCADE registry.""" + +from __future__ import annotations + +from everos.memory.cascade.registry import KIND_REGISTRY, match_kind + + +class TestKnowledgeKindRegistration: + def test_knowledge_document_registered(self) -> None: + names = [k.name for k in KIND_REGISTRY] + assert "knowledge_document" in names + + def test_knowledge_topic_registered(self) -> None: + names = [k.name for k in KIND_REGISTRY] + assert "knowledge_topic" in names + + def test_match_index_md(self) -> None: + spec = match_kind( + "default_app/default_project/knowledge/Sports/Olympics/index.md" + ) + assert spec is not None + assert spec.name == "knowledge_document" + + def test_match_topic_md(self) -> None: + spec = match_kind( + "default_app/default_project/knowledge/Sports/Olympics/1_Budget.md" + ) + assert spec is not None + assert spec.name == "knowledge_topic" + + def test_knowledge_document_has_no_lance_schema(self) -> None: + spec = next(k for k in KIND_REGISTRY if k.name == "knowledge_document") + assert spec.lance_schema is None + assert spec.lance_repo is None + + def test_knowledge_topic_has_lance_schema(self) -> None: + spec = next(k for k in KIND_REGISTRY if k.name == "knowledge_topic") + assert spec.lance_schema is not None + assert spec.lance_repo is not None diff --git a/tests/unit/test_memory/test_cascade/test_scanner_unit.py b/tests/unit/test_memory/test_cascade/test_scanner_unit.py index 78fed0d..764923e 100644 --- a/tests/unit/test_memory/test_cascade/test_scanner_unit.py +++ b/tests/unit/test_memory/test_cascade/test_scanner_unit.py @@ -13,7 +13,6 @@ from pathlib import Path import pytest from everos.core.persistence import MemoryRoot -from everos.memory.cascade import scanner as scanner_module from everos.memory.cascade.scanner import CascadeScanner, _collect_scan_inputs @@ -113,26 +112,16 @@ async def test_run_loop_swallows_scan_exception( scanner = CascadeScanner(mr, scan_interval_seconds=0.05) call_count = {"n": 0} - second_scan = asyncio.Event() - logged_errors: list[str] = [] async def fake_scan() -> list: # type: ignore[type-arg] call_count["n"] += 1 if call_count["n"] == 1: raise RuntimeError("simulated scanner failure") - second_scan.set() return [] - def fake_exception(_event: str, *, error: str) -> None: - logged_errors.append(error) - monkeypatch.setattr(scanner, "scan_once", fake_scan) - monkeypatch.setattr(scanner_module.logger, "exception", fake_exception) await scanner.start() - try: - await asyncio.wait_for(second_scan.wait(), timeout=1.0) - finally: - await scanner.stop() - - assert logged_errors == ["simulated scanner failure"] + # Let the loop iterate at least twice (interval is 50ms). + await asyncio.sleep(0.2) + await scanner.stop() assert call_count["n"] >= 2 # second call ran despite first throwing diff --git a/tests/unit/test_memory/test_extract/test_parser/test_enrich.py b/tests/unit/test_memory/test_extract/test_parser/test_enrich.py index 7e47ead..8915da5 100644 --- a/tests/unit/test_memory/test_extract/test_parser/test_enrich.py +++ b/tests/unit/test_memory/test_extract/test_parser/test_enrich.py @@ -1,4 +1,4 @@ -"""Tests for enrich_content_items (everalgo.parser.aparse is monkeypatched).""" +"""Tests for enrich_content_items (component.parser.aparse_file is monkeypatched).""" from __future__ import annotations @@ -7,20 +7,29 @@ from typing import Any import pytest -# ``everalgo.parser`` ships under the ``[multimodal]`` extra (see -# pyproject.toml). CI doesn't install that extra by default, and these -# tests monkeypatch ``everalgo.parser.aparse`` — which requires the -# module to actually be importable, otherwise ``monkeypatch.setattr`` -# fails at resolve-time. Skip the whole module when the optional -# dependency isn't present; we still run when ``multimodal`` is installed. pytest.importorskip("everalgo.parser") from everalgo.llm import LLMError # noqa: E402 from everalgo.types import ParsedContent # noqa: E402 +from everos.component import parser as _parser_mod # noqa: E402 from everos.core.errors import UnsupportedModalityError # noqa: E402 from everos.memory.extract.parser import enrich_content_items # noqa: E402 +_APARSE_FILE_TARGET = "everos.component.parser.aparse_file" + + +@pytest.fixture(autouse=True) +def _ensure_parser_module_imported() -> None: + """Force ``everos.component.parser`` into sys.modules before monkeypatch. + + ``enrich_content_items`` does ``from everos.component.parser import + aparse_file`` inside its body. If the module hasn't been imported yet + when monkeypatch runs, the ``from`` import creates a fresh binding + to the real function, bypassing the patch. + """ + assert _parser_mod is not None + def _img_item() -> dict[str, Any]: return { @@ -45,38 +54,38 @@ def _html_uri_item() -> dict[str, Any]: async def test_enrich_backfills_parsed_content( monkeypatch: pytest.MonkeyPatch, ) -> None: - async def fake_aparse(raw_file: Any, *, llm: Any) -> ParsedContent: + async def fake_aparse(raw_file: Any) -> ParsedContent: return ParsedContent(text="OCR RESULT") - monkeypatch.setattr("everalgo.parser.aparse", fake_aparse) + monkeypatch.setattr(_APARSE_FILE_TARGET, fake_aparse) items: list[dict[str, Any]] = [{"type": "text", "text": "hi"}, _img_item()] - await enrich_content_items(items, llm=object(), max_concurrency=2) + await enrich_content_items(items, max_concurrency=2) assert items[1]["parsed_content"] == "OCR RESULT" assert items[1]["parse_status"] == "success" - assert "parsed_content" not in items[0] # text item untouched + assert "parsed_content" not in items[0] async def test_enrich_unsupported_modality_raises( monkeypatch: pytest.MonkeyPatch, ) -> None: - async def fake_aparse(raw_file: Any, *, llm: Any) -> ParsedContent: - raise NotImplementedError("video deferred") + async def fake_aparse(raw_file: Any) -> ParsedContent: + raise UnsupportedModalityError("video deferred") - monkeypatch.setattr("everalgo.parser.aparse", fake_aparse) + monkeypatch.setattr(_APARSE_FILE_TARGET, fake_aparse) with pytest.raises(UnsupportedModalityError): - await enrich_content_items([_img_item()], llm=object()) + await enrich_content_items([_img_item()]) async def test_enrich_transient_llm_error_degrades( monkeypatch: pytest.MonkeyPatch, ) -> None: - async def fake_aparse(raw_file: Any, *, llm: Any) -> ParsedContent: + async def fake_aparse(raw_file: Any) -> ParsedContent: raise LLMError("provider down") - monkeypatch.setattr("everalgo.parser.aparse", fake_aparse) + monkeypatch.setattr(_APARSE_FILE_TARGET, fake_aparse) items = [_img_item()] - await enrich_content_items(items, llm=object()) # must not raise + await enrich_content_items(items) assert items[0]["parse_status"] == "failed" assert "parsed_content" not in items[0] @@ -85,22 +94,16 @@ async def test_enrich_transient_llm_error_degrades( async def test_enrich_html_base64_routes_as_html_bytes( monkeypatch: pytest.MonkeyPatch, ) -> None: - """A type=html base64 item reaches the parser as html-extension bytes. - - Locks the "normal HTML file call" contract: base64 + ext=html maps to - a RawFile the parser dispatches as HTML (vs the 415 that a text-only - html item produces — see test_ingest for that negative path). - """ seen: dict[str, Any] = {} - async def fake_aparse(raw_file: Any, *, llm: Any) -> ParsedContent: + async def fake_aparse(raw_file: Any) -> ParsedContent: seen["extension"] = raw_file.extension seen["content"] = raw_file.content return ParsedContent(text="HTML PARSED") - monkeypatch.setattr("everalgo.parser.aparse", fake_aparse) + monkeypatch.setattr(_APARSE_FILE_TARGET, fake_aparse) items = [_html_b64_item()] - await enrich_content_items(items, llm=object()) + await enrich_content_items(items) assert items[0]["parsed_content"] == "HTML PARSED" assert items[0]["parse_status"] == "success" @@ -111,22 +114,16 @@ async def test_enrich_html_base64_routes_as_html_bytes( async def test_enrich_http_uri_routes_as_uri( monkeypatch: pytest.MonkeyPatch, ) -> None: - """An http(s) uri item reaches the parser as a uri RawFile (no bytes). - - Proves everos forwards uri-backed items to the parser, which is what - drives everalgo's URL-fetch dispatch path (http/https only; file:// is - rejected downstream). - """ seen: dict[str, Any] = {} - async def fake_aparse(raw_file: Any, *, llm: Any) -> ParsedContent: + async def fake_aparse(raw_file: Any) -> ParsedContent: seen["uri"] = raw_file.uri seen["content"] = raw_file.content return ParsedContent(text="URL PARSED") - monkeypatch.setattr("everalgo.parser.aparse", fake_aparse) + monkeypatch.setattr(_APARSE_FILE_TARGET, fake_aparse) items = [_html_uri_item()] - await enrich_content_items(items, llm=object()) + await enrich_content_items(items) assert items[0]["parsed_content"] == "URL PARSED" assert items[0]["parse_status"] == "success" @@ -137,47 +134,32 @@ async def test_enrich_http_uri_routes_as_uri( async def test_enrich_html_text_only_raises_unsupported( monkeypatch: pytest.MonkeyPatch, ) -> None: - """type=html carrying only ``text`` (no uri/base64) is undispatchable. - - Any non-text item is routed to the parser, which needs a fetchable or - decodable payload; a bare ``text`` has neither, so it surfaces as a - MultimodalError (the route maps it to HTTP 415). To inline HTML *as - text*, callers must use ``type="text"`` instead. - """ - - async def fake_aparse(raw_file: Any, *, llm: Any) -> ParsedContent: + async def fake_aparse(raw_file: Any) -> ParsedContent: return ParsedContent(text="should-not-be-reached") - monkeypatch.setattr("everalgo.parser.aparse", fake_aparse) + monkeypatch.setattr(_APARSE_FILE_TARGET, fake_aparse) with pytest.raises(UnsupportedModalityError): - await enrich_content_items( - [{"type": "html", "text": "

hi

"}], llm=object() - ) + await enrich_content_items([{"type": "html", "text": "

hi

"}]) async def test_enrich_file_uri_hydrates_and_parses( monkeypatch: pytest.MonkeyPatch, tmp_path: Any, ) -> None: - """A ``file://`` item is read locally and handed to the parser as bytes. - - Proves EverOS hydrates the file (everalgo never sees the path / fs) — the - parser receives ``content`` bytes, not a uri. - """ seen: dict[str, Any] = {} - async def fake_aparse(raw_file: Any, *, llm: Any) -> ParsedContent: + async def fake_aparse(raw_file: Any) -> ParsedContent: seen["content"] = raw_file.content seen["uri"] = raw_file.uri return ParsedContent(text="FILE PARSED") - monkeypatch.setattr("everalgo.parser.aparse", fake_aparse) + monkeypatch.setattr(_APARSE_FILE_TARGET, fake_aparse) f = tmp_path / "doc.html" f.write_bytes(b"hello") items = [{"type": "html", "uri": f"file://{f}"}] - await enrich_content_items(items, llm=object()) + await enrich_content_items(items) assert items[0]["parsed_content"] == "FILE PARSED" assert items[0]["parse_status"] == "success" - assert seen["content"] == b"hello" # hydrated, not a pointer + assert seen["content"] == b"hello" assert seen["uri"] == "" diff --git a/tests/unit/test_memory/test_extract/test_pipeline/test_user_memory_emits.py b/tests/unit/test_memory/test_extract/test_pipeline/test_user_memory_emits.py index b2db224..265f47d 100644 --- a/tests/unit/test_memory/test_extract/test_pipeline/test_user_memory_emits.py +++ b/tests/unit/test_memory/test_extract/test_pipeline/test_user_memory_emits.py @@ -3,7 +3,6 @@ from __future__ import annotations import datetime as _dt from unittest.mock import AsyncMock, MagicMock, patch -import pytest from everalgo.types import ChatMessage, MemCell from everalgo.types import Episode as AlgoEpisode @@ -64,7 +63,6 @@ async def test_emit_pipeline_started_routes_through_engine() -> None: assert started[0].memcell is cell -@pytest.mark.asyncio async def test_emit_episode_extracted_after_md_write() -> None: """Each per-sender Episode write emits EpisodeExtracted with the md entry id.""" engine = _CapturingEngine() @@ -121,3 +119,5 @@ async def test_emit_episode_extracted_after_md_write() -> None: assert extracted[0].episode_text == "they said hello" assert extracted[0].episode_timestamp_ms == 1_700_000_000_000 assert extracted[0].owner_id == "u1" + assert extracted[0].session_id == "s1" + assert extracted[0].source == "pipeline" diff --git a/tests/unit/test_memory/test_get/test_filters_adapter.py b/tests/unit/test_memory/test_get/test_filters_adapter.py index c5dea9c..eef5deb 100644 --- a/tests/unit/test_memory/test_get/test_filters_adapter.py +++ b/tests/unit/test_memory/test_get/test_filters_adapter.py @@ -28,7 +28,8 @@ def test_no_filters_emits_base_clause() -> None: where = compile_filters_for_get(None, owner_id="u1", owner_type="user") assert where == ( "owner_id = 'u1' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default'" + "AND app_id = 'default' AND project_id = 'default' " + "AND deprecated_by IS NULL" ) @@ -37,7 +38,8 @@ def test_owner_id_quote_is_escaped() -> None: where = compile_filters_for_get(None, owner_id="o'reilly", owner_type="user") assert where == ( "owner_id = 'o''reilly' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default'" + "AND app_id = 'default' AND project_id = 'default' " + "AND deprecated_by IS NULL" ) @@ -50,8 +52,8 @@ def test_flat_multi_field_renders_implicit_and() -> None: assert "owner_type = 'user'" in where assert "session_id = 'sess_a'" in where assert "parent_id = 'mc_x'" in where - # 4 base scope clauses + 2 filter fields = 6 clauses → 5 ' AND ' joins. - assert where.count(" AND ") == 5 + # 5 base scope clauses + 2 filter fields = 7 clauses → 6 ' AND ' joins. + assert where.count(" AND ") == 6 def test_reserved_owner_id_in_filters_raises() -> None: diff --git a/tests/unit/test_memory/test_reflection/__init__.py b/tests/unit/test_memory/test_reflection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_memory/test_reflection/test_orchestrator.py b/tests/unit/test_memory/test_reflection/test_orchestrator.py new file mode 100644 index 0000000..c168204 --- /dev/null +++ b/tests/unit/test_memory/test_reflection/test_orchestrator.py @@ -0,0 +1,436 @@ +"""Tests for :class:`ReflectionOrchestrator`. + +All seven constructor dependencies are mocked. Tests verify: +- candidate selection filtering logic (INIT vs UPDATE) +- full INIT-mode flow with merge + deprecate +- UPDATE-mode old_episode passthrough +- LLM failure skips cluster gracefully +- empty candidates return empty list +""" + +from __future__ import annotations + +import datetime as _dt +from dataclasses import dataclass, field +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from everos.infra.ome.testing import FakeStrategyContext +from everos.memory._partition_locks import _reset_for_tests +from everos.memory.reflection.orchestrator import ( + _MAX_CLUSTERS_PER_RUN, + ReflectionOrchestrator, + _merged_episode_to_entry_body, + _ts_to_ms, +) + + +@pytest.fixture(autouse=True) +def _isolate_locks() -> None: + _reset_for_tests() + + +# ── Helpers ─────────────────────────────────────────────────────────────── + + +@dataclass +class _FakeAlgoResult: + """Minimal stand-in for ``everalgo.types.Episode``.""" + + owner_id: str | None + episode: str + subject: str + timestamp: int + + +@dataclass +class _FakeEpisodeRow: + """Minimal stand-in for a LanceDB Episode row.""" + + id: str + entry_id: str + owner_id: str + owner_type: str = "user" + app_id: str = "default" + project_id: str = "default" + session_id: str | None = "s_test" + timestamp: _dt.datetime = _dt.datetime(2026, 6, 1, tzinfo=_dt.UTC) + parent_type: str = "memcell" + parent_id: str = "mc_aaa" + sender_ids: list[str] = field(default_factory=list) + subject: str | None = "test subject" + summary: str | None = None + episode: str = "test episode text" + episode_tokens: str = "test episode text" + md_path: str = "/tmp/test.md" + content_sha256: str = "abc123" + deprecated_by: str | None = None + vector: list[float] | None = None + + +def _make_episode_row( + entry_id: str = "ep_20260601_0001", + parent_id: str = "mc_aaa", + parent_type: str = "memcell", + owner_id: str = "u_alice", + **kwargs: object, +) -> _FakeEpisodeRow: + return _FakeEpisodeRow( + id=f"{owner_id}_{entry_id}", + entry_id=entry_id, + owner_id=owner_id, + parent_id=parent_id, + parent_type=parent_type, + **kwargs, + ) + + +def _make_entry_id(formatted: str = "ep_20260614_0001") -> MagicMock: + eid = MagicMock() + eid.format.return_value = formatted + eid.date = _dt.date(2026, 6, 14) + return eid + + +def _build_orchestrator( + *, + cluster_repo: MagicMock | None = None, + episode_store: MagicMock | None = None, + atomic_fact_store: MagicMock | None = None, + episode_writer: MagicMock | None = None, + report_repo: MagicMock | None = None, + reflector: MagicMock | None = None, + embedder: MagicMock | None = None, +) -> ReflectionOrchestrator: + return ReflectionOrchestrator( + cluster_repo=cluster_repo or MagicMock(), + episode_store=episode_store or MagicMock(), + atomic_fact_store=atomic_fact_store or MagicMock(), + episode_writer=episode_writer or MagicMock(), + report_repo=report_repo or MagicMock(), + reflector=reflector or MagicMock(), + embedder=embedder or MagicMock(), + ) + + +# ── Tests ───────────────────────────────────────────────────────────────── + + +async def test_select_candidates_init_and_update() -> None: + """Unreflected clusters with >=2 members are INIT candidates. + Reflected clusters with >1 member are UPDATE candidates. + """ + cluster_repo = MagicMock() + report_repo = MagicMock() + + report_repo.list_reflected_cluster_ids = AsyncMock(return_value={"cl_reflected"}) + cluster_repo.list_ids_and_member_counts = AsyncMock( + return_value=[ + ("cl_new_3", 3), + ("cl_new_1", 1), # only 1 member -> skip + ("cl_reflected", 2), # reflected + 2 members -> UPDATE + ("cl_reflected_1", 1), # reflected + 1 member -> skip + ] + ) + + orch = _build_orchestrator(cluster_repo=cluster_repo, report_repo=report_repo) + result = await orch._select_candidates( + owner_id="u_alice", + kind="user_memory", + app_id="default", + project_id="default", + ) + + assert result == ["cl_new_3", "cl_reflected"] + + +async def test_select_candidates_respects_max_limit() -> None: + """More than ``_MAX_CLUSTERS_PER_RUN`` candidates are truncated.""" + cluster_repo = MagicMock() + report_repo = MagicMock() + report_repo.list_reflected_cluster_ids = AsyncMock(return_value=set()) + cluster_repo.list_ids_and_member_counts = AsyncMock( + return_value=[(f"cl_{i:03d}", i + 2) for i in range(_MAX_CLUSTERS_PER_RUN + 5)] + ) + + orch = _build_orchestrator(cluster_repo=cluster_repo, report_repo=report_repo) + result = await orch._select_candidates( + owner_id="u_alice", + kind="user_memory", + app_id="default", + project_id="default", + ) + assert len(result) == _MAX_CLUSTERS_PER_RUN + + +async def test_empty_candidates_returns_empty() -> None: + """No qualifying clusters -> run() returns empty list immediately.""" + cluster_repo = MagicMock() + report_repo = MagicMock() + report_repo.list_reflected_cluster_ids = AsyncMock(return_value=set()) + cluster_repo.list_ids_and_member_counts = AsyncMock( + return_value=[("cl_only_one", 1)] + ) + + orch = _build_orchestrator(cluster_repo=cluster_repo, report_repo=report_repo) + ctx = FakeStrategyContext() + reports = await orch.run(ctx=ctx, owner_id="u_alice") + assert reports == [] + + +async def test_run_init_mode_merges_and_deprecates() -> None: + """Full INIT flow: 2 episode members -> merge -> write -> deprecate.""" + cluster_repo = MagicMock() + episode_store = MagicMock() + atomic_fact_store = MagicMock() + episode_writer = MagicMock() + report_repo = MagicMock() + reflector = MagicMock() + embedder = MagicMock() + + # SELECT: 1 candidate cluster. + report_repo.list_reflected_cluster_ids = AsyncMock(return_value=set()) + cluster_repo.list_ids_and_member_counts = AsyncMock(return_value=[("cl_abc", 2)]) + + # Step 0: orphan detection returns empty. + episode_store.find_where = AsyncMock(return_value=[]) + + # Step 1: cluster members (episode type). + cluster_repo.get_members_with_type = AsyncMock( + return_value=[("ep_20260601_0001", "episode"), ("ep_20260601_0002", "episode")] + ) + + # Step 2: fetch episodes by entry_id. + ep1 = _make_episode_row( + entry_id="ep_20260601_0001", parent_id="mc_001", owner_id="u_alice" + ) + ep2 = _make_episode_row( + entry_id="ep_20260601_0002", parent_id="mc_002", owner_id="u_alice" + ) + episode_store.find_by_owner_entries = AsyncMock(return_value=[ep1, ep2]) + + # Step 4: algo reflector returns merged episode. + algo_result = _FakeAlgoResult( + owner_id=None, + episode="merged episode text", + subject="merged subject", + timestamp=1717200000000, + ) + reflector.areflect = AsyncMock(return_value=algo_result) + + # Step 5: episode writer returns entry id. + entry_id_mock = _make_entry_id("ep_20260614_0001") + episode_writer.append_entries = AsyncMock(return_value=[entry_id_mock]) + episode_writer.patch_frontmatter = AsyncMock() + + # Step 6: wait_for_event succeeds. + ctx = FakeStrategyContext() + + # Step 7: deprecate -> cluster operations. + cluster_repo.remove_members = AsyncMock() + cluster_repo.add_member = AsyncMock() + cluster_repo.update_metadata = AsyncMock() + embedder.embed = AsyncMock(return_value=[0.1] * 1024) + + # LanceDB episode store update for deprecation. + episode_store.update = AsyncMock() + + # Atomic fact store. + atomic_fact_store.update = AsyncMock() + + # Report repo. + report_repo.create = AsyncMock() + + orch = _build_orchestrator( + cluster_repo=cluster_repo, + episode_store=episode_store, + atomic_fact_store=atomic_fact_store, + episode_writer=episode_writer, + report_repo=report_repo, + reflector=reflector, + embedder=embedder, + ) + + reports = await orch.run(ctx=ctx, owner_id="u_alice") + + # Reflector was called in INIT mode (no old_episode kwarg). + reflector.areflect.assert_awaited_once() + call_kwargs = reflector.areflect.call_args + assert "old_episode" not in (call_kwargs.kwargs or {}) + + # Episode was written. + episode_writer.append_entries.assert_awaited_once() + + # EpisodeExtracted was emitted. + assert len(ctx.emitted) == 1 + event = ctx.emitted[0] + assert event.source == "reflection" + assert event.session_id is None + + # Cluster updated. + cluster_repo.remove_members.assert_awaited_once() + cluster_repo.add_member.assert_awaited_once_with( + "cl_abc", "ep_20260614_0001", "episode" + ) + + # Report created. + report_repo.create.assert_awaited_once() + assert len(reports) == 1 + + +async def test_run_update_mode_uses_old_episode() -> None: + """UPDATE flow: cluster has 1 merged episode + 1 original episode.""" + cluster_repo = MagicMock() + episode_store = MagicMock() + atomic_fact_store = MagicMock() + episode_writer = MagicMock() + report_repo = MagicMock() + reflector = MagicMock() + embedder = MagicMock() + + # SELECT. + report_repo.list_reflected_cluster_ids = AsyncMock(return_value={"cl_update"}) + cluster_repo.list_ids_and_member_counts = AsyncMock(return_value=[("cl_update", 2)]) + + # Orphan detection. + episode_store.find_where = AsyncMock(return_value=[]) + + # Members: both episode type (old merged + new original). + cluster_repo.get_members_with_type = AsyncMock( + return_value=[("ep_20260612_0001", "episode"), ("ep_20260613_0001", "episode")] + ) + + # Episodes. + old_merged = _make_episode_row( + entry_id="ep_20260612_0001", + parent_id="cl_update", + parent_type="cluster", + owner_id="u_alice", + episode="old merged text", + ) + new_ep = _make_episode_row( + entry_id="ep_20260613_0001", + parent_id="mc_004", + owner_id="u_alice", + episode="new episode text", + ) + episode_store.find_by_owner_entries = AsyncMock(return_value=[new_ep, old_merged]) + + # Reflector. + algo_result = _FakeAlgoResult( + owner_id=None, + episode="updated merged text", + subject="updated subject", + timestamp=1717200000000, + ) + reflector.areflect = AsyncMock(return_value=algo_result) + + # Writer. + entry_id_mock = _make_entry_id("ep_20260614_0002") + episode_writer.append_entries = AsyncMock(return_value=[entry_id_mock]) + episode_writer.patch_frontmatter = AsyncMock() + + # Deprecate deps. + cluster_repo.remove_members = AsyncMock() + cluster_repo.add_member = AsyncMock() + cluster_repo.update_metadata = AsyncMock() + embedder.embed = AsyncMock(return_value=[0.1] * 1024) + atomic_fact_store.update = AsyncMock() + episode_store.update = AsyncMock() + report_repo.create = AsyncMock() + + ctx = FakeStrategyContext() + orch = _build_orchestrator( + cluster_repo=cluster_repo, + episode_store=episode_store, + atomic_fact_store=atomic_fact_store, + episode_writer=episode_writer, + report_repo=report_repo, + reflector=reflector, + embedder=embedder, + ) + + reports = await orch.run(ctx=ctx, owner_id="u_alice") + + # Reflector called with old_episode kwarg (UPDATE mode). + reflector.areflect.assert_awaited_once() + _, kwargs = reflector.areflect.call_args + assert "old_episode" in kwargs + + assert len(reports) == 1 + + +async def test_llm_failure_skips_cluster() -> None: + """Reflector raising an exception skips the cluster, continues.""" + cluster_repo = MagicMock() + episode_store = MagicMock() + report_repo = MagicMock() + reflector = MagicMock() + + # SELECT: 1 candidate. + report_repo.list_reflected_cluster_ids = AsyncMock(return_value=set()) + cluster_repo.list_ids_and_member_counts = AsyncMock(return_value=[("cl_fail", 2)]) + + # Orphan detection. + episode_store.find_where = AsyncMock(return_value=[]) + + # Members. + cluster_repo.get_members_with_type = AsyncMock( + return_value=[("ep_001", "episode"), ("ep_002", "episode")] + ) + + # Episodes. + ep1 = _make_episode_row(entry_id="ep_001", parent_id="mc_a", owner_id="u_alice") + ep2 = _make_episode_row(entry_id="ep_002", parent_id="mc_b", owner_id="u_alice") + episode_store.find_by_owner_entries = AsyncMock(return_value=[ep1, ep2]) + + # Reflector fails. + reflector.areflect = AsyncMock(side_effect=RuntimeError("LLM timeout")) + + ctx = FakeStrategyContext() + orch = _build_orchestrator( + cluster_repo=cluster_repo, + episode_store=episode_store, + report_repo=report_repo, + reflector=reflector, + ) + + reports = await orch.run(ctx=ctx, owner_id="u_alice") + assert reports == [] + assert len(ctx.emitted) == 0 + + +# ── Unit helpers ────────────────────────────────────────────────────────── + + +def test_merged_episode_to_entry_body_shape() -> None: + """Verify the inline/sections shape for a merged episode.""" + result = _FakeAlgoResult( + owner_id=None, + episode="merged text", + subject="merged subject", + timestamp=1717200000000, + ) + inline, sections = _merged_episode_to_entry_body( + result, "cl_abc", "u_alice", "2026-06-01T00:00:00+00:00" + ) + assert inline["parent_type"] == "cluster" + assert inline["parent_id"] == "cl_abc" + assert inline["owner_id"] == "u_alice" + assert "session_id" not in inline + assert sections["Subject"] == "merged subject" + assert sections["Content"] == "merged text" + + +def test_ts_to_ms_datetime() -> None: + """datetime -> milliseconds conversion.""" + dt = _dt.datetime(2026, 6, 1, tzinfo=_dt.UTC) + ms = _ts_to_ms(dt) + assert isinstance(ms, int) + assert ms > 0 + + +def test_ts_to_ms_int_passthrough() -> None: + """int -> int passthrough.""" + assert _ts_to_ms(1717200000000) == 1717200000000 diff --git a/tests/unit/test_memory/test_search/test_agentic.py b/tests/unit/test_memory/test_search/test_agentic.py index 0886734..8464027 100644 --- a/tests/unit/test_memory/test_search/test_agentic.py +++ b/tests/unit/test_memory/test_search/test_agentic.py @@ -309,12 +309,12 @@ async def test_agentic_search_shapes_candidates_with_episode_id( # ── Metadata bridge to the everalgo _format_docs contract ────────────────── -def test_to_everalgo_doc_metadata_injects_text_and_ms_timestamp() -> None: - """Bridge adds `text` (episode body) + ms-epoch `timestamp` for _format_docs. +def test_to_everalgo_doc_metadata_bridges_episode_and_timestamp() -> None: + """Bridge restructures episode to dict and converts timestamp to ms-epoch. - Without this the sufficiency / multi-query LLM prompt falls back to the - memcell id as the doc body and renders the date as "N/A". ``episode`` is - left untouched so the reranker / shaper (both expecting a str) keep working. + ``_format_docs`` expects ``metadata["episode"] = {"subject": ..., "content": ...}`` + and a ms-epoch ``timestamp``. The flat ``episode`` string is also kept as + ``text`` for the reranker. """ original = _ts() md = { @@ -324,15 +324,21 @@ def test_to_everalgo_doc_metadata_injects_text_and_ms_timestamp() -> None: } out = _to_everalgo_doc_metadata(md) assert out["text"] == "Alice prefers oat milk" - assert out["episode"] == "Alice prefers oat milk" # untouched for rerank/shaper + assert out["episode"] == { + "subject": "Alice eats oat milk", + "content": "Alice prefers oat milk", + } assert isinstance(out["timestamp"], int) assert from_timestamp(out["timestamp"]) == original -def test_restore_shaper_metadata_reverts_ms_timestamp_to_datetime() -> None: - """The ms-epoch timestamp is reverted to the datetime the shaper requires.""" +def test_restore_shaper_metadata_reverts_bridged_fields() -> None: + """Restore reverts both ms-epoch timestamp and dict episode to shaper format.""" original = _ts() - bridged = _to_everalgo_doc_metadata({"episode": "x", "timestamp": original}) + bridged = _to_everalgo_doc_metadata( + {"episode": "x", "timestamp": original, "subject": "s"} + ) restored = _restore_shaper_metadata(bridged) assert isinstance(restored["timestamp"], _dt.datetime) assert restored["timestamp"] == original + assert restored["episode"] == "x" diff --git a/tests/unit/test_memory/test_search/test_dto.py b/tests/unit/test_memory/test_search/test_dto.py index c720380..f1a5a56 100644 --- a/tests/unit/test_memory/test_search/test_dto.py +++ b/tests/unit/test_memory/test_search/test_dto.py @@ -42,6 +42,7 @@ def test_minimal_request_uses_hybrid_default() -> None: assert req.include_profile is False assert req.filters is None assert req.radius is None + assert req.min_score is None def test_top_k_zero_rejected() -> None: @@ -77,6 +78,18 @@ def test_radius_out_of_range_rejected() -> None: SearchRequest(**_minimal_request_kwargs(), radius=-0.1) +def test_min_score_out_of_range_rejected() -> None: + with pytest.raises(ValidationError): + SearchRequest(**_minimal_request_kwargs(), min_score=1.5) + with pytest.raises(ValidationError): + SearchRequest(**_minimal_request_kwargs(), min_score=-0.1) + + +def test_min_score_in_range_accepted() -> None: + req = SearchRequest(**_minimal_request_kwargs(), min_score=0.4) + assert req.min_score == 0.4 + + def test_neither_user_id_nor_agent_id_rejected() -> None: """The xor validator requires exactly one of user_id / agent_id.""" with pytest.raises(ValidationError, match="exactly one of"): @@ -127,7 +140,6 @@ def test_response_default_arrays_present() -> None: assert resp.data.profiles == [] assert resp.data.agent_cases == [] assert resp.data.agent_skills == [] - assert resp.data.unprocessed_messages == [] def test_method_enum_serialises_to_lowercase() -> None: diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index dd304ae..1964761 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -17,7 +17,8 @@ def test_no_filters_emits_base_clause() -> None: where = compile_filters(None, owner_id="alice", owner_type="user") assert where == ( "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default'" + "AND app_id = 'default' AND project_id = 'default' " + "AND deprecated_by IS NULL" ) @@ -240,5 +241,14 @@ def test_empty_and_array_skips_combinator() -> None: where = compile_filters(node, owner_id="alice", owner_type="user") assert where == ( "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default'" + "AND app_id = 'default' AND project_id = 'default' " + "AND deprecated_by IS NULL" ) + + +# ── Deprecated exclusion ────────────────────────────────────────────── + + +def test_compile_filters_excludes_deprecated_by_default() -> None: + result = compile_filters(None, owner_id="u_a", owner_type="user") + assert "deprecated_by IS NULL" in result diff --git a/tests/unit/test_memory/test_search/test_hierarchy.py b/tests/unit/test_memory/test_search/test_hierarchy.py index 5ef8a99..7ce3c89 100644 --- a/tests/unit/test_memory/test_search/test_hierarchy.py +++ b/tests/unit/test_memory/test_search/test_hierarchy.py @@ -4,6 +4,12 @@ White-box surfaces accessed: - ``_hierarchy_eviction_pass`` (internal, tested directly for unit coverage) - ``hierarchy_retrieve_episodes`` (public function, tested with stubbed I/O) +Layer 4 uses hierarchical fact eviction: parent episode and its facts are +calibrated to an LR probability via ``cosine_to_lr_score`` and compete on that +single scale, so the expected scores below are computed with the same helper +rather than hard-coded — the assertions track the calibration, not magic +numbers. + All I/O (fact_recaller, episode_recaller) is injected via AsyncMock stubs. No LanceDB or network calls are made. """ @@ -14,9 +20,11 @@ import datetime as _dt from unittest.mock import AsyncMock, MagicMock import pytest +from everalgo.rank.fusion import cosine_to_lr_score from everalgo.types import Candidate, FactCandidate from everos.memory.search.hierarchy import ( + _build_ep_to_fact_parents, _hierarchy_eviction_pass, hierarchy_retrieve_episodes, ) @@ -33,22 +41,26 @@ def _episode_candidate( ep_id: str = "ep-1", score: float = 0.7, memcell_id: str = "mc-1", + entry_id: str | None = None, ) -> Candidate: + metadata = { + "parent_id": memcell_id, + "owner_id": "u1", + "owner_type": "user", + "session_id": "sess-1", + "timestamp": _ts(), + "episode": "Some episode text.", + "sender_ids": ["u1"], + "subject": "Test subject", + "summary": "Test summary", + } + if entry_id is not None: + metadata["entry_id"] = entry_id return Candidate( id=ep_id, score=score, source="vector", - metadata={ - "parent_id": memcell_id, - "owner_id": "u1", - "owner_type": "user", - "session_id": "sess-1", - "timestamp": _ts(), - "episode": "Some episode text.", - "sender_ids": ["u1"], - "subject": "Test subject", - "summary": "Test summary", - }, + metadata=metadata, ) @@ -78,9 +90,7 @@ def _make_recallers( fact_recaller.facts_for_episodes = AsyncMock(return_value=facts_for_episodes or {}) episode_recaller = MagicMock() - episode_recaller.fetch_by_parent_ids = AsyncMock( - return_value=fetched_episodes or [] - ) + episode_recaller.fetch_by_entry_ids = AsyncMock(return_value=fetched_episodes or []) return fact_recaller, episode_recaller @@ -89,107 +99,286 @@ def _make_recallers( class TestHierarchyEvictionPass: - def test_fact_wins_emits_atomic_fact_scored_item(self) -> None: - episode = _episode_candidate(ep_id="ep-1", score=0.5) + def test_fact_wins_emits_atomic_fact_at_lr_score(self) -> None: + # Fact cosine (0.9) > parent cosine (0.5) → fact wins; alpha=1.0 so the + # emitted score is the fact's own LR-calibrated value. + episode = _episode_candidate(ep_id="ep-1") fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.9) - result = _hierarchy_eviction_pass([episode], {"ep-1": [fact]}) + result = _hierarchy_eviction_pass( + [episode], + {"ep-1": [fact]}, + ep_cosine={"ep-1": 0.5}, + ep_bm25={}, + ) assert len(result) == 1 item = result[0] assert item.item_type == "atomic_fact" assert item.id == "fact-1" - assert item.score == pytest.approx(0.9) + assert item.score == pytest.approx(cosine_to_lr_score(0.9, 0.0)) - def test_episode_wins_emits_episode_scored_item(self) -> None: - episode = _episode_candidate(ep_id="ep-1", score=0.8) + def test_episode_wins_emits_episode_at_parent_lr_score(self) -> None: + # Fact cosine (0.6) < parent cosine (0.8) → episode wins at parent_lr. + episode = _episode_candidate(ep_id="ep-1") fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.6) - result = _hierarchy_eviction_pass([episode], {"ep-1": [fact]}) + result = _hierarchy_eviction_pass( + [episode], + {"ep-1": [fact]}, + ep_cosine={"ep-1": 0.8}, + ep_bm25={}, + ) assert len(result) == 1 item = result[0] assert item.item_type == "episode" assert item.id == "ep-1" - assert item.score == pytest.approx(0.8) + assert item.score == pytest.approx(cosine_to_lr_score(0.8, 0.0)) - def test_no_facts_emits_episode(self) -> None: - episode = _episode_candidate(ep_id="ep-1", score=0.7) + def test_no_facts_emits_episode_at_parent_lr(self) -> None: + episode = _episode_candidate(ep_id="ep-1") - result = _hierarchy_eviction_pass([episode], {}) + result = _hierarchy_eviction_pass( + [episode], + {}, + ep_cosine={"ep-1": 0.7}, + ep_bm25={}, + ) assert len(result) == 1 assert result[0].item_type == "episode" assert result[0].id == "ep-1" + assert result[0].score == pytest.approx(cosine_to_lr_score(0.7, 0.0)) def test_ordering_preserved_matches_input_order(self) -> None: - ep_a = _episode_candidate(ep_id="ep-a", score=0.9, memcell_id="mc-a") - ep_b = _episode_candidate(ep_id="ep-b", score=0.8, memcell_id="mc-b") - ep_c = _episode_candidate(ep_id="ep-c", score=0.7, memcell_id="mc-c") + ep_a = _episode_candidate(ep_id="ep-a", memcell_id="mc-a") + ep_b = _episode_candidate(ep_id="ep-b", memcell_id="mc-b") + ep_c = _episode_candidate(ep_id="ep-c", memcell_id="mc-c") merged = [ep_a, ep_b, ep_c] - result = _hierarchy_eviction_pass(merged, {}) + result = _hierarchy_eviction_pass( + merged, + {}, + ep_cosine={"ep-a": 0.9, "ep-b": 0.8, "ep-c": 0.7}, + ep_bm25={}, + ) assert [r.id for r in result] == ["ep-a", "ep-b", "ep-c"] def test_parent_episode_id_set_on_evicted_fact(self) -> None: - episode = _episode_candidate(ep_id="ep-1", score=0.4) + episode = _episode_candidate(ep_id="ep-1") fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.8) - result = _hierarchy_eviction_pass([episode], {"ep-1": [fact]}) + result = _hierarchy_eviction_pass( + [episode], + {"ep-1": [fact]}, + ep_cosine={"ep-1": 0.4}, + ep_bm25={}, + ) assert result[0].parent_episode_id == "ep-1" def test_episode_wins_parent_episode_id_is_none(self) -> None: - episode = _episode_candidate(ep_id="ep-1", score=0.9) + episode = _episode_candidate(ep_id="ep-1") fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.5) - result = _hierarchy_eviction_pass([episode], {"ep-1": [fact]}) + result = _hierarchy_eviction_pass( + [episode], + {"ep-1": [fact]}, + ep_cosine={"ep-1": 0.9}, + ep_bm25={}, + ) assert result[0].parent_episode_id is None def test_multiple_episodes_mixed_eviction(self) -> None: - ep1 = _episode_candidate(ep_id="ep-1", score=0.5, memcell_id="mc-1") - ep2 = _episode_candidate(ep_id="ep-2", score=0.8, memcell_id="mc-2") - ep3 = _episode_candidate(ep_id="ep-3", score=0.6, memcell_id="mc-3") + ep1 = _episode_candidate(ep_id="ep-1", memcell_id="mc-1") + ep2 = _episode_candidate(ep_id="ep-2", memcell_id="mc-2") + ep3 = _episode_candidate(ep_id="ep-3", memcell_id="mc-3") fact1 = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.9) fact2 = _fact_candidate(fact_id="fact-2", parent_episode_id="ep-2", score=0.4) result = _hierarchy_eviction_pass( [ep1, ep2, ep3], {"ep-1": [fact1], "ep-2": [fact2]}, + ep_cosine={"ep-1": 0.5, "ep-2": 0.8, "ep-3": 0.6}, + ep_bm25={}, ) assert len(result) == 3 - assert result[0].item_type == "atomic_fact" + assert result[0].item_type == "atomic_fact" # 0.9 > 0.5 assert result[0].id == "fact-1" - assert result[1].item_type == "episode" + assert result[1].item_type == "episode" # 0.4 < 0.8 assert result[1].id == "ep-2" - assert result[2].item_type == "episode" + assert result[2].item_type == "episode" # no fact assert result[2].id == "ep-3" - def test_best_fact_is_first_element_used_for_comparison(self) -> None: - episode = _episode_candidate(ep_id="ep-1", score=0.7) + def test_best_fact_across_window_used_for_comparison(self) -> None: + episode = _episode_candidate(ep_id="ep-1") best_fact = _fact_candidate( - fact_id="fact-best", parent_episode_id="ep-1", score=0.8 + fact_id="fact-best", parent_episode_id="ep-1", score=0.85 ) - second_fact = _fact_candidate( - fact_id="fact-second", parent_episode_id="ep-1", score=0.3 + weak_fact = _fact_candidate( + fact_id="fact-weak", parent_episode_id="ep-1", score=0.3 ) - result = _hierarchy_eviction_pass([episode], {"ep-1": [best_fact, second_fact]}) + result = _hierarchy_eviction_pass( + [episode], + {"ep-1": [best_fact, weak_fact]}, + ep_cosine={"ep-1": 0.7}, + ep_bm25={}, + ) assert result[0].item_type == "atomic_fact" assert result[0].id == "fact-best" - def test_fact_score_equal_to_episode_score_episode_wins(self) -> None: - episode = _episode_candidate(ep_id="ep-1", score=0.7) + def test_fact_equal_to_parent_does_not_evict(self) -> None: + # Blend must strictly beat parent_lr; equal scores keep the episode. + episode = _episode_candidate(ep_id="ep-1") fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.7) - result = _hierarchy_eviction_pass([episode], {"ep-1": [fact]}) + result = _hierarchy_eviction_pass( + [episode], + {"ep-1": [fact]}, + ep_cosine={"ep-1": 0.7}, + ep_bm25={}, + ) assert result[0].item_type == "episode" + def test_alpha_blends_parent_and_child(self) -> None: + # alpha=0.5 → score = 0.5*child_lr + 0.5*parent_lr. + episode = _episode_candidate(ep_id="ep-1") + fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.9) + + result = _hierarchy_eviction_pass( + [episode], + {"ep-1": [fact]}, + ep_cosine={"ep-1": 0.5}, + ep_bm25={}, + alpha=0.5, + ) + + parent_lr = cosine_to_lr_score(0.5, 0.0) + child_lr = cosine_to_lr_score(0.9, 0.0) + expected = 0.5 * child_lr + 0.5 * parent_lr + assert result[0].item_type == "atomic_fact" + assert result[0].score == pytest.approx(expected) + + def test_bm25_raises_calibrated_score(self) -> None: + # BM25 is folded into both parent and child calibration (children + # inherit parent_bm25), so it does not change the intra-episode + # parent-vs-fact outcome at alpha=1 — it lifts the absolute LR score. + episode = _episode_candidate(ep_id="ep-1") + + without_bm25 = _hierarchy_eviction_pass( + [episode], {}, ep_cosine={"ep-1": 0.5}, ep_bm25={} + ) + with_bm25 = _hierarchy_eviction_pass( + [episode], {}, ep_cosine={"ep-1": 0.5}, ep_bm25={"ep-1": 50.0} + ) + + assert with_bm25[0].score > without_bm25[0].score + assert with_bm25[0].score == pytest.approx(cosine_to_lr_score(0.5, 50.0)) + + def test_facts_per_episode_window_caps_competition(self) -> None: + # A high-scoring fact beyond the window must not win. + episode = _episode_candidate(ep_id="ep-1") + in_window = _fact_candidate(fact_id="in", parent_episode_id="ep-1", score=0.55) + out_window = _fact_candidate( + fact_id="out", parent_episode_id="ep-1", score=0.99 + ) + + result = _hierarchy_eviction_pass( + [episode], + {"ep-1": [in_window, out_window]}, + ep_cosine={"ep-1": 0.6}, + ep_bm25={}, + facts_per_episode=1, + ) + + # Only ``in`` (0.55) is in the 1-fact window and it loses to 0.6 → + # episode wins; the 0.99 ``out`` fact is never considered. + assert result[0].item_type == "episode" + + +# ── _build_ep_to_fact_parents unit tests ──────────────────────────────── + + +class TestBuildEpToFactParents: + """Unit tests for the dual parent_id mapping builder.""" + + def test_entry_id_and_parent_id_both_collected(self) -> None: + """Post-1.5 episode with entry_id and parent_id both present.""" + ep = _episode_candidate(ep_id="ep-1", memcell_id="mc-1", entry_id="ep_entry_1") + + result = _build_ep_to_fact_parents([ep]) + + assert result == {"ep-1": ["ep_entry_1", "mc-1"]} + + def test_memcell_id_only_no_entry_id(self) -> None: + """Pre-1.5 episode: only parent_id (memcell_id), no entry_id.""" + ep = _episode_candidate(ep_id="ep-1", memcell_id="mc-1") + + result = _build_ep_to_fact_parents([ep]) + + assert result == {"ep-1": ["mc-1"]} + + def test_entry_id_equals_parent_id_no_duplicate(self) -> None: + """When entry_id == parent_id, only one value in the list.""" + ep = _episode_candidate(ep_id="ep-1", memcell_id="same_id", entry_id="same_id") + + result = _build_ep_to_fact_parents([ep]) + + assert result == {"ep-1": ["same_id"]} + + def test_missing_parent_id_skipped(self) -> None: + """Episode with no parent_id and no entry_id is excluded.""" + ep = Candidate( + id="ep-orphan", + score=0.5, + source="vector", + metadata={"owner_id": "u1"}, + ) + + result = _build_ep_to_fact_parents([ep]) + + assert result == {} + + def test_empty_string_parent_id_skipped(self) -> None: + """Empty string parent_id is filtered out.""" + ep = _episode_candidate(ep_id="ep-1", memcell_id="") + + result = _build_ep_to_fact_parents([ep]) + + assert result == {} + + def test_empty_string_entry_id_skipped(self) -> None: + """Empty string entry_id is filtered; parent_id still collected.""" + ep = _episode_candidate(ep_id="ep-1", memcell_id="mc-1", entry_id="") + + result = _build_ep_to_fact_parents([ep]) + + assert result == {"ep-1": ["mc-1"]} + + def test_multiple_episodes_independent(self) -> None: + ep_a = _episode_candidate( + ep_id="ep-a", memcell_id="mc-a", entry_id="ep_entry_a" + ) + ep_b = _episode_candidate(ep_id="ep-b", memcell_id="mc-b") + + result = _build_ep_to_fact_parents([ep_a, ep_b]) + + assert result == { + "ep-a": ["ep_entry_a", "mc-a"], + "ep-b": ["mc-b"], + } + + def test_empty_list_returns_empty_dict(self) -> None: + result = _build_ep_to_fact_parents([]) + assert result == {} + # ── hierarchy_retrieve_episodes integration-style unit tests ───────────── @@ -241,27 +430,25 @@ class TestHierarchyRetrieveEpisodes: episode_item = result[0] assert episode_item.id == "ep-1" assert episode_item.atomic_facts == [] + # Episode-win score is the LR-calibrated parent score (cosine 0.8, bm25 0.8). + assert episode_item.score == pytest.approx(cosine_to_lr_score(0.8, 0.8)) async def test_happy_path_fact_evicts_episode_nested_in_result(self) -> None: - ep = _episode_candidate(ep_id="ep-2", score=0.6, memcell_id="mc-2") + ep = _episode_candidate(ep_id="ep-2", score=0.5, memcell_id="mc-2") fact = _fact_candidate(fact_id="fact-2", parent_episode_id="ep-2", score=0.95) + # No Layer-2 boost (dense_facts empty) → ep_cosine comes from the dense + # episode recall (0.5); sparse empty → parent_bm25 = 0.0. The Layer-4 + # fact (0.95) calibrates above the parent and evicts it. fact_recaller, episode_recaller = _make_recallers( - dense_facts=[ - Candidate( - id="fact-2", - score=0.95, - source="vector", - metadata={"parent_id": "mc-2"}, - ) - ], - fetched_episodes=[ep], + dense_facts=[], + fetched_episodes=[], facts_for_episodes={"ep-2": [fact]}, ) result = await hierarchy_retrieve_episodes( query="test query", - sparse=[ep], + sparse=[], dense=[ep], query_vector=[0.1, 0.2, 0.3], fact_recaller=fact_recaller, @@ -275,4 +462,44 @@ class TestHierarchyRetrieveEpisodes: assert episode_item.atomic_facts != [] nested_fact = episode_item.atomic_facts[0] assert nested_fact.id == "fact-2" - assert nested_fact.score == pytest.approx(0.95) + # Evicted-fact score is its own LR-calibrated value (alpha default 1.0). + assert episode_item.score == pytest.approx(cosine_to_lr_score(0.95, 0.0)) + + async def test_min_score_filters_below_threshold(self) -> None: + ep = _episode_candidate(ep_id="ep-1", score=0.5, memcell_id="mc-1") + fact_recaller, episode_recaller = _make_recallers() + + produced = cosine_to_lr_score(0.5, 0.0) # episode-win score (sparse empty) + + result = await hierarchy_retrieve_episodes( + query="test query", + sparse=[], + dense=[ep], + query_vector=[0.1, 0.2, 0.3], + fact_recaller=fact_recaller, + episode_recaller=episode_recaller, + where="owner_id = 'u1'", + top_k=10, + min_score=produced + 0.05, + ) + + assert result == [] + + async def test_min_score_none_keeps_all(self) -> None: + ep = _episode_candidate(ep_id="ep-1", score=0.5, memcell_id="mc-1") + fact_recaller, episode_recaller = _make_recallers() + + result = await hierarchy_retrieve_episodes( + query="test query", + sparse=[], + dense=[ep], + query_vector=[0.1, 0.2, 0.3], + fact_recaller=fact_recaller, + episode_recaller=episode_recaller, + where="owner_id = 'u1'", + top_k=10, + min_score=None, + ) + + assert len(result) == 1 + assert result[0].id == "ep-1" diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index 93248fe..07b86e9 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -48,6 +48,7 @@ def _episode_row( "subject": f"subj {eid}", "summary": f"summary {eid}", "episode": f"body {eid}", + "entry_id": eid, "parent_id": memcell_id if memcell_id is not None else f"mc_{eid}", }, ) @@ -113,11 +114,15 @@ class _StubEpisodeRecaller: async def fetch_by_parent_ids( self, parent_ids: Sequence[str], where: str ) -> list[Candidate]: - # Index dense rows by their parent_id (memcell id) so the maxsim - # path's reverse-resolve has something to return. by_parent = {str(c.metadata.get("parent_id", "")): c for c in self._dense} return [by_parent[p] for p in parent_ids if p in by_parent] + async def fetch_by_entry_ids( + self, entry_ids: Sequence[str], where: str + ) -> list[Candidate]: + by_entry = {str(c.metadata.get("entry_id", "")): c for c in self._dense} + return [by_entry[e] for e in entry_ids if e in by_entry] + class _StubAtomicFactRecaller: kind: ClassVar[str] = "atomic_fact" @@ -140,16 +145,15 @@ class _StubAtomicFactRecaller: async def facts_for_episodes( self, - ep_to_memcell: Mapping[str, str], + ep_to_parents: Mapping[str, Sequence[str]], where: str, *, per_episode: int, query_vector: Any = None, ) -> dict[str, list[FactCandidate]]: - # ``query_vector`` accepted to match the real recaller signature # Accepted to match the real recaller signature; stub doesn't use it. return { - eid: self._facts_map.get(eid, [])[:per_episode] for eid in ep_to_memcell + eid: self._facts_map.get(eid, [])[:per_episode] for eid in ep_to_parents } @@ -428,8 +432,8 @@ async def test_vector_maxsim_atomic_max_pools_facts_to_episodes( monkeypatch: pytest.MonkeyPatch, ) -> None: """``vector_strategy=maxsim_atomic`` should ANN atomic_facts → max-pool by - memcell parent → reverse-resolve to episode, ordering episodes by the - per-memcell maximum fact score.""" + episode entry_id → resolve to episode, ordering episodes by the + per-episode maximum fact score.""" from everos.config.settings import load_settings monkeypatch.setenv("EVEROS_SEARCH__VECTOR_STRATEGY", "maxsim_atomic") @@ -442,9 +446,9 @@ async def test_vector_maxsim_atomic_max_pools_facts_to_episodes( _episode_row("ep_B", memcell_id="mc_B"), ], atomic_fact_dense=[ - _atomic_fact_row("f_A1", parent_id="mc_A", score=0.95), - _atomic_fact_row("f_A2", parent_id="mc_A", score=0.40), - _atomic_fact_row("f_B1", parent_id="mc_B", score=0.75), + _atomic_fact_row("f_A1", parent_id="ep_A", score=0.95), + _atomic_fact_row("f_A2", parent_id="ep_A", score=0.40), + _atomic_fact_row("f_B1", parent_id="ep_B", score=0.75), ], embedding=_StubEmbedding(), ) diff --git a/tests/unit/test_memory/test_search/test_recall_agent_skill.py b/tests/unit/test_memory/test_search/test_recall_agent_skill.py index 153efaf..9239fa3 100644 --- a/tests/unit/test_memory/test_search/test_recall_agent_skill.py +++ b/tests/unit/test_memory/test_search/test_recall_agent_skill.py @@ -65,7 +65,7 @@ def _skill_row( @pytest.fixture(autouse=True) async def _reset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """Isolate LanceDB under tmp memory root per test.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) lancedb_manager._conn = None lancedb_manager._tables.clear() yield diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index 0270388..f2e80b8 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -1,17 +1,18 @@ """Real-LanceDB tests for ``AtomicFactRecaller.facts_for_episodes``. -The MRAG bridge is the only path that links facts back to episodes, and +The memcell bridge is the only path that links facts back to episodes, and the previous ``parent_type='episode' AND parent_id IN (episode_ids)`` query never matched: cascade writes facts with ``parent_type='memcell'``, ``parent_id=memcell_id``. The fixed version -takes an ``episode → memcell`` map from the caller, queries by the -deduped memcell set, and re-buckets results under every episode that -shares each memcell. +takes an ``episode_id → [parent_id, ...]`` map from the caller (dual +parent_id: entry_id for post-1.5 facts, memcell_id for pre-1.5 facts), +queries by ``parent_id IN (all_parent_ids)`` and regroups by episode. These tests exercise the real LanceDB query path (no recaller stubs): -- shared memcell → fact appears under both episodes, -- distinct memcells → facts bucket exclusively to their owning episode, -- empty / unknown memcells → empty result, no LanceDB call surprise. +- shared parent → fact appears under both episodes, +- distinct parents → facts bucket exclusively to their owning episode, +- dual parent_id (entry_id + memcell_id) → facts from both eras found, +- empty / unknown parents → empty result, no LanceDB call surprise. """ from __future__ import annotations @@ -71,7 +72,7 @@ def _fact_row( @pytest.fixture(autouse=True) async def _reset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """Isolate LanceDB to a tmp memory root per test.""" - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) lancedb_manager._conn = None lancedb_manager._tables.clear() yield @@ -98,13 +99,13 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: ] ) - ep_to_memcell = { - "alice_ep_a": "mc_shared", - "alice_ep_b": "mc_shared", - "alice_ep_c": "mc_other", + ep_to_parents = { + "alice_ep_a": ["mc_shared"], + "alice_ep_b": ["mc_shared"], + "alice_ep_c": ["mc_other"], } where = "owner_id = 'alice' AND owner_type = 'user'" - out = await _recaller().facts_for_episodes(ep_to_memcell, where, per_episode=10) + out = await _recaller().facts_for_episodes(ep_to_parents, where, per_episode=10) assert sorted(out.keys()) == ["alice_ep_a", "alice_ep_b", "alice_ep_c"] assert sorted(f.id for f in out["alice_ep_a"]) == ["alice_af_1", "alice_af_2"] @@ -119,7 +120,9 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: async def test_facts_for_episodes_returns_empty_for_no_episodes() -> None: - out = await _recaller().facts_for_episodes({}, "owner_id = 'alice'", per_episode=10) + out: dict = await _recaller().facts_for_episodes( + {}, "owner_id = 'alice'", per_episode=10 + ) assert out == {} @@ -130,7 +133,7 @@ async def test_facts_for_episodes_skips_unknown_memcells() -> None: ) out = await _recaller().facts_for_episodes( - {"alice_ep_a": "mc_a", "alice_ep_b": "mc_missing"}, + {"alice_ep_a": ["mc_a"], "alice_ep_b": ["mc_missing"]}, "owner_id = 'alice' AND owner_type = 'user'", per_episode=10, ) @@ -159,15 +162,15 @@ async def test_facts_for_episodes_filters_by_where_clause() -> None: ) out = await _recaller().facts_for_episodes( - {"alice_ep_a": "mc_a"}, + {"alice_ep_a": ["mc_a"]}, "owner_id = 'alice' AND owner_type = 'user'", per_episode=10, ) assert [f.id for f in out["alice_ep_a"]] == ["alice_af_1"] -async def test_facts_for_episodes_drops_empty_memcell_ids() -> None: - """Episodes whose parent_id is missing (empty string) are dropped silently. +async def test_facts_for_episodes_drops_empty_parent_ids() -> None: + """Episodes whose parent_id list contains only empty strings are dropped. Real-world cause: a candidate row that lost its ``parent_id`` (data corruption, manual edit). The bridge must not crash and must not @@ -179,14 +182,14 @@ async def test_facts_for_episodes_drops_empty_memcell_ids() -> None: ) out = await _recaller().facts_for_episodes( - {"alice_ep_a": ""}, + {"alice_ep_a": [""]}, "owner_id = 'alice' AND owner_type = 'user'", per_episode=10, ) assert out == {} -# ── MRAG fact-level scoring (regression for query_vector handling) ───── +# ── Hierarchical fact-level scoring (regression for query_vector handling) ── def _unit_vector(direction: int, dim: int = 1024) -> list[float]: @@ -206,7 +209,7 @@ async def test_facts_for_episodes_assigns_real_cosine_score_with_query_vector() """Regression: ``query_vector`` triggers cosine ANN, not flat scan. Pre-fix, ``facts_for_episodes`` only ran ``where parent_id IN (...)`` - and emitted every fact with ``score=0.0`` — the MRAG fact-level + and emitted every fact with ``score=0.0`` — the hierarchical fact-level ranking collapsed to insertion order. Post-fix, ``query_vector`` flows into ``.nearest_to(...).distance_type('cosine')`` and each fact lands with its real query↔fact relevance score. @@ -227,7 +230,7 @@ async def test_facts_for_episodes_assigns_real_cosine_score_with_query_vector() await atomic_fact_repo.upsert([row_a, row_b]) out = await _recaller().facts_for_episodes( - {"alice_ep_a": "mc_shared"}, + {"alice_ep_a": ["mc_shared"]}, "owner_id = 'alice' AND owner_type = 'user'", per_episode=10, query_vector=_unit_vector(0), @@ -255,10 +258,70 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: await atomic_fact_repo.upsert([row]) out = await _recaller().facts_for_episodes( - {"alice_ep_a": "mc_a"}, + {"alice_ep_a": ["mc_a"]}, "owner_id = 'alice' AND owner_type = 'user'", per_episode=10, # no query_vector ) assert out["alice_ep_a"][0].score == 0.0 + + +# ── Dual parent_id (post-1.5 migration) ──────────────────────────────── + + +async def test_facts_for_episodes_dual_parent_id_finds_both_eras() -> None: + """Dual parent_id: facts linked by entry_id AND memcell_id are both found. + + Post-1.5 facts use ``parent_id = episode_entry_id``; pre-1.5 facts + use ``parent_id = memcell_id``. The caller provides both candidate + parent_ids in the list, and facts from both eras appear in the + result bucket. + """ + await atomic_fact_repo.upsert( + [ + _fact_row(fid="alice_af_old", memcell_id="mc_1", fact="old era fact"), + _fact_row(fid="alice_af_new", memcell_id="ep_entry_1", fact="new era fact"), + ] + ) + + out = await _recaller().facts_for_episodes( + {"alice_ep_a": ["ep_entry_1", "mc_1"]}, + "owner_id = 'alice' AND owner_type = 'user'", + per_episode=10, + ) + + assert sorted(f.id for f in out["alice_ep_a"]) == [ + "alice_af_new", + "alice_af_old", + ] + + +async def test_facts_for_episodes_multiple_parent_ids_dedup_across_episodes() -> None: + """Two episodes sharing one parent_id each see the same fact pool. + + Episode A has [ep_entry_1, mc_shared]; Episode B has [ep_entry_2, mc_shared]. + A fact with parent_id=mc_shared should appear under both episodes. + """ + await atomic_fact_repo.upsert( + [ + _fact_row(fid="alice_af_1", memcell_id="mc_shared", fact="shared fact"), + _fact_row(fid="alice_af_2", memcell_id="ep_entry_1", fact="ep_a only fact"), + ] + ) + + out = await _recaller().facts_for_episodes( + { + "alice_ep_a": ["ep_entry_1", "mc_shared"], + "alice_ep_b": ["ep_entry_2", "mc_shared"], + }, + "owner_id = 'alice' AND owner_type = 'user'", + per_episode=10, + ) + + # Both episodes get the shared fact + assert "alice_af_1" in {f.id for f in out["alice_ep_a"]} + assert "alice_af_1" in {f.id for f in out["alice_ep_b"]} + # Only ep_a gets the ep_entry_1-linked fact + assert "alice_af_2" in {f.id for f in out["alice_ep_a"]} + assert "alice_af_2" not in {f.id for f in out.get("alice_ep_b", [])} diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 3471448..f579e72 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -1,4 +1,4 @@ -"""Unit tests for ``EpisodeRecaller.fetch_all_for_owner``.""" +"""Unit tests for ``EpisodeRecaller.fetch_all_for_owner`` and ``fetch_by_entry_ids``.""" from __future__ import annotations @@ -12,7 +12,13 @@ from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.episode import EpisodeRecaller -def _make_row(ep_id: str, mc_id: str) -> dict[str, Any]: +def _make_row( + ep_id: str, + mc_id: str, + *, + parent_type: str = "memcell", + entry_id: str = "", +) -> dict[str, Any]: """Build a minimal episode LanceDB row dict for test fixtures.""" return { "id": ep_id, @@ -25,6 +31,8 @@ def _make_row(ep_id: str, mc_id: str) -> dict[str, Any]: "summary": f"summary {ep_id}", "episode": f"body {ep_id}", "parent_id": mc_id, + "parent_type": parent_type, + "entry_id": entry_id or ep_id, } @@ -41,10 +49,10 @@ def recaller() -> EpisodeRecaller: return EpisodeRecaller(RecallerDeps(tokenizer=tok)) -async def test_fetch_all_for_owner_returns_memcell_keyed_candidates( +async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( recaller: EpisodeRecaller, ) -> None: - """id must equal parent_id (memcell_id) so acluster_retrieve membership works.""" + """id must equal entry_id so acluster_retrieve membership works.""" rows = [ _make_row("ep_1", "mc_1"), _make_row("ep_2", "mc_2"), @@ -58,7 +66,7 @@ async def test_fetch_all_for_owner_returns_memcell_keyed_candidates( assert len(result) == 2 ids = {c.id for c in result} - assert ids == {"mc_1", "mc_2"}, "id must be memcell_id, not episode_id" + assert ids == {"ep_1", "ep_2"}, "id must be entry_id" async def test_fetch_all_for_owner_stores_episode_id_in_metadata( @@ -77,13 +85,10 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( assert result[0].metadata["parent_id"] == "mc_xyz" -async def test_fetch_all_for_owner_skips_rows_without_parent_id( +async def test_fetch_all_for_owner_skips_rows_without_entry_id( recaller: EpisodeRecaller, ) -> None: - """Rows without parent_id are silently skipped. - - They are incomplete episode records. - """ + """Rows without entry_id are silently skipped.""" rows = [ { "id": "ep_bad", @@ -95,7 +100,7 @@ async def test_fetch_all_for_owner_skips_rows_without_parent_id( "subject": "", "summary": "", "episode": "", - # no parent_id key + "parent_id": "mc_x", }, ] with patch( @@ -106,3 +111,92 @@ async def test_fetch_all_for_owner_skips_rows_without_parent_id( result = await recaller.fetch_all_for_owner("owner_id = 'alice'") assert result == [] + + +async def test_fetch_all_for_owner_merged_episode_uses_entry_id( + recaller: EpisodeRecaller, +) -> None: + """Merged episodes (parent_type=cluster) must use entry_id as Candidate.id. + + This ensures acluster_retrieve membership matching works for + member_type=episode cluster members whose member_id is the episode's + entry_id, not the cluster_id stored in parent_id. + """ + rows = [ + _make_row( + "ep_merged", + "cluster_abc", + parent_type="cluster", + entry_id="entry_xyz", + ), + ] + with patch( + "everos.memory.search.recall.episode.get_table", + new_callable=AsyncMock, + return_value=_mock_table(rows), + ): + result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + + assert len(result) == 1 + assert result[0].id == "entry_xyz", "merged episode id must be entry_id" + assert result[0].metadata["episode_id"] == "ep_merged" + + +async def test_fetch_all_for_owner_mixed_regular_and_merged( + recaller: EpisodeRecaller, +) -> None: + """Mixed rows: both regular and merged episodes key by entry_id.""" + rows = [ + _make_row("ep_regular", "mc_1"), + _make_row( + "ep_merged", + "cluster_99", + parent_type="cluster", + entry_id="entry_42", + ), + ] + with patch( + "everos.memory.search.recall.episode.get_table", + new_callable=AsyncMock, + return_value=_mock_table(rows), + ): + result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + + assert len(result) == 2 + ids = {c.id for c in result} + assert ids == {"ep_regular", "entry_42"} + + +async def test_fetch_by_entry_ids_returns_candidates( + recaller: EpisodeRecaller, +) -> None: + """fetch_by_entry_ids queries by entry_id and returns valid candidates.""" + rows = [ + _make_row( + "ep_merged", + "cluster_abc", + parent_type="cluster", + entry_id="entry_xyz", + ), + ] + mock_tbl = MagicMock() + mock_tbl.query.return_value.where.return_value.limit.return_value.to_list = ( + AsyncMock(return_value=rows) + ) + with patch( + "everos.memory.search.recall.episode.get_table", + new_callable=AsyncMock, + return_value=mock_tbl, + ): + result = await recaller.fetch_by_entry_ids(["entry_xyz"], "owner_id = 'alice'") + + assert len(result) == 1 + assert result[0].id == "ep_merged" + + +async def test_fetch_by_entry_ids_empty_input_returns_empty( + recaller: EpisodeRecaller, +) -> None: + """Empty entry_ids list short-circuits without querying.""" + result = await recaller.fetch_by_entry_ids([], "owner_id = 'alice'") + assert result == [] diff --git a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py new file mode 100644 index 0000000..d40d8a4 --- /dev/null +++ b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py @@ -0,0 +1,227 @@ +"""Unit tests for ``KnowledgeTopicRecaller``. + +Verifies dual-column BM25 + cosine ANN recall, using ``unittest.mock`` +to patch ``get_table`` so no real LanceDB connection is needed. + +White-box surfaces touched: + - ``everos.memory.search.recall.knowledge_topic.get_table`` (patched) + - ``KnowledgeTopicRecaller.sparse_recall`` — queries both BM25 columns + - ``KnowledgeTopicRecaller.dense_recall`` — cosine ANN with distance→score +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from everos.component.tokenizer import Tokenizer +from everos.memory.search.recall.base import RecallerDeps +from everos.memory.search.recall.knowledge_topic import KnowledgeTopicRecaller + +_MODULE = "everos.memory.search.recall.knowledge_topic" + + +class _WhitespaceTokenizer(Tokenizer): + """Splits on whitespace — predictable token output for assertions.""" + + def tokenize(self, text: str) -> list[str]: + return text.split() + + +def _make_row( + rid: str, *, score: float = 1.0, distance: float | None = None +) -> dict[str, Any]: + """Build a minimal LanceDB row dict.""" + row: dict[str, Any] = { + "id": rid, + "app_id": "app", + "project_id": "proj", + "doc_id": "doc_1", + "category_id": "cat_1", + "topic_name": f"Topic {rid}", + "topic_path": f"/root/{rid}", + "depth": 1, + "parent_node_id": "", + "summary": f"Summary of {rid}", + "summary_tokens": f"summary {rid}", + "content_tokens": f"content {rid}", + "content_labels": [], + "md_path": f"knowledge/default/{rid}.md", + "content_sha256": "a" * 64, + } + if distance is not None: + row["_distance"] = distance + else: + row["_score"] = score + return row + + +def _mock_bm25_table( + summary_rows: list[dict[str, Any]], + content_rows: list[dict[str, Any]], +) -> MagicMock: + """Build a table mock whose BM25 results differ per column. + + The first ``nearest_to_text`` call (summary_tokens) returns + ``summary_rows``; the second (content_tokens) returns ``content_rows``. + ``asyncio.gather`` fires both concurrently, so we use ``side_effect`` + on the chain rather than recording call order. + """ + summary_chain = MagicMock() + summary_chain.where.return_value.limit.return_value.to_list = AsyncMock( + return_value=summary_rows + ) + + content_chain = MagicMock() + content_chain.where.return_value.limit.return_value.to_list = AsyncMock( + return_value=content_rows + ) + + tbl = MagicMock() + tbl.query.return_value.nearest_to_text.side_effect = [summary_chain, content_chain] + return tbl + + +def _mock_ann_table(rows: list[dict[str, Any]]) -> MagicMock: + """Build a table mock for ANN (dense) queries.""" + tbl = MagicMock() + ann = tbl.query.return_value.nearest_to.return_value + chain = ann.distance_type.return_value.where.return_value.limit.return_value + chain.to_list = AsyncMock(return_value=rows) + return tbl + + +@pytest.fixture() +def recaller() -> KnowledgeTopicRecaller: + return KnowledgeTopicRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) + + +_WHERE = "app_id = 'app' AND project_id = 'proj'" + + +# --------------------------------------------------------------------------- +# sparse_recall — dual-column BM25 +# --------------------------------------------------------------------------- + + +async def test_sparse_recall_queries_both_columns( + recaller: KnowledgeTopicRecaller, +) -> None: + """``nearest_to_text`` must be called once per BM25 column.""" + tbl = _mock_bm25_table( + summary_rows=[_make_row("t1", score=0.9)], + content_rows=[_make_row("t2", score=0.7)], + ) + with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + result = await recaller.sparse_recall("topic query", _WHERE, limit=10) + + # nearest_to_text called twice (once per column) + assert tbl.query.return_value.nearest_to_text.call_count == 2 + ids = {c.id for c in result} + assert ids == {"t1", "t2"} + + +async def test_sparse_recall_merges_by_max_score( + recaller: KnowledgeTopicRecaller, +) -> None: + """When the same id appears in both columns, keep the higher score.""" + shared_id = "topic_shared" + summary_rows = [_make_row(shared_id, score=0.5)] + content_rows = [_make_row(shared_id, score=0.9)] + + tbl = _mock_bm25_table(summary_rows, content_rows) + with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + result = await recaller.sparse_recall("overlap", _WHERE, limit=10) + + assert len(result) == 1 + assert result[0].id == shared_id + assert result[0].score == pytest.approx(0.9) + assert result[0].source == "keyword" + + +async def test_sparse_recall_returns_sorted_by_score( + recaller: KnowledgeTopicRecaller, +) -> None: + """Merged results must be sorted descending by score, truncated to limit.""" + summary_rows = [ + _make_row("a", score=0.3), + _make_row("b", score=0.8), + ] + content_rows = [ + _make_row("c", score=0.6), + ] + tbl = _mock_bm25_table(summary_rows, content_rows) + with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + result = await recaller.sparse_recall("query", _WHERE, limit=2) + + assert len(result) == 2 + assert result[0].id == "b" + assert result[1].id == "c" + + +async def test_sparse_recall_empty_query_returns_empty( + recaller: KnowledgeTopicRecaller, +) -> None: + """Empty tokenisation short-circuits — no LanceDB query is issued.""" + tok = MagicMock(spec=Tokenizer) + tok.tokenize.return_value = [] + r = KnowledgeTopicRecaller(RecallerDeps(tokenizer=tok)) + + with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + result = await r.sparse_recall("", _WHERE, limit=10) + + assert result == [] + mock_gt.assert_not_called() + + +# --------------------------------------------------------------------------- +# dense_recall — cosine ANN +# --------------------------------------------------------------------------- + + +async def test_dense_recall_cosine_conversion( + recaller: KnowledgeTopicRecaller, +) -> None: + """``_distance`` is converted to similarity: score = 1.0 - distance.""" + rows = [ + _make_row("t1", distance=0.2), + _make_row("t2", distance=0.5), + ] + tbl = _mock_ann_table(rows) + with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=10) + + assert len(result) == 2 + scores = {c.id: c.score for c in result} + assert scores["t1"] == pytest.approx(0.8) + assert scores["t2"] == pytest.approx(0.5) + assert all(c.source == "vector" for c in result) + + +async def test_dense_recall_empty_vector_returns_empty( + recaller: KnowledgeTopicRecaller, +) -> None: + """Empty vector short-circuits — no LanceDB query is issued.""" + with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + result = await recaller.dense_recall([], _WHERE, limit=10) + + assert result == [] + mock_gt.assert_not_called() + + +async def test_dense_recall_metadata_excludes_noise_columns( + recaller: KnowledgeTopicRecaller, +) -> None: + """``vector`` and ``_distance`` must not appear in ``Candidate.metadata``.""" + row = _make_row("t1", distance=0.3) + row["vector"] = [0.0] * 1024 + + tbl = _mock_ann_table([row]) + with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=5) + + assert len(result) == 1 + assert "vector" not in result[0].metadata + assert "_distance" not in result[0].metadata diff --git a/tests/unit/test_memory/test_search/test_recall_or_semantics.py b/tests/unit/test_memory/test_search/test_recall_or_semantics.py index fb176e3..3e35cde 100644 --- a/tests/unit/test_memory/test_search/test_recall_or_semantics.py +++ b/tests/unit/test_memory/test_search/test_recall_or_semantics.py @@ -76,7 +76,7 @@ def _episode_row( @pytest.fixture(autouse=True) async def _reset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) lancedb_manager._conn = None lancedb_manager._tables.clear() yield diff --git a/tests/unit/test_memory/test_search/test_recall_profile.py b/tests/unit/test_memory/test_search/test_recall_profile.py index df84b71..0563d00 100644 --- a/tests/unit/test_memory/test_search/test_recall_profile.py +++ b/tests/unit/test_memory/test_search/test_recall_profile.py @@ -44,7 +44,7 @@ def _profile_row( @pytest.fixture(autouse=True) async def _reset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) lancedb_manager._conn = None lancedb_manager._tables.clear() yield diff --git a/tests/unit/test_memory/test_strategies/test_extract_agent_case.py b/tests/unit/test_memory/test_strategies/test_extract_agent_case.py index 2c8885b..42642ae 100644 --- a/tests/unit/test_memory/test_strategies/test_extract_agent_case.py +++ b/tests/unit/test_memory/test_strategies/test_extract_agent_case.py @@ -68,7 +68,7 @@ def _algo_case( async def test_strategy_meta_is_attached() -> None: - meta = extract_agent_case._ome_strategy_meta # type: ignore[attr-defined] + meta = extract_agent_case.meta assert meta.name == "extract_agent_case" assert AgentPipelineStarted in meta.trigger.on assert meta.emits == frozenset({AgentCaseExtracted}) diff --git a/tests/unit/test_memory/test_strategies/test_extract_agent_skill.py b/tests/unit/test_memory/test_strategies/test_extract_agent_skill.py index 7b584f4..84abf5b 100644 --- a/tests/unit/test_memory/test_strategies/test_extract_agent_skill.py +++ b/tests/unit/test_memory/test_strategies/test_extract_agent_skill.py @@ -26,12 +26,12 @@ from everalgo.clustering import Cluster as AlgoCluster from everalgo.types import AgentSkill as AlgoAgentSkill from everos.component.embedding import ( - EmbeddingError, EmbeddingNotConfiguredError, + EmbeddingServiceError, ) from everos.infra.ome.testing import FakeStrategyContext +from everos.memory._partition_locks import _reset_for_tests from everos.memory.events import SkillClusterUpdated -from everos.memory.strategies._partition_locks import _reset_for_tests from everos.memory.strategies.extract_agent_skill import ( MAX_SKILLS_IN_PROMPT, MAX_SUPPORTING_CASES, @@ -135,7 +135,7 @@ def _algo_skill(name: str = "summarise_doc") -> AlgoAgentSkill: async def test_strategy_meta_is_attached() -> None: - meta = extract_agent_skill._ome_strategy_meta # type: ignore[attr-defined] + meta = extract_agent_skill.meta assert meta.name == "extract_agent_skill" assert SkillClusterUpdated in meta.trigger.on assert meta.emits == frozenset() @@ -328,7 +328,7 @@ async def test_select_existing_skills_falls_back_to_scalar_when_embed_fails() -> scalar_skills = [_lance_skill(name=f"s{i}") for i in range(MAX_SKILLS_IN_PROMPT)] mock_embedder = MagicMock() - mock_embedder.embed = AsyncMock(side_effect=EmbeddingError("provider down")) + mock_embedder.embed = AsyncMock(side_effect=EmbeddingServiceError("provider down")) with ( patch( diff --git a/tests/unit/test_memory/test_strategies/test_extract_atomic_facts.py b/tests/unit/test_memory/test_strategies/test_extract_atomic_facts.py index d0e0cd8..09ab3cb 100644 --- a/tests/unit/test_memory/test_strategies/test_extract_atomic_facts.py +++ b/tests/unit/test_memory/test_strategies/test_extract_atomic_facts.py @@ -1,86 +1,58 @@ from __future__ import annotations import importlib +from collections.abc import Mapping from unittest.mock import AsyncMock, patch import pytest import structlog.testing -from everalgo.types import AtomicFact, ChatMessage, MemCell +from everalgo.types import AtomicFact from everos.infra.ome.testing import FakeStrategyContext -from everos.memory.events import UserPipelineStarted +from everos.memory.events import EpisodeExtracted from everos.memory.strategies.extract_atomic_facts import extract_atomic_facts mod = importlib.import_module("everos.memory.strategies.extract_atomic_facts") -def _two_user_memcell() -> MemCell: - return MemCell( - items=[ - ChatMessage( - id="m1", - role="user", - content="hi from alice", - timestamp=1_700_000_000_000, - sender_id="u_alice", - ), - ChatMessage( - id="m2", - role="user", - content="hi from bob", - timestamp=1_700_000_001_000, - sender_id="u_bob", - ), - ChatMessage( - id="m3", - role="assistant", - content="hello both", - timestamp=1_700_000_002_000, - sender_id="agent", - ), - ], - timestamp=1_700_000_002_000, - ) +def _fact(text: str) -> AtomicFact: + return AtomicFact(owner_id=None, content=text, timestamp=1_700_000_000_000) -def _fact(owner_id: str | None, text: str) -> AtomicFact: - return AtomicFact(owner_id=owner_id, content=text, timestamp=1_700_000_000_000) - - -def _event() -> UserPipelineStarted: - return UserPipelineStarted( - memcell_id="mc_a", session_id="s1", memcell=_two_user_memcell() +def _event( + *, + owner_id: str = "u_alice", + memcell_id: str = "mc_a", + session_id: str = "s1", + episode_text: str = "alice likes hiking and lives in tokyo", + episode_timestamp_ms: int = 1_700_000_000_000, +) -> EpisodeExtracted: + return EpisodeExtracted( + memcell_id=memcell_id, + episode_entry_id="ep_20260517_0001", + episode_text=episode_text, + episode_timestamp_ms=episode_timestamp_ms, + owner_id=owner_id, + session_id=session_id, ) async def test_strategy_meta_is_attached() -> None: - meta = extract_atomic_facts._ome_strategy_meta # type: ignore[attr-defined] + meta = extract_atomic_facts.meta assert meta.name == "extract_atomic_facts" - assert UserPipelineStarted in meta.trigger.on + assert EpisodeExtracted in meta.trigger.on assert meta.emits == frozenset() assert meta.max_retries == 2 -async def test_extracts_once_and_fans_out_per_sender( +async def test_extracts_from_episode_text_and_writes_under_event_owner( monkeypatch: pytest.MonkeyPatch, ) -> None: - """One LLM call per memcell; same fact list re-written under each sender. - - The algo prompt is subject-agnostic (only ``INPUT_TEXT`` + ``TIME`` - placeholders), so re-running it per sender would burn LLM tokens - and let non-determinism drift the per-sender md files apart. The - strategy calls ``aextract`` once with ``sender_id=None`` and - broadcasts the resulting list — every user sender gets its own md - entries pointing at the same fact bodies. - - Per-owner batching: the strategy collects each sender's full fact - list and issues one :meth:`append_entries` per owner (not N single - appends), so the call shape is one batch call per sender. - """ + """Single LLM call on episode_text; all facts written under event.owner_id.""" monkeypatch.setattr(mod, "_writer", None, raising=False) generic_facts = [ - _fact(None, "alice mentioned a weekend trip to tokyo"), - _fact(None, "bob said he needs hiking gear"), + _fact("alice mentioned a weekend trip to tokyo"), + _fact("alice said she needs hiking gear"), ] with ( @@ -96,108 +68,46 @@ async def test_extracts_once_and_fans_out_per_sender( ) as mock_wcls, structlog.testing.capture_logs() as captured, ): - mock_cls.return_value.aextract = AsyncMock(return_value=generic_facts) + mock_cls.return_value.aextract_from_text = AsyncMock(return_value=generic_facts) mock_wcls.return_value.append_entries = AsyncMock(return_value=[]) await extract_atomic_facts(_event(), FakeStrategyContext()) - # Exactly one LLM call, parameterised with sender_id=None. - assert mock_cls.return_value.aextract.await_count == 1 - call = mock_cls.return_value.aextract.call_args - assert call.kwargs["sender_id"] is None + # Exactly one LLM call with the episode text. + assert mock_cls.return_value.aextract_from_text.await_count == 1 + call = mock_cls.return_value.aextract_from_text.call_args + assert call.args[0] == "alice likes hiking and lives in tokyo" + assert call.kwargs["timestamp"] == 1_700_000_000_000 - # 2 senders → 2 batch calls; each batch carries this sender's 2 facts - # (same generic body re-used). - assert mock_wcls.return_value.append_entries.call_count == 2 - batch_calls = mock_wcls.return_value.append_entries.call_args_list - batched_owners = sorted(c.args[0] for c in batch_calls) - assert batched_owners == ["u_alice", "u_bob"] - # Flatten items across batches: (owner, fact_text) pairs. - flat = sorted( - (c.args[0], sections["Fact"]) - for c in batch_calls - for inline, sections in c.args[1] - ) - assert flat == [ - ("u_alice", "alice mentioned a weekend trip to tokyo"), - ("u_alice", "bob said he needs hiking gear"), - ("u_bob", "alice mentioned a weekend trip to tokyo"), - ("u_bob", "bob said he needs hiking gear"), + # Single owner → one batch call with 2 facts. + assert mock_wcls.return_value.append_entries.call_count == 1 + batch_call = mock_wcls.return_value.append_entries.call_args + assert batch_call.args[0] == "u_alice" + items: list[tuple[Mapping, Mapping]] = batch_call.args[1] + assert len(items) == 2 + + for inline, _sections in items: + assert inline["owner_id"] == "u_alice" + assert inline["session_id"] == "s1" + assert inline["parent_type"] == "episode" + assert inline["parent_id"] == "ep_20260517_0001" + + fact_texts = sorted(sections["Fact"] for _, sections in items) + assert fact_texts == [ + "alice mentioned a weekend trip to tokyo", + "alice said she needs hiking gear", ] matching = [e for e in captured if e.get("event") == "atomic_facts_extracted"] assert matching, "expected atomic_facts_extracted log line" record = matching[0] - assert record["count"] == 4 - assert sorted(record["owner_ids"]) == ["u_alice", "u_bob"] + assert record["count"] == 2 + assert record["owner_id"] == "u_alice" -async def test_writes_md_for_each_fact( +async def test_skips_when_extractor_returns_empty( monkeypatch: pytest.MonkeyPatch, ) -> None: - facts = [ - _fact("u_alice", "alice likes hiking"), - _fact("u_alice", "alice lives in tokyo"), - ] - - monkeypatch.setattr(mod, "_writer", None, raising=False) - with ( - patch( - "everos.memory.strategies.extract_atomic_facts.get_llm_client", - return_value=object(), - ), - patch( - "everos.memory.strategies.extract_atomic_facts.AtomicFactExtractor" - ) as mock_cls, - patch( - "everos.memory.strategies.extract_atomic_facts.AtomicFactWriter" - ) as mock_wcls, - ): - mock_cls.return_value.aextract = AsyncMock(return_value=facts) - mock_wcls.return_value.append_entries = AsyncMock(return_value=[]) - - event = UserPipelineStarted( - memcell_id="mc_a", - session_id="s1", - memcell=MemCell( - items=[ - ChatMessage( - id="m1", - role="user", - content="hi", - timestamp=1_700_000_000_000, - sender_id="u_alice", - ) - ], - timestamp=1_700_000_000_000, - ), - ) - await extract_atomic_facts(event, FakeStrategyContext()) - - # Single sender (u_alice) → one batch call with 2 items. - assert mock_wcls.return_value.append_entries.call_count == 1 - batch_call = mock_wcls.return_value.append_entries.call_args - assert batch_call.args[0] == "u_alice" - items = batch_call.args[1] - assert len(items) == 2 - for (inline, sections), fact in zip(items, facts, strict=True): - assert inline["owner_id"] == "u_alice" - assert inline["session_id"] == "s1" - assert inline["parent_type"] == "memcell" - assert inline["parent_id"] == "mc_a" - assert "sender_ids" not in inline - assert sections == {"Fact": fact.content} - - -async def test_skips_when_memcell_has_no_messages( - monkeypatch: pytest.MonkeyPatch, -) -> None: - event = UserPipelineStarted( - memcell_id="mc_b", - session_id="s1", - memcell=MemCell(items=[], timestamp=1_700_000_000_000), - ) - monkeypatch.setattr(mod, "_writer", None, raising=False) with ( patch( @@ -212,12 +122,40 @@ async def test_skips_when_memcell_has_no_messages( ) as mock_wcls, structlog.testing.capture_logs() as captured, ): - mock_cls.return_value.aextract = AsyncMock(return_value=[]) + mock_cls.return_value.aextract_from_text = AsyncMock(return_value=[]) mock_wcls.return_value.append_entries = AsyncMock(return_value=[]) - ctx = FakeStrategyContext() - await extract_atomic_facts(event, ctx) + await extract_atomic_facts(_event(), FakeStrategyContext()) matching = [e for e in captured if e.get("event") == "atomic_facts_extracted"] assert matching, "log line should still fire (count=0)" assert matching[0]["count"] == 0 mock_wcls.return_value.append_entries.assert_not_called() + + +async def test_passes_app_id_and_project_id_to_writer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(mod, "_writer", None, raising=False) + facts = [_fact("some fact")] + + with ( + patch( + "everos.memory.strategies.extract_atomic_facts.get_llm_client", + return_value=object(), + ), + patch( + "everos.memory.strategies.extract_atomic_facts.AtomicFactExtractor" + ) as mock_cls, + patch( + "everos.memory.strategies.extract_atomic_facts.AtomicFactWriter" + ) as mock_wcls, + ): + mock_cls.return_value.aextract_from_text = AsyncMock(return_value=facts) + mock_wcls.return_value.append_entries = AsyncMock(return_value=[]) + + event = _event() + await extract_atomic_facts(event, FakeStrategyContext()) + + batch_call = mock_wcls.return_value.append_entries.call_args + assert batch_call.kwargs["app_id"] == "default" + assert batch_call.kwargs["project_id"] == "default" diff --git a/tests/unit/test_memory/test_strategies/test_extract_foresight.py b/tests/unit/test_memory/test_strategies/test_extract_foresight.py index fc036e4..065064a 100644 --- a/tests/unit/test_memory/test_strategies/test_extract_foresight.py +++ b/tests/unit/test_memory/test_strategies/test_extract_foresight.py @@ -59,7 +59,7 @@ def _event() -> UserPipelineStarted: async def test_strategy_meta_is_attached() -> None: - meta = extract_foresight._ome_strategy_meta # type: ignore[attr-defined] + meta = extract_foresight.meta assert meta.name == "extract_foresight" assert UserPipelineStarted in meta.trigger.on assert meta.emits == frozenset() diff --git a/tests/unit/test_memory/test_strategies/test_extract_user_profile.py b/tests/unit/test_memory/test_strategies/test_extract_user_profile.py index 7b6cfad..3b756b1 100644 --- a/tests/unit/test_memory/test_strategies/test_extract_user_profile.py +++ b/tests/unit/test_memory/test_strategies/test_extract_user_profile.py @@ -20,8 +20,8 @@ from everalgo.types import Profile as AlgoProfile from everos.infra.ome.testing import FakeStrategyContext from everos.infra.persistence.markdown import UserProfileFrontmatter +from everos.memory._partition_locks import _reset_for_tests from everos.memory.events import ProfileClusterUpdated -from everos.memory.strategies._partition_locks import _reset_for_tests from everos.memory.strategies.extract_user_profile import extract_user_profile @@ -54,6 +54,15 @@ def _algo_cluster(*, cluster_id: str, members: list[str], last_ts: int) -> AlgoC ) +def _episode_row(entry_id: str, parent_id: str) -> MagicMock: + """Stand-in for a LanceDB Episode row with parent_type=memcell.""" + row = MagicMock() + row.entry_id = entry_id + row.parent_type = "memcell" + row.parent_id = parent_id + return row + + def _memcell_row(memcell_id: str, *, sender_id: str, ts_ms: int) -> MagicMock: """Stand-in for a sqlite Memcell row — only ``payload_json`` is read.""" cell = MemCell( @@ -75,7 +84,7 @@ def _memcell_row(memcell_id: str, *, sender_id: str, ts_ms: int) -> MagicMock: async def test_strategy_meta_is_attached() -> None: - meta = extract_user_profile._ome_strategy_meta # type: ignore[attr-defined] + meta = extract_user_profile.meta assert meta.name == "extract_user_profile" assert ProfileClusterUpdated in meta.trigger.on assert meta.emits == frozenset() @@ -89,10 +98,11 @@ async def test_init_mode_writes_profile_when_no_existing( """No prior profile → ProfileExtractor invoked without ``old_profile``.""" cluster = _algo_cluster( cluster_id="cl_user00000001", - members=["mc_aaaaaaaaaaa1"], + members=["ep_20260101_0001"], last_ts=1_700_000_001_000, ) - rows = [ + ep_rows = [_episode_row("ep_20260101_0001", "mc_aaaaaaaaaaa1")] + mc_rows = [ _memcell_row("mc_aaaaaaaaaaa1", sender_id="u_alice", ts_ms=1_700_000_001_000) ] new_profile = AlgoProfile.model_validate( @@ -109,6 +119,9 @@ async def test_init_mode_writes_profile_when_no_existing( patch( "everos.memory.strategies.extract_user_profile.cluster_repo" ) as mock_cluster_repo, + patch( + "everos.memory.strategies.extract_user_profile.episode_repo" + ) as mock_episode_repo, patch( "everos.memory.strategies.extract_user_profile.memcell_repo" ) as mock_memcell_repo, @@ -127,7 +140,8 @@ async def test_init_mode_writes_profile_when_no_existing( ) as mock_writer_cls, ): mock_cluster_repo.list_for_owner = AsyncMock(return_value=[cluster]) - mock_memcell_repo.find_by_ids = AsyncMock(return_value=rows) + mock_episode_repo.find_by_owner_entries = AsyncMock(return_value=ep_rows) + mock_memcell_repo.find_by_ids = AsyncMock(return_value=mc_rows) mock_reader_cls.return_value.read = AsyncMock(return_value=None) mock_writer_cls.return_value.write = AsyncMock(return_value=None) mock_extractor_cls.return_value.aextract = AsyncMock(return_value=new_profile) @@ -162,10 +176,11 @@ async def test_update_mode_rehydrates_old_profile( """Existing profile → algo Profile rehydrated and passed as old_profile.""" cluster = _algo_cluster( cluster_id="cl_user00000001", - members=["mc_aaaaaaaaaaa1"], + members=["ep_20260101_0001"], last_ts=1_700_000_002_000, ) - rows = [ + ep_rows = [_episode_row("ep_20260101_0001", "mc_aaaaaaaaaaa1")] + mc_rows = [ _memcell_row("mc_aaaaaaaaaaa1", sender_id="u_alice", ts_ms=1_700_000_002_000) ] existing_fm = UserProfileFrontmatter( @@ -190,6 +205,9 @@ async def test_update_mode_rehydrates_old_profile( patch( "everos.memory.strategies.extract_user_profile.cluster_repo" ) as mock_cluster_repo, + patch( + "everos.memory.strategies.extract_user_profile.episode_repo" + ) as mock_episode_repo, patch( "everos.memory.strategies.extract_user_profile.memcell_repo" ) as mock_memcell_repo, @@ -208,7 +226,8 @@ async def test_update_mode_rehydrates_old_profile( ) as mock_writer_cls, ): mock_cluster_repo.list_for_owner = AsyncMock(return_value=[cluster]) - mock_memcell_repo.find_by_ids = AsyncMock(return_value=rows) + mock_episode_repo.find_by_owner_entries = AsyncMock(return_value=ep_rows) + mock_memcell_repo.find_by_ids = AsyncMock(return_value=mc_rows) mock_reader_cls.return_value.read = AsyncMock( return_value=(existing_fm, "prior summary") ) @@ -238,7 +257,7 @@ async def test_skips_when_no_members(monkeypatch: pytest.MonkeyPatch) -> None: # to a non-existent value to drop everything. stale_cluster = _algo_cluster( cluster_id="cl_other000001", - members=["mc_other00000"], + members=["ep_other00000"], last_ts=1_600_000_000_000, ) existing_fm = UserProfileFrontmatter( @@ -308,16 +327,19 @@ async def _run_serialisation_probe( ) cluster_a = _algo_cluster( - cluster_id="cl_a", members=["mc_a"], last_ts=1_700_000_000_000 + cluster_id="cl_a", members=["ep_a"], last_ts=1_700_000_000_000 ) cluster_b = _algo_cluster( - cluster_id="cl_b", members=["mc_b"], last_ts=1_700_000_000_000 + cluster_id="cl_b", members=["ep_b"], last_ts=1_700_000_000_000 ) with ( patch( "everos.memory.strategies.extract_user_profile.cluster_repo" ) as mock_cluster_repo, + patch( + "everos.memory.strategies.extract_user_profile.episode_repo" + ) as mock_episode_repo, patch( "everos.memory.strategies.extract_user_profile.memcell_repo" ) as mock_memcell_repo, @@ -340,6 +362,11 @@ async def _run_serialisation_probe( [cluster_a] if owner == owner_a else [cluster_b] ) ) + mock_episode_repo.find_by_owner_entries = AsyncMock( + side_effect=lambda _owner, ids, **_kw: [ + _episode_row(ids[0], f"mc_{ids[0]}") + ] + ) mock_memcell_repo.find_by_ids = AsyncMock( side_effect=lambda ids: [ _memcell_row(ids[0], sender_id="sender", ts_ms=1_700_000_000_000) diff --git a/tests/unit/test_memory/test_strategies/test_partition_locks.py b/tests/unit/test_memory/test_strategies/test_partition_locks.py index 1202dd7..949cf29 100644 --- a/tests/unit/test_memory/test_strategies/test_partition_locks.py +++ b/tests/unit/test_memory/test_strategies/test_partition_locks.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`everos.memory.strategies._partition_locks`. +"""Tests for :mod:`everos.memory._partition_locks`. The helper is the foundation under every strategy that performs a read → modify → write on shared state; its own behaviour (lock reuse, @@ -12,7 +12,7 @@ import asyncio import pytest -from everos.memory.strategies._partition_locks import ( +from everos.memory._partition_locks import ( _reset_for_tests, get_partition_lock, ) diff --git a/tests/unit/test_memory/test_strategies/test_reflect_episodes.py b/tests/unit/test_memory/test_strategies/test_reflect_episodes.py new file mode 100644 index 0000000..85e0341 --- /dev/null +++ b/tests/unit/test_memory/test_strategies/test_reflect_episodes.py @@ -0,0 +1,31 @@ +"""Tests for the ``reflect_episodes`` Cron strategy. + +Verifies decorator metadata (name, trigger type, emits, enabled flag). +The strategy body is a thin entry point — orchestrator logic is tested +separately in ``test_reflection/test_orchestrator.py``. +""" + +from __future__ import annotations + +import inspect + +from everos.infra.ome.triggers import Cron +from everos.memory.events import EpisodeExtracted +from everos.memory.strategies.reflect_episodes import reflect_episodes + + +async def test_strategy_meta_is_attached() -> None: + """Decorator stamps the expected StrategyMeta on the function.""" + meta = reflect_episodes.meta + assert meta.name == "reflect_episodes" + assert isinstance(meta.trigger, Cron) + assert meta.trigger.expr == "0 2 * * 1" + assert meta.emits == frozenset({EpisodeExtracted}) + assert meta.max_retries == 1 + assert meta.enabled is False + + +async def test_strategy_is_callable() -> None: + """The Strategy wrapper must be callable (delegates to async func).""" + assert callable(reflect_episodes) + assert inspect.iscoroutinefunction(reflect_episodes.meta.func) diff --git a/tests/unit/test_memory/test_strategies/test_registration.py b/tests/unit/test_memory/test_strategies/test_registration.py index 44db2ae..717e091 100644 --- a/tests/unit/test_memory/test_strategies/test_registration.py +++ b/tests/unit/test_memory/test_strategies/test_registration.py @@ -13,6 +13,7 @@ from everos.memory.strategies import ( extract_atomic_facts, extract_foresight, extract_user_profile, + reflect_episodes, trigger_profile_clustering, trigger_skill_clustering, ) @@ -27,8 +28,9 @@ def test_strategies_are_re_exported_from_package() -> None: (extract_agent_skill, "extract_agent_skill"), (trigger_profile_clustering, "trigger_profile_clustering"), (extract_user_profile, "extract_user_profile"), + (reflect_episodes, "reflect_episodes"), ]: - assert fn._ome_strategy_meta.name == name # type: ignore[attr-defined] + assert fn.meta.name == name async def test_get_engine_registers_all_strategies( @@ -53,4 +55,5 @@ async def test_get_engine_registers_all_strategies( "extract_agent_skill", "trigger_profile_clustering", "extract_user_profile", + "reflect_episodes", } diff --git a/tests/unit/test_memory/test_strategies/test_strategies_persistence.py b/tests/unit/test_memory/test_strategies/test_strategies_persistence.py index ed9baf4..cc01313 100644 --- a/tests/unit/test_memory/test_strategies/test_strategies_persistence.py +++ b/tests/unit/test_memory/test_strategies/test_strategies_persistence.py @@ -16,12 +16,27 @@ from everos.infra.persistence.markdown import ( AtomicFactReader, ForesightReader, ) -from everos.memory.events import AgentPipelineStarted, UserPipelineStarted +from everos.memory.events import ( + AgentPipelineStarted, + EpisodeExtracted, + UserPipelineStarted, +) from everos.memory.strategies.extract_agent_case import extract_agent_case from everos.memory.strategies.extract_atomic_facts import extract_atomic_facts from everos.memory.strategies.extract_foresight import extract_foresight +def _episode_event_for(owner: str) -> EpisodeExtracted: + return EpisodeExtracted( + memcell_id="mc_a", + episode_entry_id="ep_20260517_0001", + episode_text="hi", + episode_timestamp_ms=1_700_000_000_000, + owner_id=owner, + session_id="s1", + ) + + def _event_for(owner: str) -> UserPipelineStarted: return UserPipelineStarted( memcell_id="mc_a", @@ -102,8 +117,8 @@ async def test_atomic_facts_round_trip( "everos.memory.strategies.extract_atomic_facts.AtomicFactExtractor" ) as mock_ext, ): - mock_ext.return_value.aextract = AsyncMock(return_value=facts) - await extract_atomic_facts(_event_for("u_alice"), FakeStrategyContext()) + mock_ext.return_value.aextract_from_text = AsyncMock(return_value=facts) + await extract_atomic_facts(_episode_event_for("u_alice"), FakeStrategyContext()) reader = AtomicFactReader(root=MemoryRoot(root=tmp_path)) path = reader.path_for("u_alice") diff --git a/tests/unit/test_memory/test_strategies/test_strategy_to_handler_contract.py b/tests/unit/test_memory/test_strategies/test_strategy_to_handler_contract.py index 7abd076..e0a68df 100644 --- a/tests/unit/test_memory/test_strategies/test_strategy_to_handler_contract.py +++ b/tests/unit/test_memory/test_strategies/test_strategy_to_handler_contract.py @@ -32,7 +32,11 @@ from everos.memory.cascade.handlers import ( HandlerDeps, ) from everos.memory.cascade.handlers._daily_log_base import ParsedEntry -from everos.memory.events import AgentPipelineStarted, UserPipelineStarted +from everos.memory.events import ( + AgentPipelineStarted, + EpisodeExtracted, + UserPipelineStarted, +) from everos.memory.strategies.extract_agent_case import extract_agent_case from everos.memory.strategies.extract_atomic_facts import extract_atomic_facts from everos.memory.strategies.extract_foresight import extract_foresight @@ -56,6 +60,17 @@ class _StubEmbedder(EmbeddingProvider): return [await self.embed(t) for t in texts] +def _episode_event(owner_id: str) -> EpisodeExtracted: + return EpisodeExtracted( + memcell_id="mc_a", + episode_entry_id="ep_20260517_0001", + episode_text="hi", + episode_timestamp_ms=1_700_000_000_000, + owner_id=owner_id, + session_id="s1", + ) + + def _event(owner_id: str) -> UserPipelineStarted: return UserPipelineStarted( memcell_id="mc_a", @@ -132,8 +147,8 @@ async def test_atomic_fact_strategy_md_feeds_handler_with_content( "everos.memory.strategies.extract_atomic_facts.AtomicFactExtractor" ) as mock_ext, ): - mock_ext.return_value.aextract = AsyncMock(return_value=facts) - await extract_atomic_facts(_event("u_alice"), FakeStrategyContext()) + mock_ext.return_value.aextract_from_text = AsyncMock(return_value=facts) + await extract_atomic_facts(_episode_event("u_alice"), FakeStrategyContext()) handler = AtomicFactHandler( HandlerDeps( diff --git a/tests/unit/test_memory/test_strategies/test_trigger_profile_clustering.py b/tests/unit/test_memory/test_strategies/test_trigger_profile_clustering.py index 2469f76..bfce577 100644 --- a/tests/unit/test_memory/test_strategies/test_trigger_profile_clustering.py +++ b/tests/unit/test_memory/test_strategies/test_trigger_profile_clustering.py @@ -16,8 +16,8 @@ import structlog.testing from everalgo.clustering import Cluster as AlgoCluster from everos.infra.ome.testing import FakeStrategyContext +from everos.memory._partition_locks import _reset_for_tests from everos.memory.events import EpisodeExtracted, ProfileClusterUpdated -from everos.memory.strategies._partition_locks import _reset_for_tests from everos.memory.strategies.trigger_profile_clustering import ( trigger_profile_clustering, ) @@ -32,24 +32,27 @@ def _event( *, owner_id: str = "u_alice", memcell_id: str = "mc_aaaaaaaaaaa1", + episode_entry_id: str = "ep_20260517_0001", episode_text: str = "alice likes hiking", episode_timestamp_ms: int = 1_700_000_001_000, ) -> EpisodeExtracted: return EpisodeExtracted( memcell_id=memcell_id, - episode_entry_id="ep_20260517_0001", + episode_entry_id=episode_entry_id, episode_text=episode_text, episode_timestamp_ms=episode_timestamp_ms, owner_id=owner_id, + session_id="s_test", ) async def test_strategy_meta_is_attached() -> None: - meta = trigger_profile_clustering._ome_strategy_meta # type: ignore[attr-defined] + meta = trigger_profile_clustering.meta assert meta.name == "trigger_profile_clustering" assert EpisodeExtracted in meta.trigger.on assert meta.emits == frozenset({ProfileClusterUpdated}) assert meta.max_retries == 2 + assert meta.applies_to is not None @pytest.mark.asyncio @@ -90,7 +93,7 @@ async def test_creates_new_cluster_when_no_existing( assert new_cluster.id == "cl_newuser00001" assert new_cluster.count == 1 assert new_cluster.last_ts == 1_700_000_001_000 - assert new_cluster.members == ["mc_aaaaaaaaaaa1"] + assert new_cluster.members == ["ep_20260517_0001"] assert new_cluster.preview == ["alice likes hiking"] assert existing == [] @@ -101,7 +104,7 @@ async def test_creates_new_cluster_when_no_existing( "owner_id": "u_alice", "owner_type": "user", "kind": "user_memory", - "member_type": "memcell", + "member_type": "episode", "app_id": "default", "project_id": "default", } @@ -129,7 +132,7 @@ async def test_merges_into_existing_cluster_when_algo_matches() -> None: count=1, last_ts=1_700_000_000_000, preview=["earlier episode"], - members=["mc_zzzzzzzzzzz0"], + members=["ep_20260517_0000"], ) merged_cluster = AlgoCluster( id="cl_existing0001", @@ -137,7 +140,7 @@ async def test_merges_into_existing_cluster_when_algo_matches() -> None: count=2, last_ts=1_700_000_001_000, preview=["earlier episode", "alice likes hiking"], - members=["mc_zzzzzzzzzzz0", "mc_aaaaaaaaaaa1"], + members=["ep_20260517_0000", "ep_20260517_0001"], ) with ( @@ -174,7 +177,7 @@ async def _run_serialisation_probe(owner_a: str, owner_b: str) -> list[str]: """Drive two trigger_profile_clustering runs and record entry/exit order.""" log: list[str] = [] - def mock_cluster_by_geometry(_new_cluster, _existing): + def mock_cluster_by_geometry(_new_cluster, _existing, **_kw): # Sync, matching the real algo signature (must not be awaited). return None @@ -208,11 +211,11 @@ async def _run_serialisation_probe(owner_a: str, owner_b: str) -> list[str]: await asyncio.gather( trigger_profile_clustering( - _event(owner_id=owner_a, memcell_id="mc_run_a"), + _event(owner_id=owner_a, episode_entry_id="ep_run_a"), FakeStrategyContext(), ), trigger_profile_clustering( - _event(owner_id=owner_b, memcell_id="mc_run_b"), + _event(owner_id=owner_b, episode_entry_id="ep_run_b"), FakeStrategyContext(), ), ) @@ -223,13 +226,31 @@ async def test_partition_lock_serialises_runs_on_same_owner() -> None: """Two runs sharing ``owner_id`` must not overlap critical sections.""" log = await _run_serialisation_probe("u_alice", "u_alice") assert log in ( - ["enter:mc_run_a", "leave:mc_run_a", "enter:mc_run_b", "leave:mc_run_b"], - ["enter:mc_run_b", "leave:mc_run_b", "enter:mc_run_a", "leave:mc_run_a"], + ["enter:ep_run_a", "leave:ep_run_a", "enter:ep_run_b", "leave:ep_run_b"], + ["enter:ep_run_b", "leave:ep_run_b", "enter:ep_run_a", "leave:ep_run_a"], ) async def test_partition_lock_lets_different_owners_run_in_parallel() -> None: """Runs on distinct ``owner_id`` must overlap (no false serialisation).""" log = await _run_serialisation_probe("u_alice", "u_bob") - assert log.index("enter:mc_run_a") < log.index("leave:mc_run_b") - assert log.index("enter:mc_run_b") < log.index("leave:mc_run_a") + assert log.index("enter:ep_run_a") < log.index("leave:ep_run_b") + assert log.index("enter:ep_run_b") < log.index("leave:ep_run_a") + + +async def test_applies_to_rejects_non_pipeline_source() -> None: + """Events with source != 'pipeline' must not pass the applies_to gate.""" + meta = trigger_profile_clustering.meta + pipeline_event = _event() + assert meta.applies_to(pipeline_event) is True + + reflection_event = EpisodeExtracted( + memcell_id="mc_merged", + episode_entry_id="ep_20260517_0002", + episode_text="merged narrative", + episode_timestamp_ms=1_700_000_001_000, + owner_id="u_alice", + session_id="reflection", + source="reflection", + ) + assert meta.applies_to(reflection_event) is False diff --git a/tests/unit/test_memory/test_strategies/test_trigger_skill_clustering.py b/tests/unit/test_memory/test_strategies/test_trigger_skill_clustering.py index 71ad15c..2856056 100644 --- a/tests/unit/test_memory/test_strategies/test_trigger_skill_clustering.py +++ b/tests/unit/test_memory/test_strategies/test_trigger_skill_clustering.py @@ -16,8 +16,8 @@ import structlog.testing from everalgo.clustering import Cluster as AlgoCluster from everos.infra.ome.testing import FakeStrategyContext +from everos.memory._partition_locks import _reset_for_tests from everos.memory.events import AgentCaseExtracted, SkillClusterUpdated -from everos.memory.strategies._partition_locks import _reset_for_tests from everos.memory.strategies.trigger_skill_clustering import ( trigger_skill_clustering, ) @@ -47,7 +47,7 @@ def _event( async def test_strategy_meta_is_attached() -> None: - meta = trigger_skill_clustering._ome_strategy_meta # type: ignore[attr-defined] + meta = trigger_skill_clustering.meta assert meta.name == "trigger_skill_clustering" assert AgentCaseExtracted in meta.trigger.on assert meta.emits == frozenset({SkillClusterUpdated}) @@ -163,7 +163,7 @@ async def test_merges_into_existing_cluster_when_algo_matches() -> None: preview=["earlier intent"], members=["ac_20260517_0000"], ) - # Simulate merge behavior: id passes through from existing, members appended. + # Simulate everalgo _merge: id passes through from existing, members appended. merged_cluster = AlgoCluster( id="cl_existing0001", centroid=np.array([0.17] * 1024, dtype=np.float32), diff --git a/tests/unit/test_service/test_knowledge_create.py b/tests/unit/test_service/test_knowledge_create.py new file mode 100644 index 0000000..d391e62 --- /dev/null +++ b/tests/unit/test_service/test_knowledge_create.py @@ -0,0 +1,281 @@ +"""Unit tests for :func:`everos.service.knowledge.create_document`. + +White-box surfaces: ``KnowledgeExtractor.aextract`` (mocked), +``KnowledgeWriter.write`` (mocked), ``knowledge_document_repo.doc_id_exists`` +(mocked), ``ensure_taxonomy`` / ``parse_taxonomy`` (mocked). +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest +from everalgo.types import KnowledgeMemory, ParsedContent + +from everos.service.knowledge import ( + CreateDocumentResult, + DuplicateDocumentError, + ExtractionEmptyError, + create_document, +) + +# ── Fixtures ───────────────────────────────────────────────────────────── + + +def _make_memory( + doc_id: str = "d_abc123000000", + topic_index: int = 0, + category_id: str = "Technology", +) -> KnowledgeMemory: + return KnowledgeMemory( + doc_id=doc_id, + topic_index=topic_index, + topic=f"Topic {topic_index}", + topic_path=f"Topic {topic_index}", + summary=f"Summary for topic {topic_index}", + content=f"Content for topic {topic_index}", + depth=0 if topic_index == 0 else 1, + category_id=category_id, + ) + + +def _make_memories( + doc_id: str = "d_abc123000000", + count: int = 3, + category_id: str = "Technology", +) -> list[KnowledgeMemory]: + """Root (index 0) + ``count-1`` topic nodes.""" + return [ + _make_memory(doc_id=doc_id, topic_index=i, category_id=category_id) + for i in range(count) + ] + + +@pytest.fixture +def mock_extractor() -> AsyncMock: + return AsyncMock() + + +@pytest.fixture +def knowledge_dir(tmp_path: Path) -> Path: + return tmp_path / "knowledge" + + +# ── Shared patch targets ───────────────────────────────────────────────── + +_MOD = "everos.service.knowledge" + + +# ── Tests ──────────────────────────────────────────────────────────────── + + +async def test_create_document_success( + mock_extractor: AsyncMock, + knowledge_dir: Path, +) -> None: + """Happy path: extractor returns 3 memories, writer writes, result OK.""" + memories = _make_memories(count=3) + mock_extractor.aextract.return_value = memories + doc_dir = knowledge_dir / "Technology" / "test_doc" + + with ( + patch(f"{_MOD}.ensure_taxonomy") as mock_ensure, + patch(f"{_MOD}.parse_taxonomy", return_value=[]), + patch(f"{_MOD}.KnowledgeWriter") as mock_writer_cls, + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_writer_cls.write = AsyncMock(return_value=doc_dir) + mock_repo.doc_id_exists = AsyncMock(return_value=False) + + result = await create_document( + extractor=mock_extractor, + parsed=ParsedContent(text="Some document content"), + title="Test Doc", + knowledge_dir=knowledge_dir, + doc_id="d_abc123000000", + source_name="test.pdf", + source_type="file", + ) + + assert isinstance(result, CreateDocumentResult) + assert result.doc_id == "d_abc123000000" + assert result.category_id == "Technology" + assert result.topic_count == 2 # 3 memories, 1 root (index 0) + assert result.source_name == "test.pdf" + assert result.md_path == str(doc_dir) + mock_ensure.assert_called_once_with(knowledge_dir) + mock_writer_cls.write.assert_awaited_once() + + +async def test_create_document_empty_result_raises( + mock_extractor: AsyncMock, + knowledge_dir: Path, +) -> None: + """Extractor returns empty list -> ExtractionEmptyError.""" + mock_extractor.aextract.return_value = [] + + with ( + patch(f"{_MOD}.ensure_taxonomy"), + patch(f"{_MOD}.parse_taxonomy", return_value=[]), + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_repo.doc_id_exists = AsyncMock(return_value=False) + + with pytest.raises(ExtractionEmptyError): + await create_document( + extractor=mock_extractor, + parsed=ParsedContent(text="Empty doc"), + title="Empty", + knowledge_dir=knowledge_dir, + doc_id="d_abc123000000", + ) + + +async def test_create_document_mints_doc_id( + mock_extractor: AsyncMock, + knowledge_dir: Path, +) -> None: + """No doc_id provided -> one is minted (starts with 'd_', 14 chars).""" + memories = _make_memories(count=1, doc_id="d_placeholder0") + mock_extractor.aextract.return_value = memories + doc_dir = knowledge_dir / "Technology" / "test_doc" + + with ( + patch(f"{_MOD}.ensure_taxonomy"), + patch(f"{_MOD}.parse_taxonomy", return_value=[]), + patch(f"{_MOD}.KnowledgeWriter") as mock_writer_cls, + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_writer_cls.write = AsyncMock(return_value=doc_dir) + mock_repo.doc_id_exists = AsyncMock(return_value=False) + + result = await create_document( + extractor=mock_extractor, + parsed=ParsedContent(text="Content"), + title="Title", + knowledge_dir=knowledge_dir, + # doc_id intentionally omitted + ) + + assert result.doc_id.startswith("d_") + assert len(result.doc_id) == 14 # "d_" + 12 hex chars + + +async def test_create_document_uses_provided_doc_id( + mock_extractor: AsyncMock, + knowledge_dir: Path, +) -> None: + """Explicit doc_id='d_existing123' -> passed to extractor, not minted.""" + provided_id = "d_existing123" + memories = _make_memories(count=1, doc_id=provided_id) + mock_extractor.aextract.return_value = memories + doc_dir = knowledge_dir / "Technology" / "test_doc" + + with ( + patch(f"{_MOD}.ensure_taxonomy"), + patch(f"{_MOD}.parse_taxonomy", return_value=[]), + patch(f"{_MOD}.KnowledgeWriter") as mock_writer_cls, + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_writer_cls.write = AsyncMock(return_value=doc_dir) + mock_repo.doc_id_exists = AsyncMock(return_value=False) + + result = await create_document( + extractor=mock_extractor, + parsed=ParsedContent(text="Content"), + title="Title", + knowledge_dir=knowledge_dir, + doc_id=provided_id, + ) + + assert result.doc_id == provided_id + # Verify the extractor received the provided doc_id. + call_kwargs = mock_extractor.aextract.call_args + assert call_kwargs.kwargs["doc_id"] == provided_id + + +async def test_create_document_empty_category_fallback( + mock_extractor: AsyncMock, + knowledge_dir: Path, +) -> None: + """Memories with empty category_id -> writer receives 'Others'.""" + memories = _make_memories(count=2, category_id="") + mock_extractor.aextract.return_value = memories + doc_dir = knowledge_dir / "Others" / "test_doc" + + with ( + patch(f"{_MOD}.ensure_taxonomy"), + patch(f"{_MOD}.parse_taxonomy", return_value=[]), + patch(f"{_MOD}.KnowledgeWriter") as mock_writer_cls, + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_writer_cls.write = AsyncMock(return_value=doc_dir) + mock_repo.doc_id_exists = AsyncMock(return_value=False) + + result = await create_document( + extractor=mock_extractor, + parsed=ParsedContent(text="Content"), + title="Title", + knowledge_dir=knowledge_dir, + doc_id="d_abc123000000", + ) + + assert result.category_id == "Others" + # Verify the memories passed to writer have the fallback category. + write_call = mock_writer_cls.write.call_args + written_memories = write_call.args[0] + for m in written_memories: + assert m.category_id == "Others" + + +async def test_create_document_rejects_duplicate_doc_id( + mock_extractor: AsyncMock, + knowledge_dir: Path, +) -> None: + """Providing an already-existing doc_id raises DuplicateDocumentError.""" + with ( + patch(f"{_MOD}.ensure_taxonomy"), + patch(f"{_MOD}.parse_taxonomy", return_value=[]), + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_repo.doc_id_exists = AsyncMock(return_value=True) + + with pytest.raises(DuplicateDocumentError): + await create_document( + extractor=mock_extractor, + parsed=ParsedContent(text="Content"), + title="Title", + knowledge_dir=knowledge_dir, + doc_id="d_existing123", + ) + + +async def test_create_document_ensures_taxonomy( + mock_extractor: AsyncMock, + knowledge_dir: Path, +) -> None: + """ensure_taxonomy called with knowledge_dir.""" + memories = _make_memories(count=1) + mock_extractor.aextract.return_value = memories + doc_dir = knowledge_dir / "Technology" / "test_doc" + + with ( + patch(f"{_MOD}.ensure_taxonomy") as mock_ensure, + patch(f"{_MOD}.parse_taxonomy", return_value=[]) as mock_parse, + patch(f"{_MOD}.KnowledgeWriter") as mock_writer_cls, + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_writer_cls.write = AsyncMock(return_value=doc_dir) + mock_repo.doc_id_exists = AsyncMock(return_value=False) + + await create_document( + extractor=mock_extractor, + parsed=ParsedContent(text="Content"), + title="Title", + knowledge_dir=knowledge_dir, + doc_id="d_abc123000000", + ) + + mock_ensure.assert_called_once_with(knowledge_dir) + mock_parse.assert_called_once_with(knowledge_dir / ".taxonomy.md") diff --git a/tests/unit/test_service/test_knowledge_crud.py b/tests/unit/test_service/test_knowledge_crud.py new file mode 100644 index 0000000..e808a5f --- /dev/null +++ b/tests/unit/test_service/test_knowledge_crud.py @@ -0,0 +1,345 @@ +"""Unit tests for knowledge CRUD service functions. + +White-box surfaces mocked: + ``knowledge_document_repo`` (get_by_doc_id, list_documents, upsert_from_handler) + ``knowledge_topic_sqlite_repo`` (get_topics_by_doc_id, get_topics_by_ids, + count_by_doc_id) + ``anyio.Path.is_dir`` + ``anyio.to_thread.run_sync`` for delete_document. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest + +from everos.component.utils.datetime import get_utc_now +from everos.infra.persistence.sqlite.repos.knowledge import DocumentListPage +from everos.infra.persistence.sqlite.tables.knowledge import ( + KnowledgeDocumentRow, + KnowledgeTopicRow, +) +from everos.service.knowledge import ( + DeleteResult, + DocumentDetail, + DocumentListResult, + DocumentNotFoundError, + PatchResult, + TopicDetail, + TopicNotFoundError, + delete_document, + get_document, + get_topic, + list_documents, + patch_document, +) + +_MOD = "everos.service.knowledge" + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _doc_row( + doc_id: str = "d_testdoc00001", + category_id: str = "Technology", + title: str = "Test Doc", + summary: str = "A test document.", + source_name: str | None = "test.pdf", + source_type: str | None = "file", + md_path: str = "/tmp/knowledge/Technology/d_testdoc00001", +) -> KnowledgeDocumentRow: + now = get_utc_now() + return KnowledgeDocumentRow( + doc_id=doc_id, + app_id="app1", + project_id="proj1", + category_id=category_id, + title=title, + summary=summary, + source_name=source_name, + source_type=source_type, + md_path=md_path, + created_at=now, + updated_at=now, + ) + + +def _topic_row( + node_id: str = "n_topic0001", + doc_id: str = "d_testdoc00001", + topic_index: int = 1, + topic_name: str = "Introduction", + topic_path: str = "Test Doc > Introduction", + depth: int = 1, + parent_node_id: str | None = None, + children_node_ids: str | None = None, + content_labels: str | None = None, +) -> KnowledgeTopicRow: + now = get_utc_now() + return KnowledgeTopicRow( + node_id=node_id, + doc_id=doc_id, + app_id="app1", + project_id="proj1", + category_id="Technology", + topic_index=topic_index, + topic_name=topic_name, + topic_path=topic_path, + depth=depth, + parent_node_id=parent_node_id, + children_node_ids=children_node_ids, + summary="Intro summary.", + content="Intro content.", + content_labels=content_labels, + md_path="/tmp/knowledge/Technology/d_testdoc00001", + created_at=now, + updated_at=now, + ) + + +# ── get_document ────────────────────────────────────────────────────────────── + + +async def test_get_document_success() -> None: + """Returns DocumentDetail with topics mapped from node_id → topic_id.""" + doc = _doc_row() + topics = [ + _topic_row(node_id="n_001", topic_index=1), + _topic_row(node_id="n_002", topic_index=2, topic_name="Background"), + ] + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo, + ): + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=doc) + mock_topic_repo.get_topics_by_doc_id = AsyncMock(return_value=topics) + + result = await get_document("d_testdoc00001", "app1", "proj1") + + assert isinstance(result, DocumentDetail) + assert result.doc_id == "d_testdoc00001" + assert result.category_id == "Technology" + assert result.title == "Test Doc" + assert len(result.topics) == 2 + assert result.topics[0].topic_id == "n_001" + assert result.topics[1].topic_id == "n_002" + assert result.topics[1].topic_name == "Background" + mock_doc_repo.get_by_doc_id.assert_awaited_once_with("d_testdoc00001") + mock_topic_repo.get_topics_by_doc_id.assert_awaited_once_with("d_testdoc00001") + + +async def test_get_document_not_found() -> None: + """Raises DocumentNotFoundError when the doc_id does not exist.""" + with patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo: + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=None) + + with pytest.raises(DocumentNotFoundError): + await get_document("d_missing", "app1", "proj1") + + +# ── get_topic ───────────────────────────────────────────────────────────────── + + +async def test_get_topic_success() -> None: + """Returns TopicDetail with parsed JSON children_node_ids and content_labels.""" + children = ["n_child1", "n_child2"] + labels = ["concept", "definition"] + topic = _topic_row( + node_id="n_001", + parent_node_id="n_root", + children_node_ids=json.dumps(children), + content_labels=json.dumps(labels), + ) + + with patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo: + mock_topic_repo.get_topics_by_ids = AsyncMock(return_value=[topic]) + + result = await get_topic("n_001", "app1", "proj1") + + assert isinstance(result, TopicDetail) + assert result.topic_id == "n_001" + assert result.parent_topic_id == "n_root" + assert result.children_topic_ids == children + assert result.content_labels == labels + mock_topic_repo.get_topics_by_ids.assert_awaited_once_with(["n_001"]) + + +async def test_get_topic_empty_json_fields() -> None: + """Returns empty lists when children_node_ids and content_labels are None.""" + topic = _topic_row(node_id="n_leaf", children_node_ids=None, content_labels=None) + + with patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo: + mock_topic_repo.get_topics_by_ids = AsyncMock(return_value=[topic]) + + result = await get_topic("n_leaf", "app1", "proj1") + + assert result.children_topic_ids == [] + assert result.content_labels == [] + assert result.parent_topic_id is None + + +async def test_get_topic_not_found() -> None: + """Raises TopicNotFoundError when topic_id does not exist.""" + with patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo: + mock_topic_repo.get_topics_by_ids = AsyncMock(return_value=[]) + + with pytest.raises(TopicNotFoundError): + await get_topic("n_missing", "app1", "proj1") + + +# ── delete_document ─────────────────────────────────────────────────────────── + + +async def test_delete_document_success(tmp_path: Path) -> None: + """Calls rmtree on the document directory and returns topic count.""" + doc_dir = tmp_path / "d_testdoc00001" + doc_dir.mkdir() + doc = _doc_row(md_path=str(doc_dir)) + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo, + patch(f"{_MOD}.anyio") as mock_anyio, + ): + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=doc) + mock_topic_repo.count_by_doc_id = AsyncMock(return_value=3) + + mock_path_instance = AsyncMock() + mock_path_instance.is_dir = AsyncMock(return_value=True) + mock_anyio.Path.return_value = mock_path_instance + mock_anyio.to_thread.run_sync = AsyncMock(return_value=None) + + result = await delete_document("d_testdoc00001", "app1", "proj1") + + assert isinstance(result, DeleteResult) + assert result.doc_id == "d_testdoc00001" + assert result.deleted_topics == 3 + mock_anyio.to_thread.run_sync.assert_awaited_once() + + +async def test_delete_document_idempotent() -> None: + """Returns deleted_topics=0 without error when document does not exist.""" + with patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo: + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=None) + + result = await delete_document("d_missing", "app1", "proj1") + + assert isinstance(result, DeleteResult) + assert result.doc_id == "d_missing" + assert result.deleted_topics == 0 + + +# ── list_documents ──────────────────────────────────────────────────────────── + + +async def test_list_documents_returns_paginated_result() -> None: + """Returns DocumentListResult with correct pagination metadata.""" + rows = [ + _doc_row(doc_id="d_doc1", title="Alpha"), + _doc_row(doc_id="d_doc2", title="Beta"), + ] + page_result = DocumentListPage(rows=rows, total=10) + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo, + ): + mock_doc_repo.list_documents = AsyncMock(return_value=page_result) + mock_topic_repo.count_by_doc_id = AsyncMock(return_value=4) + + result = await list_documents( + "app1", + "proj1", + category_id=None, + page=2, + page_size=2, + sort_by="title", + sort_order="asc", + ) + + assert isinstance(result, DocumentListResult) + assert result.total == 10 + assert result.page == 2 + assert result.page_size == 2 + assert len(result.documents) == 2 + assert result.documents[0].doc_id == "d_doc1" + assert result.documents[0].topic_count == 4 + assert result.documents[1].doc_id == "d_doc2" + mock_doc_repo.list_documents.assert_awaited_once_with( + app_id="app1", + project_id="proj1", + category_id=None, + page=2, + page_size=2, + sort_by="title", + sort_order="asc", + ) + + +# ── patch_document ──────────────────────────────────────────────────────────── + + +async def test_patch_document_title_updates_correctly() -> None: + """Returns PatchResult with updated_fields=['title'] on title change.""" + doc = _doc_row(title="Old Title") + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}._update_index_frontmatter", new_callable=AsyncMock), + patch(f"{_MOD}.get_utc_now") as mock_now, + ): + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=doc) + mock_doc_repo.upsert_from_handler = AsyncMock(return_value=None) + fixed_now = get_utc_now() + mock_now.return_value = fixed_now + + result = await patch_document( + "d_testdoc00001", "app1", "proj1", title="New Title" + ) + + assert isinstance(result, PatchResult) + assert result.doc_id == "d_testdoc00001" + assert "title" in result.updated_fields + assert "category_id" not in result.updated_fields + assert result.updated_at == fixed_now + mock_doc_repo.upsert_from_handler.assert_awaited_once() + + +async def test_patch_document_no_changes_returns_empty_fields() -> None: + """Returns PatchResult with empty updated_fields when nothing changed.""" + doc = _doc_row(title="Same Title", category_id="Technology") + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}.get_utc_now") as mock_now, + ): + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=doc) + mock_doc_repo.upsert_from_handler = AsyncMock(return_value=None) + mock_now.return_value = get_utc_now() + + result = await patch_document( + "d_testdoc00001", + "app1", + "proj1", + title="Same Title", + category_id="Technology", + ) + + assert result.updated_fields == [] + mock_doc_repo.upsert_from_handler.assert_not_awaited() + + +async def test_patch_document_not_found_raises() -> None: + """Raises DocumentNotFoundError when doc_id does not exist in SQLite or md.""" + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}._locate_index_md", new_callable=AsyncMock) as mock_locate, + ): + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=None) + mock_locate.return_value = None + + with pytest.raises(DocumentNotFoundError): + await patch_document("d_missing", "app1", "proj1", title="New") diff --git a/tests/unit/test_service/test_knowledge_search.py b/tests/unit/test_service/test_knowledge_search.py new file mode 100644 index 0000000..8cc6daa --- /dev/null +++ b/tests/unit/test_service/test_knowledge_search.py @@ -0,0 +1,442 @@ +"""Unit tests for knowledge search service. + +White-box surfaces mocked: + ``acategory_retrieve`` (the everalgo facade) + ``knowledge_document_repo`` (get_documents_by_ids) + ``_get_embedding`` (embedding provider) + ``_get_reranker`` (rerank provider) + ``load_settings`` (knowledge search settings) +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from everalgo.types import Candidate + +from everos.component.utils.datetime import get_utc_now +from everos.core.errors import ConfigurationError +from everos.infra.persistence.sqlite.tables.knowledge import ( + KnowledgeDocumentRow, +) +from everos.service.knowledge import ( + DocumentContext, + SearchKnowledgeResult, + compile_knowledge_where, + search_knowledge, +) + +_MOD = "everos.service.knowledge" +_CONFIG_MOD = "everos.config" + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def _candidate( + node_id: str = "n_001", + score: float = 0.85, + source: str = "keyword", + doc_id: str = "d_testdoc00001", + category_id: str = "Technology", + topic_name: str = "Neural Networks", + topic_path: str = "Technology / Neural Networks", + depth: int = 1, + summary: str = "Overview of neural networks.", + content: str = "", +) -> Candidate: + return Candidate( + id=node_id, + score=score, + source=source, + metadata={ + "doc_id": doc_id, + "category_id": category_id, + "topic_name": topic_name, + "topic_path": topic_path, + "depth": depth, + "summary": summary, + "content": content, + }, + ) + + +def _doc_row( + doc_id: str = "d_testdoc00001", + title: str = "AI Handbook", + summary: str = "A handbook on AI.", +) -> KnowledgeDocumentRow: + now = get_utc_now() + return KnowledgeDocumentRow( + doc_id=doc_id, + app_id="default", + project_id="default", + category_id="Technology", + title=title, + summary=summary, + source_name="ai.pdf", + source_type="file", + md_path="/tmp/knowledge/Technology/d_testdoc00001", + created_at=now, + updated_at=now, + ) + + +def _mock_settings() -> MagicMock: + """Build a mock Settings with knowledge.search defaults.""" + s = MagicMock() + s.knowledge.search.recall_n = 200 + s.knowledge.search.rerank_n = 50 + s.knowledge.search.mass_top_m = 50 + s.knowledge.search.lam = 0.1 + s.knowledge.search.top_k_cap = 100 + s.embedding.model = "" + s.embedding.api_key = None + return s + + +def _patch_stack( + facade_return: list[Candidate] | None = None, + doc_rows: list[KnowledgeDocumentRow] | None = None, + embed_vector: list[float] | None = None, +): + """Return a dict of patches for common mocks. + + ``acategory_retrieve`` is mocked at the module level — its internal + behavior (recall -> rollup -> rerank -> boost) is tested in everalgo. + """ + acategory = AsyncMock(return_value=facade_return or []) + + doc_repo = AsyncMock() + doc_repo.get_documents_by_ids = AsyncMock(return_value=doc_rows or []) + + embedder = AsyncMock() + embedder.embed = AsyncMock(return_value=embed_vector or [0.1] * 1024) + + reranker = AsyncMock() + + recaller = AsyncMock() + + settings = _mock_settings() + + return { + "acategory": acategory, + "doc_repo": doc_repo, + "embedder": embedder, + "reranker": reranker, + "recaller": recaller, + "settings": settings, + } + + +# ── compile_knowledge_where ────────────────────────────────────────────────── + + +class TestCompileKnowledgeWhere: + def test_basic_clause(self) -> None: + result = compile_knowledge_where("myapp", "myproj") + assert result == "app_id = 'myapp' AND project_id = 'myproj'" + + def test_defaults(self) -> None: + result = compile_knowledge_where("default", "default") + assert "app_id = 'default'" in result + assert "project_id = 'default'" in result + + def test_rejects_invalid_app_id_with_sql_injection(self) -> None: + with pytest.raises(ValueError, match="app_id"): + compile_knowledge_where("app'; DROP TABLE --", "proj") + + def test_rejects_invalid_project_id_with_sql_injection(self) -> None: + with pytest.raises(ValueError, match="project_id"): + compile_knowledge_where("app", "proj'); DELETE FROM--") + + def test_rejects_empty_app_id(self) -> None: + with pytest.raises(ValueError, match="app_id"): + compile_knowledge_where("", "proj") + + def test_rejects_empty_project_id(self) -> None: + with pytest.raises(ValueError, match="project_id"): + compile_knowledge_where("app", "") + + def test_accepts_valid_ids_with_special_chars(self) -> None: + result = compile_knowledge_where("my_app.v2", "project-1") + assert "my_app.v2" in result + assert "project-1" in result + + def test_accepts_valid_ids_with_at_plus(self) -> None: + result = compile_knowledge_where("app@org+v1", "proj_1") + assert "app@org+v1" in result + assert "proj_1" in result + + +# ── search_knowledge ───────────────────────────────────────────────────────── + + +class TestSearchKnowledgeFacadeWiring: + """Verify search_knowledge delegates to acategory_retrieve correctly.""" + + async def test_calls_facade_with_config_params(self) -> None: + c = _candidate(node_id="n_001", score=0.9, content="Neural net content.") + mocks = _patch_stack(facade_return=[c], doc_rows=[_doc_row()]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch( + "everalgo.rank.acategory_retrieve", mocks["acategory"] + ) as mock_facade, + ): + await search_knowledge(query="neural nets", method="keyword") + + mock_facade.assert_awaited_once() + call_kwargs = mock_facade.call_args + assert call_kwargs[0][0] == "neural nets" + assert call_kwargs[1]["recall_n"] == 200 + assert call_kwargs[1]["rerank_n"] == 50 + assert call_kwargs[1]["mass_top_m"] == 50 + assert call_kwargs[1]["lam"] == pytest.approx(0.1) + assert call_kwargs[1]["top_n"] == 10 + + async def test_top_n_capped_by_top_k_cap(self) -> None: + mocks = _patch_stack(doc_rows=[_doc_row()]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch( + "everalgo.rank.acategory_retrieve", mocks["acategory"] + ) as mock_facade, + ): + # top_k=200 but top_k_cap=100 → effective_k=100 + await search_knowledge(query="test", method="keyword", top_k=200) + + assert mock_facade.call_args[1]["top_n"] == 100 + + +class TestSearchKnowledgeResults: + """Verify result assembly from facade output.""" + + async def test_returns_hits_with_scores(self) -> None: + c1 = _candidate(node_id="n_001", score=0.9, content="Content A.") + c2 = _candidate( + node_id="n_002", score=0.7, topic_name="CNNs", content="Content B." + ) + mocks = _patch_stack(facade_return=[c1, c2], doc_rows=[_doc_row()]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge(query="neural networks", method="keyword") + + assert isinstance(result, SearchKnowledgeResult) + assert len(result.hits) == 2 + assert result.total == 2 + + async def test_empty_results(self) -> None: + mocks = _patch_stack() + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge(query="nothing", method="keyword") + + assert result.hits == [] + assert result.total == 0 + + +class TestSearchKnowledgeIncludeContent: + async def test_content_populated_when_true(self) -> None: + content_text = "Full content of neural networks topic." + c = _candidate(node_id="n_001", score=0.9, content=content_text) + mocks = _patch_stack(facade_return=[c], doc_rows=[_doc_row()]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge( + query="neural nets", method="keyword", include_content=True + ) + + assert result.hits[0].content == content_text + + async def test_content_none_when_false(self) -> None: + c = _candidate(node_id="n_001", score=0.9, content="Some content.") + mocks = _patch_stack(facade_return=[c], doc_rows=[_doc_row()]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge( + query="neural nets", method="keyword", include_content=False + ) + + assert result.hits[0].content is None + + +class TestSearchKnowledgeScoreThreshold: + async def test_filters_low_score_candidates(self) -> None: + high = _candidate(node_id="n_001", score=0.9) + low = _candidate(node_id="n_002", score=0.1, topic_name="Low") + mocks = _patch_stack(facade_return=[high, low], doc_rows=[_doc_row()]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge( + query="neural nets", method="keyword", score_threshold=0.5 + ) + + assert len(result.hits) == 1 + assert result.hits[0].topic_id == "n_001" + + async def test_no_filtering_when_threshold_none(self) -> None: + c1 = _candidate(node_id="n_001", score=0.9) + c2 = _candidate(node_id="n_002", score=0.1, topic_name="Low") + mocks = _patch_stack(facade_return=[c1, c2], doc_rows=[_doc_row()]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge(query="test", method="keyword") + + assert len(result.hits) == 2 + + +class TestSearchKnowledgeDocumentContext: + async def test_hits_carry_document_context(self) -> None: + c = _candidate(node_id="n_001", score=0.9) + doc = _doc_row(title="AI Handbook", summary="A handbook on AI.") + mocks = _patch_stack(facade_return=[c], doc_rows=[doc]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge(query="AI", method="keyword") + + hit = result.hits[0] + assert isinstance(hit.document, DocumentContext) + assert hit.document.doc_id == "d_testdoc00001" + assert hit.document.title == "AI Handbook" + assert hit.document.summary == "A handbook on AI." + + +class TestSearchKnowledgeTookMs: + async def test_took_ms_positive(self) -> None: + mocks = _patch_stack() + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge(query="test", method="keyword") + + assert result.took_ms >= 0 + + +class TestSearchKnowledgeReturnFields: + """Verify all SearchHit fields are correctly populated.""" + + async def test_all_hit_fields_populated(self) -> None: + c = _candidate( + node_id="n_001", + score=0.85, + source="keyword", + doc_id="d_testdoc00001", + category_id="Technology", + topic_name="Neural Networks", + topic_path="Technology / Neural Networks", + depth=1, + summary="Overview of neural networks.", + ) + mocks = _patch_stack(facade_return=[c], doc_rows=[_doc_row()]) + + with ( + patch(f"{_MOD}._build_recaller", return_value=mocks["recaller"]), + patch(f"{_MOD}.knowledge_document_repo", mocks["doc_repo"]), + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + patch("everalgo.rank.acategory_retrieve", mocks["acategory"]), + ): + result = await search_knowledge(query="neural", method="keyword") + + hit = result.hits[0] + assert hit.topic_id == "n_001" + assert hit.category_id == "Technology" + assert hit.topic_name == "Neural Networks" + assert hit.topic_path == "Technology / Neural Networks" + assert hit.depth == 1 + assert hit.summary == "Overview of neural networks." + assert hit.retrieval_method == "keyword" + assert hit.source == "keyword" + + +class TestSearchKnowledgeProviderRequired: + """Missing providers raise ConfigurationError (HTTP 500 CONFIGURATION_ERROR).""" + + async def test_raises_without_embedding(self) -> None: + mocks = _patch_stack() + + with ( + patch(f"{_MOD}._get_embedding", return_value=None), + patch(f"{_MOD}._get_reranker", return_value=mocks["reranker"]), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + pytest.raises(ConfigurationError, match="Embedding provider"), + ): + await search_knowledge(query="test", method="keyword") + + async def test_raises_without_reranker(self) -> None: + mocks = _patch_stack() + + with ( + patch(f"{_MOD}._get_embedding", return_value=mocks["embedder"]), + patch(f"{_MOD}._get_reranker", return_value=None), + patch(f"{_CONFIG_MOD}.load_settings", return_value=mocks["settings"]), + pytest.raises(ConfigurationError, match="Rerank provider"), + ): + await search_knowledge(query="test", method="keyword") diff --git a/tests/unit/test_service/test_knowledge_search_degradation.py b/tests/unit/test_service/test_knowledge_search_degradation.py new file mode 100644 index 0000000..4b20e20 --- /dev/null +++ b/tests/unit/test_service/test_knowledge_search_degradation.py @@ -0,0 +1,41 @@ +"""Verify search requires embedding and reranker — no silent degradation. + +An unconfigured provider is a configuration fault, not a transient service +outage, so it surfaces as ``ConfigurationError`` (HTTP 500 +CONFIGURATION_ERROR) rather than a retryable ``*ServiceError``. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import pytest + +from everos.core.errors import ConfigurationError + +_MOD = "everos.service.knowledge" + + +async def test_search_without_embedding_raises() -> None: + """Search without an embedding provider raises ConfigurationError.""" + from everos.service.knowledge import search_knowledge + + with ( + patch(f"{_MOD}._get_embedding", return_value=None), + pytest.raises(ConfigurationError, match="[Ee]mbedding"), + ): + await search_knowledge(query="test", method="vector") + + +async def test_search_without_reranker_raises() -> None: + """Search without a reranker raises ConfigurationError.""" + from everos.service.knowledge import search_knowledge + + mock_embedder = AsyncMock() + mock_embedder.embed = AsyncMock(return_value=[0.1] * 1024) + with ( + patch(f"{_MOD}._get_embedding", return_value=mock_embedder), + patch(f"{_MOD}._get_reranker", return_value=None), + pytest.raises(ConfigurationError, match="[Rr]erank"), + ): + await search_knowledge(query="test", method="keyword") diff --git a/tests/unit/test_service/test_original_file_storage.py b/tests/unit/test_service/test_original_file_storage.py new file mode 100644 index 0000000..d2ffa20 --- /dev/null +++ b/tests/unit/test_service/test_original_file_storage.py @@ -0,0 +1,448 @@ +"""Integration-level tests for original file storage. + +Tests the full call chain: create_document → _write_document → +_write_original_file → filesystem, and get_document → _resolve_original_file_path. + +Only the LLM extractor is mocked; KnowledgeWriter and filesystem are real. +This catches wiring bugs (wrong path passed between functions) that +isolated unit tests miss. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest +from everalgo.types import CategorySpec, KnowledgeMemory, ParsedContent + +from everos.service.knowledge import ( + CategoryOverview, + DocumentDetail, + DocumentOverviewItem, + create_document, + get_document, + list_categories, + replace_document, +) + +_MOD = "everos.service.knowledge" +_ORIGINAL_DIR = "_original" + + +def _make_memories(doc_id: str, category_id: str = "Sports") -> list[KnowledgeMemory]: + return [ + KnowledgeMemory( + doc_id=doc_id, + topic_index=0, + topic="Root Topic", + topic_path="Root Topic", + summary="Root summary.", + content="", + depth=0, + category_id=category_id, + ), + KnowledgeMemory( + doc_id=doc_id, + topic_index=1, + topic="Sub Topic", + topic_path="Root Topic > Sub Topic", + summary="Sub summary.", + content="Detailed content here.", + depth=1, + parent_index=0, + children_index=[], + category_id=category_id, + ), + ] + + +@pytest.fixture +def knowledge_dir(tmp_path: Path) -> Path: + d = tmp_path / "knowledge" + d.mkdir() + return d + + +# ── TC-1: create_document full chain writes _original/ ────────────────────── + + +async def test_create_document_writes_original_file( + knowledge_dir: Path, +) -> None: + """Full chain: create_document → KnowledgeWriter.write → _write_original_file. + + Only the extractor is mocked. KnowledgeWriter and filesystem are real. + This catches path wiring bugs between _write_document and _write_original_file. + """ + doc_id = "d_test00000001" + file_content = b"original PDF binary data" + memories = _make_memories(doc_id) + mock_ext = AsyncMock() + mock_ext.aextract.return_value = memories + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + patch(f"{_MOD}._mint_doc_id", return_value=doc_id), + ): + mock_repo.doc_id_exists = AsyncMock(return_value=False) + + result = await create_document( + extractor=mock_ext, + parsed=ParsedContent(text="some content"), + title="Test Doc", + knowledge_dir=knowledge_dir, + source_name="report.pdf", + source_type="file", + doc_id=doc_id, + category_id="Sports", + file_content=file_content, + ) + + # _original/ must be inside the document directory, not the category directory + doc_dir = Path(result.md_path) + original_file = doc_dir / _ORIGINAL_DIR / "report.pdf" + assert original_file.is_file(), f"Expected {original_file} to exist" + assert original_file.read_bytes() == file_content + + # Category directory must NOT contain _original/ + category_dir = doc_dir.parent + assert not (category_dir / _ORIGINAL_DIR).exists(), ( + f"_original/ landed in category dir {category_dir}, not doc dir" + ) + + +# ── TC-2: create_document without file_content skips _original/ ───────────── + + +async def test_create_document_without_file_content_no_original( + knowledge_dir: Path, +) -> None: + """No file_content → no _original/ directory created.""" + doc_id = "d_test00000002" + memories = _make_memories(doc_id) + mock_ext = AsyncMock() + mock_ext.aextract.return_value = memories + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + patch(f"{_MOD}._mint_doc_id", return_value=doc_id), + ): + mock_repo.doc_id_exists = AsyncMock(return_value=False) + + result = await create_document( + extractor=mock_ext, + parsed=ParsedContent(text="some content"), + title="No Original", + knowledge_dir=knowledge_dir, + source_name="test.txt", + source_type="file", + doc_id=doc_id, + category_id="Sports", + ) + + doc_dir = Path(result.md_path) + assert not (doc_dir / _ORIGINAL_DIR).exists() + + +# ── TC-3: get_document returns original_file_path when file exists ────────── + + +async def test_get_document_returns_original_file_path( + knowledge_dir: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """get_document derives original_file_path from md_path + source_name.""" + from everos.component.utils.datetime import get_utc_now + from everos.config import load_settings + from everos.core.persistence import MemoryRoot + from everos.infra.persistence.sqlite.tables.knowledge import ( + KnowledgeDocumentRow, + ) + + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + load_settings.cache_clear() + MemoryRoot._instance = None + + # Set up: doc dir with _original/ and a fake SQLite row + doc_rel = Path("app/proj/knowledge/Sports/my_doc") + doc_abs = tmp_path / doc_rel + doc_abs.mkdir(parents=True) + original_dir = doc_abs / _ORIGINAL_DIR + original_dir.mkdir() + (original_dir / "report.pdf").write_bytes(b"pdf bytes") + (doc_abs / "index.md").write_text("---\ntype: knowledge_document\n---\n") + + now = get_utc_now() + row = KnowledgeDocumentRow( + doc_id="d_test00000003", + app_id="app", + project_id="proj", + category_id="Sports", + title="Test", + summary="Summary", + source_name="report.pdf", + source_type="file", + md_path=str(doc_rel / "index.md"), + created_at=now, + updated_at=now, + ) + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo, + ): + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=row) + mock_topic_repo.get_topics_by_doc_id = AsyncMock(return_value=[]) + + detail = await get_document("d_test00000003", "app", "proj") + + assert isinstance(detail, DocumentDetail) + assert detail.original_file_path is not None + assert detail.original_file_path == str(original_dir / "report.pdf") + + load_settings.cache_clear() + MemoryRoot._instance = None + + +# ── TC-4: get_document returns None for legacy doc (no _original/) ────────── + + +async def test_get_document_returns_none_for_legacy_doc( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Legacy documents without _original/ get original_file_path=None.""" + from everos.component.utils.datetime import get_utc_now + from everos.config import load_settings + from everos.core.persistence import MemoryRoot + from everos.infra.persistence.sqlite.tables.knowledge import ( + KnowledgeDocumentRow, + ) + + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + load_settings.cache_clear() + MemoryRoot._instance = None + + doc_rel = Path("app/proj/knowledge/Sports/legacy_doc") + doc_abs = tmp_path / doc_rel + doc_abs.mkdir(parents=True) + + now = get_utc_now() + row = KnowledgeDocumentRow( + doc_id="d_legacy000001", + app_id="app", + project_id="proj", + category_id="Sports", + title="Legacy", + summary="Old doc", + source_name="old.pdf", + source_type="file", + md_path=str(doc_rel / "index.md"), + created_at=now, + updated_at=now, + ) + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo, + ): + mock_doc_repo.get_by_doc_id = AsyncMock(return_value=row) + mock_topic_repo.get_topics_by_doc_id = AsyncMock(return_value=[]) + + detail = await get_document("d_legacy000001", "app", "proj") + + assert detail.original_file_path is None + + load_settings.cache_clear() + MemoryRoot._instance = None + + +# ── TC-5: replace_document writes new _original/ ──────────────────────────── + + +async def test_replace_document_writes_new_original( + knowledge_dir: Path, +) -> None: + """replace_document writes new original file after atomic replacement.""" + from everos.component.utils.datetime import get_utc_now + from everos.infra.persistence.sqlite.tables.knowledge import ( + KnowledgeDocumentRow, + ) + + doc_id = "d_repl00000001" + old_memories = _make_memories(doc_id) + new_memories = _make_memories(doc_id) + new_file = b"new version binary" + + # Phase 1: create the original document + mock_ext = AsyncMock() + mock_ext.aextract.return_value = old_memories + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_repo.doc_id_exists = AsyncMock(return_value=False) + old_result = await create_document( + extractor=mock_ext, + parsed=ParsedContent(text="old content"), + title="Old Doc", + knowledge_dir=knowledge_dir, + source_name="v1.pdf", + doc_id=doc_id, + category_id="Sports", + file_content=b"old version binary", + ) + + old_dir = Path(old_result.md_path) + assert (old_dir / _ORIGINAL_DIR / "v1.pdf").read_bytes() == b"old version binary" + + # Phase 2: replace with new content + now = get_utc_now() + existing_row = KnowledgeDocumentRow( + doc_id=doc_id, + app_id="default", + project_id="default", + category_id="Sports", + title="Old Doc", + summary="Old summary", + source_name="v1.pdf", + source_type="file", + md_path=str(old_dir / "index.md"), + created_at=now, + updated_at=now, + ) + + mock_ext.aextract.return_value = new_memories + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_repo.get_by_doc_id = AsyncMock(return_value=existing_row) + new_result = await replace_document( + extractor=mock_ext, + parsed=ParsedContent(text="new content"), + title="New Doc", + doc_id=doc_id, + knowledge_dir=knowledge_dir, + source_name="v2.pdf", + category_id="Sports", + file_content=new_file, + ) + + new_dir = Path(new_result.md_path) + assert (new_dir / _ORIGINAL_DIR / "v2.pdf").read_bytes() == new_file + + +# ── TC-6: delete removes _original/ with doc dir ──────────────────────────── + + +async def test_delete_removes_original_with_doc_dir( + knowledge_dir: Path, +) -> None: + """rmtree on doc dir clears _original/ naturally.""" + doc_id = "d_del000000001" + memories = _make_memories(doc_id) + mock_ext = AsyncMock() + mock_ext.aextract.return_value = memories + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_repo.doc_id_exists = AsyncMock(return_value=False) + result = await create_document( + extractor=mock_ext, + parsed=ParsedContent(text="content"), + title="To Delete", + knowledge_dir=knowledge_dir, + source_name="file.pdf", + doc_id=doc_id, + category_id="Sports", + file_content=b"data", + ) + + doc_dir = Path(result.md_path) + assert (doc_dir / _ORIGINAL_DIR / "file.pdf").is_file() + + shutil.rmtree(doc_dir) + assert not list(doc_dir.parent.glob(doc_dir.name)) + + +# ── TC-7: shutil.move preserves _original/ ────────────────────────────────── + + +async def test_move_preserves_original(knowledge_dir: Path) -> None: + """PATCH category move (shutil.move) keeps _original/ intact.""" + doc_id = "d_move00000001" + memories = _make_memories(doc_id) + file_content = b"important data" + mock_ext = AsyncMock() + mock_ext.aextract.return_value = memories + + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_repo.doc_id_exists = AsyncMock(return_value=False) + result = await create_document( + extractor=mock_ext, + parsed=ParsedContent(text="content"), + title="To Move", + knowledge_dir=knowledge_dir, + source_name="file.pdf", + doc_id=doc_id, + category_id="Sports", + file_content=file_content, + ) + + old_dir = Path(result.md_path) + new_dir = knowledge_dir / "Finance" / old_dir.name + new_dir.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(old_dir), str(new_dir)) + + assert (new_dir / _ORIGINAL_DIR / "file.pdf").read_bytes() == file_content + assert not list(old_dir.parent.glob(old_dir.name)) + + +# ── TC-8: DocumentOverviewItem slim fields ────────────────────────────────── + + +def test_document_overview_item_slim_fields() -> None: + """DocumentOverviewItem has exactly 5 fields, no summary/source/updated_at.""" + from everos.component.utils.datetime import get_utc_now + + item = DocumentOverviewItem( + doc_id="d_abc", + category_id="Tech", + title="T", + topic_count=1, + created_at=get_utc_now(), + ) + fields = {f.name for f in item.__dataclass_fields__.values()} + assert fields == {"doc_id", "category_id", "title", "topic_count", "created_at"} + + +# ── TC-9: list_categories returns document_count ──────────────────────────── + + +async def test_list_categories_document_count() -> None: + """list_categories merges taxonomy specs with SQLite counts.""" + specs = [ + CategorySpec(id="Tech", description="Technology"), + CategorySpec(id="Empty", description="No docs"), + ] + counts = {"Tech": 3} + + with ( + patch(f"{_MOD}.MemoryRoot"), + patch(f"{_MOD}.ensure_taxonomy", new_callable=AsyncMock), + patch(f"{_MOD}.parse_taxonomy", new_callable=AsyncMock, return_value=specs), + patch(f"{_MOD}.knowledge_document_repo") as mock_repo, + ): + mock_repo.count_by_category = AsyncMock(return_value=counts) + result = await list_categories("app", "proj") + + assert len(result) == 2 + assert all(isinstance(c, CategoryOverview) for c in result) + assert result[0].document_count == 3 + assert result[1].document_count == 0 diff --git a/uv.lock b/uv.lock index 1bcb1e9..80558ad 100644 --- a/uv.lock +++ b/uv.lock @@ -443,29 +443,29 @@ wheels = [ [[package]] name = "everalgo-agent-memory" -version = "0.2.0" +version = "0.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "everalgo-boundary" }, { name = "everalgo-clustering" }, { name = "everalgo-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/a4/0bbe1a5a4a32458da79741abc3db6a4ab29353c49646a34cb98b10fbb9dc/everalgo_agent_memory-0.2.0.tar.gz", hash = "sha256:0b89d2d731b718ced62374d6488b0239b05ad89d06b8b5181e768d6ca3aa215d", size = 51371, upload-time = "2026-05-27T12:28:47.587Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/94/6eead80b95d0c26e750563e18b097c710ae29467d14f719efb2d72ed12df/everalgo_agent_memory-0.3.1.tar.gz", hash = "sha256:ff89fd0608530440bb21123eb76b9b5ade4d171c9c04f6407cf038b84eb6df15", size = 71944, upload-time = "2026-06-15T07:09:24.532Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/14/625ce508cccbfcecf61900338fb12cc0e252d9ddee6f3b57f32d1dea816a/everalgo_agent_memory-0.2.0-py3-none-any.whl", hash = "sha256:f4052a48b07a6f4c44facb1c6c4842249a235b6c18d0d19fb217584c5a37f150", size = 39543, upload-time = "2026-05-27T12:28:46.63Z" }, + { url = "https://files.pythonhosted.org/packages/d8/2e/3d892f31ecfbac8f127f4664877bed6a5ad74bd8c101e106ab583a5e5ca2/everalgo_agent_memory-0.3.1-py3-none-any.whl", hash = "sha256:4e8c2cf06ec4699199de5c769aa7c5ac30fdf61086526edbbd0b68bee5ec9219", size = 54895, upload-time = "2026-06-15T07:09:23.401Z" }, ] [[package]] name = "everalgo-boundary" -version = "0.2.0" +version = "0.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "asgiref" }, { name = "everalgo-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/4e/b086857a79aee04d9f919ec9c82aec113c86a1da3c38bdb90d5b808c1921/everalgo_boundary-0.2.0.tar.gz", hash = "sha256:a04a3fa130cb58d0987c72ff603cb2790e07fceb9a81e65984b6b8a0a4e36ff9", size = 21819, upload-time = "2026-05-27T12:11:01.211Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/6e/3c60e7c253948e42906108bff3f8227733804801c3ea96e04f7895366686/everalgo_boundary-0.2.1.tar.gz", hash = "sha256:23a93bb36b06251e5a85765f68640c55bcfe0f1faf8b025e44ef8857a5ce36f9", size = 21946, upload-time = "2026-06-15T06:14:36.962Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/a5/cdb26abc99b2655e6ee5f5aa7fcd1c7082a5f69a1012bec72a1dc385db51/everalgo_boundary-0.2.0-py3-none-any.whl", hash = "sha256:50f1c3aade13b24bf2b14d581ce492aad2a3a0e999003e43ddf3b338258e3d40", size = 17185, upload-time = "2026-05-27T12:11:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/73/6f/6cb00d36ee007360ac0e2e62c44da782ef881f0f5def9f9d5a64142d88fb/everalgo_boundary-0.2.1-py3-none-any.whl", hash = "sha256:95ac8982291041b5641b13c915790ef20cca8c44018c2001ff608b13c7ae6b8d", size = 17192, upload-time = "2026-06-15T06:14:35.921Z" }, ] [[package]] @@ -483,21 +483,36 @@ wheels = [ [[package]] name = "everalgo-core" -version = "0.2.0" +version = "0.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "openai" }, { name = "pydantic" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/15/9a/aba38b7fadbd54a33dccc4e8b6da284906b44b54bcead98389b27e05583a/everalgo_core-0.2.0.tar.gz", hash = "sha256:26bb4d4597d4cd165f3a727f7c8084d9393705772073d12e460bfba9c90df689", size = 51938, upload-time = "2026-05-27T12:04:20.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/d7/395c0504b426bf25cd9af60d5bab8f9c3131a6dee53a16b2a5a6a477b6c8/everalgo_core-0.3.0.tar.gz", hash = "sha256:cd91204a336ad459ae1c03eda97cdb0575534b675523cb775188debacc8f241b", size = 54469, upload-time = "2026-06-16T06:30:06.801Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/31/67be2640a4317acbb968216f27e4089d02d4af11579ae6a86a1838e6087a/everalgo_core-0.2.0-py3-none-any.whl", hash = "sha256:8b2f357e4ff2f1a6559970cbf1c1de9eac2da097d100a9a2c69a7f3576a556eb", size = 41694, upload-time = "2026-05-27T12:04:19.637Z" }, + { url = "https://files.pythonhosted.org/packages/2b/5d/ad60747004d873b23443412a74449b39640f08c4f6548fd4442641c3ff17/everalgo_core-0.3.0-py3-none-any.whl", hash = "sha256:5a25b784a2d24e28a762fee5751174a68c2eda168c353c33740ca3183069c02b", size = 44118, upload-time = "2026-06-16T06:30:05.678Z" }, +] + +[[package]] +name = "everalgo-knowledge" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asgiref" }, + { name = "everalgo-core" }, + { name = "everalgo-parser" }, + { name = "tiktoken" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/4f/e18401ef276536bce0f30dca8dd358450f5eb38cb8fee7a85e6d7d80bc0a/everalgo_knowledge-0.1.1.tar.gz", hash = "sha256:64bcf2c88a4507a3bf704d7c7813956ede55a4575cf63424786f1fa65f4b3200", size = 50172, upload-time = "2026-06-16T06:54:52.969Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/8d/3f860b72987f2028facf51733e8e3d85d722808136ea3dd1314bf462b7bd/everalgo_knowledge-0.1.1-py3-none-any.whl", hash = "sha256:6aa6fc70a65e75f380e4997dc58b69d8f5da7effcf5c4fbcdebd7fe58ca01a36", size = 34318, upload-time = "2026-06-16T06:54:51.911Z" }, ] [[package]] name = "everalgo-parser" -version = "0.2.0" +version = "0.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "asgiref" }, @@ -506,9 +521,9 @@ dependencies = [ { name = "httpx" }, { name = "pillow" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/ce/09978db4b171582e087cde8ed3f2770834d76c3123cc33001440ea5c71f8/everalgo_parser-0.2.0.tar.gz", hash = "sha256:cb554f8b060aed4ad773bc3d70550c70aecec206b83ed976a2c4bd1722f7c21f", size = 684801, upload-time = "2026-05-27T12:10:58.651Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/14/2f54d4c108ad5b5be1ac0efb7ada9b2ee2cc1802018d9141ffc7fea939e4/everalgo_parser-0.2.1.tar.gz", hash = "sha256:5fec4d4c5743514a2cdbf756a34bc6ef987ea7503082768a8d6d764032a78992", size = 686233, upload-time = "2026-06-15T06:15:13.175Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/cf/cab1e5aa973ddaf24ef6eeeb8dbc9c74d5f8acd13b6ede46f90cd4cec421/everalgo_parser-0.2.0-py3-none-any.whl", hash = "sha256:d051c6c1213d0e184ee7b510afaa01c9c528fc8417b774d8f1559cd5dd83a22c", size = 29683, upload-time = "2026-05-27T12:10:57.379Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6d/0a79a847e48d4417846916229ad90b0f13e7d348f31c84af30f31a7e0607/everalgo_parser-0.2.1-py3-none-any.whl", hash = "sha256:3876c898d835e2ae4e3bd7f2f61585de9ff84c13da54f7bbd658f795f8df007f", size = 30954, upload-time = "2026-06-15T06:15:12.102Z" }, ] [package.optional-dependencies] @@ -518,34 +533,34 @@ svg = [ [[package]] name = "everalgo-rank" -version = "0.3.0" +version = "0.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "asgiref" }, { name = "everalgo-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ed/99/35296a7f5dc66d49a3849267b166a3f0224566107b9dafbe2fcedb85c3a0/everalgo_rank-0.3.0.tar.gz", hash = "sha256:46e1045ecb5760641fdf6a82fd6d20568a0a93b356c2c421bfb616ee30e3baf4", size = 47798, upload-time = "2026-05-28T07:24:04.01Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/1d/138ca9dd92e916e840f2b00e9ce4fe2f06aba49bf0b2412bb1cdefe295af/everalgo_rank-0.4.1.tar.gz", hash = "sha256:0c4c0e72f11530ac2bcc7bca3aa1eaecbee1b5667811fd88ea15c12a1fb7cf19", size = 56960, upload-time = "2026-06-24T02:53:33.106Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f4/4af5f39c0047962464e81a01e56f27873454c41ac994163723236efc5831/everalgo_rank-0.3.0-py3-none-any.whl", hash = "sha256:02a89455aba75ce9ec7a2d6aed0c57ef1ab586c7351aff3a952e4cc338dc1469", size = 39276, upload-time = "2026-05-28T07:24:02.893Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1c/5e829ef1176a3e5d030b4a34ff5a0475dcc480d30fdcad7055d48b003e1e/everalgo_rank-0.4.1-py3-none-any.whl", hash = "sha256:675a8189d9ae3824c76d21d9bc409e0c62a3fc73023d02747bcff0b3982a2c92", size = 43021, upload-time = "2026-06-24T02:53:31.475Z" }, ] [[package]] name = "everalgo-user-memory" -version = "0.2.0" +version = "0.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "asgiref" }, { name = "everalgo-boundary" }, { name = "everalgo-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c8/aa/844bc3f89653af4d0e861664dd04882bbbbd913565a315502dfdd53a2b16/everalgo_user_memory-0.2.0.tar.gz", hash = "sha256:3a68447e5449bd99983eaca87cdfa8d04a7ac55ecb75e27e8800440f92c147f5", size = 52348, upload-time = "2026-05-27T12:28:35.364Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/81/bbb1d9681d59a0237d4a8ff7f11e317c754a518f285da18e05015a443201/everalgo_user_memory-0.3.1.tar.gz", hash = "sha256:ae7a2582c1b15a4303fb576fa67c518511ca9aa63572ae2addf975856ddfb321", size = 56065, upload-time = "2026-06-24T02:53:41.311Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/31/64/220a4465a46d35cf3574d4c26723926ca5cafd0e29ce00996d1757e091b0/everalgo_user_memory-0.2.0-py3-none-any.whl", hash = "sha256:1ba12252b8069dda9209ffdadb317df8f464dedafef20bc8b039fdc26a98c136", size = 51065, upload-time = "2026-05-27T12:28:34.436Z" }, + { url = "https://files.pythonhosted.org/packages/2a/7b/22491eeec4db1b7fa38dc12db301fe9fecb18f29b3c40fd6e3975505d3f7/everalgo_user_memory-0.3.1-py3-none-any.whl", hash = "sha256:ef9f3a1573b301222f669e7b29bd554b1b68602a5185b36845023e5e573f05b1", size = 54389, upload-time = "2026-06-24T02:53:40.392Z" }, ] [[package]] name = "everos" -version = "1.0.1" +version = "1.1.0" source = { editable = "." } dependencies = [ { name = "aiosqlite" }, @@ -553,8 +568,7 @@ dependencies = [ { name = "anyio" }, { name = "apscheduler" }, { name = "everalgo-agent-memory" }, - { name = "everalgo-boundary" }, - { name = "everalgo-core" }, + { name = "everalgo-knowledge" }, { name = "everalgo-rank" }, { name = "everalgo-user-memory" }, { name = "fastapi" }, @@ -567,6 +581,7 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-dotenv" }, + { name = "python-multipart" }, { name = "pyyaml" }, { name = "sqlmodel" }, { name = "structlog" }, @@ -601,12 +616,11 @@ requires-dist = [ { name = "alembic", specifier = ">=1.13.0" }, { name = "anyio", specifier = ">=4.0" }, { name = "apscheduler", specifier = ">=3.10.4,<4.0" }, - { name = "everalgo-agent-memory", specifier = "==0.2.0" }, - { name = "everalgo-boundary", specifier = "==0.2.0" }, - { name = "everalgo-core", specifier = "==0.2.0" }, - { name = "everalgo-parser", extras = ["svg"], marker = "extra == 'multimodal'", specifier = ">=0.1.0" }, - { name = "everalgo-rank", specifier = "==0.3.0" }, - { name = "everalgo-user-memory", specifier = "==0.2.0" }, + { name = "everalgo-agent-memory", specifier = "==0.3.1" }, + { name = "everalgo-knowledge", specifier = "==0.1.1" }, + { name = "everalgo-parser", extras = ["svg"], marker = "extra == 'multimodal'", specifier = ">=0.2.1" }, + { name = "everalgo-rank", specifier = "==0.4.1" }, + { name = "everalgo-user-memory", specifier = "==0.3.1" }, { name = "fastapi", specifier = ">=0.104.0" }, { name = "greenlet", specifier = ">=3.0" }, { name = "jieba", specifier = "==0.42.1" }, @@ -617,6 +631,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, { name = "structlog", specifier = ">=24.0.0" }, @@ -1818,6 +1833,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, +] + [[package]] name = "pywin32" version = "311"