xiaoxue_admin/EverOS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore(release): update EverOS to 1.1.1 (#327)

Browse Source
This commit is contained in:
Elliot Chen
2026-07-07 18:30:03 +08:00
committed by GitHub
parent 66a201e164
commit 15efd1198c
106 changed files with 4414 additions and 4696 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
-24
View File
@@ -30,13 +30,6 @@ EVEROS_LLM__MODEL=openai/gpt-4.1-mini
EVEROS_LLM__API_KEY=
EVEROS_LLM__BASE_URL=https://openrouter.ai/api/v1
# Aliyun Bailian (DashScope) alternative — one key also covers embedding +
# rerank below, so the same value goes in all three API_KEY slots.
# Uncomment these three lines and comment out the OpenRouter lines above:
# EVEROS_LLM__MODEL=qwen-plus
# EVEROS_LLM__API_KEY=<DASHSCOPE_API_KEY>
# EVEROS_LLM__BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
# ─── Multimodal LLM (independent from [llm]; vision/audio capable) ────
# Separate model for parsing multimodal content items (image / pdf /
@@ -66,12 +59,6 @@ EVEROS_MULTIMODAL__BASE_URL=https://openrouter.ai/api/v1
EVEROS_EMBEDDING__MODEL=Qwen/Qwen3-Embedding-4B
EVEROS_EMBEDDING__API_KEY=
EVEROS_EMBEDDING__BASE_URL=https://api.deepinfra.com/v1/openai
# Aliyun Bailian (DashScope) alternative. Uncomment these three lines and
# comment out the DeepInfra embedding lines above:
# EVEROS_EMBEDDING__MODEL=text-embedding-v4
# EVEROS_EMBEDDING__API_KEY=<DASHSCOPE_API_KEY>
# EVEROS_EMBEDDING__BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
# Runtime knobs — uncomment to override defaults (30s / 3 / 10 / 5):
# EVEROS_EMBEDDING__TIMEOUT_SECONDS=30
# EVEROS_EMBEDDING__MAX_RETRIES=3
@@ -81,21 +68,10 @@ EVEROS_EMBEDDING__BASE_URL=https://api.deepinfra.com/v1/openai
# ─── Rerank (OpenAI-protocol /rerank) ────────────────
# base_url should point at the rerank endpoint (e.g. .../v1/rerank).
# EVEROS_RERANK__PROVIDER selects the request-shape: "deepinfra" (default),
# "vllm", or "dashscope". Left unset, it is inferred from the base_url host
# (dashscope.aliyuncs.com -> dashscope, *.deepinfra.com -> deepinfra).
EVEROS_RERANK__MODEL=Qwen/Qwen3-Reranker-4B
EVEROS_RERANK__API_KEY=
EVEROS_RERANK__BASE_URL=https://api.deepinfra.com/v1/inference
# Aliyun Bailian (DashScope) alternative. EverOS currently supports
# DashScope rerank via gte-rerank-v2; provider is auto-inferred from host,
# so EVEROS_RERANK__PROVIDER is optional. Uncomment these three lines and
# comment out the DeepInfra rerank lines above:
# EVEROS_RERANK__MODEL=gte-rerank-v2
# EVEROS_RERANK__API_KEY=<DASHSCOPE_API_KEY>
# EVEROS_RERANK__BASE_URL=https://dashscope.aliyuncs.com
# Runtime knobs — uncomment to override defaults (30s / 3 / 10 / 5):
# EVEROS_RERANK__TIMEOUT_SECONDS=30
# EVEROS_RERANK__MAX_RETRIES=3
.github/workflows/ci.yml
+46 -1
View File
@@ -1,3 +1,6 @@
# CI is a thin wrapper over the Makefile — single source of truth for commands.
# See docs/engineering.md and the Makefile for what each target does.
name: CI
on:
@@ -32,7 +35,7 @@ jobs:
- name: Install dependencies (frozen)
run: make install-deps
- name: Lint (ruff + import-linter + datetime + openapi drift)
- name: Lint (ruff + import-linter + repo gates + datetime + openapi drift)
run: make lint
unit:
@@ -56,6 +59,27 @@ jobs:
- name: Unit tests
run: make test
unit-py313:
name: unit tests (3.13)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v8.2.0
with:
enable-cache: true
cache-dependency-glob: uv.lock
- name: Set up Python
run: uv python install 3.13
- name: Install dependencies (frozen)
run: make install-deps
- name: Unit tests
run: make test
integration:
name: integration tests
runs-on: ubuntu-latest
@@ -77,6 +101,27 @@ jobs:
- name: Integration tests
run: make integration
integration-py313:
name: integration tests (3.13)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v8.2.0
with:
enable-cache: true
cache-dependency-glob: uv.lock
- name: Set up Python
run: uv python install 3.13
- name: Install dependencies (frozen)
run: make install-deps
- name: Integration tests
run: make integration
package:
name: package build
runs-on: ubuntu-latest
.gitignore
+11 -6
View File
@@ -107,6 +107,7 @@ celerybeat.pid
# Environments
.env
.env.*
!.env.example
.venv
env/
venv/
@@ -258,13 +259,17 @@ evaluation/locomo_evaluation/data/locomo10.json
# Legacy src kept locally for migration reference; not under version control.
src_old/
# Benchmark checkpoints — Phase-level intermediate JSON dumps produced by
# tests/test_locomo.py / tests/run_locomo_batch.sh. Per-conv final results
# (benchmark_results/run_*/convN.json) and reports (REPORT.md / REPORT.html)
# stay tracked; checkpoints are large, regeneratable, and not useful in
# code review.
# Benchmark results — JSONL outputs, reports, and run specs produced by
# benchmarks/run.py. These are large, regeneratable, and not useful in
# code review. Legacy benchmark_results/ and benchmark_checkpoints/ kept
# for any lingering local data from pre-refactor runs.
benchmarks/results/
benchmark_results/
benchmark_checkpoints/
# Benchmark credentials — API keys + base URLs; covered by the top-level
# .env pattern but listed explicitly for discoverability.
benchmarks/.env
# Local everos runtime data (memory root, indexes, OME state)
.everos/
.everos*/
CHANGELOG.md
+53 -1
View File
@@ -7,6 +7,57 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
## [1.1.1] - 2026-07-06
### Added
- **DashScope rerank provider** — Aliyun Bailian `gte-rerank-v2` adapter;
configure with `rerank.provider = "dashscope"` in `everos.toml`.
- **`everos demo` TUI command** — Textual-based interactive CLI demo for
showcasing EverOS core features.
- **Benchmark runner** — full LoCoMo benchmark suite: `benchmarks/run.py`
with TOML configuration, automated ingestion, search evaluation, and
scoring.
- **Hybrid search: heap-expand algorithm** — rewrote `hierarchy.py` to
heap-driven lazy expansion with global top-N competition, replacing the
serial four-layer pipeline.
### Fixed
- **Knowledge: atomic upsert prevents StaleDataError** — cascade handler
switched from get→update to `INSERT ... ON CONFLICT DO UPDATE`, fixing
concurrent cascade race conditions.
- **API: OpenAPI version read from `__version__`** — no longer hardcoded to
`0.1.0`; version now stays in sync with `pyproject.toml`.
- **Profile middleware no longer swallows exceptions** — inner handler
errors now re-raise correctly instead of silently returning HTTP 200.
### Performance
- **Cascade optimize throttle 1s → 10s** — reduced unnecessary LanceDB
`optimize()` I/O by raising the minimum interval between calls.
### CI / Build
- **CI Python version matrix** — test and integration jobs now run on both
Python 3.12 and 3.13.
- **pyproject.toml improvements** — added `project.urls`, `Typing :: Typed`
classifier, relaxed `jieba` version constraint, removed unused
`python-dotenv` dependency, cleaned up sdist include list, added `RUF`
lint rules and coverage configuration.
- **`make ci` includes coverage** — `ci` target now runs
`lint + test + integration + cov`.
### Documentation
- Fixed stale references across 13 files (v1.1.0 freshness sweep).
- Added GitHub sync guide (`docs/github-sync.md`).
- Added v1.1.0 release notes and v1.0.0 migration guide as standalone docs.
- Added `README.zh-CN.md` (Chinese README).
- Expanded `QUICKSTART.md` with source install instructions and `uv run`
usage notes.
- Clarified cascade `optimize()` semantics in docstrings and runbook.
## [1.1.0] - 2026-06-24
### Added
@@ -164,7 +215,8 @@ for AI agents.
- **Decoupled algorithms** — memory extraction algorithms live in the standalone
`everalgo-*` libraries published on PyPI.
[Unreleased]: https://github.com/EverMind-AI/everos/compare/v1.1.0...HEAD
[Unreleased]: https://github.com/EverMind-AI/everos/compare/v1.1.1...HEAD
[1.1.1]: https://github.com/EverMind-AI/everos/compare/v1.1.0...v1.1.1
[1.1.0]: https://github.com/EverMind-AI/everos/compare/v1.0.1...v1.1.0
[1.0.1]: https://github.com/EverMind-AI/everos/releases/tag/v1.0.1
[1.0.0]: https://github.com/EverMind-AI/everos/releases/tag/v1.0.0
QUICKSTART.md
+163 -202
View File
@@ -10,107 +10,147 @@ front of your agent.
## Prerequisites
- **Python 3.12+**
- **An LLM provider key and endpoint** — for memory extraction. OpenRouter,
OpenAI, and other OpenAI-compatible providers work when you set
`base_url`.
- **A multimodal provider key and endpoint** — needed only when parsing
image / pdf / audio content items.
- **Embedding and rerank provider keys and endpoints** — for search. DeepInfra
works for the embedding + rerank path; vLLM and DashScope are also supported
for rerank.
Many deployments use two distinct keys by reusing one LLM key for `[llm]` and
`[multimodal]`, and one DeepInfra key for `[embedding]` and `[rerank]`. Any
setting can live in TOML or be overridden by the matching `EVEROS_*`
environment variable.
- **API keys** for three capabilities: a chat LLM (memory extraction),
an embedding model (vector retrieval), and a reranker. Any
OpenAI-compatible endpoint works.
## 1. Install
**From PyPI** (users):
```bash
pip install everos
# or: uv pip install everos
```
## 2. Configure
Generate the starter config and fill in provider credentials:
**From source** (contributors / developers):
```bash
everos init # writes ~/.everos/everos.toml + ome.toml (use --root to relocate)
# Edit ~/.everos/everos.toml and fill the provider fields:
# [llm] api_key + base_url (chat LLM)
# [multimodal] api_key + base_url (optional parser LLM)
# [embedding] model + api_key + base_url
# [rerank] model + api_key + base_url (provider can be inferred or set)
git clone https://github.com/EverMind-AI/EverOS.git
cd EverOS
uv sync # install all deps into .venv
```
`everos init` generates two files: `everos.toml` (provider settings)
and `ome.toml` (offline memory engine strategy config, hot-reloaded).
Because `everos.toml` holds API keys, consider restricting access
after editing: `chmod 600 ~/.everos/everos.toml`.
> **Note:** source install creates a `.venv` virtualenv. Subsequent
> `everos` commands need either `uv run everos ...` or activate the venv
> first (`source .venv/bin/activate`).
The shipped template sets model defaults for `[llm]` (`gpt-4.1-mini`) and
`[multimodal]` (`google/gemini-3-flash-preview`); `[embedding]` and
`[rerank]` ship no model default — set `model` + `base_url` for those two
sections yourself (for example DeepInfra's `Qwen/Qwen3-Embedding-4B` /
`Qwen/Qwen3-Reranker-4B`). `[llm]` and `[multimodal]` ship model defaults,
but still need `api_key` and `base_url` before those capabilities are used.
## 2. Configure
> **Where config lives** — `everos init` writes into the memory root
> (`~/.everos` by default; relocate with `everos init --root <path>` and
> start the server with the matching `everos server start --root <path>`).
> `everos server start` reads `<root>/everos.toml` and exits with an error
> if it is missing. Any setting can also be overridden by an `EVEROS_*`
> environment variable (e.g. `EVEROS_LLM__API_KEY`) — handy for containers
```bash
everos init # default root: ~/.everos
everos init --root /data/everos # or specify a custom root
```
> **Root directory** — defaults to `~/.everos`. Use `--root <path>` to
> relocate; all subsequent commands (`server start`, `cascade status`,
> etc.) must use the matching `--root`. Any setting in `everos.toml` can
> also be overridden via `EVEROS_*` environment variables for containers
> and CI.
This creates `everos.toml` and `ome.toml` under the root directory.
Open `everos.toml` and fill in three sections — here's the minimum
viable config:
```toml
[llm]
model = "gpt-4.1-mini" # or your preferred model
base_url = "https://openrouter.ai/api/v1" # any OpenAI-compatible endpoint
api_key = "sk-..." # your API key
[embedding]
model = "Qwen/Qwen3-Embedding-4B"
base_url = "https://api.deepinfra.com/v1/openai"
api_key = "..."
[rerank]
provider = "deepinfra"
model = "Qwen/Qwen3-Reranker-4B"
base_url = "https://api.deepinfra.com/v1/inference"
api_key = "..."
```
The generated file pre-fills recommended `model` and `base_url`
defaults — just drop in your API keys. Any OpenAI-compatible endpoint
works.
> **Multimodal** (`[multimodal]`) is optional — only needed when
> ingesting image / pdf / audio content items. See
> [docs/multimodal.md](docs/multimodal.md) for setup.
## 3. Start the server
Check your file descriptor limit — EverOS opens many LanceDB segment
files under concurrent search + indexing. Platform defaults:
**macOS 256** · **Linux 1024** · **Windows 8192**. If yours is below
4096, raise it before starting:
Run these in the **same terminal** where you will start the server —
`ulimit` is per-shell-session, not global:
```bash
everos server start
ulimit -n # check current limit
ulimit -n 4096 # raise if needed
everos server start [--root <path>] # must be in the same session
```
You should see (port and host are configurable):
> **No side effects** — `ulimit -n` only raises the per-process ceiling.
> It does not pre-allocate memory or file handles, and has zero
> performance cost. For Linux production, set `LimitNOFILE=65536` in
> your systemd unit file.
You should see:
```
starting everos on 127.0.0.1:8000
INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
```
- Default bind is `127.0.0.1` (loopback only). To expose the API
elsewhere, put your own auth/gateway in front first
([see SECURITY.md](SECURITY.md)).
- The cascade index daemon runs **in the same process** as a FastAPI
lifespan coroutine — you don't need a separate worker.
- The server runs in the foreground; **open a second terminal** for the
steps below, and use `Ctrl+C` to stop the server when you're done.
The server runs in the foreground. **Open a second terminal** for the
steps below.
In the second terminal, verify the server is up:
Verify it's up:
```bash
$ curl http://127.0.0.1:8000/health
{"status":"ok"}
curl http://127.0.0.1:8000/health
# {"status":"ok"}
```
## 4. Add a conversation
EverOS ingests memory at the **conversation level**, not as standalone
sentences: you POST a batch of `messages` tied to a `session_id`, and
the server accumulates them until the boundary detector trips (you can
also force a flush — see step 5).
Send messages to the server — one at a time or in batches. Each batch
belongs to a `session_id`, which represents one conversation thread.
Timestamps are Unix epoch in **milliseconds** (UTC).
First, a chat about climbing:
```bash
TS=$(($(date +%s)*1000)) # Unix epoch in **milliseconds** (v1 contract)
TS=$(($(date +%s)*1000))
curl -X POST http://127.0.0.1:8000/api/v1/memory/add \
-H 'Content-Type: application/json' \
-d "{
\"session_id\": \"demo-001\",
\"app_id\": \"default\",
\"project_id\": \"default\",
\"messages\": [
{\"sender_id\": \"alice\", \"role\": \"user\", \"timestamp\": $TS, \"content\": \"I love climbing in Yosemite every spring.\"},
{\"sender_id\": \"alice\", \"role\": \"user\", \"timestamp\": $((TS+10000)), \"content\": \"My favorite coffee shop is Blue Bottle in SOMA.\"},
{\"sender_id\": \"alice\", \"role\": \"user\", \"timestamp\": $((TS+20000)), \"content\": \"I bike to work most days.\"}
{\"sender_id\": \"alice\", \"role\": \"user\", \"timestamp\": $TS, \"content\": \"I just got back from a week in Yosemite. The climbing was incredible.\"},
{\"sender_id\": \"agent1\", \"role\": \"assistant\", \"timestamp\": $((TS+10000)), \"content\": \"That sounds amazing! Which routes did you do?\"},
{\"sender_id\": \"alice\", \"role\": \"user\", \"timestamp\": $((TS+20000)), \"content\": \"Mostly cracks on El Cap. I go every spring — it's my favorite season there.\"}
]
}"
# → status: "accumulated"
```
Now the topic shifts to work:
```bash
curl -X POST http://127.0.0.1:8000/api/v1/memory/add \
-H 'Content-Type: application/json' \
-d "{
\"session_id\": \"demo-001\",
\"messages\": [
{\"sender_id\": \"alice\", \"role\": \"user\", \"timestamp\": $((TS+60000)), \"content\": \"By the way, I switched to biking to work last month. Loving it so far.\"},
{\"sender_id\": \"agent1\", \"role\": \"assistant\", \"timestamp\": $((TS+70000)), \"content\": \"How long is your commute?\"},
{\"sender_id\": \"alice\", \"role\": \"user\", \"timestamp\": $((TS+80000)), \"content\": \"About 25 minutes. I stop at Blue Bottle in SOMA for coffee most mornings.\"}
]
}"
```
@@ -119,52 +159,53 @@ Response:
```json
{
"request_id": "bf86e4e857834eba804841f8bff29106",
"data": {
"message_count": 3,
"status": "accumulated"
"status": "extracted"
}
}
```
`status: "accumulated"` means the three messages are in the session
buffer, but the boundary detector hasn't decided to extract a memory
cell yet. For a quick demo we'll force it.
EverOS detected a topic shift (climbing → commute) and automatically
extracted the earlier conversation into memory.
## 5. Force boundary extraction
The `status` field tells you what happened:
| Status | Meaning |
|---|---|
| `accumulated` | Messages buffered, still part of the same topic. |
| `extracted` | Topic shift detected — memory extracted from the buffer. |
> For the full API contract, see [docs/openapi.json](docs/openapi.json).
## 5. Flush (manual extraction)
If you want to extract memory without waiting for a topic shift — for
example at the end of a session — call `/flush`:
```bash
curl -X POST http://127.0.0.1:8000/api/v1/memory/flush \
-H 'Content-Type: application/json' \
-d '{"session_id":"demo-001","app_id":"default","project_id":"default"}'
-d '{"session_id":"demo-001"}'
```
Response (this takes a few seconds — one LLM call for extraction):
```json
{
"request_id": "ec0e7a00c3bd4b00bb21212a411b7763",
"data": {
"status": "extracted"
}
}
```
`status: "extracted"` means at least one memory cell was carved out and
written to disk + indexed.
This forces extraction of whatever is still in the buffer.
> `/flush` is **OSS-only**. The cloud edition decides boundary timing
> server-side and does not expose this endpoint.
## 6. Search the memory you just added
## 6. Search
```bash
curl -X POST http://127.0.0.1:8000/api/v1/memory/search \
-H 'Content-Type: application/json' \
-d '{
"user_id": "alice",
"app_id": "default",
"project_id": "default",
"query": "Where do I like to climb?",
"top_k": 5
}'
@@ -174,158 +215,78 @@ Response (trimmed):
```json
{
"request_id": "b53a3a94a080472d97692c503c88afdf",
"data": {
"episodes": [
{
"id": "alice_ep_20260528_00000002",
"user_id": "alice",
"session_id": "demo-001",
"summary": "On May 28, 2026 ... Alice shared that she loves climbing in Yosemite every spring ...",
"score": 0.6284722685813904,
"summary": "... Alice shared that she loves climbing in Yosemite every spring ...",
"score": 0.628,
"atomic_facts": [
{
"id": "alice_af_20260528_00000016",
"content": "Alice said she loves climbing in Yosemite every spring.",
"score": 0.6284722685813904
"score": 0.628
}
]
}
],
"profiles": [],
"agent_cases": [],
"agent_skills": []
]
}
}
```
The hybrid retrieval (BM25 + vector + scalar) returns the episode
that contains the climbing fact, with the matching atomic fact nested
under it. Other response arrays (`profiles` / `agent_cases` /
`agent_skills`) are always present for client-side symmetry, populated
only when the requested kind matches.
Hybrid retrieval (BM25 + vector + scalar) returns the matching episode
with its atomic facts nested under it.
## 7. Your memory is just Markdown
This is what makes EverOS different — your memory persists as plain
Markdown files on disk:
This is what makes EverOS different — memory persists as plain Markdown:
```
$ tree ~/.everos -L 5 -a
~/.everos
├── default_app/ ← app_id ("default" → "default_app")
│ └── default_project/ ← project_id ("default" → "default_project")
│ ├── users/
│ │ └── alice/ ← user_id (mirror dir: agents/<agent_id>/)
│ │ ├── episodes/
│ │ │ └── episode-2026-05-28.md
│ │ ├── .atomic_facts/ ← hidden (dot-prefix)
│ │ │ └── atomic_fact-2026-05-28.md
│ │ ├── .foresights/
│ │ │ └── foresight-2026-05-28.md
│ │ └── user.md ← profile
│ └── knowledge/ ← shared knowledge base (v1.1+)
│ └── .taxonomy.md
├── everos.toml ← provider config (API keys)
<root>/ ← ~/.everos or your --root path
├── default_app/ ← app_id ("default" → "default_app")
│ └── default_project/ ← project_id ("default" → "default_project")
│ ├── users/<user_id>/
│ │ ├── user.md ← profile
│ │ ├── episodes/ ← daily-log episodes
│ │ ├── .atomic_facts/ ← nested facts (dot-hidden)
│ │ └── .foresights/ ← predictive memory (dot-hidden)
│ ├── agents/<agent_id>/
│ │ ├── agent.md
│ │ ├── .cases/ ← task cases
│ │ └── skills/ ← procedural memories
│ └── knowledge/ ← shared knowledge base
├── everos.toml ← provider config
├── ome.toml ← strategy config (hot-reloaded)
├── .index/ ← derived indexes (rebuildable from md)
│ ├── sqlite/system.db
│ └── lancedb/*.lance/
│ └── lancedb/
└── .tmp/
```
The `default` scope id materialises as `default_app` / `default_project`
on disk (with the `_app` / `_project` suffix) so the default space is
visually distinct from any user-named space. Any other id maps to itself
(e.g. `app_id: "my-app"` → `my-app/`).
Top-level `.index/` holds SQLite + LanceDB **derived** indexes — wipe it
and the cascade daemon rebuilds everything from the Markdown alone.
Read the episode we just created:
```
$ cat ~/.everos/default_app/default_project/users/alice/episodes/episode-2026-05-28.md
---
id: episode_log_alice_2026-05-28
type: episode_daily
file_type: episode_daily
schema_version: 1
user_id: alice
track: user
date: '2026-05-28'
entry_count: 1
last_appended_at: '2026-05-28T08:32:24.966944+00:00'
---
<!-- entry:ep_20260528_00000002 -->
## ep_20260528_00000002
**owner_id**: alice
**session_id**: demo-001
**timestamp**: 2026-05-28T08:32:13+00:00
**parent_type**: memcell
**parent_id**: mc_3779c20f1c53
**sender_ids**: [alice]
### Subject
Alice's Outdoor Activities and Daily Routine on May 28, 2026 Morning
### Content
On May 28, 2026 at 8:32 AM UTC, Alice shared that she loves climbing in
Yosemite every spring, highlighting a recurring seasonal outdoor activity.
She also mentioned that her favorite coffee shop is Blue Bottle located in
SOMA, indicating a preferred local spot. Additionally, Alice stated that
she bikes to work most days, revealing a habitual commuting practice.
<!-- /entry:ep_20260528_00000002 -->
```
Every memory entry is a plain Markdown file you can:
- `cat` / `grep` / `vim` directly — no driver, no service to query
- Version with Git (or rsync to backup)
- Open the `~/.everos/default_app/default_project/users/alice/` folder
in Obsidian (the dotfile directories stay hidden by default)
Every memory entry is a plain Markdown file you can directly read and
edit — no database driver needed.
## Stopping the server
`Ctrl+C` in the server terminal. Uvicorn catches `SIGINT` and shuts each
lifespan provider down in reverse order (cascade → LanceDB → SQLite →
LLM → metrics) before exiting.
`Ctrl+C` in the server terminal.
## Next steps
- **Integrate into your agent** — wrap the three endpoints (`/add`,
`/flush`, `/search`) in a thin Python client (`httpx.AsyncClient`) and
call them from your agent loop.
- **App + project scope** — set `app_id` / `project_id` to anything
other than `"default"` to partition memory spaces inside one server.
- **Knowledge base** — upload documents (PDF / HTML / DOCX) via
`/api/v1/knowledge/documents` and search them with hybrid retrieval
at `/api/v1/knowledge/search`. Ships with a 20-category default
taxonomy. See [docs/knowledge.md](docs/knowledge.md).
- **Reflection** — offline memory self-improvement that consolidates
related episodes. Disabled by default; enable in `ome.toml`
(`[strategies.reflect_episodes] enabled = true`). Changes are
hot-reloaded, no server restart needed.
- **Multi-modal messages** — `messages[].content` accepts a list of
typed `ContentItem`s (`text` / `image` / `audio` / `doc` / `pdf` /
`html` / `email`) for non-text input. Install the optional extra
to enable parsing:
`uv pip install 'everos[multimodal]'`. Office documents
(`doc` / `docx` / `xls` / `ppt` / `…`) additionally need
**LibreOffice** on the host (`brew install --cask libreoffice` /
`apt-get install libreoffice`) — without it those uploads return
HTTP 503 (`CAPABILITY_UNAVAILABLE`); PDF / image / audio / HTML
still work.
- **Filter DSL and search modes** — `/search` supports a filter DSL
(`AND` / `OR` / scalar predicates) and four methods (`HYBRID` /
`KEYWORD` / `VECTOR` / `AGENTIC`). The OpenAPI docs UI is served at
`/docs` only when the server runs with `ENV=DEV`; the default (`prod`)
serves the API without the docs UI. The schema also lives at
[docs/openapi.json](docs/openapi.json).
- **Architecture** — see [docs/architecture.md](docs/architecture.md)
for the DDD layering and cascade design, and
[docs/storage_layout.md](docs/storage_layout.md) for the on-disk
layout.
- **Found a bug?** — open an issue (see [CONTRIBUTING.md](CONTRIBUTING.md);
external pull requests are not merged).
- **Integrate into your agent** — wrap `/add`, `/flush`, `/search` in a
thin HTTP client and call them from your agent loop.
- **App + project scope** — pass `app_id` / `project_id` in your API
requests to partition memory spaces inside one server (defaults to
`"default"` when omitted).
- **Knowledge base** — upload documents via
`/api/v1/knowledge/documents` and search with hybrid retrieval. See
[docs/knowledge.md](docs/knowledge.md).
- **Reflection** — offline memory consolidation; enable in `ome.toml`.
See [docs/reflection.md](docs/reflection.md).
- **Multimodal** — ingest image / pdf / audio / office documents. See
[docs/multimodal.md](docs/multimodal.md).
- **Search modes** — four methods (`HYBRID` / `KEYWORD` / `VECTOR` /
`AGENTIC`) with a filter DSL. See [docs/openapi.json](docs/openapi.json)
for the full API schema.
- **Architecture** — [docs/architecture.md](docs/architecture.md) for
DDD layering; [docs/storage_layout.md](docs/storage_layout.md) for
on-disk layout.
- **Found a bug?** — [open an issue](CONTRIBUTING.md).
SECURITY.md
+4 -5
View File
@@ -2,7 +2,7 @@
## Supported Versions
EverOS is released and in the v1.x stable line. Security fixes are applied to the
EverOS is released and at v1.0.0 (stable). Security fixes are applied to the
latest release line only.
| Version | Supported |
@@ -39,9 +39,8 @@ following in mind:
is loopback-only. Only set the bind to `0.0.0.0` (or any routable interface)
after you have placed your own gateway / auth layer in front;
`everos server start` will log a warning when you bind to `0.0.0.0`.
- Secrets (LLM / embedding / rerank API keys) normally live in your local
`<root>/everos.toml`, or in `EVEROS_*` environment variables for container
deployments. Protect those values as you would any credential. EverOS never
transmits them anywhere except the providers you configure.
- Secrets (LLM / embedding API keys) live in your local `.env`; protect that
file as you would any credential. EverOS never transmits them anywhere except
the providers you configure.
- Memory content is stored as plaintext `.md` files; apply OS-level file
permissions or disk encryption if your data is sensitive.
benchmarks/.env.example
+6
View File
@@ -0,0 +1,6 @@
# Provider connection info for Answer + Judge LLM phases.
# API keys are comma-separated for key rotation (round-robin failover).
ANSWER_API_KEY=sk-...
ANSWER_BASE_URL=https://openrouter.ai/api/v1
JUDGE_API_KEY=sk-...
JUDGE_BASE_URL=https://openrouter.ai/api/v1
benchmarks/README.md
+338
View File
@@ -0,0 +1,338 @@
# Running the LoCoMo Benchmark
EverOS ships a self-contained runner for the
[LoCoMo](https://github.com/snap-research/locomo) (Long Conversation Memory)
benchmark ([Maharana et al., 2024](https://arxiv.org/abs/2402.17753)).
LoCoMo evaluates how well a memory system retrieves facts from long
multi-session dialogues across four question categories: **single-hop**,
**multi-hop**, **open-domain**, and **temporal**. This guide walks through
reproducing EverOS's LoCoMo retrieval scores locally.
## Pipeline at a glance
Each conversation runs through a four-stage pipeline:
```
ADD ──► wait_ready ──► SEARCH ──► ANSWER ──► JUDGE
│ │ │ │ │
│ ingest msgs & query EverOS generate LLM-as-judge
│ flush per-session per QA pair answers majority vote
│ into EverOS from (judge_runs×)
│ context
▼
cascade + OME drain
(per-conv polling)
```
- **ADD** — sends LoCoMo sessions to EverOS (`/add` + `/flush`), then polls
cascade and OME queues until the conversation's data is fully indexed.
- **SEARCH** — queries EverOS `/search` for each QA question.
- **ANSWER** — feeds retrieved episodes to an LLM to generate an answer.
- **JUDGE** — an LLM judge scores each answer as CORRECT or WRONG against
the gold answer; runs `judge_runs` times per question and majority-votes.
Stages are **independently re-runnable** — each reads from and writes to
JSONL files, so you can re-judge with a different model without re-ingesting
or re-searching.
Multiple conversations run **in parallel** via `conversations_concurrency`.
Within each conversation, search and eval questions run concurrently via
`search_concurrency` and `eval_concurrency`.
## Contents
- [Prerequisites](#prerequisites)
- [Configuration](#configuration)
- [1. Prepare the dataset](#1-prepare-the-dataset)
- [2. Start the server](#2-start-the-server)
- [3. Run the benchmark](#3-run-the-benchmark)
- [4. Output](#4-output)
- [CLI reference](#cli-reference)
- [Notes](#notes)
---
## Prerequisites
- A working EverOS installation — complete **all steps** in
[QUICKSTART.md](../QUICKSTART.md) (configure providers, start server,
verify search works — not just `/health`)
- Python 3.12+ with `tqdm` installed (`pip install tqdm`)
- EverOS configured for chat-only extraction — in your `everos.toml`:
```toml
[memorize]
mode = "chat"
```
And in `ome.toml`, disable strategies the benchmark does not use:
```toml
[strategies.extract_foresight]
enabled = false
[strategies.extract_user_profile]
enabled = false
```
This keeps episode extraction, `extract_atomic_facts`, and
`trigger_profile_clustering` (agentic search relies on clusters),
while cutting unnecessary LLM calls from foresight and profile
extraction.
- Copy `benchmarks/.env.example` → `benchmarks/.env` and fill in your API
keys:
```bash
cp benchmarks/.env.example benchmarks/.env
# Edit benchmarks/.env:
ANSWER_API_KEY=sk-... # LLM for generating answers
ANSWER_BASE_URL=https://openrouter.ai/api/v1
JUDGE_API_KEY=sk-... # LLM for judging answers
JUDGE_BASE_URL=https://openrouter.ai/api/v1
```
Keys are comma-separated for round-robin failover (e.g.
`ANSWER_API_KEY=sk-aaa,sk-bbb`). More keys raise the effective RPM
ceiling, which lets you increase `eval_concurrency` in `config.toml` for
faster answer/judge throughput.
## Configuration
The only required configuration is provider credentials — copy
`benchmarks/.env.example` → `benchmarks/.env` and fill in your API keys
(already done in [Prerequisites](#prerequisites)).
Everything else has sensible defaults in `benchmarks/config.toml` — see
the comments in that file for tunable parameters.
## 1. Prepare the dataset
LoCoMo 10 contains 10 multi-session conversations (~50 sessions each,
~150 QA pairs per conversation across 4 categories, adversarial
category excluded).
```bash
mkdir -p data
curl -o data/locomo10.json \
https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json
```
## 2. Start the server
Raise the file descriptor limit **before** starting — concurrent agentic
searches open many LanceDB segment files simultaneously (EverOS compacts
segments automatically, but burst concurrency during benchmark can exceed
the default macOS limit of 256):
```bash
ulimit -n 10240
everos server start [--root <path>]
```
> **Important:** if you use a custom `--root`, pass the same path to the
> benchmark runner via `--everos-root` — the runner polls the cascade and
> OME databases under that root to know when data is ready. A mismatch
> causes silent readiness false-positives.
## 3. Run the benchmark
All runs require `--run-name`, which becomes the `project_id` used for data
isolation (see [Run isolation](#run-isolation) below).
**Smoke test first** — verify end-to-end connectivity before a full run:
```bash
python benchmarks/run.py --run-name smoke --smoke [--everos-root <path>]
```
**Full run (all 10 conversations):**
```bash
python benchmarks/run.py --run-name locomo-agentic [--everos-root <path>]
```
**Single conversation:**
```bash
python benchmarks/run.py --run-name locomo-agentic --conv 0 [--everos-root <path>]
```
**Skip ingest, re-run search + answer + judge:**
```bash
python benchmarks/run.py --run-name locomo-agentic --stages search answer judge
```
**Re-judge only (reuse existing answer JSONL):**
```bash
python benchmarks/run.py --run-name locomo-agentic --stages judge
```
## 4. Output
Output root is `benchmarks/results/<run-name>/`:
```
benchmarks/results/<run-name>/
├── run_spec.json # reproducibility snapshot (git hash, config, stages)
├── conv0/
│ ├── search_<method>.jsonl # per-question search results
│ ├── answer_<method>.jsonl # per-question generated answers
│ ├── judge_<method>.jsonl # per-question judge verdicts
│ └── error.log # only on failure — full traceback
├── conv1/ … conv9/
├── report.json # aggregate accuracy by method + category
└── report.txt # human-readable accuracy table
```
`report.json` and `report.txt` are written after all conversations finish
(only when the `judge` stage is included).
**Sample `report.txt`:**
```
================================================================
EverOS LoCoMo Benchmark Report
================================================================
Run Info
Run name: locomo-agentic
Generated: 2026-06-28T14:30:00+00:00
Git hash: abc1234
EverOS version: 1.1.0
Python: 3.12.11
Conversations: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Stages: ['add', 'search', 'answer', 'judge']
Configuration
Answer model: gpt-4.1-mini
Judge model: gpt-4o-mini
Judge runs: 3
Top-k: 10
Eval owner: speaker_a
----------------------------------------------------------------
Method: agentic
----------------------------------------------------------------
Max accuracy: 93.4% (best of 3 judge runs / mean / majority)
Majority: 93.3% (1437/1540)
Mean accuracy: 93.3% (avg across 3 judge runs)
Per category:
1. single-hop 94.0% (265/282)
2. multi-hop 91.0% (292/321)
3. open-domain 80.2% (77/96)
4. temporal 95.5% (803/841)
Per conversation:
conv0 93.4% (142/152)
conv1 96.3% (78/81)
...
Search: 1540 queries, avg 23.1s, p50 19.4s, max 142.3s
Answer: 1540 questions, avg 4.7s, 7,224,168 tokens
Judge: 1540 questions × 3 runs, 2,335,683 tokens, unanimous 95.2%
Total tokens: 9,559,851
```
## CLI reference
| Flag | Default | Description |
|---|---|---|
| `--run-name` | *(required)* | Run name — maps to `project_id` for data isolation |
| `--conv` | `0 1 2 … 9` | Conversation indices to run |
| `--stages` | `add search answer judge` | Pipeline stages to execute |
| `--config` | `config` | TOML config name (without `.toml` extension) |
| `--base-url` | `http://localhost:8000` | EverOS server address |
| `--everos-root` | `~/.everos` | EverOS root path (for cascade/OME queue polling) |
| `--data-path` | `data/locomo10.json` | Path to LoCoMo dataset JSON |
| `--smoke` | off | Smoke mode: 2 convs, first 50 msgs each, 10 QA (stratified), `judge_runs=1` |
## Notes
### Evaluation methodology
The runner uses an **LLM-as-Judge** approach: a judge LLM receives the
question, the gold answer, and the generated answer, then outputs `CORRECT`
or `WRONG`. Each question is judged `judge_runs` times (default 3); the
final verdict is a **majority vote**. Accuracy = correct / total per method
and per category.
The four LoCoMo question categories test different retrieval capabilities:
| Category | Name | Tests |
|---|---|---|
| 1 | single-hop | Direct fact retrieval from one episode |
| 2 | multi-hop | Reasoning across multiple episodes |
| 3 | open-domain | General knowledge grounded in conversation |
| 4 | temporal | Time-sensitive questions requiring date reasoning |
Category 5 (adversarial — questions with no answer in the conversation) is
excluded from evaluation.
### Run isolation
Each benchmark run is scoped by three identifiers:
| Scope | Value | Purpose |
|---|---|---|
| `app_id` | `locomo_benchmark` | Fixed; separates benchmark data from production |
| `project_id` | `--run-name` value | Per-experiment isolation |
| `owner_id` | `<speaker>_conv<N>` | Per-conversation memory partition |
Two runs with the **same** `--run-name` share the same memory corpus —
useful when re-running later stages, but problematic if you want a clean
ingest. Use distinct names (e.g. `locomo-agentic`, `locomo-hybrid`) for
independent experiments.
### Stage independence
Each stage reads from and writes to JSONL files in `conv<N>/`. This means:
- `--stages search` reads from the EverOS server (requires prior `add`).
- `--stages answer` reads `search_<method>.jsonl` (requires prior `search`).
- `--stages judge` reads `answer_<method>.jsonl` (requires prior `answer`).
You can swap the judge model and re-run `--stages judge` without touching
ingest or search.
### Smoke mode
`--smoke` is a **pipeline sanity check**, not a scored run. It forces:
- 2 conversations (conv 0, 1) running in parallel
- First 50 messages each (across however many sessions that covers)
- 10 QA pairs per conversation, stratified-sampled to cover all categories
- `judge_runs=1` (no majority vote)
Use it to verify end-to-end connectivity before committing to a full run.
### Runtime estimates
Rough estimates with default settings (varies by provider latency):
| Scope | Time | Token cost (approx.) |
|---|---|---|
| Smoke | 2–5 min | ~80k tokens |
| Single conv (full) | 15–30 min | ~1M tokens |
| Full 10-conv run | 2–4 hours | ~10M tokens |
The `add` + `wait_ready` phase dominates wall-clock time; LLM calls
(answer + judge) dominate token cost.
### Troubleshooting
| Symptom | Cause | Fix |
|---|---|---|
| `Connection refused` on run | EverOS server not running | `everos server start` |
| `ANSWER_API_KEY not set` | Missing `.env` | Copy `.env.example` → `.env`, fill keys |
| `Timeout after 1800s` in wait_ready | Cascade/OME still processing | Increase `cascade_timeout` in config.toml or check server logs |
| `OME task(s) failed` warning | OME strategy crashed | Check `everos cascade status`; data may be incomplete |
| `Missing search_*.jsonl` | Running `answer` without prior `search` | Add `search` to `--stages` or run it first |
| `Too many open files (os error 24)` | LanceDB FD exhaustion from concurrent searches | Lower `search_concurrency` in config.toml (agentic needs more FDs per query) or raise `ulimit -n` |
| Low accuracy across all categories | Embedding/rerank not configured | Verify `everos.toml` has working embedding + rerank providers |
| `conv<N>/error.log` exists | Unhandled exception in that conversation | Read the traceback; other conversations are unaffected |
benchmarks/__init__.py
View File
benchmarks/config.py
+157
View File
@@ -0,0 +1,157 @@
"""Benchmark configuration.
Frozen Pydantic model providing all tunable parameters for the LoCoMo
benchmark pipeline. Defaults are aligned with the upstream evaluation
reference so that numbers are directly comparable.
"""
from __future__ import annotations
import tomllib
from pathlib import Path
from typing import Literal
from pydantic import BaseModel, ConfigDict
class BenchmarkConfig(BaseModel):
"""Immutable benchmark configuration.
Args:
cascade_timeout: Max seconds to wait for cascade queue to drain after flush.
batch_size: Messages per /add request.
methods: Comma-separated search methods.
top_k: Number of episodes to retrieve per question.
eval_owner: Which speaker's memory partition to query.
answer_model: LLM model for the Answer phase.
answer_temperature: Sampling temperature for answers.
answer_max_tokens: Max output tokens per answer call.
answer_timeout: Per-request timeout (seconds) for the answer LLM.
answer_max_retries: Retry budget for the answer phase.
judge_model: LLM model for the Judge phase.
judge_temperature: Sampling temperature for judging.
judge_timeout: Per-request timeout (seconds) for the judge LLM.
judge_max_retries: Retry budget for the judge phase.
judge_runs: Independent judge evaluations per question (majority vote).
conversations_concurrency: How many conversations run at the same time.
eval_concurrency: How many questions are processed in parallel within each conversation.
"""
model_config = ConfigDict(frozen=True)
# --- EverOS server ---
cascade_timeout: int = 7200
batch_size: int = 25
# --- Search ---
methods: str = "agentic"
top_k: int = 10
eval_owner: Literal["speaker_a", "speaker_b"] = "speaker_a"
# --- Answer LLM ---
answer_model: str = "gpt-4.1-mini"
answer_temperature: float = 0.0
answer_max_tokens: int = 32768
answer_timeout: float = 300.0
answer_max_retries: int = 5
# --- Judge LLM ---
judge_model: str = "gpt-4o-mini"
judge_temperature: float = 0.0
judge_timeout: float = 300.0
judge_max_retries: int = 5
judge_runs: int = 3
# --- Concurrency ---
conversations_concurrency: int = 10
eval_concurrency: int = 20
search_concurrency: int = 5
@property
def parsed_methods(self) -> list[str]:
"""Split comma-separated methods string into a list."""
return [m.strip() for m in self.methods.split(",") if m.strip()]
@classmethod
def from_toml(
cls, name: str = "config", *, config_dir: Path | None = None
) -> BenchmarkConfig:
"""Load config from a TOML file.
Args:
name: Config name without .toml extension.
config_dir: Directory containing config files.
Falls back to ``benchmarks/`` relative to the repo root.
Raises:
FileNotFoundError: When the TOML file does not exist.
"""
if config_dir is None:
config_dir = Path(__file__).parent
path = config_dir / f"{name}.toml"
if not path.exists():
raise FileNotFoundError(f"Config file not found: {path}")
with open(path, "rb") as f:
overrides = tomllib.load(f)
return cls(**overrides)
class SearchResult(BaseModel):
"""One QA pair's search stage output."""
model_config = ConfigDict(frozen=True)
index: int
question: str
golden_answer: str
category: int | None
evidence: list[str]
episodes: list[dict]
profiles: list[dict]
search_time_s: float
method: str
class AnswerResult(BaseModel):
"""One QA pair's answer stage output."""
model_config = ConfigDict(frozen=True)
index: int
question: str
golden_answer: str
category: int | None
generated_answer: str
answer_time_s: float
answer_attempts: int
answer_tokens: int = 0
class JudgeResult(BaseModel):
"""One QA pair's judge stage output."""
model_config = ConfigDict(frozen=True)
index: int
question: str
golden_answer: str
generated_answer: str
category: int | None
is_correct: bool
judgments: list[bool]
judge_tokens: int = 0
class RunSpec(BaseModel):
"""Reproducibility snapshot serialized at run start."""
model_config = ConfigDict(frozen=True)
run_name: str
config: dict
conversations: list[int]
stages: list[str]
git_hash: str
python_version: str
everos_version: str
started_at: str
benchmarks/config.toml
+33
View File
@@ -0,0 +1,33 @@
# Benchmark configuration — single source of truth.
#
# All parameters aligned with the upstream evaluation reference so that
# benchmark numbers are directly comparable to published LoCoMo results.
# BenchmarkConfig reads this file on every run.
# --- EverOS server ---
cascade_timeout = 7200 # max seconds to wait for cascade + OME to complete
batch_size = 25 # messages per /add request (everos caps at 500)
# --- Search ---
methods = "agentic" # comma-separated: keyword, vector, hybrid, agentic
top_k = 10 # episodes to retrieve per question
eval_owner = "speaker_a" # which speaker's memory partition to query
# --- Answer LLM ---
answer_model = "gpt-4.1-mini" # LLM for generating answers from retrieved context
answer_temperature = 0.0 # deterministic answers
answer_max_tokens = 32768 # max output tokens per answer call
answer_timeout = 300.0 # per-request socket timeout (seconds)
answer_max_retries = 5 # retry budget (HTTP + empty-response retries)
# --- Judge LLM ---
judge_model = "gpt-4o-mini" # LLM judge for correctness evaluation
judge_temperature = 0.0 # deterministic judging
judge_timeout = 300.0 # per-request socket timeout (seconds)
judge_max_retries = 5 # retry budget
judge_runs = 3 # majority vote across N independent judge runs
# --- Concurrency ---
conversations_concurrency = 10 # how many conversations run at the same time
search_concurrency = 5 # concurrent /search requests per conversation
eval_concurrency = 20 # concurrent answer/judge LLM calls per conversation
benchmarks/run.py
+1840
View File
File diff suppressed because it is too large Load Diff
docs/api.md
+1 -1
View File
@@ -688,7 +688,7 @@ ranked; `score` is `null`. Ignored when `agent_id` is set.
**`enable_llm_rerank`** — Opt-in LLM rerank pass for
`method: "hybrid"`. Applies to `agent_case` and `agent_skill` fusion
only; the episode hierarchy path has built-in fact eviction and
only; the episode hybrid path has built-in fact eviction and
ignores this flag. Adds one LLM call per request. Ignored by
`keyword` / `vector` (no fusion to rerank) and `agentic` (uses its
own cross-encoder loop).
docs/architecture.md
+1 -1
View File
@@ -282,5 +282,5 @@ prevent external types from leaking into upper layers:
## Further reading
- [docs/overview.md](overview.md) — vision and scope
- [docs/engineering.md](engineering.md) — contributor engineering reference: build, test, CI, conventions
- [docs/engineering.md](engineering.md) — engineering tooling and CI / CD
- [.claude/rules/architecture.md](../.claude/rules/architecture.md) — short-form rules for Claude Code
docs/configuration.md
+3 -13
View File
@@ -105,7 +105,7 @@ everos init --root /data/everos
|---|---|---|---|---|
| `model` | string | `"gpt-4.1-mini"` | No | LLM model identifier. |
| `api_key` | string | — | **Yes** | API key for the LLM provider. |
| `base_url` | string | — | **Yes** | Endpoint URL (OpenAI-compatible). |
| `base_url` | string | — | No | Custom endpoint URL (OpenAI-compatible). |
### `[multimodal]`
@@ -113,7 +113,7 @@ everos init --root /data/everos
|---|---|---|---|---|
| `model` | string | `"google/gemini-3-flash-preview"` | No | Multimodal parsing model. |
| `api_key` | string | — | **Yes** | API key. |
| `base_url` | string | — | **Yes** | Endpoint URL. |
| `base_url` | string | — | No | Custom endpoint URL. |
| `max_concurrency` | int | `4` | No | Max parallel parsing requests. |
| `file_uri_allow_dirs` | list[string] | `[]` | No | Allowlisted base dirs for `file://` URIs. Empty = allow any readable file. |
| `file_uri_max_bytes` | int | `52428800` | No | Max size (bytes) of a `file://` asset; larger files are rejected. |
@@ -134,7 +134,7 @@ everos init --root /data/everos
| Field | Type | Default | Required | Description |
|---|---|---|---|---|
| `provider` | string | inferred | No | Rerank provider: `deepinfra`, `vllm`, or `dashscope`. |
| `provider` | string | `"deepinfra"` | No | Rerank provider: `deepinfra` or `vllm`. |
| `model` | string | — | **Yes** | Reranker model identifier. |
| `api_key` | string | — | **Yes** | API key. |
| `base_url` | string | — | **Yes** | Rerank endpoint URL. |
@@ -164,12 +164,6 @@ everos init --root /data/everos
| `threshold` | float | `0.65` | Cosine similarity threshold for clustering (0–1). |
| `time_window_days` | float | `7.0` | Max age gap between cluster members. |
### `[search]`
| Field | Type | Default | Description |
|---|---|---|---|
| `vector_strategy` | string | `"maxsim_atomic"` | Vector retrieval path: `maxsim_atomic` (finer-grained) or `episode` (legacy). |
### `[knowledge]`
| Field | Type | Default | Description |
@@ -210,13 +204,9 @@ variables directly:
ENV EVEROS_ROOT=/data/everos
ENV EVEROS_LLM__API_KEY=sk-...
ENV EVEROS_LLM__MODEL=gpt-4o
ENV EVEROS_LLM__BASE_URL=https://api.openai.com/v1
ENV EVEROS_EMBEDDING__MODEL=text-embedding-3-large
ENV EVEROS_EMBEDDING__API_KEY=sk-...
ENV EVEROS_EMBEDDING__BASE_URL=https://api.openai.com/v1
ENV EVEROS_RERANK__MODEL=Qwen/Qwen3-Reranker-4B
ENV EVEROS_RERANK__API_KEY=...
ENV EVEROS_RERANK__BASE_URL=https://api.deepinfra.com/v1/inference
ENV EVEROS_API__HOST=0.0.0.0
ENV EVEROS_API__PORT=8000
```
docs/github-sync.md
+132 -53
View File
@@ -1,77 +1,156 @@
# GitHub Sync Guard
This page records the GitHub-only files that must be preserved when refreshing
this public repository from the internal GitLab source/export.
This page records the rules for refreshing the public GitHub repository
(EverMind-AI/EverOS) from the internal GitLab source.
## Rule
## Core Principle
When a GitLab export, archive, rsync, or scripted copy is used to refresh the
GitHub repository, do **not** blindly overwrite GitHub-only contributor,
automation, and public-facing workflow files.
GitLab `dev` is the source of truth. GitHub `main` is the public release
mirror. The two repos have **no shared commit history** — GitHub receives
packaged exports, not git pushes.
These files describe the public GitHub workflow: protected `main`, scoped
branches, GitHub pull requests, GitHub Actions, and community contribution
rules. Internal workflow text can mislead contributors and AI coding tools if
it is copied into GitHub.
---
## Preserve These GitHub Files
## 1. Must Delete on GitHub
Keep the GitHub versions of these files unless a maintainer deliberately updates
them for the public GitHub workflow:
These files exist on GitHub but are obsolete — delete them during sync:
- `CLAUDE.md`
- `CONTRIBUTING.md`
- `.github/PULL_REQUEST_TEMPLATE.md`
- `.github/BRANCH_PROTECTION.md`
- `.github/workflows/ci.yml`
- `.github/workflows/docs.yml`
- `.github/workflows/commits.yml`
- `.github/ISSUE_TEMPLATE/**`
- `.claude/skills/commit/SKILL.md`
- `.claude/skills/new-branch/SKILL.md`
- `.claude/skills/pr/SKILL.md`
| File | Reason |
|---|---|
| `docs/locomo_benchmark.md` | Replaced by `benchmarks/README.md` |
| `tests/run_locomo_10x3.sh` | Replaced by `benchmarks/run.py` |
| `tests/run_locomo_batch.sh` | Replaced by `benchmarks/run.py` |
| `tests/run_locomo_full.sh` | Replaced by `benchmarks/run.py` |
| `tests/test_locomo.py` | Replaced by `benchmarks/run.py` |
| `src/everos/memory/strategies/_partition_locks.py` | Wrong path; correct location is `src/everos/memory/_partition_locks.py` |
## 2. Must Preserve on GitHub (Do NOT Overwrite)
These files have GitHub-specific content (branch model, PR workflow,
community-facing text). Keep the GitHub versions:
### Workflow & CI
- `CLAUDE.md` — says `main` branch, not `dev`/`master`
- `CONTRIBUTING.md` — says "curated PR contributions", not "no external PRs"
- `.github/PULL_REQUEST_TEMPLATE.md` — GitHub PR template
- `.github/BRANCH_PROTECTION.md` — GitHub branch protection rules
- `.github/workflows/ci.yml` — targets `main`, uses `actions/checkout@v6`
- `.github/workflows/docs.yml` — GitHub-only doc checks
- `.github/workflows/commits.yml` — GitHub-only commit lint
- `.github/ISSUE_TEMPLATE/**` — `.yml` format (GitLab uses `.md`)
- `.github/dependabot.yml` — GitHub-only dependency scanning
### Claude Code rules & skills
- `.claude/rules/*.md` — GitHub versions are simplified for contributors
- `.claude/skills/commit/SKILL.md` — Conventional Commits (vs Gitmoji)
- `.claude/skills/new-branch/SKILL.md` — branches from `main` (vs `dev`)
- `.claude/skills/pr/SKILL.md` — GitHub PR (vs GitLab MR)
- `.claude/settings.json` — GitHub permission set (includes `gh pr`)
### GitHub-only scripts & tests
- `scripts/check_commit_messages.py`
- `scripts/check_deprecated_names.py`
- `scripts/check_docs.py`
- `scripts/check_github_contributor_docs.py`
- `tests/unit/test_scripts/test_check_github_contributor_docs.py`
- `scripts/check_pr_title.py`
- `scripts/check_repo_assets.py`
- `tests/unit/test_scripts/test_check_*.py`
### Root docs (GitHub version has different content)
- `README.md` — GitHub version has banner, badges, use-case gallery
- `Makefile` — GitHub version has extra targets (docs-check, package, etc.)
## 3. Overwrite from GitLab
Everything not in sections 1 or 2 should be overwritten from GitLab,
including but not limited to:
- All `src/everos/**` source code
- All `tests/unit/**` and `tests/integration/**` and `tests/e2e/**`
- `pyproject.toml`, `uv.lock`
- `QUICKSTART.md`, `SECURITY.md`
- `CHANGELOG.md`, `CITATION.md`, `ACKNOWLEDGMENTS.md`
- `docs/*.md` (except those listed in Must Delete)
- `docs/openapi.json`
- `benchmarks/**`
- `.gitignore`
- `.gitlab-ci.yml` (harmless on GitHub, good to keep in sync)
## 4. Do NOT Sync to GitHub
These files are GitLab-internal and should not appear on GitHub:
| File | Reason |
|---|---|
| `.gitlab/merge_request_templates/` | GitLab-only |
| `.vscode/` | IDE preference |
| `.claude/skills/release/SKILL.md` | Internal release workflow |
| `local/` | Temporary design docs |
| `evaluation/` | Empty module |
## 5. Do NOT Sync to GitLab
These files are GitHub-only and should stay only on GitHub:
| Category | Files |
|---|---|
| use-cases/ | 90 files (claude-code-plugin, game-of-throne-demo, openher) |
| GitHub CI scripts | `scripts/check_*.py` + tests |
| GitHub CI workflows | `commits.yml`, `docs.yml` |
| Config | `.gitlint`, `.env.example` (root level) |
| Docs | `docs/use-cases.md` |
## Safe Sync Pattern
Prefer an explicit exclude list in the sync job rather than manual cleanup after
the overwrite. Example shape:
```bash
# 1. Clone both repos
git clone <gitlab> gitlab-export
git clone <github> github-checkout
# 2. Sync with excludes
rsync -a --delete \
--exclude '.git' \
--exclude 'CLAUDE.md' \
--exclude 'CONTRIBUTING.md' \
--exclude '.github/PULL_REQUEST_TEMPLATE.md' \
--exclude '.github/BRANCH_PROTECTION.md' \
--exclude '.github/workflows/ci.yml' \
--exclude '.github/workflows/docs.yml' \
--exclude '.github/workflows/commits.yml' \
--exclude '.github/ISSUE_TEMPLATE/' \
--exclude '.claude/skills/commit/SKILL.md' \
--exclude '.claude/skills/new-branch/SKILL.md' \
--exclude '.claude/skills/pr/SKILL.md' \
--exclude 'scripts/check_github_contributor_docs.py' \
--exclude 'tests/unit/test_scripts/test_check_github_contributor_docs.py' \
<gitlab-export>/ <github-checkout>/
```
--exclude 'README.md' \
--exclude 'Makefile' \
--exclude '.github/' \
--exclude '.claude/rules/' \
--exclude '.claude/skills/' \
--exclude '.claude/settings.json' \
--exclude 'scripts/check_*.py' \
--exclude 'tests/unit/test_scripts/' \
--exclude 'use-cases/' \
--exclude 'local/' \
--exclude 'evaluation/' \
--exclude '.vscode/' \
--exclude '.gitlab/' \
--exclude '.gitlab-ci.yml' \
gitlab-export/ github-checkout/
After any sync, run:
# 3. Delete obsolete files
cd github-checkout
rm -f docs/locomo_benchmark.md
rm -f tests/run_locomo_10x3.sh tests/run_locomo_batch.sh tests/run_locomo_full.sh
rm -f tests/test_locomo.py
rm -f src/everos/memory/strategies/_partition_locks.py
```bash
make docs-check
# 4. Verify
make lint
make test
```
The `check_github_contributor_docs.py` gate catches the known failure mode:
public contributor docs drifting back to an internal branch/review model.
## Review Checklist
Before opening a sync PR:
Before opening the sync PR on GitHub:
- Confirm `CLAUDE.md` says branches are created from `main`.
- Confirm `.claude/skills/pr/SKILL.md` creates PRs with `--base main`.
- Confirm `CONTRIBUTING.md` says GitHub pull request, not internal review
terminology.
- Confirm `make docs-check` and `make lint` both pass.
- [ ] `CLAUDE.md` says branches are created from `main`
- [ ] `.claude/skills/pr/SKILL.md` creates PRs with `--base main`
- [ ] `CONTRIBUTING.md` says "GitHub pull request", not internal terminology
- [ ] Obsolete files from section 1 are deleted
- [ ] `make lint` and `make test` pass
- [ ] No `.gitlab-ci.yml`, `.vscode/`, `local/`, `evaluation/` leaked in
docs/index.md
+5 -6
View File
@@ -12,7 +12,6 @@ before wiring it into a real workflow.
| Doc | Purpose |
|---|---|
| [everos-demo.md](everos-demo.md) | `everos demo` — local educational TUI to feel the memory lifecycle before configuring keys |
| [use-cases.md](use-cases.md) | Worked examples and integrations showing what persistent memory enables, to study and adapt |
## Reference
@@ -51,16 +50,16 @@ specific thing (drain a queue, recover from a stuck row, etc.).
|---|---|
| [cascade_runbook.md](cascade_runbook.md) | Cascade subsystem ops — drain queue, recover stuck rows |
| [github-sync.md](github-sync.md) | Guardrails for refreshing GitHub from internal exports without overwriting GitHub-only workflow files |
| [locomo_benchmark.md](locomo_benchmark.md) | LoCoMo benchmark — run and evaluate |
| [benchmarks/README.md](../benchmarks/README.md) | LoCoMo benchmark — run and evaluate |
## Engineering
## Engineering / Internal
For contributors working on the framework itself — how to build, test,
and ship a change.
For maintainers and contributors working on the framework itself,
not for using it.
| Doc | Purpose |
|---|---|
| [engineering.md](engineering.md) | Contributor engineering reference — build, test, CI gates, branch & commit conventions |
| [engineering.md](engineering.md) | Engineering & dev-efficiency infrastructure (CI / tooling / Claude Code) |
## See also
docs/locomo_benchmark.md
-137
View File
@@ -1,137 +0,0 @@
# Running the LoCoMo Benchmark
This guide walks through reproducing EverOS's LoCoMo retrieval scores
locally using the `hybrid` and `agentic` search methods.
## Contents
- [Prerequisites](#prerequisites)
- [1. Prepare the dataset](#1-prepare-the-dataset)
- [2. Start the server](#2-start-the-server)
- [3. Run `hybrid`](#3-run-hybrid)
- [4. Run `agentic`](#4-run-agentic)
- [5. Where the results land](#5-where-the-results-land)
- [Notes](#notes)
---
## Prerequisites
- Python **3.12**, [uv](https://docs.astral.sh/uv/)
- LLM / embedding credentials configured via `everos.toml` in the memory
root (generated by `everos init`) or `EVEROS_*` env vars:
- `EVEROS_LLM__MODEL`, `EVEROS_LLM__API_KEY`, `EVEROS_LLM__BASE_URL`
- `EVEROS_EMBEDDING__*`
- `EVEROS_RERANK__*`
- The benchmark driver also reads `LLM_API_KEY` / `ANSWER_MODEL` /
`JUDGE_MODEL` for the answer + judge passes.
Install the project:
```bash
uv sync
```
## 1. Prepare the dataset
Place the LoCoMo file at `data/locomo10.json` (the dataset is
distributed by the LoCoMo authors, not this repo). Override the path
later with `--data-path` if you keep it elsewhere.
## 2. Start the server
```bash
EVEROS_ROOT=~/.everos \
uv run everos server start --port 8000
```
`EVEROS_ROOT` isolates one benchmark's corpus from another —
change it (or `rm -rf` it) whenever you want a clean run.
Leave the server running in one terminal; run the benchmark from
another.
## 3. Run `hybrid`
Single conversation:
```bash
bash tests/run_locomo_batch.sh \
--conv-indices 0 \
--methods hybrid \
--base-url http://localhost:8000 \
--top-k 10
```
All 10 conversations, 2-way parallel:
```bash
bash tests/run_locomo_batch.sh \
--conv-indices 0-9 \
--methods hybrid \
--base-url http://localhost:8000 \
--top-k 10 \
--concurrency 2
```
The wrapper picks up `EVEROS_ROOT` from the environment so the
cascade poll path matches the server's data root. If you set them
differently, pass `--corpus-path` explicitly.
## 4. Run `agentic`
Same wrapper, swap `--methods`:
```bash
bash tests/run_locomo_batch.sh \
--conv-indices 0-9 \
--methods agentic \
--base-url http://localhost:8000 \
--top-k 10 \
--concurrency 2
```
You can also benchmark multiple methods in one go — they share the
same ingested corpus:
```bash
bash tests/run_locomo_batch.sh \
--conv-indices 0-9 \
--methods hybrid,agentic \
--base-url http://localhost:8000 \
--top-k 10 \
--concurrency 2
```
## 5. Where the results land
Default output root is `benchmark_results/run_<timestamp>/`. Override
with `--output-root`:
```
<output_root>/
├── conv0.json … conv9.json # per-conv summary + per-question details
├── conv0.log … conv9.log # per-conv stdout (only in --concurrency >1 mode)
└── conv0_checkpoints/ … # incremental search/answer/eval JSON
```
An aggregate accuracy table prints at the end of the wrapper run.
## Notes
- **Smoke mode**: add `--smoke` for a quick sanity check — 2 sessions
× 5 QA with reduced wait times. Not a scored run, just pipeline
verification. Pass it to `test_locomo.py` directly or via the batch
wrapper's `--` passthrough:
```bash
uv run python tests/test_locomo.py --smoke --base-url http://localhost:8000
# or via the wrapper
bash tests/run_locomo_batch.sh --conv-indices 0 --methods hybrid -- --smoke
```
- **Re-running on the same corpus**: add `--skip-add` to skip ingest and
reuse what's already in `~/.everos`. Useful when comparing methods
side by side.
- **Judge variance**: `--judge-runs 3` runs the judge three times per
question and majority-votes; slower but reduces LLM-judge noise.
docs/migration-to-1.0.0.md
+2 -2
View File
@@ -59,8 +59,8 @@ For new integrations:
## Benchmark Guidance
The current LoCoMo reproduction path is documented in
[locomo_benchmark.md](locomo_benchmark.md). The benchmark driver uses
the 1.0.0 server API: add, flush, search, answer, and evaluate.
[benchmarks/README.md](../benchmarks/README.md). The benchmark driver uses
the server API: add, flush, search, answer, and evaluate.
Old HyperMem / pre-1.0.0 evaluation pipeline reports should not be used
as 1.0.0 bug reports unless they can be reproduced with the current
docs/multimodal.md
+20 -19
View File
@@ -91,19 +91,19 @@ audio / HTML / email parsing is unaffected.
### Configure the multimodal LLM
The parser uses its own LLM section, independent from `[llm]`. The model
must accept OpenAI `image_url` parts. Configure these in `everos.toml`
(under `[multimodal]`) or via env vars:
The parser uses its own LLM section (`[multimodal]` in `everos.toml`),
independent from `[llm]`. The model must accept OpenAI `image_url`
parts. Fill in three fields in `everos.toml`:
```bash
EVEROS_MULTIMODAL__MODEL=google/gemini-3-flash-preview
EVEROS_MULTIMODAL__API_KEY=<your key>
EVEROS_MULTIMODAL__BASE_URL=https://openrouter.ai/api/v1
```toml
[multimodal]
model = "google/gemini-3-flash-preview" # must support image_url parts
base_url = "https://openrouter.ai/api/v1"
api_key = "<your key>"
```
The default targets Gemini via OpenRouter so a single key covers both
chat extraction and multimodal parsing. See
[Configuration reference](#configuration-reference) for the full list.
See [Configuration reference](#configuration-reference) for the full
field list.
## Supported modalities
@@ -264,17 +264,18 @@ httpx.post(
## Configuration reference
All fields bind from the environment via the parent `Settings`
(`EVEROS_MULTIMODAL__<FIELD>`) or the `[multimodal]` TOML section.
All fields live under `[multimodal]` in `everos.toml`. Each can also
be overridden via `EVEROS_MULTIMODAL__<FIELD>` env vars (useful for
containers and CI).
| Env var | Default | Meaning |
| Field | Default | Meaning |
|---|---|---|
| `EVEROS_MULTIMODAL__MODEL` | `google/gemini-3-flash-preview` | Parsing model; must accept `image_url` parts |
| `EVEROS_MULTIMODAL__API_KEY` | — | API key for the multimodal endpoint |
| `EVEROS_MULTIMODAL__BASE_URL` | `None` | OpenAI-compatible base URL |
| `EVEROS_MULTIMODAL__MAX_CONCURRENCY` | `4` | Cap on parallel multimodal calls within one extraction |
| `EVEROS_MULTIMODAL__FILE_URI_MAX_BYTES` | `52428800` (50 MiB) | Max size of a `file://` asset |
| `EVEROS_MULTIMODAL__FILE_URI_ALLOW_DIRS` | `[]` (any) | JSON list of allowlisted base dirs for `file://` URIs |
| `model` | `google/gemini-3-flash-preview` | Parsing model; must accept `image_url` parts |
| `base_url` | `https://openrouter.ai/api/v1` | OpenAI-compatible base URL |
| `api_key` | — (required) | API key for the endpoint above |
| `max_concurrency` | `4` | Cap on parallel multimodal calls within one extraction |
| `file_uri_max_bytes` | `52428800` (50 MiB) | Max size of a `file://` asset |
| `file_uri_allow_dirs` | `[]` (any) | Allowlisted base dirs for `file://` URIs |
## Errors and limits
docs/openapi.json
+2 -2
View File
@@ -3,7 +3,7 @@
"info": {
"title": "everos",
"description": "md-first memory extraction framework",
"version": "1.1.0"
"version": "1.1.1"
},
"paths": {
"/health": {
@@ -2622,7 +2622,7 @@
"enable_llm_rerank": {
"type": "boolean",
"title": "Enable Llm Rerank",
"description": "Opt-in LLM rerank pass for HYBRID. Applies to agent_case and agent_skill fusion only; the episode hierarchy path has built-in fact eviction and ignores this flag. Ignored by keyword / vector / agentic.",
"description": "Opt-in LLM rerank pass for HYBRID. Applies to agent_case and agent_skill fusion only; the episode hybrid path has built-in fact eviction and ignores this flag. Ignored by keyword / vector / agentic.",
"default": false
},
"filters": {
pyproject.toml
+29 -8
View File
@@ -1,6 +1,6 @@
[project]
name = "everos"
version = "1.1.0"
version = "1.1.1"
description = "EverOS — local-first markdown memory framework for AI agents and user chats; lightweight, dev-friendly, small-team"
license = {text = "Apache-2.0"}
readme = "README.md"
@@ -15,13 +15,15 @@ classifiers = [
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"Typing :: Typed",
]
dependencies = [
# Core data validation & config
"pydantic>=2.7.1",
"pydantic-settings>=2.0.0",
"python-dotenv>=1.0.1",
# Storage stack (md-first three-piece set)
"lancedb>=0.13.0", # Vector + BM25 + scalar filter (Arrow-based)
@@ -46,12 +48,12 @@ dependencies = [
"structlog>=24.0.0",
"prometheus-client>=0.20.0",
# CLI
# CLI + TUI
"typer>=0.12.0",
"textual>=8.2.7",
# Tokenization (BM25 Chinese support)
"jieba==0.42.1",
"jieba>=0.42.1,<1.0",
# OME (Offline Memory Evolution) async scheduler & file I/O
"apscheduler>=3.10.4,<4.0", # async strategy scheduler chassis
@@ -69,6 +71,12 @@ dependencies = [
[project.optional-dependencies]
multimodal = ["everalgo-parser[svg]>=0.2.1"] # [svg] bundles cairosvg → SVG works by default
[project.urls]
Homepage = "https://evermind.ai"
Repository = "https://github.com/EverMind-AI/EverOS"
Changelog = "https://github.com/EverMind-AI/EverOS/blob/main/CHANGELOG.md"
"Bug Tracker" = "https://github.com/EverMind-AI/EverOS/issues"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
@@ -90,10 +98,8 @@ packages = ["src/everos"]
include = [
"/src",
"/tests",
"/data",
"/docs",
"/pyproject.toml",
"/uv.lock",
"/README.md",
"/QUICKSTART.md",
"/CHANGELOG.md",
@@ -121,14 +127,20 @@ target-version = "py312"
extend-exclude = ["src_old"]
[tool.ruff.lint]
select = ["E", "F", "I", "N", "UP", "B", "SIM", "ASYNC"]
select = ["E", "F", "I", "N", "UP", "B", "SIM", "ASYNC", "RUF"]
ignore = [
"RUF001", # ambiguous Unicode in strings (intentional × – symbols)
"RUF002", # ambiguous Unicode in docstrings
"RUF003", # ambiguous Unicode in comments
"RUF012", # mutable class attribute default (SQLModel requires this)
]
[tool.ruff.lint.per-file-ignores]
# LoCoMo benchmark embeds LLM prompts (ANSWER_PROMPT / JUDGE_*_PROMPT) and
# verbose argparse help strings as multi-line string literals. Line breaks
# would change what the LLM actually sees or noise the --help output, so
# E501 is silenced for this file only.
"tests/test_locomo.py" = ["E501"]
"benchmarks/run.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["tests"]
@@ -161,6 +173,15 @@ source = ["src/everos"]
branch = true
omit = ["**/__init__.py"]
[tool.coverage.report]
fail_under = 80
show_missing = true
exclude_lines = [
"if TYPE_CHECKING:",
"@abstractmethod",
"pragma: no cover",
]
[tool.importlinter]
root_packages = ["everos"]
src/everos/component/embedding/factory.py
+2 -3
View File
@@ -36,8 +36,7 @@ def build_embedding_provider(
"Embedding model is not configured "
"(set EVEROS_EMBEDDING__MODEL or [embedding] model in user toml)"
)
api_key = settings.api_key.get_secret_value() if settings.api_key else ""
if not api_key:
if not settings.api_key or not settings.api_key.get_secret_value():
raise ValueError(
"Embedding api_key is not configured (set EVEROS_EMBEDDING__API_KEY)"
)
@@ -47,7 +46,7 @@ def build_embedding_provider(
)
return OpenAIEmbeddingProvider(
model=settings.model,
api_key=api_key,
api_key=settings.api_key.get_secret_value(),
base_url=settings.base_url,
dim=dim,
timeout=settings.timeout_seconds,
src/everos/component/llm/factory.py
+2 -3
View File
@@ -28,8 +28,7 @@ def build_llm_provider(settings: LLMSettings) -> LLMClient:
Raises:
ValueError: If ``api_key`` or ``base_url`` is unset.
"""
api_key = settings.api_key.get_secret_value() if settings.api_key else ""
if not api_key:
if not settings.api_key or not settings.api_key.get_secret_value():
raise ValueError(
"LLM api_key is not configured "
"(set EVEROS_LLM__API_KEY or [llm] api_key in user toml)"
@@ -41,6 +40,6 @@ def build_llm_provider(settings: LLMSettings) -> LLMClient:
)
return OpenAIProvider(
model=settings.model,
api_key=api_key,
api_key=settings.api_key.get_secret_value(),
base_url=settings.base_url,
)
src/everos/component/rerank/__init__.py
+2 -1
View File
@@ -6,9 +6,10 @@ Public surface:
- :class:`RerankResult` / :class:`RerankServiceError` — value type + error.
- :class:`RerankError` — backward-compat alias for :class:`RerankServiceError`.
- :class:`DeepInfraRerankProvider` — DeepInfra inference-API rerank.
- :class:`DashScopeRerankProvider` — Aliyun Bailian / DashScope rerank.
- :class:`VllmRerankProvider` — OpenAI-compat ``/v1/rerank`` (vLLM,
self-hosted, other compatible servers).
- :class:`DashScopeRerankProvider` — DashScope (Aliyun Bailian)
``gte-rerank-v2`` native text-rerank endpoint.
- :func:`build_rerank_provider` — settings-driven factory that picks
the concrete provider via ``settings.rerank.provider``.
src/everos/component/rerank/factory.py
+9 -56
View File
@@ -3,10 +3,9 @@
The ``provider`` field on :class:`RerankSettings` selects which concrete
implementation to build:
- ``"deepinfra"`` → :class:`DeepInfraRerankProvider`
- ``"vllm"`` → :class:`VllmRerankProvider`
- ``"dashscope"`` → :class:`DashScopeRerankProvider`
(Aliyun Bailian ``gte-rerank-v2``)
- ``"deepinfra"`` → :class:`DeepInfraRerankProvider`
- ``"vllm"`` → :class:`VllmRerankProvider`
- ``"dashscope"`` → :class:`DashScopeRerankProvider`
Adding a new provider = one match arm here + one new file under
:mod:`everos.component.rerank`.
@@ -14,42 +13,13 @@ Adding a new provider = one match arm here + one new file under
from __future__ import annotations
from urllib.parse import urlparse
from everos.config import RerankSettings
from everos.core.observability.logging import get_logger
from .dashscope_provider import DashScopeRerankProvider
from .deepinfra_provider import DeepInfraRerankProvider
from .protocol import RerankProvider
from .vllm_provider import VllmRerankProvider
logger = get_logger(__name__)
# host substring -> provider. Ordered most-specific first; matched against
# the ``base_url`` host so a Bailian / DeepInfra URL routes to the right
# request-shape without the operator also having to set ``provider``.
_PROVIDER_HOST_HINTS: tuple[tuple[str, str], ...] = (
("dashscope.aliyuncs.com", "dashscope"),
("deepinfra.com", "deepinfra"),
)
# Fallback when the host matches no hint and ``provider`` is unset. Keeps
# the historical default so existing configs without ``provider`` are
# unaffected.
_DEFAULT_PROVIDER = "deepinfra"
def _infer_provider(base_url: str) -> str | None:
"""Infer the rerank provider from the ``base_url`` host, or ``None``."""
host = (urlparse(base_url).hostname or "").lower()
if not host:
return None
for needle, provider in _PROVIDER_HOST_HINTS:
if host == needle or host.endswith(f".{needle}"):
return provider
return None
def build_rerank_provider(settings: RerankSettings) -> RerankProvider:
"""Build a rerank provider from settings.
@@ -64,14 +34,8 @@ def build_rerank_provider(settings: RerankSettings) -> RerankProvider:
Raises:
ValueError: If ``model`` or ``base_url`` is unset, or if
``provider`` does not match a known implementation.
``api_key`` is required for ``deepinfra`` and ``dashscope``;
optional (empty string) for ``vllm`` self-hosted endpoints.
``dashscope`` currently supports ``gte-rerank-v2`` only.
Notes:
When ``settings.provider`` is ``None`` the provider is inferred
from the ``base_url`` host (see :data:`_PROVIDER_HOST_HINTS`),
falling back to ``"deepinfra"`` for unrecognized hosts.
``api_key`` is required for ``deepinfra``; optional (empty
string) for ``vllm`` self-hosted endpoints.
"""
if not settings.model:
raise ValueError(
@@ -84,18 +48,7 @@ def build_rerank_provider(settings: RerankSettings) -> RerankProvider:
)
api_key = settings.api_key.get_secret_value() if settings.api_key else ""
provider = settings.provider
if provider is None:
inferred = _infer_provider(settings.base_url)
provider = inferred or _DEFAULT_PROVIDER
logger.info(
"rerank_provider_inferred",
provider=provider,
inferred=inferred is not None,
base_url=settings.base_url,
)
if provider == "deepinfra":
if settings.provider == "deepinfra":
if not api_key:
raise ValueError(
"DeepInfra rerank api_key is not configured "
@@ -110,7 +63,7 @@ def build_rerank_provider(settings: RerankSettings) -> RerankProvider:
batch_size=settings.batch_size,
max_concurrent=settings.max_concurrent,
)
if provider == "vllm":
if settings.provider == "vllm":
return VllmRerankProvider(
model=settings.model,
api_key=api_key,
@@ -120,7 +73,7 @@ def build_rerank_provider(settings: RerankSettings) -> RerankProvider:
batch_size=settings.batch_size,
max_concurrent=settings.max_concurrent,
)
if provider == "dashscope":
if settings.provider == "dashscope":
if not api_key:
raise ValueError(
"DashScope rerank api_key is not configured "
@@ -135,4 +88,4 @@ def build_rerank_provider(settings: RerankSettings) -> RerankProvider:
batch_size=settings.batch_size,
max_concurrent=settings.max_concurrent,
)
raise ValueError(f"unknown rerank provider: {provider!r}")
raise ValueError(f"unknown rerank provider: {settings.provider!r}")
src/everos/config/default.toml
+22 -39
View File
@@ -54,18 +54,18 @@ cache_size_kb = 2048
# Provider-agnostic OpenAI-protocol client config. Override via env:
# EVEROS_LLM__MODEL, EVEROS_LLM__API_KEY, EVEROS_LLM__BASE_URL
# Or set the field directly in this file (<root>/everos.toml).
model = "gpt-4.1-mini"
# api_key = ""
# base_url = ""
model = "openai/gpt-4.1-mini"
api_key = ""
base_url = "https://openrouter.ai/api/v1"
[multimodal]
# Independent LLM for multimodal parsing (everalgo-parser); must accept
# image / pdf / audio image_url parts. Override via env:
# EVEROS_MULTIMODAL__MODEL, EVEROS_MULTIMODAL__API_KEY, EVEROS_MULTIMODAL__BASE_URL
model = "google/gemini-3-flash-preview"
api_key = ""
base_url = "https://openrouter.ai/api/v1"
max_concurrency = 4
# api_key = ""
# base_url = ""
# file:// content-item support (read locally by EverOS, not everalgo).
# file_uri_allow_dirs: empty = allow any readable file (local-first default);
# list base dirs to confine reads when the API is exposed.
@@ -75,11 +75,10 @@ max_concurrency = 4
[embedding]
# OpenAI-compatible embedding endpoint. Override via env:
# EVEROS_EMBEDDING__MODEL, EVEROS_EMBEDDING__API_KEY, EVEROS_EMBEDDING__BASE_URL
# model / api_key / base_url have no shipped defaults — must be set
# (env or user toml) before the embedding capability is used.
# model = "Qwen/Qwen3-Embedding-4B"
# api_key = ""
# base_url = "https://api.example.com/v1"
# model and base_url ship with recommended defaults; api_key must be set.
model = "Qwen/Qwen3-Embedding-4B"
api_key = ""
base_url = "https://api.deepinfra.com/v1/openai"
timeout_seconds = 30.0
max_retries = 3
batch_size = 10
@@ -92,17 +91,10 @@ max_concurrent = 5
# `provider` picks the request-shape:
# - "deepinfra" -> POST {base_url}/{model} (DeepInfra inference API)
# - "vllm" -> POST {base_url}/rerank (OpenAI-compat rerank endpoint)
# - "dashscope" -> POST {base_url}/api/v1/services/rerank/text-rerank/text-rerank
# (Aliyun Bailian native text-rerank; currently
# "gte-rerank-v2", base_url "https://dashscope.aliyuncs.com")
# Left unset, the factory infers the provider from the base_url host
# (dashscope.aliyuncs.com -> dashscope, *.deepinfra.com -> deepinfra) and
# falls back to "deepinfra" for unknown hosts. Set it explicitly to
# override — required for self-hosted "vllm" on an arbitrary host.
# provider = "deepinfra"
# model = "Qwen/Qwen3-Reranker-4B"
# api_key = ""
# base_url = "https://api.deepinfra.com/v1/inference"
provider = "deepinfra"
model = "Qwen/Qwen3-Reranker-4B"
api_key = ""
base_url = "https://api.deepinfra.com/v1/inference"
timeout_seconds = 30.0
max_retries = 3
batch_size = 10
@@ -113,16 +105,6 @@ max_concurrent = 5
hard_token_limit = 65536
hard_msg_limit = 500
[search]
# Vector retrieval strategy when SearchMethod.VECTOR is selected.
# "maxsim_atomic" (default): ANN over atomic_fact.vector (pool=top_k*20),
# max-pool the per-fact cosine by parent memcell, then reverse-resolve
# to episode rows. MaxSim over atomic facts; +0.6pp over the legacy
# episode-vector path on LoCoMo, at the cost of one extra LanceDB scan.
# "episode": single-vector ANN over episode.vector (legacy path).
# Override via EVEROS_SEARCH__VECTOR_STRATEGY.
vector_strategy = "maxsim_atomic"
[memorize]
# Conversation mode. Selects the boundary detector and which pipelines run:
# "chat" -> BoundaryDetector + user_memory only
@@ -131,14 +113,6 @@ vector_strategy = "maxsim_atomic"
# requires a restart. Override via EVEROS_MEMORIZE__MODE.
mode = "agent"
# Maximum wall-clock for one memorize() invocation while holding the
# per-session lock. On timeout the outer asyncio.timeout cancels the call
# and the lock auto-releases so subsequent concurrent /add on the same
# session aren't deadlocked. Covers boundary LLM + memcell writes +
# synchronous portion of pipeline dispatch.
# Override via EVEROS_MEMORIZE__SESSION_LOCK_TIMEOUT_SECONDS.
session_lock_timeout_seconds = 360.0
[knowledge]
# Max bytes for an uploaded knowledge document (default 50 MiB). Oversized
# uploads are rejected with HTTP 422 before parsing/extraction. Note: the
@@ -155,6 +129,14 @@ lambda = 0.1
mass_top_m = 50
top_k_cap = 100
# Maximum wall-clock for one memorize() invocation while holding the
# per-session lock. On timeout the outer asyncio.timeout cancels the call
# and the lock auto-releases so subsequent concurrent /add on the same
# session aren't deadlocked. Covers boundary LLM + memcell writes +
# synchronous portion of pipeline dispatch.
# Override via EVEROS_MEMORIZE__SESSION_LOCK_TIMEOUT_SECONDS.
session_lock_timeout_seconds = 360.0
[clustering]
# Geometry-clustering: cosine similarity threshold and time window.
# Episodes older than ``time_window_days`` from the newest cluster
@@ -162,3 +144,4 @@ top_k_cap = 100
# Override via EVEROS_CLUSTERING__THRESHOLD, EVEROS_CLUSTERING__TIME_WINDOW_DAYS.
threshold = 0.65
time_window_days = 7.0
src/everos/config/default_ome.toml
+15 -24
View File
@@ -26,36 +26,30 @@
# ── User-memory pipeline ────────────────────────────────────────────────
# Atomic fact extraction (runs per memcell). Disable to skip fact mining.
# [strategies.extract_atomic_facts]
# Atomic fact extraction runs per memcell. Always enabled — search
# depends on atomic facts for MaxSim retrieval.
# Foresight extraction (runs per memcell). Heavy LLM call — set
# enabled = false to skip in evaluation / benchmark runs.
# [strategies.extract_foresight]
# enabled = true
# Foresight extraction (runs per memcell). Heavy LLM call — common to
# disable in evaluation / benchmark runs.
# [strategies.extract_foresight]
# enabled = false
# Profile clustering trigger (counter-gated; fires once N user memcells
# accumulate). Lower the threshold to cluster more aggressively.
# Profile clustering trigger (fires on each EpisodeExtracted event).
# [strategies.trigger_profile_clustering]
# enabled = true
# [strategies.trigger_profile_clustering.gate]
# threshold = 5
# User-profile extraction (runs after clustering trigger fires). Common
# to disable in evaluation runs where ground-truth profiles aren't measured.
# User-profile extraction (runs after clustering trigger fires). Set
# enabled = false to skip in evaluation runs.
# [strategies.extract_user_profile]
# enabled = false
# enabled = true
# ── Reflection ──────────────────────────────────────────────────────────
# Offline memory consolidation. Disabled by default — uncomment to enable.
# Once enabled, runs on the default schedule: weekly, Monday 02:00.
# Run Reflection sparingly — at most once a week. Triggering it too
# often degrades quality, since each run is a lossy LLM re-merge.
# The cron expression can be overridden without restarting the server.
# Offline memory consolidation. Disabled by default. To enable,
# uncomment and set enabled = true. Runs weekly Monday 02:00.
# Run sparingly — at most once a week. Each run is a lossy LLM re-merge.
# [strategies.reflect_episodes]
# enabled = true
# enabled = false
# cron = "0 2 * * 1"
# ── Agent-memory pipeline ───────────────────────────────────────────────
@@ -64,12 +58,9 @@
# [strategies.extract_agent_case]
# enabled = true
# Skill clustering trigger (counter-gated; fires once N agent cases
# accumulate per agent).
# Skill clustering trigger (fires on each AgentCaseExtracted event).
# [strategies.trigger_skill_clustering]
# enabled = true
# [strategies.trigger_skill_clustering.gate]
# threshold = 5
# Agent skill extraction (runs after skill clustering trigger fires).
# [strategies.extract_agent_skill]
src/everos/config/settings.py
+5 -36
View File
@@ -188,7 +188,7 @@ class EmbeddingSettings(BaseModel):
timeout_seconds: float = Field(default=30.0, gt=0)
max_retries: int = Field(default=3, ge=0)
batch_size: int = Field(default=10, ge=1)
max_concurrent: int = Field(default=5, ge=1)
max_concurrent: int = Field(default=50, ge=1)
class RerankSettings(BaseModel):
@@ -197,16 +197,8 @@ class RerankSettings(BaseModel):
Unlike LLM / embedding (single OpenAI-compatible shape), rerank API
schemas differ between providers — DeepInfra uses ``POST {base_url}/
{model}`` with a custom body, vLLM uses ``POST {base_url}/rerank``
with ``{model, query, documents}``, DashScope (Aliyun Bailian)
``gte-rerank-v2`` uses ``POST {base_url}/api/v1/services/rerank/
text-rerank/text-rerank`` with a nested ``{model, input, parameters}``
body. ``provider`` picks which client implementation the factory builds.
``provider`` defaults to ``None`` — the factory then infers it from
the ``base_url`` host (e.g. ``dashscope.aliyuncs.com`` → DashScope,
``*.deepinfra.com`` → DeepInfra), falling back to ``"deepinfra"`` when
the host is unrecognized. Set ``provider`` explicitly to override the
inference (required for self-hosted ``vllm`` on an arbitrary host).
with ``{model, query, documents}``. ``provider`` picks which client
implementation the factory builds.
Env binding:
EVEROS_RERANK__PROVIDER
@@ -219,14 +211,14 @@ class RerankSettings(BaseModel):
EVEROS_RERANK__MAX_CONCURRENT
"""
provider: Literal["deepinfra", "vllm", "dashscope"] | None = None
provider: Literal["deepinfra", "vllm", "dashscope"] = "deepinfra"
model: str | None = None
api_key: SecretStr | None = None
base_url: str | None = None
timeout_seconds: float = Field(default=30.0, gt=0)
max_retries: int = Field(default=3, ge=0)
batch_size: int = Field(default=10, ge=1)
max_concurrent: int = Field(default=5, ge=1)
max_concurrent: int = Field(default=50, ge=1)
class BoundaryDetectionSettings(BaseModel):
@@ -276,28 +268,6 @@ class ClusteringSettings(BaseModel):
time_window_days: float = Field(default=7.0, gt=0)
class SearchSettings(BaseModel):
"""Search-pipeline policy knobs.
``vector_strategy`` selects the read path taken by
``SearchMethod.VECTOR``:
- ``"maxsim_atomic"`` (default) — ANN over ``atomic_fact.vector``
(recall pool ``top_k * 20``, capped at 2000), max-pool the per-fact
cosine by parent memcell, then reverse-resolve the top memcells back
to episode rows. MaxSim over atomic facts; trades one extra LanceDB
scan for finer-grained semantic match on long episodes.
- ``"episode"`` — single-vector ANN over ``episode.vector`` (one vector
per episode = the embedded Content section). The legacy path; kept
so deployments can opt out via env.
Env binding:
EVEROS_SEARCH__VECTOR_STRATEGY={episode,maxsim_atomic}
"""
vector_strategy: Literal["episode", "maxsim_atomic"] = "maxsim_atomic"
class LanceDBSettings(BaseModel):
"""LanceDB tunables.
@@ -393,7 +363,6 @@ class Settings(BaseSettings):
boundary_detection: BoundaryDetectionSettings = BoundaryDetectionSettings()
memorize: MemorizeSettings = MemorizeSettings()
clustering: ClusteringSettings = ClusteringSettings()
search: SearchSettings = SearchSettings()
multimodal: MultimodalSettings = MultimodalSettings()
knowledge: KnowledgeSettings = KnowledgeSettings()
src/everos/core/middleware/profile.py
+7 -3
View File
@@ -62,8 +62,12 @@ class ProfileMiddleware(BaseHTTPMiddleware):
profiler.start()
logger.info("profile_started", method=request.method, path=request.url.path)
try:
await call_next(request)
response = await call_next(request)
except Exception:
logger.exception("profile_request_failed")
profiler.stop()
raise
profiler.stop()
return HTMLResponse(content=profiler.output_html(), status_code=200)
return HTMLResponse(
content=profiler.output_html(),
status_code=response.status_code,
)
src/everos/core/persistence/__init__.py
+3 -3
View File
@@ -82,9 +82,9 @@ __all__ = [
"ParsedMarkdown",
"Relationship",
"RepoBase",
"SQLModel",
"SkillPathMixin",
"StructuredEntry",
"SQLModel",
"UserScopedFrontmatter",
"Vector",
"app_dir_name",
@@ -94,11 +94,11 @@ __all__ = [
"dump_frontmatter",
"find_entry",
"memory_root_lock",
"project_dir_name",
"project_id_from_dir",
"open_lancedb_connection",
"parse_frontmatter",
"parse_structured_entry",
"project_dir_name",
"project_id_from_dir",
"render_structured_entry",
"session_scope",
"split_entries",
src/everos/core/persistence/lancedb/repository.py
+27 -13
View File
@@ -176,25 +176,39 @@ class LanceRepoBase[T: BaseLanceTable]:
async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None:
"""Compact fragments + merge new data into the FTS / vector indexes.
LanceDB's ``merge_insert`` writes new data into a fresh fragment.
The FTS (BM25) index built by :meth:`ensure_fts_indexes` only
covers fragments visible at index-build time, so rows written
after the initial build can become **invisible to BM25 queries**
until ``optimize()`` runs and merges those fragments into the
index segment that the query engine reads.
``optimize()`` is a **performance + storage-hygiene** operation,
**not** a correctness/visibility one. LanceDB's ``merge_insert``
writes new data into a fresh fragment that the FTS / vector
indexes don't cover yet; queries stay correct regardless because
LanceDB transparently brute-force flat-scans that unindexed tail
and unions it with the indexed hits. Verified on lancedb 0.30.2:
after ``create_index`` + ``merge_insert`` (no ``optimize()``), a
``nearest_to_text`` for a token present only in the new rows
returns those rows immediately.
Symptom this guards against (verified on LoCoMo conv0): after
steady-state cascade ingest, ``nearest_to_text("any_common_word")``
returns 0 hits even though the column literally contains the
token in 100% of rows — the new fragments simply hadn't been
indexed.
(Older lancedb — at/below the ``>=0.13.0`` floor this repo once
pinned — did **not** flat-scan the FTS tail, so post-build rows
were genuinely invisible to BM25 until ``optimize()``; that is
the behaviour the historical LoCoMo-conv0 note described. The
flat-scan fallback closed that gap, so optimize is now purely
about keeping that tail small.)
What ``optimize()`` actually buys on the current stack:
- **Query speed** — the unindexed tail is flat-scanned on every
query; merging it into the index keeps that scan bounded as
ingest accumulates.
- **Storage hygiene** — with ``cleanup_older_than`` it prunes
replaced fragments / stale manifests / dead index files,
bounding the on-disk file count (and FD usage at scan time).
Cascade triggers this through a per-kind throttle + trailing
edge scheduler (``CascadeWorker._schedule_optimize``): at most
one run per ~1s window per kind, decoupled from the drain
one run per throttle window per kind, decoupled from the drain
loop, with a 60s heartbeat sweep as a safety net. Cost is
O(N) data-rewrite per optimized fragment; the throttle is how
we cap it under sustained write pressure.
we cap it under sustained write pressure. Because visibility no
longer depends on it, the throttle window can be generous.
Args:
cleanup_older_than: When set, also prune (physically delete)
src/everos/core/persistence/markdown/entries.py
+1 -1
View File
@@ -85,7 +85,7 @@ class EntryId:
f"{self.prefix}_{self.date.strftime(_DATE_FMT)}_{self.seq:0{_SEQ_DIGITS}d}"
)
def __str__(self) -> str: # noqa: D401
def __str__(self) -> str:
return self.format()
@classmethod
src/everos/entrypoints/api/app.py
+2 -1
View File
@@ -11,7 +11,6 @@ import os
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from everos import __version__
from everos.core.lifespan import (
LifespanProvider,
MetricsLifespanProvider,
@@ -88,6 +87,8 @@ def create_app(
OmeLifespanProvider(),
]
from everos import __version__
app = FastAPI(
title="everos",
version=__version__,
src/everos/entrypoints/api/lifespans/ome.py
+2 -2
View File
@@ -27,13 +27,13 @@ class OmeLifespanProvider(LifespanProvider):
async def startup(self, app: FastAPI) -> Any:
svc = importlib.import_module("everos.service.memorize")
engine = svc._get_engine() # noqa: SLF001 — service-internal accessor
engine = svc._get_engine()
await engine.start()
logger.info("ome_engine_started")
return engine
async def shutdown(self, app: FastAPI) -> None:
svc = importlib.import_module("everos.service.memorize")
engine = svc._get_engine() # noqa: SLF001
engine = svc._get_engine()
await engine.stop()
logger.info("ome_engine_stopped")
src/everos/entrypoints/api/routes/ome.py
+1 -1
View File
@@ -32,7 +32,7 @@ class TriggerResponse(BaseModel):
async def trigger(req: TriggerRequest) -> TriggerResponse:
"""Manually trigger a registered OME strategy and wait for completion."""
# Deferred: avoid importing heavy OME engine at module level.
from everos.service.memorize import _get_engine # noqa: SLF001
from everos.service.memorize import _get_engine
engine = _get_engine()
try:
src/everos/infra/ome/__init__.py
+1 -1
View File
@@ -49,9 +49,9 @@ __all__ = [
"IdleTick",
"Immediate",
"ManualTick",
"OfflineEngine",
"OMEConfig",
"OMEError",
"OfflineEngine",
"RunRecord",
"RunStatus",
"StartupValidationError",
src/everos/infra/ome/_background/config_reloader.py
+3 -3
View File
@@ -153,7 +153,7 @@ def _apply_one_strategy(
if _needs_aps_reschedule(meta.trigger, new_trigger):
_maybe_reschedule_aps(engine, name, new_trigger)
registry.replace(name, new_meta)
except Exception as e: # noqa: BLE001
except Exception as e:
# User-fixable config error (typo / type mismatch / APS runtime
# failure) — log + skip this strategy's atomic group, never the loop.
logger.warning(
@@ -228,12 +228,12 @@ class ConfigReloader:
"""Initial load + per-FS-change reload; survives single-iteration failures."""
try:
await self._load_once()
except Exception: # noqa: BLE001
except Exception:
logger.exception("config_reload_iteration_failed")
async for _changes in awatch(self._path, debounce=self._debounce_ms):
try:
await self._load_once()
except Exception: # noqa: BLE001
except Exception:
logger.exception("config_reload_iteration_failed")
async def _load_once(self) -> None:
src/everos/infra/ome/_background/crash_recovery.py
+1 -1
View File
@@ -70,7 +70,7 @@ async def scan_and_resume(
old_run_id=rec.run_id,
new_run_id=new_run_id,
)
except Exception: # noqa: BLE001
except Exception:
logger.exception(
"crash_recovery_resume_failed",
strategy_name=rec.strategy_name,
src/everos/infra/ome/_background/idle_scanner.py
+1 -1
View File
@@ -52,7 +52,7 @@ class IdleScanner:
idle_seconds=self._trigger.idle_seconds,
)
)
except Exception: # noqa: BLE001
except Exception:
logger.exception(
"idle_emit_failed",
strategy_name=self._name,
src/everos/infra/ome/_dispatch/dispatcher.py
+1 -1
View File
@@ -159,7 +159,7 @@ def _safe_applies(meta: StrategyMeta, event: BaseEvent) -> bool:
"""
try:
return _applies(meta.applies_to, event)
except Exception: # noqa: BLE001
except Exception:
logger.exception(
"applies_to_callable_raised",
strategy_name=meta.name,
src/everos/infra/ome/_dispatch/runner.py
+3 -3
View File
@@ -191,7 +191,7 @@ class Runner:
except StrategyContractError as e:
await self._terminate_dead_letter(current_run_id, _format_error(e))
return True
except Exception as e: # noqa: BLE001
except Exception as e:
err = _format_error(e)
if attempt < max_retries_snapshot:
await self._rec.mark_failed(
@@ -237,7 +237,7 @@ class Runner:
max_retries_snapshot=max_retries_snapshot,
event_id=event_id,
)
except Exception: # noqa: BLE001
except Exception:
logger.exception(
"mark_running_failed",
run_id=run_id,
@@ -264,7 +264,7 @@ class Runner:
return
try:
self._on_dead_letter(rec)
except Exception: # noqa: BLE001
except Exception:
logger.exception("on_dead_letter_failed")
src/everos/infra/ome/testing/harness.py
+3 -3
View File
@@ -54,7 +54,7 @@ class StrategyTestHarness:
try:
await self._engine.stop()
finally:
shutil.rmtree(self._tmpdir, ignore_errors=True) # noqa: SLF001
shutil.rmtree(self._tmpdir, ignore_errors=True)
def register(self, strategy: Strategy) -> None:
"""Register a :class:`Strategy` returned by ``@offline_strategy``.
@@ -96,7 +96,7 @@ class StrategyTestHarness:
if not await self._engine.wait_idle(timeout=timeout):
raise TimeoutError(
f"drain: engine still has "
f"{self._engine._active_runs} in-flight runs after {timeout}s" # noqa: SLF001
f"{self._engine._active_runs} in-flight runs after {timeout}s"
)
async def list_runs(
@@ -113,7 +113,7 @@ class StrategyTestHarness:
Returns:
A list of RunRecord objects.
"""
return await self._engine._run_record_store.list_runs( # noqa: SLF001
return await self._engine._run_record_store.list_runs(
strategy_name=strategy_name,
status=status,
)
src/everos/infra/persistence/lancedb/__init__.py
+1 -1
View File
@@ -26,7 +26,7 @@ first access; row population is the cascade daemon's job (see
# Importing ``tables`` registers every business :class:`BaseLanceTable`
# schema so callers can rely on the package alone to surface every schema.
from . import tables as tables # noqa: F401
from . import tables as tables
from .lancedb_manager import dispose_connection as dispose_connection
from .lancedb_manager import get_connection as get_connection
from .lancedb_manager import get_table as get_table
src/everos/infra/persistence/lancedb/tables/episode.py
+1
View File
@@ -81,3 +81,4 @@ class Episode(BaseLanceTable):
supersedes this row. ``NULL`` means the row is still active."""
vector: Vector(_DIM) # type: ignore[valid-type]
subject_vector: Vector(_DIM) | None = None # type: ignore[valid-type]
src/everos/infra/persistence/markdown/writers/base.py
+2 -2
View File
@@ -175,7 +175,7 @@ class BaseDailyWriter:
frontmatter_updates = self._frontmatter_updates(
scope_id, bucket, next_count=base_count + len(items)
)
await self._writer._append_entries_unlocked( # noqa: SLF001
await self._writer._append_entries_unlocked(
path,
rendered,
frontmatter_updates=frontmatter_updates,
@@ -227,7 +227,7 @@ class BaseDailyWriter:
scope_id, bucket, next_count=count + 1
)
await self._writer._append_entries_unlocked( # noqa: SLF001
await self._writer._append_entries_unlocked(
path,
[(entry_body, eid)],
frontmatter_updates=frontmatter_updates,
src/everos/infra/persistence/sqlite/__init__.py
+1 -1
View File
@@ -30,7 +30,7 @@ not need to manage either.
# Importing ``tables`` registers every business SQLModel in
# ``SQLModel.metadata`` so ``SqliteLifespanProvider.startup`` can
# ``create_all`` without callers having to import each model module.
from . import tables as tables # noqa: F401
from . import tables as tables
from .repos import DocumentListPage as DocumentListPage
from .repos import DocumentUpsertPayload as DocumentUpsertPayload
from .repos import QueueSummary as QueueSummary
src/everos/infra/persistence/sqlite/repos/knowledge.py
+56 -82
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
import dataclasses
from sqlalchemy import asc, delete, desc, func, select
from sqlalchemy.dialects.sqlite import insert
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from everos.component.utils.datetime import get_utc_now
@@ -69,20 +69,6 @@ class TopicUpsertPayload:
md_path: str
def _apply_document_fields(
row: KnowledgeDocumentRow, payload: DocumentUpsertPayload
) -> None:
"""Copy payload fields onto an existing SQLAlchemy row."""
row.app_id = payload.app_id
row.project_id = payload.project_id
row.category_id = payload.category_id
row.title = payload.title
row.summary = payload.summary
row.source_name = payload.source_name
row.source_type = payload.source_type
row.md_path = payload.md_path
class _KnowledgeDocumentRepo(RepoBase[KnowledgeDocumentRow]):
"""SQLite repository for ``knowledge_documents`` table."""
@@ -200,61 +186,36 @@ class _KnowledgeDocumentRepo(RepoBase[KnowledgeDocumentRow]):
async def upsert_from_handler(self, payload: DocumentUpsertPayload) -> None:
"""Insert or update a document row from the cascade handler.
Use SQLite's atomic ``ON CONFLICT`` form rather than a
load-mutate-commit cycle. Category moves can produce concurrent
cascade events for the old and new ``index.md`` paths; an ORM row
loaded before a delete can otherwise raise ``StaleDataError`` on
flush.
Uses SQLite ``INSERT ... ON CONFLICT DO UPDATE`` to avoid the
``StaleDataError`` that occurs when a concurrent cascade handler
deletes the row between a ``get`` and ``commit``.
"""
now = get_utc_now()
values = {
**dataclasses.asdict(payload),
"created_at": now,
"updated_at": now,
}
stmt = sqlite_insert(KnowledgeDocumentRow).values(**values)
stmt = stmt.on_conflict_do_update(
index_elements=["doc_id"],
set_={
"app_id": stmt.excluded.app_id,
"project_id": stmt.excluded.project_id,
"category_id": stmt.excluded.category_id,
"title": stmt.excluded.title,
"summary": stmt.excluded.summary,
"source_name": stmt.excluded.source_name,
"source_type": stmt.excluded.source_type,
"md_path": stmt.excluded.md_path,
"updated_at": stmt.excluded.updated_at,
},
)
async with session_scope(self._factory) as s:
now = get_utc_now()
values = {
**dataclasses.asdict(payload),
"created_at": now,
"updated_at": now,
}
stmt = insert(KnowledgeDocumentRow).values(**values)
update_fields = {
key: values[key]
for key in (
"app_id",
"project_id",
"category_id",
"title",
"summary",
"source_name",
"source_type",
"md_path",
"updated_at",
)
}
await s.execute(
stmt.on_conflict_do_update(
index_elements=["doc_id"],
set_=update_fields,
)
)
await s.execute(stmt)
await s.commit()
def _apply_topic_fields(row: KnowledgeTopicRow, payload: TopicUpsertPayload) -> None:
"""Copy payload fields onto an existing SQLAlchemy row."""
row.doc_id = payload.doc_id
row.app_id = payload.app_id
row.project_id = payload.project_id
row.category_id = payload.category_id
row.topic_index = payload.topic_index
row.topic_name = payload.topic_name
row.topic_path = payload.topic_path
row.depth = payload.depth
row.parent_node_id = payload.parent_node_id
row.children_node_ids = payload.children_node_ids
row.summary = payload.summary
row.content = payload.content
row.content_labels = payload.content_labels
row.md_path = payload.md_path
class _KnowledgeTopicRepo(RepoBase[KnowledgeTopicRow]):
"""SQLite repository for ``knowledge_topics`` table."""
@@ -316,25 +277,38 @@ class _KnowledgeTopicRepo(RepoBase[KnowledgeTopicRow]):
async def upsert_from_handler(self, payload: TopicUpsertPayload) -> None:
"""Insert or update a topic row from the cascade handler.
Checks by ``node_id`` (PK). If the row exists, all mutable
columns are overwritten and ``updated_at`` bumps. If not, a
fresh row is inserted with both ``created_at`` and
``updated_at`` set to now.
Uses SQLite ``INSERT ... ON CONFLICT DO UPDATE`` to avoid
``StaleDataError`` from concurrent cascade handlers.
"""
now = get_utc_now()
values = {
**dataclasses.asdict(payload),
"created_at": now,
"updated_at": now,
}
stmt = sqlite_insert(KnowledgeTopicRow).values(**values)
stmt = stmt.on_conflict_do_update(
index_elements=["node_id"],
set_={
"doc_id": stmt.excluded.doc_id,
"app_id": stmt.excluded.app_id,
"project_id": stmt.excluded.project_id,
"category_id": stmt.excluded.category_id,
"topic_index": stmt.excluded.topic_index,
"topic_name": stmt.excluded.topic_name,
"topic_path": stmt.excluded.topic_path,
"depth": stmt.excluded.depth,
"parent_node_id": stmt.excluded.parent_node_id,
"children_node_ids": stmt.excluded.children_node_ids,
"summary": stmt.excluded.summary,
"content": stmt.excluded.content,
"content_labels": stmt.excluded.content_labels,
"md_path": stmt.excluded.md_path,
"updated_at": stmt.excluded.updated_at,
},
)
async with session_scope(self._factory) as s:
existing = await s.get(KnowledgeTopicRow, payload.node_id)
now = get_utc_now()
if existing is not None:
_apply_topic_fields(existing, payload)
existing.updated_at = now
s.add(existing)
else:
row = KnowledgeTopicRow(
**dataclasses.asdict(payload),
created_at=now,
updated_at=now,
)
s.add(row)
await s.execute(stmt)
await s.commit()
src/everos/infra/persistence/sqlite/tables/unprocessed_buffer.py
+3 -3
View File
@@ -15,7 +15,7 @@ from everos.core.persistence.sqlite.base import UtcDateTimeColumn
class UnprocessedBuffer(BaseTable, table=True):
"""One row per unprocessed message. PK ``message_id``."""
"""One row per unprocessed message. PK ``(message_id, app_id, project_id)``."""
__tablename__ = "unprocessed_buffer" # type: ignore[assignment]
__table_args__ = (
@@ -32,8 +32,8 @@ class UnprocessedBuffer(BaseTable, table=True):
)
message_id: str = Field(primary_key=True)
app_id: str = Field(default="default")
project_id: str = Field(default="default")
app_id: str = Field(default="default", primary_key=True)
project_id: str = Field(default="default", primary_key=True)
"""App / project scope segments (default ``"default"``)."""
session_id: str = Field(index=True)
track: str = Field(index=True)
src/everos/memory/cascade/handlers/episode.py
+21 -3
View File
@@ -21,7 +21,8 @@ writers must match):
``sections``:
- ``Subject`` (optional): one-line topic.
- ``Subject`` (optional): one-line topic — embedded into
``subject_vector`` and appended to the BM25 tokenization source.
- ``Summary`` (optional): condensed narrative.
- ``Content``: full episode narrative — fed to the embedder AND the
tokenizer for the ``episode_tokens`` BM25 field.
@@ -29,6 +30,8 @@ writers must match):
from __future__ import annotations
import asyncio
from everos.infra.persistence.lancedb import Episode, ParentType, episode_repo
from ._common import parse_inline_list, require_iso_timestamp
@@ -66,8 +69,22 @@ class EpisodeHandler(BaseDailyLogHandler):
) -> Episode:
s = entry.structured
text = s.sections.get("Content", "").strip()
tokens = self._deps.tokenizer.tokenize(text)
vector = await self._deps.embedder.embed(text)
subject_text = s.sections.get("Subject", "").strip()
# Embed content and subject concurrently; skip subject embed when absent.
if subject_text:
vector, subject_vector = await asyncio.gather(
self._deps.embedder.embed(text),
self._deps.embedder.embed(subject_text),
)
else:
vector = await self._deps.embedder.embed(text)
subject_vector = None
# BM25 tokenization covers both body and subject keywords.
tokenize_source = f"{text} {subject_text}" if subject_text else text
tokens = self._deps.tokenizer.tokenize(tokenize_source)
return Episode(
id=f"{owner_id}_{entry.entry_id}",
entry_id=entry.entry_id,
@@ -87,4 +104,5 @@ class EpisodeHandler(BaseDailyLogHandler):
md_path=md_path,
content_sha256=entry.content_sha256,
vector=vector,
subject_vector=subject_vector,
)
src/everos/memory/cascade/scanner.py
+1 -1
View File
@@ -105,7 +105,7 @@ class CascadeScanner:
while not self._stop.is_set():
try:
await self.scan_once()
except Exception as exc: # noqa: BLE001 — never crash the daemon
except Exception as exc:
logger.exception("cascade_scanner_failed", error=str(exc))
try:
await asyncio.wait_for(self._stop.wait(), timeout=self._interval)
src/everos/memory/cascade/watcher.py
+1 -1
View File
@@ -140,7 +140,7 @@ async def _enqueue_async(
change_type=change_type,
mtime=mtime,
)
except Exception as exc: # noqa: BLE001 — defensive: never crash watcher
except Exception as exc:
logger.warning(
"cascade_watcher_upsert_failed",
md_path=rel,
src/everos/memory/cascade/worker.py
+34 -23
View File
@@ -22,10 +22,14 @@ prefix but the handlers themselves are independent.
After a batch completes, each kind that mutated its LanceDB table is
passed to :meth:`_schedule_optimize` — a per-kind throttle + trailing
edge scheduler that fires LanceDB ``optimize()`` as a separate task,
so the drain loop is never blocked by index maintenance. A 60-second
heartbeat sweeps every kind through the same gate so unindexed
fragments don't linger after a worker restart even without new
writes. See :meth:`_schedule_optimize` for the exact semantics.
so the drain loop is never blocked by index maintenance. ``optimize()``
is a performance/storage-hygiene step, **not** a visibility one: new
rows are searchable immediately via LanceDB's flat-scan over the
unindexed tail (see :meth:`LanceRepoBase.optimize`), so optimizing only
keeps that tail small and prunes dead files. A 60-second heartbeat
sweeps every kind through the same gate so an unindexed tail doesn't
accumulate after a worker restart even without new writes. See
:meth:`_schedule_optimize` for the exact semantics.
A separate 12-hour loop (:meth:`_rebuild_loop`) does a full
``drop_index + create_index`` per kind to bound the **active** index
@@ -57,7 +61,7 @@ DEFAULT_BATCH_SIZE = 50
DEFAULT_MAX_RETRY = 3
DEFAULT_POLL_INTERVAL_SECONDS = 1.0
DEFAULT_RETRY_BACKOFF_SECONDS = 2.0
DEFAULT_OPTIMIZE_MIN_INTERVAL_SECONDS = 1.0
DEFAULT_OPTIMIZE_MIN_INTERVAL_SECONDS = 10.0
DEFAULT_OPTIMIZE_HEARTBEAT_SECONDS = 60.0
DEFAULT_OPTIMIZE_REBUILD_INTERVAL_SECONDS = 12 * 60 * 60.0
"""How often (per kind) to do a full ``drop_index + create_index`` rebuild.
@@ -97,7 +101,7 @@ exhausts file descriptors at index-scan time (observed: macOS / Linux
default ``ulimit -n`` of 1024 — the ``os error 24`` reported in CI).
The prune itself is cheap when scoped to recent versions; we just don't
want to pay it on every 1-second throttle tick. 5 minutes is the
want to pay it on every optimize throttle tick. 5 minutes is the
shortest interval that comfortably outlives any in-flight query / index
build, while keeping the on-disk footprint bounded. It is also passed
as ``cleanup_older_than`` itself (semantically: "the retention window
@@ -231,10 +235,12 @@ class CascadeWorker:
For each kind that mutated its LanceDB table this batch,
:meth:`_schedule_optimize` records a throttled optimize
intent. The actual ``optimize()`` runs as a separate task so
drain throughput is decoupled from index maintenance; callers
that need a "fully indexed" barrier (CLI ``cascade sync``)
must call :meth:`_flush_optimizers` — :meth:`drain_until_empty`
does this on their behalf.
drain throughput is decoupled from index maintenance. Drained
rows are already searchable at this point (flat-scan over the
unindexed tail); callers that additionally want the index fully
merged before returning (CLI ``cascade sync``) call
:meth:`_flush_optimizers` — :meth:`drain_until_empty` does this
on their behalf.
"""
batch = await md_change_state_repo.claim_pending_batch(self._batch_size)
if not batch:
@@ -254,8 +260,9 @@ class CascadeWorker:
cheap safety net).
Awaits :meth:`_flush_optimizers` before returning so callers
(CLI ``cascade sync``) can rely on FTS being up-to-date when
the call completes.
(CLI ``cascade sync``) get a fully merged index — not for
visibility (the data is already searchable) but so ``sync``
returns a deterministically optimized state.
"""
total = 0
for _ in range(max_passes):
@@ -272,7 +279,7 @@ class CascadeWorker:
while not self._stop.is_set():
try:
processed = await self.drain_once()
except Exception as exc: # noqa: BLE001 — never crash the daemon
except Exception as exc:
logger.exception("cascade_worker_drain_failed", error=str(exc))
processed = 0
if processed == 0:
@@ -330,7 +337,7 @@ class CascadeWorker:
new_retry_count=retry_count,
)
return None
except Exception as exc: # noqa: BLE001 — surface as unrecoverable
except Exception as exc:
last_error = f"{type(exc).__name__}: {exc}"
logger.exception(
"cascade_worker_unrecoverable",
@@ -483,7 +490,7 @@ class CascadeWorker:
kind=kind,
pruned=should_prune,
)
except Exception as exc: # noqa: BLE001 — never crash the daemon
except Exception as exc:
logger.warning(
"cascade_lancedb_optimize_failed",
kind=kind,
@@ -496,11 +503,13 @@ class CascadeWorker:
Sweeps every kind through :meth:`_schedule_optimize` once per
``optimize_heartbeat_seconds``. Without this, a worker that
restarts with stale unindexed fragments (e.g. after a crash
between write and optimize) would only catch up once new
writes arrive. The sweep goes through the same throttle gate
so it can never storm — kinds with an in-flight optimize or
a fresh ``last_run_at`` are coalesced.
restarts with an unindexed tail (e.g. after a crash between
write and optimize) would only merge it in once new writes
arrive — those rows stay searchable meanwhile (flat-scan), but
the tail keeps the scan slow and the dead files on disk; the
sweep bounds both. It goes through the same throttle gate so it
can never storm — kinds with an in-flight optimize or a fresh
``last_run_at`` are coalesced.
"""
while not self._stop.is_set():
if await self._wait_or_stop(self._optimize_heartbeat):
@@ -571,7 +580,7 @@ class CascadeWorker:
try:
await rebuild_task
logger.info("cascade_lancedb_rebuilt", kind=kind)
except Exception as exc: # noqa: BLE001 — never crash the daemon
except Exception as exc:
logger.warning(
"cascade_lancedb_rebuild_failed",
kind=kind,
@@ -586,8 +595,10 @@ class CascadeWorker:
Drain-loop path is fire-and-forget for throughput; this is the
explicit barrier used by CLI ``cascade sync`` and worker
shutdown to ensure FTS is consistent with the data on disk
before the call returns.
shutdown to let in-flight optimizes finish merging the unindexed
tail before the call returns. Not a visibility barrier — drained
rows are searchable via flat-scan regardless; this just yields a
fully merged index (and, on shutdown, no orphaned mid-write).
Exceptions from optimize tasks are already logged in
:meth:`_run_optimize_once`; ``return_exceptions=True`` here
src/everos/memory/get/filters_adapter.py
-2
View File
@@ -25,7 +25,6 @@ def compile_filters_for_get(
owner_type: str,
app_id: str = "default",
project_id: str = "default",
exclude_deprecated: bool = True,
) -> str:
"""Compile ``/get`` filters via the shared ``compile_filters`` path.
@@ -38,5 +37,4 @@ def compile_filters_for_get(
owner_type=owner_type,
app_id=app_id,
project_id=project_id,
exclude_deprecated=exclude_deprecated,
)
src/everos/memory/get/manager.py
-1
View File
@@ -76,7 +76,6 @@ class GetManager:
owner_type=req.owner_type,
app_id=req.app_id,
project_id=req.project_id,
exclude_deprecated=req.memory_type == GetMemoryType.EPISODE,
)
match req.memory_type:
src/everos/memory/search/adapter.py
+4 -4
View File
@@ -6,8 +6,8 @@ Passing ``AGENTIC`` here is a caller contract violation and raises
``ValueError`` as a defensive guard.
* ``KEYWORD`` / ``VECTOR`` → ``None`` → manager skips ``everalgo.rank``.
* ``HYBRID`` → ``"hierarchy"`` (episode / atomic_fact) — four-layer pipeline
(RRF → MaxSim → RRF merge → single-pass eviction)
* ``HYBRID`` → ``"hierarchy"`` (episode / atomic_fact) — heap-expand
pipeline (RRF-ordered expansion → LR-calibrated global top-N competition)
or ``"vector_anchored"`` (agent_case) — everalgo vector-anchored fusion (alpha=0.7)
or ``"skill_hybrid"`` (agent_skill) — custom rrf → cross-encoder rerank → optional
verify.
@@ -30,8 +30,8 @@ def resolve_pipeline(
``pipeline_signal`` of ``None`` means "do not call ``everalgo.rank.arank``;
the manager runs single-route recall and returns directly".
``"hierarchy"`` routes to the four-layer episode pipeline in
``memory.search.hierarchy`` (RRF → MaxSim → RRF merge → eviction).
``"hierarchy"`` routes to the heap-expand episode pipeline in
``memory.search.hierarchy`` (RRF → LR → heap expansion → eviction).
``"vector_anchored"`` routes to ``everalgo.rank.arank`` with vector-anchored
fusion (alpha=0.7, saturation_k=5.0) — matches the opensource case retrieval.
``"skill_hybrid"`` routes to the custom skill hybrid orchestrator in
src/everos/memory/search/agentic.py
+27 -5
View File
@@ -20,6 +20,7 @@ before calling ``shape_episode_from_candidate``.
from __future__ import annotations
import asyncio
import datetime as _dt
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Any
@@ -31,12 +32,15 @@ from everalgo.rank.maxsim import amaxsim_retrieve
from everalgo.types import Candidate
from everos.component.utils.datetime import from_timestamp, to_timestamp_ms
from everos.core.observability.logging import get_logger
from everos.infra.persistence.sqlite import cluster_repo
from everos.memory.search.callbacks import build_rerank_fn
from everos.memory.search.shaper import shape_episode_from_candidate
from .dto import SearchEpisodeItem
logger = get_logger(__name__)
if TYPE_CHECKING:
from everalgo.clustering import Cluster
from everalgo.llm.protocols import LLMClient
@@ -76,6 +80,8 @@ async def search_episodes_agentic(
*,
owner_id: str,
where: str,
app_id: str = "default",
project_id: str = "default",
episode_recaller: EpisodeRecaller,
atomic_fact_recaller: AtomicFactRecaller,
embed_query_fn: Callable[[str], Awaitable[list[float]]],
@@ -101,15 +107,23 @@ async def search_episodes_agentic(
Empty when no clusters exist or retrieval returns nothing.
"""
# 1. Fact-level child retrieve closures (dense + sparse via atomic_fact table).
# 1. Fact-level child retrieve closures (dense + sparse via both tables).
async def _fact_dense(q: str, k: int) -> list[Candidate]:
vec = await embed_query_fn(q)
if not vec:
return []
return await atomic_fact_recaller.dense_recall(vec, where, limit=k)
fact_results, ep_results = await asyncio.gather(
atomic_fact_recaller.dense_recall(vec, where, limit=k),
episode_recaller.dense_recall_subject_as_child(vec, where, limit=k),
)
return fact_results + ep_results
async def _fact_sparse(q: str, k: int) -> list[Candidate]:
return await atomic_fact_recaller.sparse_recall(q, where, limit=k)
fact_results, ep_results = await asyncio.gather(
atomic_fact_recaller.sparse_recall(q, where, limit=k),
episode_recaller.sparse_recall_as_child(q, where, limit=k),
)
return fact_results + ep_results
# 2. parent_fetch: maps entry_ids (from atomic_fact.parent_id) to episodes.
# Atomic facts always point to episodes via entry_id regardless of
@@ -168,7 +182,12 @@ async def search_episodes_agentic(
# Reshape metadata to the everalgo doc contract so the sufficiency /
# multi-query LLM prompt (rendered by ``_format_docs``) sees the episode
# body and a ms-epoch date instead of the memcell id.
clusters: list[Cluster] = await cluster_repo.list_for_owner(owner_id, "user_memory")
clusters: list[Cluster] = await cluster_repo.list_for_owner(
owner_id,
"user_memory",
app_id=app_id,
project_id=project_id,
)
raw_all_docs = await episode_recaller.fetch_all_for_owner(where)
all_docs: list[Candidate] = [
c.model_copy(update={"metadata": _to_everalgo_doc_metadata(c.metadata)})
@@ -192,7 +211,7 @@ async def search_episodes_agentic(
)
# 8. aagentic_retrieve — benchmark cluster main path.
candidates, _decision = await aagentic_retrieve(
candidates, decision = await aagentic_retrieve(
query,
base_retrieve=cluster_scoped,
round2_retrieve=hybrid_full,
@@ -207,6 +226,9 @@ async def search_episodes_agentic(
rrf_k=_HYBRID_RRF_K,
)
round_tag = "round1" if decision.is_sufficient else "round2"
logger.info("agentic_search_decision", round=round_tag, query=query[:80])
# 9. Shape: remap id from memcell_id -> episode_id, then build DTO.
return _shape_results(candidates)
src/everos/memory/search/dto.py
+4 -4
View File
@@ -81,19 +81,19 @@ class SearchRequest(BaseModel):
top_k: int = -1
radius: float | None = Field(default=None, ge=0.0, le=1.0)
min_score: float | None = Field(default=None, ge=0.0, le=1.0)
"""Post-fusion relevance floor for the episode HYBRID (hierarchy) path.
"""Post-fusion relevance floor for the episode HYBRID path.
Applied after Layer 4 against the LR-calibrated score in ``[0, 1]``:
Applied after heap-expand against the LR-calibrated score in ``[0, 1]``:
items scoring below this value are dropped. Independent of ``radius``
(which gates raw cosine at recall time); ``None`` disables the floor.
Only the episode hierarchy path consumes it — other methods ignore it.
Only the episode hybrid path consumes it — other methods ignore it.
"""
include_profile: bool = False
enable_llm_rerank: bool = Field(
default=False,
description=(
"Opt-in LLM rerank pass for HYBRID. Applies to agent_case "
"and agent_skill fusion only; the episode hierarchy path "
"and agent_skill fusion only; the episode hybrid path "
"has built-in fact eviction and ignores this flag. "
"Ignored by keyword / vector / agentic."
),
src/everos/memory/search/filters.py
+2 -4
View File
@@ -35,7 +35,7 @@ import datetime as _dt
from typing import Any, Final
from everos.component.utils.datetime import from_timestamp, to_iso_format
from everos.core.errors import FilterError as FilterError # noqa: F401
from everos.core.errors import FilterError as FilterError
from .dto import FilterNode
@@ -92,7 +92,6 @@ def compile_filters(
owner_type: str,
app_id: str = "default",
project_id: str = "default",
exclude_deprecated: bool = True,
) -> str:
"""Compile a request's filters into a single LanceDB ``where`` string.
@@ -108,9 +107,8 @@ def compile_filters(
f"owner_type = '{owner_type}'",
f"app_id = '{_escape_str(app_id)}'",
f"project_id = '{_escape_str(project_id)}'",
"deprecated_by IS NULL",
]
if exclude_deprecated:
base.append("deprecated_by IS NULL")
if node is None:
return " AND ".join(base)
compiled = _compile_node(node.model_dump(exclude_none=True))
src/everos/memory/search/hierarchy.py
+165 -275
View File
@@ -1,295 +1,35 @@
"""Hierarchical episode retrieval — two-path recall fused with per-fact eviction.
"""Heap-expand hierarchy — heap-driven lazy expansion with global top-N competition.
Episode HYBRID search path: combines episode-level hybrid recall (Layer 1)
with fact-driven MaxSim re-scoring (Layer 2), merges via RRF (Layer 3), then
runs a hierarchical fact eviction where parent episode and its facts compete on a
single LR-calibrated scale and the best fact replaces the episode when it
wins (Layer 4).
Replaces the four-layer serial pipeline (hierarchy.py) with a heap-based
approach where RRF orders the expansion priority and LR-calibrated scores
drive the global competition between episodes and their atomic facts.
Uses everalgo operators as pure algorithm primitives; all I/O is injected
via recaller callbacks. No changes to the everalgo library are required.
Pure synchronous module — zero I/O, zero async. Designed for zero-change
migration to everalgo.
"""
from __future__ import annotations
import heapq
from typing import TYPE_CHECKING
from everalgo.rank import amaxsim_retrieve
from everalgo.rank.fusion import cosine_to_lr_score, rrf
from everalgo.rank.fusion import cosine_to_lr_score, lr, rrf
from everalgo.types import Candidate, FactCandidate, ScoredItem
from everos.core.observability.logging import get_logger
from .dto import SearchEpisodeItem
from .shaper import reshape_hybrid_output
if TYPE_CHECKING:
from collections.abc import Sequence
from everos.memory.search.recall.atomic_fact import AtomicFactRecaller
from everos.memory.search.recall.episode import EpisodeRecaller
logger = get_logger(__name__)
_HIERARCHY_ALPHA = 1.0
_HIERARCHY_FACTS_PER_EPISODE = 3
from everalgo.rank.weight import LRCoefs
async def hierarchy_retrieve_episodes(
query: str,
*,
sparse: list[Candidate],
dense: list[Candidate],
query_vector: list[float],
fact_recaller: AtomicFactRecaller,
episode_recaller: EpisodeRecaller,
where: str,
top_k: int,
fact_child_candidates: int = 200,
alpha: float = _HIERARCHY_ALPHA,
min_score: float | None = None,
) -> list[SearchEpisodeItem]:
"""Run the four-layer hierarchical episode retrieval pipeline.
Layer 1: RRF fusion over pre-recalled sparse + dense episode candidates.
Layer 2: MaxSim re-score via atomic-fact child retrieval (fact cosine ANN
→ group by parent memcell → episode re-score by best fact).
Layer 3: RRF merge of Layer-1 and Layer-2 results, sliced to top_k.
Layer 4: Pre-fetch facts for merged episodes, then hierarchical eviction —
parent and facts compete on one LR-calibrated scale; the best
fact replaces its episode when it wins.
def build_ep_to_fact_parents(
episodes: list[Candidate],
) -> dict[str, list[str]]:
"""Map episode candidate id to possible fact parent_id values.
Args:
query: Raw query string passed to amaxsim_retrieve.
sparse: BM25 episode candidates from the caller's recall phase.
dense: Vector ANN episode candidates from the caller's recall phase.
query_vector: Pre-computed query embedding; reused for fact ANN recall
and per-fact scoring in facts_for_episodes.
fact_recaller: AtomicFactRecaller instance for child retrieval and
facts_for_episodes.
episode_recaller: EpisodeRecaller instance for MaxSim parent fetch.
where: LanceDB filter clause (owner scope, tenant, etc.).
top_k: Maximum number of items in the final merged slice before eviction.
fact_child_candidates: How many atomic-fact ANN candidates to pull in
Layer 2. Default 200.
alpha: Child (fact) weight in the Layer-4 LR-scale blend. Default
``_HIERARCHY_ALPHA``.
min_score: Optional post-Layer-4 relevance floor on the LR-calibrated
score in ``[0, 1]``; items below it are dropped. ``None`` disables.
episodes: Episode candidate list.
Returns:
Shaped SearchEpisodeItem list (episodes with nested atomic_facts),
sorted by score descending, each carrying an LR-calibrated score.
"""
# Layer 1 — episode RRF fusion
layer1_episodes = rrf(sparse, dense)
# Layer 2 — MaxSim re-score via atomic-fact child retrieval
layer2_episodes = await _maxsim_episode_rescore(
query=query,
query_vector=query_vector,
fact_recaller=fact_recaller,
episode_recaller=episode_recaller,
where=where,
child_candidates=fact_child_candidates,
)
# Layer 3 — RRF merge of episode-level results, slice to top_k
merged = rrf(layer1_episodes, layer2_episodes)[:top_k]
if not merged:
logger.info("hierarchy_retrieve_empty_merge", top_k=top_k)
return []
# Layer 4a — pre-fetch facts for merged episodes
ep_to_parents = _build_ep_to_fact_parents(merged)
episode_to_facts = await fact_recaller.facts_for_episodes(
ep_to_parents,
where,
per_episode=max(top_k * 2, 20),
query_vector=query_vector,
)
ep_cosine: dict[str, float] = {}
for c in (*dense, *layer2_episodes):
if c.id:
ep_cosine[c.id] = max(ep_cosine.get(c.id, 0.0), c.score)
ep_bm25 = {c.id: c.score for c in sparse if c.id}
scored_items = _hierarchy_eviction_pass(
merged,
episode_to_facts,
ep_cosine=ep_cosine,
ep_bm25=ep_bm25,
alpha=alpha,
)
# Build episode pool for orphan fact parent lookup.
# Include layer2_episodes so episodes surfaced only via MaxSim path
# (not in the original sparse/dense recall) can still serve as parent.
episode_pool = {c.id: c for c in (*sparse, *dense, *layer2_episodes)}
shaped = reshape_hybrid_output(scored_items, episode_pool=episode_pool)
# Post-Layer-4 relevance floor on the LR-calibrated score.
if min_score is not None:
shaped = [item for item in shaped if item.score >= min_score]
return shaped
def _hierarchy_eviction_pass(
merged: list[Candidate],
episode_to_facts: dict[str, list[FactCandidate]],
*,
ep_cosine: dict[str, float],
ep_bm25: dict[str, float],
alpha: float = _HIERARCHY_ALPHA,
facts_per_episode: int = _HIERARCHY_FACTS_PER_EPISODE,
) -> list[ScoredItem]:
"""Hierarchical fact eviction: parent and facts compete on one LR-calibrated scale.
For each merged episode the parent and its candidate facts are calibrated
to an LR probability via ``cosine_to_lr_score`` so a raw fact cosine and an
episode's recall relevance become directly comparable (replacing the prior
cosine-vs-RRF comparison, which mixed scales). Each fact's blended score is
``alpha * child_lr + (1 - alpha) * parent_lr``; the single best-scoring
fact replaces the episode (eviction) when it beats the parent's own LR
score, otherwise the episode is emitted at ``parent_lr``.
Args:
merged: RRF-merged episode candidates, ordered by descending score.
Their ``.score`` (RRF) is used only for ordering, not for scoring.
episode_to_facts: Map from episode id to its pre-fetched FactCandidates,
sorted by cosine similarity descending.
ep_cosine: Per-episode best cosine relevance (dense / MaxSim routes).
ep_bm25: Per-episode BM25 score (sparse route); ``0.0`` when absent.
alpha: Child (fact) weight in the blend; ``1.0`` lets the fact's own
calibrated relevance fully drive the blended score.
facts_per_episode: Max facts per episode entered into the competition.
Returns:
Mixed list of ScoredItem instances (episodes and atomic_facts), each
carrying an LR-calibrated ``score`` in ``[0, 1]``, ready for
reshape_hybrid_output.
"""
out: list[ScoredItem] = []
for episode in merged:
parent_bm25 = ep_bm25.get(episode.id, 0.0)
parent_cosine = ep_cosine.get(episode.id, 0.0)
parent_lr = cosine_to_lr_score(parent_cosine, parent_bm25)
# A fact must strictly beat the parent's LR score to evict it.
best_fact: FactCandidate | None = None
best_blended = parent_lr
for fact in episode_to_facts.get(episode.id, [])[:facts_per_episode]:
child_lr = cosine_to_lr_score(fact.score, parent_bm25)
blended = alpha * child_lr + (1.0 - alpha) * parent_lr
if blended > best_blended:
best_blended = blended
best_fact = fact
if best_fact is not None:
# Fact wins: emit fact at its blended score; episode becomes orphan parent.
out.append(
ScoredItem(
id=best_fact.id,
score=best_blended,
item_type="atomic_fact",
metadata=best_fact.metadata,
parent_episode_id=episode.id,
)
)
logger.debug(
"hierarchy_eviction_fact_wins",
episode_id=episode.id,
fact_id=best_fact.id,
fact_score=best_blended,
episode_score=parent_lr,
)
else:
# Episode wins: emit episode at its LR-calibrated parent score.
out.append(
ScoredItem(
id=episode.id,
score=parent_lr,
item_type="episode",
metadata=dict(episode.metadata),
parent_episode_id=None,
)
)
return out
# ── Internal helpers ─────────────────────────────────────────────────────
async def _maxsim_episode_rescore(
*,
query: str,
query_vector: list[float],
fact_recaller: AtomicFactRecaller,
episode_recaller: EpisodeRecaller,
where: str,
child_candidates: int,
) -> list[Candidate]:
"""Run amaxsim_retrieve to produce MaxSim-rescored episode candidates.
Atomic facts serve as child documents (their metadata["parent_id"] is
the episode entry_id). Episodes are fetched as parents via
episode_recaller.fetch_by_entry_ids.
``amaxsim_retrieve`` calls ``child_retrieve`` exactly once with the
original query string. We reuse the pre-computed ``query_vector`` to
avoid a redundant embed call.
Args:
query: Raw query string (passed verbatim to amaxsim_retrieve).
query_vector: Pre-computed query embedding; used directly for child
ANN recall, bypassing a second embed call.
fact_recaller: Provides the child ANN retrieval function.
episode_recaller: Provides the parent fetch function.
where: LanceDB filter clause.
child_candidates: Number of atomic-fact candidates to pull per call.
Returns:
Episode candidates re-scored by their best matching atomic fact.
"""
async def child_retrieve(_q: str, n: int) -> Sequence[Candidate]:
# amaxsim_retrieve calls this exactly once with the original query string.
# Reuse the pre-computed query_vector instead of re-embedding.
return await fact_recaller.dense_recall(query_vector, where, limit=n)
async def parent_fetch(entry_ids: list[str]) -> list[Candidate]:
return await episode_recaller.fetch_by_entry_ids(entry_ids, where)
return await amaxsim_retrieve(
query,
child_retrieve=child_retrieve,
parent_fetch=parent_fetch,
top_n=50,
child_candidates=child_candidates,
)
def _build_ep_to_fact_parents(episodes: list[Candidate]) -> dict[str, list[str]]:
"""Map episode candidate id to all possible fact parent_id values.
New facts (post-1.5): parent_id = episode entry_id.
Old facts (pre-1.5): parent_id = memcell_id (episode.parent_id).
Both are collected so the IN query covers both eras without backfill.
Invariant: entry_id (ep_*) and memcell_id (mc_*) namespaces never
overlap, so mixing them in one IN clause is safe.
Args:
episodes: Merged episode candidate list.
Returns:
Dict mapping episode LanceDB id to a list of candidate parent_ids
(entry_id and/or memcell_id).
Dict mapping episode id to parent_ids (entry_id and/or memcell_id).
"""
result: dict[str, list[str]] = {}
for ep in episodes:
@@ -303,3 +43,153 @@ def _build_ep_to_fact_parents(episodes: list[Candidate]) -> dict[str, list[str]]
if parents:
result[ep.id] = parents
return result
def heap_expand(
*,
sparse: list[Candidate],
dense: list[Candidate],
episode_to_facts: dict[str, list[FactCandidate]],
top_k: int = 10,
alpha: float = 1.0,
rrf_k: int = 60,
max_convergence_rounds: int = 10,
facts_per_episode: int = 3,
lr_coefs: LRCoefs | None = None,
) -> list[ScoredItem]:
"""Heap-expand hierarchy: RRF orders expansion, LR scores drive competition.
Args:
sparse: BM25 episode candidates (descending by BM25 score).
dense: Vector ANN episode candidates (descending by cosine).
episode_to_facts: Pre-fetched facts per episode, each list sorted
by cosine descending.
top_k: Maximum items in the final output.
alpha: Fact weight in the blend (1.0 = fact score only).
rrf_k: RRF constant (default 60).
max_convergence_rounds: Stop after this many consecutive rounds
with no top-N change.
facts_per_episode: Max facts per episode to enter competition.
lr_coefs: Override LR coefficients; None uses production defaults.
Returns:
Mixed list of ScoredItem (episodes + atomic_facts), sorted by
score descending. Ready for ``reshape_hybrid_output``.
"""
if not sparse and not dense:
return []
# Phase 1 — dual fusion
bm25_scores = {c.id: c.score for c in sparse}
lr_results = lr(dense, sparse, coefs=lr_coefs)
episode_scores = {c.id: c.score for c in lr_results}
rrf_results = rrf(sparse, dense, k=rrf_k)
if not rrf_results:
return []
# Phase 2 — heap + top-N init
heap: list[tuple[float, str]] = []
for doc in rrf_results:
heapq.heappush(heap, (-doc.score, doc.id))
# topn: {id: (Candidate, lr_score, item_type, source_episode_id)}
topn: dict[str, tuple[Candidate | FactCandidate, float, str, str]] = {}
for doc in rrf_results[:top_k]:
topn[doc.id] = (doc, episode_scores.get(doc.id, 0.0), "episode", doc.id)
# Phase 3 — heap convergence loop
convergence_count = 0
while heap and convergence_count < max_convergence_rounds:
_, episode_id = heapq.heappop(heap)
prev_keys = set(topn.keys())
_expand_one_episode(
episode_id,
topn=topn,
episode_to_facts=episode_to_facts,
bm25_scores=bm25_scores,
episode_scores=episode_scores,
alpha=alpha,
facts_per_episode=facts_per_episode,
top_k=top_k,
lr_coefs=lr_coefs,
)
if set(topn.keys()) == prev_keys:
convergence_count += 1
else:
convergence_count = 0
# Phase 4 — output
sorted_entries = sorted(topn.values(), key=lambda v: v[1], reverse=True)
result: list[ScoredItem] = []
for item, score, item_type, source_ep_id in sorted_entries:
if item_type == "episode":
result.append(
ScoredItem(
id=item.id,
score=score,
item_type="episode",
metadata=dict(item.metadata),
)
)
else:
result.append(
ScoredItem(
id=item.id,
score=score,
item_type="atomic_fact",
metadata=dict(item.metadata),
parent_episode_id=source_ep_id,
)
)
return result
def _expand_one_episode(
episode_id: str,
*,
topn: dict[str, tuple[Candidate | FactCandidate, float, str, str]],
episode_to_facts: dict[str, list[FactCandidate]],
bm25_scores: dict[str, float],
episode_scores: dict[str, float],
alpha: float,
facts_per_episode: int,
top_k: int,
lr_coefs: LRCoefs | None,
) -> None:
"""Expand one episode's facts and compete with top-N in place."""
# Pre-fetch 2× candidates so LR scoring can filter to the best N.
facts = episode_to_facts.get(episode_id, [])[: facts_per_episode * 2]
if not facts:
return
parent_bm25 = bm25_scores.get(episode_id, 0.0)
parent_lr = episode_scores.get(episode_id, 0.0)
scored: list[tuple[float, FactCandidate]] = []
for fact in facts:
child_lr = cosine_to_lr_score(fact.score, parent_bm25, coefs=lr_coefs)
blended = alpha * child_lr + (1.0 - alpha) * parent_lr
scored.append((blended, fact))
scored.sort(key=lambda t: t[0], reverse=True)
top_facts = scored[:facts_per_episode]
min_topn_score = min((v[1] for v in topn.values()), default=0.0) if topn else 0.0
any_entered = False
for fact_score, fact in top_facts:
if fact_score <= 0.0:
continue
if len(topn) < top_k or fact_score > min_topn_score:
topn[fact.id] = (fact, fact_score, "atomic_fact", episode_id)
any_entered = True
if len(topn) > top_k:
worst_id = min(topn, key=lambda k: topn[k][1])
del topn[worst_id]
min_topn_score = min((v[1] for v in topn.values()), default=0.0)
if any_entered and episode_id in topn:
del topn[episode_id]
src/everos/memory/search/manager.py
+23 -19
View File
@@ -12,8 +12,8 @@ guards (embedding / cross-encoder / LLM) raise early when a method is
selected without its prerequisites.
``HYBRID`` defaults to **no LLM rerank** — the response comes back
straight after the four-layer hierarchy pipeline (RRF → MaxSim →
RRF merge → single-pass fact eviction). ``enable_llm_rerank`` is
straight after the heap-expand pipeline (RRF-ordered expansion → LR-calibrated
global top-N competition with fact eviction). ``enable_llm_rerank`` is
**ignored** for the hierarchy path. ``AGENTIC`` keeps its own
internal cross-encoder rerank loop; the flag is ignored there.
@@ -33,10 +33,10 @@ from collections.abc import Sequence
from typing import TYPE_CHECKING
from everalgo.rank import DEFAULT_RANK_CONFIG, RankConfig, arank
from everalgo.rank.fusion import rrf
from everalgo.types import Candidate, RankInput
from everos.component.utils.datetime import to_display_tz
from everos.config import load_settings
from everos.core.observability.logging import get_logger
from everos.core.observability.tracing import gen_request_id
from everos.infra.persistence.sqlite import (
@@ -60,8 +60,9 @@ from .dto import (
UnprocessedMessageDTO,
)
from .filters import compile_filters
from .hierarchy import hierarchy_retrieve_episodes
from .hierarchy import build_ep_to_fact_parents, heap_expand
from .shaper import (
reshape_hybrid_output,
shape_agent_case_from_candidate,
shape_agent_skill_from_candidate,
shape_episode_from_candidate,
@@ -161,7 +162,6 @@ class SearchManager:
owner_type=req.owner_type,
app_id=req.app_id,
project_id=req.project_id,
exclude_deprecated=req.owner_type == "user",
)
self._validate_components(req)
@@ -251,6 +251,8 @@ class SearchManager:
req.query,
owner_id=req.owner_id,
where=where,
app_id=req.app_id,
project_id=req.project_id,
episode_recaller=self._ep,
atomic_fact_recaller=self._fact,
embed_query_fn=self._embedding.embed, # type: ignore[union-attr]
@@ -269,14 +271,8 @@ class SearchManager:
cands = await self._ep.sparse_recall(
req.query, where, limit=self._recall_limit(req.top_k)
)
elif load_settings().search.vector_strategy == "maxsim_atomic":
cands = await self._maxsim_atomic_recall(req, where, top_k)
else:
vector = await self._embed_query(req.query)
cands = await self._ep.dense_recall(
vector, where, limit=self._recall_limit(req.top_k)
)
cands = self._apply_radius(cands, _effective_radius(req))
cands = await self._maxsim_atomic_recall(req, where, top_k)
# ``atomic_facts`` stays empty: facts come back only when the HYBRID
# pipeline surfaces them with a score (see ``reshape_hybrid_output``).
# Single-route recall has no per-fact score against the query, so
@@ -294,17 +290,25 @@ class SearchManager:
)
if fusion_mode == "hierarchy":
return await hierarchy_retrieve_episodes(
req.query,
rrf_candidates = rrf(sparse, dense)
ep_to_parents = build_ep_to_fact_parents(rrf_candidates)
episode_to_facts = await self._fact.facts_for_episodes(
ep_to_parents,
where,
per_episode=max(top_k * 2, 20),
query_vector=query_vector,
)
scored = heap_expand(
sparse=sparse,
dense=dense,
query_vector=query_vector,
fact_recaller=self._fact,
episode_recaller=self._ep,
where=where,
episode_to_facts=episode_to_facts,
top_k=top_k,
min_score=req.min_score,
)
episode_pool = {c.id: c for c in (*sparse, *dense)}
shaped = reshape_hybrid_output(scored, episode_pool=episode_pool)
if req.min_score is not None:
shaped = [s for s in shaped if s.score >= req.min_score]
return shaped
# rrf / lr: standard everalgo fusion path (fallback).
output = await arank(
src/everos/memory/search/recall/base.py
+1 -1
View File
@@ -37,7 +37,7 @@ from everos.component.tokenizer import Tokenizer
# is huge (1024 floats); ``_distance`` belongs to LanceDB's query engine
# and is converted into ``score`` before the row leaves the recaller.
_NOISE_COLUMNS: frozenset[str] = frozenset(
{"vector", "_distance", "_score", "created_at", "updated_at"}
{"vector", "subject_vector", "_distance", "_score", "created_at", "updated_at"}
)
src/everos/memory/search/recall/episode.py
+62
View File
@@ -17,6 +17,19 @@ from .base import (
)
def _inject_parent_id(candidates: list[Candidate]) -> list[Candidate]:
"""Wrap candidates with ``parent_id`` for MaxSim group-by-parent."""
return [
Candidate(
id=c.id,
score=c.score,
source=c.source,
metadata={**c.metadata, "parent_id": c.metadata.get("entry_id", c.id)},
)
for c in candidates
]
def _q(value: str) -> str:
return value.replace("'", "''")
@@ -63,6 +76,7 @@ class EpisodeRecaller:
rows = (
await table.query()
.nearest_to(list(vector))
.column("vector")
.distance_type("cosine")
.where(where)
.limit(limit)
@@ -77,6 +91,54 @@ class EpisodeRecaller:
for r in rows
]
async def sparse_recall_as_child(
self, query: str, where: str, *, limit: int
) -> list[Candidate]:
"""Sparse recall returning episodes as MaxSim child candidates."""
return _inject_parent_id(await self.sparse_recall(query, where, limit=limit))
async def dense_recall_as_child(
self, vector: Sequence[float], where: str, *, limit: int
) -> list[Candidate]:
"""Dense recall (body vector ANN) returning as MaxSim children."""
return _inject_parent_id(await self.dense_recall(vector, where, limit=limit))
async def dense_recall_subject(
self, vector: Sequence[float], where: str, *, limit: int
) -> list[Candidate]:
"""ANN over the ``subject_vector`` column.
Rows with ``subject_vector=None`` are naturally excluded by
LanceDB ANN.
"""
if not vector:
return []
table = await get_table(Episode.TABLE_NAME, Episode)
rows = (
await table.query()
.nearest_to(list(vector))
.column("subject_vector")
.distance_type("cosine")
.where(where)
.limit(limit)
.to_list()
)
return [
row_to_candidate(
r,
source="vector",
score=cosine_score_from_distance(r.get("_distance")),
)
for r in rows
]
async def dense_recall_subject_as_child(
self, vector: Sequence[float], where: str, *, limit: int
) -> list[Candidate]:
"""Subject-vector ANN returning as MaxSim children."""
candidates = await self.dense_recall_subject(vector, where, limit=limit)
return _inject_parent_id(candidates)
async def fetch_all_for_owner(self, where: str) -> list[Candidate]:
"""Flat scan — all episodes for this owner, keyed by entry_id.
src/everos/memory/strategies/_partition_locks.py
-66
View File
@@ -1,66 +0,0 @@
"""Per-strategy partition locks for serialising RMW critical sections.
The OME engine intentionally does NOT serialise concurrent runs of the
same strategy
(``local/specs/2026-04-27-ome-tech-design.md`` §4.5.2: the business logic
must guard itself, deciding inside the strategy body via
``async with lock`` bucketed by a business key). Offline strategies whose
body is a read →
modify → write on shared state (cluster rows, user.md, SKILL.md)
serialise on a business key (``owner_id`` / ``agent_id``) here.
Mirrors :mod:`everos.service._session_lock` (and
:class:`everos.core.persistence.markdown.writer.MarkdownWriter`'s
per-path lock pool): one ``asyncio.Lock`` per
``(strategy_name, partition_key)`` pair, **never evicted** — a lock
with pending waiters must outlive any dict entry that points to it,
otherwise GC racing waiters can drop the lock mid-flight (CPython
bpo-28427). The pool grows with the live partition-key set, which in
practice is bounded by the agent / user / cluster counts a single
everos process owns.
No acquire timeout: an OME strategy run has no upstream client
waiting on it, so timing out a queued caller would only convert
"slow" into a permanent ``dead_letter`` data-loss (`max_retries`
exhaustion). The LLM client owns the per-request timeout
(`component.llm.openai_provider`, default 60s) — that is the layer
that breaks a stuck LLM call, not this one. If a genuinely hung
strategy holds the lock indefinitely it surfaces as a stuck queue
under process-level monitoring; the recovery is a process restart,
not a silent data drop.
Cross-process safety is out of scope: everos is single-process by
design (see ``CLAUDE.md`` deployment notes); the enterprise edition
layers a distributed coordinator on top.
"""
from __future__ import annotations
import asyncio
_pools: dict[str, dict[str, asyncio.Lock]] = {}
def get_partition_lock(strategy_name: str, partition_key: str) -> asyncio.Lock:
"""Return the lock for ``(strategy_name, partition_key)``; create on first use.
``dict.setdefault`` is atomic under single-threaded asyncio — no
``await`` runs between the nested ``setdefault`` calls, so the
"check then insert" pair is indivisible. No meta-lock is needed.
Callers acquire the lock with ``async with``; the lock object is
cached forever (see module docstring on bpo-28427), and the inner
asyncio queue gives FIFO fairness across waiters on the same key.
"""
return _pools.setdefault(strategy_name, {}).setdefault(
partition_key, asyncio.Lock()
)
def _reset_for_tests() -> None:
"""Test-only: drop every registered lock pool.
Used by test fixtures that need a clean lock registry between
cases (no inherited holders, no inherited waiters).
"""
_pools.clear()
src/everos/service/__init__.py
+1 -1
View File
@@ -53,12 +53,12 @@ __all__ = [
"CategoryOverview",
"CreateDocumentResult",
"DeleteResult",
"DuplicateDocumentError",
"DocumentContext",
"DocumentDetail",
"DocumentListResult",
"DocumentNotFoundError",
"DocumentOverviewItem",
"DuplicateDocumentError",
"ExtractionEmptyError",
"MemorizeResult",
"PatchResult",
src/everos/service/knowledge.py
+3 -3
View File
@@ -960,7 +960,7 @@ _embedding_resolved = False
def _get_embedding() -> EmbeddingProvider | None:
"""Build the embedding client on first call. ``None`` when not configured."""
global _embedding, _embedding_resolved # noqa: PLW0603
global _embedding, _embedding_resolved
if _embedding_resolved:
return _embedding
@@ -990,7 +990,7 @@ _recaller_resolved = False
def _build_recaller() -> KnowledgeTopicRecaller:
"""Return the shared :class:`KnowledgeTopicRecaller`, building it on first call."""
global _recaller, _recaller_resolved # noqa: PLW0603
global _recaller, _recaller_resolved
if _recaller_resolved:
return _recaller # type: ignore[return-value] -- guarded by _recaller_resolved
@@ -1014,7 +1014,7 @@ _reranker_resolved = False
def _get_reranker() -> RerankProvider | None:
"""Build the rerank client on first call. ``None`` when not configured."""
global _reranker, _reranker_resolved # noqa: PLW0603
global _reranker, _reranker_resolved
if _reranker_resolved:
return _reranker
src/everos/service/search.py
+6 -14
View File
@@ -59,14 +59,10 @@ def _get_embedding() -> EmbeddingProvider | None:
from everos.config import load_settings
cfg = load_settings().embedding
api_key = cfg.api_key.get_secret_value() if cfg.api_key else ""
if not cfg.model or not api_key or not cfg.base_url:
if not cfg.model or not cfg.api_key or not cfg.api_key.get_secret_value():
logger.warning(
"embedding_not_configured",
hint=(
"set [embedding] model / api_key / base_url to enable "
"vector / hybrid search"
),
hint="set [embedding] model / api_key to enable vector / hybrid search",
)
_embedding = None
else:
@@ -86,14 +82,11 @@ def _get_reranker() -> RerankProvider | None:
from everos.config import load_settings
cfg = load_settings().rerank
api_key = cfg.api_key.get_secret_value() if cfg.api_key else ""
if not cfg.model or not cfg.base_url or (cfg.provider != "vllm" and not api_key):
has_key = cfg.api_key and cfg.api_key.get_secret_value()
if not cfg.model or not cfg.base_url or not has_key:
logger.warning(
"rerank_not_configured",
hint=(
"set [rerank] model / api_key / base_url to enable "
"agentic search; api_key is optional only for provider='vllm'"
),
hint="set [rerank] model / api_key / base_url to enable agentic search",
)
_reranker = None
else:
@@ -113,8 +106,7 @@ def _get_llm_client() -> LLMClient | None:
from everos.config import load_settings
cfg = load_settings().llm
api_key = cfg.api_key.get_secret_value() if cfg.api_key else ""
if not api_key or not cfg.base_url:
if not cfg.api_key or not cfg.api_key.get_secret_value() or not cfg.base_url:
logger.warning(
"llm_not_configured",
hint="set [llm] api_key / base_url to enable hybrid / agentic search",
src/everos/templates/env.template
-24
View File
@@ -30,13 +30,6 @@ EVEROS_LLM__MODEL=openai/gpt-4.1-mini
EVEROS_LLM__API_KEY=
EVEROS_LLM__BASE_URL=https://openrouter.ai/api/v1
# Aliyun Bailian (DashScope) alternative — one key also covers embedding +
# rerank below, so the same value goes in all three API_KEY slots.
# Uncomment these three lines and comment out the OpenRouter lines above:
# EVEROS_LLM__MODEL=qwen-plus
# EVEROS_LLM__API_KEY=<DASHSCOPE_API_KEY>
# EVEROS_LLM__BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
# ─── Multimodal LLM (independent from [llm]; vision/audio capable) ────
# Separate model for parsing multimodal content items (image / pdf /
@@ -66,12 +59,6 @@ EVEROS_MULTIMODAL__BASE_URL=https://openrouter.ai/api/v1
EVEROS_EMBEDDING__MODEL=Qwen/Qwen3-Embedding-4B
EVEROS_EMBEDDING__API_KEY=
EVEROS_EMBEDDING__BASE_URL=https://api.deepinfra.com/v1/openai
# Aliyun Bailian (DashScope) alternative. Uncomment these three lines and
# comment out the DeepInfra embedding lines above:
# EVEROS_EMBEDDING__MODEL=text-embedding-v4
# EVEROS_EMBEDDING__API_KEY=<DASHSCOPE_API_KEY>
# EVEROS_EMBEDDING__BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
# Runtime knobs — uncomment to override defaults (30s / 3 / 10 / 5):
# EVEROS_EMBEDDING__TIMEOUT_SECONDS=30
# EVEROS_EMBEDDING__MAX_RETRIES=3
@@ -81,21 +68,10 @@ EVEROS_EMBEDDING__BASE_URL=https://api.deepinfra.com/v1/openai
# ─── Rerank (OpenAI-protocol /rerank) ────────────────
# base_url should point at the rerank endpoint (e.g. .../v1/rerank).
# EVEROS_RERANK__PROVIDER selects the request-shape: "deepinfra" (default),
# "vllm", or "dashscope". Left unset, it is inferred from the base_url host
# (dashscope.aliyuncs.com -> dashscope, *.deepinfra.com -> deepinfra).
EVEROS_RERANK__MODEL=Qwen/Qwen3-Reranker-4B
EVEROS_RERANK__API_KEY=
EVEROS_RERANK__BASE_URL=https://api.deepinfra.com/v1/inference
# Aliyun Bailian (DashScope) alternative. EverOS currently supports
# DashScope rerank via gte-rerank-v2; provider is auto-inferred from host,
# so EVEROS_RERANK__PROVIDER is optional. Uncomment these three lines and
# comment out the DeepInfra rerank lines above:
# EVEROS_RERANK__MODEL=gte-rerank-v2
# EVEROS_RERANK__API_KEY=<DASHSCOPE_API_KEY>
# EVEROS_RERANK__BASE_URL=https://dashscope.aliyuncs.com
# Runtime knobs — uncomment to override defaults (30s / 3 / 10 / 5):
# EVEROS_RERANK__TIMEOUT_SECONDS=30
# EVEROS_RERANK__MAX_RETRIES=3
tests/e2e/conftest.py
-4
View File
@@ -35,7 +35,6 @@ import asyncio
import importlib
import json
from collections.abc import AsyncIterator, Awaitable, Callable
from importlib import resources
from pathlib import Path
import httpx
@@ -125,9 +124,6 @@ async def core_pipeline_runtime(
"""
monkeypatch.setenv("EVEROS_ROOT", str(tmp_path))
default_ome = resources.files("everos.config").joinpath("default_ome.toml")
(tmp_path / "ome.toml").write_text(default_ome.read_text(encoding="utf-8"))
from everos.config import load_settings
load_settings.cache_clear()
tests/test_reflection_e2e.py → tests/e2e/test_reflection_e2e.py
+4 -4
View File
@@ -17,11 +17,11 @@ import time
from pathlib import Path
from typing import Any
# Path setup for sibling-module import (test_locomo lives in the same dir).
# load_dotenv() is deferred to main() to avoid module-level side effects.
sys.path.insert(0, str(Path(__file__).resolve().parent))
# benchmarks/run.py is the benchmark runner; add repo root to sys.path so
# the benchmarks package is importable from any working directory.
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from test_locomo import ( # noqa: E402
from benchmarks.run import (
ANSWER_PROMPT,
JUDGE_SYSTEM_PROMPT,
JUDGE_USER_PROMPT,
tests/e2e/test_search_endpoint_e2e.py
+18 -21
View File
@@ -352,10 +352,9 @@ async def test_vector_search_returns_episode_hits(
) -> None:
"""``method=vector`` embeds the query and ranks by cosine.
Seeds atomic_facts alongside episodes because the default
``vector_strategy = "maxsim_atomic"`` (config/default.toml) walks
Seeds atomic_facts alongside episodes because the MaxSim path walks
atomic_fact ANN → max-pool by parent_id → fetch episodes; an
episode-only corpus would return 0 hits under that strategy.
episode-only corpus would return 0 hits.
"""
await _seed_episodes(_eps_for_owner(search_seed, "caroline"))
await _seed_atomic_facts(_facts_for_owner(search_seed, "caroline"))
@@ -419,17 +418,15 @@ async def test_agentic_search_returns_episode_hits(
# ── Agent owner_type dispatch (separate path: agent_case + agent_skill) ─
async def _seed_one_agent_corpus(
owner: str = "a1", *, use_real_embeddings: bool = False
) -> None:
async def _seed_one_agent_corpus(owner: str = "a1") -> None:
"""Single seed used by the parametrized agent dispatch test.
One case + one skill sharing surface tokens with the test query
("refactor authentication") so BM25 deterministically hits both
tables. Dense / agentic methods exercise the same rows and opt into
real embeddings so LanceDB's ``nearest_to`` can rank them (zero
vectors are undefined under cosine distance — the dense path returns
0 hits for them).
tables; dense / agentic methods exercise the same rows. Both rows
are embedded with the real embedder so LanceDB's ``nearest_to``
can rank them (zero vectors are undefined under cosine distance —
the dense path returns 0 hits for them).
"""
from everos.service.search import _get_embedding
@@ -438,16 +435,16 @@ async def _seed_one_agent_corpus(
skill_desc = "refactor authentication middleware reliably"
skill_body = "step-by-step approach for auth refactors"
embedder = _get_embedding() if use_real_embeddings else None
embedder = _get_embedding()
if embedder is not None:
case_vec, skill_vec = await embedder.embed_batch(
[f"{case_intent}\n{case_approach}", f"{skill_desc}\n{skill_body}"]
)
else:
# Keyword-only default runs offline in CI; live dense variants
# pass real embeddings via ``use_real_embeddings=True``.
case_vec = [1.0, *([0.0] * 1023)]
skill_vec = [1.0, *([0.0] * 1023)]
# No embedder credentials → leave zeros; only keyword assertions
# will pass, vector/hybrid/agentic methods are skipped anyway.
case_vec = [0.0] * 1024
skill_vec = [0.0] * 1024
await _seed_agent_cases(
[
@@ -511,7 +508,7 @@ async def test_search_agent_dispatch_per_method(
All methods must enforce the owner_type hard partition:
``episodes`` / ``profiles`` stay empty.
"""
await _seed_one_agent_corpus(use_real_embeddings=method != "keyword")
await _seed_one_agent_corpus()
resp = await _post(
client,
@@ -1290,7 +1287,7 @@ async def test_search_filter_error_returns_422(
# FastAPI's default ``{"detail": ...}``). The FilterError text
# lands in ``error.message``.
body = resp.json()
assert body["error"]["code"] == "INVALID_INPUT"
assert body["error"]["code"] == "HTTP_ERROR"
assert "this_field_does_not_exist" in body["error"]["message"]
@@ -1310,10 +1307,10 @@ async def test_vector_search_with_session_filter(
recall query (not bypassed by the dense path). Half the seed gets
a target session, half gets another; only target hits may come back.
The default ``vector_strategy = "maxsim_atomic"`` filters atomic_facts
first (then max-pools to episodes), so the per-episode session_id
mutation has to propagate to each fact via its parent memcell id —
otherwise the where clause drops every fact and recall returns 0.
The MaxSim path filters atomic_facts first (then max-pools to
episodes), so the per-episode session_id mutation has to propagate
to each fact via its parent memcell id — otherwise the where clause
drops every fact and recall returns 0.
"""
base = _eps_for_owner(search_seed, "caroline")
facts = _facts_for_owner(search_seed, "caroline")
tests/integration/test_cascade_all_kinds_consistency.py
+2 -2
View File
@@ -215,12 +215,12 @@ _KIND_CASES: list[_DailyLogKindCase] = [
async def _wait_path_done(md_path: str, *, deadline: float = 15.0) -> None:
async with asyncio.timeout(deadline):
while True: # noqa: ASYNC110 - polling cascade state
while True:
row = await md_change_state_repo.get_by_id(md_path)
if row is not None:
break
await asyncio.sleep(0.05)
while True: # noqa: ASYNC110 - polling cascade state
while True:
row = await md_change_state_repo.get_by_id(md_path)
if row is not None and row.status in ("done", "failed"):
break
tests/integration/test_cascade_cli_integration.py
+14 -5
View File
@@ -60,6 +60,10 @@ async def _dispose_all() -> None:
await dispose_engine()
def _strip_ansi(value: str) -> str:
return re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", value)
def test_status_on_empty_queue(cli_runtime: Path) -> None:
"""``cascade status`` boots the runtime + prints zeros for a fresh DB."""
result = CliRunner().invoke(cascade_mod.app, ["status"])
@@ -121,12 +125,17 @@ def test_sync_with_path_outside_root_errors(
other.write_text("# unrelated\n")
result = CliRunner().invoke(cascade_mod.app, ["sync", str(other)])
assert result.exit_code != 0
# Typer.BadParameter surfaces in stderr / mixed output. Rich may wrap
# the error box at different terminal widths, so assert the stable
# semantic fragments instead of their exact adjacency.
# Typer.BadParameter surfaces in stderr / mixed output. The rich
# error box wraps the message at terminal width and pads each line
# with ``│`` (U+2502 box-drawing); so ``not under`` and
# ``memory root`` end up separated by spaces *plus* box characters
# *plus* a newline. ``\s`` doesn't match ``│``, so widen to
# ``[^\w]+`` (anything that isn't an alnum / underscore) — that
# tolerates the rich frame without falsely matching real text
# between the two tokens.
output = result.stdout + (result.stderr or "")
assert re.search(r"\bnot under\b", output), output
assert re.search(r"\bmemory root\b", output), output
plain_output = _strip_ansi(output)
assert re.search(r"not under[^\w]+memory root", plain_output), output
def test_sync_with_unmatched_path(
tests/integration/test_cascade_scenarios.py
+21 -10
View File
@@ -127,12 +127,12 @@ async def _wait_path_done(md_path: str, *, deadline: float = 15.0) -> None:
last-second re-enqueue (e.g. atomic-replace echo).
"""
async with asyncio.timeout(deadline):
while True: # noqa: ASYNC110 - polling cascade state
while True:
row = await md_change_state_repo.get_by_id(md_path)
if row is not None:
break
await asyncio.sleep(0.05)
while True: # noqa: ASYNC110 - polling cascade state
while True:
row = await md_change_state_repo.get_by_id(md_path)
if row is not None and row.status in ("done", "failed"):
break
@@ -400,14 +400,25 @@ async def test_modify_existing_entry_content_reindexes(
assert new_text != text
mw = MarkdownWriter(memory_root)
await mw.write(absolute, new_text)
# The edit reuses md_path; row status flips back to pending then
# to done again. Poll until content_sha256 actually changes.
await asyncio.sleep(0.3)
await _wait_drain()
# The edit reuses md_path; watcher enqueue can lag behind the write,
# so queue-empty is not a sufficient barrier. Poll for the externally
# visible condition this scenario cares about: LanceDB rows reflect the
# rewritten entry bodies.
async with asyncio.timeout(15.0):
while True:
await _wait_drain()
rows_after = await atomic_fact_repo.find_where(
f"md_path = '{md_path}'", limit=10
)
if len(rows_after) == 3 and all(
r.content_sha256 != sha_before.get(r.entry_id)
and "EDITED" in r.fact
and "ORIGINAL" not in r.fact
for r in rows_after
):
break
await asyncio.sleep(0.05)
rows_after = await atomic_fact_repo.find_where(
f"md_path = '{md_path}'", limit=10
)
assert len(rows_after) == 3
sha_after = {r.entry_id: r.content_sha256 for r in rows_after}
fact_after = {r.entry_id: r.fact for r in rows_after}
@@ -616,7 +627,7 @@ async def test_scanner_recovers_missed_delete(
await asyncio.sleep(0.1)
async with asyncio.timeout(5.0):
while True: # noqa: ASYNC110 - polling cascade state
while True:
row = await md_change_state_repo.get_by_id(md_path)
if row is not None and row.status == "done":
break
tests/integration/test_ome_strategies_integration.py
+2 -2
View File
@@ -217,7 +217,7 @@ async def _setup_system_db_schema(monkeypatch: pytest.MonkeyPatch) -> None:
from everos.infra.persistence.sqlite import sqlite_manager
if sqlite_manager._engine is not None: # noqa: SLF001
if sqlite_manager._engine is not None:
await sqlite_manager.dispose_engine()
monkeypatch.setattr(sqlite_manager, "_engine", None, raising=False)
monkeypatch.setattr(sqlite_manager, "_session_factory", None, raising=False)
@@ -231,7 +231,7 @@ async def _teardown_system_db_schema() -> None:
the event loop (counterpart of :func:`_setup_system_db_schema`)."""
from everos.infra.persistence.sqlite import sqlite_manager
if sqlite_manager._engine is not None: # noqa: SLF001
if sqlite_manager._engine is not None:
await sqlite_manager.dispose_engine()
tests/integration/test_reflection_integration.py
+2 -2
View File
@@ -182,7 +182,7 @@ async def _setup_sqlite(monkeypatch: pytest.MonkeyPatch) -> None:
"""Reset the sqlite_manager singleton and create_all tables."""
from everos.infra.persistence.sqlite import sqlite_manager
if sqlite_manager._engine is not None: # noqa: SLF001
if sqlite_manager._engine is not None:
await sqlite_manager.dispose_engine()
monkeypatch.setattr(sqlite_manager, "_engine", None, raising=False)
monkeypatch.setattr(sqlite_manager, "_session_factory", None, raising=False)
@@ -194,7 +194,7 @@ async def _setup_sqlite(monkeypatch: pytest.MonkeyPatch) -> None:
async def _teardown_sqlite() -> None:
from everos.infra.persistence.sqlite import sqlite_manager
if sqlite_manager._engine is not None: # noqa: SLF001
if sqlite_manager._engine is not None:
await sqlite_manager.dispose_engine()
tests/run_locomo_10x3.sh
-264
View File
@@ -1,264 +0,0 @@
#!/usr/bin/env bash
# Run the LoCoMo benchmark across all 10 conversations × 3 retrieval
# methods (keyword, vector, hybrid).
#
# Wraps tests/run_locomo_batch.sh with the defaults that match the
# everos post-fix benchmark protocol:
# - all 10 LoCoMo conversations (conv 0..9)
# - keyword + vector + hybrid (agentic is skipped — costs 2-3× more
# LLM tokens and the rerank loop hasn't been benchmarked yet)
# - speaker_a partition (the LoCoMo "Plan C" single-owner eval)
# - judge runs = 1 (single-pass LLM judge, no majority vote)
# - top-K 10
#
# Two ingest modes:
#
# --skip-add (default) reuse the corpus that already lives at
# ~/.everos-report-corpus. Skips the
# ~5 min/conv ingest phase × 10 = ~50 min
# saved. Note: the existing corpus may
# still carry artefacts from the OLD code
# (conv-5 missing episode rows,
# MRAG score=0.0 facts). For a strictly
# clean benchmark of the *fixed* code,
# use --fresh-corpus instead.
#
# --fresh-corpus wipe ~/.everos-report-corpus, restart
# the server, and re-ingest every conv
# with the current bug-fixed cascade.
# Adds ~50 min to the run.
#
# Server must already be running on :8000 with the current code loaded
# (i.e. the OR + optimize fixes). Health check confirmed before launch.
#
# Output structure:
#
# benchmark_results/run_<ts>_10x3/
# ├── conv0.json ... conv9.json ← per-conv final results
# ├── conv0_checkpoints/ ... ← phase-level snapshots
# └── SUMMARY.md ← cross-conv accuracy table
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." &> /dev/null && pwd)"
# ── Defaults ──────────────────────────────────────────────────────────
BASE_URL="${BASE_URL:-http://localhost:8000}"
DATA_PATH="${DATA_PATH:-data/locomo10.json}"
MEMORY_ROOT="${EVEROS_ROOT:-$HOME/.everos-report-corpus}"
MODE="skip-add" # default; toggle via --fresh-corpus
TS="$(date +%Y%m%d_%H%M%S)"
OUTPUT_ROOT="$REPO_ROOT/benchmark_results/run_${TS}_10x3"
# ── Parse args ────────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
case "$1" in
--skip-add) MODE="skip-add"; shift ;;
--fresh-corpus) MODE="fresh"; shift ;;
--base-url) BASE_URL="$2"; shift 2 ;;
--memory-root) MEMORY_ROOT="$2"; shift 2 ;;
--output-root) OUTPUT_ROOT="$2"; shift 2 ;;
-h|--help)
grep -E "^# " "$0" | sed 's/^# //;s/^#//'
exit 0
;;
*)
echo "unknown arg: $1" >&2
exit 1
;;
esac
done
# ── Preflight ─────────────────────────────────────────────────────────
echo "═════════════════════════════════════════════════════════════════"
echo " LoCoMo 10 × 3 benchmark"
echo "═════════════════════════════════════════════════════════════════"
echo " mode: $MODE"
echo " base_url: $BASE_URL"
echo " memory_root: $MEMORY_ROOT"
echo " output: $OUTPUT_ROOT"
echo
# 1. Server up?
if ! curl -fsS -o /dev/null "$BASE_URL/health" 2>/dev/null; then
echo "❌ server at $BASE_URL is not responding"
echo " start with: EVEROS_ROOT=$MEMORY_ROOT PYTHONPATH=src \\"
echo " python -m everos.entrypoints.cli.main server start --port 8000"
exit 1
fi
echo "✓ server healthy"
# 2. LLM env (test_locomo.py reads bare LLM_* — bridge from EVEROS_LLM__*)
if [[ -z "${LLM_API_KEY:-}" ]] || [[ -z "${LLM_BASE_URL:-}" ]] || [[ -z "${LLM_MODEL:-}" ]]; then
if [[ -f "$REPO_ROOT/.env" ]]; then
set -a
# shellcheck source=/dev/null
source <(grep -E "^EVEROS_LLM__" "$REPO_ROOT/.env" | sed 's/EVEROS_LLM__/LLM_/')
set +a
fi
fi
if [[ -z "${LLM_API_KEY:-}" ]]; then
echo "❌ LLM_API_KEY not set (and .env has no EVEROS_LLM__API_KEY to bridge from)"
exit 1
fi
echo "✓ LLM credentials: model=$LLM_MODEL @ $LLM_BASE_URL"
echo
# 3. Fresh corpus mode → wipe + restart server
if [[ "$MODE" == "fresh" ]]; then
echo "═════════════════════════════════════════════════════════════════"
echo " --fresh-corpus: wiping $MEMORY_ROOT and restarting server"
echo "═════════════════════════════════════════════════════════════════"
# Find and kill existing server (best-effort)
pids="$(pgrep -f "everos.entrypoints.cli.main server" || true)"
if [[ -n "$pids" ]]; then
echo " stopping server pid(s): $pids"
# shellcheck disable=SC2086
kill $pids
sleep 3
fi
rm -rf "$MEMORY_ROOT"
mkdir -p "$MEMORY_ROOT"
# Restart in background; the server picks up the empty memory root.
echo " starting fresh server..."
(
cd "$REPO_ROOT"
EVEROS_ROOT="$MEMORY_ROOT" \
PYTHONPATH=src \
nohup python -m everos.entrypoints.cli.main server start --port 8000 \
> /tmp/everos-server-${TS}.log 2>&1 &
echo " server pid=$!"
)
# Wait for lifespan ready
for i in $(seq 1 60); do
if curl -fsS -o /dev/null "$BASE_URL/health" 2>/dev/null; then
echo " server ready after ${i}s"
break
fi
sleep 1
done
if ! curl -fsS -o /dev/null "$BASE_URL/health" 2>/dev/null; then
echo "❌ server failed to come up; see /tmp/everos-server-${TS}.log"
exit 1
fi
echo
fi
# ── Build the batch invocation ────────────────────────────────────────
BATCH_ARGS=(
--conv-indices 0-9
--methods keyword,vector,hybrid
--base-url "$BASE_URL"
--top-k 10
--eval-owner speaker_a
--judge-runs 1
--output-root "$OUTPUT_ROOT"
)
[[ "$MODE" == "skip-add" ]] && BATCH_ARGS+=( --skip-add )
echo "═════════════════════════════════════════════════════════════════"
echo " Launching: tests/run_locomo_batch.sh ${BATCH_ARGS[*]}"
echo "═════════════════════════════════════════════════════════════════"
echo
cd "$REPO_ROOT"
bash tests/run_locomo_batch.sh "${BATCH_ARGS[@]}"
# ── Summary markdown ──────────────────────────────────────────────────
echo
echo "═════════════════════════════════════════════════════════════════"
echo " Rendering SUMMARY.md"
echo "═════════════════════════════════════════════════════════════════"
python - <<PYEOF
import json
from pathlib import Path
root = Path("$OUTPUT_ROOT")
out_md = root / "SUMMARY.md"
files = sorted(root.glob("conv*.json"))
if not files:
print(f"no result files under {root}")
raise SystemExit
methods_seen: list[str] = []
for p in files:
d = json.load(open(p))
for m in d["methods"]:
if m not in methods_seen:
methods_seen.append(m)
cat_names = {"1": "single-hop", "2": "multi-hop", "3": "open-domain", "4": "temporal"}
lines: list[str] = []
lines.append(f"# LoCoMo 10×3 — run_${TS}\n")
lines.append(
f"- mode: `{'$MODE'}`\n"
f"- base_url: \`$BASE_URL\`\n"
f"- memory_root: \`$MEMORY_ROOT\`\n"
f"- methods: \`{', '.join(methods_seen)}\`\n"
)
# Per-conv table
lines.append("\n## Per-conv accuracy\n\n")
lines.append("| conv | " + " | ".join(f"**{m}**" for m in methods_seen) + " |\n")
lines.append("|---|" + "|".join(["---"] * len(methods_seen)) + "|\n")
agg_correct = {m: 0 for m in methods_seen}
agg_total = {m: 0 for m in methods_seen}
cat_correct: dict[str, dict[str, int]] = {m: {} for m in methods_seen}
cat_total: dict[str, dict[str, int]] = {m: {} for m in methods_seen}
for p in files:
d = json.load(open(p))
cells = []
for m in methods_seen:
mr = d["methods"].get(m)
if mr is None:
cells.append("—")
continue
s = mr["summary"]
cells.append(f"{s['accuracy']*100:.1f}%")
agg_correct[m] += s["correct"]
agg_total[m] += s["total"]
for cat, st in s["category_stats"].items():
cat_correct[m][cat] = cat_correct[m].get(cat, 0) + st["correct"]
cat_total[m][cat] = cat_total[m].get(cat, 0) + st["total"]
lines.append(f"| {p.stem} | " + " | ".join(cells) + " |\n")
# Overall
overall = []
for m in methods_seen:
if agg_total[m]:
overall.append(f"**{agg_correct[m]/agg_total[m]*100:.1f}%**")
else:
overall.append("—")
lines.append(f"| **OVERALL** | " + " | ".join(overall) + " |\n")
# Per-category
lines.append("\n## Per-category accuracy (across all 10 convs)\n\n")
lines.append("| cat | kind | " + " | ".join(f"**{m}**" for m in methods_seen) + " |\n")
lines.append("|---|---|" + "|".join(["---"] * len(methods_seen)) + "|\n")
for cat in ["1", "2", "3", "4"]:
cells = []
for m in methods_seen:
tot = cat_total[m].get(cat, 0)
if tot:
cells.append(f"{cat_correct[m][cat] / tot * 100:.1f}%")
else:
cells.append("—")
lines.append(f"| {cat} | {cat_names[cat]} | " + " | ".join(cells) + " |\n")
out_md.write_text("".join(lines), encoding="utf-8")
print(f" → {out_md}")
print()
print("".join(lines))
PYEOF
echo
echo "Done."
tests/run_locomo_batch.sh
-287
View File
@@ -1,287 +0,0 @@
#!/usr/bin/env bash
# Batch driver for LoCoMo benchmark across multiple conversations + methods.
#
# Wraps tests/test_locomo.py in an outer ``--conv-index`` loop. test_locomo.py
# already loops over ``--methods`` internally, so one invocation per
# conversation runs the full method matrix for that conv.
#
# Per-conv outputs (separate JSON + checkpoint dir) live under
# ``benchmark_results/run_<timestamp>/conv<N>.json`` so reports never collide.
# An aggregate accuracy table is printed at the end.
#
# Examples
# ────────
# # all 10 convs, hybrid only:
# bash tests/run_locomo_batch.sh --conv-indices 0-9 --methods hybrid
#
# # 3 specific convs, two methods, skip the ~5min Add phase (corpus already loaded):
# bash tests/run_locomo_batch.sh \
# --conv-indices 0,3,7 --methods keyword,hybrid --skip-add
#
# # one conv, all 4 methods comparison:
# bash tests/run_locomo_batch.sh --conv-indices 0 --methods keyword,vector,hybrid,agentic
set -euo pipefail
# ── Defaults (override via flags) ─────────────────────────────────────
BASE_URL="${BASE_URL:-http://localhost:8000}"
DATA_PATH="${DATA_PATH:-data/locomo10.json}"
CONV_INDICES="${CONV_INDICES:-0}"
METHODS="${METHODS:-hybrid}"
TOP_K="${TOP_K:-10}"
EVAL_OWNER="${EVAL_OWNER:-speaker_a}"
JUDGE_RUNS="${JUDGE_RUNS:-1}"
SKIP_ADD="false"
OUTPUT_ROOT=""
CONCURRENCY="${CONCURRENCY:-1}"
# Default to polling cascade pending==0 (not fixed sleep). Falls back to
# ~/.everos to match the server's default data root; override via env or
# EVEROS_ROOT (which the server consumes). post-flush-wait becomes
# the MAX wait when corpus-path is set.
CORPUS_PATH="${CORPUS_PATH:-${EVEROS_ROOT:-$HOME/.everos}}"
POST_FLUSH_WAIT="${POST_FLUSH_WAIT:-600}"
EXTRA_ARGS=()
usage() {
cat <<EOF
Usage: bash tests/run_locomo_batch.sh [options]
--conv-indices <spec> conv list — "0,1,2" | "0-9" | "all" (default: $CONV_INDICES)
--methods <list> comma-separated, e.g. "keyword,hybrid" (default: $METHODS)
--base-url <url> everos server (default: $BASE_URL)
--data-path <file> LoCoMo dataset path (default: $DATA_PATH)
--top-k <int> per-question recall depth (default: $TOP_K)
--eval-owner <a|b> speaker_a | speaker_b (default: $EVAL_OWNER)
--judge-runs <int> LLM judge majority-vote runs (default: $JUDGE_RUNS)
--skip-add reuse existing corpus, skip ingest
--output-root <dir> parent dir for results
(default: benchmark_results/run_<ts>)
--concurrency <N> run up to N convs in parallel (default: 1 = serial)
per-conv stdout/stderr is redirected to
\$OUTPUT_ROOT/conv<i>.log so streams don't interleave
-h | --help show this help
-- everything after is forwarded to test_locomo.py
Any positional or unknown arg goes through to test_locomo.py untouched.
EOF
}
while [[ $# -gt 0 ]]; do
case "$1" in
--conv-indices) CONV_INDICES="$2"; shift 2 ;;
--methods) METHODS="$2"; shift 2 ;;
--base-url) BASE_URL="$2"; shift 2 ;;
--data-path) DATA_PATH="$2"; shift 2 ;;
--top-k) TOP_K="$2"; shift 2 ;;
--eval-owner) EVAL_OWNER="$2"; shift 2 ;;
--judge-runs) JUDGE_RUNS="$2"; shift 2 ;;
--skip-add) SKIP_ADD="true"; shift ;;
--output-root) OUTPUT_ROOT="$2"; shift 2 ;;
--concurrency) CONCURRENCY="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
--) shift; EXTRA_ARGS+=("$@"); break ;;
*) EXTRA_ARGS+=("$1"); shift ;;
esac
done
# ── Expand conv-indices spec ──────────────────────────────────────────
expand_indices() {
local spec="$1"
if [[ "$spec" == "all" ]]; then
echo "0 1 2 3 4 5 6 7 8 9"
return
fi
if [[ "$spec" =~ ^([0-9]+)-([0-9]+)$ ]]; then
seq "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
return
fi
echo "$spec" | tr ',' ' '
}
INDICES=$(expand_indices "$CONV_INDICES")
TS="$(date +%Y%m%d_%H%M%S)"
OUTPUT_ROOT="${OUTPUT_ROOT:-benchmark_results/run_${TS}}"
mkdir -p "$OUTPUT_ROOT"
# ── Plan banner ───────────────────────────────────────────────────────
echo "═════════════════════════════════════════════════════════════════"
echo " LoCoMo batch run"
echo "═════════════════════════════════════════════════════════════════"
printf " base_url : %s\n" "$BASE_URL"
printf " conv_indices : %s\n" "$(echo "$INDICES" | tr '\n' ' ')"
printf " methods : %s\n" "$METHODS"
printf " top_k : %s\n" "$TOP_K"
printf " eval_owner : %s\n" "$EVAL_OWNER"
printf " judge_runs : %s\n" "$JUDGE_RUNS"
printf " skip_add : %s\n" "$SKIP_ADD"
printf " concurrency : %s\n" "$CONCURRENCY"
printf " output_root : %s\n" "$OUTPUT_ROOT"
[[ ${#EXTRA_ARGS[@]} -gt 0 ]] && printf " forwarded args : %s\n" "${EXTRA_ARGS[*]}"
echo
# ── Build per-conv command and launch ────────────────────────────────
#
# bash 3.2 (macOS default) lacks namerefs (`local -n`) and `wait -n`, so
# build_cmd populates a global array CMD and the parallel scheduler
# uses a poll-loop with `kill -0` instead of `wait -n`.
build_cmd() {
local _ci="$1"
CMD=(
PYTHONPATH=src
python tests/test_locomo.py
--base-url "$BASE_URL"
--data-path "$DATA_PATH"
--conv-index "$_ci"
--methods "$METHODS"
--top-k "$TOP_K"
--eval-owner "$EVAL_OWNER"
--judge-runs "$JUDGE_RUNS"
--output "$OUTPUT_ROOT/conv${_ci}.json"
--checkpoint-dir "$OUTPUT_ROOT/conv${_ci}_checkpoints"
--corpus-path "$CORPUS_PATH"
--post-flush-wait "$POST_FLUSH_WAIT"
--quiet
)
[[ "$SKIP_ADD" == "true" ]] && CMD+=( --skip-add )
[[ ${#EXTRA_ARGS[@]} -gt 0 ]] && CMD+=( "${EXTRA_ARGS[@]}" )
# Final no-op: the trailing [[ ]] above can be false (e.g. no extra
# args), which would make the function's exit status non-zero and
# trip `set -e` in the caller. Explicit success keeps the contract.
return 0
}
FAILED=()
if [[ "$CONCURRENCY" -le 1 ]]; then
# ── Serial path (legacy behaviour) ──────────────────────────────────
for CI in $INDICES; do
echo "═════════════════════════════════════════════════════════════════"
echo " conv $CI → $OUTPUT_ROOT/conv${CI}.json"
echo "═════════════════════════════════════════════════════════════════"
build_cmd "$CI"
set +e
env "${CMD[@]}"
rc=$?
set -e
if [[ $rc -ne 0 ]]; then
FAILED+=("conv${CI}")
fi
done
else
# ── Parallel path: job pool of $CONCURRENCY workers ─────────────────
#
# Each conv runs in its own python process, streaming to a per-conv
# log file (conv<i>.log) so interleaved stdout doesn't turn into
# confetti. Status is collected via `wait $pid`; one conv's failure
# does not abort the rest.
echo "─────────────────────────────────────────────────────────────────"
echo " Parallel mode: up to $CONCURRENCY convs concurrent"
echo " Per-conv logs: $OUTPUT_ROOT/conv<i>.log"
echo "─────────────────────────────────────────────────────────────────"
# Parallel arrays (no associative arrays in bash 3.2).
RUN_PIDS=()
RUN_CIS=()
# Wait for *any* worker to exit, reap it, prune the slot, record
# failures. Polls because `wait -n` is bash 4.3+.
reap_one() {
while true; do
local idx
for idx in "${!RUN_PIDS[@]}"; do
local pid="${RUN_PIDS[$idx]}"
if ! kill -0 "$pid" 2>/dev/null; then
set +e
wait "$pid"
local rc=$?
set -e
local ci="${RUN_CIS[$idx]}"
if [[ $rc -eq 0 ]]; then
echo " ✓ conv${ci} done (pid $pid)"
else
echo " ✗ conv${ci} failed (pid $pid, status $rc) — see $OUTPUT_ROOT/conv${ci}.log"
FAILED+=("conv${ci}")
fi
unset 'RUN_PIDS[idx]'
unset 'RUN_CIS[idx]'
# Re-pack arrays so ${#RUN_PIDS[@]} stays accurate.
RUN_PIDS=("${RUN_PIDS[@]}")
RUN_CIS=("${RUN_CIS[@]}")
return 0
fi
done
sleep 2
done
}
for CI in $INDICES; do
build_cmd "$CI"
LOG="$OUTPUT_ROOT/conv${CI}.log"
echo " → launching conv${CI} (log: $LOG)"
env "${CMD[@]}" > "$LOG" 2>&1 &
pid=$!
RUN_PIDS+=("$pid")
RUN_CIS+=("$CI")
if [[ ${#RUN_PIDS[@]} -ge $CONCURRENCY ]]; then
reap_one
fi
done
# Drain the remaining workers.
while [[ ${#RUN_PIDS[@]} -gt 0 ]]; do
reap_one
done
fi
if [[ ${#FAILED[@]} -gt 0 ]]; then
echo
echo "⚠ ${#FAILED[@]} conv(s) failed: ${FAILED[*]}"
fi
# ── Aggregate summary ─────────────────────────────────────────────────
echo
echo "═════════════════════════════════════════════════════════════════"
echo " Aggregate accuracy"
echo "═════════════════════════════════════════════════════════════════"
python - <<EOF
import json
from pathlib import Path
root = Path("$OUTPUT_ROOT")
files = sorted(root.glob("conv*.json"))
if not files:
print(" (no result files found)")
raise SystemExit
# header
methods_seen = []
for p in files:
d = json.load(open(p))
for m in d["methods"]:
if m not in methods_seen:
methods_seen.append(m)
w = max(20, max(len(p.stem) + 4 for p in files))
header = f"{'conversation':<{w}} " + " ".join(f"{m:>10}" for m in methods_seen)
print(header)
print("─" * len(header))
for p in files:
d = json.load(open(p))
label = p.stem
cells = []
for m in methods_seen:
mr = d["methods"].get(m)
if mr is None:
cells.append(f"{'—':>10}")
else:
raw = mr["summary"]["accuracy"]
acc = float(str(raw).rstrip("%")) if isinstance(raw, str) else float(raw) * 100
cells.append(f"{acc:>9.1f}%")
print(f"{label:<{w}} " + " ".join(cells))
print()
print(f" detailed JSONs: {root}/conv*.json")
print(f" phase checkpoints: {root}/conv*_checkpoints/")
EOF
tests/run_locomo_full.sh
-156
View File
@@ -1,156 +0,0 @@
#!/usr/bin/env bash
# ──────────────────────────────────────────────────────────────────────
# Full LoCoMo Benchmark Runner
#
# Usage:
# bash tests/run_locomo_full.sh # default: 5 runs × hybrid,agentic
# bash tests/run_locomo_full.sh --runs 3 # 3 runs
# bash tests/run_locomo_full.sh --methods hybrid # hybrid only
# bash tests/run_locomo_full.sh --skip-add # reuse existing data
# ──────────────────────────────────────────────────────────────────────
set -euo pipefail
RUNS="${RUNS:-5}"
METHODS="${METHODS:-hybrid,agentic}"
SKIP_ADD=""
DATA_PATH="data/locomo10.json"
CONVS=10
POST_FLUSH_WAIT=180
JUDGE_MODEL="gpt-4o-mini"
JUDGE_RUNS=5
TOP_K=10
OUTPUT_DIR="benchmark_results"
SEARCH_CONCURRENCY=1
# Parse args
while [[ $# -gt 0 ]]; do
case "$1" in
--runs) RUNS="$2"; shift 2 ;;
--methods) METHODS="$2"; shift 2 ;;
--skip-add) SKIP_ADD="--skip-add"; shift ;;
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
--post-flush-wait) POST_FLUSH_WAIT="$2"; shift 2 ;;
*) echo "Unknown arg: $1"; exit 1 ;;
esac
done
TS=$(date +%Y%m%d_%H%M%S)
RUN_DIR="${OUTPUT_DIR}/run_${TS}"
mkdir -p "$RUN_DIR"
echo "════════════════════════════════════════════════════════════════"
echo " LoCoMo Full Benchmark"
echo " Runs: $RUNS | Methods: $METHODS | Convs: $CONVS"
echo " Judge: $JUDGE_MODEL (${JUDGE_RUNS} runs/question)"
echo " Output: $RUN_DIR"
echo "════════════════════════════════════════════════════════════════"
# Phase 1: Add all 10 conversations (once)
if [[ -z "$SKIP_ADD" ]]; then
echo ""
echo "──── Phase 1: Loading all $CONVS conversations ────"
for conv in $(seq 0 $((CONVS - 1))); do
echo " Loading conv $conv..."
uv run python tests/test_locomo.py \
--conv-index "$conv" \
--methods hybrid \
--data-path "$DATA_PATH" \
--post-flush-wait "$POST_FLUSH_WAIT" \
--judge-model "$JUDGE_MODEL" \
--judge-runs 1 \
--top-k 1 \
--quiet \
--search-concurrency 1 \
--checkpoint-dir "$RUN_DIR/load_conv${conv}" \
2>&1 | tail -5
echo " conv $conv loaded."
done
echo " All conversations loaded."
SKIP_ADD="--skip-add"
fi
# Phase 2: Run benchmark (search + answer + judge)
IFS=',' read -ra METHOD_LIST <<< "$METHODS"
for run_idx in $(seq 1 "$RUNS"); do
for method in "${METHOD_LIST[@]}"; do
echo ""
echo "══════════════════════════════════════════════════════════"
echo " Run $run_idx/$RUNS — method=$method"
echo "══════════════════════════════════════════════════════════"
run_out="$RUN_DIR/${method}_run${run_idx}"
mkdir -p "$run_out"
summary_file="$run_out/summary.json"
all_correct=0
all_total=0
for conv in $(seq 0 $((CONVS - 1))); do
conv_out="$run_out/conv${conv}"
result_file="$conv_out/${method}_results.json"
echo " conv $conv ($method, run $run_idx)..."
uv run python tests/test_locomo.py \
--conv-index "$conv" \
--methods "$method" \
--data-path "$DATA_PATH" \
--skip-add \
--judge-model "$JUDGE_MODEL" \
--judge-runs "$JUDGE_RUNS" \
--top-k "$TOP_K" \
--quiet \
--search-concurrency "$SEARCH_CONCURRENCY" \
--checkpoint-dir "$conv_out" \
--output "$result_file" \
2>&1 | grep -E "Overall:|Done:" | head -3
# Extract accuracy from result JSON
if [[ -f "$result_file" ]]; then
conv_correct=$(python3 -c "
import json, sys
d = json.load(open('$result_file'))
s = d.get('methods', {}).get('$method', {}).get('summary', {})
print(s.get('correct', 0))
" 2>/dev/null || echo 0)
conv_total=$(python3 -c "
import json, sys
d = json.load(open('$result_file'))
s = d.get('methods', {}).get('$method', {}).get('summary', {})
print(s.get('total', 0))
" 2>/dev/null || echo 0)
all_correct=$((all_correct + conv_correct))
all_total=$((all_total + conv_total))
fi
done
# Write run summary
if [[ $all_total -gt 0 ]]; then
accuracy=$(python3 -c "print(f'{$all_correct / $all_total * 100:.1f}')")
else
accuracy="0.0"
fi
echo "{\"method\": \"$method\", \"run\": $run_idx, \"correct\": $all_correct, \"total\": $all_total, \"accuracy\": $accuracy}" > "$summary_file"
echo " ── Run $run_idx $method: $all_correct / $all_total ($accuracy%) ──"
done
done
# Final summary
echo ""
echo "════════════════════════════════════════════════════════════════"
echo " Final Results"
echo "════════════════════════════════════════════════════════════════"
for method in "${METHOD_LIST[@]}"; do
echo " $method:"
for run_idx in $(seq 1 "$RUNS"); do
summary="$RUN_DIR/${method}_run${run_idx}/summary.json"
if [[ -f "$summary" ]]; then
python3 -c "
import json
d = json.load(open('$summary'))
print(f\" Run {d['run']}: {d['correct']}/{d['total']} ({d['accuracy']}%)\")
"
fi
done
done
echo "════════════════════════════════════════════════════════════════"
tests/test_locomo.py
-1714
View File
File diff suppressed because it is too large Load Diff
tests/unit/test_benchmark_config.py
+163
View File
@@ -0,0 +1,163 @@
"""Unit tests for BenchmarkConfig loading."""
from __future__ import annotations
from pathlib import Path
import pytest
def test_defaults_construct_without_error() -> None:
"""BenchmarkConfig() with defaults must construct and have all fields."""
from benchmarks.config import BenchmarkConfig
cfg = BenchmarkConfig()
assert isinstance(cfg.methods, str)
assert cfg.top_k > 0
assert cfg.cascade_timeout > 0
assert cfg.batch_size > 0
assert isinstance(cfg.answer_model, str)
assert isinstance(cfg.judge_model, str)
assert cfg.judge_runs >= 1
assert cfg.conversations_concurrency >= 1
assert cfg.eval_concurrency >= 1
assert cfg.search_concurrency >= 1
def test_from_toml_loads_default_config() -> None:
"""from_toml('config') loads benchmarks/config.toml successfully."""
from benchmarks.config import BenchmarkConfig
cfg = BenchmarkConfig.from_toml("config")
assert isinstance(cfg.methods, str)
assert isinstance(cfg.judge_model, str)
def test_from_toml_override(tmp_path: Path) -> None:
"""A custom TOML overrides specific fields, rest stay default."""
from benchmarks.config import BenchmarkConfig
(tmp_path / "custom.toml").write_text(
'methods = "hybrid"\ntop_k = 20\n', encoding="utf-8"
)
cfg = BenchmarkConfig.from_toml("custom", config_dir=tmp_path)
assert cfg.methods == "hybrid"
assert cfg.top_k == 20
assert cfg.judge_model == "gpt-4o-mini" # untouched
def test_from_toml_missing_file_raises() -> None:
"""from_toml with nonexistent name raises FileNotFoundError."""
from benchmarks.config import BenchmarkConfig
with pytest.raises(FileNotFoundError):
BenchmarkConfig.from_toml("nonexistent")
def test_frozen_model() -> None:
"""BenchmarkConfig instances are immutable."""
from benchmarks.config import BenchmarkConfig
cfg = BenchmarkConfig()
with pytest.raises(Exception): # noqa: B017 — ValidationError
cfg.top_k = 999
def test_concurrency_fields_exist() -> None:
"""Concurrency fields exist and are positive; no legacy 'concurrency' attr."""
from benchmarks.config import BenchmarkConfig
cfg = BenchmarkConfig()
assert cfg.conversations_concurrency >= 1
assert cfg.eval_concurrency >= 1
assert cfg.search_concurrency >= 1
assert not hasattr(cfg, "concurrency")
def test_search_result_frozen() -> None:
from benchmarks.config import SearchResult
sr = SearchResult(
index=0,
question="q",
golden_answer="a",
category=1,
evidence=[],
episodes=[],
profiles=[],
search_time_s=0.5,
method="agentic",
)
assert sr.index == 0
with pytest.raises(Exception): # noqa: B017
sr.index = 99
def test_search_result_jsonl_roundtrip() -> None:
from benchmarks.config import SearchResult
sr = SearchResult(
index=0,
question="q",
golden_answer="a",
category=1,
evidence=["e1"],
episodes=[{"id": "ep1"}],
profiles=[],
search_time_s=0.5,
method="agentic",
)
line = sr.model_dump_json()
restored = SearchResult.model_validate_json(line)
assert restored == sr
def test_answer_result_tracks_tokens() -> None:
from benchmarks.config import AnswerResult
ar = AnswerResult(
index=0,
question="q",
golden_answer="a",
category=1,
generated_answer="ans",
answer_time_s=1.0,
answer_attempts=1,
answer_tokens=500,
)
assert ar.answer_tokens == 500
def test_judge_result_jsonl_roundtrip() -> None:
from benchmarks.config import JudgeResult
jr = JudgeResult(
index=0,
question="q",
golden_answer="a",
generated_answer="ans",
category=1,
is_correct=True,
judgments=[True, True, False],
judge_tokens=200,
)
line = jr.model_dump_json()
restored = JudgeResult.model_validate_json(line)
assert restored == jr
assert restored.is_correct is True
def test_run_spec_includes_git_hash() -> None:
from benchmarks.config import RunSpec
spec = RunSpec(
run_name="v1",
config={},
conversations=[0, 1],
stages=["add", "search"],
git_hash="abc1234",
python_version="3.12.11",
everos_version="1.1.0",
started_at="2026-06-28T22:30:00Z",
)
assert spec.git_hash == "abc1234"
tests/unit/test_component/test_embedding/test_factory.py
-6
View File
@@ -24,12 +24,6 @@ def test_raises_when_api_key_missing() -> None:
build_embedding_provider(s)
def test_raises_when_api_key_empty() -> None:
s = EmbeddingSettings(model="m", api_key=SecretStr(""), base_url="https://x")
with pytest.raises(ValueError, match="EVEROS_EMBEDDING__API_KEY"):
build_embedding_provider(s)
def test_raises_when_base_url_missing() -> None:
s = EmbeddingSettings(model="m", api_key=SecretStr("k"), base_url=None)
with pytest.raises(ValueError, match="EVEROS_EMBEDDING__BASE_URL"):
tests/unit/test_component/test_llm/test_factory.py
-6
View File
@@ -16,12 +16,6 @@ def test_raises_when_api_key_missing() -> None:
build_llm_provider(s)
def test_raises_when_api_key_empty() -> None:
s = LLMSettings(model="m", api_key=SecretStr(""), base_url="https://x")
with pytest.raises(ValueError, match="EVEROS_LLM__API_KEY"):
build_llm_provider(s)
def test_raises_when_base_url_missing() -> None:
s = LLMSettings(model="m", api_key=SecretStr("k"), base_url=None)
with pytest.raises(ValueError, match="EVEROS_LLM__BASE_URL"):
tests/unit/test_component/test_rerank/test_factory.py
-89
View File
@@ -6,7 +6,6 @@ import pytest
from pydantic import SecretStr
from everos.component.rerank import (
DashScopeRerankProvider,
DeepInfraRerankProvider,
VllmRerankProvider,
build_rerank_provider,
@@ -34,17 +33,6 @@ def test_deepinfra_requires_api_key() -> None:
build_rerank_provider(s)
def test_deepinfra_rejects_empty_api_key() -> None:
s = RerankSettings(
provider="deepinfra",
model="m",
api_key=SecretStr(""),
base_url="https://api.deepinfra.com/v1/inference",
)
with pytest.raises(ValueError, match="EVEROS_RERANK__API_KEY"):
build_rerank_provider(s)
def test_deepinfra_builds_provider() -> None:
s = RerankSettings(
provider="deepinfra",
@@ -77,80 +65,3 @@ def test_vllm_with_api_key() -> None:
)
p = build_rerank_provider(s)
assert isinstance(p, VllmRerankProvider)
def test_dashscope_requires_api_key() -> None:
s = RerankSettings(
provider="dashscope",
model="gte-rerank-v2",
api_key=None,
base_url="https://dashscope.aliyuncs.com",
)
with pytest.raises(ValueError, match="EVEROS_RERANK__API_KEY"):
build_rerank_provider(s)
def test_dashscope_builds_provider() -> None:
s = RerankSettings(
provider="dashscope",
model="gte-rerank-v2",
api_key=SecretStr("k"),
base_url="https://dashscope.aliyuncs.com",
)
p = build_rerank_provider(s)
assert isinstance(p, DashScopeRerankProvider)
def test_dashscope_supports_gte_rerank_v2_only() -> None:
s = RerankSettings(
provider="dashscope",
model="qwen3-rerank",
api_key=SecretStr("k"),
base_url="https://dashscope.aliyuncs.com",
)
with pytest.raises(ValueError, match="gte-rerank-v2 only"):
build_rerank_provider(s)
def test_infers_dashscope_from_base_url() -> None:
"""provider unset + DashScope host -> DashScopeRerankProvider."""
s = RerankSettings(
provider=None,
model="gte-rerank-v2",
api_key=SecretStr("k"),
base_url="https://dashscope.aliyuncs.com",
)
assert isinstance(build_rerank_provider(s), DashScopeRerankProvider)
def test_infers_deepinfra_from_base_url() -> None:
"""provider unset + DeepInfra host -> DeepInfraRerankProvider."""
s = RerankSettings(
provider=None,
model="m",
api_key=SecretStr("k"),
base_url="https://api.deepinfra.com/v1/inference",
)
assert isinstance(build_rerank_provider(s), DeepInfraRerankProvider)
def test_unknown_host_falls_back_to_deepinfra() -> None:
"""provider unset + unrecognized host -> historical deepinfra default."""
s = RerankSettings(
provider=None,
model="m",
api_key=SecretStr("k"),
base_url="https://rerank.internal.example/v1",
)
assert isinstance(build_rerank_provider(s), DeepInfraRerankProvider)
def test_explicit_provider_overrides_inference() -> None:
"""Explicit provider wins even when the host hints another."""
s = RerankSettings(
provider="vllm",
model="m",
api_key=SecretStr("k"),
base_url="https://dashscope.aliyuncs.com",
)
assert isinstance(build_rerank_provider(s), VllmRerankProvider)
tests/unit/test_config/test_settings.py
+12 -37
View File
@@ -13,10 +13,12 @@ from everos.config.settings import resolve_root
@pytest.fixture(autouse=True)
def _isolate_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Strip EVEROS_* env vars and move CWD away from any config file."""
"""Strip EVEROS_* env vars and pin root to tmp_path so no external
everos.toml is ever discovered."""
for key in list(os.environ):
if key.startswith("EVEROS_"):
monkeypatch.delenv(key, raising=False)
monkeypatch.setenv("EVEROS_ROOT", str(tmp_path))
monkeypatch.chdir(tmp_path)
load_settings.cache_clear()
@@ -107,47 +109,20 @@ def test_load_settings_is_cached() -> None:
def test_embedding_rerank_defaults() -> None:
s = Settings()
assert s.embedding.model is None
assert s.embedding.api_key is None
assert s.embedding.model == "Qwen/Qwen3-Embedding-4B"
assert s.embedding.base_url == "https://api.deepinfra.com/v1/openai"
assert s.embedding.api_key.get_secret_value() == ""
assert s.embedding.timeout_seconds == 30.0
assert s.rerank.model is None
assert s.rerank.provider is None
assert s.rerank.model == "Qwen/Qwen3-Reranker-4B"
assert s.rerank.base_url == "https://api.deepinfra.com/v1/inference"
assert s.rerank.api_key.get_secret_value() == ""
assert s.rerank.timeout_seconds == 30.0
assert s.llm.api_key.get_secret_value() == ""
def test_dashscope_one_key_can_configure_llm_embedding_and_rerank(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""One DashScope key value can be reused across all three clients."""
key = "sk-dashscope"
compatible_base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1"
monkeypatch.setenv("EVEROS_LLM__MODEL", "qwen-plus")
monkeypatch.setenv("EVEROS_LLM__API_KEY", key)
monkeypatch.setenv("EVEROS_LLM__BASE_URL", compatible_base_url)
monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "text-embedding-v4")
monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", key)
monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", compatible_base_url)
monkeypatch.setenv("EVEROS_RERANK__MODEL", "gte-rerank-v2")
monkeypatch.setenv("EVEROS_RERANK__API_KEY", key)
monkeypatch.setenv("EVEROS_RERANK__BASE_URL", "https://dashscope.aliyuncs.com")
s = Settings()
assert s.llm.api_key is not None
assert s.embedding.api_key is not None
assert s.rerank.api_key is not None
assert s.llm.api_key.get_secret_value() == key
assert s.embedding.api_key.get_secret_value() == key
assert s.rerank.api_key.get_secret_value() == key
assert s.llm.base_url == compatible_base_url
assert s.embedding.base_url == compatible_base_url
assert s.rerank.provider is None
assert s.rerank.base_url == "https://dashscope.aliyuncs.com"
def test_resolve_root_default() -> None:
def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None:
"""No --root, no EVEROS_ROOT → ~/.everos."""
monkeypatch.delenv("EVEROS_ROOT", raising=False)
assert resolve_root() == Path("~/.everos").expanduser().resolve()
tests/unit/test_core/test_middleware/test_profile.py
+2 -3
View File
@@ -107,7 +107,7 @@ async def test_enabled_with_query_returns_html_when_inner_raises(
pytest.skip("pyinstrument not installed in this env")
# Rebuild a tiny app whose route raises so the middleware's except branch
# fires; the profile HTML is still emitted regardless.
# fires; the middleware re-raises so the error propagates normally.
app = FastAPI()
app.add_middleware(ProfileMiddleware)
@@ -120,8 +120,7 @@ async def test_enabled_with_query_returns_html_when_inner_raises(
base_url="http://test",
) as c:
resp = await c.get("/bang?profile=true")
assert resp.status_code == 200
assert "text/html" in resp.headers.get("content-type", "")
assert resp.status_code == 500
async def test_enabled_without_pyinstrument(monkeypatch: pytest.MonkeyPatch) -> None:
tests/unit/test_entrypoints/test_api/test_lifespans/test_ome.py
+2 -2
View File
@@ -29,7 +29,7 @@ async def test_lifespan_starts_and_stops_engine(
engine = await provider.startup(app)
assert engine is not None
assert engine._started is True # noqa: SLF001 — test introspection
assert engine._started is True
await provider.shutdown(app)
assert engine._started is False # noqa: SLF001
assert engine._started is False
tests/unit/test_entrypoints/test_cli/test_main.py
-1
View File
@@ -13,7 +13,6 @@ def test_help_exits_zero() -> None:
assert "everos" in result.stdout
assert "server" in result.stdout
assert "cascade" in result.stdout
assert "demo" in result.stdout
assert "config" in result.stdout
tests/unit/test_infra/test_lancedb/test_tables/test_content_sha256.py
+1 -1
View File
@@ -146,5 +146,5 @@ def test_content_sha256_required(factory) -> None: # type: ignore[no-untyped-de
row = factory()
kwargs = row.model_dump()
del kwargs["content_sha256"]
with pytest.raises(Exception): # noqa: B017,PT011
with pytest.raises(Exception): # noqa: B017
type(row).model_validate(kwargs)
tests/unit/test_infra/test_markdown/test_writers/test_base.py
+3 -3
View File
@@ -130,7 +130,7 @@ async def test_current_count_hook_can_be_overridden(root: MemoryRoot) -> None:
class _ConstantCount(BaseDailyWriter):
schema = _UserDemoFrontmatter
async def _current_count(self, path): # noqa: ANN001
async def _current_count(self, path):
return 41 # always claim 41 existing entries
writer = _ConstantCount(root)
@@ -144,7 +144,7 @@ async def test_frontmatter_updates_hook_supplies_defaults(root: MemoryRoot) -> N
class _WithDefaults(BaseDailyWriter):
schema = _UserDemoFrontmatter
def _frontmatter_updates(self, scope_id, date, *, next_count): # noqa: ANN001
def _frontmatter_updates(self, scope_id, date, *, next_count):
return {
"user_id": scope_id,
"entry_count": next_count,
@@ -167,7 +167,7 @@ async def test_explicit_frontmatter_updates_skip_hook(root: MemoryRoot) -> None:
class _WithDefaults(BaseDailyWriter):
schema = _UserDemoFrontmatter
def _frontmatter_updates(self, scope_id, date, *, next_count): # noqa: ANN001
def _frontmatter_updates(self, scope_id, date, *, next_count):
return {"marker": "from-hook"}
writer = _WithDefaults(root)
tests/unit/test_infra/test_ome/test_e2e.py
+1 -1
View File
@@ -178,7 +178,7 @@ async def test_cron_strategy_executes_when_cron_entry_fires() -> None:
h.register(cron_job)
await h.start()
# Directly invoke what APS would call; bypass scheduler timing.
await _cron_entry(h._engine._engine_id, "cron_e2e") # noqa: SLF001
await _cron_entry(h._engine._engine_id, "cron_e2e")
await h.drain(timeout=5)
runs = await h.list_runs("cron_e2e")
tests/unit/test_infra/test_ome/test_records.py
+1 -1
View File
@@ -104,7 +104,7 @@ def test_run_record_rejects_negative_int(field: str) -> None:
def test_run_record_rejects_naive_started_at() -> None:
naive = datetime(2026, 5, 12, 12, 0, 0) # noqa: DTZ001 — deliberately naive
naive = datetime(2026, 5, 12, 12, 0, 0)
with pytest.raises(ValidationError, match="started_at"):
RunRecord(**_ok_kwargs(started_at=naive))
tests/unit/test_memory/test_cascade/test_handler_episode.py
+39 -2
View File
@@ -153,7 +153,8 @@ async def test_added_entry_upserts_typed_row(
assert outcome.upserted == 1
assert outcome.deleted == 0
assert outcome.skipped == 0
assert embedder.calls == 1
# Subject "Test" is present → two embed calls (content + subject).
assert embedder.calls == 2
assert len(fake_repo.upserts) == 1
row = fake_repo.upserts[0][0]
assert row.owner_id == "u1"
@@ -165,12 +166,15 @@ async def test_added_entry_upserts_typed_row(
assert row.parent_id == "mc_test_parent"
assert row.parent_type == "memcell"
assert row.episode == "hello world"
assert row.episode_tokens == "hello world"
# episode_tokens includes subject keywords appended after content tokens.
assert row.episode_tokens == "hello world Test"
assert row.subject == "Test"
assert row.md_path == rel
assert row.entry_id.startswith("ep_")
assert row.id == f"u1_{row.entry_id}"
assert len(row.vector) == 1024
assert row.subject_vector is not None
assert len(row.subject_vector) == 1024
async def test_unchanged_entry_is_skipped_no_embed_call(
@@ -211,7 +215,40 @@ async def test_modified_entry_reembeds(
outcome = await handler.handle_added_or_modified(rel)
assert outcome.upserted == 1
assert outcome.skipped == 0
# Subject "Test" is present → two embed calls (content + subject).
assert embedder.calls == 2
async def test_no_subject_skips_subject_embed(
memory_root: MemoryRoot, fake_repo: _FakeEpisodeRepo
) -> None:
"""When Subject is absent, subject_vector is None; only one embed call is made."""
today = _dt.date(2026, 5, 14)
writer = EpisodeWriter(memory_root)
await writer.append_entry(
"u2",
inline={
"owner_id": "u2",
"session_id": "s2",
"timestamp": "2026-05-14T10:00:00+00:00",
"parent_type": "memcell",
"parent_id": "mc_no_subject",
"sender_ids": ["u2"],
},
sections={"Content": "content only"},
date=today,
)
rel = "default_app/default_project/users/u2/episodes/episode-2026-05-14.md"
handler, embedder = _build_handler(memory_root)
outcome = await handler.handle_added_or_modified(rel)
assert outcome.upserted == 1
# No Subject → single embed call for content only.
assert embedder.calls == 1
row = fake_repo.upserts[0][0]
assert row.subject_vector is None
assert row.episode_tokens == "content only"
# ── deletion paths ───────────────────────────────────────────────────────
tests/unit/test_memory/test_cascade/test_worker.py
+1 -1
View File
@@ -72,7 +72,7 @@ class _FakeRepo:
class _OkHandler(Handler):
def __init__(self) -> None: # noqa: D401 — no deps needed
def __init__(self) -> None:
pass
async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome:
tests/unit/test_memory/test_extract/test_parser/test_enrich.py
+5 -5
View File
@@ -9,12 +9,12 @@ import pytest
pytest.importorskip("everalgo.parser")
from everalgo.llm import LLMError # noqa: E402
from everalgo.types import ParsedContent # noqa: E402
from everalgo.llm import LLMError
from everalgo.types import ParsedContent
from everos.component import parser as _parser_mod # noqa: E402
from everos.core.errors import UnsupportedModalityError # noqa: E402
from everos.memory.extract.parser import enrich_content_items # noqa: E402
from everos.component import parser as _parser_mod
from everos.core.errors import UnsupportedModalityError
from everos.memory.extract.parser import enrich_content_items
_APARSE_FILE_TARGET = "everos.component.parser.aparse_file"
tests/unit/test_memory/test_extract/test_pipeline/test_user_memory_emits.py
+2 -2
View File
@@ -46,7 +46,7 @@ async def test_emit_pipeline_started_routes_through_engine() -> None:
)
cell = _sample_memcell()
await pipeline._emit_pipeline_started( # noqa: SLF001 — test introspection
await pipeline._emit_pipeline_started(
memcell_id="mc_a",
session_id="s1",
app_id="claude_code",
@@ -101,7 +101,7 @@ async def test_emit_episode_extracted_after_md_write() -> None:
algo_ep = AlgoEpisode(
owner_id="u1", episode="they said hello", timestamp=1_700_000_000_000
)
with patch.object( # noqa: SLF001
with patch.object(
pipeline._ep_ext, "aextract", new=AsyncMock(return_value=algo_ep)
):
outcome = await pipeline.run(
tests/unit/test_memory/test_search/conftest.py
+1 -25
View File
@@ -1,27 +1,3 @@
"""Shared fixtures for ``memory.search`` unit tests.
The project default is ``EVEROS_SEARCH__VECTOR_STRATEGY=maxsim_atomic`` —
that path queries both the ``atomic_fact`` table and the ``episode`` table
to do MaxSim. The existing VECTOR-route tests in ``test_manager.py`` were
written against the legacy single-vector ``episode`` path and stub only the
episode recaller (atomic_fact recaller is a no-data stub).
Force the legacy ``episode`` strategy by default for these tests so they
keep asserting against the dense-recall path they were designed to cover.
MaxSim-specific tests opt back into ``maxsim_atomic`` by overriding the env
var inside their own body.
"""
"""Shared fixtures for ``memory.search`` unit tests."""
from __future__ import annotations
import pytest
from everos.config.settings import load_settings
@pytest.fixture(autouse=True)
def _force_episode_vector_strategy(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("EVEROS_SEARCH__VECTOR_STRATEGY", "episode")
load_settings.cache_clear()
yield
load_settings.cache_clear()
tests/unit/test_memory/test_search/test_agentic.py
+25 -1
View File
@@ -71,12 +71,25 @@ class _StubEpisodeRecaller:
async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]:
return list(self._all_docs)
async def sparse_recall_as_child(self, *_: Any, **__: Any) -> list[Candidate]:
return []
async def dense_recall_subject_as_child(
self, *_: Any, **__: Any
) -> list[Candidate]:
return []
async def fetch_by_parent_ids(
self, parent_ids: Sequence[str], where: str
) -> list[Candidate]:
"""Returns Candidate with id=episode_id (real LanceDB id)."""
return [self._by_parent[p] for p in parent_ids if p in self._by_parent]
async def fetch_by_entry_ids(
self, entry_ids: Sequence[str], where: str
) -> list[Candidate]:
return [self._by_parent[e] for e in entry_ids if e in self._by_parent]
async def fetch_all_for_owner(self, where: str) -> list[Candidate]:
"""Returns Candidate with id=memcell_id and metadata['episode_id']."""
return list(self._all_docs)
@@ -213,6 +226,8 @@ async def test_agentic_search_wires_benchmark_hyperparams(
"What did Alice eat?",
owner_id="alice",
where="owner_id = 'alice' AND owner_type = 'user'",
app_id="test_app",
project_id="test_proj",
episode_recaller=ep_recaller,
atomic_fact_recaller=fact_recaller,
embed_query_fn=fake_embed,
@@ -252,6 +267,8 @@ async def test_agentic_search_loads_user_memory_clusters(
"q",
owner_id="alice",
where="owner_id = 'alice' AND owner_type = 'user'",
app_id="bench_app",
project_id="bench_proj",
episode_recaller=ep_recaller,
atomic_fact_recaller=fact_recaller,
embed_query_fn=fake_embed,
@@ -260,7 +277,12 @@ async def test_agentic_search_loads_user_memory_clusters(
top_k=10,
)
mock_list.assert_called_once_with("alice", "user_memory")
mock_list.assert_called_once_with(
"alice",
"user_memory",
app_id="bench_app",
project_id="bench_proj",
)
async def test_agentic_search_shapes_candidates_with_episode_id(
@@ -290,6 +312,8 @@ async def test_agentic_search_shapes_candidates_with_episode_id(
"What did Alice eat?",
owner_id="alice",
where="owner_id = 'alice' AND owner_type = 'user'",
app_id="test_app",
project_id="test_proj",
episode_recaller=ep_recaller,
atomic_fact_recaller=fact_recaller,
embed_query_fn=fake_embed,
tests/unit/test_memory/test_search/test_hierarchy.py
+120 -431
View File
@@ -1,49 +1,34 @@
"""Unit tests for ``memory.search.hierarchy``.
White-box surfaces accessed:
- ``_hierarchy_eviction_pass`` (internal, tested directly for unit coverage)
- ``hierarchy_retrieve_episodes`` (public function, tested with stubbed I/O)
Layer 4 uses hierarchical fact eviction: parent episode and its facts are
calibrated to an LR probability via ``cosine_to_lr_score`` and compete on that
single scale, so the expected scores below are computed with the same helper
rather than hard-coded — the assertions track the calibration, not magic
numbers.
All I/O (fact_recaller, episode_recaller) is injected via AsyncMock stubs.
No LanceDB or network calls are made.
"""
"""Unit tests for ``memory.search.hierarchy``."""
from __future__ import annotations
import datetime as _dt
from unittest.mock import AsyncMock, MagicMock
import pytest
from everalgo.rank.fusion import cosine_to_lr_score
from everalgo.types import Candidate, FactCandidate
from everos.memory.search.hierarchy import (
_build_ep_to_fact_parents,
_hierarchy_eviction_pass,
hierarchy_retrieve_episodes,
build_ep_to_fact_parents,
heap_expand,
)
# ── Fixtures / helpers ───────────────────────────────────────────────────
# ── Helpers ──────────────────────────────────────────────────────────────
def _ts() -> _dt.datetime:
return _dt.datetime(2026, 1, 1, tzinfo=_dt.UTC)
def _episode_candidate(
def _ep(
*,
ep_id: str = "ep-1",
score: float = 0.7,
memcell_id: str = "mc-1",
entry_id: str | None = None,
source: str = "vector",
) -> Candidate:
metadata = {
metadata: dict = {
"parent_id": memcell_id,
"owner_id": "u1",
"owner_type": "user",
@@ -56,15 +41,10 @@ def _episode_candidate(
}
if entry_id is not None:
metadata["entry_id"] = entry_id
return Candidate(
id=ep_id,
score=score,
source="vector",
metadata=metadata,
)
return Candidate(id=ep_id, score=score, source=source, metadata=metadata)
def _fact_candidate(
def _fact(
*,
fact_id: str = "fact-1",
parent_episode_id: str = "ep-1",
@@ -78,428 +58,137 @@ def _fact_candidate(
)
def _make_recallers(
*,
dense_facts: list[Candidate] | None = None,
fetched_episodes: list[Candidate] | None = None,
facts_for_episodes: dict[str, list[FactCandidate]] | None = None,
) -> tuple[MagicMock, MagicMock]:
"""Build stubbed fact_recaller and episode_recaller."""
fact_recaller = MagicMock()
fact_recaller.dense_recall = AsyncMock(return_value=dense_facts or [])
fact_recaller.facts_for_episodes = AsyncMock(return_value=facts_for_episodes or {})
episode_recaller = MagicMock()
episode_recaller.fetch_by_entry_ids = AsyncMock(return_value=fetched_episodes or [])
return fact_recaller, episode_recaller
# ── _hierarchy_eviction_pass unit tests ─────────────────────────────────
class TestHierarchyEvictionPass:
def test_fact_wins_emits_atomic_fact_at_lr_score(self) -> None:
# Fact cosine (0.9) > parent cosine (0.5) → fact wins; alpha=1.0 so the
# emitted score is the fact's own LR-calibrated value.
episode = _episode_candidate(ep_id="ep-1")
fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.9)
result = _hierarchy_eviction_pass(
[episode],
{"ep-1": [fact]},
ep_cosine={"ep-1": 0.5},
ep_bm25={},
)
assert len(result) == 1
item = result[0]
assert item.item_type == "atomic_fact"
assert item.id == "fact-1"
assert item.score == pytest.approx(cosine_to_lr_score(0.9, 0.0))
def test_episode_wins_emits_episode_at_parent_lr_score(self) -> None:
# Fact cosine (0.6) < parent cosine (0.8) → episode wins at parent_lr.
episode = _episode_candidate(ep_id="ep-1")
fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.6)
result = _hierarchy_eviction_pass(
[episode],
{"ep-1": [fact]},
ep_cosine={"ep-1": 0.8},
ep_bm25={},
)
assert len(result) == 1
item = result[0]
assert item.item_type == "episode"
assert item.id == "ep-1"
assert item.score == pytest.approx(cosine_to_lr_score(0.8, 0.0))
def test_no_facts_emits_episode_at_parent_lr(self) -> None:
episode = _episode_candidate(ep_id="ep-1")
result = _hierarchy_eviction_pass(
[episode],
{},
ep_cosine={"ep-1": 0.7},
ep_bm25={},
)
assert len(result) == 1
assert result[0].item_type == "episode"
assert result[0].id == "ep-1"
assert result[0].score == pytest.approx(cosine_to_lr_score(0.7, 0.0))
def test_ordering_preserved_matches_input_order(self) -> None:
ep_a = _episode_candidate(ep_id="ep-a", memcell_id="mc-a")
ep_b = _episode_candidate(ep_id="ep-b", memcell_id="mc-b")
ep_c = _episode_candidate(ep_id="ep-c", memcell_id="mc-c")
merged = [ep_a, ep_b, ep_c]
result = _hierarchy_eviction_pass(
merged,
{},
ep_cosine={"ep-a": 0.9, "ep-b": 0.8, "ep-c": 0.7},
ep_bm25={},
)
assert [r.id for r in result] == ["ep-a", "ep-b", "ep-c"]
def test_parent_episode_id_set_on_evicted_fact(self) -> None:
episode = _episode_candidate(ep_id="ep-1")
fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.8)
result = _hierarchy_eviction_pass(
[episode],
{"ep-1": [fact]},
ep_cosine={"ep-1": 0.4},
ep_bm25={},
)
assert result[0].parent_episode_id == "ep-1"
def test_episode_wins_parent_episode_id_is_none(self) -> None:
episode = _episode_candidate(ep_id="ep-1")
fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.5)
result = _hierarchy_eviction_pass(
[episode],
{"ep-1": [fact]},
ep_cosine={"ep-1": 0.9},
ep_bm25={},
)
assert result[0].parent_episode_id is None
def test_multiple_episodes_mixed_eviction(self) -> None:
ep1 = _episode_candidate(ep_id="ep-1", memcell_id="mc-1")
ep2 = _episode_candidate(ep_id="ep-2", memcell_id="mc-2")
ep3 = _episode_candidate(ep_id="ep-3", memcell_id="mc-3")
fact1 = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.9)
fact2 = _fact_candidate(fact_id="fact-2", parent_episode_id="ep-2", score=0.4)
result = _hierarchy_eviction_pass(
[ep1, ep2, ep3],
{"ep-1": [fact1], "ep-2": [fact2]},
ep_cosine={"ep-1": 0.5, "ep-2": 0.8, "ep-3": 0.6},
ep_bm25={},
)
assert len(result) == 3
assert result[0].item_type == "atomic_fact" # 0.9 > 0.5
assert result[0].id == "fact-1"
assert result[1].item_type == "episode" # 0.4 < 0.8
assert result[1].id == "ep-2"
assert result[2].item_type == "episode" # no fact
assert result[2].id == "ep-3"
def test_best_fact_across_window_used_for_comparison(self) -> None:
episode = _episode_candidate(ep_id="ep-1")
best_fact = _fact_candidate(
fact_id="fact-best", parent_episode_id="ep-1", score=0.85
)
weak_fact = _fact_candidate(
fact_id="fact-weak", parent_episode_id="ep-1", score=0.3
)
result = _hierarchy_eviction_pass(
[episode],
{"ep-1": [best_fact, weak_fact]},
ep_cosine={"ep-1": 0.7},
ep_bm25={},
)
assert result[0].item_type == "atomic_fact"
assert result[0].id == "fact-best"
def test_fact_equal_to_parent_does_not_evict(self) -> None:
# Blend must strictly beat parent_lr; equal scores keep the episode.
episode = _episode_candidate(ep_id="ep-1")
fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.7)
result = _hierarchy_eviction_pass(
[episode],
{"ep-1": [fact]},
ep_cosine={"ep-1": 0.7},
ep_bm25={},
)
assert result[0].item_type == "episode"
def test_alpha_blends_parent_and_child(self) -> None:
# alpha=0.5 → score = 0.5*child_lr + 0.5*parent_lr.
episode = _episode_candidate(ep_id="ep-1")
fact = _fact_candidate(fact_id="fact-1", parent_episode_id="ep-1", score=0.9)
result = _hierarchy_eviction_pass(
[episode],
{"ep-1": [fact]},
ep_cosine={"ep-1": 0.5},
ep_bm25={},
alpha=0.5,
)
parent_lr = cosine_to_lr_score(0.5, 0.0)
child_lr = cosine_to_lr_score(0.9, 0.0)
expected = 0.5 * child_lr + 0.5 * parent_lr
assert result[0].item_type == "atomic_fact"
assert result[0].score == pytest.approx(expected)
def test_bm25_raises_calibrated_score(self) -> None:
# BM25 is folded into both parent and child calibration (children
# inherit parent_bm25), so it does not change the intra-episode
# parent-vs-fact outcome at alpha=1 — it lifts the absolute LR score.
episode = _episode_candidate(ep_id="ep-1")
without_bm25 = _hierarchy_eviction_pass(
[episode], {}, ep_cosine={"ep-1": 0.5}, ep_bm25={}
)
with_bm25 = _hierarchy_eviction_pass(
[episode], {}, ep_cosine={"ep-1": 0.5}, ep_bm25={"ep-1": 50.0}
)
assert with_bm25[0].score > without_bm25[0].score
assert with_bm25[0].score == pytest.approx(cosine_to_lr_score(0.5, 50.0))
def test_facts_per_episode_window_caps_competition(self) -> None:
# A high-scoring fact beyond the window must not win.
episode = _episode_candidate(ep_id="ep-1")
in_window = _fact_candidate(fact_id="in", parent_episode_id="ep-1", score=0.55)
out_window = _fact_candidate(
fact_id="out", parent_episode_id="ep-1", score=0.99
)
result = _hierarchy_eviction_pass(
[episode],
{"ep-1": [in_window, out_window]},
ep_cosine={"ep-1": 0.6},
ep_bm25={},
facts_per_episode=1,
)
# Only ``in`` (0.55) is in the 1-fact window and it loses to 0.6 →
# episode wins; the 0.99 ``out`` fact is never considered.
assert result[0].item_type == "episode"
# ── _build_ep_to_fact_parents unit tests ────────────────────────────────
# ── build_ep_to_fact_parents ─────────────────────────────────────────────
class TestBuildEpToFactParents:
"""Unit tests for the dual parent_id mapping builder."""
def test_entry_id_and_parent_id_both_collected(self) -> None:
"""Post-1.5 episode with entry_id and parent_id both present."""
ep = _episode_candidate(ep_id="ep-1", memcell_id="mc-1", entry_id="ep_entry_1")
result = _build_ep_to_fact_parents([ep])
assert result == {"ep-1": ["ep_entry_1", "mc-1"]}
def test_memcell_id_only_no_entry_id(self) -> None:
"""Pre-1.5 episode: only parent_id (memcell_id), no entry_id."""
ep = _episode_candidate(ep_id="ep-1", memcell_id="mc-1")
result = _build_ep_to_fact_parents([ep])
def test_entry_id_only(self) -> None:
"""entry_id equal to parent_id → deduped to a single parent."""
ep = _ep(ep_id="ep-1", memcell_id="ent-1", entry_id="ent-1")
result = build_ep_to_fact_parents([ep])
assert result == {"ep-1": ["ent-1"]}
def test_parent_id_only(self) -> None:
ep = _ep(ep_id="ep-1", memcell_id="mc-1")
result = build_ep_to_fact_parents([ep])
assert result == {"ep-1": ["mc-1"]}
def test_entry_id_equals_parent_id_no_duplicate(self) -> None:
"""When entry_id == parent_id, only one value in the list."""
ep = _episode_candidate(ep_id="ep-1", memcell_id="same_id", entry_id="same_id")
def test_both_entry_and_parent(self) -> None:
ep = _ep(ep_id="ep-1", memcell_id="mc-1", entry_id="ent-1")
result = build_ep_to_fact_parents([ep])
assert result == {"ep-1": ["ent-1", "mc-1"]}
result = _build_ep_to_fact_parents([ep])
def test_empty_list(self) -> None:
assert build_ep_to_fact_parents([]) == {}
assert result == {"ep-1": ["same_id"]}
def test_no_ids_in_metadata(self) -> None:
ep = Candidate(id="ep-1", score=0.5, metadata={})
assert build_ep_to_fact_parents([ep]) == {}
def test_missing_parent_id_skipped(self) -> None:
"""Episode with no parent_id and no entry_id is excluded."""
ep = Candidate(
id="ep-orphan",
score=0.5,
source="vector",
metadata={"owner_id": "u1"},
# ── heap_expand ──────────────────────────────────────────────────────────
class TestHeapExpand:
def test_empty_inputs_returns_empty(self) -> None:
assert heap_expand(sparse=[], dense=[], episode_to_facts={}) == []
def test_episodes_only_no_facts_sorted_by_lr(self) -> None:
"""No facts → all episodes survive, sorted by LR score descending."""
sparse = [_ep(ep_id="ep-a", score=5.0), _ep(ep_id="ep-b", score=3.0)]
dense = [_ep(ep_id="ep-a", score=0.8), _ep(ep_id="ep-b", score=0.6)]
result = heap_expand(sparse=sparse, dense=dense, episode_to_facts={}, top_k=2)
assert len(result) == 2
assert result[0].id == "ep-a"
assert result[1].id == "ep-b"
assert all(r.item_type == "episode" for r in result)
assert result[0].score == pytest.approx(cosine_to_lr_score(0.8, 5.0))
assert result[1].score == pytest.approx(cosine_to_lr_score(0.6, 3.0))
def test_fact_evicts_parent_episode(self) -> None:
"""A high-scoring fact enters top-N and evicts its parent episode."""
sparse = [_ep(ep_id="ep-1", score=2.0)]
dense = [_ep(ep_id="ep-1", score=0.5)]
facts = {"ep-1": [_fact(fact_id="f1", parent_episode_id="ep-1", score=0.95)]}
result = heap_expand(
sparse=sparse, dense=dense, episode_to_facts=facts, top_k=2
)
result = _build_ep_to_fact_parents([ep])
fact_items = [r for r in result if r.item_type == "atomic_fact"]
ep_items = [r for r in result if r.item_type == "episode"]
assert len(fact_items) == 1
assert fact_items[0].id == "f1"
assert fact_items[0].parent_episode_id == "ep-1"
assert len(ep_items) == 0
assert result == {}
def test_empty_string_parent_id_skipped(self) -> None:
"""Empty string parent_id is filtered out."""
ep = _episode_candidate(ep_id="ep-1", memcell_id="")
result = _build_ep_to_fact_parents([ep])
assert result == {}
def test_empty_string_entry_id_skipped(self) -> None:
"""Empty string entry_id is filtered; parent_id still collected."""
ep = _episode_candidate(ep_id="ep-1", memcell_id="mc-1", entry_id="")
result = _build_ep_to_fact_parents([ep])
assert result == {"ep-1": ["mc-1"]}
def test_multiple_episodes_independent(self) -> None:
ep_a = _episode_candidate(
ep_id="ep-a", memcell_id="mc-a", entry_id="ep_entry_a"
)
ep_b = _episode_candidate(ep_id="ep-b", memcell_id="mc-b")
result = _build_ep_to_fact_parents([ep_a, ep_b])
assert result == {
"ep-a": ["ep_entry_a", "mc-a"],
"ep-b": ["mc-b"],
def test_global_competition_fact_evicts_weaker_episode(self) -> None:
"""Fact from ep-a can push ep-b out of top-N if ep-b scores lower."""
sparse = [
_ep(ep_id="ep-a", score=4.0),
_ep(ep_id="ep-b", score=1.0),
]
dense = [
_ep(ep_id="ep-a", score=0.7),
_ep(ep_id="ep-b", score=0.3),
]
facts = {
"ep-a": [
_fact(fact_id="f1", parent_episode_id="ep-a", score=0.95),
_fact(fact_id="f2", parent_episode_id="ep-a", score=0.90),
],
}
def test_empty_list_returns_empty_dict(self) -> None:
result = _build_ep_to_fact_parents([])
assert result == {}
# ── hierarchy_retrieve_episodes integration-style unit tests ─────────────
class TestHierarchyRetrieveEpisodes:
"""Integration-style unit tests with fully stubbed I/O.
amaxsim_retrieve and rrf are exercised with real implementations but
all LanceDB / network calls are replaced by AsyncMock.
"""
async def test_empty_sparse_dense_returns_empty_list(self) -> None:
fact_recaller, episode_recaller = _make_recallers()
result = await hierarchy_retrieve_episodes(
query="test query",
sparse=[],
dense=[],
query_vector=[0.1, 0.2, 0.3],
fact_recaller=fact_recaller,
episode_recaller=episode_recaller,
where="owner_id = 'u1'",
top_k=10,
)
assert result == []
async def test_happy_path_episode_wins_no_nested_facts(self) -> None:
ep = _episode_candidate(ep_id="ep-1", score=0.8, memcell_id="mc-1")
fact_recaller, episode_recaller = _make_recallers(
dense_facts=[],
fetched_episodes=[],
facts_for_episodes={},
)
result = await hierarchy_retrieve_episodes(
query="test query",
sparse=[ep],
dense=[ep],
query_vector=[0.1, 0.2, 0.3],
fact_recaller=fact_recaller,
episode_recaller=episode_recaller,
where="owner_id = 'u1'",
top_k=10,
result = heap_expand(
sparse=sparse, dense=dense, episode_to_facts=facts, top_k=1
)
assert len(result) == 1
episode_item = result[0]
assert episode_item.id == "ep-1"
assert episode_item.atomic_facts == []
# Episode-win score is the LR-calibrated parent score (cosine 0.8, bm25 0.8).
assert episode_item.score == pytest.approx(cosine_to_lr_score(0.8, 0.8))
assert result[0].item_type == "atomic_fact"
assert result[0].id == "f1"
async def test_happy_path_fact_evicts_episode_nested_in_result(self) -> None:
ep = _episode_candidate(ep_id="ep-2", score=0.5, memcell_id="mc-2")
fact = _fact_candidate(fact_id="fact-2", parent_episode_id="ep-2", score=0.95)
def test_top_k_caps_output(self) -> None:
sparse = [_ep(ep_id=f"ep-{i}", score=float(5 - i)) for i in range(5)]
dense = [_ep(ep_id=f"ep-{i}", score=0.9 - i * 0.1) for i in range(5)]
# No Layer-2 boost (dense_facts empty) → ep_cosine comes from the dense
# episode recall (0.5); sparse empty → parent_bm25 = 0.0. The Layer-4
# fact (0.95) calibrates above the parent and evicts it.
fact_recaller, episode_recaller = _make_recallers(
dense_facts=[],
fetched_episodes=[],
facts_for_episodes={"ep-2": [fact]},
result = heap_expand(sparse=sparse, dense=dense, episode_to_facts={}, top_k=3)
assert len(result) == 3
def test_convergence_stops_loop(self) -> None:
"""With max_convergence_rounds=1, loop stops after 1 round of no change."""
sparse = [_ep(ep_id="ep-1", score=2.0), _ep(ep_id="ep-2", score=1.0)]
dense = [_ep(ep_id="ep-1", score=0.5), _ep(ep_id="ep-2", score=0.4)]
result = heap_expand(
sparse=sparse,
dense=dense,
episode_to_facts={},
top_k=2,
max_convergence_rounds=1,
)
result = await hierarchy_retrieve_episodes(
query="test query",
sparse=[],
dense=[ep],
query_vector=[0.1, 0.2, 0.3],
fact_recaller=fact_recaller,
episode_recaller=episode_recaller,
where="owner_id = 'u1'",
top_k=10,
assert len(result) == 2
def test_alpha_blending(self) -> None:
"""alpha=0.5 blends child and parent LR scores equally."""
sparse = [_ep(ep_id="ep-1", score=2.0)]
dense = [_ep(ep_id="ep-1", score=0.5)]
facts = {"ep-1": [_fact(fact_id="f1", parent_episode_id="ep-1", score=0.95)]}
result = heap_expand(
sparse=sparse,
dense=dense,
episode_to_facts=facts,
top_k=2,
alpha=0.5,
)
assert len(result) == 1
episode_item = result[0]
assert episode_item.atomic_facts != []
nested_fact = episode_item.atomic_facts[0]
assert nested_fact.id == "fact-2"
# Evicted-fact score is its own LR-calibrated value (alpha default 1.0).
assert episode_item.score == pytest.approx(cosine_to_lr_score(0.95, 0.0))
async def test_min_score_filters_below_threshold(self) -> None:
ep = _episode_candidate(ep_id="ep-1", score=0.5, memcell_id="mc-1")
fact_recaller, episode_recaller = _make_recallers()
produced = cosine_to_lr_score(0.5, 0.0) # episode-win score (sparse empty)
result = await hierarchy_retrieve_episodes(
query="test query",
sparse=[],
dense=[ep],
query_vector=[0.1, 0.2, 0.3],
fact_recaller=fact_recaller,
episode_recaller=episode_recaller,
where="owner_id = 'u1'",
top_k=10,
min_score=produced + 0.05,
)
assert result == []
async def test_min_score_none_keeps_all(self) -> None:
ep = _episode_candidate(ep_id="ep-1", score=0.5, memcell_id="mc-1")
fact_recaller, episode_recaller = _make_recallers()
result = await hierarchy_retrieve_episodes(
query="test query",
sparse=[],
dense=[ep],
query_vector=[0.1, 0.2, 0.3],
fact_recaller=fact_recaller,
episode_recaller=episode_recaller,
where="owner_id = 'u1'",
top_k=10,
min_score=None,
)
assert len(result) == 1
assert result[0].id == "ep-1"
child_lr = cosine_to_lr_score(0.95, 2.0)
parent_lr = cosine_to_lr_score(0.5, 2.0)
expected = 0.5 * child_lr + 0.5 * parent_lr
fact_items = [r for r in result if r.item_type == "atomic_fact"]
assert fact_items
assert fact_items[0].score == pytest.approx(expected)
tests/unit/test_memory/test_search/test_manager.py
+106 -102
View File
@@ -6,7 +6,8 @@ candidate list; the manager's job is to:
* honour the ``owner_type`` hard partition,
* run KEYWORD as sparse-only and leave ``atomic_facts`` empty,
* run VECTOR as dense-only (and refuse when no embedding is wired),
* run VECTOR via MaxSim (ANN atomic_facts -> max-pool -> resolve episodes)
and refuse when no embedding is wired,
* let HYBRID run without an LLM by default; require LLM only when the
caller sets ``enable_llm_rerank=True``,
* refuse AGENTIC when reranker / LLM prerequisites are missing,
@@ -330,86 +331,6 @@ async def test_user_keyword_filters_compile_pinned_owner() -> None:
assert "owner_type = 'user'" in recaller.last_where
# ── VECTOR: requires embedding ────────────────────────────────────────
async def test_vector_method_requires_embedding() -> None:
mgr = _build_manager() # embedding=None by default
with pytest.raises(RuntimeError, match="embedding"):
await mgr.search(_user_req(method=SearchMethod.VECTOR))
async def test_vector_method_runs_dense_only_with_embedding() -> None:
mgr = _build_manager(
episode_sparse=[_episode_row("should_not_appear")],
episode_dense=[_episode_row("ep_dense")],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR))
assert [e.id for e in resp.data.episodes] == ["ep_dense"]
async def test_vector_radius_filter_drops_below_threshold() -> None:
mgr = _build_manager(
episode_dense=[
_episode_row("ep_low", score=0.3),
_episode_row("ep_high", score=0.9),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, radius=0.5))
assert [e.id for e in resp.data.episodes] == ["ep_high"]
async def test_unlimited_mode_applies_default_radius_for_vector() -> None:
"""``top_k=-1`` without an explicit radius gets the project default 0.5.
Mirrors enterprise's auto-floor behaviour — unlimited mode must not
return arbitrarily low-similarity tail.
"""
mgr = _build_manager(
episode_dense=[
_episode_row("ep_low", score=0.3), # below default 0.5 → dropped
_episode_row("ep_mid", score=0.55), # above default → kept
_episode_row("ep_high", score=0.9),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=-1))
assert [e.id for e in resp.data.episodes] == ["ep_mid", "ep_high"]
async def test_unlimited_mode_explicit_radius_overrides_default() -> None:
"""Caller-supplied radius (even ``0.0``) wins over the unlimited default."""
mgr = _build_manager(
episode_dense=[
_episode_row("ep_low", score=0.2),
_episode_row("ep_high", score=0.9),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=-1, radius=0.1))
# 0.1 threshold keeps both rows (the default 0.5 would have dropped ep_low).
assert {e.id for e in resp.data.episodes} == {"ep_low", "ep_high"}
async def test_normal_mode_keeps_full_pool_when_no_radius() -> None:
"""``top_k > 0`` without a radius applies no threshold — truncation handles tail."""
mgr = _build_manager(
episode_dense=[
_episode_row("ep_low", score=0.2),
_episode_row("ep_high", score=0.9),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=10))
# No radius default in normal mode → both kept.
assert {e.id for e in resp.data.episodes} == {"ep_low", "ep_high"}
# ── VECTOR + maxsim_atomic strategy ─────────────────────────────────────
def _atomic_fact_row(fid: str, *, parent_id: str, score: float) -> Candidate:
"""Atomic-fact candidate emitted by ``AtomicFactRecaller.dense_recall``."""
return Candidate(
@@ -428,18 +349,107 @@ def _atomic_fact_row(fid: str, *, parent_id: str, score: float) -> Candidate:
)
async def test_vector_maxsim_atomic_max_pools_facts_to_episodes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""``vector_strategy=maxsim_atomic`` should ANN atomic_facts → max-pool by
episode entry_id → resolve to episode, ordering episodes by the
per-episode maximum fact score."""
from everos.config.settings import load_settings
# ── VECTOR (MaxSim atomic) ────────────────────────────────────────────
monkeypatch.setenv("EVEROS_SEARCH__VECTOR_STRATEGY", "maxsim_atomic")
load_settings.cache_clear()
# Two episodes; each has two atomic facts under it. The max fact score
# per memcell is what should end up as the episode's score.
async def test_vector_method_requires_embedding() -> None:
mgr = _build_manager() # embedding=None by default
with pytest.raises(RuntimeError, match="embedding"):
await mgr.search(_user_req(method=SearchMethod.VECTOR))
async def test_vector_method_returns_episodes_via_maxsim() -> None:
mgr = _build_manager(
episode_sparse=[_episode_row("should_not_appear")],
episode_dense=[_episode_row("ep_dense")],
atomic_fact_dense=[
_atomic_fact_row("f1", parent_id="ep_dense", score=0.85),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR))
assert [e.id for e in resp.data.episodes] == ["ep_dense"]
async def test_vector_radius_filter_drops_below_threshold() -> None:
mgr = _build_manager(
episode_dense=[
_episode_row("ep_low"),
_episode_row("ep_high"),
],
atomic_fact_dense=[
_atomic_fact_row("f_low", parent_id="ep_low", score=0.3),
_atomic_fact_row("f_high", parent_id="ep_high", score=0.9),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, radius=0.5))
assert [e.id for e in resp.data.episodes] == ["ep_high"]
async def test_unlimited_mode_applies_default_radius_for_vector() -> None:
"""``top_k=-1`` without an explicit radius gets the project default 0.5.
Mirrors enterprise's auto-floor behaviour — unlimited mode must not
return arbitrarily low-similarity tail.
"""
mgr = _build_manager(
episode_dense=[
_episode_row("ep_low"),
_episode_row("ep_mid"),
_episode_row("ep_high"),
],
atomic_fact_dense=[
_atomic_fact_row("f_low", parent_id="ep_low", score=0.3), # below 0.5
_atomic_fact_row("f_mid", parent_id="ep_mid", score=0.55), # above 0.5
_atomic_fact_row("f_high", parent_id="ep_high", score=0.9),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=-1))
# Ordered by max-pooled fact score descending.
assert [e.id for e in resp.data.episodes] == ["ep_high", "ep_mid"]
async def test_unlimited_mode_explicit_radius_overrides_default() -> None:
"""Caller-supplied radius (even ``0.0``) wins over the unlimited default."""
mgr = _build_manager(
episode_dense=[
_episode_row("ep_low"),
_episode_row("ep_high"),
],
atomic_fact_dense=[
_atomic_fact_row("f_low", parent_id="ep_low", score=0.2),
_atomic_fact_row("f_high", parent_id="ep_high", score=0.9),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=-1, radius=0.1))
# 0.1 threshold keeps both rows (the default 0.5 would have dropped ep_low).
assert {e.id for e in resp.data.episodes} == {"ep_low", "ep_high"}
async def test_normal_mode_keeps_full_pool_when_no_radius() -> None:
"""``top_k > 0`` without a radius applies no threshold — truncation handles tail."""
mgr = _build_manager(
episode_dense=[
_episode_row("ep_low"),
_episode_row("ep_high"),
],
atomic_fact_dense=[
_atomic_fact_row("f_low", parent_id="ep_low", score=0.2),
_atomic_fact_row("f_high", parent_id="ep_high", score=0.9),
],
embedding=_StubEmbedding(),
)
resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=10))
# No radius default in normal mode -> both kept.
assert {e.id for e in resp.data.episodes} == {"ep_low", "ep_high"}
async def test_vector_maxsim_max_pools_facts_to_episodes() -> None:
"""ANN atomic_facts -> max-pool by episode entry_id -> resolve to
episode, ordering episodes by the per-episode maximum fact score."""
mgr = _build_manager(
episode_dense=[
_episode_row("ep_A", memcell_id="mc_A"),
@@ -460,14 +470,8 @@ async def test_vector_maxsim_atomic_max_pools_facts_to_episodes(
assert eps[1].score == pytest.approx(0.75)
async def test_vector_maxsim_atomic_returns_empty_when_no_facts(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""No fact recall → no memcells to score → empty episode list."""
from everos.config.settings import load_settings
monkeypatch.setenv("EVEROS_SEARCH__VECTOR_STRATEGY", "maxsim_atomic")
load_settings.cache_clear()
async def test_vector_returns_empty_when_no_facts() -> None:
"""No fact recall -> no episodes to score -> empty episode list."""
mgr = _build_manager(
episode_dense=[_episode_row("ep_A", memcell_id="mc_A")],
atomic_fact_dense=[],
@@ -503,9 +507,9 @@ async def test_hybrid_requires_llm_when_enable_llm_rerank_true() -> None:
async def test_user_hybrid_episode_fuses_and_evicts_facts() -> None:
"""HYBRID episode path: hierarchy pipeline (RRF -> MaxSim -> merge -> eviction).
"""HYBRID episode path: heap-expand pipeline (RRF -> LR -> expansion).
ep_1 has a fact scoring higher than the RRF score -> fact evicts episode.
ep_1 has a fact scoring higher than its LR score -> fact evicts episode.
ep_2 has no facts -> episode emitted as-is.
"""
ep1 = _episode_row("ep_1", score=0.8, memcell_id="mc_1")

Some files were not shown because too many files have changed in this diff Show More