Merge pull request #2220 from MemPalace/fix/windows-hybrid-search-empty-results

fix(tests): harden hybrid search against empty Windows Chroma reads

.agents/plugins/marketplace.json

@@ -8,11 +8,11 @@
       "name": "mempalace",
       "source": {
         "source": "local",
-        "path": "./.codex-plugin"
+        "path": "./"
       },
       "policy": {
         "installation": "AVAILABLE",
-        "authentication": "NONE"
+        "authentication": "ON_INSTALL"
       },
       "category": "Coding"
     }

.claude-plugin/README.md

@@ -1,6 +1,6 @@
 # MemPalace Claude Code Plugin
 
-A Claude Code plugin that gives your AI a persistent memory system. Mine projects and conversations into a searchable palace backed by ChromaDB, with 35 MCP tools, auto-save hooks, and 5 guided skills.
+A Claude Code plugin that gives your AI a persistent memory system. Mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and 5 guided skills.
 
 ## Prerequisites
 
@@ -51,7 +51,7 @@ Set the `MEMPAL_DIR` environment variable to a directory path to automatically r
 
 ## MCP Server
 
-The plugin automatically configures a local MCP server with 34 tools for storing, searching, and managing memories. No manual MCP setup is required -- `/mempalace:init` handles everything.
+The plugin automatically configures a local MCP server with 36 tools for storing, searching, and managing memories. No manual MCP setup is required -- `/mempalace:init` handles everything.
 
 ## Full Documentation
 

.claude-plugin/marketplace.json

@@ -8,8 +8,8 @@
     {
       "name": "mempalace",
       "source": "./.claude-plugin",
-      "description": "AI memory system — mine projects and conversations into a searchable palace. 35 MCP tools, auto-save hooks, guided setup.",
-      "version": "3.5.0",
+      "description": "AI memory system — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, guided setup.",
+      "version": "3.7.0",
       "author": {
         "name": "milla-jovovich"
       }

.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "mempalace",
-  "version": "3.5.0",
-  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 35 MCP tools, auto-save hooks, and guided setup.",
+  "version": "3.7.0",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.",
   "author": {
     "name": "milla-jovovich"
   },

.codex-plugin/README.md

@@ -1,6 +1,6 @@
 # MemPalace - Codex CLI Plugin
 
-Give your AI a persistent memory -- mine projects and conversations into a searchable palace backed by ChromaDB, with 35 MCP tools, auto-save hooks, and guided skills.
+Give your AI a persistent memory -- mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and guided skills.
 
 ## Prerequisites
 

.codex-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "mempalace",
-  "version": "3.5.0",
-  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 35 MCP tools, auto-save hooks, and guided setup.",
+  "version": "3.7.0",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.",
   "author": {
     "name": "milla-jovovich"
   },
@@ -18,16 +18,11 @@
     "search"
   ],
   "skills": "./skills/",
-  "hooks": "./hooks.json",
-  "mcpServers": {
-    "mempalace": {
-      "command": "mempalace-mcp"
-    }
-  },
+  "mcpServers": "./.mcp.json",
   "interface": {
     "displayName": "MemPalace",
     "shortDescription": "AI memory system for Codex",
-    "longDescription": "Give your AI a persistent memory — mine projects and conversations into a searchable palace backed by ChromaDB, with 35 MCP tools, auto-save hooks, and guided skills.",
+    "longDescription": "Give your AI a persistent memory — mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and guided skills.",
     "developerName": "milla-jovovich",
     "category": "Coding",
     "capabilities": [

.cursor-plugin/README.md

@@ -1,6 +1,6 @@
 # MemPalace Cursor Plugin
 
-A Cursor IDE plugin that gives your agent a persistent memory system. Auto-registers the `mempalace-mcp` server (35 MCP tools), ships 5 slash commands, two model-invocable skills (setup/mining/search and a recall protocol), and an optional recall rule.
+A Cursor IDE plugin that gives your agent a persistent memory system. Auto-registers the `mempalace-mcp` server (36 MCP tools), ships 5 slash commands, two model-invocable skills (setup/mining/search and a recall protocol), and an optional recall rule.
 
 > Hooks (auto-save + session-start memory recall) are shipped separately under `hooks/cursor/` so the plugin is safe to install in any Cursor workspace without touching the agent loop. See [Hooks](#hooks-optional) below.
 
@@ -87,7 +87,7 @@ This plugin ships `mcp.json` at the plugin root, so Cursor auto-loads the `mempa
 }
 ```
 
-All 34 MemPalace MCP tools (`mempalace_search`, `mempalace_add_drawer`, `mempalace_diary_write`, `mempalace_check_duplicate`, `mempalace_diary_read`, …) become available to the agent immediately. No manual `~/.cursor/mcp.json` edit required.
+All 36 MemPalace MCP tools (`mempalace_search`, `mempalace_add_drawer`, `mempalace_diary_write`, `mempalace_check_duplicate`, `mempalace_diary_read`, …) become available to the agent immediately. No manual `~/.cursor/mcp.json` edit required.
 
 If the server doesn't appear, confirm `mempalace-mcp` is on the user `$PATH`:
 

.cursor-plugin/marketplace.json

@@ -8,7 +8,7 @@
     {
       "name": "mempalace",
       "source": ".",
-      "description": "AI memory system — mine projects and conversations into a searchable palace. 35 MCP tools, slash commands, and a guided skill for Cursor.",
+      "description": "AI memory system — mine projects and conversations into a searchable palace. 36 MCP tools, slash commands, and a guided skill for Cursor.",
       "author": {
         "name": "milla-jovovich"
       }

.cursor-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "mempalace",
-  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 35 MCP tools, slash commands, and a guided skill for Cursor.",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, slash commands, and a guided skill for Cursor.",
   "author": {
     "name": "milla-jovovich"
   },

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
       matrix:
         python-version: ["3.9", "3.11", "3.13"]
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
@@ -24,24 +24,29 @@ jobs:
   test-windows:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v6
         with:
           python-version: "3.13"
           cache: 'pip'
       - run: pip install -e ".[dev]"
-      # ChromaDB's rust HNSW core intermittently fails compaction on Windows
-      # ("Failed to apply logs to the hnsw segment writer") regardless of our
-      # code — a long-standing, non-reproducible-on-Linux/macOS flake. Retry
-      # ONLY that specific transient error (via --only-rerun) so real,
+      # ChromaDB's rust HNSW core intermittently fails compaction or reader
+      # initialization on Windows ("Failed to apply logs to the hnsw segment
+      # writer" / "Error creating hnsw segment reader: Nothing found on disk")
+      # regardless of our code — long-standing, non-reproducible-on-Linux/macOS
+      # flakes. Retry ONLY those specific transient errors (via --only-rerun) so real,
       # deterministic failures still fail on the first run. Linux/macOS jobs
       # deliberately run with no reruns so genuine regressions surface there.
-      - run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10 --reruns 2 --reruns-delay 5 --only-rerun "Failed to apply logs to the hnsw segment writer"
+      - run: >-
+          python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace
+          --cov-report=term-missing --cov-fail-under=80 --durations=10 --reruns 2
+          --reruns-delay 5 --only-rerun
+          "Failed to apply logs to the hnsw segment writer|Error creating hnsw segment reader: Nothing found on disk"
 
   test-macos:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v6
         with:
           python-version: "3.13"
@@ -51,7 +56,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v6
         with:
           python-version: "3.11"
@@ -59,6 +64,10 @@ jobs:
       # Keep this pin identical to the `ruff==` pin in pyproject.toml
       # ([project.optional-dependencies].dev and [dependency-groups].dev)
       # so CI and `pip install -e ".[dev]"` format/lint identically.
-      - run: pip install "ruff==0.15.14"
+      # This drifted once (CI on 0.15.14 against pyproject's 0.15.20) and the
+      # drift was invisible: the lint job never installs from pyproject, so a
+      # dependabot bump to the pin there passes CI without the new version
+      # ever running. test_ruff_pins_match asserts the two stay equal.
+      - run: pip install "ruff==0.16.1"
       - run: ruff check .
       - run: ruff format --check .

.github/workflows/deploy-docs.yml

@@ -23,7 +23,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           fetch-depth: 0
 

.github/workflows/docker-publish.yml

@@ -16,20 +16,61 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
+  # Run the image before anyone can publish it. A green `build` only proves the
+  # Dockerfile compiles — it never started a container, so defects that break
+  # the first documented command shipped anyway (#2187, #2188). This builds
+  # amd64 natively, loads it into the local daemon, and exercises the README
+  # paths: CLI mine + verbatim search across two containers, a real MCP stdio
+  # handshake, and `compose config` on both shipped Compose files.
+  smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      # amd64 only and `load: true`: the smoke run needs a real image in the
+      # local daemon, and buildx cannot load a multi-arch manifest. Shares the
+      # `build` job's gha cache scope, so this is mostly a cache hit.
+      - name: Build image for smoke test
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64
+          push: false
+          load: true
+          tags: mempalace:smoke
+          # Read the publish job's cache too (its amd64 layers are identical),
+          # but write to a private scope so an amd64-only export never lands on
+          # top of that job's multi-arch one. Same split as `scope=gpu` below.
+          cache-from: |
+            type=gha
+            type=gha,scope=smoke
+          cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=smoke' || '' }}
+
+      # Reaches Chroma's S3 once to fetch the ~80 MB embedding model, so it is
+      # network-dependent; that download is itself part of a user's first run.
+      - name: Smoke test
+        run: ./scripts/docker-smoke.sh mempalace:smoke
+
   build:
     runs-on: ubuntu-latest
+    # Never publish an image the smoke test has not cleared.
+    needs: smoke
     permissions:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       # Needed for the emulated linux/arm64 build on real pushes.
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       # Only authenticate + push for in-repo events. Fork PRs lack the
       # packages:write token, so they build (to validate the Dockerfile) but
@@ -79,10 +120,10 @@ jobs:
   build-gpu:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Build GPU image (validation only — not published)
         uses: docker/build-push-action@v7

.github/workflows/publish.yml

@@ -73,7 +73,7 @@ jobs:
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
           echo "Resolved tag: $tag"
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           # Fully-qualified refs/tags/ so an unqualified name can't resolve to a
           # same-named *branch* instead of the tag (checkout prefers branches).

.github/workflows/version-guard.yml

@@ -16,7 +16,7 @@ jobs:
   check-versions:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Extract versions from all sources
         id: versions

.gitignore

@@ -2,6 +2,7 @@
 dist/
 build/
 __pycache__/
+.hypothesis/
 *.pyc
 .pytest_cache/
 mempal.yaml

.mcp.json

@@ -0,0 +1,7 @@
+{
+  "mcpServers": {
+    "mempalace": {
+      "command": "mempalace-mcp"
+    }
+  }
+}

CHANGELOG.md

@@ -10,6 +10,123 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ---
 
+## [3.7.0] — 2026-08-11
+
+### Features
+
+- **Embeddings via any OpenAI-compatible `/v1/embeddings` endpoint.** New `embedding_model: "openai-compat"` option computes embeddings on a server (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or a self-hosted endpoint) instead of a local ONNX model — useful for larger/multilingual embedders such as Qwen3-Embedding, or GPU offload. New `OpenAICompatEmbeddingFunction` in [`mempalace/embedding.py`](mempalace/embedding.py) speaks the standard `/v1/embeddings` protocol over stdlib `urllib` (no new dependency), batches requests, re-sorts the response by `index`, and L2-normalizes for the cosine collection. Endpoint settings are resolved by `MempalaceConfig` as a single source of truth — `embedding_api_url` / `embedding_api_model` / `embedding_api_key` in `config.json`, each overridable via the matching `MEMPALACE_EMBEDDING_API_*` env var. The embedding function's `name()` encodes the model id so changing it forces `mempalace repair rebuild-index` (different vector space). Mirrors the existing `openai-compat` LLM provider naming; stays local when the endpoint is on your machine/LAN. (#1559)
+- **`mempalace_search` / `mempalace search` — `since`/`before` date window.** Semantic search is poor at temporal queries ("what did we discuss this week?" scores ~0.35 even when matching drawers exist), so the search surfaces now accept the same `[since, before)` window `list_drawers` gained in #1128: inclusive/exclusive ISO bounds compared wall-clock against each drawer's `filed_at`, undated drawers excluded while a bound is active. The window applies on every candidate path (vector, `candidate_strategy="union"`, and the BM25-only fallback); the vector candidate pool widens under an active window (ChromaDB cannot range-compare string metadata server-side), and a full pool is flagged via `date_filter_pool_truncated` instead of passing silently. Shared parsing lives in the new `mempalace.date_window` module, also backing the `list_drawers` filter. (#463)
+- **Hermes agent memory provider (core).** In-package Hermes `MemoryProvider` files live turns through the shared `file_conversation_exchange()` path (same metadata as convo mining), with a background worker so the agent loop never blocks on Chroma. Wake-up L1 uses the long-lived collection only — a second PersistentClient is not opened, avoiding local SQLite races. Install/backfill/docs remain a stacked follow-up. (#1915, #2215)
+- **Explicit RFC 002 source adapters on `mempalace mine`.** `mempalace mine <source> --source <adapter>` resolves adapters through the registry, holds the palace writer lease for the full ingest, and keeps dry-runs inert (no real backend/KG open). Legacy `--mode` paths are unchanged. (#2068, #2062)
+- **MCP refuses writes when the served library is no longer the one that started.** Detects mempalace (and chromadb when that backend is active) version drift or uninstall after upgrade without restarting the server; opt-out via `MEMPALACE_MCP_ALLOW_STALE_LIBRARY`. (#2081, #899)
+- **Hook write-routing through the daemon.** Background hook saves and mines can honor the shared write-routing policy so multi-session setups serialize mutations through one local owner instead of racing the palace. (#2030, #1963)
+
+### Performance
+
+- **EmbeddingGemma groups documents by size before sub-batching.** The tokenizer pads every row of a sub-batch to the longest sequence in it, so arrival order decided the bill: one long verbatim message dragged a whole sub-batch up to its own length. Measured over 43,157 `sweep` drawers from 160 Claude Code transcripts, padded token slots drop 39.7% and the quadratic attention term 45.0%. Vectors move by at most one float32 ULP (1.2e-07 absolute, cosine 0.99999992), which is reduction-order rounding and not a change of meaning. Applies to `embedding_model: embeddinggemma` only; the default MiniLM embedder pads to a fixed width and was never affected. (#2104)
+- **HNSW capacity probes are cached** and invalidated by palace file signature, so repeated MCP status/taxonomy paths no longer re-scan native segment files on every call. (#2051, #1471)
+- **`chunk_text` line numbering is O(N)** via incremental tallies, fixing multi-second hangs on large sources. (#2054, #2055)
+
+### Bug Fixes
+
+- **Orphaned per-source-file mine locks are reaped instead of accumulating forever.** `_cleanup_mine_lock_file` reclaims a lock correctly on the happy path, but only for the specific lock its own `mine_lock` context manager just released — a process killed abruptly (SIGKILL, force-quit, host crash) never reaches that cleanup, and nothing else revisited the file afterward. One long-lived installation was found with 5,636 stale entries in `~/.mempalace/locks/`, the oldest several months old, none held by any live process. `mine_lock` now opportunistically reaps locks older than an hour via the same nonblocking-flock-reacquire safety check `_cleanup_mine_lock_file` already uses, throttled to once per 15 minutes so it costs nothing on the common path. `mine_palace_*.lock` (the newer per-palace lock) is untouched — it has its own lifecycle and holder tracking.
+- **Windows MCP stdout capture falls back cleanly** when fd-level redirection is unavailable, and fails closed if protocol stdout cannot be restored after a successful redirect. (#2211, #2210)
+- **Claude Code `subagents/` transcripts are skipped by default** when mining conversations (98%+ noise on typical workspaces); `--include-subagents` opts back in. (#1330, #1217)
+- **Worktree transcript cwd no longer mints a throwaway wing** — wing is derived from the project root above `.claude/worktrees/`. (#2206)
+- **Markdown emphasis/bold no longer scores as emotional content** in the general extractor. (#2199, #2197)
+- **Encoding hardening on Windows locales:** pin `encoding=utf-8` on config/dialect opens; safer mojibake repair that does not destroy clean Portuguese/Vietnamese/Turkish prose; repair-encoding CLI reconfigures stdio like other entry points; remaining non-ASCII CLI symbols replaced for GBK consoles. (#2098, #2208, #2194, #1104, #1034, #2193)
+- **`rebuild_index` holds the palace writer lease** for the full snapshot→rebuild/swap cycle so concurrent writers cannot recreate HNSW divergence mid-repair. (#2195)
+- **`MEMPALACE_PALACE_PATH` is restored after each service entrypoint** so multi-call processes do not leak the first call's palace into the second. (#2192)
+- **systemd unit uses `Restart=always`** and documents `MEMPALACE_MCP_IDLE_HOURS=0` so the idle watchdog does not leave a dedicated server down after a clean exit. (#2203, #2204)
+- **Mining works again on the default Chroma backend.** Once Chroma began declaring `requires_explicit_embeddings`, every write started routing through `EmbeddingCollection`, whose `_embed_texts` built rows with `list(ndarray)` — that unpacks into `np.float32` *scalars*, which chromadb rejects outright (`Expected embeddings to be a list of floats or ints, a list of lists, a numpy array, or a list of numpy arrays`). `mine`, and every other write against a default palace, aborted on the first drawer. Vectors now convert to real Python floats. The suite was structurally blind to this: conftest's autouse embedding fixture replaces `_embed_texts` itself for every module outside `test_embedding` / `test_embeddinggemma`, so the defective function was never executed under test — the regression tests therefore live in `test_embedding.py`, where that stub does not apply. (#2187)
+- **ChatGPT data exports are parsed instead of stored as raw JSON.** A real `conversations.json` is a top-level array of conversations, which no parser claimed, so `mine --mode convos` chunked the raw JSON and lost every speaker turn while reporting success. Each conversation now normalizes to its own transcript, as Claude.ai privacy exports already do, so per-conversation dedup survives a re-export. The ChatGPT parser also type-checks its nested shapes, so an unrelated array carrying a `mapping` key is declined instead of raising. (#2160)
+- **Local backends enforce process-lifetime single-writer ownership.** File-backed and unknown backends require one writer owner for the full process lifetime (daemon holds the lease until workers exit; writable MCP HTTP acquires ownership before bind and refuses startup when blocked). Read-only MCP may coexist; `sqlite_exact` opens genuine query-only/immutable readers; remote Milvus/Zilliz remain multi-process. Addresses multi-writer SQLite/WAL corruption from MCP HTTP + daemon + mine topologies. (#2079, #2045)
+- **Chroma HNSW write defaults match chromadb** (`batch_size=100` / `sync_threshold=1000`) instead of the old 2/2 bloat guard that rewrote segments thousands of times on large mines. (#2107, #2106)
+- **Repair and recovery are safer under contention.** `repair --mode from-sqlite` takes the mine-lock before archiving; rebuilds preserve a verified temp collection when the live swap fails; sparse drawers with zero `embedding_metadata` rows are no longer dropped; truncated ID pagination fails loud instead of pretending success. (#2109, #2086, #2087)
+- **`repair --mode from-sqlite --dry-run` is a true preview.** It no longer archives or re-embeds; it prints per-collection would-be counts from SQLite ground truth and exits without touching the palace. Unreadable counts fail closed instead of inventing zeros. (#2133, #2095, #1654)
+- **`repair --dry-run` is a true preview in the default (legacy) mode too.** That path ignored the flag entirely and ran the real rebuild — deleting any existing `<palace>.backup`, copying the palace over it, and re-filing the drawers collection. It now prints a read-only plan and exits without opening a chromadb client, which is itself a write to `chroma.sqlite3`. The plan names the live-collection delete the rebuild performs, warns when an existing backup would be destroyed, and reports the truncation guard as disabled when `--confirm-truncation-ok` is set. An isolated FTS5 inverted-index error is reported as auto-healable instead of raising the manual-recovery abort a real run never reaches, unreadable counts fail closed with a non-zero exit, and the `--dry-run` help no longer claims to be `--mode max-seq-id` only. (#2144)
+- **`repair` and `migrate` survive a socket or a named pipe in the palace directory.** Both take a whole-directory `shutil.copytree` backup before they overwrite the palace, and `copytree` cannot duplicate a Unix domain socket left beside `chroma.sqlite3`, nor a named pipe: it copied everything else and then raised `shutil.Error`, so the command died at the backup step before any rebuild ran, and re-running only repeated it. Neither entry carries palace data, so both are now skipped and named in the output while the backup completes. Device nodes are skipped too, because the copy dereferences them instead of failing on them. Every other copy failure still aborts the command, including a symlink whose target cannot be read: no errno separates a deleted target from one on a volume that is not mounted, so an entry that may have held data is left for the copy to fail on. (#2207)
+- **HNSW divergence is preflighted before remaining `col.count()` crash sites** across mine, dedup, migrate, repair, and palace helpers. (#2093)
+- **Re-mine and conversation ingest no longer lose or duplicate drawers.** Content-hash dedup prevents duplicate LLM conversation drawers; sweeper drawers are excluded from convo extract-mode purge scope and failed purges abort; search returns round-trippable `drawer_id` values for `get_drawer`. (#2050, #2125, #2089, #2090, #2044, #2080)
+- **MCP and daemon lifecycle harden multi-agent use.** Read-only mode refuses config and checkpoint-ack tools that rewrite host state; stdio MCP exits on stdin EOF/broken pipe so orphaned sessions release locks; daemon jobs refused the palace lock are deferred instead of failed permanently. (#2126, #2103, #2101, #2072, #2029, #2014)
+- **Entity-candidate extraction no longer hangs on long ASCII runs** (base64, minified blobs) while preserving CJK/non-ASCII text. (#2127, #2065, #2063)
+- **Mining windowing rejects `chunk_overlap` above half the chunk size**, stopping infinite chunk_text loops. (#2056, #2058)
+- **`docker-compose.yml` is valid again.** The `environment:` key was declared with only comments beneath it, which YAML parses as null, so Compose rejected the whole file (`services.mcp.environment must be a mapping`) — every documented Compose command failed before starting. The key is commented out along with its examples, which now use mapping syntax so uncommenting them yields a valid block. (#2188)
+
+### Documentation
+
+- Operator write-routing / single-writer recovery notes in `docs/write-routing-policy.md`. (#2079)
+- Remote-server guide wording for read-only tools that change host state. (#2126)
+- **The README's Docker section leads with the published image.** It previously documented only `docker build`, even though `ghcr.io/mempalace/mempalace` ships multi-arch — and a clone builds `develop`, not the release, so a build and a pull could differ silently. It now covers what the omissions actually cost: the MCP client config mounts a transcripts directory (without one the server starts and every mine finds nothing), the first embedding call downloads ~80 MB into `/data` and looks like a hang, bind mounts keep host ownership against the image's uid 1000 so a `0700` directory fails with a bare `PermissionError` on Linux (and `--user` is the wrong fix — `/data` is mode 700 owned by that uid), mining sources can be mounted read-only, and the GPU image is x86_64-only. (#2196)
+
+### Internal
+
+- **The Docker workflow runs the image before publishing it.** It previously built both images without ever starting a container or parsing a Compose file, so a green run only proved the Dockerfile compiled — which is how two defects that break the first documented command shipped past it. `scripts/docker-smoke.sh` now validates both Compose files, checks entrypoint dispatch, mines a mounted directory, asserts the drawer reads back verbatim from a separate container, and drives a real MCP stdio handshake; publication is gated on it. The script runs the same way locally: `scripts/docker-smoke.sh <image>`. (#2189)
+
+---
+
+## [3.6.0] — 2026-07-14
+
+### Features
+
+- **Turnkey secure remote / team server.** `mempalace serve` exposes the full MCP surface over HTTP with secure defaults: non-loopback binds require a bearer token, generated tokens are stored with restrictive permissions and passed through the child environment rather than argv, native TLS 1.2+ is supported, and `--read-only` both hides and refuses mutating tools. Docker Compose, systemd, and environment templates are included for deployment. (#1877, #1897, #1900)
+
+- **Milvus storage backend.** The new opt-in `milvus` backend supports embedded Milvus Lite, self-hosted Milvus, and Zilliz Cloud through `pymilvus`, with COSINE vector search, native BM25 lexical search for new collections, namespace isolation, target-mismatch protection, and an optional `milvus` dependency extra. (#1899)
+
+- **Atomic knowledge-graph fact replacement.** `supersede()` / `mempalace_kg_supersede` closes an open fact and opens its successor at one shared instant, so model, employer, address, and other single-valued facts can change without a hand-written invalidate/add race. Temporal queries now use half-open intervals at timestamp precision, returning only the successor at the transition boundary while preserving whole-day semantics for date-only facts. (#1913)
+
+- **Conversation chronology is preserved.** Conversation drawers now retain transcript time as `authored_at` alongside ingest time, search results expose it, and exact hybrid-score ties prefer the more recently authored drawer. An idempotent, dry-run-first backfill script adds the metadata to existing palaces without re-embedding. `mempalace_list_drawers` also accepts inclusive `since` and exclusive `before` filing-time bounds. (#1890, #1891)
+
+- **Mined sessions populate the associative graph.** A precision-biased, no-LLM structural extractor records code symbols, URLs, paths, and qualified identifiers as drawer entities; conversation mining then derives hallways from them. The new `mempalace hallways` CLI command exposes the graph. (#1894, #1895)
+
+- **Per-project mining exclusions.** `exclude_patterns` in `mempalace.yaml` filters pre-scanned project files without changing the corrected `--limit` semantics. (#1213, #1953)
+
+- **LaTeX project coverage.** `.tex` and `.bib` files are now treated as readable prose sources. (#1901)
+
+### Performance
+
+- **MCP initialization no longer waits for palace-wide integrity work.** stdio answers the JSON-RPC `initialize` request immediately while startup preflight runs on a background thread; a tool arriving mid-probe joins the same verdict instead of launching duplicate work. The startup SQLite probe is also skipped above a configurable size limit (512 MiB by default), while repair preflights remain strict. (#1911, #1987)
+
+- **Qdrant metadata counts use server-side facets.** Taxonomy, overview, and status paths can count metadata remotely instead of fetching and tallying every record; the embedding wrapper now forwards the facet and bulk-metadata capabilities correctly. (#1868, #1898)
+
+- **pgvector metadata-only fetches omit document payloads**, reducing transfer and decoding work on enumeration paths. (#1892)
+
+### Bug Fixes
+
+- **Chroma recovery distinguishes derived-index damage from data loss.** Valid all-layer-0 HNSW segments with an empty `link_lists.bin` are no longer quarantined repeatedly; isolated FTS5 inverted-index corruption is rebuilt from intact content during repair and after mining; embedded NUL bytes are sanitized before reaching ChromaDB. (#1716, #1872, #1878, #1927, #1928)
+
+- **SQLite recovery is bounded, atomic, and honest.** Integrity checks wait up to 15 seconds for transient writer contention instead of reporting a healthy palace as corrupt. In-place recovery archives use atomic `os.rename` rather than `shutil.move`'s destructive copy/delete fallback. `repair --mode from-sqlite` now rebuilds FTS5, vacuums, and requires a clean final `PRAGMA quick_check`; cleanup failures return non-zero with recovery details instead of printing a false success banner. (#1945, #2015, #2017)
+
+- **CLI search checks HNSW divergence before opening ChromaDB.** A diverged palace routes directly to the existing SQLite BM25 fallback, avoiding embedder or collection initialization against damaged native index state while leaving healthy Chroma and non-Chroma backends on their normal vector path. (#2016)
+
+- **Writer leases recover instead of stranding servers read-only.** A server refused while a peer owns the palace now retries on each mutating call and promotes itself after the peer exits. Status remains read-only and cannot acquire the writer lease, the in-process re-entrant holder set is released even across asynchronous interruption, and `checkpoint` / `delete_by_source` are correctly classified as mutations for both read-only serving and peer-writer protection. (#1923, #1930, #1934, #1960, #1970, #1971)
+
+- **Status reports only checks that actually ran.** Non-Chroma backends now mark the Chroma SQLite integrity check as not applicable rather than claiming an unperformed success. (#1931, #1946)
+
+- **Conversation transcripts are treated as mutable.** Appended or rewritten sessions are purged and re-filed based on modification time, preventing later turns from being silently skipped. Claude Code `tool-results/` sidecars are excluded from conversation scans so raw machine dumps cannot flood the embedding space. (#1957, #2010)
+
+- **Explicit palace selection now scopes derived graph state.** Project, conversation, and format mining write hallways and tunnels beside the selected `--palace` instead of leaking them into the ambient default palace. (#2018)
+
+- **Remote and local server hardening.** Non-loopback HTTP now requires a token unless an explicit insecure override is supplied; optional-provider probes cannot reuse an ambient external key before the user selects that provider; hook transcript paths and file-copy/repair paths receive stricter local validation and no-follow handling. Importing the MCP server no longer clobbers the host application's root logger, HTTP writes can re-enter the process-wide palace lock safely, and routine client disconnects no longer emit server tracebacks. (#1859, #1860, #1864, #1885, #2003, #2004)
+
+- **Mining and configuration correctness.** Oversized files produce a visible stderr warning, `~` in configured palace paths expands consistently, wing slugs handle special characters, and Windows background daemon / synchronous hook mines use `CREATE_NO_WINDOW`. (#923, #1852, #1857, #1863, #1865)
+
+- **`mempalace init` handles non-ASCII `.gitignore` files on Windows.** The project-file ignore guard now reads and appends UTF-8 explicitly instead of relying on locale defaults such as GBK. (#1648)
+
+- **L1 wake-up surfaces the latest moments.** Drawers with equal importance are now ordered by `filed_at` recency rather than insertion order, so startup context prefers recent memories instead of the oldest ones. (#1630)
+
+- **Backend detection requires a SQLite magic header** before classifying a target as Chroma or `sqlite_exact`, preventing unrelated files from being mistaken for a palace. (#1893, #1896)
+
+### Documentation
+
+- Added a storage-backend configuration reference, a remote/team-server deployment guide, `authored_at` migration guidance, and refreshed the OpenClaw integration for the full 36-tool MCP surface. (#1719, #1877, #1890, #1904, #1905)
+
+### Internal
+
+- Expanded WAL crash-safety, idempotence, and redaction-path coverage (#1869); updated GitHub Actions and Ruff; repaired `uv.lock` drift so it again matches the Ruff 0.15.20 pin and includes Python 3.9 dependency markers introduced by the Milvus resolution.
+
+---
+
 ## [3.5.0] — 2026-06-22
 
 ### Features
@@ -568,7 +685,10 @@ Initial public release.
 
 ---
 
-[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.4.1...HEAD
+[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.7.0...HEAD
+[3.7.0]: https://github.com/MemPalace/mempalace/compare/v3.6.0...v3.7.0
+[3.6.0]: https://github.com/MemPalace/mempalace/compare/v3.5.0...v3.6.0
+[3.5.0]: https://github.com/MemPalace/mempalace/compare/v3.4.1...v3.5.0
 [3.4.1]: https://github.com/MemPalace/mempalace/compare/v3.4.0...v3.4.1
 [3.4.0]: https://github.com/MemPalace/mempalace/compare/v3.3.6...v3.4.0
 [3.3.6]: https://github.com/MemPalace/mempalace/compare/v3.3.5...v3.3.6

CONTRIBUTING.md

@@ -119,7 +119,7 @@ If you're planning a significant change, open an issue first to discuss the appr
 
 - **Discord**: [Join us](https://discord.com/invite/ycTQQCu6kn)
 - **Issues**: Bug reports and feature requests welcome
-- **Discussions**: For questions and ideas
+- **Questions and ideas**: Please use Discord or open an issue while GitHub Discussions are not enabled for this repo
 
 ## License
 

README.md

@@ -72,75 +72,103 @@ pip install mempalace
 ### Docker
 
 A container image is also available for running the MCP server or the CLI
-without a local Python toolchain. Everything persists under `/data` (palace,
-config, and the cached embedding model), so mount a volume there.
+without a local Python toolchain. Multi-arch (amd64 + arm64), so it runs
+natively on Apple Silicon:
 
 ```bash
-# Build the image (CPU; bundles the `extract` + `spellcheck` extras)
-docker build -t mempalace .
-
-# MCP server over stdio — note the `-i` flag (JSON-RPC needs stdin)
-docker run -i --rm -v mempalace-data:/data mempalace
-
-# Run any CLI command instead (mount the host directory you want to mine)
-docker run --rm -v mempalace-data:/data -v /path/to/project:/work mempalace mine /work
-docker run --rm -v mempalace-data:/data mempalace search "why GraphQL"
+docker pull ghcr.io/mempalace/mempalace:latest
 ```
 
-Wire it into an MCP client (e.g. Claude Code) as a stdio server:
+Everything persists under `/data` — palace, config, and the cached embedding
+model — so mount a volume there and reuse it across runs:
+
+```bash
+# MCP server over stdio — note the `-i` flag (JSON-RPC needs stdin)
+docker run -i --rm -v mempalace-data:/data ghcr.io/mempalace/mempalace
+
+# Run any CLI command instead. The container only sees what you mount, so
+# mount the directory you want to mine — read-only is enough, mining never
+# writes to the source.
+docker run --rm -v mempalace-data:/data -v /path/to/project:/work:ro \
+  ghcr.io/mempalace/mempalace mine /work
+docker run --rm -v mempalace-data:/data ghcr.io/mempalace/mempalace search "why GraphQL"
+```
+
+The first command that needs embeddings downloads the model into `/data`
+(~80 MB for the default `minilm`, ~300 MB for `embeddinggemma`). It is a
+one-off as long as the volume persists, but it does mean the first call is
+slow and needs network — worth knowing before assuming a hung container.
+
+Wire it into an MCP client (e.g. Claude Code) as a stdio server. Mount
+anything you want the server to be able to mine — it cannot reach your
+transcripts otherwise:
 
 ```json
 {
   "mcpServers": {
     "mempalace": {
       "command": "docker",
-      "args": ["run", "-i", "--rm", "-v", "mempalace-data:/data", "mempalace"]
+      "args": [
+        "run", "-i", "--rm",
+        "-v", "mempalace-data:/data",
+        "-v", "/absolute/path/to/.claude/projects:/transcripts:ro",
+        "ghcr.io/mempalace/mempalace"
+      ]
     }
   }
 }
 ```
 
-`docker compose run --rm mcp` works too (see `docker-compose.yml`). For
-CUDA-accelerated embeddings, build the GPU variant with
-`docker build -f Dockerfile.gpu -t mempalace:gpu .` and run it with
-`--gpus all`. Customise the bundled extras at build time, e.g.
-`docker build --build-arg EXTRAS="extract,spellcheck" -t mempalace .`.
+Use a real absolute path there — `~` and `$HOME` are not expanded by every
+MCP client. Paths are container paths from then on: mine `/transcripts`, not
+`~/.claude/projects`.
+
+**Mount permissions on Linux.** The image runs as uid 1000 and bind mounts
+keep their host ownership, so a mounted directory has to be readable by that
+uid — an ordinary `0755` checkout is fine, a `0700` directory is not, and the
+failure surfaces as `PermissionError: [Errno 13]` rather than anything about
+Docker. Docker Desktop maps uids on macOS and Windows, so this only bites on
+Linux. Do **not** work around it with `--user`: `/data` is owned by uid 1000
+inside the image, so another uid cannot write the palace at all.
+
+`docker compose run --rm mcp` works too (see `docker-compose.yml`), and
+`deploy/docker-compose.server.yml` stands up the team server. To build the
+image yourself instead of pulling — required for the GPU variant, which is not
+published:
+
+```bash
+docker build -t mempalace .                                  # CPU
+docker build --build-arg EXTRAS="extract,spellcheck" -t mempalace .
+docker build -f Dockerfile.gpu -t mempalace:gpu .            # CUDA; run with --gpus all
+```
+
+The GPU image is x86_64-only: `onnxruntime-gpu` publishes no aarch64 Linux
+wheels, so that last build fails on an ARM host (including Apple Silicon) with
+a dependency-resolution error rather than an obvious one.
+
+Note that a build from a clone uses whatever branch you checked out; `develop`
+is the default branch, so pull the published image if you want the released
+version.
 
 ## Storage backends
 
-ChromaDB is the default. For the pluggable-backend preview, MemPalace also
-ships `sqlite_exact` for local exact-vector correctness checks, and two opt-in
-external service backends — `qdrant` (REST) and `pgvector` (Postgres). The two
-external backends exercise the storage contract on different substrates (a
-REST/dict store and a SQL/JSONB store), so it is not accidentally shaped around
-one vendor.
+ChromaDB is the default and needs no configuration. MemPalace also ships a
+pluggable backend contract, exercised across deliberately different substrates
+so the contract is never accidentally shaped around one vendor. Every
+non-default backend is opt-in.
 
-```bash
-# local no-service backend
-mempalace mine ~/projects/myapp --backend sqlite_exact
+| Backend | Mode | Install | Namespaces | Lexical | Configure with |
+| ------- | ---- | ------- | :--------: | :-----: | -------------- |
+| `chroma` _(default)_ | Local (embedded) | bundled | – | ✓ | – |
+| `sqlite_exact` | Local (exact) | bundled | – | ✓ | – |
+| `milvus` | Local (Lite) · Server opt-in | `mempalace[milvus]` | ✓ | ✓ | `MEMPALACE_MILVUS_URI` |
+| `qdrant` | Server (REST) | bundled | ✓ | ✓ | `MEMPALACE_QDRANT_URL` |
+| `pgvector` | Server (Postgres) | `mempalace[pgvector]` | ✓ | ✓ | `MEMPALACE_PGVECTOR_DSN` |
 
-# Qdrant backend, defaulting to http://localhost:6333
-MEMPALACE_QDRANT_URL=http://localhost:6333 \
-  mempalace mine ~/projects/myapp --backend qdrant
-
-# Postgres + pgvector backend, defaulting to postgresql://localhost:5432/mempalace
-#   needs the optional driver: pip install mempalace[pgvector]
-#   and the `vector` extension available on the server
-MEMPALACE_PGVECTOR_DSN=postgresql://localhost:5432/mempalace \
-  mempalace mine ~/projects/myapp --backend pgvector
-```
-
-Qdrant can also be configured with `MEMPALACE_QDRANT_API_KEY`,
-`MEMPALACE_QDRANT_NAMESPACE`, and `MEMPALACE_QDRANT_TIMEOUT`; pgvector with
-`MEMPALACE_PGVECTOR_NAMESPACE`. Both external backends isolate tenants by
-namespace (advertised via the `supports_namespace_isolation` capability) and
-write a local marker (`qdrant_backend.json` / `pgvector_backend.json`) to guard
-against silently opening a palace against the wrong server.
-
-When `MEMPALACE_QDRANT_URL` or `MEMPALACE_PGVECTOR_DSN` points anywhere other
-than your own local or trusted self-hosted service, MemPalace will send and
-store verbatim drawer text and metadata there. That is an explicit opt-in
-backend choice, never the default.
+Select with `--backend <name>`, `MEMPALACE_BACKEND=<name>`, or
+`"backend": "<name>"` in `config.json`. See
+[Storage backends](/guide/configuration#storage-backends) for connection
+variables, namespace behavior, and deployment notes.
 
 ## Quickstart
 
@@ -225,8 +253,9 @@ Usage and tool reference:
 
 ## MCP server
 
-35 MCP tools cover palace reads/writes, knowledge-graph operations,
-cross-wing navigation, drawer management, and agent diaries. Installation
+44 MCP tools cover palace reads/writes, knowledge-graph operations,
+cross-wing navigation, drawer management, agent diaries, and agent
+coordination (logstream events + artifact handoffs). Installation
 and the full tool list:
 [mempalaceofficial.com/reference/mcp-tools](https://mempalaceofficial.com/reference/mcp-tools.html).
 
@@ -264,6 +293,7 @@ verbatim drawer per user/assistant message, idempotent and resume-safe.
 - Python 3.9+
 - A vector-store backend (ChromaDB by default)
 - ~300 MB disk for the embedding model. Onboarding (`python -m mempalace.onboarding`) offers `embeddinggemma-300m` (multilingual, 100+ languages, recommended) or `all-MiniLM-L6-v2` (English-only, ~30 MB). See the docstring at [`mempalace/embedding.py`](mempalace/embedding.py) for details and migration notes.
+- Optional — compute embeddings on a server instead of locally. Set `embedding_model: "openai-compat"` in `~/.mempalace/config.json` together with `embedding_api_url` / `embedding_api_model` (and `embedding_api_key` if the server needs auth) to use any OpenAI-compatible `/v1/embeddings` endpoint — LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or a self-hosted server (e.g. a larger multilingual or GPU-served embedder). Each key is overridable via the matching `MEMPALACE_EMBEDDING_API_*` env var. When the endpoint is on your machine or LAN, no content leaves your network. Switching to it requires `mempalace repair rebuild-index` (different vector space).
 
 No API key is required for the core benchmark path.
 
@@ -285,7 +315,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
 MIT — see [LICENSE](LICENSE).
 
 <!-- Link Definitions -->
-[version-shield]: https://img.shields.io/badge/version-3.5.0-4dc9f6?style=flat-square&labelColor=0a0e14
+[version-shield]: https://img.shields.io/badge/version-3.7.0-4dc9f6?style=flat-square&labelColor=0a0e14
 [release-link]: https://github.com/MemPalace/mempalace/releases
 [python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8
 [python-link]: https://www.python.org/

benchmarks/locomo_bench.py

@@ -665,7 +665,7 @@ def run_benchmark(
                 palace_cache = json.load(f)
             print(f"  Palace cache: {len(palace_cache)} room assignments loaded")
 
-    rerank_label = f" + LLM re-rank ({llm_model.split('-')[1]})" if llm_rerank_enabled else ""
+    rerank_label = f" + LLM re-rank ({llm_model})" if llm_rerank_enabled else ""
 
     print(f"\n{'=' * 60}")
     print("  MemPal × LoCoMo Benchmark")

benchmarks/longmemeval_bench.py

@@ -3015,7 +3015,7 @@ def run_benchmark(
 
         if unique_sessions and api_key and not skip_precompute:
             print(
-                f"  Diary ingest: pre-computing {len(unique_sessions)} sessions with {llm_model.split('-')[1]}..."
+                f"  Diary ingest: pre-computing {len(unique_sessions)} sessions with {llm_model}..."
             )
             done = 0
             cache_path = Path(diary_cache_file) if diary_cache_file else None
@@ -3051,9 +3051,8 @@ def run_benchmark(
     print(f"  Data:        {Path(data_file).name}")
     print(f"  Questions:   {len(data)}")
     print(f"  Granularity: {granularity}")
-    model_short = llm_model.split("-")[1] if "-" in llm_model else llm_model
-    rerank_label = f" + LLM re-rank ({model_short})" if llm_rerank_enabled else ""
-    diary_label = f" [diary ingest: {model_short}]" if mode == "diary" else ""
+    rerank_label = f" + LLM re-rank ({llm_model})" if llm_rerank_enabled else ""
+    diary_label = f" [diary ingest: {llm_model}]" if mode == "diary" else ""
     print(f"  Mode:        {mode}{diary_label}{rerank_label}")
     print(f"{'─' * 60}\n")
 

deploy/docker-compose.server.yml

@@ -0,0 +1,71 @@
+# MemPalace remote team server — MCP over HTTP, backed by a central Qdrant.
+#
+# One command stands up a shared memory service a whole team's AI clients
+# connect to. Embeddings are still produced locally inside the mempalace
+# container; only your own Qdrant ever receives the vectors and text.
+#
+#   1. cp deploy/server.env.example deploy/.env  &&  edit deploy/.env
+#      (at minimum set MEMPALACE_MCP_HTTP_TOKEN to a long random secret)
+#   2. docker compose -f deploy/docker-compose.server.yml --env-file deploy/.env up -d
+#   3. connect a client (see the Remote / Team Server guide):
+#      claude mcp add --transport http mempalace http://YOUR_HOST:8765/mcp \
+#        --header "Authorization: Bearer $MEMPALACE_MCP_HTTP_TOKEN"
+#
+# SECURITY: this exposes plaintext HTTP on :8765. For anything beyond a trusted
+# private network, put a TLS-terminating reverse proxy in front (nginx/Caddy/
+# Traefik) and only expose the proxy. The bearer token is mandatory for the
+# network-exposed (0.0.0.0) bind.
+
+services:
+  qdrant:
+    image: qdrant/qdrant:latest
+    restart: unless-stopped
+    volumes:
+      - qdrant-storage:/qdrant/storage
+    # Not published to the host: only the mempalace service reaches it over the
+    # internal compose network. Uncomment to inspect Qdrant directly.
+    # ports:
+    #   - "6333:6333"
+
+  mempalace:
+    image: ghcr.io/mempalace/mempalace:latest
+    restart: unless-stopped
+    depends_on:
+      - qdrant
+    command:
+      - serve
+      - --host
+      - "0.0.0.0"
+      - --port
+      - "8765"
+      - --backend
+      - qdrant
+      # Uncomment to expose recall without write access to most clients:
+      # - --read-only
+    environment:
+      # Required for the network-exposed bind. Set in deploy/.env.
+      MEMPALACE_MCP_HTTP_TOKEN: ${MEMPALACE_MCP_HTTP_TOKEN:?set MEMPALACE_MCP_HTTP_TOKEN in deploy/.env}
+      MEMPALACE_QDRANT_URL: http://qdrant:6333
+      MEMPALACE_QDRANT_API_KEY: ${MEMPALACE_QDRANT_API_KEY:-}
+      # Set to cuda/dml/coreml on an accelerated host (see Dockerfile.gpu).
+      MEMPALACE_EMBEDDING_DEVICE: ${MEMPALACE_EMBEDDING_DEVICE:-auto}
+    ports:
+      - "8765:8765"
+    volumes:
+      - mempalace-data:/data
+    healthcheck:
+      # The image has no curl; use Python (always present). /healthz needs no auth.
+      # If you enable TLS on the server itself, switch this to https + ssl context.
+      test:
+        - CMD
+        - python
+        - -c
+        - "import urllib.request,sys; sys.exit(0) if urllib.request.urlopen('http://127.0.0.1:8765/healthz').read().strip()==b'ok' else sys.exit(1)"
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 40s
+
+volumes:
+  qdrant-storage:
+  mempalace-data:

deploy/mempalace-server.service

@@ -0,0 +1,51 @@
+# MemPalace remote MCP server — systemd unit template.
+#
+# Install:
+#   sudo useradd --system --home /var/lib/mempalace --shell /usr/sbin/nologin mempalace
+#   sudo install -d -o mempalace -g mempalace -m 750 /var/lib/mempalace /etc/mempalace
+#   sudo cp deploy/server.env.example /etc/mempalace/server.env   # then edit + chmod 600
+#   sudo install -m 600 -o mempalace -g mempalace /etc/mempalace/server.env /etc/mempalace/server.env
+#   # install mempalace into a venv on PATH, or adjust ExecStart to its absolute path
+#   sudo cp deploy/mempalace-server.service /etc/systemd/system/
+#   sudo systemctl daemon-reload && sudo systemctl enable --now mempalace-server
+#
+# This binds 0.0.0.0:8765 and requires MEMPALACE_MCP_HTTP_TOKEN (set in the
+# EnvironmentFile). Front it with a TLS-terminating reverse proxy, or set
+# MEMPALACE_MCP_TLS_CERT / _KEY in the EnvironmentFile for native TLS.
+
+[Unit]
+Description=MemPalace remote MCP server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=exec
+User=mempalace
+Group=mempalace
+EnvironmentFile=/etc/mempalace/server.env
+ExecStart=mempalace serve --host 0.0.0.0 --port 8765
+# The idle watchdog exits with status 0, which on-failure reads as an
+# intentional stop, so the server would stay down. server.env disables the
+# watchdog; Restart=always covers a deployment that has not set it.
+Restart=always
+RestartSec=2
+
+# --- Hardening ---------------------------------------------------------------
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+RestrictNamespaces=true
+LockPersonality=true
+MemoryDenyWriteExecute=false
+# The palace and any local state live here; everything else is read-only.
+ReadWritePaths=/var/lib/mempalace
+StateDirectory=mempalace
+
+[Install]
+WantedBy=multi-user.target

deploy/server.env.example

@@ -0,0 +1,35 @@
+# MemPalace remote server environment.
+# Copy to deploy/.env (compose) or /etc/mempalace/server.env (systemd) and edit.
+# Keep this file readable only by the service account: chmod 600.
+
+# --- Required for a network-exposed (0.0.0.0) bind ------------------------------
+# Clients send: Authorization: Bearer <this value>. Generate a strong secret:
+#   openssl rand -hex 32
+MEMPALACE_MCP_HTTP_TOKEN=
+
+# --- Storage backend ------------------------------------------------------------
+# The team server should use a networked backend so the palace is shared.
+MEMPALACE_BACKEND=qdrant
+MEMPALACE_QDRANT_URL=http://qdrant:6333
+# MEMPALACE_QDRANT_API_KEY=
+
+# --- Embedding ------------------------------------------------------------------
+# auto | cpu | cuda | dml | coreml. Use cuda on a GPU host (needs the GPU image).
+MEMPALACE_EMBEDDING_DEVICE=auto
+
+# --- Optional: native TLS (otherwise terminate TLS at a reverse proxy) ----------
+# Point these at a PEM cert/key the service account can read. When set, serve
+# speaks HTTPS directly and clients connect to https://...
+# MEMPALACE_MCP_TLS_CERT=/etc/mempalace/tls/cert.pem
+# MEMPALACE_MCP_TLS_KEY=/etc/mempalace/tls/key.pem
+
+# --- Server lifecycle -----------------------------------------------------------
+# The MCP server exits by itself after this many idle hours (default 8) to free
+# the ChromaDB/HNSW file handles that per-session stdio servers leave behind.
+# A dedicated always-on server does not need that, so 0 disables it. Note that
+# /healthz traffic does not count as activity; only MCP requests reset the timer.
+MEMPALACE_MCP_IDLE_HOURS=0
+
+# --- Palace location (systemd / bare-metal) -------------------------------------
+# In Docker the palace lives on the mempalace-data volume by default.
+# MEMPALACE_PALACE_PATH=/var/lib/mempalace/palace

docker-compose.yml

@@ -21,12 +21,15 @@ services:
     tty: false
     volumes:
       - mempalace-data:/data
-    environment:
-      # Everything defaults correctly from HOME=/data (palace ->
-      # /data/.mempalace/palace, config -> /data/.mempalace, model cache ->
-      # /data/.cache). Override here only for non-default locations, e.g.:
-      # - MEMPALACE_EMBEDDING_MODEL=embeddinggemma   # multilingual (default: minilm)
-      # - MEMPALACE_PALACE_PATH=/data/custom/palace
+    # Everything defaults correctly from HOME=/data (palace ->
+    # /data/.mempalace/palace, config -> /data/.mempalace, model cache ->
+    # /data/.cache), so no environment block is needed. Uncomment the key
+    # *together with* an entry under it for non-default locations — a bare
+    # `environment:` with only comments beneath parses as null, and Compose
+    # rejects the whole file with "services.mcp.environment must be a mapping".
+    # environment:
+    #   MEMPALACE_EMBEDDING_MODEL: embeddinggemma   # multilingual (default: minilm)
+    #   MEMPALACE_PALACE_PATH: /data/custom/palace
 
 volumes:
   mempalace-data:

docs/authored-at.md

@@ -0,0 +1,45 @@
+# Authored date (`authored_at`)
+
+Conversation transcripts carry a per-line ISO-8601 `timestamp` (both Claude Code and
+Codex JSONL). The miner records the most recent one per file as the drawer's
+**`authored_at`** — when the content was actually written.
+
+This is distinct from the ingest date:
+
+| Field | Meaning |
+|-------|---------|
+| `filed_at` / result `created_at` | When the drawer was **mined** (written to the palace). A bulk re-mine collapses these to a single instant. |
+| `authored_at` | When the underlying content was **written**, recovered from the transcript timestamps. Survives re-mining. |
+
+`authored_at` is surfaced in search results (and shown in the CLI `search` output), and is
+used as a deterministic tie-break in hybrid ranking: candidates with identical scores order
+with the more recently authored drawer first. Drawers without per-line timestamps (e.g.
+markdown) fall back to `filed_at`.
+
+## Backfilling existing memory
+
+New mines populate `authored_at` automatically. Drawers mined before this feature only have
+`filed_at`. Re-mining does **not** fix them — the scanner skips files already mined at the
+current `NORMALIZE_VERSION`. Two options:
+
+1. **In-place backfill (recommended — no re-embedding).** `scripts/backfill_authored_at.py`
+   reads each convos drawer's source transcript and updates only the `authored_at` metadata.
+   Idempotent and safe to re-run; embeddings are untouched.
+
+   ```bash
+   python scripts/backfill_authored_at.py \
+       --palace ~/.mempalace/palace \
+       --sessions ~/.claude --sessions ~/.codex          # dry run
+   python scripts/backfill_authored_at.py \
+       --palace ~/.mempalace/palace \
+       --sessions ~/.claude --sessions ~/.codex --apply  # write
+   ```
+
+   For the Docker MCP image, mount the volume and session dirs read-only — see the header of
+   `scripts/backfill_authored_at.py` for the exact `docker run` invocation.
+
+   > Back up first: `tar czf palace-backup.tgz -C <palace-dir> .` (or snapshot the
+   > `mempalace-data` volume).
+
+2. **Drop and recreate.** Delete the affected drawers and re-mine the transcripts; the fresh
+   mine stamps `authored_at`. Simpler, but re-embeds everything.

docs/hook-write-routing.md

@@ -0,0 +1,112 @@
+# Hook write routing
+
+Hook-triggered writes use the shared write-routing policy introduced for the
+Tier 3 rollout tracked in #1963.
+
+## Scope
+
+This applies to every routine write initiated by the Python hook layer:
+
+- Stop-hook diary checkpoints;
+- transcript/conversation ingest;
+- project auto-ingest;
+- SessionEnd final flushes;
+- PreCompact transcript ingest;
+- PreCompact synchronous project mining.
+
+It does not change CLI write routing. CLI adoption is a separate follow-up.
+
+## Policy behavior
+
+### `direct`
+
+Hooks use the existing direct in-process or subprocess paths.
+
+The daemon is not probed.
+
+### `prefer`
+
+Hooks use the daemon when it is already healthy.
+
+If the daemon is unavailable, hooks retain the historical direct fallback.
+
+### `require`
+
+Hooks use the daemon when it is already healthy.
+
+If the daemon is unavailable:
+
+- no in-process ChromaDB write runs;
+- no direct `mempalace mine` subprocess is started;
+- no daemon is cold-started from the hook;
+- the hook log records the skipped operation;
+- the hook returns a visible `systemMessage`;
+- the Stop save marker is not advanced, allowing a later retry.
+
+## Why hooks do not start the daemon
+
+Hooks operate under strict latency budgets. Starting a Python daemon and its
+storage dependencies from a Stop or SessionEnd hook can exceed that budget.
+
+A supervised installation using `require` must start the daemon earlier, for
+example at login, plugin initialization, or session setup:
+
+    mempalace daemon start
+
+SessionStart performs a fast health probe in `require` mode and warns early if
+the required daemon is unavailable.
+
+## One decision per hook event
+
+A Stop or SessionEnd event may perform several writes:
+
+1. diary checkpoint;
+2. transcript ingest;
+3. project auto-ingest.
+
+The route is resolved once and stored in a context-local value for the whole
+write burst. This avoids repeated health probes and prevents different writes
+from selecting inconsistent routes during the same event.
+
+## Submission ambiguity
+
+Once a daemon submission is attempted, an error never triggers direct
+fallback. The daemon may have accepted the job before the client observed the
+failure; retrying directly could duplicate content.
+
+## Invalid configuration
+
+An explicitly invalid routing policy fails closed: hook writes are blocked and
+no direct ChromaDB fallback is attempted.
+
+An unrelated configuration read/runtime failure preserves the historical
+direct-save behavior so a final checkpoint is not lost because of an
+independent configuration failure.
+
+## Backward compatibility
+
+The default policy remains `direct`.
+
+Legacy settings remain supported through the shared policy resolver:
+
+- `MEMPALACE_HOOKS_DAEMON=true` maps to `prefer`;
+- `hooks.daemon: true` maps to `prefer`;
+- false values map to `direct`.
+
+## Configuration examples
+
+Prefer the daemon but permit direct fallback:
+
+    MEMPALACE_HOOK_WRITE_ROUTING=prefer
+
+Require the daemon and prohibit direct writers:
+
+    MEMPALACE_HOOK_WRITE_ROUTING=require
+
+Configuration file:
+
+    {
+      "write_routing": {
+        "hooks": "require"
+      }
+    }

docs/rfcs/001-storage-backend-plugin-spec.md

@@ -0,0 +1,670 @@
+# RFC 001 — Storage Backend Plugin Specification
+
+- **Status:** Accepted (2026-06-07)
+- **Tracking issue:** [#737](https://github.com/MemPalace/mempalace/issues/737)
+- **Supersedes:** The informal seam introduced by [#413](https://github.com/MemPalace/mempalace/pull/413)
+- **Related:** [#266](https://github.com/MemPalace/mempalace/issues/266), [#574](https://github.com/MemPalace/mempalace/pull/574), [#643](https://github.com/MemPalace/mempalace/pull/643), [#665](https://github.com/MemPalace/mempalace/pull/665), [#697](https://github.com/MemPalace/mempalace/pull/697), [#700](https://github.com/MemPalace/mempalace/pull/700), [#381](https://github.com/MemPalace/mempalace/pull/381), [#1679](https://github.com/MemPalace/mempalace/pull/1679)
+- **Spec version:** `1.0`
+
+> **Implementation status (2026-08-11).** The §1–2 contract surface (`PalaceRef`,
+> typed results, `BaseBackend` / `BaseCollection`, capability tokens, and most of
+> the §10 seam cleanup) landed ahead of this merge via [#1679](https://github.com/MemPalace/mempalace/pull/1679),
+> which ships in-tree `pgvector`, `qdrant`, and `sqlite_exact` backends. The three
+> areas originally deferred as follow-ups have since landed: embedder-identity
+> (§1.5 / §5, [#1731](https://github.com/MemPalace/mempalace/pull/1731) /
+> [#1734](https://github.com/MemPalace/mempalace/pull/1734)), maintenance hooks
+> (§7.3, [#1732](https://github.com/MemPalace/mempalace/pull/1732)), and searcher
+> metric-awareness (§10, [#1727](https://github.com/MemPalace/mempalace/pull/1727)).
+> Accepting this RFC pins the contract; remaining gaps are evolutionary, not
+> blocking.
+
+## Summary
+
+A formal contract for MemPalace storage backends so third parties can ship `pip install mempalace-<name>` packages that drop into the core without patches. The spec defines the collection interface, the backend lifecycle, registration via Python entry points, configuration shape, a required test contract, and a migration path between backends.
+
+It also sets up MemPalace to run as a long-lived daemon that manages many palaces, where different palaces may route to different backends.
+
+## Motivation
+
+Six backend PRs are currently in flight. Each one solves the same problem six different ways — different method signatures, different registration mechanisms, different embedder ownership, incompatible where-clause dialects, no shared test suite. The ad-hoc `BaseCollection` ABC merged in #413 was deliberately minimal and deferred every non-obvious decision. This RFC closes the open decisions so backend authors can build to a stable contract.
+
+## Goals
+
+1. A backend ships as a standalone Python package; installing it is sufficient to use it.
+2. All callers in MemPalace core go through the collection interface. No direct `chromadb` imports outside `mempalace/backends/chroma.py`.
+3. Backends are interchangeable: every backend passes the same shared test suite, and `mempalace migrate` supports lossless movement between them when source/target capabilities allow, with explicit re-embedding as the fallback (§8.2).
+4. The model scales from single-user local (one backend, one palace, no config) to a daemon serving many palaces with heterogeneous backends.
+5. Chroma's current dict-shaped return values are not the long-term contract. Typed results are spec v1.
+
+## Non-goals
+
+- Defining the embedder pipeline in detail. The embedder is a separate contract this spec depends on but does not specify.
+- Defining the sync subsystem. This spec only declares the capability flag and the minimal hook a sync subsystem will read.
+- Specifying wire protocol for a future networked daemon. That is a separate RFC.
+
+---
+
+## 1. Collection contract
+
+### 1.1 Required methods
+
+All backends implement `BaseCollection` with kwargs-only signatures:
+
+```python
+class BaseCollection(ABC):
+    @abstractmethod
+    def add(
+        self,
+        *,
+        documents: list[str],
+        ids: list[str],
+        metadatas: list[dict] | None = None,
+        embeddings: list[list[float]] | None = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def upsert(
+        self,
+        *,
+        documents: list[str],
+        ids: list[str],
+        metadatas: list[dict] | None = None,
+        embeddings: list[list[float]] | None = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def query(
+        self,
+        *,
+        query_texts: list[str] | None = None,
+        query_embeddings: list[list[float]] | None = None,
+        n_results: int = 10,
+        where: dict | None = None,
+        where_document: dict | None = None,
+        include: list[str] | None = None,
+    ) -> QueryResult: ...
+
+    @abstractmethod
+    def get(
+        self,
+        *,
+        ids: list[str] | None = None,
+        where: dict | None = None,
+        where_document: dict | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
+        include: list[str] | None = None,
+    ) -> GetResult: ...
+
+    @abstractmethod
+    def delete(
+        self,
+        *,
+        ids: list[str] | None = None,
+        where: dict | None = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def count(self) -> int: ...
+```
+
+### 1.2 Optional methods (default implementations on the ABC)
+
+```python
+def estimated_count(self) -> int:
+    return self.count()
+
+def close(self) -> None:
+    return None
+
+def health(self) -> HealthStatus:
+    return HealthStatus.ok()
+
+def update(
+    self,
+    *,
+    ids: list[str],
+    documents: list[str] | None = None,
+    metadatas: list[dict] | None = None,
+    embeddings: list[list[float]] | None = None,
+) -> None:
+    """Partial update of existing rows. At least one of documents/metadatas/embeddings must be non-None.
+
+    Default implementation: get(ids=...), merge the provided fields, upsert. Non-atomic
+    and does two round-trips. Backends advertising `supports_update` MUST override with
+    an atomic, single-round-trip implementation.
+    """
+    ...  # default impl in the ABC
+```
+
+Backends with cheap approximate counters override `estimated_count`. Backends that hold connections must override `close`. Backends with native partial-update primitives (Postgres `UPDATE`, Lance `merge_insert`) override `update` and advertise `supports_update`; the token signals "atomic + single round-trip," not "supports partial updates at all" — the default implementation already supports them, just non-atomically.
+
+### 1.3 Typed results (replaces Chroma dict shape)
+
+```python
+@dataclass(frozen=True)
+class QueryResult:
+    ids: list[list[str]]                              # outer = queries, inner = hits
+    documents: list[list[str]]
+    metadatas: list[list[dict]]
+    distances: list[list[float]]
+    embeddings: list[list[list[float]]] | None = None
+
+@dataclass(frozen=True)
+class GetResult:
+    ids: list[str]
+    documents: list[str]
+    metadatas: list[dict]
+    embeddings: list[list[float]] | None = None
+```
+
+On empty results: return a result object with empty inner lists, never raise. Specifically, an empty query returns `QueryResult(ids=[[]], documents=[[]], metadatas=[[]], distances=[[]])` — the outer dimension is the number of query vectors issued; the inner dimension is hits per query and may be zero.
+
+`include` controls which fields are populated. Fields not in `include` are populated with empty lists of the correct outer shape; they are never `None` (except `embeddings`, which is `None` when not requested).
+
+### 1.4 Where-clause dialect
+
+**Required operators:** `$eq`, `$ne`, `$in`, `$nin`, `$and`, `$or`, `$contains`.
+
+Backends that do not support full-text natively MUST still implement `$contains` via payload string match — correctness is required; performance is not. `supports_contains_fast` (§2.1) is the only performance floor the spec promises. Without it, callers and benchmarks MUST assume `$contains` is O(n). This is an intentional split: `$contains` is a correctness requirement, `contains_fast` is the performance boundary, and the gap between scan and indexed FTS is too large for the spec to paper over.
+
+**Unknown operators:** backends MUST raise `UnsupportedFilterError`. Silent dropping is forbidden — it produces incorrect results.
+
+**Optional operators:** `$gt`, `$gte`, `$lt`, `$lte`. Backends either implement them or reject with `UnsupportedFilterError`. Advertised via capabilities.
+
+### 1.5 Embeddings
+
+#### Signature compliance (all backends)
+
+All backends MUST accept a pre-computed `embeddings=` argument on `add` / `upsert` without raising. This is signature compliance only — it does not guarantee the vectors are persisted (see passthrough below). Capability token: `supports_embeddings_in`.
+
+Backends MUST NOT hardcode embedding models or dimensions. Model selection is the embedder's responsibility (§4).
+
+#### Passthrough vs re-embed (separate guarantee)
+
+Accepting the argument is not the same as honoring it. Two distinct semantics, distinguished by capability:
+
+- **`supports_embeddings_passthrough`** — when `embeddings=` is provided, the backend MUST persist those vectors as-is and MUST NOT re-embed from text. This is the stronger guarantee lossless migration depends on.
+- **No `supports_embeddings_passthrough`** — the backend always re-embeds from text at write time. Provided `embeddings=` is accepted (signature compliance) but discarded. Migration *to* such a backend is re-embedding, not lossless transfer.
+
+`supports_migration_export` (source-side bulk read) MUST be paired with `supports_embeddings_passthrough` (target-side lossless write) for a migration to be labeled lossless. The `mempalace migrate` CLI refuses to run between backends where the target lacks `supports_embeddings_passthrough` unless `--accept-re-embed` is passed, which records re-embedding in the target palace's migration log.
+
+#### Dimension check (all backends, required)
+
+Backends MUST validate embedding dimension on first write to a new collection and on open of an existing collection, and MUST raise `DimensionMismatchError` on mismatch. Silent acceptance of mismatched dimensions produces unrecoverable corruption.
+
+#### Model identity check (all backends, three-state)
+
+Dimension matching is necessary but not sufficient. Swapping to a different model that happens to share a dimension (e.g., both 384-d) silently degrades retrieval without tripping `DimensionMismatchError`. Backends MUST persist `embedder.model_name` alongside the collection on first write and MUST check it on subsequent open. Three outcomes:
+
+| State | Condition | Required behavior |
+|---|---|---|
+| `known_match` | Stored name equals current `embedder.model_name` | Proceed normally. |
+| `known_mismatch` | Stored name exists and differs from current | Raise `EmbedderIdentityMismatchError`. Override only via explicit CLI `--force-model-swap`, which writes the swap to the palace's migration log and updates the stored identity. |
+| `unknown` | No model name recorded (legacy collection, pre-v1 palace) | Do not hard-fail — emit a `EmbedderIdentityUnknownWarning` on first open. The resolved identity is recorded on the next successful write, reindex, or migration, transitioning the palace to `known_match` going forward. CLI exposes `mempalace palace set-embedder --model NAME` for explicit resolution. |
+
+The `unknown` state exists because existing palaces from #413 and earlier have no recorded identity; hard-failing them on upgrade would be hostile. Once recorded, subsequent opens are strict.
+
+An injected embedder that exposes no usable `model_name` (empty or `None`) resolves to `unknown` rather than being a hard error — the backend persists no identity, emits `EmbedderIdentityUnknownWarning`, and records identity on the first open against an embedder that *does* report a name. A nameless embedder is therefore a degraded-but-valid mode, not a rejection; operators promote it with `mempalace palace set-embedder --model NAME`.
+
+> **Follow-up dependency.** The `MUST persist embedder.model_name` rule above is only satisfiable once the `Embedder` protocol (§5) is a normative contract — a backend cannot persist an identity it is never handed. §5 now pins the minimal protocol (`model_name`, `dimension`, `embed`) as sufficient for this section; the embedder-identity enforcement is tracked as follow-up implementation work (the current in-tree backends defer it). See §5.
+
+#### `server_embedder` backends are not exempt
+
+A backend advertising `server_embedder` (§2.1) provides its own embedder and MAY ignore the `embedder=` kwarg passed to `get_collection`. That does **not** exempt it from the dimension and identity rules above. Such backends MUST:
+
+- Expose an effective `model_name: str` and `dimension: int` describing the embedder actually in use (via `BaseCollection.effective_embedder_identity() -> EmbedderIdentity`).
+- Persist that effective identity on first write and validate it on open, per the three-state rules above.
+- Raise `DimensionMismatchError` and `EmbedderIdentityMismatchError` on conflicts between the effective identity and any injected `embedder` (if one was passed) or between the stored identity and the current effective identity.
+
+`server_embedder` documents where the embedding happens; it never suspends the safety contract. A backend that cannot report its effective embedder identity does not qualify for the `server_embedder` capability.
+
+---
+
+## 2. Backend contract
+
+### 2.1 Identity and capabilities
+
+```python
+class BaseBackend(ABC):
+    name: ClassVar[str]                       # "chroma", "postgres", "qdrant", ...
+    spec_version: ClassVar[str] = "1.0"       # which spec version this backend targets
+    capabilities: ClassVar[frozenset[str]]
+    distance_metric: ClassVar[str] = "cosine" # "cosine" | "l2" | "ip" (inner product)
+```
+
+`distance_metric` declares the space the backend's `distances` are reported in.
+It is **not** a capability token (it is a single value, not a boolean), so it is a
+class attribute. Core search code MUST convert a backend's reported distance to a
+similarity using this declaration rather than assuming cosine — see §10, which
+adds `searcher.py` to the cleanup precisely because `_hybrid_rank` currently
+hard-codes `max(0, 1 - distance)` (cosine-only). All in-tree backends are `cosine`
+today, so the assumption is latent, not yet wrong; the declaration makes a
+non-cosine backend (e.g. a dot-product store) correct rather than silently
+mis-ranked.
+
+Defined capability tokens (v1):
+
+| Token | Meaning |
+|---|---|
+| `supports_embeddings_in` | Accepts pre-computed `embeddings=` without raising (signature compliance; MUST be true for all backends) |
+| `supports_embeddings_passthrough` | Persists provided `embeddings=` as-is without re-embedding (required for lossless migration target) |
+| `supports_embeddings_out` | Returns embeddings when `include=["embeddings"]` is requested |
+| `supports_estimated_count` | `estimated_count()` is meaningfully cheaper than `count()` |
+| `supports_update` | `update()` is atomic and single-round-trip (vs the ABC default of get+merge+upsert) |
+| `supports_metadata_filters` | Implements the required where-clause subset (§1.4) |
+| `supports_range_filters` | Implements `$gt` / `$gte` / `$lt` / `$lte` |
+| `supports_contains_fast` | `$contains` is indexed (vs scan-based) |
+| `supports_server_side_indexes` | Exposes index creation / maintenance to operators |
+| `supports_migration_export` | Implements a bulk read path suitable for `mempalace migrate` |
+| `supports_change_feed` | Exposes `changes_since(cursor)` for the sync subsystem |
+| `supports_sync` | Implies `supports_change_feed` plus idempotent upserts under conflicts |
+| `requires_external_service` | Needs a running server (e.g., Postgres, hosted Qdrant) |
+| `local_mode` | Persists to `palace.local_path` |
+| `server_mode` | Connects to an external server; `palace.namespace` is used |
+| `server_embedder` | Backend provides its own embedder (may ignore injected one) |
+| `supports_namespace_isolation` | Enforces `PalaceRef.namespace` as a hard isolation boundary (§4.4). Multi-tenant deployments MAY rely on it for tenant isolation; backends without it MUST NOT be relied on for that. |
+
+A backend may advertise both `local_mode` and `server_mode` (e.g., Chroma with either `PersistentClient` or `HttpClient`).
+
+Capability tokens are free-form strings, not an enum — third-party backends may declare novel capabilities for their ecosystem. Core MemPalace only inspects the tokens listed above.
+
+### 2.2 Palace references
+
+A backend serves palaces, not raw filesystem paths. This is the central change from #413.
+
+```python
+@dataclass(frozen=True)
+class PalaceRef:
+    id: str                          # stable identity, used as cache key
+    local_path: str | None = None    # filesystem root, if this palace is local
+    namespace: str | None = None     # server-side namespace/prefix, if applicable
+```
+
+Rules:
+- `id` is always present. It is the key the backend uses to cache open handles.
+- Local-only backends read `local_path`. If `local_path is None` they raise `PalaceNotFoundError`.
+- Server-only backends read `namespace`. If `namespace is None` they derive one deterministically from `id`.
+- Mixed-mode backends may use both (e.g., a local cache alongside a server store).
+
+### 2.3 Methods
+
+```python
+class BaseBackend(ABC):
+    @abstractmethod
+    def get_collection(
+        self,
+        *,
+        palace: PalaceRef,
+        collection_name: str,
+        create: bool,
+        embedder: Embedder | None = None,
+        options: dict | None = None,
+    ) -> BaseCollection: ...
+
+    def close_palace(self, palace: PalaceRef) -> None:
+        """Evict a single palace's cached handles. Default: no-op."""
+        return None
+
+    def close(self) -> None:
+        """Shut down the entire backend instance. Default: no-op."""
+        return None
+
+    def health(self, palace: PalaceRef | None = None) -> HealthStatus:
+        """Return health. With palace=None, probe the backend itself."""
+        return HealthStatus.ok()
+```
+
+### 2.4 Semantics of `create`
+
+- `create=False` on a nonexistent palace MUST raise `PalaceNotFoundError` (subclass of `FileNotFoundError` for backwards compatibility with the #413 seam).
+- `create=True` MUST be idempotent — calling it repeatedly with the same arguments produces the same state and does not corrupt existing data.
+- `create=True` on local backends creates the directory with `0700` permissions (matches the existing Chroma behavior).
+
+**Multiple collections per palace.** `get_collection` is keyed by `collection_name`, and a palace MAY hold more than one collection. Backends MUST support N collections per palace, addressed by distinct `collection_name` values, with the §2.5 isolation guarantee applying per `(palace.id, collection_name)`. The "palace" is not 1:1 with a collection: production already splits verbatim drawers from short, query-term-saturated session-recovery checkpoints into sibling collections (`mempalace_drawers` vs `mempalace_session_recovery`) so the latter can't dominate vector top-N. Backends like Postgres (schema/table naming) and Qdrant (collection naming) handle this trivially; a backend author MUST NOT assume one collection per palace and design themselves into a corner. No signature changes — this is already implicit in `collection_name`; it is stated here so it is a contract fact, not a convention.
+
+### 2.5 Concurrency
+
+A backend instance is long-lived and serves many palaces. Backends MUST be thread-safe for concurrent `get_collection` calls across different `PalaceRef.id` values. Collection handles for the same `(palace.id, collection_name)` MAY be cached internally and returned on subsequent calls.
+
+Backends MAY assume a single thread accesses a given `BaseCollection` instance at a time. MemPalace core serializes access per palace; backend authors are not required to make individual collections thread-safe.
+
+### 2.6 Lifecycle
+
+1. `__init__`: lightweight. No I/O, no network connections. A backend instance may be constructed and never used.
+2. First call to `get_collection`: may open connections, create schemas, etc. All I/O is lazy.
+3. `close_palace(palace)`: releases cached handles for one palace. Safe to call on a palace that was never opened.
+4. `close()`: releases all resources. After `close()`, further calls MUST raise `BackendClosedError`.
+
+There is no explicit `connect()` — it is always implicit and lazy, matching current Chroma behavior.
+
+---
+
+## 3. Registration and discovery
+
+### 3.1 Entry points (primary mechanism)
+
+Third-party backends ship as installable packages:
+
+```toml
+# pyproject.toml of mempalace-postgres
+[project.entry-points."mempalace.backends"]
+postgres = "mempalace_postgres:PostgresBackend"
+```
+
+MemPalace discovers backends at process start via `importlib.metadata.entry_points(group="mempalace.backends")`. No patches to the core are required.
+
+### 3.2 In-tree registry (secondary)
+
+For tests and local development:
+
+```python
+from mempalace.backends.registry import register
+
+register("my-experimental-backend", MyBackend)
+```
+
+Entry-point discovery and explicit `register()` populate the same registry. Explicit registration wins on name conflict.
+
+### 3.3 Selection priority
+
+When resolving a palace's backend, priority (highest first):
+
+1. Explicit `backend=` kwarg to `Palace(...)` or CLI `--backend`
+2. Per-palace `backend` key in config (see §4)
+3. `MEMPALACE_BACKEND` environment variable
+4. Auto-detect from on-disk artifacts: `chroma.sqlite3` → `chroma`, `*.lance` → `lance`, etc. Backends declare detection hints via an optional `BaseBackend.detect(path: str) -> bool` classmethod.
+5. Default: `chroma`.
+
+**Auto-detection is strictly a migration/upgrade compatibility path, not a general selection mechanism.** It exists so existing palaces from v3.x keep opening without forced config migration. For *new* palaces, explicit configuration or CLI flag always wins — creating a palace without a resolved backend from (1)–(3) falls through to default (5), never to detection (4). Auto-detection fires only when a local path is presented AND no earlier rule has chosen a backend AND the path already contains backend-identifiable artifacts.
+
+Note the interaction with a globally-set `MEMPALACE_BACKEND`: rule (3) sits above detection (4), so a user who exports `MEMPALACE_BACKEND=postgres` and then opens a palace containing on-disk Chroma artifacts gets postgres — the env var wins and detection is skipped. That is intended (explicit configuration overrides detection), but it means **setting `MEMPALACE_BACKEND` globally overrides existing-palace auto-detection; users opening pre-existing palaces of mixed backends should leave it unset** and rely on per-palace config or detection.
+
+---
+
+## 4. Configuration
+
+### 4.1 Shape
+
+```json
+{
+  "backends": {
+    "chroma": { "type": "chroma" },
+    "pg_prod": {
+      "type": "postgres",
+      "dsn": "postgresql://...",
+      "pool_size": 10
+    }
+  },
+  "palaces": {
+    "work": {
+      "backend": "pg_prod",
+      "namespace": "work"
+    },
+    "personal": {
+      "backend": "chroma",
+      "local_path": "~/.mempalace/personal"
+    }
+  },
+  "embedder": {
+    "type": "onnx",
+    "model": "all-MiniLM-L6-v2"
+  }
+}
+```
+
+Single-user local mode: all of this is optional. The absence of a config file yields one Chroma backend, one palace at the default path, with the default embedder.
+
+### 4.2 Environment variables
+
+- `MEMPALACE_BACKEND` — shortcut for the default backend type when there is no config.
+- `MEMPALACE_<NAME>_*` — per-backend secrets and connection info (e.g., `MEMPALACE_POSTGRES_DSN`, `MEMPALACE_QDRANT_URL`, `MEMPALACE_QDRANT_API_KEY`).
+- `<NAME>` is the backend's **type** name (the `type` field in §4.1 — `postgres`, `qdrant`), uppercased, not the per-instance config key. So the `pg_prod` instance in §4.1 reads `MEMPALACE_POSTGRES_*`, not `MEMPALACE_PG_PROD_*` — instances of the same type share one env namespace, and connection-specific values that differ per instance (distinct DSNs) belong in the config file's per-backend block, not in env. Hyphens in a type name are normalized to underscores for the env prefix (`my-backend` → `MEMPALACE_MY_BACKEND_*`).
+- Secrets MUST be readable from env vars; config files are for structure, env vars for credentials.
+
+### 4.3 Backend-specific options
+
+The `options` kwarg to `get_collection` is a free-form dict. Each backend documents its accepted keys. Unknown keys MUST be ignored (forward compatibility), but the backend MAY log a warning.
+
+### 4.4 Multi-tenancy (absorbs #697)
+
+Per-tenant collection-name prefixing is not a backend concern. It is handled by the resolver layer above backends: `PalaceRef.namespace` carries the tenant identifier. The `collection_prefix` concept from #697 dissolves into this model.
+
+**Isolation contract.** `PalaceRef.id` is the *required* isolation key for every backend: within a single backend instance, a record written for one `id` MUST NOT be returned, modified, or deleted by an operation issued for a different `id`. Cross-palace access is a spec violation. This is the non-negotiable blanket MUST; multi-tenant deployments may cite it as their primary partition boundary even when they do not use `namespace`.
+
+`namespace` is *additional* partitioning. A backend that advertises `supports_namespace_isolation` (§2.1) MUST extend the same guarantee to namespaces:
+
+> A record written under one `namespace` MUST NOT be returned, modified, or deleted by an operation issued under a different `namespace` within the same backend instance. Cross-namespace access is a spec violation, not a caller misconfiguration.
+
+This is what hosted multi-tenant deployments cite as the basis for *namespace*-level tenant isolation. Authorization (which namespaces a given request may touch) stays on the deployment side; the backend's job is to guarantee no bleed *within* the instance once the namespace is fixed.
+
+Making namespace isolation a declared capability rather than a blanket `MUST` is deliberate: path-rooted local backends (e.g. `chroma`, `sqlite_exact`) already isolate by `local_path` / `id`, and forcing them to re-implement a second axis would be ceremony with no security gain.
+
+**Conformance arms.** Self-attestation is not enough. The isolation suite (`tests/_backend_conformance.py` / `assert_partition_isolation`) has two distinct arms:
+
+1. **Cross-`id` isolation** — required of every backend. Two `PalaceRef`s that differ only in `id` MUST NOT see each other's records.
+2. **Same-`id` / different-`namespace` isolation** — required of every backend that advertises `supports_namespace_isolation`. Two refs that share `id` (and, when applicable, `local_path`) but differ in `namespace` MUST NOT see each other's records. Distinct from arm 1; a backend that only passes arm 1 does not get to claim the capability.
+
+**No silent drop.** A backend that does **not** advertise `supports_namespace_isolation` MUST NOT silently accept and ignore a populated `namespace` — that is a latent cross-namespace leak, same spirit as `UnsupportedFilterError` for unknown operators. Such backends MUST either raise (e.g. `UnsupportedCapabilityError`) when `PalaceRef.namespace` is non-`None`, or honor the namespace and advertise the capability. Callers targeting path-rooted backends MUST leave `namespace` as `None`.
+
+---
+
+## 5. Embedder contract (minimal, normative here)
+
+§1.5 makes persisting and checking `embedder.model_name` a hard `MUST`. A backend cannot satisfy that against an embedder that has no identity to read, so the minimal protocol below is **normative for this spec** — not deferred. A fuller Embedder RFC (batching, async, pooling, multi-vector) is tracked separately, but it is *additive*: §1.5 conformance depends only on the three members here.
+
+```python
+class Embedder(Protocol):
+    model_name: str          # stable identity persisted and checked per §1.5
+    dimension: int           # validated per §1.5 dimension check
+    def embed(self, texts: list[str]) -> list[list[float]]: ...
+```
+
+Backends receive an `Embedder` via `get_collection(embedder=...)`. Backends with the `server_embedder` capability MAY ignore the injected embedder but MUST still expose an effective `model_name` / `dimension` (§1.5). An embedder whose `model_name` is empty or `None` is handled as the §1.5 `unknown` state, not a hard error.
+
+> **Follow-up.** The full Embedder RFC is the only external contract §1.5 leans on; it is tracked as a hard, blocking dependency of the §1.5 *implementation* (not of this spec's acceptance — the minimal protocol above closes the contract gap). Tracking issue: see §13. The current in-tree backends defer embedder-identity enforcement until that work lands.
+
+---
+
+## 6. Sync (capability declaration only)
+
+The sync subsystem is out of scope for this spec. What this spec defines:
+
+- `supports_sync` capability flag (§2.1) — a backend advertising it agrees to implement idempotent upserts under conflict and to expose change data.
+- Optional method on `BaseCollection`:
+  ```python
+  def changes_since(self, cursor: SyncCursor) -> Iterator[Change]: ...
+  ```
+- Backends without `supports_change_feed` / `supports_sync` are rejected by the sync subsystem at bind time.
+
+Local single-user deployments never load the sync subsystem; non-sync-capable backends cost them nothing.
+
+---
+
+## 7. Testing contract
+
+### 7.1 The abstract suite
+
+MemPalace ships `mempalace.backends.testing.AbstractBackendContractSuite` — a pytest mixin. Every backend package ships a concrete subclass:
+
+```python
+from mempalace.backends.testing import AbstractBackendContractSuite
+
+class TestPostgresBackend(AbstractBackendContractSuite):
+    @pytest.fixture
+    def backend(self, tmp_path):
+        return PostgresBackend(dsn=os.environ["TEST_PG_DSN"])
+```
+
+The suite covers:
+- Round-trip for every required method
+- Empty-result shape (outer dimension preserved, inner lists empty)
+- `create=False` on missing palace raises `PalaceNotFoundError`
+- `create=True` is idempotent
+- Full required where-clause subset including `$contains`
+- Unknown operator raises `UnsupportedFilterError`
+- Dimension-mismatch detection
+- Unicode text and unicode IDs
+- Large batch writes (10k+ items)
+- Delete-then-query consistency
+- `close()` releases handles and further calls raise `BackendClosedError`
+- Concurrent `get_collection` across different palaces is safe
+- Isolation arm 1: cross-`PalaceRef.id` (every backend)
+- Isolation arm 2: same-`id` / different-`namespace` (only when `supports_namespace_isolation` is advertised; self-attestation without this arm is a conformance failure)
+- Non-advertising backends raise on a populated `namespace` rather than silently accepting it (§4.4)
+
+### 7.2 Parametrized core suite
+
+The existing MemPalace test suite is parametrized over all registered backends when `MEMPALACE_TEST_ALL_BACKENDS=1` is set in the environment. This is the "strongest parity claim" — if a backend passes the full core suite, it is drop-in compatible. This is expensive; local development defaults to Chroma only, CI runs all backends on a scheduled job.
+
+### 7.3 Benchmark methodology hooks
+
+Backend-to-backend comparisons are meaningless without accounting for per-backend maintenance state. Postgres with stale planner stats behaves very differently from Postgres post-`VACUUM ANALYZE`; HNSW-based stores behave differently before and after index compaction.
+
+Backends MAY implement `maintenance_state()` returning a structured dict describing the current state (e.g., `{"autovacuum_age_seconds": 42, "last_analyze": "...", "index_build_complete": true}`), and `run_maintenance(kind: str)` to trigger supported kinds. Both are optional.
+
+Supported maintenance kinds MUST be advertised via a class-level frozenset:
+
+```python
+class BaseBackend(ABC):
+    maintenance_kinds: ClassVar[frozenset[str]] = frozenset()
+```
+
+The spec reserves the kind names `"analyze"` (update planner/query statistics), `"compact"` (reclaim space, rewrite storage), and `"reindex"` (rebuild secondary indexes). Backends MAY add their own kinds; the reserved names MUST mean what the spec says if advertised. A backend that has no analogue for a reserved kind MUST omit it from `maintenance_kinds` rather than declaring it as a no-op — otherwise a benchmark harness sees `"analyze"` advertised and assumes it did what the spec says when the implementation did nothing.
+
+`run_maintenance(kind)` MUST raise `UnsupportedMaintenanceKindError` when called with a kind not in `maintenance_kinds`. Advertising a kind without implementing it is a conformance failure.
+
+**`run_maintenance` is observable, not fire-and-forget.** This resolves the §12 open question, and it is driven by a production failure: on a lazy-index path, multiple daemon writers crossing the index-build threshold in the same window each issued the build, stacked an `ACCESS EXCLUSIVE` lock, and blocked writes for the whole build (fixed in-backend with a session-level advisory lock). A pure fire-and-forget call reproduces that race — if the call cannot report "already running," concurrent callers re-trigger the build. Therefore:
+
+- `run_maintenance(kind)` MUST be safe to call concurrently. A backend MUST serialize same-kind maintenance internally (advisory lock, build flag, or equivalent) so a second caller does not start a duplicate operation.
+- It returns a structured `MaintenanceResult` (e.g. `{"kind": str, "status": "ran" | "already_running" | "noop", "stats": {...}}`) rather than `None`. `stats` is free-form per kind (rows analyzed, bytes reclaimed, fragments merged). `already_running` is how a concurrent caller learns it must not re-trigger.
+
+This makes the maintenance hook the operator-safe path the lazy-index concurrency wedge requires, and gives the benchmark harness (above) machine-readable phase data.
+
+The benchmark harness under [benchmarks/](../../benchmarks/) records `maintenance_state()` alongside every latency/recall measurement it publishes. Published numbers MUST include three phases: immediately after bulk load, after the backend's native background maintenance has caught up, and after `run_maintenance(kind)` has been called for each kind in `maintenance_kinds`. Harnesses rely on this advertisement to decide what to call — they MUST NOT assume kind names. This prevents comparing an un-`ANALYZE`d Postgres to a settled Chroma and calling the former slow.
+
+### 7.4 ID stability for non-string-ID backends
+
+Backends requiring UUID IDs (Qdrant) use a canonical namespace:
+
+```python
+NAMESPACE_MEMPALACE = uuid.UUID("c06c3fc7-5c14-4dc4-84c2-24a5f72d8dc1")
+backend_id = uuid.uuid5(NAMESPACE_MEMPALACE, original_id)
+```
+
+The namespace UUID is fixed at spec v1 adoption and recorded here — once, for all time. This value is the one the in-tree `qdrant` backend already shipped with in #1679, so existing Qdrant palaces' point IDs already derive from it; promoting the shipped constant to canonical (rather than minting a fresh one) avoids re-deriving any deployed palace's IDs. New UUID-ID backends MUST use this exact namespace. This resolves the #700 vs #381 divergence.
+
+---
+
+## 8. Migration
+
+### 8.1 The CLI
+
+```
+mempalace migrate --palace PATH --from chroma --to postgres
+mempalace migrate --all --to lance
+```
+
+Implementation is backend-agnostic: reads from source via `BaseCollection.get(include=["documents", "metadatas", "embeddings"])`, writes to target via `BaseCollection.upsert(...)` with the original embeddings. No backend-specific migration code.
+
+### 8.2 Lossless vs re-embed
+
+Migration is labeled **lossless** only when:
+
+- The source advertises `supports_migration_export` (bulk read includes embeddings), AND
+- The target advertises `supports_embeddings_passthrough` (persists provided embeddings as-is), AND
+- Source and target agree on `embedder.model_name` (or `--force-model-swap` is explicit).
+
+If the target lacks `supports_embeddings_passthrough`, `mempalace migrate` refuses to run. Passing `--accept-re-embed` overrides — the migration proceeds but re-embeds from document text at write time, and the migration record labels the result as re-embedded rather than lossless. Retrieval quality may shift.
+
+A backend that persists the exact float32 vector in its own durable store (rather than re-embedding from text on read) satisfies **both** sides of the lossless pairing: it qualifies for `supports_migration_export` as a source (it can hand back the verbatim vector it stored) and for `supports_embeddings_passthrough` as a target (it persists provided vectors as-is). Such "rank-from-stored-vectors" backends are first-class migration endpoints in both directions — the pairing in this section is about the two *capabilities* being present, not about any particular index structure. An exact-vector store therefore makes `chroma → <store>` and `<store> → chroma` lossless under §8 in both directions, given model-identity agreement.
+
+### 8.3 Safety
+
+- Source is never modified. Migration is read-only against the source backend.
+- Target palace must not already exist unless `--overwrite` is passed.
+- A successful migration writes a `.mempalace-migration.json` record into the target palace containing: source backend name, source path/ref, timestamp, row count, `lossless: true|false`, source and target `embedder.model_name`, and whether `--force-model-swap` or `--accept-re-embed` was used.
+
+### 8.4 Verification
+
+After migration, run `mempalace verify --palace PATH --against SOURCE_PATH --source-backend chroma`. This samples N rows and confirms round-trip parity (ids match, documents match, embedding cosine similarity ≥ 0.999 when the migration was lossless; a looser document-overlap check when re-embedded).
+
+---
+
+## 9. Versioning and compatibility
+
+- `BaseBackend.spec_version` declares which spec version a backend implements.
+- MemPalace refuses to load a backend declaring a different major version. The failure is loud and names the mismatch, e.g.:
+  ```python
+  raise BackendVersionMismatchError(
+      f"backend {name!r} targets spec {backend.spec_version!r}; "
+      f"this MemPalace implements major version {CORE_MAJOR!r}. "
+      f"Install a build of {name!r} that targets spec {CORE_MAJOR}.x."
+  )
+  ```
+- Minor versions are additive (new optional methods, new capability tokens). Backends declaring an older minor continue to work.
+- This is spec v1.0.
+
+---
+
+## 10. Cleanup prerequisite (mostly landed via #1679)
+
+**Status update (2026-06-07).** The bulk of this cleanup landed in [#1679](https://github.com/MemPalace/mempalace/pull/1679): the direct `chromadb` *client* imports across `repair.py`, `dedup.py`, `cli.py`, `mcp_server.py`, and `migrate.py` were routed through `BaseCollection`, and the dict-to-typed-result migration (§1.3) shipped. What remains is narrow:
+
+- **Residual exception-class imports.** `mcp_server.py` and `repair.py` still `from chromadb.errors import NotFoundError` to catch Chroma's not-found error. These are catch-site couplings, not client construction; they should resolve to a backend-neutral `PalaceNotFoundError` / collection-not-found exception from the contract (§2.4) so non-Chroma palaces raise the same type.
+- **`mcp_server._get_client()` caching.** Caches a `PersistentClient` at module scope and invalidates it on `chroma.sqlite3` inode or mtime changes (merged via [#757](https://github.com/MemPalace/mempalace/pull/757)). Both the cache and the stat-based freshness check are Chroma-specific. They should migrate into `ChromaBackend.get_collection()` (§2.5, handle caching) and `ChromaBackend.close_palace()` (§2.6, explicit flush) — other backends do not have a single on-disk SQLite file to stat. The `mempalace_reconnect` MCP tool then becomes a thin wrapper around `backend.close_palace(palace_ref)`.
+
+**`searcher.py` is the highest-leakage module still coupled, and #1679 did not cover it** (flagged by two backend authors — kostadis-ntnx on this RFC, jphein on #1679). Two concrete couplings:
+
+- `_hybrid_rank()` hard-codes `vec_sim = max(0, 1 - distance)`, i.e. it assumes cosine distance ∈ [0, 2]. For a backend whose metric isn't cosine (dot-product, L2) the ranking is silently wrong. Fix: make the conversion metric-aware off the backend-declared `distance_metric` (§2.1) rather than hard-coding cosine. All in-tree backends are cosine today, so this is latent — but it is a contract gap, not a coincidence to keep relying on.
+- The BM25 survival fallback opens `chroma.sqlite3` directly and queries Chroma's FTS5 `embedding_fulltext_search` shadow table. On any non-Chroma palace this path is dead. Fix: route lexical fallback through `BaseCollection` (§1.4 `$contains` / a lexical capability), or gate it on a backend-declared capability so backends whose index self-heals never request it.
+
+This `searcher.py` work is tracked as follow-up implementation (see §13) — it is the remaining piece that promotes a non-Chroma backend from "stores and retrieves" to a first-class search peer.
+
+---
+
+## 11. Impact on in-flight PRs
+
+**Update (2026-06-07).** The first wave of in-tree backends — `pgvector`, `qdrant`, `sqlite_exact` — landed via [#1679](https://github.com/MemPalace/mempalace/pull/1679) implementing the §1–2 contract surface directly, ahead of this RFC's merge. That reshapes the table below: the community PRs now rebase against both the spec *and* the shipped in-tree backends, and the prediction that `collection_prefix` would dissolve into `PalaceRef.namespace` (#697) held.
+
+| PR | Status | Effort to align |
+|---|---|---|
+| [#574](https://github.com/MemPalace/mempalace/pull/574) LanceDB | Open | Closest to final shape. Needs `PalaceRef` and typed results (now both shipped in-tree — rebase against the merged ABC). |
+| [#665](https://github.com/MemPalace/mempalace/pull/665) Postgres (`pg_sorted_heap`) | Open | The in-tree `pgvector` backend (#1679) now occupies the basic Postgres slot; #665 rebases as the optional `pg_sorted_heap` performance variant. Decouple embedder; adopt the merged `PalaceRef`. Its session-level advisory-lock fix for the lazy-index concurrency wedge is the production basis for §7.3's observable `run_maintenance`. |
+| [#700](https://github.com/MemPalace/mempalace/pull/700) Qdrant | Open | Largely superseded by the in-tree `qdrant` backend (#1679), which already uses the §7.4 canonical namespace. Reconcile or close in favor of the merged backend. |
+| [#381](https://github.com/MemPalace/mempalace/pull/381) Qdrant (older) | Open | Same as #700; subclass the merged `BaseCollection` rather than a bare `Protocol`, or close as superseded. |
+| [#643](https://github.com/MemPalace/mempalace/pull/643) PalaceStore | Closed | POC; the parametrized-test approach it explored became the standard (§7.2). |
+| [#697](https://github.com/MemPalace/mempalace/pull/697) Chroma HttpClient + prefix | Closed | `collection_prefix` dissolved into `PalaceRef.namespace` + `supports_namespace_isolation` (§4.4), as predicted. |
+
+---
+
+## 12. Resolved decisions
+
+The three questions that were open at draft are resolved for v1, informed by the backend-author reviews:
+
+- **`changes_since` accepts a collection filter — yes.** It takes an optional `collection_name` so a sync caller can request changes for one collection (LanceDB and others already track changes per table; filtering by collection is natural). Additive on the §6 optional hook.
+- **Per-palace capabilities — no; capabilities stay static/class-level.** Making `supports_*` palace-dependent turns every capability check into "maybe — depends," pushing state tracking onto every caller. A backend that builds indexes lazily either guarantees the index exists when the capability is needed (and advertises it) or doesn't advertise it. The one isolation-by-deployment case (`supports_contains_fast` where an FTS index may or may not exist) is handled by the backend either guaranteeing the floor or omitting the token — not by per-palace variance. (Note: `supports_namespace_isolation` is still a static, class-level token — it declares what the backend *enforces*, not a per-palace fact.)
+- **`run_maintenance(kind)` returns a structured, observable result — resolved in §7.3.** It is not fire-and-forget: it returns `MaintenanceResult` and must serialize concurrent same-kind calls (the production lazy-index concurrency wedge is why). See §7.3.
+
+No remaining blockers. Genuinely additive future items (new optional methods, new tokens) land under the minor-version rule (§9).
+
+---
+
+## 13. Rollout
+
+The original draft sequenced cleanup → spec → ChromaBackend → in-flight rebase → migrate CLI. In practice the contract surface and the first backends shipped together in [#1679](https://github.com/MemPalace/mempalace/pull/1679) ahead of this merge, so the remaining sequence is what's left, not the whole list.
+
+**Done (via #1679):**
+1. ✅ §10 seam cleanup (client imports routed through `BaseCollection`; typed-result migration §1.3).
+2. ✅ `BaseBackend` / `BaseCollection`, `PalaceRef`, capability tokens, entry-point/registry discovery.
+3. ✅ In-tree `pgvector`, `qdrant`, `sqlite_exact` backends + `tests/test_backend_conformance.py`.
+4. ✅ Canonical §7.4 namespace pinned to the shipped value.
+
+**Remaining (tracked as follow-up issues):**
+5. **Embedder-identity contract (§1.5 / §5)** — [#1724](https://github.com/MemPalace/mempalace/issues/1724). Land the minimal `Embedder` protocol as a real injected dependency; persist + check `model_name` per the three-state model. *Hard dependency for §1.5 conformance.*
+6. **Maintenance hooks (§7.3)** — [#1725](https://github.com/MemPalace/mempalace/issues/1725). Implement `maintenance_kinds` / `maintenance_state()` / observable `run_maintenance()` in the in-tree backends; wire the §7.3 advisory-lock serialization (`pg_sorted_heap` / #665 has the reference fix).
+7. **`searcher.py` metric-awareness + lexical fallback (§10)** — [#1726](https://github.com/MemPalace/mempalace/issues/1726). Make `_hybrid_rank` read `distance_metric`; route the BM25 fallback through `BaseCollection` instead of `chroma.sqlite3`.
+8. Residual §10 items: backend-neutral not-found exception; migrate `mcp_server._get_client()` caching into `ChromaBackend`.
+9. Finish the `mempalace migrate` CLI (§8) and the parametrized core suite gate (§7.2, `MEMPALACE_TEST_ALL_BACKENDS=1` in CI).
+10. Rebase / reconcile the remaining community backend PRs (§11) against the merged contract.
+11. Update [ROADMAP.md](../../ROADMAP.md) with spec v1.0 adoption under v4.0.0-alpha.

docs/rfcs/003-agent-logstream-coordination.md

@@ -0,0 +1,505 @@
+# RFC 003: Agent Logstream Coordination
+
+Status: Implemented (phases 1-5). Phase 5 SSE shipped as GET /logstream/stream on the hub HTTP transport (bearer-authenticated, event_list filter set, since_event_id/Last-Event-ID resume, 15s heartbeats, bounded clients); logstream tools additionally dispatch outside the global HTTP request lock so long-polls cannot starve the hub. Server-side cursor storage and drawer compaction remain future work.
+Owner: Claude Fable 5
+Created: 2026-07-01
+Branch: `feat/shared-brain-dogfood`
+
+## Summary
+
+MemPalace should become a local-first coordination substrate for agents, not only a long-term recall system. Agents running on different machines should be able to delegate work, wait for replies, exchange patches or other artifacts, and subscribe to realtime coordination messages through the shared MemPalace hub.
+
+This RFC proposes **MemPalace Logstream**: a small append-only event layer served by the existing MemPalace MCP HTTP hub. It preserves the core MemPalace promise of durable, local, exact storage while adding coordination primitives agents can use without Igor manually relaying messages between them.
+
+## Motivation
+
+The shared-brain dogfood exposed a real workflow gap:
+
+1. Mac Codex wrote a delegation packet into `shared_agent_brain/delegation`.
+2. Windows Codex found it, did the work, and wrote back status.
+3. Windows Codex forgot to push the code.
+4. Mac Codex could read the status but could not receive the actual patch through a formal channel.
+
+The memory layer worked. The coordination layer was implicit and fragile.
+
+We need a first-class way for agents to:
+
+- create durable work packets;
+- subscribe or wait for matching replies;
+- submit patch/file artifacts before they are committed;
+- acknowledge, apply, reject, or supersede handoffs;
+- keep enough structured metadata to avoid ambiguity across machines, branches, and worktrees.
+
+## Non-Goals
+
+- Do not build a cloud service.
+- Do not require external APIs, hosted queues, Redis, Postgres, or SaaS infra.
+- Do not replace Git for final source control.
+- Do not make agents apply patches silently without explicit caller intent.
+- Do not weaken verbatim drawer storage. Event payloads and artifacts must remain exact.
+- Do not add telemetry or phone-home behavior.
+
+## Design Principles
+
+- **Local-first**: the event bus runs inside the existing MemPalace hub and is reachable over loopback, LAN, or tailnet according to the user's deployment.
+- **Append-only by default**: events are immutable. Corrections are new events that reference prior events.
+- **Exact payloads**: event bodies and artifacts are stored verbatim.
+- **Structured envelopes**: agents should not infer core routing fields from prose.
+- **Durable before realtime**: every realtime message must also be recoverable after reconnect.
+- **Small useful v1**: favor long-poll and polling over a large streaming subsystem if that ships faster.
+- **No hidden authority**: patch application and destructive actions must be explicit local operations.
+
+## Core Concepts
+
+### Event
+
+An event is a structured coordination message. It has routing metadata plus an optional verbatim body.
+
+Required fields:
+
+- `id`: stable event id, generated by the server.
+- `type`: event type, such as `task.request`, `task.reply`, `patch.ready`, `patch.applied`.
+- `stream`: logical stream name. Suggested format: `project/<project-name>` or `shared_agent_brain`.
+- `room`: sub-channel inside a stream, such as `delegation`, `patches`, `reviews`, `status`.
+- `from_agent`: writer identity, supplied by caller.
+- `created_at`: server timestamp.
+
+Common optional fields:
+
+- `to_agent`: target agent or `*`.
+- `correlation_id`: task or conversation id tying request/reply events together.
+- `branch`: Git branch, when relevant.
+- `base_commit`: Git commit the work started from.
+- `status`: `open`, `claimed`, `ready`, `applied`, `blocked`, `failed`, `superseded`.
+- `artifact_ids`: patch/file artifact references.
+- `body`: verbatim human-readable content.
+
+### Artifact
+
+An artifact is exact content attached to an event. Examples:
+
+- unified diff patches;
+- generated files;
+- test logs;
+- benchmark reports;
+- screenshots encoded by reference, not necessarily inline;
+- structured JSON payloads.
+
+Artifacts must have:
+
+- `id`;
+- `kind`: `patch`, `file`, `log`, `json`, `note`;
+- `sha256`;
+- `size_bytes`;
+- `content`;
+- `created_at`;
+- `created_by`.
+
+### Cursor
+
+Each agent may maintain a cursor per stream so it can ask, "what happened since I last checked?"
+
+Cursor state can be server-side later. For v1, it is acceptable for the client to pass `since_event_id` or `since_created_at`.
+
+## Event Envelope
+
+Example event:
+
+```json
+{
+  "id": "evt_01J...",
+  "type": "patch.ready",
+  "stream": "project/mempalace",
+  "room": "patches",
+  "from_agent": "windows-codex",
+  "to_agent": "mac-codex",
+  "correlation_id": "task_01J...",
+  "branch": "feat/shared-brain-dogfood",
+  "base_commit": "2668053",
+  "status": "ready",
+  "artifact_ids": ["art_01J..."],
+  "body": "Search ranking patch is ready. Tests passed on Windows.",
+  "created_at": "2026-07-01T20:00:00Z"
+}
+```
+
+Example patch artifact:
+
+```json
+{
+  "id": "art_01J...",
+  "kind": "patch",
+  "sha256": "abc123...",
+  "size_bytes": 18422,
+  "created_by": "windows-codex",
+  "created_at": "2026-07-01T20:00:00Z",
+  "content": "diff --git a/mempalace/searcher.py b/mempalace/searcher.py\n..."
+}
+```
+
+## Storage Model
+
+Use SQLite in the active palace directory:
+
+- `logstream.sqlite3`
+
+Tables:
+
+```sql
+CREATE TABLE events (
+  id TEXT PRIMARY KEY,
+  type TEXT NOT NULL,
+  stream TEXT NOT NULL,
+  room TEXT NOT NULL,
+  from_agent TEXT NOT NULL,
+  to_agent TEXT,
+  correlation_id TEXT,
+  branch TEXT,
+  base_commit TEXT,
+  status TEXT,
+  body TEXT NOT NULL DEFAULT '',
+  created_at TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX events_stream_created_idx ON events(stream, created_at);
+CREATE INDEX events_correlation_idx ON events(correlation_id, created_at);
+CREATE INDEX events_to_agent_idx ON events(to_agent, created_at);
+CREATE INDEX events_type_idx ON events(type, created_at);
+
+CREATE TABLE artifacts (
+  id TEXT PRIMARY KEY,
+  kind TEXT NOT NULL,
+  sha256 TEXT NOT NULL,
+  size_bytes INTEGER NOT NULL,
+  content TEXT NOT NULL,
+  created_by TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX artifacts_sha256_idx ON artifacts(sha256);
+
+CREATE TABLE event_artifacts (
+  event_id TEXT NOT NULL,
+  artifact_id TEXT NOT NULL,
+  PRIMARY KEY (event_id, artifact_id)
+);
+```
+
+Rationale:
+
+- Keep coordination separate from Chroma/vector search for reliability and latency.
+- Avoid forcing realtime coordination through semantic search.
+- Make append/list/wait cheap and deterministic.
+- Optionally mirror summaries into normal drawers later for recall, but do not depend on it.
+
+## MCP Tool Surface
+
+### `mempalace_event_append`
+
+Append one event.
+
+Input:
+
+```json
+{
+  "type": "task.request",
+  "stream": "project/mempalace",
+  "room": "delegation",
+  "from_agent": "mac-codex",
+  "to_agent": "windows-codex",
+  "correlation_id": "task_...",
+  "branch": "feat/shared-brain-dogfood",
+  "base_commit": "2668053",
+  "status": "open",
+  "body": "Please fix search echo ranking.",
+  "metadata": {}
+}
+```
+
+Returns:
+
+```json
+{
+  "success": true,
+  "event": { "...": "..." }
+}
+```
+
+### `mempalace_event_list`
+
+List events with structured filters.
+
+Filters:
+
+- `stream`
+- `room`
+- `type`
+- `to_agent`
+- `from_agent`
+- `correlation_id`
+- `since_event_id`
+- `since_created_at`
+- `limit`
+
+Default limit: 50.
+
+### `mempalace_event_wait`
+
+Block until a matching event exists or timeout expires.
+
+Input:
+
+```json
+{
+  "stream": "project/mempalace",
+  "to_agent": "mac-codex",
+  "correlation_id": "task_...",
+  "type": "patch.ready",
+  "since_event_id": "evt_...",
+  "timeout_ms": 60000
+}
+```
+
+Implementation for v1:
+
+- simple polling loop inside the MCP request;
+- sleep interval 250-1000 ms with jitter;
+- max timeout 5 minutes;
+- return `{ "timed_out": true, "events": [] }` rather than error on timeout.
+
+`mempalace_event_wait` remains the polling MCP tool. Clients that can hold an
+HTTP stream should use `GET /logstream/stream` SSE for live tailing.
+
+### `mempalace_event_ack`
+
+Append an acknowledgement event.
+
+Input:
+
+```json
+{
+  "event_id": "evt_...",
+  "from_agent": "mac-codex",
+  "status": "applied",
+  "body": "Patch applied and tests passed."
+}
+```
+
+This should create a new `event.ack` event with `correlation_id` copied from the target event.
+
+### `mempalace_artifact_put`
+
+Store exact artifact content.
+
+Input:
+
+```json
+{
+  "kind": "patch",
+  "created_by": "windows-codex",
+  "content": "diff --git ...",
+  "metadata": {
+    "branch": "feat/shared-brain-dogfood",
+    "base_commit": "2668053"
+  }
+}
+```
+
+Returns `id`, `sha256`, `size_bytes`.
+
+### `mempalace_artifact_get`
+
+Fetch an artifact by id. Returns exact content and metadata.
+
+### `mempalace_patch_submit`
+
+Convenience wrapper:
+
+1. store patch artifact;
+2. append `patch.ready` event referencing the artifact;
+3. return both ids.
+
+This is optional for v1 if `artifact_put` + `event_append` are enough.
+
+## HTTP Endpoint Surface
+
+MCP tools are sufficient for v1. Add HTTP streaming later.
+
+Future endpoints:
+
+- `GET /logstream/events?stream=...&since=...`
+- `GET /logstream/stream?stream=...` using Server-Sent Events.
+
+Do not add unauthenticated logstream endpoints. They expose work metadata and patch contents.
+
+## Agent Workflow Examples
+
+### Delegation With Wait
+
+Mac Codex:
+
+1. `mempalace_event_append(type=task.request, to_agent=windows-codex, correlation_id=task_123)`
+2. `mempalace_event_wait(correlation_id=task_123, type=patch.ready, to_agent=mac-codex, timeout_ms=300000)`
+3. `mempalace_artifact_get(artifact_id)`
+4. Apply patch locally.
+5. Run tests.
+6. `mempalace_event_ack(status=applied)`.
+
+Windows Codex:
+
+1. `mempalace_event_wait(to_agent=windows-codex, type=task.request, timeout_ms=300000)`
+2. Do work.
+3. `mempalace_patch_submit(type=patch.ready, to_agent=mac-codex, correlation_id=task_123)`.
+
+### Status-Only Handoff
+
+If an agent cannot produce a patch, it can still append `task.reply` or `task.blocked` with exact notes and evidence.
+
+## Implementation Plan
+
+### Phase 1: Durable Logstream Core
+
+Add:
+
+- `mempalace/logstream.py`
+- SQLite schema creation/migration inside the active palace dir.
+- Pure Python API:
+  - `append_event(...)`
+  - `list_events(...)`
+  - `wait_events(...)`
+  - `put_artifact(...)`
+  - `get_artifact(...)`
+  - `ack_event(...)`
+
+Requirements:
+
+- no Chroma dependency;
+- no vector index open;
+- safe under concurrent HTTP requests;
+- append-only events;
+- validate/sanitize string fields with existing config helpers where appropriate;
+- cap body/artifact sizes with explicit errors.
+
+Suggested limits:
+
+- event body: 256 KiB default;
+- artifact content: 4 MiB default;
+- wait timeout: 300 seconds max.
+
+### Phase 2: MCP Tools
+
+Add tools in `mempalace/mcp_server.py`:
+
+- `mempalace_event_append`
+- `mempalace_event_list`
+- `mempalace_event_wait`
+- `mempalace_event_ack`
+- `mempalace_artifact_put`
+- `mempalace_artifact_get`
+- optionally `mempalace_patch_submit`
+
+Read-only mode:
+
+- `event_list`, `event_wait`, and `artifact_get` are allowed.
+- `event_append`, `event_ack`, `artifact_put`, and `patch_submit` are mutating tools and must be hidden/refused in read-only mode.
+
+SQLite integrity gate:
+
+- Logstream should not depend on Chroma SQLite integrity.
+- However, if the active palace path is unsafe/missing, fail clearly.
+
+### Phase 3: CLI Utilities
+
+Add:
+
+- `mempalace logstream append`
+- `mempalace logstream list`
+- `mempalace logstream wait`
+- `mempalace artifact put`
+- `mempalace artifact get`
+
+Keep CLI output scriptable with `--json`.
+
+### Phase 4: Docs and Skills
+
+Docs:
+
+- `website/concepts/agent-logstream.md`
+- `website/reference/mcp-tools.md` entries.
+- Update `website/guide/remote-server.md` to mention agent coordination over the hub.
+
+Agent skills/prompts:
+
+- Update MemPalace recall/search skills to mention logstream for active coordination.
+- Add a brief "when delegating work, use event_append/event_wait" protocol.
+
+### Phase 5: Realtime Stream
+
+Only after Phase 1-4 are solid:
+
+- add SSE endpoint;
+- per-agent cursor helpers;
+- maybe background compaction of old event payloads into normal drawers for long-term recall.
+
+## Test Plan
+
+Unit tests:
+
+- schema initializes in empty palace dir;
+- append/list round trip;
+- filters by stream, room, type, to_agent, correlation_id;
+- wait returns immediately when event already exists;
+- wait times out cleanly;
+- artifact put/get preserves exact bytes/text;
+- artifact hash is stable;
+- ack creates new event and does not mutate target event;
+- read-only mode hides/refuses mutating tools;
+- size limits reject oversized event bodies/artifacts.
+
+Integration tests:
+
+- two threads: one waits, one appends, waiter returns matching event;
+- patch submit stores artifact and event atomically enough that readers never see a dangling artifact id;
+- MCP HTTP server can append from one client and wait/list from another;
+- no auth headers/request bodies appear in `/statusz`.
+
+Dogfood test:
+
+1. Mac agent creates task request for Windows.
+2. Windows agent waits for it.
+3. Windows agent submits patch artifact.
+4. Mac agent waits, gets patch, verifies hash, applies patch manually, runs tests.
+5. Mac agent acknowledges `applied`.
+
+## Open Questions
+
+- Should logstream events also be mirrored into Chroma drawers for semantic recall, or should structured search remain the primary access path?
+- Should artifact content support binary data in v1, or should v1 be UTF-8 text only?
+- Should agent identities be free-form strings initially, or tied to configured local agent profiles?
+- Should `event_wait` be allowed over stdio MCP clients that may have shorter host timeouts?
+- Should events have retention/archival controls, or stay forever by default like memory?
+
+## Claude Fable 5 Instructions
+
+Please implement Phase 1 and Phase 2 first. Keep the patch narrow and avoid unrelated refactors.
+
+Start by creating `mempalace/logstream.py` with the durable SQLite core and focused tests. Then expose MCP tools in `mempalace/mcp_server.py` with read-only and mutating-tool semantics. Only add CLI/docs after the core and MCP tools are green.
+
+Before finishing:
+
+- run focused logstream tests;
+- run relevant MCP server tests;
+- run `ruff check` on touched Python files;
+- write a MemPalace delegation reply with the implementation status, changed files, and exact verification commands.
+
+## Implementation Notes
+
+Windows dogfood verification by `windows-codex` on 2026-07-02:
+
+- Branch/base: `feat/shared-brain-dogfood` at `1ff3125`.
+- OS: Microsoft Windows 11 Pro Insider Preview 10.0.29576 build 29576 64-bit.
+- Python: 3.12.11 via `uv run python`.
+- Command: `uv run pytest tests/test_logstream.py tests/test_mcp_logstream.py tests/test_cli_logstream.py -q`.
+- Result: 72 passed in 2.93s.

docs/rfcs/004-replicated-palace.md

@@ -0,0 +1,630 @@
+# RFC 004: The Replicated Palace
+
+Status: Draft complete — all sections drafted (storage: mac-claude; transport/lifecycle/appendix: windows-claude); awaiting Igor's review
+
+Shipped so far: step 0 (logstream multi-master replication) and step 1 (memory read replicas — snapshot pull, local fold, distributed embedding), plus the transport seam and the estate endpoint. Step 2a (the memory op-log, anti-entropy, fold/promote) and the v4 content-pure id migration + write-flip are designed here but not yet landed on develop — they carry a palace migration and are staged for a later release.
+Owners: mac-claude (storage layers, §6–§9), windows-claude (transport & lifecycle, §5 and Appendix A), decided by Igor
+Created: 2026-07-02
+Branch: `feat/shared-brain-dogfood`
+Prior art: RFC 003 (logstream), the `rfc004_replicated_palace_position` correlation thread (position debate, verbatim in the logstream)
+
+## Summary
+
+Each human has ONE palace — an extension of their brain — replicated in full
+across every machine they own. Agents always talk to the MemPalace service on
+`127.0.0.1`; services converge with each other over an encrypted mesh. The
+hub stops being a dependency and becomes a role (rendezvous, and the home of
+*shared* palaces). The design is judged against offline operation as the
+default posture, not as an edge case.
+
+One sentence: **N equal replicas of the facts, each with locally-derived
+indexes and a local writer, converging through provenance-stamped ops over
+the mesh — with origin as the home-of-record for source-bound maintenance.**
+
+## Motivation
+
+The shared-brain dogfood proved the hub topology works — and watched it fail:
+when the hub machine slept, every other machine lost recall, capture, and
+coordination simultaneously. A brain does not stop remembering because
+another brain is asleep. Concretely, today a remote machine without the hub
+has **no palace at all**.
+
+The mission statement ("memory is identity") implies the requirement
+directly: you do not rent your identity, and you do not park it on a single
+machine.
+
+## Requirements
+
+The availability invariant, and the offline requirements distilled from the
+fleet (logstream correlation `rfc004_replicated_palace_position`):
+
+- **R0 — Mission invariants hold everywhere**: verbatim always, local-first,
+  zero external API for core operations, hooks < 500 ms, startup injection
+  < 100 ms. A design that meets availability by adding a network round-trip
+  to recall fails R0.
+- **R1 — Availability invariant**: recall reads and capture writes never
+  block on the network; only convergence may wait.
+- **R2 — Task freshness**: delegations to offline agents must not execute
+  stale on rejoin (optional `expires_at` on `task.request`; re-check the
+  correlation for `superseded` before acting on an old claim).
+- **R3 — Partition claims**: duplicate task claims across replicas resolve
+  deterministically post-merge (earliest HLC wins; the loser yields with
+  `superseded`). Append-only makes double work safe-but-wasteful, never
+  corrupting.
+- **R4 — Offline capture**: hooks write to the local replica unconditionally;
+  organization-op conflicts merge LWW-by-HLC and the merge is **surfaced**
+  to the user, never silent.
+- **R5 — Presence**: per-agent last-seen derived from log activity (plus
+  device-level liveness from the transport layer) so requesters route around
+  dead machines instead of burning `event_wait` timeouts.
+- **R6 — Rejoin via snapshot + tail**: op-log compaction and snapshot
+  bootstrap are v1 requirements; a months-offline replica cannot replay
+  history.
+- **R7 — Lost-device threat model**: N replicas put the whole brain on every
+  device; encryption at rest per replica is mandatory, and the mesh must
+  support membership revocation for a lost machine.
+- **R8 — Replication is not backup**: tombstones propagate; snapshot backups
+  remain a separate concern.
+
+## Non-Goals
+
+- **Replacing the hub for shared palaces.** Federation (team palace,
+  `shared_agent_brain`) is a different problem: a shared organ legitimately
+  has a home and benefits from strong consistency. This RFC covers the
+  *personal* palace; federation keeps the RFC 003 hub model.
+- **Cloud as a system of record** — constitutionally excluded. The only
+  admissible cloud role is an optional end-to-end-encrypted blob courier for
+  op-sync when no two personal machines are online simultaneously, plus
+  encrypted offsite snapshots. Zero knowledge; never queryable server-side.
+- **Thin/partial replicas (phone-class devices) in v1.** They remain remote
+  clients of a nearby full replica; partial replication is future work.
+- **Multi-user merge and authorization semantics.** One human, N devices.
+  Trust boundary = palace boundary. Mesh membership is full-replica trust,
+  not family/team scoped read or write permission.
+
+## Architecture Overview
+
+```
+  Machine A (mac)                Machine B (windows)          Machine C (laptop)
+  ┌─────────────────────┐        ┌─────────────────────┐      ┌──────────────────┐
+  │ agents → 127.0.0.1  │        │ agents → 127.0.0.1  │      │ agents → local   │
+  │ ┌─────────────────┐ │  ops   │ ┌─────────────────┐ │ ops  │ ┌──────────────┐ │
+  │ │ mempalace svc   │◀┼───────▶│ │ mempalace svc   │◀┼─────▶│ │ mempalace svc│ │
+  │ │  op-log (SoT)   │ │  mesh  │ │  op-log (SoT)   │ │ mesh │ │  op-log (SoT)│ │
+  │ │  derived index  │ │        │ │  derived index  │ │      │ │ derived index│ │
+  │ └─────────────────┘ │        │ └─────────────────┘ │      │ └──────────────┘ │
+  └─────────────────────┘        └─────────────────────┘      └──────────────────┘
+         encrypted mesh transport (Layer 1) · anti-entropy op sync (Layer 2)
+```
+
+Three layers, separable by design:
+
+1. **Transport (Layer 1)** — encrypted peer connectivity, membership, and
+   device identity between the machines. Owner: windows-claude (§5).
+2. **Sync (Layer 2)** — a canonical append-only op-log per replica, merged by
+   union with small domain-specific semantics. Owner: mac-claude (§6).
+3. **Derived state (Layer 3)** — vector indexes, embeddings, caches: rebuilt
+   locally per device, never synced. Owner: mac-claude (§7).
+
+The decisive property: **sync the facts, derive the senses.** Ops are
+kilobytes; HNSW graphs are gigabytes. Every machine remembers everything;
+each machine senses with its own hardware.
+
+## Alternatives Considered
+
+| Dimension | Replicated mesh | Self-hosted central server | Cloud backend |
+|---|---|---|---|
+| Offline / partition | Full function, converge later | Dead when server/link down | Dead without internet |
+| Recall latency | Local, sub-ms–ms | LAN ms / tailnet 10–30 ms | 50–150 ms+ |
+| Privacy | Never leaves your devices | Your hardware, one exposed box | Provider sees plaintext or E2EE cripples search |
+| Durability | N live replicas (+R8 backups) | One box | Best-in-class |
+| Ops burden on user | ~zero if software earns it | Forever (patching, TLS, backups) | ~zero |
+| Engineering complexity | High, paid once by us | ~zero (exists today) | Zero for user, trust cost |
+| Consistency | Eventual + merge semantics | Strong, trivially | Strong |
+| Teams / sharing | Wrong tool | Natural | Natural |
+| Exit / lock-in | SQLite files you hold | Files you hold | Provider's mercy |
+
+Verdicts: the mesh is the only option satisfying R0+R1 for the personal
+palace (a cloud round-trip spends the entire hook budget on network; central
+fails the availability invariant we watched fail in production). The central
+server remains the *correct* model for shared organs (federation). Cloud is
+admissible only as the E2EE courier of the Non-Goals section.
+
+## §5. Layer 1: Transport
+
+Layer 1 exists so Layer 2 never has to think about networks, keys, or which
+machines are awake. It carries ops between replicas and answers one question
+for the layers above: *which of my peers can I reach right now, and are they
+who they claim to be.* Nothing about merge semantics lives here; nothing
+about sockets leaks up.
+
+### 5.1 The seam (what Layer 2 is allowed to assume)
+
+Layer 2 sees peers only through this interface — the entire contract:
+
+```
+Transport {
+  self(): ReplicaId                      // stable, = mesh node identity
+  peers(): Map<ReplicaId, Presence>      // membership + liveness snapshot
+  onPresenceChange(cb)                   // SWIM/heartbeat deltas → R5
+
+  request(peer, path, body): Response    // one authenticated round-trip
+  openStream(peer, path): EventStream    // long-lived, resumable by cursor
+  onInbound(path, handler)               // serve anti-entropy pulls
+}
+```
+
+Two channel shapes cover every need: **request** for anti-entropy pulls
+(`GET /sync/ops?origin=X&after=N`, §6.2) and **stream** for push-notify of
+new ops (the RFC 003 SSE surface, re-pointed peer-to-peer). Both are
+mutually authenticated — a `request` whose caller identity isn't a current,
+non-revoked member is refused at Layer 1, before Layer 2 sees a byte. That
+refusal is the replication ACL (R7) and it is the *only* authorization Layer
+2 relies on: an op that arrived is an op from a trusted replica.
+
+This is deliberately a v1 device-trust boundary, not a user/role permission
+model. Admitting a replica means admitting it to the entire personal palace,
+verbatim content included. Delegated, family, team, or other partial-access
+workflows need a separate palace-level capability layer; they must not be
+modeled as "just add this device to the replication mesh."
+
+`ReplicaId` is the transport's node identity — no separate replica registry
+to drift. Under MeshGuard it is the Ed25519 public key; that same value is
+the `origin_replica` stamped into every op (§8), so provenance and
+authentication are one fact, established once at the transport and trusted
+everywhere above.
+
+### 5.2 Peer addressing & rendezvous
+
+Replicas are named by identity, never by address — a laptop's IP changes
+between café and home; its `ReplicaId` does not. Address discovery is the
+transport's job:
+
+- **MeshGuard / Tailscale**: the mesh's own coordination plane maps identity
+  → current endpoint(s); we never hardcode IPs. A replica "moves" and peers
+  re-resolve transparently.
+- **Bare LAN** (no mesh): mDNS/`_mempalace._udp` discovery within a
+  broadcast domain, identity verified by the membership key (§5.5) — an
+  address hint is never a trust grant.
+- **No global directory.** Rendezvous is peer-to-peer within the mesh
+  membership set. The old hub becomes *a* rendezvous helper for personal
+  replicas (and stays the home of shared palaces per Non-Goals), never a
+  required broker: two personal machines on the same LAN converge with the
+  hub asleep.
+
+### 5.3 NAT traversal posture
+
+Convergence must survive both machines being behind NAT on different
+networks (R1 across the internet, not just the LAN):
+
+1. **Direct** when a route exists (same LAN, or one side reachable).
+2. **Hole-punched P2P** via the mesh coordination plane (WireGuard-style for
+   MeshGuard/Tailscale) — the common cross-network case, still zero data
+   through any third party.
+3. **Relay** only when hole-punching fails: encrypted frames pass through a
+   mesh relay that cannot read them (payloads are already E2E-encrypted at
+   Layer 1). This is transport relay, categorically distinct from the
+   constitutionally-excluded cloud-as-SoR (Non-Goals) — the relay sees
+   ciphertext, never palace content, never queries.
+4. **Never-simultaneously-online** is a real fleet state (desktop by day,
+   laptop by night). Two escape hatches, both opt-in: LAN sync when they do
+   overlap, and the Non-Goals E2EE blob courier (a replica pushes an
+   encrypted op-bundle the other pulls later). The courier is a §5 consumer
+   of the same op ranges, not a new path.
+
+### 5.4 Connection lifecycle
+
+- **Discover → authenticate → sync-on-connect → tail.** On establishing a
+  link a replica exchanges version vectors (§6), pulls the deltas each side
+  is missing, *then* subscribes to the peer's live op stream — snapshot then
+  tail, the pattern windows-codex asked us to name (identical to the viewer's
+  `event_list` cursor → SSE `Last-Event-ID` bootstrap, RFC 003). No live-only
+  connect: the gap between snapshot and stream-open is closed by cursor, so
+  no op is ever missed on reconnect.
+- **Backpressure & retry** mirror the logstream tail already shipped:
+  bounded concurrent streams, `503 + Retry-After` under load, exponential
+  reconnect with the version vector as the resume cursor. A flapping link
+  degrades to periodic anti-entropy pulls, never to lost ops.
+- **Idempotent by construction.** Re-delivering an op is a no-op (union
+  merge, §6.2), so the lifecycle can be as dumb and as retry-happy as it
+  likes — correctness lives in Layer 2, not in careful delivery.
+
+### 5.5 Replica join / leave / revoke ceremony
+
+Membership *is* trust; there is no other authorization in the system.
+
+- **Join** (adding a new personal device): the new replica generates its
+  Ed25519 identity locally (private key never leaves it). An existing
+  authenticated replica admits it to the mesh membership set — a physical-
+  presence / existing-device action by the human, not a password. First sync
+  is a full snapshot + tail (§6, R6). Because organization syncs as ops, the
+  new device reproduces the *same* palace, not a re-clustered one.
+- **Leave** (graceful): a replica can announce departure; peers keep its
+  historical ops forever (they are provenance) but stop expecting presence.
+- **Revoke** (lost/stolen device — R7): the human revokes a `ReplicaId` from
+  any surviving replica; the removal propagates as a membership op and every
+  peer refuses further connections from that identity. Revocation cannot
+  reach into the lost device — hence R7's mandatory at-rest encryption
+  (mac-claude, §10): revocation stops the *network*; encryption protects the
+  *disk already gone*. Ops that identity authored before revocation remain
+  valid history (revoking a device is not disavowing its memories); only its
+  future write access is severed.
+
+### 5.6 Key custody
+
+- **Device key**: Ed25519 private key per replica, generated on-device,
+  non-exportable, OS keystore where available (Keychain / DPAPI / kernel
+  keyring). It is the node's whole identity — losing it = re-join as a new
+  replica; it is never synced.
+- **Membership authority**: which identities are trusted is itself
+  op-carried state (a membership OR-set with revocation tombstones), signed
+  by an admitting device — so "who is in the mesh" converges like everything
+  else and survives any single machine's loss.
+- **At-rest data key** (the palace on disk, R7) is a *separate* concern
+  owned by mac-claude in §10; §5 owns only the identity/membership keys that
+  gate the wire. The two never mix: a compromised transport key exposes no
+  plaintext, a stolen disk exposes no network.
+
+### 5.7 Transport fallback matrix
+
+Per-link, best available wins; the seam (§5.1) makes the choice invisible
+to Layer 2.
+
+| Situation | Transport | Notes |
+|---|---|---|
+| Same LAN, mesh up | MeshGuard direct | Lowest latency; hub not involved |
+| Cross-network, both online | MeshGuard hole-punched (Tailscale bridge if MeshGuard not yet integrated) | E2E, P2P |
+| Hole-punch fails | Encrypted mesh relay | Ciphertext only; distinct from cloud-SoR |
+| Mesh unavailable, same LAN | Bare LAN + mDNS, membership-key auth | Degraded discovery, full trust |
+| Never simultaneously online | E2EE blob courier (Non-Goals) | Async op-bundle exchange |
+| MeshGuard pre-integration | Tailscale (today's dogfood transport) | Ship Sequencing step 0–1 before MeshGuard lands |
+
+MeshGuard is the target; **Tailscale is the shipping fallback that unblocks
+Sequencing steps 0–1 today** (it already carries this very logstream). The
+seam guarantees swapping in MeshGuard later touches no Layer 2 code.
+
+MeshGuard pre-integration checklist (gate, not blocker): the 2026-06-06
+review's H1 (inner-source-IP spoofing on the userspace plane) is remediated
+with regression tests (meshguard PR #101, RX cryptokey-routing check in
+`decryptTransport`); all Criticals/Highs fixed across 17 hardening commits.
+Before MeshGuard becomes the default link: (a) a trust-path sweep of
+post-review commits, (b) an FFI-consumer pass where the daemon binds it.
+Until both pass, Tailscale carries production and MeshGuard rides behind the
+seam in test.
+
+### 5.8 Failure detection → presence (R5)
+
+Presence has two sources; the transport fuses them so the layers above ask
+one question:
+
+- **Device liveness** (Layer 1): SWIM-style failure detection from the mesh
+  (MeshGuard membership; heartbeat pings on bare LAN). Answers "is the
+  machine reachable *now*," sub-second, and drives reconnect.
+- **Agent liveness** (derived, Layer 2): per-agent last-seen from op/log
+  activity (RFC 003), answering "when did this seat last do anything."
+
+`Presence = { reachable: bool (device), lastSeen: hlc (agent), replicaId }`.
+Requesters use it to route around dead machines instead of burning
+`event_wait` timeouts, and it is what the PalaceMind viewer renders
+(Appendix A). This is also the substrate for **R2 (task freshness)** and
+**R3 (partition claims)** on the wire:
+
+- **R2**: a `task.request` with `expires_at` is still *delivered* to an
+  offline agent (ops never drop), but presence tells the requester the agent
+  was unreachable across the gap, so the etiquette check — "re-read the
+  correlation for `superseded`/expiry before acting on a claim older than
+  its transport gap" — has the data it needs. Layer 1 supplies the gap;
+  Layer 2 supplies the rule.
+- **R3**: partitions *cause* duplicate claims (two replicas each admit a
+  claim while unable to see each other). Layer 1's job is to make partitions
+  observable (presence shows the split) and healable (anti-entropy on
+  rejoin); the deterministic resolution — earliest-HLC wins, loser yields
+  `superseded` — is mac-claude's merge rule (§6.2). Presence makes the
+  window small; the merge rule makes the outcome safe.
+
+## §6. Layer 2: The Canonical Op-Log
+
+### 6.1 The op envelope
+
+Every mutation of the palace becomes an immutable op:
+
+```json
+{
+  "op_id": "op_<origin>_<counter>",
+  "origin_replica": "<transport identity of the writing machine>",
+  "author_agent": "mac-claude",
+  "hlc": "0189f3a2-0007-mac",
+  "authored_at": "2026-07-02T21:14:09Z",
+  "kind": "drawer.add | drawer.revise | drawer.tombstone | org.file | org.move | org.tunnel.add | org.tunnel.remove | kg.assert | kg.close | kg.entity.upsert | registry.entity.upsert | event.append | artifact.put | ...",
+  "payload": { "...kind-specific, verbatim content inline or by sha256..." }
+}
+```
+
+- `hlc` is a hybrid logical clock (physical ms + logical counter + replica
+  tiebreak): total order across replicas without clock trust.
+- Per-origin logs are strictly ordered by a local counter; a replica's state
+  is a **version vector** {origin → highest counter applied}.
+- Storage: `oplog.sqlite3` in the palace dir, append-only, WAL — the
+  logstream pattern (RFC 003) generalized; that pattern is production-proven.
+
+### 6.2 Merge semantics (complete list — nothing else exists)
+
+| State | Op kinds | Merge rule |
+|---|---|---|
+| Drawer content | add / revise / tombstone | Grow-only set of content-addressed revisions; head = latest by HLC; tombstone hides, never deletes (verbatim survives) |
+| Artifacts | artifact.put | True G-set, union by sha256 — conflicts impossible |
+| Organization | org.file / org.move / org.tunnel.* | LWW-by-HLC register per drawer (placement) / OR-set (tunnels); merges surfaced to user (R4) |
+| Knowledge graph | kg.assert / kg.close / kg.entity.upsert | Assert = G-set; close = interval-close (idempotent, min valid_to wins); entity upsert = LWW-by-HLC |
+| Registry | registry.entity.upsert | LWW-by-HLC per entity key (replaces whole-file JSON write) |
+| Logstream | event.append | Append-only union; cross-replica order by HLC; per-origin `seq` preserved; consumer contract additive (`origin_replica`, `hlc` are new fields) |
+| Diary | drawer.add in diary rooms | Same as drawer content (already append-only) |
+
+`org.tunnel.remove` is the remove half of the tunnel OR-set: it hides the edge
+from the current organization view, not from history. If MemPalace adopts a
+first-class dormant-tunnel product state, ship it as an explicit state op
+(`org.tunnel.set_state(active|dormant)` or equivalent) rather than overloading
+remove with dormancy semantics.
+
+Anti-entropy: peers exchange version vectors and pull missing per-origin
+ranges (`GET /sync/ops?origin=X&after=N`), push-notified over the existing
+SSE channel. No broker, no framework: automerge/yjs are document-CRDTs
+(wrong shape, heavy deps); cr-sqlite is a native extension whose generic
+table-CRDTs know nothing of id purity or verbatim; file-level sync of live
+SQLite corrupts. The merge logic above is ~hundreds of lines we fully own.
+
+Future domain-specific state needs the same explicit treatment before it
+enters Layer 2. Closet/card semantics, succession edges, contradiction
+intervals, or other semantic structures are not automatically drawer
+placement registers. Each adopted state must name its own op kinds, conflict
+surface, and any merge-exempt or intentionally non-LWW transitions; otherwise
+replicas can silently erase meaning while still "converging."
+
+### 6.3 Id purity (prerequisite, not footnote)
+
+Verified against the code (2026-07-02): today's drawer identities are NOT
+content-addressed. Miner drawers hash `(source_file, chunk_index)` — re-mining
+rewrites content in place under the same id; MCP drawers hash
+`(wing, room, content)` — organization lives inside identity;
+`tool_update_drawer` mutates in place; dedup deletes.
+
+**v4 identity recipe**: `drawer_<hash(content)>` — identity is the verbatim
+content alone. Organization (wing/room) becomes op-carried metadata
+(`org.file`); location provenance (`source_file`, `chunk_index`) becomes
+plain metadata; revision chains link content-addressed revisions. `ids.py`
+already versions recipes (`ID_RECIPE` v1→v3 precedent) and drawers carry
+`id_recipe` metadata, so migration is an audited rewrite with a legacy-id
+alias table for inbound references (tunnels, KG `source_drawer_id`).
+
+### 6.4 Mutable-state inventory → op mapping (verified, with file refs)
+
+| Today (mutable) | Where | Becomes |
+|---|---|---|
+| `tool_update_drawer` in-place update/upsert | mcp_server.py | `drawer.revise` (new content-addressed revision) |
+| Miner re-mine upsert over same id | miner.py:1336,1478 | `drawer.revise` at origin replica only (§8) |
+| `delete_drawer` / `delete_by_source` / dedup batch delete | dedup.py:127 | `drawer.tombstone` (hide, never destroy) |
+| Entity registry whole-file `json.dumps` | entity_registry.py:328 | `registry.entity.upsert` op stream |
+| `hallways.json` whole-file rewrite | hallways.py:140 | `org.tunnel.add/remove` OR-set ops |
+| KG `invalidate` UPDATE of valid_to; entities INSERT OR REPLACE | knowledge_graph.py | `kg.close` interval op; `kg.entity.upsert` |
+| repair / migrate / dedup wholesale rewrites | repair.py, migrate.py | replica-local maintenance of derived state (never synced) |
+
+Hardest today, cleanest after: the two whole-file JSONs (currently pure
+last-writer-wins with silent loss) gain real merge semantics for free.
+
+## §7. Layer 3: Derived State
+
+The vector store stops being the system of record — that single change
+dissolves the fleet-level writer lease, the stdio proxy's reason to exist,
+and the cross-machine integrity gates. Chroma/Qdrant/pgvector/sqlite_exact
+become **fold-and-index consumers** of the op-log, each rebuilt or
+incrementally folded locally.
+
+- Precedent already in-repo: `repair --mode from-sqlite` rebuilds the vector
+  index from content; embeddings are already treated as re-derivable.
+- Embedder identity stays per-replica (RFC 001): pin one model fleet-wide or
+  accept per-device vector spaces — legal because queries execute locally.
+- **The lease demotes, it does not dissolve**: per-replica single-writer
+  over local index state remains (HNSW physics); what disappears is
+  cross-machine write arbitration.
+- Organization is NOT derived state. Filing decisions sync as ops (§6.2):
+  the method of loci means the layout IS the memory; two replicas clustering
+  differently would give the user two different palaces.
+
+## §8. Provenance & Source-Bound Maintenance
+
+Every op carries `(origin_replica, author_agent, hlc, authored_at)` —
+simultaneously the sync unit, the conflict tiebreak, the audit trail, and
+the answer to "which machine/agent did this memory come from."
+
+Local references are replica-local: a drawer mined from `P:\...` on Windows
+references a path that exists only there. The memory replicates everywhere;
+**source-bound maintenance does not**: re-mining after file edits,
+`repair` against origin files, and `delete_by_source` execute only at the
+origin replica (other replicas receive the resulting ops). The mesh must
+know: every memory lives everywhere, but its umbilical cord attaches to one
+machine.
+
+## §9. Sequencing
+
+0. **Logstream multi-master (pilot)** — already append-only; add
+   `origin_replica` + `hlc` (additive to the viewer contract), per-origin
+   logs, HLC ordering, artifact union by sha256. Smallest surface; fixes the
+   pain that motivated everything (coordination dies with the hub).
+1. **Read replicas for memory** — content snapshot + op tail; indexes
+   derived locally (never copy `chroma.sqlite3` — that replicates Chroma's
+   fragility to N machines). Cheap availability for recall; builds the R6
+   snapshot machinery.
+2. **Canonical op-log + v4 id migration** — §6 in full; backends demoted to
+   derived consumers. Decided 2026-07-02 (Igor): ships as **2a**
+   (drawers + KG ops — the waiting customers: mining promotion and the
+   multi-writer foundation) followed by **2b** (registry + hallways/tunnels
+   op conversion); superseded revisions are kept **forever** (verbatim
+   maximalism — search surfaces head revisions only; no GC path exists);
+   the v4 migration runs **staged on a palace copy first**, validated, then
+   live with a timestamped backup and a brief read-only window; the mac
+   origin runs a **dual-write shadow period** (Chroma writes + op emission,
+   divergence detectable) before cutover. Step 3's write-flip on remote
+   replicas begins only **after the local-capture promotion validates**
+   end-to-end. Two commitments this step MUST honor:
+   - **Local-capture promotion.** A step-1 replica may mine machine-local
+     data (projects, conversations) into its own palace before step 2
+     exists — such drawers carry no `replica_origin` stamp, so read-replica
+     reconciliation cannot touch them. Step 2 ships a one-time promotion
+     pass: every unstamped local drawer becomes a `drawer.add` op under the
+     replica's identity and flows to the whole mesh. Capture-now is
+     forward-compatible by contract, not by luck; nothing mined early is
+     ever re-mined or lost.
+   - **v4 ids are the cross-machine dedup.** Content-pure identity means
+     the same content mined on two machines yields the same drawer id; the
+     grow-only merge collapses duplicates into one drawer with multiple
+     provenance records. The migration is the dedup mechanism — no separate
+     dedup pass across origins.
+3. **Full multi-writer** — every replica captures locally, all converge.
+
+Each step ships value alone; none blocks on Layer 1 choice (seam, §5).
+
+## §10. Security — shared ownership
+
+Threat model: N replicas mean the entire brain — every drawer, verbatim —
+sits on every device. Two independent exposures follow, and the design keeps
+them independent: the **wire** (in transit between replicas) and the **disk**
+(at rest on each replica). A compromise of one must not yield the other.
+
+### 10.1 Transport-side (windows-claude)
+
+- **In transit**: every op crosses the mesh E2E-encrypted (MeshGuard /
+  WireGuard-class); relays and couriers (§5.3) see ciphertext only. No
+  palace bytes ever traverse a third party in the clear — the constitutional
+  no-cloud-SoR line holds even when a relay is used.
+- **Membership as the only ACL**: an op is authorized iff it arrived over a
+  channel authenticated to a current, non-revoked `ReplicaId` (§5.1). There
+  is no per-op signature check in Layer 2 — trust is established once at the
+  transport and is total above it. Membership itself is convergent,
+  revocation-aware state (§5.6).
+- **Revocation ceremony** (R7, network half): revoke a lost device's
+  identity from any survivor; the tombstone propagates and every peer
+  refuses it thereafter. This severs *future* access only — see 10.3.
+- **Identity-key custody**: per-device Ed25519 key, on-device, non-exportable
+  (§5.6). Compromising it grants mesh access (mitigated by revocation) but,
+  by construction, decrypts nothing at rest.
+
+### 10.2 At-rest side (mac-claude)
+
+Two tiers, both grounded in one rule: **data keys are per-replica and never
+traverse the mesh.** Ops arrive over the wire (10.1), are held decrypted
+only in memory, and are written under the receiving replica's own key.
+Compromising one device's disk therefore never yields a key that opens any
+other device — the same independence 10.3 demands between wire and disk
+holds between disks.
+
+- **Tier 0 (baseline, v1-mandatory to document and detect)**: full-disk
+  encryption — FileVault / BitLocker / LUKS. Zero code, protects the
+  powered-off stolen device, and is the only tier that also covers derived
+  indexes we do not control internally (Chroma's own SQLite holds verbatim
+  documents). Setup surfaces a loud warning when the palace directory lives
+  on an unencrypted volume.
+- **Tier 1 (target): per-palace data key over the canonical stores.** A
+  symmetric data key encrypts `oplog.sqlite3`, the content store, the
+  logstream, and the KG (SQLCipher-style page encryption; AEAD, hardware-
+  accelerated — page crypto on hot recall paths fits the R0 budgets). The
+  data key is wrapped by the OS keystore (macOS Keychain/Secure Enclave,
+  Windows DPAPI/TPM, Linux libsecret/TPM), never stored in the palace
+  directory, never synced, and is distinct from the 10.1 identity key by
+  construction — neither derives from the other.
+- **Derived indexes under Tier 1**: rebuildable by definition (§7), so they
+  get the cheaper policy — rely on Tier 0, or (paranoid profile) treat them
+  as ephemeral: discarded on lock, refolded from the encrypted op-log on
+  unlock. Never the canonical stores' key.
+- **Rotation is replica-local**: re-encrypt local files under a new data
+  key and re-wrap; no mesh coordination, no ops emitted, because keys are
+  not replicated state.
+- **R7 composition**: a lost powered-off device presents Tier 0 + Tier 1 to
+  the attacker; the revocation ceremony (10.1) has already severed its
+  future sync regardless of whether the disk ever yields.
+
+### 10.3 The seam between them
+
+Revocation stops the network; encryption protects the disk that is already
+gone. A stolen, powered-off device is defended only by 10.2; a live device
+still on the mesh is defended by 10.2 (unlocked-disk exposure) *and* 10.1
+(revocation cuts its future sync). Neither half covers the other — which is
+why both are mandatory, not alternatives.
+
+Explicitly: **replication ≠ backup (R8)**. Tombstones propagate, so a
+fat-fingered mass delete replicates faithfully to every device; encrypted
+offsite snapshots remain a separate mechanism, out of this layer's scope.
+
+## Open Questions
+
+- HLC skew bounds and how loudly to surface clock anomalies.
+- Op-log compaction policy (R6): checkpoint cadence for snapshot bootstrap.
+  (Content-revision retention is settled: superseded revisions are kept
+  forever; compaction only concerns op-replay bootstrap cost, never
+  content.)
+- Does the E2EE cloud courier ship in v1 or wait for demand?
+- Partial replicas for phone-class devices (deferred; Non-Goals).
+- Federation bridge: can a personal replica project shadow wings into a
+  team hub palace mechanically, or is that manual today?
+
+## Appendix A: PalaceMind
+
+The desktop app is where replication stops being infrastructure and becomes
+something the human can see and trust. Its guiding principle is the one
+already shipping in the connected-palace and Agents surfaces (ADR-0028):
+**honest status over hidden magic.** Four consumer surfaces.
+
+### A.1 Replica status (generalizing Live/Polling)
+
+The Agents viewer already renders a `Live · seq N` vs `Polling` honesty flag
+off the logstream. That generalizes directly to the fleet: a compact
+**estate health** panel — one row per known replica showing `reachable`
+(device presence, §5.8), last-seen, and **version-vector drift** ("this
+device is 14 ops behind mac"). Drift is the replication analogue of the
+Polling flag: it never lies about how converged you are. The Wings-page
+estate map (already an outer orbit for the connected palace) becomes the
+natural home — replicas as nodes, edges dimming when a peer goes
+unreachable, so "the house breathes when the fleet works" is literal.
+
+### A.2 Merge surfacing (R4 — the load-bearing UX)
+
+R4 forbids silent organization merges: when two replicas filed the same
+drawer differently during a partition and LWW-by-HLC picks a winner, the
+loser must be *shown*, not swallowed. PalaceMind owns that moment:
+
+- A quiet, dismissible notice — "This capture was filed in *Meds* on your
+  laptop and *Billing* here; kept *Billing* (more recent). Move it?" — with
+  one-click accept-the-other. Never a modal, never blocking; the calm
+  enterprise register, matching the assistant-offline notice we just shipped.
+- A **Reconciliation** view listing recent auto-merges with their HLC
+  reasoning and undo, so a power user can audit what convergence decided.
+  The op-log makes this free: every merge is two ops and a rule.
+
+### A.3 Presence rendering (R5)
+
+Presence (§5.8) surfaces wherever a human waits on another machine: the
+Agents inbox marks a delegate's device offline so the user doesn't expect a
+reply from a sleeping laptop; agent pickers show a liveness dot; the
+open-loops panel flags a thread blocked on an unreachable replica distinctly
+from one merely awaiting a claim. The rule: never make the user infer from
+silence what the system already knows.
+
+### A.4 Multi-origin logstream
+
+The viewer must not fracture when ops carry `origin_replica` + `hlc`
+(additive fields, per §6.2 — the contract stays compatible):
+
+- **Order by HLC**, not per-origin `seq`, once events span replicas; keep
+  `seq` as the per-origin resume cursor.
+- **Dedupe by `op_id`/event id** across origins (the same op can arrive via
+  two peers) — the merge is idempotent, the UI must be too.
+- **Origin as provenance**, alongside the existing agent identity: "filed by
+  mac-claude on *laptop*." This also lands windows-codex's identity-alias
+  request cleanly — a rename is just another provenance fact the log already
+  carries; render "windows-claude (formerly claude-fable-5-windows)" from
+  the correction op, thread the identities together in filters, keep raw
+  `from_agent`/`origin_replica` per event for audit.
+
+Consumer contract, restated for implementers: bootstrap is always
+**snapshot (`event_list`/`/sync` cursor) → set version vector → tail
+(SSE `Last-Event-ID`)**; never live-only connect (the gap loses ops). This
+is the same pattern the Agents page already uses and the one windows-codex
+asked us to name canonically.

docs/rfcs/005-agent-identity-routing.md

@@ -0,0 +1,272 @@
+# RFC 005: Agent Identity & Routing
+
+Status: Draft — for Igor's review
+Owner: mac-claude (backend/hub owner; identity is palace/fleet infrastructure, not a meshguard concern)
+Created: 2026-07-04
+Branch: `feat/shared-brain-dogfood`
+Prior art: RFC 003 (logstream — the routing surface this refines), RFC 004 (the replicated palace — provides the stable replica identity a host label derives from), the `rfc005_agent_identity_routing` correlation thread
+
+## Summary
+
+An agent's coordination identity is the triple **`host:agent:project`** — the
+granularity at which a working actor actually shares filesystem, local config,
+and knowledge. Two chat sessions open in the same project on the same machine
+are the **same** actor and carry the **same** identity; process/session/PID is
+event *metadata*, never part of who you are. Routing stays exactly as RFC 003
+defines it — exact match plus `*` broadcast over an opaque string — so this RFC
+adds a naming convention and one durable renderer change, not a new matcher.
+
+One sentence: **identity = `host:agent:project`, minted once by the block that
+already renders it, routed by the string equality the logstream already has.**
+
+## Motivation
+
+The shared brain identifies each agent by a flat name — `mac-claude`,
+`windows-claude`, `blade-claude`. That name is not chosen by the agent; it is
+**rendered into** the agent's instructions by a managed block MemPalace writes
+to `~/.claude/CLAUDE.md`:
+
+```
+<!-- mempalace-shared-brain:start -->
+## MemPalace shared brain
+You share a MemPalace hub with other agents. Your agent identity is
+windows-claude — use it as from_agent/created_by ...
+<!-- mempalace-shared-brain:end -->
+```
+
+The flat name conflates two things that have now come apart in the live fleet:
+
+1. **Two sessions, one name.** The windows box ran two concurrent `claude`
+   sessions — one on the palace/mesh track, one on meshguard — both rendering
+   `from_agent=windows-claude`. Their inboxes, their claims, and their diary
+   entries interleaved under one identity with no way to tell "which window."
+   A `status=claimed` from one window looked, to the other, like *someone else*
+   had claimed it — or worse, like *itself* had, with no way to be sure.
+
+2. **No project seam.** Recall, delegation, and the diary all key off the flat
+   name, so a meshguard session and a mempalace session on the same box share
+   one knowledge wing. Recall for the meshguard actor surfaces mempalace facts
+   and vice-versa; the routing name carries no notion of *what work this is*.
+
+Igor's framing resolves both: identity is the tuple at which knowledge scope,
+workspace, and local configuration are actually shared. Same box + same project
+= same actor, however many windows are open. Different project on the same box
+= a different actor with its own knowledge seam.
+
+## Requirements
+
+- **I1 — Identity is `host:agent:project`.** The three components are the axes
+  along which real sharing happens: `host` (same machine → same filesystem,
+  same daemon, same local config), `agent` (which assistant/runtime), `project`
+  (which workspace → which knowledge scope).
+- **I2 — Same-project sessions share one identity.** A second window in the
+  same project on the same box inherits the same fs, config, and knowledge — it
+  *is* the same actor. Identity must not fragment per session, per window, or
+  per PID.
+- **I3 — Session/PID is metadata, never identity.** Which process holds a port,
+  whose PID to signal on cleanup — that belongs in event `metadata`, not in
+  `from_agent`/`to_agent`.
+- **I4 — Claims are held by the identity, not the session.** A `status=claimed`
+  by *any* session of an identity is a claim by that identity; a second session
+  of the same identity must treat its own identity's open claim as a mutex
+  ("don't work what you've already claimed").
+- **I5 — No new routing primitive required.** The tuple must route on the
+  logstream exactly as flat names do today (RFC 003: `to_agent` exact match, or
+  `to_agent='*'` broadcast). Hierarchical/glob routing is explicitly deferred
+  (see Decision 1).
+- **I6 — Minted at the source, not hand-edited.** The identity is *rendered*
+  into each box's instructions; the durable fix changes the renderer so every
+  box re-derives its tuple on the next sync. No per-box hand-editing as the
+  steady state.
+- **I7 — Backward compatible.** Existing flat names (`mac-claude`, …) remain
+  valid and keep routing. Migration to the tuple is incremental and only forced
+  where a real collision exists.
+
+## Non-Goals
+
+- **A lease/lock server.** Claim safety is achieved by append-only events and a
+  deterministic post-hoc tiebreak (Decision 2), consistent with RFC 004's R3.
+  Nothing here introduces a coordinator or a mutex service.
+- **A new matcher.** Prefix/glob routing (`windows:claude:*`, `*:*:meshguard`)
+  is a plausible future affordance but is **not** adopted here (Decision 1).
+- **Cross-user identity.** One human, N devices, per RFC 004. The tuple
+  distinguishes *this* human's actors, not multiple humans.
+- **Changing how `from_agent`/`to_agent` are stored or validated.** Colons
+  already pass `_sanitize_routing`; the tuple is a legal value today.
+
+## The Identity Tuple
+
+```
+host : agent : project
+```
+
+- **`host`** — a stable, human-readable label for the machine. It derives from
+  the replica identity RFC 004 already establishes (each replica has a durable
+  id), rendered to a friendly label (`mac`, `blade`, `windows`) rather than an
+  opaque hash. One host = one replica = one local daemon.
+- **`agent`** — the assistant/runtime family (`claude`, `codex`, `hermes`, …).
+  This is the existing suffix of today's flat names, lifted out intact.
+- **`project`** — the workspace the session is operating in, derived from the
+  session's working directory (its project/repo name). This is the new axis and
+  the one that carries the knowledge seam.
+
+Today's flat names are exactly `host-agent` with the `project` axis missing.
+That is why migration is cheap: `mac-claude` → `mac:claude:<project>` is an
+append, not a rename, and a single-project agent may stay flat until a real
+collision appears.
+
+**Delimiter.** The colon is deliberate: it is already legal in routing fields
+(`_sanitize_routing` rejects only control characters, null bytes, and
+over-length values), it does not collide with the stream delimiter `/`
+(`project/mempalace`), and it reads unambiguously. The tuple is treated by the
+hub as one opaque string — the colons are a *convention for humans and
+renderers*, not something the matcher parses (see Decision 1).
+
+## Same-Session Equivalence (I2)
+
+Two sessions with the same `host:agent:project` are one actor. Concretely:
+
+- They watch the **same inbox** (`to_agent=<tuple>`, which also catches `*`).
+- They write with the **same `from_agent`**.
+- They share **one knowledge wing and one diary** (see Knowledge Partition).
+- A task claimed by one is claimed by the identity (I4).
+
+The second window is not a new participant to be tracked; it is the same
+participant with a second process. When the fleet needs to know *which process*
+(e.g. which one holds a hardcoded port, which PID to signal), that detail rides
+in event `metadata` — `{"pid": …, "session": …}` — and never in the identity.
+
+## Claim Safety Under a Shared Identity (I4, Decision 2)
+
+A shared identity introduces one race: two sessions of the same identity (or
+two replicas of the same identity, post-partition per RFC 004 R3) claim the
+same task before either sees the other's claim. Resolution, no lease server:
+
+1. **Natural mutex.** Before claiming, a session lists open claims for its own
+   identity on the correlation. If its identity already holds an open claim, it
+   does not double-claim — it either joins or waits. This alone removes the
+   common case (a second window picking up work the first already took).
+2. **Deterministic tiebreak for the residual window.** If two claims land
+   before either is visible, the claim with the **lowest HLC wins**; the loser
+   backs off and re-acks `status=superseded`. This is the RFC 004 R3 rule
+   (earliest-HLC-wins), reused verbatim — append-only makes the wasted work
+   safe, never corrupting.
+
+No new event type is required: `event.ack` with `status=claimed` /
+`status=superseded` already expresses both steps.
+
+## Knowledge Partition (I2, bonus)
+
+Keying the palace wing and the diary by the full tuple partitions recall along
+the same seam routing already uses: a `…:meshguard` actor's recall stops
+surfacing `…:mempalace` facts. Today a single-agent box interleaves both
+projects in one wing (e.g. `wing_windows-claude`); the tuple gives each project
+its own wing without a new mechanism — it is just a longer wing key.
+
+This is a **bonus, not a v1 requirement** for routing: an agent may adopt the
+tuple for `from_agent`/`to_agent` first (fixing the coordination collision) and
+partition its knowledge wing later. The two are independent adoptions.
+
+## Decisions
+
+Two choices live in the backend/hub and are resolved here so downstream
+consumers (PalaceMind's viewer, the renderer, other agents) can build against a
+fixed contract.
+
+### Decision 1 — Matcher: **no change.** Route the tuple as an opaque string.
+
+`to_agent` matching stays exactly as RFC 003 defines it:
+
+```sql
+(to_agent = ? OR to_agent = '*')
+```
+
+The tuple `host:agent:project` is matched by **string equality**; `*` remains
+the only wildcard, meaning fleet-wide broadcast. Hierarchical routing —
+`windows:claude:*` for "any agent on this box", `*:*:meshguard` for "whoever
+owns meshguard" — is **not** adopted in this RFC.
+
+*Rationale.* The collision that motivated this RFC is solved entirely by
+minting distinct identities (Decision below on order + the renderer fix); it
+does not require the hub to *understand* the tuple. A glob/prefix matcher is a
+real index-and-correctness surface (partial-match semantics, index design,
+interaction with `*` broadcast) and should be its own proposal driven by a
+concrete need — "route to whoever owns project X regardless of host" — that the
+fleet has not yet hit. Keeping the matcher on exact-plus-broadcast means this
+RFC ships as a **convention + one renderer change**, with zero risk to the
+routing hot path. If hierarchical routing is later wanted, it is additive: a new
+optional match mode, not a migration.
+
+### Decision 2 — Order: **`host:agent:project`** (host-first).
+
+*Rationale.* Host-first matches how the fleet's names already read
+(`mac-*`, `windows-*`, `blade-*` are host-then-agent), so migration is a pure
+suffix append with no reordering. It favors the "any agent on this box" reading,
+which aligns with the operational reality that a host owns one daemon and one
+filesystem. Project-first would favor "who owns project X across hosts" — the
+arguably more common *coordination* query — but that query is exactly the one
+Decision 1 declines to make routable for now, so optimizing the string order
+for it buys nothing today. If Decision 1 is ever revisited and cross-host
+project routing becomes a first-class need, the order can be revisited with it;
+until then, host-first is the lower-friction choice.
+
+## The Durable Fix: mint the identity at the renderer (I6)
+
+The identity is hardcoded because MemPalace **renders** it. The one-place fix is
+to change the renderer/installer that writes the
+`<!-- mempalace-shared-brain:start … end -->` block so it emits the tuple
+instead of the flat name:
+
+- **`host`** — derived from the local replica identity (RFC 004), mapped to a
+  friendly label.
+- **`agent`** — the runtime family, as today.
+- **`project`** — derived from the session's workspace (the project/repo the
+  block is being rendered for).
+
+Then every box re-renders to its tuple on the next sync — no hand-editing. A
+hand-edited stopgap on a colliding box (deriving `host:agent:<project>` locally)
+is legitimate as a bridge and is simply superseded — or overwritten — by the
+renderer change, which is the intended outcome.
+
+**Compatibility.** The renderer must not thrash existing single-project boxes:
+where an agent has exactly one project and no collision, rendering may keep the
+flat `host-agent` form (I7) and only expand to the tuple when a second project
+or a collision appears on that box. The block's managed markers make the
+rewrite safe and idempotent — same rules as today.
+
+## Sequencing
+
+This RFC is **deferred behind the RFC 004 write-flip**: it touches no storage or
+merge semantics, so it does not gate the flip, and the flip's cutover work takes
+priority. Once that lands:
+
+1. **Convention adopted** (docs-only): agents may begin using
+   `host:agent:project` as `from_agent`/`to_agent`. Routes today, no code
+   change (colons already valid; matcher unchanged).
+2. **Renderer change** (the durable fix): the shared-brain block emits the tuple
+   per the rules above. Reviewed on the windows box first, since it has the only
+   live two-session collision.
+3. **Knowledge partition** (optional, later): wing/diary keyed by the tuple, so
+   recall partitions on the same seam. Independent of steps 1–2.
+
+## Open Questions
+
+- **`project` derivation edge cases.** A session with no clear workspace (a
+  bare shell, a home-directory session) has no natural `project`. Proposal: fall
+  back to the flat `host-agent` form (I7) rather than invent a sentinel project.
+- **Friendly host labels.** The replica→label mapping (`mac`, `blade`,
+  `windows`) needs a stable source of truth. Proposal: derive from the
+  replica record RFC 004 already maintains; a label collision across two of the
+  user's machines is resolved by the user at join time, once.
+- **Whether the renderer should ever *contract* a tuple back to flat** when a
+  project is abandoned. Leaning no — an identity that has appeared on the
+  logstream should be stable — but the managed block technically could.
+
+## Relationship to RFC 004
+
+RFC 004 gives this RFC its `host` axis for free: a replicated palace already has
+a durable per-replica identity, and one host is one replica is one local
+daemon. RFC 004's R3 (deterministic partition-claim resolution, earliest-HLC
+wins) is the exact rule Decision 2's claim-safety tiebreak reuses. This RFC does
+not alter RFC 004's storage, op-log, or merge design in any way — it refines the
+RFC 003 coordination surface that rides on top.

docs/write-routing-policy.md

@@ -0,0 +1,156 @@
+# Write routing policy
+
+This document defines the shared policy used by the staged Tier 3 daemon
+rollout tracked in #1963.
+
+This foundation PR does not change existing hook or CLI routing. It provides
+one tested policy model that later hook and CLI PRs can consume without
+inventing different fallback rules.
+
+## Policies
+
+`direct`
+
+    Always execute through the existing direct local path.
+
+`prefer`
+
+    Use an available daemon. A caller that is allowed to start the daemon may
+    do so. Otherwise, fall back to the direct path.
+
+`require`
+
+    Use an available daemon. A caller that is allowed to start the daemon may
+    do so. If neither is possible, block the operation. Never fall back to a
+    direct ChromaDB writer.
+
+## Concrete routing outcomes
+
+The shared decision function returns one of:
+
+- `direct`
+- `daemon`
+- `blocked`
+
+It also reports whether the caller should auto-start the daemon and why the
+route was selected.
+
+Hooks generally pass `daemon_can_start=False` because hook execution has a
+tight latency budget.
+
+Interactive CLI commands can pass `daemon_can_start=True`.
+
+## Configuration
+
+Global environment policy:
+
+    MEMPALACE_WRITE_ROUTING=direct|prefer|require
+
+Hook-specific environment policy:
+
+    MEMPALACE_HOOK_WRITE_ROUTING=direct|prefer|require
+
+CLI-specific environment policy:
+
+    MEMPALACE_CLI_WRITE_ROUTING=direct|prefer|require
+
+Configuration-file shape:
+
+    {
+      "write_routing": {
+        "default": "direct",
+        "hooks": "prefer",
+        "cli": "require"
+      }
+    }
+
+## Precedence
+
+For hooks:
+
+1. `MEMPALACE_HOOK_WRITE_ROUTING`
+2. `MEMPALACE_WRITE_ROUTING`
+3. legacy `MEMPALACE_HOOKS_DAEMON`
+4. `write_routing.hooks`
+5. `write_routing.default`
+6. legacy `hooks.daemon`
+7. `direct`
+
+For CLI writes:
+
+1. `MEMPALACE_CLI_WRITE_ROUTING`
+2. `MEMPALACE_WRITE_ROUTING`
+3. `write_routing.cli`
+4. `write_routing.default`
+5. `direct`
+
+## Backward compatibility
+
+The existing `MEMPALACE_HOOKS_DAEMON` environment variable and
+`hooks.daemon` config value remain supported.
+
+Legacy true values map to `prefer`.
+
+Legacy false values map to `direct`.
+
+The existing `MempalaceConfig.hook_use_daemon` property is intentionally
+unchanged in this PR. Hook and CLI behavior remains unchanged until their
+policy-aware rollout PRs land.
+
+## Invalid policy values
+
+New policy settings accept only:
+
+- `direct`
+- `prefer`
+- `require`
+
+Invalid values fail with a source-specific error rather than silently falling
+back. This is important because silently turning a misspelled `require` into a
+direct write would violate the safety purpose of the policy.
+
+## Local backend single-writer safety
+
+File-backed backends such as `chroma`, `sqlite_exact`, and Milvus Lite support
+exactly one writable process per palace. Serializing individual calls is not
+enough because each long-lived process can retain SQLite/WAL, FTS, or vector
+index state between calls.
+
+- A writable daemon owns the palace writer lease for its full lifetime.
+- Writable MCP HTTP acquires that lease before binding, holds it through the
+  full serving lifetime, and releases it after active requests stop.
+- MCP stdio opens `sqlite_exact` read-only until it acquires the writer lease.
+  It may therefore coexist for reads; mutating tools refuse while another
+  process owns the lease and reopen writable storage after that owner exits.
+- Read-only MCP HTTP may coexist with the writer.
+- Read-only `sqlite_exact` clients use an immutable connection for a clean
+  checkpointed database, or `mode=ro` when an active writer's complete WAL
+  sidecar pair must remain visible. Both paths enable `query_only` and skip
+  schema, WAL, FTS, migration, and metadata initialization.
+- Direct CLI and hook writes must not run beside a writable daemon or MCP HTTP
+  owner. Route them through the daemon with `require` when the daemon owns the
+  palace.
+- Direct `sqlite_exact` collection mutations contend for the same palace lease,
+  and full LLM closet regeneration owns it before opening collections or
+  calling the configured model.
+
+`MEMPALACE_MCP_ALLOW_PEER_WRITER` cannot bypass this protection for local
+file-backed or unknown plugin backends. It is retained only for explicitly
+remote service backends (`qdrant`, `pgvector`, and Milvus server/Zilliz Cloud)
+that coordinate concurrent clients themselves. Milvus Lite remains protected
+as local file-backed storage.
+
+Do not delete or unlink a live palace lock to recover ownership. Stop the
+owning process cleanly; the operating system releases its lock automatically.
+If corruption is suspected, back up the palace and run integrity/repair
+operations offline, with no writable service running.
+
+## Follow-up PRs
+
+Hook-triggered writes now consume this policy; see
+`docs/hook-write-routing.md`.
+
+The remaining rollout PR will apply the policy to routine CLI writes.
+
+Maintenance operations such as repair, migration, and index rebuild are not
+ordinary routed writes. They require a separate exclusive-maintenance policy.

examples/mx3_public_shim_embeddings_rerank.md

@@ -0,0 +1,92 @@
+# MX3 Public Shim Embeddings and Rerank
+
+MemPalace can optionally route Chroma embeddings and post-retrieval rerank through the public
+MX3 shim helpers.
+
+This path is local by default. It sends the live search query and candidate hits to the configured
+shim endpoint, so remote endpoints are blocked unless you explicitly opt in, and any remote
+endpoint must use HTTPS.
+
+Public repo:
+
+- `https://github.com/grtninja/mx3-public-shim`
+
+## Hardware required
+
+This setup requires a MemryX MX3 accelerator in an M.2 slot behind the configured public-shim
+endpoint.
+
+## Install
+
+```bash
+pip install "mx3-public-shim[mempalace] @ git+https://github.com/grtninja/mx3-public-shim.git"
+```
+
+## Validate before enabling
+
+Use this path only with a hardware-backed public-shim deployment.
+
+```bash
+python -m mx3_public_shim.doctor
+```
+
+MemPalace rejects the explicit `cpu_reference` fallback, but you should still validate the
+configured endpoint before enabling it:
+
+- the runtime must expose an embeddings provider
+- the selected provider must not be `cpu_reference`
+- the endpoint should be the local public shim unless you intentionally opt in to a remote HTTPS
+  target
+
+On Windows, a healthy doctor report normally shows:
+
+- `mx3_linux` unavailable because the direct official Python runtime is Linux-only
+- `openai_compat` available and selected for `embeddings`
+- `openai_compat` available and selected for `chat`
+
+That means MemPalace is using the local public-shim OpenAI-compatible surface while the required
+hardware lives behind that endpoint.
+
+## Enable embeddings + rerank in MemPalace
+
+```bash
+export MEMPALACE_EMBEDDING_BACKEND=mx3_public_shim
+export MEMPALACE_SEARCH_RERANKER=mx3_public_shim
+export MEMPALACE_MX3_PUBLIC_SHIM_BASE_URL=http://127.0.0.1:9000/v1
+export MEMPALACE_MX3_PUBLIC_SHIM_EMBEDDING_MODEL=text-embedding-nomic-embed-text-v1.5
+export MEMPALACE_MX3_PUBLIC_SHIM_PROVIDER_ORDER=mx3_linux,openai_compat
+export MEMPALACE_MX3_PUBLIC_SHIM_REQUIRE_HARDWARE=1
+```
+
+On Windows, the direct `mx3_linux` provider is not expected to resolve; the normal path is a local
+public-shim OpenAI-compatible endpoint that is itself backed by the required hardware.
+
+If your endpoint requires authentication, also set:
+
+```bash
+export MEMPALACE_MX3_PUBLIC_SHIM_API_KEY=...
+```
+
+`MEMPALACE_EMBEDDING_BACKEND` and `MEMPALACE_SEARCH_RERANKER` can be enabled independently. Use
+both when you want MX3-backed Chroma embeddings plus post-retrieval rerank.
+
+If you intentionally point `MEMPALACE_MX3_PUBLIC_SHIM_BASE_URL` at a non-localhost endpoint, also
+set:
+
+```bash
+export MEMPALACE_MX3_PUBLIC_SHIM_ALLOW_REMOTE=1
+```
+
+## Optional example script
+
+The companion example below demonstrates the narrow Chroma seam directly:
+
+```bash
+python examples/mx3_public_shim_embeddings_rerank.py
+```
+
+Then search normally:
+
+```bash
+mempalace search "auth decisions"
+```

examples/mx3_public_shim_embeddings_rerank.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Example: query a local public-shim embeddings endpoint and pass vectors into Chroma."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from urllib import request
+
+try:
+    from chromadb import PersistentClient
+except ImportError as exc:  # pragma: no cover
+    raise SystemExit("Install chromadb before running this example.") from exc
+
+
+BASE_URL = os.environ.get("MEMPALACE_MX3_PUBLIC_SHIM_BASE_URL", "http://127.0.0.1:9000/v1")
+EMBEDDINGS_URL = BASE_URL.rstrip("/") + "/embeddings"
+EMBED_MODEL = os.environ.get(
+    "MEMPALACE_MX3_PUBLIC_SHIM_EMBEDDING_MODEL",
+    "text-embedding-nomic-embed-text-v1.5",
+)
+
+
+def fetch_embeddings(texts: list[str]) -> list[list[float]]:
+    payload = json.dumps({"model": EMBED_MODEL, "input": texts}).encode("utf-8")
+    headers = {"Content-Type": "application/json"}
+    api_key = os.environ.get("MEMPALACE_MX3_PUBLIC_SHIM_API_KEY")
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    req = request.Request(
+        EMBEDDINGS_URL,
+        data=payload,
+        headers=headers,
+        method="POST",
+    )
+    with request.urlopen(req, timeout=20) as response:
+        body = json.loads(response.read().decode("utf-8"))
+    data = body.get("data", [])
+    if len(data) != len(texts):
+        raise RuntimeError("Embedding response count did not match input count")
+    return [row["embedding"] for row in data]
+
+
+def main() -> None:
+    documents = [
+        "A local retrieval queue keeps notes that still need verification.",
+        "The workstation stores a short MX3 setup checklist for repeatable bring-up.",
+        "This note is about watering tomatoes in the backyard.",
+    ]
+    metadatas = [
+        {"wing": "verification"},
+        {"wing": "operations"},
+        {"wing": "personal"},
+    ]
+    query = "How does the system track notes that still need verification?"
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        client = PersistentClient(path=str(Path(tmpdir)))
+        collection = client.get_or_create_collection("mempalace_drawers")
+        doc_vectors = fetch_embeddings(documents)
+        collection.add(
+            ids=["1", "2", "3"],
+            documents=documents,
+            embeddings=doc_vectors,
+            metadatas=metadatas,
+        )
+        query_vector = fetch_embeddings([query])[0]
+        result = collection.query(query_embeddings=[query_vector], n_results=3)
+        print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

hooks/mempal_precompact_hook.sh

@@ -182,14 +182,14 @@ echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$
 #   1. TRANSCRIPT_PATH (from Claude Code) → parent dir, --mode convos
 #   2. MEMPAL_DIR → --mode projects
 if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
-    mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
+    "$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
         >> "$STATE_DIR/hook.log" 2>&1
 elif [ -n "$TRANSCRIPT_PATH" ]; then
     echo "[$(date '+%H:%M:%S')] Skipping missing or invalid transcript path after normalization: $TRANSCRIPT_PATH" \
         >> "$STATE_DIR/hook.log"
 fi
 if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
-    mempalace mine "$MEMPAL_DIR" --mode projects \
+    "$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \
         >> "$STATE_DIR/hook.log" 2>&1
 fi
 

hooks/mempal_save_hook.sh

@@ -265,14 +265,14 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then
     # MEMPAL_DIR is *additive*, not an override: a user with MEMPAL_DIR
     # pointed at their project still gets the active conversation mined.
     if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
-        mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
+        "$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
             >> "$STATE_DIR/hook.log" 2>&1 &
     elif [ -n "$TRANSCRIPT_PATH" ]; then
         echo "[$(date '+%H:%M:%S')] Skipping invalid transcript path: $TRANSCRIPT_PATH" \
             >> "$STATE_DIR/hook.log"
     fi
     if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
-        mempalace mine "$MEMPAL_DIR" --mode projects \
+        "$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \
             >> "$STATE_DIR/hook.log" 2>&1 &
     fi
 

integrations/openclaw/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: mempalace
 description: "MemPalace — Local AI memory with 96.6% recall. Semantic search, temporal knowledge graph, palace architecture (wings/rooms/drawers). Free, no cloud, no API keys."
-version: 3.3.0
+version: 3.7.0
 homepage: https://github.com/MemPalace/mempalace
 user-invocable: true
 metadata:
@@ -46,6 +46,10 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
 
 ## Available Tools
 
+Full MCP surface: 36 tools. Destructive or host-level tools are documented so
+you know they exist, but use them only when the user explicitly asks or when a
+tool-specific workflow below says to.
+
 ### Search & Browse
 - `mempalace_search` — Semantic search across all memories. Always start here.
   - `query` (required): natural language search — keep it short, keywords or a question. Do NOT include system prompts or conversation context.
@@ -58,6 +62,14 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
 - `mempalace_status` — Palace overview: total drawers, wings, rooms, AAAK spec
 - `mempalace_list_wings` — All wings with drawer counts
 - `mempalace_list_rooms` — Rooms within a wing (optional wing filter)
+- `mempalace_list_drawers` — Paginated drawer listing
+  - `wing`, `room`: optional filters
+  - `since`: only drawers filed on/after this ISO date/time
+  - `before`: only drawers filed before this ISO date/time
+  - `limit`: max results (default 20)
+  - `offset`: pagination offset (default 0)
+- `mempalace_get_drawer` — Fetch a single drawer by ID. Returns full verbatim content and metadata.
+  - `drawer_id` (required)
 - `mempalace_get_taxonomy` — Full wing/room/count tree
 - `mempalace_get_aaak_spec` — Get AAAK compression dialect specification
 
@@ -81,17 +93,59 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
 - `mempalace_traverse` — Walk from a room, find connected ideas across wings
   - `start_room` (required): room to start from
   - `max_hops`: connection depth (default 2)
-- `mempalace_find_tunnels` — Find rooms that bridge two wings
-  - `wing_a`, `wing_b` (required)
+- `mempalace_find_tunnels` — Find rooms that bridge two wings via *implicit* overlap (rooms whose drawers naturally share content across wings — discovered, not declared)
+  - `wing_a`, `wing_b`: optional filters; omit both to scan all wing pairs
+- `mempalace_create_tunnel` — Create an *explicit* cross-wing tunnel: a user/agent-declared link between two locations. Use when you notice content in one project relates to another (e.g. API design in `project_api` connects to schema in `project_database`).
+  - `source_wing`, `source_room`, `target_wing`, `target_room` (required)
+  - `label`: short description of the relationship
+  - `source_drawer_id`, `target_drawer_id`: anchor to specific drawers
+- `mempalace_list_tunnels` — List all explicit tunnels, optionally filtered by wing
+  - `wing`: optional filter
+- `mempalace_delete_tunnel` — Remove an explicit tunnel by ID
+  - `tunnel_id` (required)
+- `mempalace_list_hallways` — List within-wing entity hallways (entity-to-entity co-occurrence links built at mine time)
+  - `wing`: optional filter
+- `mempalace_delete_hallway` — Remove a hallway record by ID
+  - `hallway_id` (required)
+- `mempalace_follow_tunnels` — From a room, follow explicit tunnels to connected drawers in other wings
+  - `wing`, `room` (required)
 - `mempalace_graph_stats` — Graph connectivity overview
 
 ### Write
 - `mempalace_add_drawer` — Store verbatim content into a wing/room
   - `wing`, `room`, `content` (required)
   - `source_file`: optional source reference
+  - `added_by`: optional filing agent label
   - Checks for duplicates automatically
+- `mempalace_checkpoint` — Save a whole session in one call: dedup each item, file non-duplicates, then write one diary entry
+  - `items` (required): array of `{wing, room, content}`; content must be verbatim
+  - `diary`: optional `{agent_name, entry, topic?, wing?}`; entry should use AAAK format
+  - `dedup_threshold`: similarity threshold (default 0.9)
+  - `added_by`: optional filing agent label (defaults to the diary `agent_name`, else `checkpoint`)
+- `mempalace_update_drawer` — Update an existing drawer's content and/or move it to a different wing/room
+  - `drawer_id` (required)
+  - `content`, `wing`, `room`: at least one must be provided (no-op otherwise)
 - `mempalace_delete_drawer` — Remove a drawer by ID
   - `drawer_id` (required)
+
+### Ingest & Cleanup
+- `mempalace_mine` — Mine a directory into the palace. Host-level ingest; call only when the user asks to import files.
+  - `source` (required): directory to mine
+  - `mode`: `projects` (default), `convos`, or `extract`
+  - `wing`: target wing (default: source directory name)
+  - `agent`: recorded on every drawer (default `mempalace`)
+  - `limit`: max files to process (0 = all)
+  - `dry_run`: preview without writing
+  - `extract`: convos extraction strategy (`exchange` default, or `general`)
+- `mempalace_sync` — Prune drawers whose source files are gitignored, deleted, or moved. Use dry-run first.
+  - `project_dir`: optional project root scope
+  - `wing`: optional wing scope
+  - `apply`: actually delete; default is dry-run preview
+- `mempalace_delete_by_source` — Bulk-delete drawers with one exact `source_file`. Destructive; use dry-run first.
+  - `source_file` (required): exact metadata value to remove
+  - `dry_run`: preview match count and sample (default true)
+
+### Diary & Session
 - `mempalace_diary_write` — Write a session diary entry
   - `agent_name` (required): your name/identifier
   - `entry` (required): what happened, what you learned, what matters
@@ -99,6 +153,15 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
 - `mempalace_diary_read` — Read recent diary entries
   - `agent_name` (required)
   - `last_n`: number of entries (default 10)
+- `mempalace_memories_filed_away` — Acknowledge the latest silent auto-save checkpoint.
+  - Returns: how many messages were tucked into drawers since the last ack
+  - When to call: at the START of a session, to confirm prior-conversation persistence
+
+### System
+- `mempalace_hook_settings` — Get or set auto-save hook behavior. Host-level setting; do not change silently.
+  - `silent_save`: true saves directly without MCP-level clutter
+  - `desktop_toast`: true shows a desktop notification when saves complete
+- `mempalace_reconnect` — Force reconnect to the palace database after external writes or stale index state
 
 ## Setup
 

integrations/shared/coordination-protocol.md

@@ -0,0 +1,125 @@
+# MemPalace Shared-Brain Coordination Protocol
+
+The canonical protocol for agents sharing one MemPalace hub — memory
+discipline plus the logstream coordination layer (RFC 003). Like
+`recall-protocol.md`, this file is the single source of truth: skills,
+rules, and system prompts should link here or copy the System-Prompt
+Snippet below verbatim, so the protocol never drifts per-agent.
+
+## The two layers
+
+A shared palace gives every agent two distinct channels. Do not mix them:
+
+- **Memory (drawers, KG, diary)** — durable knowledge worth recalling
+  later. Searched semantically. Follow
+  [`recall-protocol.md`](recall-protocol.md).
+- **Coordination (logstream events + artifacts)** — active work moving
+  between agents *right now*: delegations, replies, patches, acks.
+  Filtered structurally, never searched semantically.
+
+Rule of thumb: if another agent should **act** on it, it is an event.
+If a future session should **know** it, it is a drawer. A concluded
+delegation usually produces both: the events carried the work; a drawer
+records the outcome.
+
+## Identity
+
+Every agent uses one stable `from_agent` identity, formatted
+`<machine>-<harness>` (e.g. `mac-claude`, `windows-codex`,
+`aero-opencode`). Never impersonate another agent; never rotate names —
+the event trail is only auditable if identities are stable.
+
+## Delegating work (requester)
+
+1. Generate a `correlation_id` for the task: `task_<short-description>`
+   plus enough entropy to be unique (e.g. `task_fix_ranking_7f3a`).
+2. `mempalace_event_append` with `type=task.request`, `stream=project/<name>`,
+   `room=delegation`, `to_agent=<worker>`, `status=open`, and a body that
+   states the goal, the branch, the base commit, and the definition of done.
+3. Wait for the reply: `mempalace_event_wait` with the `correlation_id`
+   and `to_agent=<you>`. Waits cap at 5 minutes — loop, passing
+   `since_event_id` of the last event you saw.
+4. When a `patch.ready` arrives: `mempalace_artifact_get`, verify the
+   `sha256`, apply locally, run the stated verification.
+5. Always close the loop with `mempalace_event_ack` — `status=applied`
+   on success, `status=failed` with verbatim evidence on failure.
+
+## Receiving work (worker)
+
+1. Poll or wait for `type=task.request`, `to_agent=<you>` (plus `*`
+   broadcasts are matched automatically).
+2. Claim it: `mempalace_event_ack` with `status=claimed` so no other
+   agent duplicates the work.
+3. Do the work on the stated branch/commit.
+4. Deliver through the formal channel — `mempalace_patch_submit` with the
+   diff, `correlation_id`, `branch`, and `base_commit`. **Pushing a
+   branch is not a handoff**; the event is. If you also pushed, say so
+   in the body.
+5. If blocked or unable to produce a patch, still reply:
+   `type=task.reply` with `status=blocked` or `failed` and verbatim
+   notes. Silence is the only unrecoverable failure.
+
+## Hard rules
+
+- **Never apply a patch silently.** Fetching an artifact is free;
+  applying it is an explicit local decision, stated to the user.
+- **Verify hashes.** An artifact's `sha256` must match its content
+  before you act on it.
+- **Append-only.** Never try to edit or delete events; supersede with a
+  new event (`status=superseded`) referencing the old one.
+- **Exact payloads.** Bodies and artifacts are verbatim — no summaries
+  of diffs, no truncated logs. If it is too big, store it as an
+  artifact and reference it.
+- **Close every loop.** Every `task.request` you claimed ends in an
+  `applied`, `failed`, or `blocked` — no dangling `open` tasks.
+- **File the outcome.** When a delegation concludes, write one drawer
+  (`mempalace_add_drawer`) recording what was decided/learned, so the
+  result is searchable without replaying the event trail.
+
+## System-Prompt Snippet
+
+Copy this block into an agent's system prompt / custom instructions.
+Replace `<AGENT_ID>` with the agent's stable identity.
+
+```text
+## MemPalace shared brain
+
+You share a MemPalace hub with other agents. Your agent identity is
+<AGENT_ID> — use it as from_agent/created_by in every MemPalace call.
+
+Memory (recall + writing):
+- Before answering about past work, decisions, people, or projects,
+  search the palace (mempalace_search; mempalace_kg_query for
+  relational/temporal facts). Quote results verbatim — never paraphrase
+  stored content. If the palace has nothing, say so; don't guess.
+- File durable outcomes (decisions, conclusions, learned facts) with
+  mempalace_add_drawer. When a fact changes: mempalace_kg_invalidate the
+  old fact, then mempalace_kg_add the new one. Don't file secrets or
+  tokens.
+
+Coordination (logstream):
+- Check your inbox when starting work and before long tasks:
+  mempalace_event_list with to_agent=<AGENT_ID> (new since your last
+  seen event id).
+- To delegate: mempalace_event_append (type=task.request, stream=
+  project/<name>, room=delegation, correlation_id=task_..., status=open,
+  body = goal + branch + base commit + definition of done), then
+  mempalace_event_wait on that correlation_id for the reply.
+- When you accept a task: ack it with status=claimed. Deliver code as a
+  patch via mempalace_patch_submit (never just push a branch and go
+  silent). If blocked, reply with status=blocked and verbatim notes.
+- When you receive a patch: mempalace_artifact_get, verify sha256,
+  apply only with explicit user-visible intent, run the stated tests,
+  then mempalace_event_ack with status=applied or failed.
+- Events are append-only and verbatim. Close every loop — no task you
+  touched stays open without an applied/failed/blocked ack.
+```
+
+## See also
+
+- [`recall-protocol.md`](recall-protocol.md) — the search-before-answer
+  memory protocol this composes with.
+- [Agent Logstream concepts](../../website/concepts/agent-logstream.md) —
+  event/artifact model and the full tool reference.
+- RFC 003 (`docs/rfcs/003-agent-logstream-coordination.md`) — design
+  rationale and storage model.

integrations/shared/recall-protocol.md

@@ -116,5 +116,10 @@ added through the MCP server and diary entries, which have no source file
 
 - [`integrations/openclaw/SKILL.md`](../openclaw/SKILL.md) — the original
   full-protocol skill this is distilled from.
+- [`coordination-protocol.md`](coordination-protocol.md) — the shared-brain
+  companion protocol: when agents delegate work to each other over the
+  hub, they use the logstream (`mempalace_event_append` /
+  `mempalace_event_wait`), not drawers. Recall answers questions;
+  the logstream moves work.
 - MemPalace design principles (verbatim, local-first, never summarize):
   <https://github.com/MemPalace/mempalace>

mempalace/backends/__init__.py

@@ -36,6 +36,7 @@ from .base import (
     UnsupportedMaintenanceKindError,
 )
 from .chroma import ChromaBackend, ChromaCollection
+from .milvus import MilvusBackend, MilvusCollection
 from .pgvector import PgVectorBackend, PgVectorCollection
 from .qdrant import QdrantBackend, QdrantCollection
 from .sqlite_exact import SQLiteExactBackend, SQLiteExactCollection
@@ -67,6 +68,8 @@ __all__ = [
     "LexicalHit",
     "LexicalResult",
     "MaintenanceResult",
+    "MilvusBackend",
+    "MilvusCollection",
     "PalaceNotFoundError",
     "PalaceRef",
     "PgVectorBackend",

mempalace/backends/base.py

@@ -123,10 +123,13 @@ class PalaceRef:
         access is a spec violation.
 
     Backends that do not advertise ``supports_namespace_isolation`` (e.g.
-    ``sqlite_exact``, whose isolation is the on-disk path alone) MAY ignore
-    ``namespace`` entirely; callers MUST NOT rely on it for tenant isolation
-    on such backends. Any conforming backend can self-check both guarantees by
-    running the shared assertions in ``tests/_backend_conformance.py``.
+    path-rooted ``chroma`` / ``sqlite_exact``) MUST NOT silently accept and
+    ignore a populated ``namespace`` — they MUST raise
+    :class:`UnsupportedCapabilityError` (same spirit as
+    :class:`UnsupportedFilterError`). Callers targeting those backends MUST
+    leave ``namespace`` as ``None``. Isolation conformance lives in
+    ``tests/_backend_conformance.py`` (cross-id arm for every backend;
+    same-id / different-namespace arm for advertisers only).
     """
 
     id: str
@@ -501,6 +504,15 @@ class BaseCollection(ABC):
             offset += len(batch_meta)
         return all_meta
 
+    def facet_counts(
+        self,
+        field: str,
+        where: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        """Return counts for each distinct value of a metadata field."""
+        raise UnsupportedCapabilityError("backend does not support facet_counts")
+
     def maintenance_state(self) -> dict:
         """Return a structured snapshot of this collection's maintenance state.
 
@@ -595,8 +607,9 @@ class BaseBackend(ABC):
     ``PalaceRef.namespace`` (multi-tenant / hosted deployments) MUST advertise
     the ``supports_namespace_isolation`` capability token; doing so is a
     promise to satisfy the cross-namespace guarantee and to pass the namespace
-    arm of the conformance suite. Backends without the token MAY ignore
-    ``namespace``.
+    arm of the conformance suite. Backends without the token MUST raise
+    :class:`UnsupportedCapabilityError` when ``PalaceRef.namespace`` is
+    non-``None`` rather than silently accept-and-ignore (RFC 001 §4.4).
     """
 
     name: ClassVar[str]
@@ -617,6 +630,21 @@ class BaseBackend(ABC):
     #: for anything not listed here.
     maintenance_kinds: ClassVar[frozenset[str]] = frozenset()
 
+    def require_namespace_support(self, palace: PalaceRef) -> None:
+        """Raise if ``palace.namespace`` is set but this backend does not isolate by it.
+
+        Call at the start of ``get_collection`` (and any other entry that
+        accepts a :class:`PalaceRef`) so non-advertising backends never
+        silently drop a tenant namespace (RFC 001 §4.4).
+        """
+        if palace.namespace is None:
+            return
+        if "supports_namespace_isolation" not in self.capabilities:
+            raise UnsupportedCapabilityError(
+                f"{type(self).name} does not advertise supports_namespace_isolation; "
+                f"leave PalaceRef.namespace as None (got {palace.namespace!r})"
+            )
+
     @abstractmethod
     def get_collection(
         self,

mempalace/backends/chroma.py

@@ -8,6 +8,7 @@ import math
 import os
 import pickle
 import re
+import shlex
 import sqlite3
 import time
 from collections import defaultdict
@@ -81,13 +82,52 @@ def _hnsw_link_to_data_ratio(seg_dir: str) -> Optional[float]:
     return link_size / data_size
 
 
-def _hnsw_link_lists_is_usable_for_payload(seg_dir: str) -> bool:
-    """Return False when a non-trivial HNSW payload lacks usable link lists.
+def _hnsw_metadata_marker_intact(seg_dir: str) -> bool:
+    """Return True when ``index_metadata.pickle`` bears a complete envelope.
 
-    A missing or empty link_lists.bin is acceptable only for a fresh/empty
-    segment. Once data_level0.bin has real payload, a zero-byte link_lists.bin
-    is not a harmless async-flush shape: ChromaDB can later hand the broken
-    graph to hnswlib and crash in native code.
+    ChromaDB writes ``index_metadata.pickle`` last during a persist, so an
+    intact pickle envelope — protocol marker ``0x80`` at the head, ``STOP``
+    byte ``0x2e`` at the tail — proves the flush finished. Used as a
+    persist-completion marker: when present, the segment's on-disk shape
+    (including an all-layer-0 index with an empty ``link_lists.bin``) is the
+    one chromadb intentionally serialized, not a half-written one.
+
+    Deliberately byte-sniffs only; never deserializes. Deserialization can
+    execute arbitrary code, and the byte-sniff is enough to tell a complete
+    write from truncation or zero-fill. Assumes pickle protocol >= 2.
+    """
+    meta_path = os.path.join(seg_dir, "index_metadata.pickle")
+    try:
+        if not os.path.isfile(meta_path) or os.path.getsize(meta_path) < 16:
+            return False
+        with open(meta_path, "rb") as f:
+            head = f.read(2)
+            f.seek(-1, 2)  # last byte
+            tail = f.read(1)
+    except OSError:
+        return False
+    return len(head) == 2 and head[0] == 0x80 and tail == b"\x2e"
+
+
+def _hnsw_link_lists_is_usable_for_payload(seg_dir: str) -> bool:
+    """Return False when a non-trivial HNSW payload looks like a partial flush.
+
+    A zero-byte ``link_lists.bin`` is *not* corruption on its own. hnswlib
+    stores the entire layer-0 graph inside ``data_level0.bin`` and only writes
+    ``link_lists.bin`` for elements promoted to level > 0. A small or
+    low-fanout index where every element stays on layer 0 therefore
+    serializes an empty ``link_lists.bin`` and loads/searches fine (#1716).
+    Treating that shape as corruption produced a self-perpetuating quarantine
+    loop: repair rebuilt the byte-identical all-layer-0 segment, which the next
+    cold start quarantined again, accumulating drift dirs without bound.
+
+    An empty ``link_lists.bin`` only signals trouble when the persist was
+    interrupted before it could finish. ChromaDB writes
+    ``index_metadata.pickle`` last, so an intact metadata envelope proves the
+    flush completed and the empty ``link_lists.bin`` is the legitimate
+    all-layer-0 shape. Only when there is real payload, an empty
+    ``link_lists.bin``, *and* no completion marker do we treat the segment as a
+    partial flush.
     """
     data_path = os.path.join(seg_dir, "data_level0.bin")
     link_path = os.path.join(seg_dir, "link_lists.bin")
@@ -100,10 +140,16 @@ def _hnsw_link_lists_is_usable_for_payload(seg_dir: str) -> bool:
         if data_size <= _HNSW_MISSING_METADATA_DATA_FLOOR:
             return True
 
-        return os.path.isfile(link_path) and os.path.getsize(link_path) > 0
+        if os.path.isfile(link_path) and os.path.getsize(link_path) > 0:
+            return True
     except OSError:
         return False
 
+    # Real payload with an empty/absent link_lists.bin: legitimate only when
+    # the persist completed (all-layer-0 index), proven by an intact metadata
+    # marker. Otherwise it is a half-written segment chromadb could segfault on.
+    return _hnsw_metadata_marker_intact(seg_dir)
+
 
 def _hnsw_payload_appears_sane(seg_dir: str) -> bool:
     """Return False when HNSW payload files are structurally implausible."""
@@ -114,34 +160,101 @@ def _hnsw_payload_appears_sane(seg_dir: str) -> bool:
     return ratio is None or ratio <= _HNSW_LINK_TO_DATA_MAX_RATIO
 
 
-# HNSW batch/sync thresholds applied at collection creation.
+# HNSW batch/sync thresholds applied at collection creation — chromadb's own
+# documented defaults (chromadb/api/configuration.py:263-273,
+# chromadb/api/collection_configuration.py:451-454,
+# chromadb/segment/impl/vector/hnsw_params.py:79-80, and
+# https://docs.trychroma.com/docs/collections/configure).
 #
-# chromadb's Rust HNSW segment writes index_metadata.pickle and
-# link_lists.bin only when internal counters cross both thresholds
-# (batch_size gates _apply_batch; sync_threshold gates _persist).
-# Records below both thresholds stay in memory and are lost on exit.
+# Both ran at 2 to answer #1579 (a sub-threshold mine left index_metadata.pickle
+# absent, and quarantine_stale_hnsw renamed the segment away). Three findings on
+# chromadb 1.5.9 (PersistentClient / Rust bindings, single writer) retire that:
 #
-# Previously 50k/50k to work around link_lists.bin sparse-file bloat
-# in pre-1.5.x Python chromadb (#344).  chromadb >=1.5.4 Rust bindings
-# (the minimum mempalace supports) do not exhibit that bloat; verified
-# at batch_size=2 with 20k records: link_lists.bin = 171 KB, no
-# sparse-file inflation.
+#   * 2 SITS OUTSIDE CHROMA'S OWN DECLARED VALID RANGE. hnsw_params.py:21-22
+#     validates both knobs as `isinstance(p, int) and p > 2`. Not an inequality
+#     between the two — a floor on each.
 #
-# The 50k guard caused #1579: mines under 50k drawers never triggered
-# _persist(), leaving index_metadata.pickle absent and link_lists.bin
-# empty.  quarantine_stale_hnsw then renamed the segment on every cold
-# open after a 300s mtime gap, accumulating .drift-* directories.
+#   * IT COSTS WRITE AMPLIFICATION, measured. Bytes written (/proc/self/io) for
+#     one mine of N records, 64-dim, num_threads=1, identical corpus and seed:
 #
-# Lowered to 2 (empirical Rust-side minimum for chromadb >=1.5.4; the
-# Rust bindings reject 1 with InvalidArgumentError) so any mine of 2+
-# drawers triggers a natural persist.  Existing palaces created under
-# the old 50k guard keep those thresholds in their collection metadata
-# until the user runs repair --mode from-sqlite --archive-existing.
-_HNSW_BLOAT_GUARD = {
-    "hnsw:batch_size": 2,
-    "hnsw:sync_threshold": 2,
+#         N        2/2        100/1000     ratio
+#         10,000    47.1 MB    28.1 MB     1.67x
+#         20,000   127.4 MB    60.0 MB     2.12x
+#         40,000   390.3 MB   134.0 MB     2.91x
+#
+#     Two runs at N=20,000 reproduced within 0.1%. sync_threshold dominates:
+#     3/3 measured identical to 2/2, and 1000/1000 identical to 100/1000. The
+#     ratio grows with collection size, so the cost is worst on the largest
+#     palaces.
+#
+#   * IT BUYS NOTHING. A 5-record collection at 100/1000 — far below the
+#     threshold, so no persist ever fires — reads back whole from a FRESH
+#     PROCESS (count and vector query both answer). On that artifact
+#     link_lists.bin is 0 bytes with no index_metadata.pickle, which is #1579's
+#     trigger shape exactly, and quarantine_stale_hnsw() creates no drift dir:
+#     _segment_appears_healthy reads it as never-persisted rather than as a torn
+#     persist.
+#
+# NOT claimed here: that a small sync_threshold is a known chroma failure mode.
+# No such report was found in chroma's issues or docs, and chroma's own guidance
+# runs the other way (raise sync_threshold for bulk inserts). The Python
+# persist path that #1579 and chroma#6975 describe does not execute under the
+# Rust bindings at all — hnswlib is not a dependency of this line.
+#
+# A palace keeps whatever thresholds it was created under;
+# `repair --mode from-sqlite --archive-existing` re-creates it under these.
+_HNSW_WRITE_DEFAULTS = {
+    "hnsw:batch_size": 100,
+    "hnsw:sync_threshold": 1000,
 }
 
+
+def _hnsw_creation_metadata(options: Optional[dict]) -> dict:
+    """Build the ``metadata=`` dict for a fresh collection from caller options.
+
+    Centralizes the HNSW knobs so a multi-collection palace can tune each
+    collection at creation while the base keeps every config value in the
+    ``collection_metadata`` table where the divergence guard, the cosine-space
+    detector (``ChromaCollection.distance_metric``), and ``_read_sync_threshold``
+    already read it. The legacy ``metadata=`` keys are kept deliberately: the
+    modern ``configuration=`` API stores the same parameters in
+    ``configuration_json`` instead, leaving ``collection.metadata`` empty and
+    silently blinding all of that existing tooling.
+
+    Caller option -> chromadb metadata key:
+
+    * ``hnsw_space``       -> ``hnsw:space``        (default ``"cosine"``)
+    * ``num_threads``      -> ``hnsw:num_threads``  (default 1; serializes inserts)
+    * ``ef_construction``  -> ``hnsw:construction_ef``
+    * ``max_neighbors``    -> ``hnsw:M``
+    * ``sync_threshold``   -> ``hnsw:sync_threshold``
+    * ``batch_size``       -> ``hnsw:batch_size``
+
+    ``sync_threshold``/``batch_size`` default to chromadb's own values
+    (:data:`_HNSW_WRITE_DEFAULTS`), which amortize the index flush across a
+    mine. A caller tunes them per collection; a caller writing far fewer
+    records than the threshold still keeps them (the Rust writer holds the
+    sub-threshold tail durable — see :data:`_HNSW_WRITE_DEFAULTS`).
+    ``ef_construction``/``max_neighbors`` are omitted when the caller does not
+    set them, so chromadb applies its own defaults.
+    """
+    opts = options if isinstance(options, dict) else {}
+    md: dict[str, Any] = {
+        "hnsw:space": opts.get("hnsw_space", "cosine"),
+        "hnsw:num_threads": int(opts.get("num_threads", 1)),
+        **_HNSW_WRITE_DEFAULTS,
+    }
+    if "ef_construction" in opts and opts["ef_construction"] is not None:
+        md["hnsw:construction_ef"] = int(opts["ef_construction"])
+    if "max_neighbors" in opts and opts["max_neighbors"] is not None:
+        md["hnsw:M"] = int(opts["max_neighbors"])
+    if "sync_threshold" in opts and opts["sync_threshold"] is not None:
+        md["hnsw:sync_threshold"] = int(opts["sync_threshold"])
+    if "batch_size" in opts and opts["batch_size"] is not None:
+        md["hnsw:batch_size"] = int(opts["batch_size"])
+    return md
+
+
 # Below this size, data_level0.bin is too small for a meaningful HNSW graph.
 # Used by _hnsw_link_lists_is_usable_for_payload (empty link_lists is fine
 # when data is trivially small) and _missing_dimensionality_appears_recoverable
@@ -336,17 +449,7 @@ def _segment_appears_healthy(seg_dir: str) -> bool:
     if not _hnsw_payload_appears_sane(seg_dir):
         return False
 
-    try:
-        size = os.path.getsize(meta_path)
-        if size < 16:
-            return False
-        with open(meta_path, "rb") as f:
-            head = f.read(2)
-            f.seek(-1, 2)  # last byte
-            tail = f.read(1)
-    except OSError:
-        return False
-    return len(head) == 2 and head[0] == 0x80 and tail == b"\x2e"
+    return _hnsw_metadata_marker_intact(seg_dir)
 
 
 def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 300.0) -> list[str]:
@@ -597,9 +700,10 @@ def _hnsw_element_count(palace_path: str, segment_id: str) -> Optional[int]:
 # read the collection metadata (older palaces missing the row, sqlite
 # unreadable). 2000 = 2 × chromadb's default sync_threshold of 1000.
 #
-# Why dynamic: legacy palaces may still carry ``sync_threshold = 50_000``
-# (the pre-#1579 guard), so flush-lag can grow up to 50K on those palaces.
-# New palaces use sync_threshold=2 (#1579) and flush almost immediately.
+# Why dynamic: a palace carries whatever ``sync_threshold`` it was created
+# under, and flush-lag grows to that threshold before a persist fires — up
+# to 50K on a palace created under an old large guard, and 2 on one created
+# under the small guard that #1308 traces back to.
 # A fixed 2000 floor would flag actively-written legacy palaces as
 # DIVERGED the moment their queue exceeded 10% of sqlite_count, even
 # though chromadb is behaving correctly. The floor must scale with the
@@ -691,6 +795,112 @@ def _hnsw_metadata_age_seconds(palace_path: str, segment_id: str) -> Optional[fl
         return None
 
 
+# Probe verdicts keyed by ``(palace_path, collection_name)``; each value is
+# ``(fingerprint, segment_id, status, probed_at)``. Written as one tuple
+# assignment, so a concurrent reader either sees the previous entry or the new
+# one, never a half-updated pair. Two threads that miss together simply both
+# run the probe and the last writer wins — a benign race, and cheaper than
+# serializing every reader behind a lock.
+_capacity_cache: dict[tuple[str, str], tuple[tuple, Optional[str], dict, float]] = {}
+
+# Bumped by every :func:`reset_hnsw_capacity_cache`. A probe reads it before
+# running and refuses to store its verdict if it changed meanwhile, so a probe
+# already in flight when a reset lands cannot repopulate the entry the reset
+# was meant to discard (the ``tool_reconnect`` race).
+_capacity_cache_generation = 0
+
+# A long-lived server watches one palace and a handful of collections, so the
+# map stays tiny; the bound only exists so a process that walks many palaces
+# (benchmarks, batch tooling) cannot grow it without limit.
+_CAPACITY_CACHE_MAX_ENTRIES = 32
+
+# Ceiling on how long one verdict may be reused, as a backstop for filesystems
+# whose timestamps are too coarse to notice a quick rewrite: FAT32 stores mtime
+# at 2 s granularity, exFAT at 10 ms, and a write that lands inside existing
+# sqlite pages need not change the file size either. On ext4/APFS/NTFS the
+# signature already catches every write, so this ceiling never fires in
+# practice. It bounds the worst case; it is not the freshness mechanism.
+_CAPACITY_CACHE_MAX_AGE_SECONDS = 10.0
+
+
+def _stat_signature(path: str) -> tuple[int, int, int]:
+    """Return ``(inode, mtime_ns, size)`` for ``path``, all zeros when absent.
+
+    Catches every exception, not just ``OSError``: a palace path carrying an
+    embedded null byte makes ``os.stat`` raise ``ValueError``, and
+    :func:`hnsw_capacity_status` promises never to raise. An unreadable path
+    simply yields the "absent" signature and the probe reports ``unknown``.
+    """
+    try:
+        st = os.stat(path)
+    except Exception:
+        return (0, 0, 0)
+    return (st.st_ino, st.st_mtime_ns, st.st_size)
+
+
+def _db_family_signature(palace_path: str) -> tuple:
+    """Signature of the sqlite files whose contents the probe depends on.
+
+    chromadb 1.5.x leaves ``chroma.sqlite3`` in ``journal_mode=delete``, so the
+    ``-wal`` sidecar usually does not exist and stat'ing it is one cheap miss.
+    It is covered anyway because the journal mode belongs to the database
+    rather than to this code: under WAL a writer appends rows the probe would
+    count while the main file's own mtime stays put, and the verdict would
+    otherwise be reused against data it never saw.
+
+    ``-shm`` is deliberately excluded. It is the WAL index in shared memory,
+    and sqlite restamps it every time a connection opens the database — even
+    read-only. Including it would make the probe invalidate its own cache on
+    every call, so on a WAL-mode palace the cache would never hit.
+    """
+    db_path = os.path.join(palace_path, "chroma.sqlite3")
+    return (
+        _stat_signature(db_path),
+        _stat_signature(db_path + "-wal"),
+    )
+
+
+def _pickle_signature(palace_path: str, segment_id: Optional[str]) -> tuple[int, int, int]:
+    """Signature of the segment's ``index_metadata.pickle``."""
+    if not segment_id:
+        return (0, 0, 0)
+    return _stat_signature(os.path.join(palace_path, segment_id, "index_metadata.pickle"))
+
+
+def _segment_id_safe(palace_path: str, collection_name: str) -> Optional[str]:
+    """``_vector_segment_id`` that never raises, for the pre-probe signature."""
+    try:
+        return _vector_segment_id(palace_path, collection_name)
+    except Exception:
+        return None
+
+
+def _capacity_fingerprint(palace_path: str, segment_id: Optional[str]) -> tuple:
+    """Signature over every file the probe reads: the sqlite family + the pickle.
+
+    Both halves must be captured for the same ``segment_id`` so a rewrite of
+    ``index_metadata.pickle`` is caught. The probe reads that pickle partway
+    through, then makes two more sqlite calls, so a signature taken only after
+    the probe returned would record a mid-probe pickle rewrite as "unchanged"
+    while the verdict still reflected the pre-write file (#1471 review).
+    """
+    return (_db_family_signature(palace_path), _pickle_signature(palace_path, segment_id))
+
+
+def reset_hnsw_capacity_cache() -> None:
+    """Forget every cached capacity verdict.
+
+    The signature check already picks up on-disk changes on its own; this is
+    for callers that drop all cached palace state at once (``tool_reconnect``,
+    ``_force_chroma_cache_reset``) and for tests that want a probe to run
+    unconditionally. Bumps the generation so a probe already running cannot
+    re-store the entry this call just dropped.
+    """
+    global _capacity_cache_generation
+    _capacity_cache_generation += 1
+    _capacity_cache.clear()
+
+
 def hnsw_capacity_status(palace_path: str, collection_name: str = "mempalace_drawers") -> dict:
     """Compare sqlite embedding count against HNSW element count.
 
@@ -711,7 +921,70 @@ def hnsw_capacity_status(palace_path: str, collection_name: str = "mempalace_dra
     * ``message``          — human-readable summary
 
     Never raises — a probe that throws would defeat the point.
+
+    A fully-measured verdict is cached per ``(palace_path, collection_name)``
+    and reused while every file the probe reads is unchanged on disk (#1471).
+    Each call otherwise costs a ``COUNT(*)`` over the embeddings table and a
+    full unpickle of the segment metadata — the two dominant costs — plus a few
+    small sqlite reads, on a path every search, duplicate check and status call
+    runs through. A verdict the probe could not fully measure (``sqlite_count``
+    is ``None`` from a locked database, or there is no palace yet) is returned
+    but never cached, so a transient failure cannot pin a false reading.
+
+    Freshness comes from an ``(inode, mtime_ns, size)`` signature rather than a
+    wall-clock TTL, so an external writer — ``mempalace repair``, a peer mine,
+    another process — invalidates the verdict as soon as it touches the files,
+    instead of leaving the #1222 guard blind for a fixed window.
+    ``_CAPACITY_CACHE_MAX_AGE_SECONDS`` caps how long one verdict may be
+    reused, but only as a backstop for filesystems with coarse timestamps; the
+    signature is what makes the verdict fresh.
+
+    Unlike :meth:`ChromaBackend._client`, which tolerates a 0.01 s mtime
+    epsilon to avoid rebuilding an expensive client, this compares exactly:
+    re-running the probe costs milliseconds, whereas serving one stale verdict
+    can route a query into a diverged segment.
     """
+    key = (palace_path, collection_name)
+    cached = _capacity_cache.get(key)
+    if cached is not None:
+        fingerprint, cached_segment, status, probed_at = cached
+        if time.monotonic() - probed_at <= _CAPACITY_CACHE_MAX_AGE_SECONDS:
+            if _capacity_fingerprint(palace_path, cached_segment) == fingerprint:
+                return dict(status)
+
+    generation = _capacity_cache_generation
+    # Snapshot the files the probe is about to read, before it reads them, and
+    # again after — using the segment id the probe itself resolved. Caching
+    # only when both snapshots agree makes an external write during the probe
+    # (sqlite OR the pickle) fall through uncached rather than pin a verdict
+    # the disk no longer supports.
+    before = _capacity_fingerprint(palace_path, _segment_id_safe(palace_path, collection_name))
+    out = _hnsw_capacity_status_uncached(palace_path, collection_name)
+    segment_id = out.get("segment_id")
+    after = _capacity_fingerprint(palace_path, segment_id)
+    cacheable = (
+        before == after
+        # A None sqlite_count means the probe could not read the database
+        # (transient lock/error), not a real "unknown" — pinning it would go
+        # blind for the whole ceiling. A None segment id has no pickle path to
+        # watch, so its fingerprint can never notice a first flush.
+        and out.get("sqlite_count") is not None
+        and segment_id is not None
+        # A reset that landed while this probe ran already dropped the entry
+        # it was told to; do not resurrect it.
+        and generation == _capacity_cache_generation
+    )
+    if cacheable:
+        if len(_capacity_cache) >= _CAPACITY_CACHE_MAX_ENTRIES:
+            _capacity_cache.clear()
+        _capacity_cache[key] = (after, segment_id, dict(out), time.monotonic())
+    return out
+
+
+def _hnsw_capacity_status_uncached(
+    palace_path: str, collection_name: str = "mempalace_drawers"
+) -> dict:
+    """Run the capacity probe, bypassing the cache. See :func:`hnsw_capacity_status`."""
     out: dict[str, Any] = {
         "segment_id": None,
         "sqlite_count": None,
@@ -1339,8 +1612,8 @@ class ChromaCollection(BaseCollection):
 
     @staticmethod
     def _sanitize_documents_for_chromadb(documents):
-        """Strip lone UTF-16 surrogates from every document before it reaches
-        the chromadb client.
+        """Strip lone UTF-16 surrogates and embedded NUL characters from every
+        document before it reaches the chromadb client.
 
         A single lone surrogate (U+D800–U+DFFF) raises ``UnicodeEncodeError``
         inside chromadb's encode path and aborts the *entire* add/upsert batch
@@ -1353,18 +1626,29 @@ class ChromaCollection(BaseCollection):
         directly. Sanitising here makes the chokepoint catch-all complete: the
         sibling :meth:`_sanitize_metadatas_for_chromadb` already guarantees this
         for metadata one method over; documents get the same guarantee.
+
+        A document containing an embedded NUL (U+0000) — routine in mined
+        Bash tool output — is well-formed UTF-8, unlike a lone surrogate, but
+        can corrupt the FTS5 inverted index for the whole collection rather
+        than just failing to store that one row (a ChromaDB-side bug; tracked
+        upstream). Stripping it here is the same defense-in-depth already
+        applied to surrogates: sanitize input we don't control before it
+        reaches a datastore we don't control.
         """
         if documents is None:
             return None
-        from ..config import strip_lone_surrogates
+        from ..config import strip_lone_surrogates, strip_nul_bytes
+
+        def _sanitize(d):
+            return strip_nul_bytes(strip_lone_surrogates(d))
 
         # chromadb accepts OneOrMany[Document]: a bare str is a single document,
         # not an iterable of characters. Handle it explicitly so we don't split
         # it into per-character documents — that would be exactly the kind of
         # silent corruption this method exists to prevent.
         if isinstance(documents, str):
-            return strip_lone_surrogates(documents)
-        return [strip_lone_surrogates(d) if isinstance(d, str) else d for d in documents]
+            return _sanitize(documents)
+        return [_sanitize(d) if isinstance(d, str) else d for d in documents]
 
     def add(self, *, documents, ids, metadatas=None, embeddings=None):
         kwargs: dict[str, Any] = {
@@ -1869,6 +2153,7 @@ class ChromaBackend(BaseBackend):
     name = "chroma"
     capabilities = frozenset(
         {
+            "requires_explicit_embeddings",
             "supports_embeddings_in",
             "supports_embeddings_passthrough",
             "supports_embeddings_out",
@@ -1924,6 +2209,7 @@ class ChromaBackend(BaseBackend):
             current_model = MempalaceConfig().embedding_model
         except Exception:
             current_model = "unknown"
+        rebuild_cmd = f"mempalace --palace {shlex.quote(palace_path)} repair rebuild-index"
         return (
             f"Embedding model mismatch reading palace at {palace_path!r}.\n"
             f"  Underlying ChromaDB error: {msg}\n"
@@ -1931,8 +2217,8 @@ class ChromaBackend(BaseBackend):
             f"  The palace was built with a different embedding model. Either:\n"
             f"    (a) revert the model: unset MEMPALACE_EMBEDDING_MODEL (or set "
             f"the previous value), or\n"
-            f"    (b) re-embed in place: `mempalace repair rebuild-index "
-            f"--palace {palace_path}` (writes new vectors with the current model)."
+            f"    (b) re-embed in place: `{rebuild_cmd}` "
+            f"(writes new vectors with the current model)."
         )
 
     # ------------------------------------------------------------------
@@ -2109,6 +2395,7 @@ class ChromaBackend(BaseBackend):
           — still used by callers not yet migrated.
         """
         palace_ref, collection_name, create, options = _normalize_get_collection_args(args, kwargs)
+        self.require_namespace_support(palace_ref)
 
         palace_path = palace_ref.local_path
         if palace_path is None:
@@ -2125,9 +2412,6 @@ class ChromaBackend(BaseBackend):
                 pass
 
         client = self._client(palace_path)
-        hnsw_space = "cosine"
-        if options and isinstance(options, dict):
-            hnsw_space = options.get("hnsw_space", hnsw_space)
 
         ef = self._resolve_embedding_function()
         ef_kwargs = {"embedding_function": ef} if ef is not None else {}
@@ -2138,11 +2422,7 @@ class ChromaBackend(BaseBackend):
             except _ChromaNotFoundError:
                 collection = client.create_collection(
                     collection_name,
-                    metadata={
-                        "hnsw:space": hnsw_space,
-                        "hnsw:num_threads": 1,
-                        **_HNSW_BLOAT_GUARD,
-                    },
+                    metadata=_hnsw_creation_metadata(options),
                     **ef_kwargs,
                 )
             except ValueError as e:
@@ -2191,7 +2471,26 @@ class ChromaBackend(BaseBackend):
 
     @classmethod
     def detect(cls, path: str) -> bool:
-        return os.path.isfile(os.path.join(path, "chroma.sqlite3"))
+        """Return True when ``path`` looks like a chroma palace.
+
+        Verifies the SQLite magic header rather than file presence alone.
+        Bare ``sqlite3.connect()`` against a missing path leaves a 0-byte
+        file behind (the SQLite header is written on the first statement,
+        not on connection), so file-presence alone treats those artifacts
+        as real chroma palaces and breaks multi-backend resolution. The
+        16-byte ``SQLite format 3\\x00`` magic prefix is written as soon
+        as chromadb's ``PersistentClient`` does any work, so this check
+        accepts every real chroma palace while rejecting empty / garbage
+        files. See #1893.
+        """
+        db_path = os.path.join(path, "chroma.sqlite3")
+        if not os.path.isfile(db_path):
+            return False
+        try:
+            with open(db_path, "rb") as f:
+                return f.read(16) == b"SQLite format 3\x00"
+        except OSError:
+            return False
 
     # ------------------------------------------------------------------
     # Legacy (pre-RFC 001) surface — retained while callers migrate.
@@ -2213,11 +2512,7 @@ class ChromaBackend(BaseBackend):
         ef_kwargs = {"embedding_function": ef} if ef is not None else {}
         collection = self._client(palace_path).create_collection(
             collection_name,
-            metadata={
-                "hnsw:space": hnsw_space,
-                "hnsw:num_threads": 1,
-                **_HNSW_BLOAT_GUARD,
-            },
+            metadata=_hnsw_creation_metadata({"hnsw_space": hnsw_space}),
             **ef_kwargs,
         )
         return ChromaCollection(collection, palace_path=palace_path)

mempalace/backends/embedding_wrapper.py

@@ -8,14 +8,26 @@ from .base import BaseCollection
 
 
 def _embed_texts(texts: list[str]) -> list[list[float]]:
-    """Embed ``texts`` with the configured local embedding function."""
+    """Embed ``texts`` with the configured local embedding function.
+
+    Embedding functions return ``list[np.ndarray]`` (float32). ``list(arr)``
+    would unpack that into ``np.float32`` *scalars*, which ChromaDB's
+    ``normalize_embeddings`` rejects outright ("Expected embeddings to be a
+    list of floats or ints..."), so every write through this wrapper must
+    convert to real Python floats. ``.tolist()`` does that in C; the
+    ``float(x)`` branch covers embedders that already hand back plain
+    sequences.
+    """
     if not texts:
         return []
     from ..embedding import get_embedding_function
 
     ef = get_embedding_function()
     vectors = ef(input=texts)
-    return [list(v) for v in vectors]
+    return [
+        v.tolist() if hasattr(v, "tolist") else [float(x) for x in v]  # numpy | plain sequence
+        for v in vectors
+    ]
 
 
 def _as_list(value):
@@ -155,6 +167,30 @@ class EmbeddingCollection(BaseCollection):
     def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None):
         return self._inner.lexical_search(query=query, n_results=n_results, where=where)
 
+    def facet_counts(
+        self, field: str, where: Optional[dict] = None, limit: int = 1000
+    ) -> dict[str, int]:
+        # ``BaseCollection.facet_counts`` is a concrete method that raises
+        # ``UnsupportedCapabilityError`` as its default. MRO resolves it on
+        # this subclass before ``__getattr__`` ever fires, so without an
+        # explicit forwarder every facet call against a wrapped backend
+        # (qdrant, pgvector, sqlite_exact) raises and silently degrades to
+        # client-side counting in mcp_server's try/except.
+        return self._inner.facet_counts(field, where=where, limit=limit)
+
+    def get_all_metadata(self, where: Optional[dict] = None) -> list[dict]:
+        # ``BaseCollection.get_all_metadata`` ships a concrete default that
+        # pages through ``self.get(include=["metadatas"])``. Without this
+        # forwarder, MRO resolves the call here on the subclass and runs the
+        # base default — which routes back through ``self.get()`` (the
+        # wrapper's get, then ``__getattr__`` to the inner's get). Result:
+        # the inner's overridden ``get_all_metadata`` (e.g. pgvector's
+        # ``with_document=False`` fast path from #1892) is never reached,
+        # and every metadata-only fetch transfers the full document column
+        # over the wire. Same MRO-shadow pattern as ``facet_counts`` /
+        # ``lexical_search`` above.
+        return self._inner.get_all_metadata(where=where)
+
     def update(self, *, ids, documents=None, metadatas=None, embeddings=None):
         ids = _as_list(ids)
         if documents is not None:

mempalace/backends/pgvector.py

@@ -54,6 +54,7 @@ from .base import (
     PalaceNotFoundError,
     PalaceRef,
     QueryResult,
+    UnsupportedCapabilityError,
     UnsupportedFilterError,
     _IncludeSpec,
 )
@@ -674,13 +675,19 @@ class _PgVectorClient:
         *,
         where: Optional[dict] = None,
         with_embedding: bool = False,
+        with_document: bool = True,
         limit: Optional[int] = None,
         offset: Optional[int] = None,
     ) -> list[dict]:
         qi = _quote_identifier(table)
         params: list = []
         where_sql = _where_to_sql(where, params) if where else "TRUE"
-        cols = "id, document, metadata"
+        # Project NULL into the document slot when the caller only needs
+        # metadata (e.g. mempalace_status's wing/room tally). Keeps the
+        # positional _row parser unchanged — document remains record[1] —
+        # while avoiding O(n × document_size) bytes over the wire on remote
+        # pgvector deployments. Follow-up to #1840.
+        cols = "id, document, metadata" if with_document else "id, NULL::text, metadata"
         if with_embedding:
             cols += ", embedding"
         sql = f"SELECT {cols} FROM {qi} WHERE {where_sql}"
@@ -721,6 +728,29 @@ class _PgVectorClient:
         rows = self._execute(f"SELECT count(*) FROM {_quote_identifier(table)}", fetch=True)
         return int(rows[0][0]) if rows and rows[0] else 0
 
+    def facet_counts(
+        self,
+        table: str,
+        *,
+        field: str,
+        where: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        qi = _quote_identifier(table)
+        params: list = [field]
+        where_sql = _where_to_sql(where, params) if where else "TRUE"
+        sql = (
+            f"SELECT metadata->>%s AS k, count(*) AS c "
+            f"FROM {qi} WHERE {where_sql} "
+            f"GROUP BY 1 ORDER BY 2 DESC, 1 LIMIT %s"
+        )
+        params.append(int(limit))
+        rows = self._execute(sql, params, fetch=True)
+        # Exclude NULL bucket: matches Qdrant's facet semantics; mcp_server
+        # reconciles missing values into "unknown" via the count diff
+        # (count(*) - sum(facet_values)).
+        return {str(row[0]): int(row[1]) for row in rows if row[0] is not None}
+
     def drop_table(self, table: str) -> None:
         self._execute(f"DROP TABLE IF EXISTS {_quote_identifier(table)}")
 
@@ -855,7 +885,15 @@ class PgVectorCollection(BaseCollection):
                 )
             self._known_dimension = existing_dim or dimension
 
-    def _scroll(self, *, where=None, with_embedding=False, limit=None, offset=None) -> list[dict]:
+    def _scroll(
+        self,
+        *,
+        where=None,
+        with_embedding=False,
+        with_document=True,
+        limit=None,
+        offset=None,
+    ) -> list[dict]:
         self._ensure_open()
         if not self._table_exists():
             if self._marker_exists():
@@ -865,10 +903,39 @@ class PgVectorCollection(BaseCollection):
             self._table,
             where=where,
             with_embedding=with_embedding,
+            with_document=with_document,
             limit=limit,
             offset=offset,
         )
 
+    def get_all_metadata(self, where=None) -> list[dict]:
+        """Single-pass metadata-only fetch — projects out the document column.
+
+        The base implementation pages through ``get(include=["metadatas"])``,
+        which routes here via ``_scroll`` and (pre-this-override) always sent
+        the ``document`` text over the wire even when nothing consumed it.
+        For pgvector deployments where the client is remote (TLS over WAN),
+        that meant ``mempalace_status`` transferred O(n × document_size)
+        bytes per call, dominating wall time. With ``with_document=False``
+        the SELECT replaces document with NULL, dropping the per-row payload
+        to id + metadata for every caller of this method.
+
+        Filtered fetches still need the ``_matches_where`` post-filter for
+        non-pushdown semantics (array/object values where ``metadata @> ...``
+        is broader than the exact match the caller asked for — same
+        correctness contract as #1840's filtered ``get`` path). Since that
+        post-filter only reads ``metadata``, we keep the single-scroll +
+        ``with_document=False`` fast path and just apply the filter locally
+        on the metadata dicts before returning. This extends the wire-byte
+        win to filtered callers as well.
+        """
+        _validate_where(where)
+        pushdown = None if _requires_local_filter(where) else where
+        rows = self._scroll(where=pushdown, with_document=False)
+        if where is None:
+            return [row["metadata"] for row in rows]
+        return [row["metadata"] for row in rows if _matches_where(row["metadata"], where)]
+
     def _rows(
         self,
         *,
@@ -1151,6 +1218,26 @@ class PgVectorCollection(BaseCollection):
             return 0
         return self._client.count_rows(self._table)
 
+    def facet_counts(
+        self,
+        field: str,
+        where: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        self._ensure_open()
+        # Validate the filter before the existence short-circuit so an
+        # unsupported local-only filter raises even on an unmaterialized
+        # collection — matches the order used by get()/lexical_search() and
+        # qdrant.facet_counts (PR #1868 review).
+        _validate_where(where)
+        if _requires_local_filter(where):
+            raise UnsupportedCapabilityError("facet_counts does not support local-only filters")
+        if not self._table_exists():
+            if self._marker_exists():
+                raise CollectionNotInitializedError(self._collection_name)
+            return {}
+        return self._client.facet_counts(self._table, field=field, where=where, limit=limit)
+
     def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None):
         _validate_where(where)
         pushdown = None if _requires_local_filter(where) else where
@@ -1248,6 +1335,7 @@ class PgVectorBackend(BaseBackend):
             "supports_embeddings_out",
             "supports_metadata_filters",
             "supports_lexical_search",
+            "supports_metadata_facets",
             "supports_namespace_isolation",
             "supports_server_side_indexes",
             "server_mode",

mempalace/backends/qdrant.py

@@ -40,6 +40,7 @@ from .base import (
     PalaceNotFoundError,
     PalaceRef,
     QueryResult,
+    UnsupportedCapabilityError,
     UnsupportedFilterError,
     _IncludeSpec,
 )
@@ -543,6 +544,38 @@ class _QdrantRESTClient:
         result = response.get("result") or {}
         return int(result.get("count") or 0)
 
+    def facet_counts(
+        self,
+        collection: str,
+        *,
+        field: str,
+        qdrant_filter: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        body: dict[str, Any] = {
+            "key": field,
+            "exact": True,
+            "limit": limit,
+        }
+
+        if qdrant_filter:
+            body["filter"] = qdrant_filter
+
+        response = self.request(
+            "POST",
+            f"/collections/{urlparse.quote(collection, safe='')}/facet",
+            body=body,
+        )
+
+        result = response.get("result") or {}
+        hits = result.get("hits") or []
+
+        return {
+            str(hit["value"]): int(hit.get("count") or 0)
+            for hit in hits
+            if hit.get("value") is not None
+        }
+
     def delete_collection(self, collection: str) -> None:
         self.request("DELETE", f"/collections/{urlparse.quote(collection, safe='')}")
 
@@ -1035,6 +1068,34 @@ class QdrantCollection(BaseCollection):
         rows = self._rows(where=where)
         return [row["metadata"] for row in rows]
 
+    def facet_counts(
+        self,
+        field: str,
+        where: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        self._ensure_open()
+        # Validate the filter before the existence short-circuit so an
+        # unsupported local-only filter raises regardless of whether the
+        # collection has been materialized yet — matching the order used by
+        # get()/lexical_search() above (#1835 review).
+        _validate_where(where)
+        if _requires_local_filter(where):
+            raise UnsupportedCapabilityError("facet_counts does not support local-only filters")
+        if not self._remote_exists():
+            if self._marker_exists():
+                raise CollectionNotInitializedError(self._collection_name)
+            return {}
+
+        q_filter = _qdrant_filter(where)
+
+        return self._client.facet_counts(
+            self._remote_collection,
+            field=f"{_PAYLOAD_METADATA}.{field}",
+            qdrant_filter=q_filter,
+            limit=limit,
+        )
+
     def delete(self, *, ids=None, where=None):
         _validate_where(where)
         if not self._remote_exists():
@@ -1125,6 +1186,7 @@ class QdrantBackend(BaseBackend):
             "supports_embeddings_out",
             "supports_metadata_filters",
             "supports_lexical_search",
+            "supports_metadata_facets",
             "supports_namespace_isolation",
             "server_mode",
         }

mempalace/backends/registry.py

@@ -207,6 +207,7 @@ def resolve_backend_for_palace(
 def _register_builtins() -> None:
     """Register chroma as the in-tree default."""
     from .chroma import ChromaBackend
+    from .milvus import MilvusBackend
     from .pgvector import PgVectorBackend
     from .qdrant import QdrantBackend
     from .sqlite_exact import SQLiteExactBackend
@@ -214,6 +215,8 @@ def _register_builtins() -> None:
     # Use setdefault semantics so a caller that pre-registered for tests wins.
     if "chroma" not in _registry:
         _registry["chroma"] = ChromaBackend
+    if "milvus" not in _registry:
+        _registry["milvus"] = MilvusBackend
     if "qdrant" not in _registry:
         _registry["qdrant"] = QdrantBackend
     if "sqlite_exact" not in _registry:

mempalace/backends/sqlite_exact.py

@@ -15,12 +15,14 @@ import re
 import sqlite3
 import threading
 from datetime import datetime, timezone
+from pathlib import Path
 from typing import Any, Optional
 
 import numpy as np
 
 from .base import (
     BackendClosedError,
+    BackendError,
     BaseBackend,
     BaseCollection,
     CollectionNotInitializedError,
@@ -247,9 +249,23 @@ def _validate_write_batch(
 
 
 class _SQLiteExactHandle:
-    def __init__(self, conn: sqlite3.Connection, lock: threading.RLock):
+    def __init__(
+        self,
+        conn: sqlite3.Connection,
+        lock: threading.RLock,
+        palace_path: str,
+        *,
+        read_only: bool = False,
+        immutable: bool = False,
+    ):
         self.conn = conn
         self.lock = lock
+        self.palace_path = palace_path
+        self.read_only = read_only
+        # True when opened with ``immutable=1`` because no WAL existed at connect
+        # time. A later writer can create WAL sidecars that this connection will
+        # never see, so the backend must reopen once those files appear.
+        self.immutable = immutable
         self.closed = False
 
 
@@ -264,8 +280,27 @@ class SQLiteExactCollection(BaseCollection):
             raise BackendClosedError("SQLiteExactCollection has been closed")
 
     @contextlib.contextmanager
-    def _cursor(self):
+    def _write_lock(self):
+        """Serialize this handle before taking process-wide writer ownership.
+
+        ``mine_palace_lock`` grants cross-thread re-entrant access whenever
+        this process already owns the palace. Taking it before ``handle.lock``
+        lets a waiting thread consume that re-entrant credit, outlive the
+        thread that owns the OS lease, and then mutate after the lease has been
+        released. The handle mutex must therefore be the outer context.
+        """
+        # Late import avoids a palace.py -> backend -> palace.py cycle.
+        from ..palace import mine_palace_lock
+
         with self._handle.lock:
+            self._ensure_open()
+            with mine_palace_lock(self._handle.palace_path):
+                yield
+
+    @contextlib.contextmanager
+    def _cursor(self, *, write: bool = False):
+        serialization = self._write_lock() if write else self._handle.lock
+        with serialization:
             self._ensure_open()
             cur = self._handle.conn.cursor()
             try:
@@ -345,7 +380,7 @@ class SQLiteExactCollection(BaseCollection):
     def set_embedder_identity(self, identity) -> None:
         if not identity or not identity.model_name:
             return
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             cur.execute(
                 "INSERT INTO meta(key, value) VALUES (?, ?) "
                 "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
@@ -375,7 +410,7 @@ class SQLiteExactCollection(BaseCollection):
             raise ValueError("sqlite_exact requires explicit embeddings")
         metadatas = metadatas or [{} for _ in ids]
         now = _utcnow()
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             collection_id = self._collection_id(cur)
             prepared = []
             for doc_id, doc, meta, emb in zip(ids, documents, metadatas, embeddings):
@@ -413,7 +448,7 @@ class SQLiteExactCollection(BaseCollection):
             raise ValueError("sqlite_exact requires explicit embeddings")
         metadatas = metadatas or [{} for _ in ids]
         now = _utcnow()
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             collection_id = self._collection_id(cur)
             prepared = []
             for doc_id, doc, meta, emb in zip(ids, documents, metadatas, embeddings):
@@ -457,7 +492,7 @@ class SQLiteExactCollection(BaseCollection):
         ):
             if value is not None and len(value) != n:
                 raise ValueError(f"{label} length {len(value)} does not match ids length {n}")
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             collection_id = self._collection_id(cur)
             updates = []
             for idx, doc_id in enumerate(ids):
@@ -653,7 +688,7 @@ class SQLiteExactCollection(BaseCollection):
         )
 
     def delete(self, *, ids=None, where=None):
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             collection_id = self._collection_id(cur)
             if ids is None:
                 rows = self._rows(cur, where=where)
@@ -797,17 +832,16 @@ class SQLiteExactCollection(BaseCollection):
             )
         if kind == "analyze":
             # Refresh planner stats. Concurrent runs serialize on the handle lock.
-            with self._cursor() as cur:
+            with self._cursor(write=True) as cur:
                 cur.execute("ANALYZE")
             return MaintenanceResult(kind="analyze", status="ran")
 
         # compact → VACUUM. It cannot run inside a transaction, so flip the
-        # connection to autocommit for the duration. The handle lock serializes
-        # concurrent runs in-process; SQLite's own write lock serializes across
-        # processes.
+        # connection to autocommit for the duration. _write_lock takes the
+        # handle mutex before the palace lease so a waiting thread cannot
+        # retain stale process-reentrant ownership after another thread exits.
         before = self.maintenance_state()
-        with self._handle.lock:
-            self._ensure_open()
+        with self._write_lock():
             conn = self._handle.conn
             prev_isolation = conn.isolation_level
             try:
@@ -848,6 +882,7 @@ class SQLiteExactBackend(BaseBackend):
 
     def __init__(self):
         self._clients: dict[str, _SQLiteExactHandle] = {}
+        self._read_only_clients: dict[str, _SQLiteExactHandle] = {}
         self._clients_lock = threading.RLock()
         self._closed = False
 
@@ -855,9 +890,64 @@ class SQLiteExactBackend(BaseBackend):
     def _db_path(palace_path: str) -> str:
         return os.path.join(palace_path, _DB_FILENAME)
 
-    def _connect(self, palace_path: str, create: bool):
+    @staticmethod
+    def _wal_sidecar_state(db_path: str) -> tuple[bool, bool]:
+        return (
+            os.path.isfile(f"{db_path}-wal"),
+            os.path.isfile(f"{db_path}-shm"),
+        )
+
+    @staticmethod
+    def _connect_read_only(db_path: str) -> tuple[sqlite3.Connection, bool]:
+        """Open without creating WAL files while preserving an active WAL.
+
+        Returns ``(connection, immutable)``. ``immutable`` is True when the
+        database was clean (no WAL) and was opened with ``immutable=1``.
+        """
+        wal_exists, shm_exists = SQLiteExactBackend._wal_sidecar_state(db_path)
+        if wal_exists != shm_exists:
+            raise BackendError(
+                "sqlite_exact read-only open found an incomplete WAL sidecar set; "
+                "open the palace after its writer exits cleanly or restore both "
+                "the -wal and -shm files"
+            )
+
+        db_uri = Path(db_path).resolve().as_uri()
+        if wal_exists:
+            # An active writer's uncheckpointed rows live in the WAL. With both
+            # sidecars already present, mode=ro can read them without creating
+            # filesystem state, including on a read-only mount.
+            db_uri = f"{db_uri}?mode=ro"
+            immutable = False
+        else:
+            # A clean WAL-mode database would otherwise make SQLite create new
+            # -wal/-shm files while connecting. Immutable mode is safe here
+            # only until a writer creates sidecars this connection would miss.
+            db_uri = f"{db_uri}?mode=ro&immutable=1"
+            immutable = True
+        return sqlite3.connect(db_uri, uri=True, check_same_thread=False), immutable
+
+    def _retire_read_only_handle(self, palace_path: str, handle: _SQLiteExactHandle) -> None:
+        """Drop a cached read-only handle so the next open re-evaluates WAL state."""
+        self._read_only_clients.pop(palace_path, None)
+        with handle.lock:
+            if handle.closed:
+                return
+            handle.closed = True
+            try:
+                handle.conn.close()
+            except Exception:
+                logger.debug(
+                    "Failed to close stale immutable sqlite_exact reader for %s",
+                    palace_path,
+                    exc_info=True,
+                )
+
+    def _connect(self, palace_path: str, create: bool, *, read_only: bool = False):
         if self._closed:
             raise BackendClosedError("SQLiteExactBackend has been closed")
+        if create and read_only:
+            raise ValueError("sqlite_exact read-only connections cannot create a palace")
         db_path = self._db_path(palace_path)
         if not create and not os.path.isfile(db_path):
             raise PalaceNotFoundError(db_path)
@@ -877,20 +967,52 @@ class SQLiteExactBackend(BaseBackend):
         with self._clients_lock:
             if self._closed:
                 raise BackendClosedError("SQLiteExactBackend has been closed")
-            cached = self._clients.get(palace_path)
+            clients = self._read_only_clients if read_only else self._clients
+            cached = clients.get(palace_path)
             if cached is not None and not cached.closed:
-                return cached
-            conn = sqlite3.connect(db_path, check_same_thread=False)
+                if read_only and cached.immutable:
+                    # An immutable snapshot freezes the clean-database view.
+                    # Reopen only when the complete WAL sidecar pair is present
+                    # (active writer). A partial pair is a transient mid-open
+                    # state — keep the immutable handle rather than forcing a
+                    # reconnect that would raise on the incomplete set.
+                    wal_exists, shm_exists = self._wal_sidecar_state(db_path)
+                    if wal_exists and shm_exists:
+                        self._retire_read_only_handle(palace_path, cached)
+                        cached = None
+                if cached is not None:
+                    return cached
+            if read_only:
+                conn, immutable = self._connect_read_only(db_path)
+            else:
+                conn = sqlite3.connect(db_path, check_same_thread=False)
+                immutable = False
             try:
                 conn.row_factory = sqlite3.Row
                 lock = threading.RLock()
-                handle = _SQLiteExactHandle(conn, lock)
+                handle = _SQLiteExactHandle(
+                    conn,
+                    lock,
+                    palace_path,
+                    read_only=read_only,
+                    immutable=immutable,
+                )
                 with handle.lock:
-                    self._init_schema(conn)
+                    if read_only:
+                        # ``mode=ro`` prevents filesystem writes. ``query_only``
+                        # adds a second connection-local guard so a future
+                        # refactor cannot accidentally introduce a temp/schema
+                        # write through a read-only MCP collection.
+                        conn.execute("PRAGMA query_only=ON")
+                    else:
+                        from ..palace import mine_palace_lock
+
+                        with mine_palace_lock(palace_path):
+                            self._init_schema(conn)
             except BaseException:
                 conn.close()
                 raise
-            self._clients[palace_path] = handle
+            clients[palace_path] = handle
             return handle
 
     def _init_schema(self, conn: sqlite3.Connection) -> None:
@@ -955,13 +1077,14 @@ class SQLiteExactBackend(BaseBackend):
         *args,
         **kwargs,
     ) -> SQLiteExactCollection:
-        palace, collection_name, create = self._normalize_args(args, kwargs)
+        palace, collection_name, create, read_only = self._normalize_args(args, kwargs)
+        self.require_namespace_support(palace)
         palace_path = palace.local_path
         if palace_path is None:
             raise PalaceNotFoundError("SQLiteExactBackend requires PalaceRef.local_path")
         if not create and not os.path.isdir(palace_path):
             raise PalaceNotFoundError(palace_path)
-        handle = self._connect(palace_path, create=create)
+        handle = self._connect(palace_path, create=create, read_only=read_only)
         with handle.lock:
             row = handle.conn.execute(
                 "SELECT id FROM collections WHERE name = ?",
@@ -970,11 +1093,14 @@ class SQLiteExactBackend(BaseBackend):
             if row is None:
                 if not create:
                     raise CollectionNotInitializedError(collection_name)
-                handle.conn.execute(
-                    "INSERT INTO collections(name, created_at) VALUES (?, ?)",
-                    (collection_name, _utcnow()),
-                )
-                handle.conn.commit()
+                from ..palace import mine_palace_lock
+
+                with mine_palace_lock(palace_path):
+                    handle.conn.execute(
+                        "INSERT INTO collections(name, created_at) VALUES (?, ?)",
+                        (collection_name, _utcnow()),
+                    )
+                    handle.conn.commit()
         return SQLiteExactCollection(handle, collection_name)
 
     @staticmethod
@@ -985,10 +1111,11 @@ class SQLiteExactBackend(BaseBackend):
                 raise TypeError("palace= must be a PalaceRef instance")
             collection_name = kwargs.pop("collection_name")
             create = bool(kwargs.pop("create", False))
-            kwargs.pop("options", None)
+            options = kwargs.pop("options", None) or {}
+            read_only = bool(options.get("read_only", False))
             if args or kwargs:
                 raise TypeError("unexpected arguments to get_collection")
-            return palace, collection_name, create
+            return palace, collection_name, create, read_only
         if args:
             palace_path = args[0]
             rest = list(args[1:])
@@ -996,18 +1123,32 @@ class SQLiteExactBackend(BaseBackend):
             if collection_name is None:
                 raise TypeError("collection_name is required")
             create = kwargs.pop("create", False)
+            options = kwargs.pop("options", None) or {}
+            read_only = bool(options.get("read_only", False))
             if rest:
                 create = rest.pop(0)
             if rest or kwargs:
                 raise TypeError("unexpected arguments to get_collection")
-            return PalaceRef(id=palace_path, local_path=palace_path), collection_name, bool(create)
+            return (
+                PalaceRef(id=palace_path, local_path=palace_path),
+                collection_name,
+                bool(create),
+                read_only,
+            )
         if "palace_path" in kwargs:
             palace_path = kwargs.pop("palace_path")
             collection_name = kwargs.pop("collection_name")
             create = bool(kwargs.pop("create", False))
+            options = kwargs.pop("options", None) or {}
+            read_only = bool(options.get("read_only", False))
             if kwargs:
                 raise TypeError("unexpected arguments to get_collection")
-            return PalaceRef(id=palace_path, local_path=palace_path), collection_name, create
+            return (
+                PalaceRef(id=palace_path, local_path=palace_path),
+                collection_name,
+                create,
+                read_only,
+            )
         raise TypeError("get_collection requires palace= or a positional palace_path")
 
     def close_palace(self, palace: PalaceRef | str) -> None:
@@ -1015,11 +1156,15 @@ class SQLiteExactBackend(BaseBackend):
         if path is None:
             return
         with self._clients_lock:
-            cached = self._clients.pop(path, None)
-        if cached is not None:
-            with cached.lock:
-                cached.closed = True
-                cached.conn.close()
+            cached_handles = [
+                self._clients.pop(path, None),
+                self._read_only_clients.pop(path, None),
+            ]
+        for cached in cached_handles:
+            if cached is not None:
+                with cached.lock:
+                    cached.closed = True
+                    cached.conn.close()
 
     def close(self) -> None:
         # Flip _closed under the registry lock so a concurrent _connect either
@@ -1028,8 +1173,9 @@ class SQLiteExactBackend(BaseBackend):
         # Unlocked readers of _closed elsewhere are advisory fast-fails; the
         # locked recheck in _connect is the authoritative gate.
         with self._clients_lock:
-            handles = list(self._clients.values())
+            handles = list(self._clients.values()) + list(self._read_only_clients.values())
             self._clients.clear()
+            self._read_only_clients.clear()
             self._closed = True
         for handle in handles:
             with handle.lock:
@@ -1045,7 +1191,23 @@ class SQLiteExactBackend(BaseBackend):
 
     @classmethod
     def detect(cls, path: str) -> bool:
-        return os.path.isfile(os.path.join(path, _DB_FILENAME))
+        """Return True when ``path`` looks like a sqlite_exact palace.
+
+        Verifies the SQLite magic header rather than file presence alone, for
+        the same reason as :py:meth:`mempalace.backends.chroma.ChromaBackend.detect`:
+        bare ``sqlite3.connect()`` against a missing path leaves a 0-byte file
+        behind because the SQLite header is written on the first statement,
+        not on connection. The 16-byte ``SQLite format 3\\x00`` magic prefix
+        accepts every real palace while rejecting empty / garbage files. See #1893.
+        """
+        db_path = os.path.join(path, _DB_FILENAME)
+        if not os.path.isfile(db_path):
+            return False
+        try:
+            with open(db_path, "rb") as f:
+                return f.read(16) == b"SQLite format 3\x00"
+        except OSError:
+            return False
 
     def create_collection(self, palace_path: str, collection_name: str) -> SQLiteExactCollection:
         return self.get_collection(palace_path, collection_name, create=True)

mempalace/backups.py

@@ -1,4 +1,4 @@
-"""Retention pruning for timestamped palace backups.
+"""Writing and pruning palace backups.
 
 ``mempalace migrate`` and ``mempalace repair max-seq-id`` each write a fresh,
 timestamped backup every time they run and historically never deleted the old
@@ -9,11 +9,222 @@ filling the disk. This module prunes the backup set down to a bounded count
 after each new backup is written.
 
 The retention count comes from ``MempalaceConfig.max_backups`` (default 10).
+
+``copy_palace_dir`` is the other half: the whole-directory copy that
+``mempalace repair`` in its default mode and ``mempalace migrate`` take before
+they overwrite a live palace. The other backup paths copy a single file and do
+not use it.
 """
 
 import glob
 import os
 import shutil
+import stat
+
+
+# Names for the file types a backup copy deliberately leaves behind. A socket
+# or a named pipe makes ``shutil.copytree`` record an error for that entry and
+# raise at the end. Device nodes do not, but dereferencing one copies the
+# DEVICE instead of palace data, and a link to ``/dev/zero`` is copied without
+# bound, so they are left out for the opposite reason: the copy would succeed
+# at the wrong thing.
+_UNCOPYABLE_FILE_TYPES = (
+    (stat.S_ISSOCK, "socket"),
+    (stat.S_ISFIFO, "named pipe"),
+    (stat.S_ISCHR, "character device"),
+    (stat.S_ISBLK, "block device"),
+)
+
+_UNNAMED_FILE_TYPE = "not a regular file or directory"
+
+
+def _file_type_label(mode):
+    """Name the file type in ``mode``, falling back to a generic phrase.
+
+    The fallback covers a file type this table has no name for. Nothing on
+    Linux reaches it, because ``stat`` there reports ``S_IFDOOR``,
+    ``S_IFPORT`` and ``S_IFWHT`` as 0. macOS does define ``S_IFWHT``, a
+    union-mount whiteout, which is precisely what the fallback exists for:
+    an entry no copy can carry is skipped with an honest reason rather than
+    read as copyable.
+    """
+    for is_type, label in _UNCOPYABLE_FILE_TYPES:
+        if is_type(mode):
+            return label
+    return _UNNAMED_FILE_TYPE
+
+
+def _uncopyable_reason(path, *, follow_symlinks):
+    """Name why ``path`` cannot be copied into a backup, or return ``None``.
+
+    This is an allowlist, matching how the rest of the package treats
+    directory entries: only regular files and directories are copyable, and
+    everything else is named and left out.
+
+    Args:
+        path: The directory entry to classify.
+        follow_symlinks: Mirrors the copy's own link handling. ``True`` when
+            the copy dereferences links, so the TARGET's file type decides;
+            ``False`` when it recreates them as links, in which case any
+            symlink is fine and its target is never read.
+
+    Returns:
+        A short human-readable reason, or ``None`` both when the entry is
+        copyable and when its type could not be established.
+
+    Those two ``None`` cases are deliberately the same answer: only a file
+    type this could read names a reason. An entry whose type it could not
+    read is left for the copy to attempt and, if it really is broken, to
+    fail on.
+
+    A failed ``stat`` therefore never becomes a reason to skip. It says the
+    entry cannot be resolved right now, which is not the same as holding no
+    data, and no errno separates the two: a symlink into a volume that is
+    not mounted fails exactly like one whose target was deleted, and on
+    Windows an unmapped drive letter and an unreachable network share both
+    arrive as ``ENOENT`` as well.
+    """
+    try:
+        st = os.lstat(path)
+    except OSError:
+        return None
+
+    if stat.S_ISLNK(st.st_mode):
+        if not follow_symlinks:
+            return None
+        try:
+            st = os.stat(path)
+        except OSError:
+            return None
+
+    if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode):
+        return None
+    return _file_type_label(st.st_mode)
+
+
+def _report_skipped(skipped, src, log):
+    """Tell the operator what the copy left out.
+
+    Every step that can fail is guarded on its own, because this also runs
+    after the copy has failed: both callers pass ``print``, so one line the
+    terminal will not take must cost neither the copy's own outcome nor the
+    lines after it.
+    """
+    noun = "entry" if len(skipped) == 1 else "entries"
+    lines = [f"  Backup: skipped {len(skipped)} {noun} that cannot be copied:"]
+    for path, reason in skipped:
+        try:
+            name = os.path.relpath(path, src)
+        except (OSError, ValueError):
+            # On Windows ``relpath`` rejects paths on different drives; for a
+            # relative ``src`` it also reaches ``os.getcwd()``, which can fail
+            # on a deleted directory. Neither costs us the entry's name.
+            name = path
+        lines.append(f"    {name} ({reason})")
+
+    for line in lines:
+        try:
+            log(line)
+        except UnicodeError:
+            # An stdout that cannot encode this entry's name. Dropping the
+            # line would leave a header whose count disagrees with the list
+            # under it, so the name is escaped and retried before it is
+            # given up on.
+            try:
+                log(line.encode("ascii", "backslashreplace").decode("ascii"))
+            except (OSError, UnicodeError):
+                continue
+        except OSError:
+            # A write that the device refused. A short report is buffered
+            # whole, so this arrives only once the report is long enough to
+            # flush partway through, and then the header is already out.
+            # Anything else is a caller passing something that is not a
+            # working ``log``, which should surface rather than be swallowed.
+            continue
+
+
+def copy_palace_dir(src, dst, *, symlinks=False, log=None):
+    """Copy a palace directory to ``dst``, skipping entries no copy can carry.
+
+    ``shutil.copytree`` finishes the copy but raises ``shutil.Error`` at the
+    end when a directory entry is not something it knows how to duplicate: a
+    Unix domain socket, or a named pipe. The caller has to treat that as a
+    failed backup, so the command died before the rebuild the backup was
+    guarding (#2207). Such entries are runtime artifacts of whatever process
+    created them and never hold palace data, so a backup without them is
+    still a complete backup of the palace.
+
+    Args:
+        src: Palace directory to copy.
+        dst: Destination path. Must not already exist.
+        symlinks: Passed to ``shutil.copytree``. ``True`` recreates symlinks
+            as symlinks, ``False`` copies what they point at.
+        log: Optional callable (e.g. ``print``) for human-readable progress.
+
+    Returns:
+        The list of ``(path, reason)`` pairs that were skipped: sorted within
+        each directory, in the order the copy visited the directories. Like
+        ``prune_backups``, this both returns what it did and logs it.
+
+    Every other copy failure still raises out of ``shutil.copytree``. A caller
+    about to overwrite the live palace must still stop when its safety copy
+    did not come out whole, so this narrows what the copy attempts rather
+    than swallowing what it reports.
+
+    ``shutil.copytree`` hands the callback names rather than the ``os.scandir``
+    entries it already holds, so classifying costs one extra ``os.lstat`` per
+    directory entry, and it classifies a whole directory before copying any of
+    it. An entry whose type changes inside that window is handled by the
+    earlier reading, which cuts both ways: one that became a socket still
+    aborts the copy, and one that was a socket and became a regular file is
+    skipped with its contents. Nothing in MemPalace replaces an entry that
+    way, and the skipped name is printed either way.
+    """
+    skipped = []
+    # ``shutil.copytree`` classifies the top directory before it creates the
+    # destination, so a copy that never starts still produces a skip list.
+    # Reporting one would tell the operator what a nonexistent backup is
+    # missing, so both ways that happens are excluded below: a destination
+    # already occupied (this snapshot), and a destination that could not be
+    # created at all.
+    dst_existed = os.path.lexists(dst)
+
+    def _ignore(directory, names):
+        ignored = set()
+        found = []
+        for name in names:
+            path = os.path.join(directory, name)
+            reason = _uncopyable_reason(path, follow_symlinks=not symlinks)
+            if reason is not None:
+                ignored.add(name)
+                found.append((path, reason))
+        skipped.extend(sorted(found))
+        return ignored
+
+    def _report():
+        if skipped and log and not dst_existed and os.path.isdir(dst):
+            _report_skipped(skipped, src, log)
+
+    try:
+        shutil.copytree(src, dst, symlinks=symlinks, ignore=_ignore)
+    except BaseException:
+        # Reported for a failed copy too, including an interrupted one: a
+        # half-written backup is exactly when the operator needs to know what
+        # was left out of it. But the copy's own failure is what the caller
+        # has to diagnose, so a report that fails on top of it is dropped
+        # rather than raised in its place.
+        try:
+            _report()
+        except Exception:
+            pass
+        raise
+
+    # Not suppressed here: with the backup complete and nothing destructive
+    # done yet, a ``log`` that cannot be called at all is a caller's bug worth
+    # hearing about. The terminal failures inside ``_report_skipped`` are
+    # absorbed on both paths.
+    _report()
+    return skipped
 
 
 def prune_backups(pattern, max_backups, *, log=None):

mempalace/closet_llm.py

@@ -36,6 +36,7 @@ No vendor lock-in. No hidden dependency on any specific provider. Zero deps
 added to pyproject — uses stdlib urllib.
 """
 
+import contextlib
 import json
 import os
 import re
@@ -51,6 +52,7 @@ from .palace import (
     get_closets_collection,
     get_collection,
     mine_lock,
+    mine_palace_lock,
     purge_file_closets,
     upsert_closet_lines,
 )
@@ -225,8 +227,31 @@ def regenerate_closets(
         print("or pass --endpoint / --model / --key on the CLI.")
         return {"error": "missing-config", "missing": missing}
 
-    drawers_col = get_collection(palace_path, create=False)
-    closets_col = get_closets_collection(palace_path)
+    # A full regeneration is one long-lived read/LLM/purge/upsert operation.
+    # Acquire ownership before opening either collection or calling the LLM so
+    # a conflicting local writer fails immediately. The palace lock is
+    # process-wide and re-entrant, so daemon/MCP-owned calls compose safely.
+    lease = contextlib.nullcontext() if dry_run else mine_palace_lock(palace_path)
+    with lease:
+        return _regenerate_closets_owned(
+            palace_path,
+            wing=wing,
+            sample=sample,
+            dry_run=dry_run,
+            cfg=cfg,
+        )
+
+
+def _regenerate_closets_owned(
+    palace_path,
+    *,
+    wing,
+    sample,
+    dry_run,
+    cfg: LLMConfig,
+):
+    drawers_col = get_collection(palace_path, create=False, read_only=dry_run)
+    closets_col = None if dry_run else get_closets_collection(palace_path)
 
     total = drawers_col.count()
     if total == 0:
@@ -303,6 +328,7 @@ def regenerate_closets(
         # otherwise a regex closet rebuild mid-regenerate races with our
         # purge+upsert cycle and leaves mixed regex/LLM lines.
         with mine_lock(source):
+            assert closets_col is not None
             purge_file_closets(closets_col, source)
             upsert_closet_lines(
                 closets_col,

mempalace/config.py

@@ -11,6 +11,14 @@ from datetime import date, datetime
 from functools import lru_cache
 from pathlib import Path
 
+from .write_routing import (
+    ResolvedWriteRoutingPolicy,
+    RoutingPolicyCandidate,
+    WriteRoutingError,
+    WriteRoutingPolicy,
+    resolve_write_routing_policy,
+)
+
 # ── Input validation ──────────────────────────────────────────────────────────
 # Shared sanitizers for wing/room/entity names. Prevents path traversal,
 # excessively long strings, and special characters that could cause issues
@@ -31,6 +39,22 @@ def strip_lone_surrogates(text: str) -> str:
     return _LONE_SURROGATE_RE.sub("�", text)
 
 
+# Tool output mined from real transcripts routinely embeds a NUL character
+# (U+0000) — e.g. captured Bash output where a reader raced a background
+# writer, or genuine binary/NUL-delimited command output. A document
+# containing one is otherwise valid, well-formed text (unlike a lone
+# surrogate, which is invalid UTF-8), but handing it to ChromaDB's
+# SQLite/FTS5 layer can corrupt the FTS5 inverted index for the *whole*
+# collection (``PRAGMA quick_check`` reports "malformed inverted index for
+# FTS5 table"), not just fail to store that one document. Stripping it
+# before it reaches the chromadb client is the same defense-in-depth this
+# module already applies to lone surrogates (#1235) — sanitize input we
+# don't control before it reaches a datastore we don't control.
+def strip_nul_bytes(text: str) -> str:
+    """Replace embedded NUL characters with U+FFFD before ChromaDB storage."""
+    return text.replace("\x00", "�")
+
+
 def normalize_wing_name(name: str) -> str:
     """Lower-case + collapse separators (`-`, ` `) to `_` for wing slugs.
 
@@ -197,6 +221,13 @@ def sanitize_content(value: str, max_length: int = 100_000) -> str:
 DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
 DEFAULT_COLLECTION_NAME = "mempalace_drawers"
 DEFAULT_BACKEND = "chroma"
+DEFAULT_MILVUS_CONSISTENCY_LEVEL = "Strong"
+_MILVUS_CONSISTENCY_LEVELS = {
+    "strong": "Strong",
+    "session": "Session",
+    "bounded": "Bounded",
+    "eventually": "Eventually",
+}
 
 # How many timestamped palace backups to retain before the oldest are
 # pruned. Applies to the accumulating backups written by ``mempalace
@@ -205,6 +236,15 @@ DEFAULT_BACKEND = "chroma"
 DEFAULT_MAX_BACKUPS = 10
 
 
+def normalize_milvus_consistency_level(value) -> str:
+    raw = str(value).strip() if value else DEFAULT_MILVUS_CONSISTENCY_LEVEL
+    normalized = _MILVUS_CONSISTENCY_LEVELS.get(raw.lower())
+    if normalized:
+        return normalized
+    allowed = ", ".join(_MILVUS_CONSISTENCY_LEVELS.values())
+    raise ValueError(f"milvus_consistency_level must be one of: {allowed}")
+
+
 def sqlite_read_uri(db_path: str) -> str:
     """Return a read-only ``file:`` URI for ``sqlite3.connect(..., uri=True)``.
 
@@ -310,37 +350,47 @@ class MempalaceConfig:
     Load order: env vars > config file > defaults.
     """
 
-    def __init__(self, config_dir=None):
+    def __init__(self, config_dir=None, palace_path=None):
         """Initialize config.
 
         Args:
             config_dir: Override config directory (useful for testing).
                         Defaults to ~/.mempalace.
+            palace_path: Explicit palace data directory. This is primarily
+                         used by CLI operations that received ``--palace``;
+                         it takes precedence over environment and file config.
         """
         self._config_dir = (
             Path(config_dir) if config_dir else Path(os.path.expanduser("~/.mempalace"))
         )
         self._config_file = self._config_dir / "config.json"
         self._people_map_file = self._config_dir / "people_map.json"
+        self._palace_path_override = (
+            os.path.abspath(os.path.expanduser(str(palace_path)))
+            if palace_path is not None
+            else None
+        )
         self._file_config = {}
 
         if self._config_file.exists():
             try:
-                with open(self._config_file, "r") as f:
+                with open(self._config_file, "r", encoding="utf-8") as f:
                     self._file_config = json.load(f)
-            except (json.JSONDecodeError, OSError):
+            except (json.JSONDecodeError, UnicodeDecodeError, OSError):
                 self._file_config = {}
 
     @property
     def palace_path(self):
         """Path to the memory palace data directory."""
+        if self._palace_path_override is not None:
+            return self._palace_path_override
         env_val = os.environ.get("MEMPALACE_PALACE_PATH") or os.environ.get("MEMPAL_PALACE_PATH")
         if env_val:
             # Normalize: expand ~ and collapse .. to match the CLI --palace
             # code path (mcp_server.py:62) and prevent surprise redirection
             # when the env var contains unresolved components.
             return os.path.abspath(os.path.expanduser(env_val))
-        return self._file_config.get("palace_path", DEFAULT_PALACE_PATH)
+        return os.path.expanduser(self._file_config.get("palace_path", DEFAULT_PALACE_PATH))
 
     @property
     def tunnel_file(self):
@@ -421,6 +471,56 @@ class MempalaceConfig:
             timeout = 10.0
         return timeout if timeout > 0 else 10.0
 
+    @property
+    def milvus_uri(self):
+        """Milvus endpoint for the opt-in ``milvus`` backend.
+
+        Defaults to ``None`` so selecting Milvus uses per-palace Milvus Lite at
+        ``<palace>/milvus.db``. Set this only to deliberately use a shared
+        Milvus server, Zilliz Cloud, or a custom local Lite file.
+        """
+        env_val = os.environ.get("MEMPALACE_MILVUS_URI")
+        if env_val:
+            return env_val.strip()
+        value = self._file_config.get("milvus_uri")
+        return str(value).strip() if value else None
+
+    @property
+    def milvus_token(self):
+        """Token for the opt-in ``milvus`` backend, if configured."""
+        env_val = os.environ.get("MEMPALACE_MILVUS_TOKEN")
+        if env_val:
+            return env_val
+        value = self._file_config.get("milvus_token")
+        return str(value) if value else None
+
+    @property
+    def milvus_db_name(self):
+        """Optional Milvus database name for the opt-in ``milvus`` backend."""
+        env_val = os.environ.get("MEMPALACE_MILVUS_DB_NAME")
+        if env_val:
+            return env_val.strip()
+        value = self._file_config.get("milvus_db_name")
+        return str(value).strip() if value else None
+
+    @property
+    def milvus_namespace(self):
+        """Optional Milvus collection namespace/prefix."""
+        env_val = os.environ.get("MEMPALACE_MILVUS_NAMESPACE")
+        if env_val:
+            return env_val.strip()
+        value = self._file_config.get("milvus_namespace")
+        return str(value).strip() if value else None
+
+    @property
+    def milvus_consistency_level(self):
+        """Milvus read consistency level for the opt-in ``milvus`` backend."""
+        env_val = os.environ.get("MEMPALACE_MILVUS_CONSISTENCY_LEVEL")
+        if env_val:
+            return normalize_milvus_consistency_level(env_val)
+        value = self._file_config.get("milvus_consistency_level")
+        return normalize_milvus_consistency_level(value)
+
     @property
     def pgvector_dsn(self):
         """Postgres DSN for the opt-in ``pgvector`` backend.
@@ -450,9 +550,9 @@ class MempalaceConfig:
         """Mapping of name variants to canonical names."""
         if self._people_map_file.exists():
             try:
-                with open(self._people_map_file, "r") as f:
+                with open(self._people_map_file, "r", encoding="utf-8") as f:
                     return json.load(f)
-            except (json.JSONDecodeError, OSError):
+            except (json.JSONDecodeError, UnicodeDecodeError, OSError):
                 pass
         return self._file_config.get("people_map", {})
 
@@ -522,7 +622,8 @@ class MempalaceConfig:
 
         Enforces the invariants the miner relies on:
           * ``chunk_size >= 1``
-          * ``0 <= chunk_overlap < chunk_size`` — equality would loop forever
+          * ``0 <= chunk_overlap <= chunk_size // 2``. A larger overlap can
+            loop the miner forever on short-line content (#2056)
           * ``min_chunk_size <= chunk_size`` — otherwise no chunk is ever
             large enough to file, and ingest silently produces 0 drawers
 
@@ -535,12 +636,14 @@ class MempalaceConfig:
             "min_chunk_size", DEFAULT_MIN_CHUNK_SIZE, minimum=0
         )
 
-        if chunk_overlap >= chunk_size:
-            chunk_overlap = (
-                DEFAULT_CHUNK_OVERLAP
-                if DEFAULT_CHUNK_OVERLAP < chunk_size
-                else max(0, chunk_size - 1)
-            )
+        if chunk_overlap > chunk_size // 2:
+            # Overlap past half the chunk size can hang miner.chunk_text's
+            # windowing loop on short-line content (#2056): a boundary pull can
+            # shrink a chunk below chunk_overlap, so
+            # ``start = end - chunk_overlap`` stops advancing. Repair to the
+            # default when it is still at most half, else clamp to the largest
+            # safe overlap.
+            chunk_overlap = min(DEFAULT_CHUNK_OVERLAP, chunk_size // 2)
 
         if min_chunk_size > chunk_size:
             min_chunk_size = (
@@ -556,7 +659,7 @@ class MempalaceConfig:
 
     @property
     def chunk_overlap(self) -> int:
-        """Overlap between adjacent chunks (validated, ``< chunk_size``)."""
+        """Overlap between adjacent chunks (validated, ``<= chunk_size // 2``)."""
         return self._validated_chunk_config()[1]
 
     @property
@@ -645,7 +748,10 @@ class MempalaceConfig:
 
         Values: ``"minilm"`` (ChromaDB's all-MiniLM-L6-v2 — English-only),
         ``"embeddinggemma"`` (multilingual, 100+ languages, default for
-        new installs since onboarding writes the choice). Read from env
+        new installs since onboarding writes the choice), or
+        ``"openai-compat"`` (embeddings served by an OpenAI-compatible
+        ``/v1/embeddings`` endpoint — see ``embedding_api_url`` /
+        ``embedding_api_model`` / ``embedding_api_key``). Read from env
         ``MEMPALACE_EMBEDDING_MODEL`` first, then ``embedding_model`` in
         ``config.json``, then ``"minilm"`` as a back-compat fallback for
         palaces created before onboarding asked the question.
@@ -729,6 +835,52 @@ class MempalaceConfig:
         except (OSError, NotImplementedError):
             pass
 
+    def _resolve_str_setting(self, env_var: str, config_key: str):
+        """Resolve a string setting: env var > ``config.json`` > ``None``.
+
+        Whitespace-only values are treated as unset, so a blank env var or a
+        hand-edited empty config key doesn't mask the value below it. Unlike
+        ``embedding_model`` the result is not lower-cased — URLs, model ids,
+        and API keys are case-sensitive.
+        """
+        env_val = os.environ.get(env_var)
+        if env_val and env_val.strip():
+            return env_val.strip()
+        cfg_val = self._file_config.get(config_key)
+        if isinstance(cfg_val, str) and cfg_val.strip():
+            return cfg_val.strip()
+        return None
+
+    @property
+    def embedding_api_url(self):
+        """Base URL of the OpenAI-compatible ``/v1/embeddings`` endpoint.
+
+        Used only when ``embedding_model == "openai-compat"``. Resolved from
+        env ``MEMPALACE_EMBEDDING_API_URL`` first, then ``embedding_api_url``
+        in ``config.json``; ``None`` when unset. Accepts a bare host, a
+        ``…/v1`` base, or a full endpoint URL.
+        """
+        return self._resolve_str_setting("MEMPALACE_EMBEDDING_API_URL", "embedding_api_url")
+
+    @property
+    def embedding_api_model(self):
+        """Server-side model id for the ``openai-compat`` embeddings endpoint.
+
+        Resolved from env ``MEMPALACE_EMBEDDING_API_MODEL`` first, then
+        ``embedding_api_model`` in ``config.json``; ``None`` when unset.
+        """
+        return self._resolve_str_setting("MEMPALACE_EMBEDDING_API_MODEL", "embedding_api_model")
+
+    @property
+    def embedding_api_key(self):
+        """Optional bearer token / API key for the embeddings endpoint.
+
+        Resolved from env ``MEMPALACE_EMBEDDING_API_KEY`` first, then
+        ``embedding_api_key`` in ``config.json``; ``None`` when unset (for
+        local endpoints that need no auth).
+        """
+        return self._resolve_str_setting("MEMPALACE_EMBEDDING_API_KEY", "embedding_api_key")
+
     @property
     def topic_tunnel_min_count(self):
         """Minimum number of overlapping confirmed topics required to create
@@ -785,6 +937,43 @@ class MempalaceConfig:
         )
         return DEFAULT_MAX_BACKUPS if coerced is None else coerced
 
+    @property
+    def lang_explicit(self):
+        """Primary language code when explicitly configured, else ``None``.
+
+        Resolution order: ``MEMPALACE_LANG`` / ``MEMPAL_LANG`` env var, then
+        ``config.json["lang"]``. Returns ``None`` if neither is set. Use this
+        when a caller needs to know whether the user has opted in to locale
+        behaviour (e.g. to avoid silently changing search scoring for palaces
+        that have never set a language).
+        """
+        env_val = os.environ.get("MEMPALACE_LANG") or os.environ.get("MEMPAL_LANG")
+        if env_val and env_val.strip():
+            return env_val.strip()
+        cfg = self._file_config.get("lang")
+        if isinstance(cfg, str) and cfg.strip():
+            return cfg.strip()
+        return None
+
+    @property
+    def lang(self):
+        """Primary language code for localized output and display.
+
+        Resolution order: ``lang_explicit`` (env or config.json), first entry
+        of ``entity_languages``, then ``"en"``. Always returns a non-empty
+        string so callers that need a language for display purposes never
+        have to handle ``None``. Code paths that must not silently change
+        behaviour for unconfigured palaces should read ``lang_explicit``
+        instead.
+        """
+        explicit = self.lang_explicit
+        if explicit:
+            return explicit
+        entity_langs = self.entity_languages
+        if entity_langs:
+            return entity_langs[0]
+        return "en"
+
     @property
     def hook_silent_save(self):
         """Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""
@@ -795,6 +984,102 @@ class MempalaceConfig:
         """Whether the stop hook shows a desktop notification via notify-send."""
         return self._file_config.get("hooks", {}).get("desktop_toast", False)
 
+    def resolve_write_routing(self, scope: str) -> ResolvedWriteRoutingPolicy:
+        """Resolve the configured write policy for ``hooks`` or ``cli``.
+
+        Precedence is:
+
+        1. scope-specific environment variable;
+        2. global environment variable;
+        3. legacy hook environment variable;
+        4. scope-specific config value;
+        5. global config value;
+        6. legacy hook config value;
+        7. ``direct``.
+
+        This foundation does not change current hook or CLI behavior. The
+        policy-aware consumers are introduced by follow-up PRs.
+        """
+
+        normalized_scope = str(scope).strip().lower()
+        env_names = {
+            "hooks": "MEMPALACE_HOOK_WRITE_ROUTING",
+            "cli": "MEMPALACE_CLI_WRITE_ROUTING",
+        }
+
+        if normalized_scope not in env_names:
+            raise WriteRoutingError("write routing scope must be 'hooks' or 'cli'")
+
+        routing_config = self._file_config.get("write_routing", {})
+        if routing_config is None:
+            routing_config = {}
+
+        if not isinstance(routing_config, dict):
+            raise WriteRoutingError("config write_routing must be an object")
+
+        candidates = [
+            RoutingPolicyCandidate(
+                env_names[normalized_scope],
+                os.environ.get(env_names[normalized_scope]),
+            ),
+            RoutingPolicyCandidate(
+                "MEMPALACE_WRITE_ROUTING",
+                os.environ.get("MEMPALACE_WRITE_ROUTING"),
+            ),
+        ]
+
+        if normalized_scope == "hooks":
+            candidates.append(
+                RoutingPolicyCandidate(
+                    "MEMPALACE_HOOKS_DAEMON (legacy)",
+                    os.environ.get("MEMPALACE_HOOKS_DAEMON"),
+                    legacy_boolean=True,
+                )
+            )
+
+        candidates.extend(
+            [
+                RoutingPolicyCandidate(
+                    f"config write_routing.{normalized_scope}",
+                    routing_config.get(normalized_scope),
+                ),
+                RoutingPolicyCandidate(
+                    "config write_routing.default",
+                    routing_config.get("default"),
+                ),
+            ]
+        )
+
+        if normalized_scope == "hooks":
+            hooks_config = self._file_config.get("hooks", {})
+            if hooks_config is None:
+                hooks_config = {}
+
+            if not isinstance(hooks_config, dict):
+                raise WriteRoutingError("config hooks must be an object")
+
+            candidates.append(
+                RoutingPolicyCandidate(
+                    "config hooks.daemon (legacy)",
+                    hooks_config.get("daemon"),
+                    legacy_boolean=True,
+                )
+            )
+
+        return resolve_write_routing_policy(candidates)
+
+    @property
+    def hook_write_routing(self) -> WriteRoutingPolicy:
+        """Resolved future routing policy for hook-triggered writes."""
+
+        return self.resolve_write_routing("hooks").policy
+
+    @property
+    def cli_write_routing(self) -> WriteRoutingPolicy:
+        """Resolved future routing policy for routine CLI writes."""
+
+        return self.resolve_write_routing("cli").policy
+
     @property
     def hook_use_daemon(self):
         """Whether hooks should submit save/mine work to the opt-in daemon."""
@@ -842,7 +1127,7 @@ class MempalaceConfig:
                 "topic_wings": DEFAULT_TOPIC_WINGS,
                 "hall_keywords": DEFAULT_HALL_KEYWORDS,
             }
-            with open(self._config_file, "w") as f:
+            with open(self._config_file, "w", encoding="utf-8") as f:
                 json.dump(default_config, f, indent=2)
             # Restrict config file to owner read/write only
             try:
@@ -858,7 +1143,7 @@ class MempalaceConfig:
             people_map: Dict mapping name variants to canonical names.
         """
         self._config_dir.mkdir(parents=True, exist_ok=True)
-        with open(self._people_map_file, "w") as f:
+        with open(self._people_map_file, "w", encoding="utf-8") as f:
             json.dump(people_map, f, indent=2)
         try:
             self._people_map_file.chmod(0o600)

mempalace/convo_miner.py

@@ -10,15 +10,25 @@ Same palace as project mining. Different ingest strategy.
 
 import os
 import sys
+import json
+import hashlib
 import logging
+import stat
 from pathlib import Path
 from datetime import datetime
 from collections import defaultdict
 from typing import Optional
 
+from .backends import PalaceNotFoundError
 from .collision_scan import assert_no_collisions
-from .ids import ID_RECIPE, make_convo_drawer_id, make_convo_sentinel_id
-from .normalize import normalize
+from .ids import (
+    ID_RECIPE,
+    make_convo_drawer_id,
+    make_convo_sentinel_id,
+    make_exchange_drawer_id,
+)
+from .normalize import normalize_conversations
+from .entities import entities_metadata
 from .palace import (
     NORMALIZE_VERSION,
     SKIP_DIRS,
@@ -28,6 +38,7 @@ from .palace import (
     get_collection,
     mine_lock,
     mine_palace_lock,
+    prefetch_content_hashes,
     prefetch_mined_set,
 )
 
@@ -54,6 +65,82 @@ def _detect_hall_cached(content: str) -> str:
     return max(scores, key=scores.get) if scores else "general"
 
 
+def file_conversation_exchange(
+    collection,
+    *,
+    wing: str,
+    room: str,
+    text: str,
+    source_file: str,
+    agent: str,
+    authored_at: Optional[str] = None,
+    extra_metadata: Optional[dict] = None,
+) -> Optional[str]:
+    """File one verbatim conversation exchange as a single drawer.
+
+    Canonical write path for live agent integrations (e.g. Hermes) and
+    their backfills — both must route here so routing, normalization,
+    and metadata conventions stay identical between live and historical
+    ingest. Builds the same metadata the convo miner writes so hallway
+    traversal, entity search, and since/before date filters see
+    integration drawers exactly like mined ones.
+
+    ``wing`` and ``room`` are validated with the same ``sanitize_name``
+    rules the MCP write tools apply, but a failed name falls back
+    (``wing_general`` / ``conversations``) instead of erroring: this
+    path files *live* turns, and dropping a turn over a config typo
+    would break the verbatim / 100%-recall promise. The fallback is
+    logged at warning level so the misconfiguration is visible.
+
+    ``extra_metadata`` lets callers append integration-specific fields
+    (e.g. ``source`` / ``session_id``); keys that collide with the
+    canonical fields are ignored, so it cannot be used to overwrite or
+    drop them. Returns the drawer id, or None when ``text`` is empty
+    after stripping.
+    """
+    from .config import sanitize_name
+
+    text = (text or "").strip()
+    if not text:
+        return None
+    try:
+        wing = sanitize_name(wing, "wing")
+    except ValueError:
+        logger.warning(
+            "file_conversation_exchange: invalid wing %r — filing under wing_general", wing
+        )
+        wing = "wing_general"
+    try:
+        room = sanitize_name(room, "room")
+    except ValueError:
+        logger.warning(
+            "file_conversation_exchange: invalid room %r — filing under conversations", room
+        )
+        room = "conversations"
+    filed_at = datetime.now().isoformat()
+    drawer_id = make_exchange_drawer_id(wing, room, source_file, filed_at, text)
+    metadata = {
+        "wing": wing,
+        "room": room,
+        "hall": _detect_hall_cached(text),
+        "source_file": source_file,
+        "chunk_index": 0,
+        "added_by": agent,
+        "filed_at": filed_at,
+        "entities": entities_metadata(text),
+        "authored_at": authored_at if authored_at is not None else filed_at,
+        "ingest_mode": "convos",
+        "extract_mode": "exchange",
+        "normalize_version": NORMALIZE_VERSION,
+        "id_recipe": ID_RECIPE,
+    }
+    if extra_metadata:
+        for key, value in extra_metadata.items():
+            metadata.setdefault(key, value)
+    collection.upsert(ids=[drawer_id], documents=[text], metadatas=[metadata])
+    return drawer_id
+
+
 # File types that might contain conversations
 CONVO_EXTENSIONS = {
     ".txt",
@@ -62,6 +149,16 @@ CONVO_EXTENSIONS = {
     ".jsonl",
 }
 
+# Directories inside conversation sources that never hold conversations.
+# ``tool-results``: Claude Code pages large tool outputs to
+# ``<session>/tool-results/*.txt`` inside ``~/.claude/projects/<slug>/``.
+# They are raw machine dumps referenced from the transcript JSONL — mining
+# them stores megabytes of command output as "memories" (field measurement:
+# 12.8k drawers from tool-results files on one palace; a single file
+# produced 3.6k). Extends the generic SKIP_DIRS set for the convo scanner
+# only — project mining semantics are unchanged.
+CONVO_SKIP_DIRS = SKIP_DIRS | {"tool-results"}
+
 MIN_CHUNK_SIZE = 30
 CHUNK_SIZE = 800  # chars per drawer — align with miner.py
 _LINE_GROUP_SIZE = 25  # lines per fallback group when no paragraph breaks
@@ -77,30 +174,81 @@ MAX_FILE_SIZE = 500 * 1024 * 1024  # 500 MB — skip files larger than this.
 # use also scales with source size.
 
 
-def _register_file(collection, source_file: str, wing: str, agent: str, extract_mode: str):
+def _path_within_root(path: Path, root: Path) -> bool:
+    try:
+        path.expanduser().resolve().relative_to(root.expanduser().resolve())
+        return True
+    except (OSError, ValueError):
+        return False
+
+
+def _is_regular_source_file(filepath: Path, root: Path) -> bool:
+    if not _path_within_root(filepath, root):
+        return False
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    fd = -1
+    try:
+        fd = os.open(filepath, flags)
+        st = os.fstat(fd)
+        return stat.S_ISREG(st.st_mode) and st.st_size <= MAX_FILE_SIZE
+    except OSError:
+        return False
+    finally:
+        if fd != -1:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+
+
+def _register_file(
+    collection,
+    source_file: str,
+    wing: str,
+    agent: str,
+    extract_mode: str,
+    content_hash: Optional[str] = None,
+):
     """Write a sentinel so file_already_mined() returns True for 0-chunk files.
 
     Without this, files that normalize to nothing or produce zero chunks are
     re-read and re-processed on every mine run because nothing was written to
     ChromaDB on the first pass.
+
+    Stamps source_mtime like every real drawer does, so a file that later
+    grows past the min-chunk-size floor (e.g. a short session that gets
+    extended) is correctly detected as changed on the next mine instead of
+    being skipped forever by this sentinel.
+
+    Also used to register a file recognized as a content-duplicate of an
+    already-mined transcript under a different path (see
+    ``prefetch_content_hashes``) — stamping it here means the next run skips
+    it via the cheap mtime check instead of re-normalizing and re-hashing it.
     """
+    try:
+        source_mtime = os.path.getmtime(source_file)
+    except OSError:
+        source_mtime = None
     sentinel_id = make_convo_sentinel_id(source_file, extract_mode)
+    meta = {
+        "wing": wing,
+        "room": "_registry",
+        "source_file": source_file,
+        "added_by": agent,
+        "filed_at": datetime.now().isoformat(),
+        "ingest_mode": "registry",
+        "extract_mode": extract_mode,
+        "normalize_version": NORMALIZE_VERSION,
+        "id_recipe": ID_RECIPE,
+    }
+    if source_mtime is not None:
+        meta["source_mtime"] = source_mtime
+    if content_hash is not None:
+        meta["content_hash"] = content_hash
     collection.upsert(
         documents=[f"[registry] {source_file}"],
         ids=[sentinel_id],
-        metadatas=[
-            {
-                "wing": wing,
-                "room": "_registry",
-                "source_file": source_file,
-                "added_by": agent,
-                "filed_at": datetime.now().isoformat(),
-                "ingest_mode": "registry",
-                "extract_mode": extract_mode,
-                "normalize_version": NORMALIZE_VERSION,
-                "id_recipe": ID_RECIPE,
-            }
-        ],
+        metadatas=[meta],
     )
 
 
@@ -345,17 +493,37 @@ def detect_convo_room(content: str) -> str:
 # =============================================================================
 
 
-def scan_convos(convo_dir: str) -> list:
+def scan_convos(convo_dir: str, include_subagents: bool = False) -> list:
     """Find all potential conversation files.
 
     Skips symlinks and oversized files. Each skipped symlink is logged to
     ``sys.stderr`` with a ``  SKIP: <relative-path> (symlink)`` line so the
     caller can tell why an apparent conversation directory yielded no files.
+
+    By default, directories named ``subagents`` are skipped: Claude Code
+    records Explore/Plan/Grep subagent transcripts there, and on typical
+    workspaces they outnumber main session files by one to two orders of
+    magnitude. Pass ``include_subagents=True`` to mine them anyway.
+
+    The match is case-insensitive on the directory name only (``subagents``
+    or ``Subagents``), so directories like ``mysubagents`` or
+    ``subagentsbackup`` are not affected.
     """
-    convo_path = Path(convo_dir).expanduser().resolve()
+    # A direct conversation file is a valid source. For a file, feed only
+    # its basename through the existing directory validation loop.
+    requested_path = Path(convo_dir).expanduser()
+    single_file = requested_path.is_file()
+    convo_path = (requested_path.parent if single_file else requested_path).resolve()
+    scan_entries = (
+        [(str(convo_path), [], [requested_path.name])] if single_file else os.walk(convo_path)
+    )
     files = []
-    for root, dirs, filenames in os.walk(convo_path):
-        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+    for root, dirs, filenames in scan_entries:
+        dirs[:] = [
+            d
+            for d in dirs
+            if d not in CONVO_SKIP_DIRS and (include_subagents or d.lower() != "subagents")
+        ]
         for filename in filenames:
             if filename.endswith(".meta.json"):
                 continue
@@ -369,10 +537,30 @@ def scan_convos(convo_dir: str) -> list:
                     except OSError:
                         pass
                     continue
+                # Skip files exceeding size limit, or those whose stat() raises
+                # (permission denied, racing delete, broken symlink that
+                # survived the earlier is_symlink check). Both branches log
+                # to stderr to match the SKIP: (symlink) line above; silent
+                # drops at this gate were the original #923 complaint.
                 try:
-                    if filepath.stat().st_size > MAX_FILE_SIZE:
+                    file_size = filepath.stat().st_size
+                    if file_size > MAX_FILE_SIZE:
+                        print(
+                            f"  SKIP: {filepath.name} ({file_size / (1024 * 1024):.1f} MB)"
+                            f" exceeds {MAX_FILE_SIZE // (1024 * 1024)} MB limit",
+                            file=sys.stderr,
+                        )
                         continue
-                except OSError:
+                except OSError as exc:
+                    # Prefer ``exc.strerror`` so the path isn't duplicated in
+                    # the output (see the matching comment in
+                    # ``miner.scan_project``).
+                    print(
+                        f"  SKIP: {filepath.name} (stat error: {exc.strerror or exc})",
+                        file=sys.stderr,
+                    )
+                    continue
+                if not _is_regular_source_file(filepath, convo_path):
                     continue
                 files.append(filepath)
     return files
@@ -383,12 +571,59 @@ def scan_convos(convo_dir: str) -> list:
 # =============================================================================
 
 
-def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extract_mode):
+def _extract_authored_at(filepath):
+    """Most-recent message timestamp in a transcript, used as the drawer's authored date.
+
+    Both Claude Code and Codex JSONL transcripts carry a top-level ISO-8601
+    ``timestamp`` on each line. We take the max so ``authored_at`` reflects when the
+    content was actually written, independent of when it was mined (``filed_at``).
+    This restores chronology: a session from days ago keeps its real date even when
+    re-mined today, instead of every drawer collapsing to ingest time. Returns None
+    for formats without per-line timestamps (e.g. plain ``.md``).
+    """
+    path = Path(filepath)
+    if path.suffix != ".jsonl":
+        return None
+    latest = None
+    try:
+        with path.open(encoding="utf-8", errors="ignore") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    ts = json.loads(line).get("timestamp")
+                except (ValueError, TypeError, AttributeError):
+                    continue
+                # ISO-8601 timestamps are strings; guard against a non-string
+                # ``timestamp`` so a malformed line can't raise TypeError on compare.
+                if isinstance(ts, str) and (latest is None or ts > latest):
+                    latest = ts
+    except OSError:
+        return None
+    return latest
+
+
+def _file_chunks_locked(
+    collection,
+    source_file,
+    chunks,
+    wing,
+    room,
+    agent,
+    extract_mode,
+    authored_at=None,
+    content_hash=None,
+):
     """Lock the source file, purge stale drawers, and upsert fresh chunks.
 
     Combines the per-file serialization that prevents concurrent agents from
-    duplicating work (via mine_lock) with the normalize-version rebuild
-    contract (purge-before-insert so pre-v2 drawers don't survive).
+    duplicating work (via mine_lock) with the rebuild contract
+    (purge-before-insert so stale drawers never survive) that fires on
+    either a normalize-version bump OR a changed/grown source file (mtime
+    differs from what's stored) -- transcripts are not assumed immutable,
+    since a Claude Code session keeps appending to its own file while
+    active and /compact or /clear can rewrite one in place.
 
     Returns (drawers_added, room_counts_delta, skipped).
     """
@@ -396,26 +631,47 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
     drawers_added = 0
     with mine_lock(source_file):
         # Re-check after lock — another agent may have just finished this file
-        # at the current schema. A stale-version hit here returns False, so we
+        # at the current schema/mtime. A stale hit here returns False, so we
         # still fall through to the purge+rebuild path below.
-        if file_already_mined(collection, source_file, extract_mode=extract_mode):
+        if file_already_mined(collection, source_file, check_mtime=True, extract_mode=extract_mode):
             return 0, room_counts_delta, True
 
-        # Purge stale drawers first. When the normalize schema bumps,
-        # file_already_mined() returned False for pre-v2 drawers — clean
-        # them out so the source doesn't end up with mixed old/new drawers.
+        # Purge stale drawers first. Fires both on a normalize-schema bump
+        # (file_already_mined() returned False for pre-v2 drawers) and on a
+        # changed/grown transcript (mtime differs) — clean them out so the
+        # source doesn't end up with mixed old/new drawers.
+        #
+        # A failed purge must abort this file's mine attempt rather than
+        # fall through to upsert: proceeding on top of an unpurged (or
+        # partially purged) set produces duplicate/stale drawers under
+        # mixed schema versions, with no operator-visible signal beyond a
+        # debug log (#105 — convo_miner's own instance of the same swallow
+        # already fixed for miner.py at #23). Returning here leaves the old
+        # drawers' stored mtime untouched, so the next mine still sees a
+        # mismatch and retries.
         try:
             delete_ids = _source_file_delete_ids(collection, source_file, extract_mode)
             if delete_ids:
                 collection.delete(ids=delete_ids)
-        except Exception:
+        except Exception as exc:
+            print(
+                f"  ! [skip] stale-drawer purge failed for {source_file!r} "
+                f"({exc!r}); leaving existing drawers untouched, will retry "
+                f"on the next mine",
+                file=sys.stderr,
+            )
             logger.debug("Stale-drawer purge failed for %s", source_file, exc_info=True)
+            return 0, room_counts_delta, True
 
         # Batch chunks into bounded upserts so large transcripts keep most of
         # the embedding speedup without one huge Chroma/SQLite request. Keep
         # one filed_at per source file so all transcript drawers share an
         # ingest timestamp.
         filed_at = datetime.now().isoformat()
+        try:
+            source_mtime = os.path.getmtime(source_file)
+        except OSError:
+            source_mtime = None
         for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE):
             batch_docs: list = []
             batch_ids: list = []
@@ -429,21 +685,30 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
                 )
                 batch_docs.append(chunk["content"])
                 batch_ids.append(drawer_id)
-                batch_metas.append(
-                    {
-                        "wing": wing,
-                        "room": chunk_room,
-                        "hall": _detect_hall_cached(chunk["content"]),
-                        "source_file": source_file,
-                        "chunk_index": chunk["chunk_index"],
-                        "added_by": agent,
-                        "filed_at": filed_at,
-                        "ingest_mode": "convos",
-                        "extract_mode": extract_mode,
-                        "normalize_version": NORMALIZE_VERSION,
-                        "id_recipe": ID_RECIPE,
-                    }
-                )
+                meta = {
+                    "wing": wing,
+                    "room": chunk_room,
+                    "hall": _detect_hall_cached(chunk["content"]),
+                    "source_file": source_file,
+                    "chunk_index": chunk["chunk_index"],
+                    "added_by": agent,
+                    "filed_at": filed_at,
+                    "entities": entities_metadata(chunk["content"]),
+                    "authored_at": authored_at if authored_at is not None else filed_at,
+                    "ingest_mode": "convos",
+                    "extract_mode": extract_mode,
+                    "normalize_version": NORMALIZE_VERSION,
+                    "id_recipe": ID_RECIPE,
+                }
+                if source_mtime is not None:
+                    meta["source_mtime"] = source_mtime
+                # Stamp content_hash only on chunk 0 so multi-conversation
+                # privacy-export hashes are not O(N²)-duplicated across every
+                # chunk row. ``prefetch_content_hashes`` still finds them —
+                # it scans all drawers and splits comma-joined hash fields.
+                if content_hash is not None and chunk.get("chunk_index", 0) == 0:
+                    meta["content_hash"] = content_hash
+                batch_metas.append(meta)
             assert_no_collisions(list(zip(batch_ids, batch_metas)), collection)
             try:
                 collection.upsert(
@@ -487,6 +752,54 @@ def _is_ai_tool_path(path: Path) -> bool:
     return False
 
 
+def _split_new_and_duplicate_conversations(
+    conversations: list,
+    wing: str,
+    source_file: str,
+    mined_content_hashes: dict,
+) -> tuple:
+    """Hash each conversation and split them into (new, duplicate) lists.
+
+    A conversation is a duplicate when its hash is already registered under
+    a *different* source_file in the same wing — mining the same transcript
+    into a second wing is a deliberate re-file, not a repeat, so the lookup
+    is scoped to (wing, hash). Returns ([(hash, text), ...] new, [(hash,
+    dup_source_file), ...] duplicates).
+    """
+    new_items = []
+    duplicates = []
+    for conversation in conversations:
+        content_hash = hashlib.sha256(conversation.strip().encode("utf-8")).hexdigest()
+        dup_source = mined_content_hashes.get((wing, content_hash))
+        if dup_source is None or dup_source == source_file:
+            new_items.append((content_hash, conversation))
+        else:
+            duplicates.append((content_hash, dup_source))
+    return new_items, duplicates
+
+
+def _is_unchanged_since_last_mine(source_file: str, mined_mtimes: dict) -> bool:
+    """True iff source_file was mined at the current schema AND its on-disk
+    mtime still matches what was stored -- the mtime-aware replacement for
+    "we've seen this source_file before" (transcripts are not immutable).
+
+    False (re-mine) whenever the file isn't in mined_mtimes at all, its
+    stored mtime is None (never recorded -- pre-mtime-tracking drawer, or
+    getmtime failed when it was written), or getmtime fails right now
+    (treat as changed rather than silently trusting stale data).
+    """
+    if source_file not in mined_mtimes:
+        return False
+    stored_mtime = mined_mtimes[source_file]
+    if stored_mtime is None:
+        return False
+    try:
+        current_mtime = os.path.getmtime(source_file)
+    except OSError:
+        return False
+    return abs(stored_mtime - current_mtime) < 0.001
+
+
 def _resolve_wing(convo_path: Path, wing: Optional[str]) -> str:
     """Determine the destination wing for ``mine_convos``.
 
@@ -520,12 +833,16 @@ def mine_convos(
     limit: int = 0,
     dry_run: bool = False,
     extract_mode: str = "exchange",
+    include_subagents: bool = False,
 ):
     """Mine a directory of conversation files into the palace.
 
     extract_mode:
         "exchange" — default exchange-pair chunking (Q+A = one unit)
         "general"  — general extractor: decisions, preferences, milestones, problems, emotions
+    include_subagents:
+        False (default) — skip Claude Code ``subagents/`` directories
+        True            — also mine subagent transcripts
 
     The real work is in :func:`_mine_convos_impl`; this wrapper holds the
     per-palace flock around it so two concurrent ``mempalace mine --mode
@@ -552,6 +869,7 @@ def mine_convos(
             limit=limit,
             dry_run=dry_run,
             extract_mode=extract_mode,
+            include_subagents=include_subagents,
         )
 
     with mine_palace_lock(palace_path):
@@ -563,9 +881,84 @@ def mine_convos(
             limit=limit,
             dry_run=dry_run,
             extract_mode=extract_mode,
+            include_subagents=include_subagents,
         )
 
 
+def _compute_hallways_for_wing_safe(wing, collection, drawers_filed, config=None):
+    """Auto-populate the associative graph from the entities just mined.
+
+    Best-effort: hallway computation must never fail an otherwise-good mine, and is
+    skipped when nothing new was filed.
+    """
+    if drawers_filed <= 0:
+        return
+    try:
+        from .hallways import compute_hallways_for_wing
+
+        compute_hallways_for_wing(wing, col=collection, config=config)
+    except Exception as exc:
+        print(f"  (hallways skipped: {exc})")
+
+
+def _normalize_convo_conversations(
+    filepath: Path,
+    source_file: str,
+    cfg_min_chunk_size: int,
+    collection,
+    wing: str,
+    agent: str,
+    extract_mode: str,
+    dry_run: bool,
+) -> Optional[list]:
+    """Normalize a transcript file into its individual conversations,
+    registering it as filed when there's nothing worth mining. Returns None
+    when the caller should skip the file (normalize failed, or normalized
+    content is too short to chunk).
+
+    Kept as separate conversations rather than joined into one string so
+    dedup can hash and skip per conversation — a Claude.ai privacy export
+    bundles every conversation into a single file, and hashing the joined
+    bundle means one new conversation added to a re-export changes the
+    whole-file hash and hides the conversations that didn't change.
+    """
+    try:
+        conversations = [c for c in normalize_conversations(str(filepath)) if c]
+    except (OSError, ValueError):
+        if not dry_run:
+            _register_file(collection, source_file, wing, agent, extract_mode)
+        return None
+
+    total_len = sum(len(c.strip()) for c in conversations)
+    if not conversations or total_len < cfg_min_chunk_size:
+        if not dry_run:
+            _register_file(collection, source_file, wing, agent, extract_mode)
+        return None
+
+    return conversations
+
+
+def _open_convo_collection(
+    palace_path: str,
+    *,
+    dry_run: bool,
+):
+    """Open the conversation collection without creating it during dry-run."""
+    if not dry_run:
+        return get_collection(palace_path)
+
+    try:
+        return get_collection(
+            palace_path,
+            create=False,
+            read_only=True,
+        )
+    except PalaceNotFoundError:
+        # A missing palace or uninitialized collection represents empty
+        # prior state to a dry-run. Do not create either one.
+        return None
+
+
 def _mine_convos_impl(
     convo_dir: str,
     palace_path: str,
@@ -574,10 +967,11 @@ def _mine_convos_impl(
     limit: int = 0,
     dry_run: bool = False,
     extract_mode: str = "exchange",
+    include_subagents: bool = False,
 ):
     from .config import MempalaceConfig
 
-    palace_config = MempalaceConfig()
+    palace_config = MempalaceConfig(palace_path=palace_path)
     cfg_chunk_size = palace_config.chunk_size
     # Only override convo_miner's MIN_CHUNK_SIZE when the user has set
     # min_chunk_size explicitly. min_chunk_size_explicit returns the
@@ -593,10 +987,10 @@ def _mine_convos_impl(
     convo_path = Path(convo_dir).expanduser().resolve()
     wing = _resolve_wing(convo_path, wing)
 
-    files = scan_convos(convo_dir)
+    files = scan_convos(convo_dir, include_subagents=include_subagents)
 
     print(f"\n{'=' * 55}")
-    print("  MemPalace Mine — Conversations")
+    print("  MemPalace Mine -- Conversations")
     print(f"{'=' * 55}")
     print(f"  Wing:    {wing}")
     print(f"  Source:  {convo_path}")
@@ -604,18 +998,32 @@ def _mine_convos_impl(
     print(f"  Files:   {len(files)}{limit_suffix}")
     print(f"  Palace:  {palace_path}")
     if dry_run:
-        print("  DRY RUN — nothing will be filed")
+        print("  DRY RUN -- nothing will be filed")
     print(f"{'-' * 55}\n")
 
-    collection = get_collection(palace_path) if not dry_run else None
+    collection = _open_convo_collection(
+        palace_path,
+        dry_run=dry_run,
+    )
 
-    # Bulk pre-fetch already-mined set in one paginated pass instead of
-    # `len(files)` separate WHERE-source_file queries. On a 150k-drawer
-    # palace each per-file query costs ~2s, so a 2000-file sweep used to
-    # spend >1h just deciding to skip. prefetch_mined_set() does the same
-    # decisions in a single scan; loop body becomes an O(1) set check.
-    mined_set: set[str] = (
-        prefetch_mined_set(collection, extract_mode=extract_mode) if not dry_run else set()
+    # Bulk pre-fetch already-mined source_file -> stored mtime in one
+    # paginated pass instead of `len(files)` separate WHERE-source_file
+    # queries. On a 150k-drawer palace each per-file query costs ~2s, so a
+    # 2000-file sweep used to spend >1h just deciding to skip.
+    # prefetch_mined_set() does the same decisions in a single scan; loop
+    # body becomes an O(1) dict lookup + a cheap local mtime comparison.
+    mined_mtimes: dict = (
+        prefetch_mined_set(collection, extract_mode=extract_mode) if collection is not None else {}
+    )
+    # content_hash -> source_file for transcripts already filed. Repeated
+    # exports from Claude/ChatGPT commonly land under a new filename each
+    # run even when the conversation itself is unchanged, so the
+    # source_file-keyed skip above ("mined_mtimes") never recognizes them —
+    # this catches the same conversation reappearing at a new path.
+    mined_content_hashes: dict = (
+        prefetch_content_hashes(collection, extract_mode=extract_mode)
+        if collection is not None
+        else {}
     )
 
     total_drawers = 0
@@ -628,24 +1036,59 @@ def _mine_convos_impl(
         files_processed = i
         source_file = str(filepath)
 
-        # Skip if already filed at current NORMALIZE_VERSION
-        if not dry_run and source_file in mined_set:
+        # Skip only if already filed at the current NORMALIZE_VERSION AND
+        # unchanged on disk since. Transcripts are NOT assumed immutable:
+        # a Claude Code session keeps appending to the same file while
+        # active, and /compact or /clear can rewrite one in place -- so
+        # "we've seen this source_file before" alone is not sufficient.
+        # Falling through re-mines: _file_chunks_locked purges this
+        # source_file's stale drawers before inserting fresh ones, so this
+        # never leaves duplicates behind.
+        if _is_unchanged_since_last_mine(source_file, mined_mtimes):
             files_skipped += 1
             continue
 
-        # Normalize format
-        try:
-            content = normalize(str(filepath))
-        except (OSError, ValueError):
-            if not dry_run:
-                _register_file(collection, source_file, wing, agent, extract_mode)
+        if not _is_regular_source_file(filepath, Path(convo_dir).expanduser().resolve()):
+            files_skipped += 1
             continue
 
-        if not content or len(content.strip()) < cfg_min_chunk_size:
+        conversations = _normalize_convo_conversations(
+            filepath,
+            source_file,
+            cfg_min_chunk_size,
+            collection,
+            wing,
+            agent,
+            extract_mode,
+            dry_run,
+        )
+        if conversations is None:
+            continue
+
+        # Hash and dedup per conversation, not per file: a Claude/ChatGPT
+        # privacy export bundles every conversation into one file, so a
+        # re-export that adds one new conversation changes the whole-file
+        # hash and would hide the conversations that didn't change if we
+        # hashed the joined bundle. Conversations whose hash is already
+        # filed under a different source_file in this wing are dropped;
+        # the rest are re-joined and mined as usual.
+        new_items, duplicates = _split_new_and_duplicate_conversations(
+            conversations, wing, source_file, mined_content_hashes
+        )
+        if not new_items:
             if not dry_run:
                 _register_file(collection, source_file, wing, agent, extract_mode)
+            dup_source = duplicates[0][1]
+            print(
+                f"  = [{i:4}/{len(files)}] {filepath.name[:50]:50} "
+                f"duplicate of {Path(dup_source).name}"
+            )
+            files_skipped += 1
             continue
 
+        content = "\n\n".join(text for _, text in new_items)
+        content_hash = ",".join(h for h, _ in new_items)
+
         # Chunk — either exchange pairs or general extraction
         if extract_mode == "general":
             from .general_extractor import extract_memories
@@ -676,9 +1119,9 @@ def _mine_convos_impl(
 
                 type_counts = Counter(c.get("memory_type", "general") for c in chunks)
                 types_str = ", ".join(f"{t}:{n}" for t, n in type_counts.most_common())
-                print(f"    [DRY RUN] {filepath.name} → {len(chunks)} memories ({types_str})")
+                print(f"    [DRY RUN] {filepath.name} -> {len(chunks)} memories ({types_str})")
             else:
-                print(f"    [DRY RUN] {filepath.name} → room:{room} ({len(chunks)} drawers)")
+                print(f"    [DRY RUN] {filepath.name} -> room:{room} ({len(chunks)} drawers)")
             total_drawers += len(chunks)
             # Track room counts
             if extract_mode == "general":
@@ -697,7 +1140,15 @@ def _mine_convos_impl(
         # Lock + purge stale + file fresh chunks. Lock serializes concurrent
         # agents; purge removes pre-v2 drawers so the schema bump applies.
         drawers_added, room_delta, skipped = _file_chunks_locked(
-            collection, source_file, chunks, wing, room, agent, extract_mode
+            collection,
+            source_file,
+            chunks,
+            wing,
+            room,
+            agent,
+            extract_mode,
+            authored_at=_extract_authored_at(filepath),
+            content_hash=content_hash,
         )
         if skipped:
             files_skipped += 1
@@ -705,6 +1156,8 @@ def _mine_convos_impl(
         for r, n in room_delta.items():
             room_counts[r] += n
 
+        for h, _ in new_items:
+            mined_content_hashes[(wing, h)] = source_file
         total_drawers += drawers_added
         files_mined += 1
         print(f"  + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}")
@@ -712,6 +1165,10 @@ def _mine_convos_impl(
             break
 
     if not dry_run:
+        # Compute hallways before the FTS5 validation: the latter opens a direct sqlite
+        # connection to the Chroma DB, which can invalidate the live collection handle on
+        # some Chroma builds and make the hallway fetch fail.
+        _compute_hallways_for_wing_safe(wing, collection, total_drawers, config=palace_config)
         _validate_palace_fts5_after_mine(palace_path)
 
     print(f"\n{'=' * 55}")

mempalace/daemon.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 import argparse
 import contextlib
 import json
+import math
 import os
 import secrets
 import sqlite3
@@ -28,6 +29,12 @@ from urllib import request as urlrequest
 from urllib.parse import parse_qs, urlparse
 
 from .config import MempalaceConfig
+from .palace import (
+    MineAlreadyRunning,
+    backend_requires_single_writer,
+    mine_palace_lock,
+    resolve_backend_name,
+)
 
 HOST = "127.0.0.1"
 STATE_ROOT_ENV = "MEMPALACE_DAEMON_STATE_ROOT"
@@ -39,6 +46,39 @@ DEFAULT_WAIT_TIMEOUT = 60.0 * 60.0
 HOOK_PROBE_TIMEOUT = 0.5
 TERMINAL_STATES = {"succeeded", "failed", "cancelled"}
 MAX_ATTEMPTS = 3
+# ``error_class`` marking a job refused the per-palace write lock. The lock
+# wraps the palace write itself, so a refusal means no drawer was filed and
+# re-running cannot duplicate palace content. Distinct from a crash
+# mid-execution, whose outcome is unknown -- see ``QueueStore.defer``.
+LOCK_REFUSAL_ERROR_CLASS = "LockHeldByOtherProcess"
+_DEFAULT_LOCK_BACKOFF_SECONDS = 60.0
+
+
+def _lock_defer_backoff_seconds() -> float:
+    """Seconds a job refused the lock cools off before the worker re-claims it.
+
+    The holder keeps the lock for as long as its write runs, which can be a long
+    mine, so re-claiming at once would spin without progress. Anything that is
+    not a positive, finite number yields the default: a typo must not crash the
+    daemon at import; ``0`` or a negative value would make the cooldown expire
+    instantly and spin the worker against a lock it cannot take; and ``inf``
+    would park the job until the daemon restarts, which is the dropped work
+    #2014 is about.
+    """
+    try:
+        value = float(
+            os.environ.get("MEMPALACE_DAEMON_LOCK_BACKOFF_SECONDS", "")
+            or _DEFAULT_LOCK_BACKOFF_SECONDS
+        )
+    except ValueError:
+        return _DEFAULT_LOCK_BACKOFF_SECONDS
+    if not math.isfinite(value) or value <= 0:
+        return _DEFAULT_LOCK_BACKOFF_SECONDS
+    return value
+
+
+# Override via env for operators; tests patch the module attribute directly.
+LOCK_DEFER_BACKOFF_SECONDS = _lock_defer_backoff_seconds()
 MAX_BODY_BYTES = 1 << 20  # 1 MiB cap on request bodies (auth-gated DoS guard)
 SHUTDOWN_DRAIN_SECONDS = 10.0
 # Terminal jobs are kept for diagnostics then pruned so the queue DB (which
@@ -411,36 +451,67 @@ class QueueStore:
             row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
             return self._row_to_job(row)
 
-    def claim_next(self) -> Job | None:
+    def claim_next(self, *, exclude: set[str] | None = None) -> Job | None:
         # Atomic across processes: the UPDATE only fires if the row is still
         # 'queued'. If two daemon processes SELECT the same row, the first to
         # UPDATE it flips state to 'running' (rowcount=1); the second's UPDATE
         # matches 0 rows (WHERE state='queued' is now false) and we re-loop
         # instead of double-executing the job. The in-process RLock does not
         # protect against a second OS process — this guard does.
+        #
+        # ``exclude`` skips jobs still cooling off after a lock refusal, so the
+        # ordering below cannot hand back the same refused job forever while
+        # newer work waits behind it. The cooldown is the worker's, held in
+        # memory rather than the schema (which has no migrations): a restart
+        # just retries such a job at once, costing one refusal, never the work.
+        #
+        # The filter runs in Python rather than as ``id NOT IN (?, ?, ...)``:
+        # that binds one host parameter per cooling job, and
+        # SQLITE_MAX_VARIABLE_NUMBER defaults to 999 below SQLite 3.32 (we
+        # support Python 3.9, whose bundled SQLite can predate it). Over the cap
+        # the query raises into the worker's catch-all, which retries a second
+        # later against a set that only ages out on the cooldown, so the queue
+        # would stall in cooldown-long fits rather than drain. Fetching
+        # ``len(exclude) + 1`` rows costs one parameter and is enough by
+        # pigeonhole: at most ``len(exclude)`` of those rows can be excluded, so
+        # the first one that survives the filter is the row a plain LIMIT 1 would
+        # have returned.
+        #
+        # ``error_json`` is cleared because it describes the claim that just
+        # ended: ``defer`` parks the refusal reason on a queued job, and a stale
+        # one would outlive its cause -- reporting a lock refusal for a job that
+        # later crashed, and leaking into recover_running's dead-letter (which
+        # COALESCEs it) in place of ``MaxAttemptsExceeded``.
+        exclude = exclude or set()
         with self._lock, self._connect() as conn:
-            row = conn.execute(
+            # Only the ids: the rows being skipped are here to be discarded, and
+            # jobs carry verbatim payloads, so selecting * would materialise a
+            # cooling job's user content just to read its id past it.
+            candidates = conn.execute(
                 """
-                SELECT * FROM jobs
+                SELECT id FROM jobs
                 WHERE state = 'queued'
                 ORDER BY priority DESC, created_at ASC
-                LIMIT 1
-                """
-            ).fetchone()
-            if row is None:
+                LIMIT ?
+                """,
+                (len(exclude) + 1,),
+            ).fetchall()
+            job_id = next((r["id"] for r in candidates if r["id"] not in exclude), None)
+            if job_id is None:
                 return None
             cur = conn.execute(
                 """
                 UPDATE jobs
-                SET state = 'running', started_at = ?, attempts = attempts + 1
+                SET state = 'running', started_at = ?, attempts = attempts + 1,
+                    error_json = NULL
                 WHERE id = ? AND state = 'queued'
                 """,
-                (_now(), row["id"]),
+                (_now(), job_id),
             )
             if cur.rowcount != 1:
                 # Lost the race to another process — nothing to run this iteration.
                 return None
-            claimed = conn.execute("SELECT * FROM jobs WHERE id = ?", (row["id"],)).fetchone()
+            claimed = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
             return self._row_to_job(claimed)
 
     def finish(
@@ -477,6 +548,56 @@ class QueueStore:
             row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
             return self._row_to_job(row)
 
+    def defer(
+        self,
+        job_id: str,
+        *,
+        claimed_started_at: str | None = None,
+        error: dict[str, Any] | None = None,
+    ) -> Job:
+        """Return a job refused the palace write lock to ``queued``, unspent.
+
+        ``mine_palace_lock`` wraps the palace write itself, so a refusal means
+        no drawer was filed and re-running cannot duplicate palace content.
+        That is what separates it from the failure ``MAX_ATTEMPTS`` guards -- a
+        daemon that died mid-execution, whose outcome is unknown and whose blind
+        retry would re-file verbatim content. Undoing ``claim_next``'s increment
+        keeps a palace that stays locked from spending the retry budget of work
+        that never landed.
+
+        ``error`` records why the job went back to the queue, so a job parked
+        behind a lock is distinguishable from one merely awaiting its turn.
+        ``claim_next`` clears it on the next claim, so the reason never outlives
+        the claim it describes.
+
+        The ``state = 'running'`` guard mirrors ``finish(only_if_running=True)``:
+        if shutdown already cancelled this job, deferring must not resurrect it.
+        ``claimed_started_at`` narrows the update further, to the claim that was
+        actually refused: if the row was re-queued and re-claimed in the window
+        (recover_running on another daemon start), a late defer must not throw
+        away that newer claim by re-queuing -- and refunding -- work it does not
+        own. ``None`` skips the check.
+        """
+        with self._lock, self._connect() as conn:
+            conn.execute(
+                """
+                UPDATE jobs
+                SET state = 'queued', started_at = NULL,
+                    attempts = MAX(attempts - 1, 0),
+                    error_json = ?
+                WHERE id = ? AND state = 'running'
+                  AND (? IS NULL OR started_at = ?)
+                """,
+                (
+                    json.dumps(error or {}, ensure_ascii=False) if error else None,
+                    job_id,
+                    claimed_started_at,
+                    claimed_started_at,
+                ),
+            )
+            row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
+            return self._row_to_job(row)
+
     def get(self, job_id: str) -> Job:
         with self._lock, self._connect() as conn:
             row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
@@ -523,6 +644,24 @@ class QueueStore:
         )
 
 
+def job_deferred_by_lock(job: dict[str, Any]) -> bool:
+    """True when a job's last claim was refused the palace lock.
+
+    Such a job is deferred, not failed (#2014): it went back to the queue and
+    runs once the lock frees. That makes it non-terminal, so a caller blocking
+    until ``TERMINAL_STATES`` would wait out the holder -- which can be a
+    long-lived session. Interactive callers use this to report the parked job
+    instead of stranding the terminal.
+
+    Keyed on the reason ``defer`` records, which ``claim_next`` clears on the
+    next claim: the answer is about the claim that just ended, not a live probe
+    of the lock (the holder may already have exited during the backoff).
+    """
+    if job.get("state") != "queued":
+        return False
+    return (job.get("error") or {}).get("error_class") == LOCK_REFUSAL_ERROR_CLASS
+
+
 def job_to_dict(job: Job, *, include_payload: bool = True) -> dict[str, Any]:
     out = {
         "id": job.id,
@@ -551,6 +690,9 @@ class DaemonRuntime:
         self.worker_wake = threading.Event()
         self.active_job_id: str | None = None
         self.worker_thread: threading.Thread | None = None
+        # job_id -> monotonic deadline before which a lock-refused job is not
+        # re-claimed (#2014). Worker-owned; only _worker_loop touches it.
+        self._deferred_until: dict[str, float] = {}
 
     def start_worker(self) -> threading.Thread:
         self.store.recover_running()
@@ -579,12 +721,37 @@ class DaemonRuntime:
         except Exception:  # noqa: BLE001 - a finish failure must not kill the worker
             pass
 
+    def _safe_defer(
+        self, job_id: str, *, claimed_started_at: str | None = None, error: dict | None = None
+    ) -> None:
+        try:
+            self.store.defer(job_id, claimed_started_at=claimed_started_at, error=error)
+        except Exception:  # noqa: BLE001 - a defer failure must not kill the worker
+            # The job stays 'running', so recover_running normally re-queues it
+            # on the next start and the work is still held -- mirroring
+            # _safe_finish's swallow. (recover_running does not refund the
+            # attempt, so repeated defer failures across MAX_ATTEMPTS restarts
+            # would eventually dead-letter it.)
+            pass
+
+    def _cooling_job_ids(self) -> set[str]:
+        """Ids of jobs still inside their post-refusal cooldown.
+
+        Pruned on every read, so the map tracks only jobs the worker is actually
+        holding back and cannot grow without bound.
+        """
+        now = time.monotonic()
+        self._deferred_until = {
+            job_id: until for job_id, until in self._deferred_until.items() if until > now
+        }
+        return set(self._deferred_until)
+
     def _worker_loop(self) -> None:
         from .service import execute_job
 
         while not self.shutdown_event.is_set():
             try:
-                job = self.store.claim_next()
+                job = self.store.claim_next(exclude=self._cooling_job_ids())
             except Exception:  # noqa: BLE001 - sqlite/disk errors must not kill the worker
                 self.shutdown_event.wait(1.0)
                 continue
@@ -602,6 +769,32 @@ class DaemonRuntime:
                     payload["backend"] = self.backend
                 result = execute_job(job.kind, payload)
                 ok = bool(result.get("success", True))
+                if not ok and result.get("error_class") == LOCK_REFUSAL_ERROR_CLASS:
+                    # Refused the palace write lock: no drawer was filed, so this
+                    # is a deferral, not a failure (#2014). Dead-lettering it
+                    # here would silently drop the work -- only hook-submitted
+                    # kinds get re-emitted, and the hook is not a retry
+                    # mechanism.
+                    self._safe_defer(
+                        job.id,
+                        claimed_started_at=job.started_at,
+                        error={
+                            "error_class": LOCK_REFUSAL_ERROR_CLASS,
+                            "message": result.get(
+                                "error", "palace write lock held by another process"
+                            ),
+                        },
+                    )
+                    # Cool the job off in the map instead of sleeping here. This
+                    # is the only worker, and the holder may outlive any wait we
+                    # could pick -- it keeps the lock until its write finishes,
+                    # and that write can be a long mine. Blocking would stall
+                    # every unrelated job behind a lock that has nothing to do
+                    # with them, and a job merely queued behind this one would
+                    # never be claimed, so it would never get the refusal marker
+                    # its caller waits on: the #2014 hang wearing a different hat.
+                    self._deferred_until[job.id] = time.monotonic() + LOCK_DEFER_BACKOFF_SECONDS
+                    continue
                 state = "succeeded" if ok else "failed"
                 error = None if ok else {"message": result.get("error", "job failed")}
                 self._safe_finish(job.id, state=state, result=result, error=error)
@@ -633,6 +826,49 @@ def _json_response(handler: BaseHTTPRequestHandler, status: int, payload: dict[s
     handler.close_connection = True
 
 
+def _restore_server_process_state(previous_env: dict[str, str | None], previous_umask: int) -> None:
+    for key, value in previous_env.items():
+        if value is None:
+            os.environ.pop(key, None)
+        else:
+            os.environ[key] = value
+    os.umask(previous_umask)
+
+
+def _close_writer_lease_after_worker(
+    writer_lease: contextlib.ExitStack,
+    worker: threading.Thread,
+) -> None:
+    """Retain writer ownership until a timed-out daemon worker really exits.
+
+    The normal daemon process may terminate first, in which case the operating
+    system releases the file lock. When ``run_server`` is embedded in a larger
+    process, this reaper prevents the server thread from exposing the palace to
+    another writer while the old worker is still finishing a mutation.
+    """
+
+    def _wait_and_close() -> None:
+        worker.join()
+        writer_lease.close()
+
+    threading.Thread(
+        target=_wait_and_close,
+        name="mempalace-daemon-writer-lease-reaper",
+        daemon=True,
+    ).start()
+
+
+def _close_or_defer_writer_lease(
+    writer_lease: contextlib.ExitStack,
+    runtime: "DaemonRuntime | None",
+) -> None:
+    worker = runtime.worker_thread if runtime is not None else None
+    if worker is not None and worker.is_alive():
+        _close_writer_lease_after_worker(writer_lease, worker)
+    else:
+        writer_lease.close()
+
+
 def run_server(palace_path: str, *, backend: str | None = None, port: int = 0) -> None:
     palace_path = canonical_palace_path(palace_path)
     previous_env = {
@@ -651,8 +887,31 @@ def run_server(palace_path: str, *, backend: str | None = None, port: int = 0) -
     # QueueStore (its _init_db opens the DB in WAL mode) — not only once the HTTP
     # server starts. Restored in the finally at the end of run_server.
     prev_umask = os.umask(0o077)
-    token = ensure_token(palace_path)
-    runtime = DaemonRuntime(palace_path, backend=backend)
+    runtime = None
+    writer_lease = contextlib.ExitStack()
+    try:
+        resolved_backend = resolve_backend_name(palace_path, explicit=backend)
+        if backend_requires_single_writer(resolved_backend):
+            try:
+                writer_lease.enter_context(mine_palace_lock(palace_path))
+            except MineAlreadyRunning as exc:
+                raise DaemonError(
+                    "writable daemon startup refused: another writer owns "
+                    f"local backend {resolved_backend!r} for {palace_path!r}; "
+                    "stop the existing writable MCP/direct/daemon owner, or "
+                    "route all writes through that owner"
+                ) from exc
+
+        token = ensure_token(palace_path)
+        # Backend resolution above is only the ownership decision. Preserve
+        # the caller's explicit/implicit distinction in queued payloads:
+        # DaemonRuntime historically injects a backend only when one was
+        # explicitly selected.
+        runtime = DaemonRuntime(palace_path, backend=backend)
+    except BaseException:
+        writer_lease.close()
+        _restore_server_process_state(previous_env, prev_umask)
+        raise
 
     class _Handler(BaseHTTPRequestHandler):
         protocol_version = "HTTP/1.1"
@@ -803,7 +1062,8 @@ def run_server(palace_path: str, *, backend: str | None = None, port: int = 0) -
             finally:
                 _drain_and_cleanup(runtime, palace_path, previous_env)
     finally:
-        os.umask(prev_umask)
+        _close_or_defer_writer_lease(writer_lease, runtime)
+        _restore_server_process_state(previous_env, prev_umask)
 
 
 def _drain_and_cleanup(
@@ -935,12 +1195,24 @@ class DaemonClient:
     def list_jobs(self, limit: int = 20) -> list[dict[str, Any]]:
         return self.request("GET", f"/jobs?limit={int(limit)}")["jobs"]
 
-    def wait(self, job_id: str, *, timeout: float = DEFAULT_WAIT_TIMEOUT) -> dict[str, Any]:
+    def wait(
+        self,
+        job_id: str,
+        *,
+        timeout: float = DEFAULT_WAIT_TIMEOUT,
+        stop_on_lock_deferral: bool = False,
+    ) -> dict[str, Any]:
         deadline = time.monotonic() + timeout
         while True:
             job = self.get_job(job_id)
             if job["state"] in TERMINAL_STATES:
                 return job
+            # A job parked behind the palace lock never becomes terminal on its
+            # own, so an interactive caller must be able to stop here instead of
+            # waiting out the holder (#2014). Background callers keep the old
+            # behaviour and simply wait.
+            if stop_on_lock_deferral and job_deferred_by_lock(job):
+                return job
             if time.monotonic() >= deadline:
                 raise DaemonError(f"timed out waiting for job {job_id}")
             time.sleep(0.2)
@@ -963,6 +1235,21 @@ def get_client_if_running(palace_path: str, *, health_timeout: float = 5.0) -> D
 
 
 def _detached_kwargs(log_path: Path) -> dict[str, Any]:
+    """Kwargs that spawn the daemon with a hidden console, detached from the CLI.
+
+    Shares the Windows flag logic with ``hooks_cli._detached_popen_kwargs``
+    (which takes no args and opens no log file). On Windows we use
+    ``CREATE_NO_WINDOW`` rather than ``DETACHED_PROCESS``: the latter gives the
+    child no console, so a console-subsystem grandchild later allocates a fresh
+    *visible* window (#1783); ``CREATE_NO_WINDOW`` gives a hidden console that
+    descendants inherit instead. Surviving the launching terminal is carried by
+    ``CREATE_BREAKAWAY_FROM_JOB`` (escapes the parent Job Object's kill-on-close)
+    plus the daemon never being attached to that console -- not by the console
+    flag -- while ``CREATE_NEW_PROCESS_GROUP`` isolates Ctrl-C/Ctrl-Break routing.
+    ``CREATE_NO_WINDOW`` is ignored when OR'd with ``DETACHED_PROCESS``, so this
+    replaces that flag rather than adding it. ``stdin=DEVNULL`` and stdout/stderr
+    redirected to the log avoid the #1268 parent hang.
+    """
     log_path.parent.mkdir(parents=True, exist_ok=True)
     log_fh = open(log_path, "a", encoding="utf-8")
     # The daemon log may capture verbatim content in tracebacks — owner-only.
@@ -975,7 +1262,7 @@ def _detached_kwargs(log_path: Path) -> dict[str, Any]:
     }
     if os.name == "nt":
         flags = 0
-        for name in ("DETACHED_PROCESS", "CREATE_NEW_PROCESS_GROUP", "CREATE_BREAKAWAY_FROM_JOB"):
+        for name in ("CREATE_NO_WINDOW", "CREATE_NEW_PROCESS_GROUP", "CREATE_BREAKAWAY_FROM_JOB"):
             flags |= getattr(subprocess, name, 0)
         if flags:
             kwargs["creationflags"] = flags
@@ -1105,6 +1392,7 @@ def submit_job(
     wait: bool = True,
     auto_start: bool = False,
     timeout: float = DEFAULT_WAIT_TIMEOUT,
+    stop_on_lock_deferral: bool = False,
 ) -> dict[str, Any]:
     # Strictly opt-in: callers that want the daemon auto-started must say so
     # explicitly (the CLI --daemon path passes auto_start=True). The default
@@ -1118,7 +1406,7 @@ def submit_job(
     job = client.submit(kind, payload, dedupe_key=dedupe_key, priority=priority)
     if not wait:
         return job
-    return client.wait(job["id"], timeout=timeout)
+    return client.wait(job["id"], timeout=timeout, stop_on_lock_deferral=stop_on_lock_deferral)
 
 
 def stop_daemon(palace_path: str) -> bool:

mempalace/date_window.py

@@ -0,0 +1,107 @@
+"""Shared date-window parsing for read-path filters.
+
+``list_drawers`` (#1128) and ``search`` (#463) accept the same optional
+``since``/``before`` bounds and compare them against drawer ``filed_at``
+metadata. The parsing and window semantics live here — a side-effect-free
+module — so ``searcher`` can use them without importing ``mcp_server``
+(whose import installs MCP stdio protection) and ``mcp_server`` keeps its
+existing behavior by aliasing these functions.
+
+Semantics (issue spec, #1128):
+* ``since`` is inclusive, ``before`` is exclusive: ``[since, before)``.
+* Comparison is wall-clock and timezone-naive. ``filed_at`` values are
+  written as naive local ISO strings (``datetime.now().isoformat()``)
+  almost everywhere; the one aware writer (``diary_ingest``, UTC) is
+  compared on its wall-clock fields after the offset is dropped.
+* A drawer whose ``filed_at`` is missing or unparseable is EXCLUDED
+  whenever a bound is active — a date-filtered result must never
+  silently include rows of unknown age.
+"""
+
+from datetime import datetime
+from typing import Optional
+
+
+def parse_date_bound(value: Optional[str] = None, field_name: str = "date") -> Optional[datetime]:
+    """Parse an optional ISO-8601 date/datetime filter bound.
+
+    Accepts a date (``"2026-04-01"``), a naive timestamp
+    (``"2026-04-01T09:30:00"``), or one carrying a ``Z``/``+HH:MM`` offset.
+    Returns a naive ``datetime`` for wall-clock
+    comparison against drawer ``filed_at`` values, which are stored as naive
+    local ISO strings (``datetime.now().isoformat()``). Any timezone offset on
+    the input is dropped so an aware bound never raises a ``TypeError`` against
+    a naive ``filed_at``. Comparison is therefore wall-clock, which is what the
+    local-first single-machine model wants; an offset bound is matched on its
+    wall-clock fields, not its absolute instant, so a bound whose offset differs
+    from the zone ``filed_at`` was recorded in is matched by clock time.
+    The accepted grammar is a date, an ISO timestamp (optionally fractional),
+    and an optional ``Z``/``±HH:MM`` offset; other ISO 8601 forms (basic format,
+    week dates) are outside the contract and are rejected on the Python 3.9 floor
+    even where a newer ``fromisoformat`` would accept them.
+    Blank / whitespace-only means "no filter" (``None``).
+    Raises ``ValueError`` on an unparseable value so the caller can surface a
+    clear error, mirroring the wing/room sanitizers.
+    """
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise ValueError(f"{field_name} must be an ISO date string")
+    value = value.strip()
+    if not value:
+        return None
+    # datetime.fromisoformat before Python 3.11 rejects a trailing "Z" (Zulu),
+    # and appending "+00:00" would break a date-only value on 3.9/3.10
+    # ("2026-04-01+00:00" is rejected there). Any offset is dropped below for
+    # wall-clock comparison anyway, so just strip a trailing Z/z; both date and
+    # date-time Zulu inputs then parse on the 3.9 floor.
+    iso = value[:-1] if value.endswith(("Z", "z")) else value
+    try:
+        parsed = datetime.fromisoformat(iso)
+    except ValueError as exc:
+        raise ValueError(
+            f"{field_name} must be an ISO date string "
+            f"(e.g. '2026-04-01' or '2026-04-01T09:30:00'), got {value!r}"
+        ) from exc
+    if parsed.tzinfo is not None:
+        parsed = parsed.replace(tzinfo=None)
+    return parsed
+
+
+def parse_window(since: Optional[str] = None, before: Optional[str] = None):
+    """Parse a ``[since, before)`` pair, rejecting an inverted window.
+
+    Returns ``(since_dt, before_dt)`` — either side ``None`` when absent.
+    Raises ``ValueError`` (naming the offending field or the inversion) so
+    callers surface the same message everywhere a window is accepted.
+    """
+    since_dt = parse_date_bound(since, "since")
+    before_dt = parse_date_bound(before, "before")
+    if since_dt is not None and before_dt is not None and since_dt >= before_dt:
+        raise ValueError(f"since ({since!r}) must be earlier than before ({before!r})")
+    return since_dt, before_dt
+
+
+def filed_at_in_window(
+    filed_at, since_dt: Optional[datetime], before_dt: Optional[datetime]
+) -> bool:
+    """True if a drawer's ``filed_at`` falls in ``[since, before)``.
+
+    ``since`` is inclusive and ``before`` is exclusive, matching the issue spec.
+    Parsing (``Z``/offset normalization, tz drop) is delegated to
+    ``parse_date_bound`` so a bound and a ``filed_at`` are compared
+    identically. A drawer whose ``filed_at`` is missing or unparseable cannot
+    be confirmed in-window, so it is EXCLUDED whenever a bound is active — a
+    date-filtered listing must never silently include rows of unknown age.
+    """
+    try:
+        filed_dt = parse_date_bound(filed_at, "filed_at")
+    except ValueError:
+        return False
+    if filed_dt is None:
+        return False
+    if since_dt is not None and filed_dt < since_dt:
+        return False
+    if before_dt is not None and filed_dt >= before_dt:
+        return False
+    return True

mempalace/dedup.py

@@ -50,12 +50,29 @@ def _get_palace_path():
         return os.path.join(os.path.expanduser("~"), ".mempalace", "palace")
 
 
-def get_source_groups(col, min_count=MIN_DRAWERS_TO_CHECK, source_pattern=None, wing=None):
+def get_source_groups(
+    col, min_count=MIN_DRAWERS_TO_CHECK, source_pattern=None, wing=None, palace_path=None
+):
     """Group drawers by source_file, return groups with min_count+ entries.
 
     If wing is specified, only considers drawers in that wing. This catches
     cross-wing duplicates when the same source was mined into multiple wings.
+
+    ``palace_path``, when passed, preflights HNSW divergence before count():
+    a diverged segment can hit the #1222 SIGSEGV/panic class, which a
+    try/except around count() cannot catch. Omitted by existing callers
+    that don't have a palace_path handy (e.g. tests) -- the check is simply
+    skipped in that case, matching this function's pre-existing behavior.
     """
+    if palace_path is not None:
+        from .backends.chroma import hnsw_capacity_status
+
+        capacity_info = hnsw_capacity_status(palace_path, COLLECTION_NAME)
+        if capacity_info.get("diverged"):
+            print(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+            print("  Run `mempalace repair --mode from-sqlite --archive-existing` first.")
+            return {}
+
     total = col.count()
     groups = defaultdict(list)
 
@@ -134,7 +151,7 @@ def show_stats(palace_path=None):
     palace_path = palace_path or _get_palace_path()
     col = get_collection(palace_path, COLLECTION_NAME)
 
-    groups = get_source_groups(col)
+    groups = get_source_groups(col, palace_path=palace_path)
 
     total_drawers = sum(len(ids) for ids in groups.values())
     print(f"\n  Sources with {MIN_DRAWERS_TO_CHECK}+ drawers: {len(groups)}")
@@ -166,15 +183,28 @@ def dedup_palace(
 
     col = get_collection(palace_path, COLLECTION_NAME)
 
+    # Preflight HNSW divergence before this function's own count() print --
+    # get_source_groups's palace_path guard (added alongside this one) only
+    # covers its own internal count(), not this earlier one. count() on a
+    # diverged segment can hard-crash the process (#1222); a try/except
+    # cannot catch that, so it must never be reached at all when diverged.
+    from .backends.chroma import hnsw_capacity_status
+
+    capacity_info = hnsw_capacity_status(palace_path, COLLECTION_NAME)
+    if capacity_info.get("diverged"):
+        print(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+        print("  Run `mempalace repair --mode from-sqlite --archive-existing` first.")
+        return
+
     print(f"  Palace: {palace_path}")
     print(f"  Drawers: {col.count():,}")
     print(f"  Threshold: {threshold}")
     print(f"  Mode: {'DRY RUN' if dry_run else 'LIVE'}")
-    print(f"{'─' * 55}")
+    print(f"{'-' * 55}")
 
     if wing:
         print(f"  Wing: {wing}")
-    groups = get_source_groups(col, min_count, source_pattern, wing=wing)
+    groups = get_source_groups(col, min_count, source_pattern, wing=wing, palace_path=palace_path)
     print(f"\n  Sources to check: {len(groups)}")
 
     t0 = time.time()
@@ -197,7 +227,7 @@ def dedup_palace(
 
     elapsed = time.time() - t0
 
-    print(f"\n{'─' * 55}")
+    print(f"\n{'-' * 55}")
     print(f"  Done in {elapsed:.1f}s")
     print(
         f"  Drawers: {total_kept + total_deleted:,} → {total_kept:,}  (-{total_deleted:,} removed)"

mempalace/dialect.py

@@ -357,8 +357,11 @@ class Dialect:
             "skip_names": ["Gandalf", "Sherlock"]
         }
         """
-        with open(config_path, "r") as f:
-            config = json.load(f)
+        try:
+            with open(config_path, "r", encoding="utf-8") as f:
+                config = json.load(f)
+        except UnicodeDecodeError as exc:
+            raise ValueError(f"{config_path} is not valid UTF-8 — re-save it as UTF-8") from exc
         return cls(
             entities=config.get("entities", {}),
             skip_names=config.get("skip_names", []),
@@ -381,7 +384,7 @@ class Dialect:
             "entities": canonical,
             "skip_names": self.skip_names,
         }
-        with open(config_path, "w") as f:
+        with open(config_path, "w", encoding="utf-8") as f:
             json.dump(config, f, indent=2)
 
     # === ENCODING (entity/emotion primitives) ===
@@ -776,11 +779,11 @@ class Dialect:
 
     def compress_file(self, zettel_json_path: str, output_path: str = None) -> str:
         """Read a zettel JSON file and compress it to AAAK Dialect."""
-        with open(zettel_json_path, "r") as f:
+        with open(zettel_json_path, "r", encoding="utf-8") as f:
             data = json.load(f)
         dialect = self.encode_file(data)
         if output_path:
-            with open(output_path, "w") as f:
+            with open(output_path, "w", encoding="utf-8") as f:
                 f.write(dialect)
         return dialect
 
@@ -790,14 +793,14 @@ class Dialect:
         for fname in sorted(os.listdir(zettel_dir)):
             if fname.endswith(".json"):
                 fpath = os.path.join(zettel_dir, fname)
-                with open(fpath, "r") as f:
+                with open(fpath, "r", encoding="utf-8") as f:
                     data = json.load(f)
                 dialect = self.encode_file(data)
                 all_dialect.append(dialect)
                 all_dialect.append("---")
         combined = "\n".join(all_dialect)
         if output_path:
-            with open(output_path, "w") as f:
+            with open(output_path, "w", encoding="utf-8") as f:
                 f.write(combined)
         return combined
 
@@ -824,7 +827,7 @@ class Dialect:
             if not fname.endswith(".json"):
                 continue
             fpath = os.path.join(zettel_dir, fname)
-            with open(fpath, "r") as f:
+            with open(fpath, "r", encoding="utf-8") as f:
                 data = json.load(f)
 
             file_num = fname.replace("file_", "").replace(".json", "")
@@ -846,7 +849,7 @@ class Dialect:
             if not fname.endswith(".json"):
                 continue
             fpath = os.path.join(zettel_dir, fname)
-            with open(fpath, "r") as f:
+            with open(fpath, "r", encoding="utf-8") as f:
                 data = json.load(f)
             for t in data.get("tunnels", []):
                 all_tunnels.append(t)
@@ -918,7 +921,7 @@ class Dialect:
         result = "\n".join(lines)
 
         if output_path:
-            with open(output_path, "w") as f:
+            with open(output_path, "w", encoding="utf-8") as f:
                 f.write(result)
 
         return result
@@ -1029,7 +1032,7 @@ if __name__ == "__main__":
             "skip_names": [],
         }
         out_path = "entities.json"
-        with open(out_path, "w") as f:
+        with open(out_path, "w", encoding="utf-8") as f:
             json.dump(example, f, indent=2)
         print(f"Created example config: {out_path}")
         print("Edit this file with your own entity mappings, then use --config entities.json")
@@ -1052,7 +1055,7 @@ if __name__ == "__main__":
         print(result)
 
     elif args[0] == "--stats":
-        with open(args[1], "r") as f:
+        with open(args[1], "r", encoding="utf-8") as f:
             data = json.load(f)
         json_str = json.dumps(data, indent=2)
         encoded = dialect.encode_file(data)
@@ -1060,7 +1063,7 @@ if __name__ == "__main__":
         print("=== COMPRESSION STATS ===")
         print(f"JSON:     ~{stats['original_tokens_est']:,} tokens (est)")
         print(f"AAAK:     ~{stats['summary_tokens_est']:,} tokens (est)")
-        print(f"Ratio:    {stats['size_ratio']}x (lossy — information is lost)")
+        print(f"Ratio:    {stats['size_ratio']}x (lossy -- information is lost)")
         print()
         print("=== AAAK DIALECT OUTPUT ===")
         print(encoded)

mempalace/embedding.py

@@ -1,10 +1,12 @@
 """Embedding function factory with hardware acceleration.
 
-Returns a ChromaDB-compatible embedding function bound to a user-selected
-ONNX Runtime execution provider.
+Returns a ChromaDB-compatible embedding function — either a local ONNX model
+bound to a user-selected ONNX Runtime execution provider, or an
+OpenAI-compatible HTTP ``/v1/embeddings`` endpoint.
 
-Two embedding models are available, selected via ``MEMPALACE_EMBEDDING_MODEL``
-or ``embedding_model`` in ``~/.mempalace/config.json``:
+Three embedding-model options are available, selected via
+``MEMPALACE_EMBEDDING_MODEL`` or ``embedding_model`` in
+``~/.mempalace/config.json``:
 
 * ``minilm`` (default) — ``all-MiniLM-L6-v2``, 384-dim, English-only training.
   ChromaDB's default; what every existing palace was built with.
@@ -15,6 +17,16 @@ or ``embedding_model`` in ``~/.mempalace/config.json``:
   model is lazy-downloaded from HuggingFace on first use. Switching models
   on an existing palace requires ``mempalace repair rebuild-index``
   (different vector space).
+* ``openai-compat`` — embeddings served by any OpenAI-compatible
+  ``/v1/embeddings`` endpoint (LM Studio, llama.cpp, vLLM, Ollama's OpenAI
+  shim, or a self-hosted server) instead of a local ONNX model. Useful for
+  larger / multilingual embedders (e.g. Qwen3-Embedding) or GPU offload.
+  Endpoint settings are read from ``config.json`` as ``embedding_api_url`` /
+  ``embedding_api_model`` / ``embedding_api_key`` (each overridable via the
+  matching ``MEMPALACE_EMBEDDING_API_*`` env var). Vectors are L2-normalized
+  for the cosine collection; the dimension is whatever the server returns, so
+  switching to/from this backend also requires ``mempalace repair
+  rebuild-index``. Stays local when the endpoint is on your machine/LAN.
 
 Supported devices (env ``MEMPALACE_EMBEDDING_DEVICE`` or ``embedding_device``
 in ``~/.mempalace/config.json``):
@@ -31,11 +43,14 @@ rather than hard-failing — mining must still work on a laptop without CUDA.
 
 from __future__ import annotations
 
+import hashlib
 import logging
 import os
 import threading
 from typing import Optional
 
+from .version import __version__
+
 logger = logging.getLogger(__name__)
 
 _PROVIDER_MAP = {
@@ -89,7 +104,7 @@ def _resolve_providers(device: str) -> tuple[list, str]:
     requested = _PROVIDER_MAP.get(device)
     if requested is None:
         if device not in _WARNED:
-            logger.warning("Unknown embedding_device %r — falling back to cpu", device)
+            logger.warning("Unknown embedding_device %r -- falling back to cpu", device)
             _WARNED.add(device)
         return (["CPUExecutionProvider"], "cpu")
 
@@ -216,10 +231,49 @@ _EMBEDDINGGEMMA_MAX_LEN = 2048
 # matches the internal batch size of chromadb's ONNXMiniLM_L6_V2, whose
 # chunked _forward survives the same call sites. embeddinggemma's
 # sentence_embedding output is attention-masked, so sub-batch padding
-# does not change any row's vector.
+# does not change any row's vector. __call__ decides which documents share
+# a sub-batch by size rather than by arrival order (#2104), because the run
+# is priced on that padded length and not on the document count.
 _EMBEDDINGGEMMA_BATCH_SIZE = 32
 
 
+def _sanitize_embeddinggemma_input_ids(tokenizer, input_ids, np):
+    """Replace tokenizer-only IDs that the text ONNX model cannot embed."""
+    model_vocab_size = tokenizer.get_vocab_size(with_added_tokens=False)
+    out_of_range = (input_ids < 0) | (input_ids >= model_vocab_size)
+
+    if not np.any(out_of_range):
+        return input_ids
+
+    unknown_token_id = tokenizer.token_to_id("<unk>")
+    if unknown_token_id is None or not 0 <= unknown_token_id < model_vocab_size:
+        raise RuntimeError(
+            "EmbeddingGemma tokenizer produced token IDs outside the ONNX "
+            "text vocabulary, but no valid <unk> token is available"
+        )
+
+    invalid_ids = sorted({int(token_id) for token_id in input_ids[out_of_range]})
+    warning_key = (
+        "embeddinggemma-out-of-range-token-ids",
+        model_vocab_size,
+        tuple(invalid_ids),
+    )
+
+    if warning_key not in _WARNED:
+        logger.warning(
+            "EmbeddingGemma tokenizer produced token IDs outside the ONNX "
+            "text vocabulary (size=%d): %s; remapping to <unk> (%d)",
+            model_vocab_size,
+            invalid_ids,
+            unknown_token_id,
+        )
+        _WARNED.add(warning_key)
+
+    sanitized = input_ids.copy()
+    sanitized[out_of_range] = unknown_token_id
+    return sanitized
+
+
 class EmbeddinggemmaONNX:
     """ChromaDB-compatible EF using embeddinggemma-300m ONNX (q8, MRL→384d).
 
@@ -319,6 +373,33 @@ class EmbeddinggemmaONNX:
             self._session = session
 
     def __call__(self, input: str | list[str] | None) -> list[list[float]]:  # noqa: A002 — ChromaDB EF protocol
+        """Embed ``input``, returning one vector per document in input order.
+
+        Documents are grouped by size before the sub-batch split. The
+        tokenizer pads every row of a sub-batch to the longest sequence in
+        it, and attention cost per layer is batch x heads x length^2, so one
+        long document drags a whole sub-batch up to its own length. Without
+        grouping the bill is set by arrival order: a verbatim transcript
+        whose long tool results sit between one-line replies pays the long
+        length for nearly every row (#2104).
+
+        An input that fits a single sub-batch is left in arrival order: every
+        row pads to the same width either way, so the keys would buy nothing
+        on the one-document search path.
+
+        Regrouping does not change what a row means. The model's
+        ``sentence_embedding`` output is attention-masked, so padding never
+        enters a row's values; what does move is float32 rounding, because a
+        different padded width changes the reduction order inside the GEMMs.
+        Measured against the same documents embedded in arrival order, that
+        residual peaks at one float32 ULP (1.2e-07 absolute, cosine
+        0.99999992).
+
+        The key is UTF-8 byte length rather than character count: this model
+        is multilingual, and bytes per token vary far less across scripts
+        than characters per token do. The sort is stable, so equal-size
+        documents keep arrival order and the split stays reproducible.
+        """
         if isinstance(input, str):
             # A bare string would be iterated character by character below,
             # silently producing one garbage vector per character.
@@ -330,16 +411,28 @@ class EmbeddinggemmaONNX:
             return []
         self._lazy_load()
         np = self._np
-        embeddings: list[list[float]] = []
-        # Tokenize and run per sub-batch, not over the whole input: padding
-        # is to the longest sequence in the sub-batch, and the ONNX runtime
-        # only ever holds batch_size rows of attention buffers at a time
-        # (#1770).
-        for start in range(0, len(input), self._batch_size):
-            chunk = input[start : start + self._batch_size]
-            texts = [_EMBEDDINGGEMMA_PREFIX + t for t in chunk]
+        # One sub-batch pads identically whatever the order, so the sort is
+        # only worth its keys once the input splits into several.
+        order: range | list[int] = range(len(input))
+        if len(input) > self._batch_size:
+            order = sorted(range(len(input)), key=lambda i: len(input[i].encode("utf-8")))
+        # Row i is filled by the sub-batch that carries document i. ``order``
+        # is a permutation of every index, so no placeholder survives; callers
+        # (ChromaDB included) zip the result against their ids positionally.
+        embeddings: list[list[float] | None] = [None] * len(input)
+        # Tokenize and run per sub-batch, not over the whole input: the ONNX
+        # runtime only ever holds batch_size rows of attention buffers at a
+        # time (#1770).
+        for start in range(0, len(order), self._batch_size):
+            idxs = order[start : start + self._batch_size]
+            texts = [_EMBEDDINGGEMMA_PREFIX + input[i] for i in idxs]
             encs = self._tokenizer.encode_batch(texts)
             input_ids = np.asarray([e.ids for e in encs], dtype=np.int64)
+            input_ids = _sanitize_embeddinggemma_input_ids(
+                self._tokenizer,
+                input_ids,
+                np,
+            )
             attention_mask = np.asarray([e.attention_mask for e in encs], dtype=np.int64)
             outputs = self._session.run(
                 None, {"input_ids": input_ids, "attention_mask": attention_mask}
@@ -348,7 +441,16 @@ class EmbeddinggemmaONNX:
             # L2-normalize so cosine similarity == dot product (matches what the
             # MTEB methodology assumes; ChromaDB's distance is configured for it).
             norms = np.linalg.norm(sent_emb, axis=1, keepdims=True) + 1e-12
-            embeddings.extend((sent_emb / norms).tolist())
+            rows = (sent_emb / norms).tolist()
+            if len(rows) != len(idxs):
+                # zip would truncate silently and leave a None in the result,
+                # which only surfaces far downstream in the caller's array
+                # conversion. Fail on the sub-batch that came back short.
+                raise RuntimeError(
+                    f"embeddinggemma returned {len(rows)} rows for a {len(idxs)}-document sub-batch"
+                )
+            for row_index, row in zip(idxs, rows):
+                embeddings[row_index] = row
         return embeddings
 
     def embed_query(self, input: list[str]) -> list[list[float]]:  # noqa: A002 — ChromaDB EF protocol
@@ -360,6 +462,165 @@ class EmbeddinggemmaONNX:
         return self(input)
 
 
+# ── OpenAI-compatible embedding API ──────────────────────────────────────
+# Fetch embeddings from an OpenAI-compatible ``/v1/embeddings`` server
+# (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or any compatible
+# endpoint) instead of running a model locally. Selected by
+# ``embedding_model == "openai-compat"``. Connection settings (URL, model,
+# optional key) are resolved by :class:`~mempalace.config.MempalaceConfig`
+# as the single source of truth — see ``embedding_api_url`` /
+# ``embedding_api_model`` / ``embedding_api_key`` (each env-overridable).
+_EF_API_BATCH = 64
+_EF_API_TIMEOUT = 120
+
+
+class EmbeddingAPIError(RuntimeError):
+    """Raised when the embedding API is unreachable or returns an invalid body.
+
+    Module-specific subclass mirroring ``llm_client.LLMError`` so callers can
+    distinguish embedding-endpoint failures; subclasses ``RuntimeError`` so
+    existing ``except RuntimeError`` paths still catch it.
+    """
+
+
+class OpenAICompatEmbeddingFunction:
+    """ChromaDB-compatible EF backed by an OpenAI-compatible ``/v1/embeddings``
+    endpoint (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, etc.).
+
+    Selected via ``embedding_model == "openai-compat"``. Vectors are produced
+    server-side and fetched over HTTP, which changes the vector space — so
+    ``name()`` encodes the model id: ChromaDB persists the EF name on the
+    collection and rejects mismatched reads, the signal to run ``mempalace
+    repair rebuild-index`` after changing model/endpoint. stdlib ``urllib``
+    only, no new dependency.
+    """
+
+    def __init__(self, base_url: str, model: str, api_key: Optional[str] = None):
+        self._url = self._resolve_url(base_url)
+        self._model = model
+        self._api_key = api_key
+
+    @staticmethod
+    def _resolve_url(base_url: str) -> str:
+        """Accept a base host, a ``/v1`` base, or a full endpoint URL.
+
+        Mirrors ``llm_client.OpenAICompatProvider._resolve_url`` so both sides
+        treat an ``http://host:port`` endpoint the same way.
+        """
+        url = base_url.rstrip("/")
+        if url.endswith("/embeddings"):
+            return url
+        if url.endswith("/v1"):
+            return f"{url}/embeddings"
+        return f"{url}/v1/embeddings"
+
+    def name(self) -> str:
+        # Encode the model so switching it changes the persisted EF identity
+        # and forces a rebuild_index (vectors from a different model/space are
+        # not interchangeable). ChromaDB compares this on every read.
+        return f"openai_compat_emb_{self._model}".replace("/", "_")
+
+    def embed_query(self, input):  # noqa: A002 — ChromaDB EF protocol uses `input`
+        # ChromaDB 1.5 dispatches query embedding through embed_query (add uses
+        # __call__). Mirror the EmbeddingFunction protocol default: same path.
+        return self(input)
+
+    def __call__(self, input):  # noqa: A002 — ChromaDB EF protocol uses `input`
+        import http.client
+        import json
+        from urllib.error import HTTPError, URLError
+        from urllib.request import Request, urlopen
+
+        headers = {
+            "Content-Type": "application/json",
+            # Some hosted (Cloudflare-fronted) endpoints 403 the default
+            # ``Python-urllib`` User-Agent — send our own (see issue #1570).
+            "User-Agent": f"mempalace/{__version__}",
+        }
+        if self._api_key:
+            headers["Authorization"] = f"Bearer {self._api_key}"
+
+        out: list = []
+        texts = list(input)
+        for start in range(0, len(texts), _EF_API_BATCH):
+            batch = texts[start : start + _EF_API_BATCH]
+            # encoding_format=float is explicit so a server that defaults to
+            # base64 doesn't hand back strings we'd mis-parse as vectors.
+            payload = {"model": self._model, "input": batch, "encoding_format": "float"}
+            req = Request(self._url, data=json.dumps(payload).encode("utf-8"), headers=headers)
+            try:
+                with urlopen(req, timeout=_EF_API_TIMEOUT) as resp:
+                    data = json.loads(resp.read())
+            # ValueError covers an invalid/missing URL scheme and json.JSONDecodeError;
+            # http.client.HTTPException covers low-level protocol faults (BadStatusLine,
+            # IncompleteRead) common with local/overloaded servers.
+            except (HTTPError, URLError, OSError, http.client.HTTPException, ValueError) as e:
+                raise EmbeddingAPIError(
+                    f"Embedding API request to {self._url} failed: {e}. Check that the "
+                    f"server is reachable and MEMPALACE_EMBEDDING_API_URL / embedding_api_url "
+                    f"is correct."
+                ) from e
+            out.extend(self._vectors_from_response(data, len(batch)))
+        return out
+
+    def _vectors_from_response(self, data, n: int) -> list:
+        """Validate one ``/v1/embeddings`` response and return L2-normed vectors.
+
+        Guards every way a non-conformant server could corrupt the store
+        silently: a missing/short ``data`` array, response ``index`` values
+        that aren't the contiguous ``0..n-1`` batch positions (sorting then
+        zipping positionally would otherwise misalign vectors with texts), and
+        malformed / ragged / base64 embedding payloads. All failures raise
+        :class:`EmbeddingAPIError` naming the endpoint rather than a cryptic
+        numpy error — a silent wrong result would break the 100%-recall promise.
+        """
+        import numpy as np
+
+        if not isinstance(data, dict):
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned a non-object response: {data}"
+            )
+        rows = data.get("data")
+        if not isinstance(rows, list):
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned no 'data' array: {data.get('error', data)}"
+            )
+        if len(rows) != n:
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned {len(rows)} embeddings for {n} inputs"
+            )
+        # The endpoint may return rows out of order — sort by index, then
+        # require the indices to be exactly 0..n-1 so positional alignment is
+        # provably correct (a server using absolute or duplicate indices would
+        # otherwise pass the count check yet map vectors to the wrong texts).
+        try:
+            rows = sorted(rows, key=lambda d: d.get("index", -1))
+            indices = [r.get("index") for r in rows]
+        except AttributeError as e:
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned non-object rows: {e}"
+            ) from e
+        if indices != list(range(n)):
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned non-contiguous or duplicate "
+                f"'index' values; cannot align embeddings with inputs"
+            )
+        try:
+            arr = np.asarray([r["embedding"] for r in rows], dtype=np.float32)
+        except (KeyError, TypeError, ValueError) as e:
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned malformed embeddings: {e}"
+            ) from e
+        if arr.ndim != 2:
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned non-vector embeddings (shape {arr.shape})"
+            )
+        # L2-normalize so cosine == dot product (collection uses
+        # hnsw:space=cosine), matching EmbeddinggemmaONNX above.
+        norms = np.linalg.norm(arr, axis=1, keepdims=True) + 1e-12
+        return (arr / norms).tolist()
+
+
 def get_embedding_function(device: Optional[str] = None, model: Optional[str] = None):
     """Return a cached embedding function for the requested device + model.
 
@@ -377,6 +638,41 @@ def get_embedding_function(device: Optional[str] = None, model: Optional[str] =
         if model is None:
             model = cfg.embedding_model
 
+    # OpenAI-compatible embedding API: bypasses local ONNX entirely. Checked
+    # before device→provider resolution since it needs no hardware accelerator.
+    if model == "openai-compat":
+        from .config import MempalaceConfig
+
+        cfg = MempalaceConfig()
+        url = cfg.embedding_api_url
+        if not url:
+            raise ValueError(
+                "embedding_model='openai-compat' requires an endpoint — set "
+                "embedding_api_url in ~/.mempalace/config.json or the "
+                "MEMPALACE_EMBEDDING_API_URL env var (e.g. http://host:port)"
+            )
+        api_model = cfg.embedding_api_model
+        if not api_model:
+            raise ValueError(
+                "embedding_model='openai-compat' requires a model — set "
+                "embedding_api_model in ~/.mempalace/config.json or the "
+                "MEMPALACE_EMBEDDING_API_MODEL env var"
+            )
+        api_key = cfg.embedding_api_key
+        # Include a fingerprint of the key (never the raw secret) so a token
+        # rotation busts the cache in long-lived processes (e.g. MCP server).
+        key_fp = hashlib.sha256((api_key or "").encode("utf-8")).hexdigest()[:16]
+        cache_key = ("openai-compat", url, api_model, key_fp)
+        cached = _EF_CACHE.get(cache_key)
+        if cached is not None:
+            return cached
+        ef = OpenAICompatEmbeddingFunction(base_url=url, model=api_model, api_key=api_key)
+        _EF_CACHE[cache_key] = ef
+        logger.info(
+            "Embedding function initialized (openai-compat url=%s model=%s)", url, api_model
+        )
+        return ef
+
     providers, effective = _resolve_providers(device)
     cache_key = (model, tuple(providers))
     cached = _EF_CACHE.get(cache_key)  # lock-free fast path; dict.get is GIL-atomic
@@ -406,15 +702,21 @@ def get_embedding_function(device: Optional[str] = None, model: Optional[str] =
 
 
 def describe_device(device: Optional[str] = None) -> str:
-    """Return a short human-readable label for the resolved device.
+    """Return a short human-readable label for the resolved embedding backend.
 
-    Used by the miner CLI header so users can see at a glance whether GPU
-    acceleration actually engaged.
+    Used by the miner CLI header / MCP status so users can see at a glance
+    whether GPU acceleration engaged — or, for the ``openai-compat`` backend,
+    that embeddings are served by a remote endpoint rather than local hardware
+    (in which case the ``embedding_device`` accelerator label is irrelevant).
     """
     if device is None:
         from .config import MempalaceConfig
 
-        device = MempalaceConfig().embedding_device
+        cfg = MempalaceConfig()
+        if cfg.embedding_model == "openai-compat":
+            url = cfg.embedding_api_url
+            return f"openai-compat ({url})" if url else "openai-compat"
+        device = cfg.embedding_device
     _, effective = _resolve_providers(device)
     return effective
 

mempalace/encoding_repair.py

@@ -0,0 +1,624 @@
+"""Safely repair high-confidence UTF-8 mojibake in MemPalace drawers."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import unicodedata
+from pathlib import Path
+from typing import Callable, Iterator, Optional, TextIO, Union
+
+_BACKUP_FORMAT = "mempalace-encoding-repair"
+_BACKUP_VERSION = 1
+_UNDEFINED_CP1252_BYTES = frozenset(
+    {
+        0x81,
+        0x8D,
+        0x8F,
+        0x90,
+        0x9D,
+    }
+)
+
+
+def _cp1252_character(
+    byte_value: int,
+) -> str:
+    """Map a legacy byte to the character stored in old palaces."""
+    if byte_value in _UNDEFINED_CP1252_BYTES:
+        # Pre-3.1 Windows reads could preserve these five undefined
+        # Windows-1252 byte values as their corresponding invisible
+        # C1 control code points.
+        return chr(byte_value)
+
+    return bytes([byte_value]).decode("cp1252")
+
+
+def _encode_mojibake_candidate(
+    text: str,
+) -> bytes:
+    """Recover original bytes, including undefined CP1252 values."""
+    raw = bytearray()
+
+    for character in text:
+        codepoint = ord(character)
+
+        if codepoint in _UNDEFINED_CP1252_BYTES:
+            raw.append(codepoint)
+        else:
+            raw.extend(character.encode("cp1252"))
+
+    return bytes(raw)
+
+
+_CONTINUATION_CHARS = "".join(
+    _cp1252_character(byte_value)
+    for byte_value in range(
+        0x80,
+        0xC0,
+    )
+)
+_CONTINUATION_CLASS = re.escape(_CONTINUATION_CHARS)
+
+# These are the characteristic visible lead characters produced by
+# common UTF-8-as-Windows-1252 corruption:
+#
+# C2/C3 -> Â/Ã
+# E2    -> â
+# F0    -> ð
+# EF    -> ï
+#
+# C4/C5 -> Ä/Å are deliberately excluded because strings such as Ų
+# can be legitimate scientific text.
+_HIGH_CONFIDENCE_RUN = re.compile(
+    rf"(?:"
+    rf"[ÂÃ][{_CONTINUATION_CLASS}]"
+    rf"|â[{_CONTINUATION_CLASS}]{{2}}"
+    rf"|ð[{_CONTINUATION_CLASS}]{{3}}"
+    rf"|ï[{_CONTINUATION_CLASS}]{{2}}"
+    rf")+"
+)
+
+# A single mojibake unit, so a chained run can be told apart from a lone
+# two-character window.
+_MOJIBAKE_UNIT = re.compile(
+    rf"[ÂÃ][{_CONTINUATION_CLASS}]"
+    rf"|â[{_CONTINUATION_CLASS}]{{2}}"
+    rf"|ð[{_CONTINUATION_CLASS}]{{3}}"
+    rf"|ï[{_CONTINUATION_CLASS}]{{2}}"
+)
+
+# Continuation bytes whose CP1252 character is typographic punctuation that
+# legitimately follows a letter in prose.
+#
+# The [ÂÃ] alternative above is only TWO characters wide, and Portuguese,
+# Vietnamese and Turkish really do end all-caps words in Ã/Â — IRMÃ, MAÇÃ,
+# MANHÃ, AMANHÃ, BÃO, NHÃ, HÂLÂ, IMÂ. Prose then puts a closing quote,
+# guillemet, ellipsis or dash straight after, and every one of those lives in
+# the continuation class. "«MAÇû." and "“IRMÔ" therefore match a shape that is
+# also perfectly clean text, and repairing them collapses two characters into
+# one — silently, and reported as a success.
+#
+# This is the same argument that already excluded Ä/Å as lead characters, applied
+# to the continuation side. The wider â/ð/ï windows are 3-4 characters and no
+# clean text produces those shapes, so they keep the full class.
+_PROSE_PUNCTUATION = frozenset(
+    _cp1252_character(byte_value)
+    for byte_value in (
+        0x82,  # ‚ single low-9 quotation mark
+        0x84,  # „ double low-9 quotation mark
+        0x85,  # … horizontal ellipsis
+        0x8B,  # ‹ single left-pointing angle quotation mark
+        0x91,  # ' left single quotation mark
+        0x92,  # ' right single quotation mark
+        0x93,  # " left double quotation mark
+        0x94,  # " right double quotation mark
+        0x95,  # • bullet
+        0x96,  # – en dash
+        0x97,  # — em dash
+        0x9B,  # › single right-pointing angle quotation mark
+        0xAB,  # « left-pointing double angle quotation mark
+        0xBB,  # » right-pointing double angle quotation mark
+    )
+)
+
+
+def _is_text_character(
+    character: str,
+) -> bool:
+    """Reject decode products that are invisible control characters.
+
+    Â followed by 0x80-0x9F decodes to exactly the C1 control block, so
+    "İMÂ… edildi." becomes "İM\\u0085 edildi." — visible text swapped for an
+    invisible control and reported as a repair. Natural text contains no C1
+    controls, so refusing these costs nothing.
+
+    Cf (format) stays allowed: U+FEFF is the legitimate product of the ""
+    BOM run and U+200D joins emoji sequences.
+    """
+    if character in "\t\n\r":
+        return True
+
+    return unicodedata.category(character) not in {
+        "Cc",
+        "Cs",
+        "Cn",
+    }
+
+
+def _is_ambiguous_window(
+    candidate: str,
+) -> bool:
+    """True for the one shape clean prose can also produce.
+
+    A lone [ÂÃ] followed by prose punctuation. Chained units ("ação") and the
+    3-4 character â/ð/ï windows cannot occur in clean text, so they stay
+    unconditional.
+    """
+    units = _MOJIBAKE_UNIT.findall(candidate) or [candidate]
+
+    if len(units) > 1 or units[0][0] not in "ÂÃ":
+        return False
+
+    return units[0][1] in _PROSE_PUNCTUATION
+
+
+def _preceded_by_lowercase(
+    text: str,
+    start: int,
+) -> bool:
+    """Evidence that the run is mojibake rather than clean prose.
+
+    Mojibaked letters sit INSIDE a word, so a lowercase letter runs straight into
+    the lead: "coûte", "Noël", "català»". The false-positive family is the
+    exact opposite — an ALL-CAPS word whose final letter genuinely is Ã/Â, as in
+    IRMÔ, MAÇû, MANHÃ…, HÂL», IM». Clean prose does not put a lowercase
+    letter directly before an uppercase Ã/Â.
+
+    Deliberately local. Inferring "this drawer is mojibake" from a run elsewhere
+    in the text destroys clean prose in a MIXED drawer, and drawers are mixed by
+    construction because the miner concatenates several sources into one.
+    """
+    return start > 0 and text[start - 1].isalpha() and text[start - 1].islower()
+
+
+def _decode_high_confidence_run(
+    match: re.Match,
+    text: str,
+) -> str:
+    candidate = match.group(0)
+
+    try:
+        decoded = _encode_mojibake_candidate(candidate).decode("utf-8")
+    except (
+        UnicodeEncodeError,
+        UnicodeDecodeError,
+    ):
+        return candidate
+
+    if not decoded or not all(_is_text_character(character) for character in decoded):
+        return candidate
+
+    # An ambiguous window is repaired only on positive local evidence. With no
+    # evidence the text is returned untouched: a missed repair costs nothing
+    # because the drawer is unchanged and the tool can be re-run, while a wrong
+    # repair destroys good text irrecoverably.
+    if _is_ambiguous_window(candidate) and not _preceded_by_lowercase(text, match.start()):
+        return candidate
+
+    return decoded
+
+
+def repair_mojibake_once(text: str) -> str:
+    """Repair one layer of high-confidence UTF-8-as-CP1252 mojibake."""
+    return _HIGH_CONFIDENCE_RUN.sub(
+        lambda match: _decode_high_confidence_run(match, text),
+        text,
+    )
+
+
+def repair_mojibake(
+    text: str,
+    *,
+    max_passes: int = 3,
+) -> str:
+    """Repair repeated high-confidence mojibake layers until stable."""
+    if max_passes < 1:
+        raise ValueError("max_passes must be at least 1")
+
+    current = text
+
+    for _ in range(max_passes):
+        repaired = repair_mojibake_once(current)
+
+        if repaired == current:
+            break
+
+        current = repaired
+
+    return current
+
+
+def _result_field(result, name: str):
+    if isinstance(result, dict):
+        return result.get(name)
+
+    return getattr(result, name, None)
+
+
+def _collection_name(
+    collection,
+) -> Optional[str]:
+    """Resolve a collection name through MemPalace backend wrappers."""
+
+    def resolve_name(candidate) -> Optional[str]:
+        name = getattr(
+            candidate,
+            "name",
+            None,
+        )
+
+        if callable(name):
+            try:
+                name = name()
+            except TypeError:
+                name = None
+
+        return str(name) if name else None
+
+    direct_name = resolve_name(collection)
+    if direct_name:
+        return direct_name
+
+    # ChromaCollection already provides this resolver for its wrapped
+    # chromadb collection.
+    resolver = getattr(
+        collection,
+        "_collection_name",
+        None,
+    )
+
+    if callable(resolver):
+        try:
+            resolved = resolver()
+        except (
+            AttributeError,
+            TypeError,
+        ):
+            resolved = None
+
+        if resolved:
+            return str(resolved)
+
+    # Defensive fallback for thin wrappers that expose only their inner
+    # collection object.
+    inner = getattr(
+        collection,
+        "_collection",
+        None,
+    )
+
+    if inner is not None and inner is not collection:
+        return resolve_name(inner)
+
+    return None
+
+
+def _open_private_backup(
+    path: Path,
+) -> TextIO:
+    path.parent.mkdir(
+        parents=True,
+        exist_ok=True,
+    )
+
+    descriptor = os.open(
+        str(path),
+        os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+        0o600,
+    )
+
+    try:
+        try:
+            os.chmod(path, 0o600)
+        except (
+            OSError,
+            NotImplementedError,
+        ):
+            pass
+
+        return os.fdopen(
+            descriptor,
+            "w",
+            encoding="utf-8",
+            newline="\n",
+        )
+    except Exception:
+        os.close(descriptor)
+        raise
+
+
+def _write_json_line(
+    handle: TextIO,
+    value: dict,
+) -> None:
+    handle.write(
+        json.dumps(
+            value,
+            ensure_ascii=True,
+            sort_keys=True,
+        )
+        + "\n"
+    )
+
+
+def _write_backup_header(
+    handle: TextIO,
+    collection,
+) -> None:
+    _write_json_line(
+        handle,
+        {
+            "format": _BACKUP_FORMAT,
+            "version": _BACKUP_VERSION,
+            "collection": _collection_name(collection),
+        },
+    )
+
+
+def _read_backup_header(
+    path: Path,
+) -> dict:
+    try:
+        with path.open(
+            "r",
+            encoding="utf-8",
+        ) as handle:
+            header = json.loads(handle.readline())
+    except OSError as exc:
+        raise ValueError(f"could not read repair backup: {path}") from exc
+    except (
+        json.JSONDecodeError,
+        TypeError,
+    ) as exc:
+        raise ValueError("repair backup has an invalid header") from exc
+
+    if not isinstance(
+        header,
+        dict,
+    ) or (header.get("format") != _BACKUP_FORMAT or header.get("version") != _BACKUP_VERSION):
+        raise ValueError("unsupported repair backup format")
+
+    return header
+
+
+def _iter_backup_records(
+    path: Path,
+) -> Iterator[tuple[str, str]]:
+    _read_backup_header(path)
+
+    with path.open(
+        "r",
+        encoding="utf-8",
+    ) as handle:
+        # Skip the validated header.
+        handle.readline()
+
+        for line_number, line in enumerate(
+            handle,
+            start=2,
+        ):
+            if not line.strip():
+                continue
+
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"invalid backup JSON at line {line_number}") from exc
+
+            drawer_id = record.get("id") if isinstance(record, dict) else None
+            document = record.get("original_document") if isinstance(record, dict) else None
+
+            if not isinstance(
+                drawer_id,
+                str,
+            ) or not isinstance(
+                document,
+                str,
+            ):
+                raise ValueError(f"invalid repair backup record at line {line_number}")
+
+            yield drawer_id, document
+
+
+def repair_collection(
+    collection,
+    *,
+    apply: bool = False,
+    page_size: int = 500,
+    backup_path: Optional[Union[str, Path]] = None,
+    on_change: Optional[Callable[[str, str, str], None]] = None,
+) -> dict:
+    """Scan a collection and optionally repair high-confidence mojibake."""
+    if page_size < 1:
+        raise ValueError("page_size must be at least 1")
+
+    if apply and backup_path is None:
+        raise ValueError("backup_path is required when apply=True")
+
+    backup = Path(backup_path) if backup_path is not None else None
+    backup_handle: Optional[TextIO] = None
+    backup_used: Optional[str] = None
+
+    scanned = 0
+    changed = 0
+    updated = 0
+    offset = 0
+
+    try:
+        while True:
+            page = collection.get(
+                limit=page_size,
+                offset=offset,
+                include=["documents"],
+            )
+            ids = list(
+                _result_field(
+                    page,
+                    "ids",
+                )
+                or []
+            )
+            documents = list(
+                _result_field(
+                    page,
+                    "documents",
+                )
+                or []
+            )
+
+            if len(ids) != len(documents):
+                raise RuntimeError("collection returned misaligned ids and documents")
+
+            if not ids:
+                break
+
+            page_changes = []
+
+            for drawer_id, document in zip(
+                ids,
+                documents,
+            ):
+                scanned += 1
+
+                if not isinstance(
+                    document,
+                    str,
+                ):
+                    continue
+
+                repaired = repair_mojibake(document)
+
+                if repaired == document:
+                    continue
+
+                item = (
+                    str(drawer_id),
+                    document,
+                    repaired,
+                )
+                page_changes.append(item)
+                changed += 1
+
+                if on_change is not None:
+                    on_change(*item)
+
+            if apply and page_changes:
+                if backup_handle is None:
+                    assert backup is not None
+
+                    backup_handle = _open_private_backup(backup)
+                    _write_backup_header(
+                        backup_handle,
+                        collection,
+                    )
+                    backup_used = str(backup)
+
+                for (
+                    drawer_id,
+                    original,
+                    _repaired,
+                ) in page_changes:
+                    _write_json_line(
+                        backup_handle,
+                        {
+                            "id": drawer_id,
+                            "original_document": (original),
+                        },
+                    )
+
+                # Originals must reach durable storage before their
+                # live collection rows are overwritten.
+                backup_handle.flush()
+                os.fsync(backup_handle.fileno())
+
+                collection.update(
+                    ids=[item[0] for item in page_changes],
+                    documents=[item[2] for item in page_changes],
+                )
+                updated += len(page_changes)
+
+            offset += len(ids)
+
+            if len(ids) < page_size:
+                break
+    finally:
+        if backup_handle is not None:
+            backup_handle.close()
+
+    return {
+        "scanned": scanned,
+        "changed": changed,
+        "updated": updated,
+        "backup_path": backup_used,
+    }
+
+
+def restore_collection(
+    collection,
+    backup_path: Union[str, Path],
+    *,
+    batch_size: int = 500,
+) -> dict:
+    """Restore original documents from an encoding-repair backup."""
+    if batch_size < 1:
+        raise ValueError("batch_size must be at least 1")
+
+    path = Path(backup_path)
+    header = _read_backup_header(path)
+
+    backup_collection = header.get("collection")
+    target_collection = _collection_name(collection)
+
+    if backup_collection and target_collection and backup_collection != target_collection:
+        raise ValueError(
+            f"backup belongs to collection {backup_collection!r}, not {target_collection!r}"
+        )
+
+    # Validate the complete file before performing the first restore write.
+    validated = sum(1 for _record in _iter_backup_records(path))
+
+    restored = 0
+    batch_ids = []
+    batch_documents = []
+
+    for (
+        drawer_id,
+        document,
+    ) in _iter_backup_records(path):
+        batch_ids.append(drawer_id)
+        batch_documents.append(document)
+
+        if len(batch_ids) < batch_size:
+            continue
+
+        collection.update(
+            ids=batch_ids,
+            documents=batch_documents,
+        )
+        restored += len(batch_ids)
+        batch_ids = []
+        batch_documents = []
+
+    if batch_ids:
+        collection.update(
+            ids=batch_ids,
+            documents=batch_documents,
+        )
+        restored += len(batch_ids)
+
+    return {
+        "validated": validated,
+        "restored": restored,
+    }

mempalace/entities.py

@@ -0,0 +1,71 @@
+"""No-LLM structural entity extraction for the associative graph.
+
+Pulls deterministic, *structural* tokens from text — author-quoted code spans, URLs,
+file paths, qualified identifiers, and CamelCase symbols — to populate the ``entities``
+drawer-metadata field that hallways/tunnels consume. Structural-only by design: no
+wordlists, no NLP models, no domain vocabulary, so it stays language-neutral and
+predictable, and biases to precision (only tokens that are unambiguously "a thing being
+referred to") over recall.
+
+The output format matches what ``hallways._parse_entities`` expects: a ``;``-joined string.
+"""
+
+import re
+
+# Author-quoted code spans are the highest-signal structural marker: `foo`, `obj.method()`.
+_BACKTICK = re.compile(r"`([^`\n]{2,64})`")
+# URLs.
+_URL = re.compile(r"https?://[^\s)>\]}\"']+")
+# Paths with a separator and a short extension: rag/foo.py, a/b/c.tsx.
+_PATH = re.compile(r"\b[\w.-]+/[\w./-]*\.[A-Za-z][A-Za-z0-9]{0,4}\b")
+# Qualified dotted identifiers; each segment starts with a letter and is >=2 chars, so
+# "1.2.3", "e.g", and "i.e" are excluded: module.func, pkg.Class.method.
+_QUALIFIED = re.compile(r"\b[A-Za-z][A-Za-z0-9_]+(?:\.[A-Za-z][A-Za-z0-9_]+)+\b")
+# CamelCase with >=2 humps — strongly code-specific: ChromaBackend, MemoryStack.
+_CAMEL = re.compile(r"\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b")
+# snake_case (must contain an underscore, so it can't match plain English): do_thing,
+# _extract_authored_at. The optional leading/trailing `_?` matches dunder-style names
+# whose underscore would otherwise fall outside the `\b` boundary (`_` is a word char).
+_SNAKE = re.compile(r"\b_?[a-z][a-z0-9]*(?:_[a-z0-9]+)+_?\b")
+
+_MAX_ENTITIES = 24
+_MIN_LEN = 2
+_MAX_LEN = 64
+
+
+def _clean(token):
+    # `;` is the entities-metadata separator, so it must never survive inside an entity
+    # (e.g. a URL query string or a backtick span) or it would split the field.
+    return token.replace(";", " ").strip().strip("`.,:()[]{}<>\"'").strip()
+
+
+def extract_structural_entities(text, max_entities=_MAX_ENTITIES):
+    """Return up to ``max_entities`` structural entities from ``text``.
+
+    Deterministic and order-stable: entities are ranked by occurrence count (ties broken
+    by first appearance), deduplicated case-insensitively, preserving the first-seen
+    surface form.
+    """
+    if not text:
+        return []
+    counts = {}
+    order = {}
+    seq = 0
+    for pattern in (_BACKTICK, _URL, _PATH, _QUALIFIED, _CAMEL, _SNAKE):
+        for match in pattern.finditer(text):
+            token = _clean(match.group(1) if pattern is _BACKTICK else match.group(0))
+            if not (_MIN_LEN <= len(token) <= _MAX_LEN):
+                continue
+            key = token.lower()
+            if key not in counts:
+                counts[key] = 0
+                order[key] = (seq, token)
+                seq += 1
+            counts[key] += 1
+    ranked = sorted(order, key=lambda k: (-counts[k], order[k][0]))
+    return [order[k][1] for k in ranked[:max_entities]]
+
+
+def entities_metadata(text, max_entities=_MAX_ENTITIES):
+    """``;``-joined entity string for drawer metadata, or ``""`` when none are found."""
+    return ";".join(extract_structural_entities(text, max_entities=max_entities))

mempalace/entity_detector.py

@@ -215,6 +215,8 @@ PROSE_EXTENSIONS = {
     ".md",
     ".rst",
     ".csv",
+    ".tex",
+    ".bib",
 }
 
 READABLE_EXTENSIONS = {
@@ -270,6 +272,34 @@ SKIP_FILENAMES = {
 # ==================== CANDIDATE EXTRACTION ====================
 
 
+# Entity-candidate matching is ReDoS-prone: the English candidate pattern's
+# nested alternation (i18n/en.json) backtracks catastrophically on a long
+# unbroken run of printable ASCII (base64, minified JS, hashes, data URIs),
+# pinning a mine on one ~5000-char window for hours (#2063). Such a run is never
+# a name, so it is collapsed to a space before single-word matching. Scope:
+#   * ASCII-only ([!-~]): non-ASCII scripts (CJK, Cyrillic, Devanagari,
+#     accented Latin) are deliberately left untouched — a CJK paragraph is one
+#     unbroken run with no ASCII whitespace, and those locales' candidate
+#     patterns are bounded and never backtrack, so collapsing them would
+#     silently destroy their entity detection.
+#   * The threshold sits above the 20-char cap of the simple-name pattern
+#     ([A-Z][a-z]{1,19}), so no real single-word name is dropped. A CamelCase
+#     code identifier glued inside a long ASCII run (a path/URL/dotted name) is
+#     dropped as noise; whitespace-delimited proper nouns are unaffected.
+_MAX_CANDIDATE_TOKEN_LEN = 24
+_LONG_ASCII_RUN_RX = re.compile(rf"[!-~]{{{_MAX_CANDIDATE_TOKEN_LEN},}}")
+
+
+def _collapse_long_ascii_runs(text: str) -> str:
+    """Collapse a long unbroken run of printable ASCII to a single space (#2063).
+
+    Applied only before single-word candidate matching — the whitespace-delimited
+    multi-word patterns cannot backtrack on such runs. See ``_LONG_ASCII_RUN_RX``
+    for scope and rationale.
+    """
+    return _LONG_ASCII_RUN_RX.sub(" ", text)
+
+
 def extract_candidates(text: str, languages=("en",)) -> dict:
     """
     Extract all capitalized proper noun candidates from text.
@@ -294,13 +324,15 @@ def extract_candidates(text: str, languages=("en",)) -> dict:
     for compound, n in compound_counts.items():
         counts[compound] += n
 
-    # Single-word candidates — one pre-wrapped pattern per language
+    # Single-word candidates — one pre-wrapped pattern per language.
+    # Collapse long ASCII blobs first (base64/minified) to defuse ReDoS (#2063).
+    candidate_text = _collapse_long_ascii_runs(working_text)
     for wrapped_pat in patterns["candidate_patterns"]:
         try:
             rx = re.compile(wrapped_pat)
         except re.error:
             continue
-        for word in rx.findall(working_text):
+        for word in rx.findall(candidate_text):
             wl = word.lower()
             if wl in stopwords:
                 continue
@@ -716,7 +748,7 @@ def _print_entity_list(entities: list, label: str):
         print("    (none detected)")
         return
     for i, e in enumerate(entities):
-        confidence_bar = "●" * int(e["confidence"] * 5) + "○" * (5 - int(e["confidence"] * 5))
+        confidence_bar = "#" * int(e["confidence"] * 5) + "." * (5 - int(e["confidence"] * 5))
         signals_str = ", ".join(e["signals"][:2]) if e["signals"] else ""
         print(f"    {i + 1:2}. {e['name']:20} [{confidence_bar}] {signals_str}")
 
@@ -736,7 +768,7 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     Pass yes=True to auto-accept all detected entities without prompting.
     """
     print(f"\n{'=' * 58}")
-    print("  MemPalace — Entity Detection")
+    print("  MemPalace -- Entity Detection")
     print(f"{'=' * 58}")
     print("\n  Scanned your files. Here's what we found:\n")
 
@@ -766,7 +798,7 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
             "topics": confirmed_topics,
         }
 
-    print(f"\n{'─' * 58}")
+    print(f"\n{'-' * 58}")
     print("  Options:")
     print("    [enter]  Accept all")
     print("    [edit]   Remove wrong entries or reclassify uncertain")
@@ -781,9 +813,9 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     if choice == "edit":
         # Handle uncertain first
         if detected["uncertain"]:
-            print("\n  Uncertain entities — classify each:")
+            print("\n  Uncertain entities -- classify each:")
             for e in detected["uncertain"]:
-                ans = input(f"    {e['name']} — (p)erson, (r)project, or (s)kip? ").strip().lower()
+                ans = input(f"    {e['name']} -- (p)erson, (r)project, or (s)kip? ").strip().lower()
                 if ans == "p":
                     confirmed_people.append(e["name"])
                 elif ans == "r":

mempalace/exporter.py

@@ -84,7 +84,7 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -
     total = col.count()
 
     if total == 0:
-        print("  Palace is empty — nothing to export.")
+        print("  Palace is empty -- nothing to export.")
         return {"wings": 0, "rooms": 0, "drawers": 0}
 
     _reject_symlink(output_dir, "output_dir")

mempalace/format_miner.py

@@ -362,7 +362,7 @@ def extract_text(
         logger.info("skip:unreadable (file gone after scan) %s", p)
         return None, ExtractionStatus.SKIP_UNREADABLE
     except OSError as exc:
-        logger.info("skip:unreadable %s — %s", p, exc)
+        logger.info("skip:unreadable %s -- %s", p, exc)
         return None, ExtractionStatus.SKIP_UNREADABLE
 
     # Fringe Case 5 — empty file. Skip silently.
@@ -433,9 +433,9 @@ def extract_text(
         # Fringe Case 4 vs Case 10: encrypted vs generic crash, by message.
         msg = str(exc)
         if _ENCRYPTED_PATTERNS.search(msg):
-            logger.info("skip:encrypted %s — %s", p, msg[:120])
+            logger.info("skip:encrypted %s -- %s", p, msg[:120])
             return None, ExtractionStatus.SKIP_ENCRYPTED
-        logger.warning("skip:extraction_error %s — %s: %s", p, type(exc).__name__, msg[:200])
+        logger.warning("skip:extraction_error %s -- %s: %s", p, type(exc).__name__, msg[:200])
         return None, ExtractionStatus.SKIP_EXTRACTION_ERROR
 
     # Either transformer can legitimately return None / empty (malformed
@@ -443,7 +443,7 @@ def extract_text(
     # so the caller knows to skip rather than file an empty drawer.
     if not text:
         transformer = "striprtf" if is_rtf else "markitdown"
-        logger.info("skip:extraction_error %s — %s returned None/empty", p, transformer)
+        logger.info("skip:extraction_error %s -- %s returned None/empty", p, transformer)
         return None, ExtractionStatus.SKIP_EXTRACTION_ERROR
 
     return text, ExtractionStatus.OK
@@ -750,7 +750,7 @@ def mine_formats(
     # min_chunk_size) are now threaded through chunk_text below, so users
     # who customized their config see the effect in format-mode mining.
     # Per PR #1555 review (Gemini #3).
-    palace_config = MempalaceConfig()
+    palace_config = MempalaceConfig(palace_path=palace_path)
 
     format_path = Path(format_dir).expanduser().resolve()
     if not wing:
@@ -803,7 +803,7 @@ def mine_formats(
         files = scan_formats(format_path)
 
         print(f"\n{'=' * 55}")
-        print("  MemPalace Mine — Format extraction")
+        print("  MemPalace Mine -- Format extraction")
         print(f"{'=' * 55}")
         print(f"  Wing:    {wing}")
         print(f"  Source:  {format_path}")
@@ -811,7 +811,7 @@ def mine_formats(
         print(f"  Files:   {len(files)}{limit_suffix}")
         print(f"  Palace:  {palace_path}")
         if dry_run:
-            print("  DRY RUN — nothing will be filed")
+            print("  DRY RUN -- nothing will be filed")
         print(f"{'-' * 55}\n")
 
         collection = get_collection(palace_path) if not dry_run else None
@@ -876,7 +876,7 @@ def mine_formats(
                 files_with_text += 1
 
                 if dry_run:
-                    print(f"    [DRY RUN] {filepath.name} → {len(chunks)} drawers")
+                    print(f"    [DRY RUN] {filepath.name} -> {len(chunks)} drawers")
                     total_drawers += len(chunks)
                     files_mined += 1
                     if limit > 0 and files_mined >= limit:
@@ -945,7 +945,7 @@ def mine_formats(
         # skipped quietly.
         if not dry_run:
             try:
-                tunnels_added = _compute_topic_tunnels_for_wing(wing)
+                tunnels_added = _compute_topic_tunnels_for_wing(wing, config=palace_config)
                 if tunnels_added:
                     print(f"\n  Topic tunnels: +{tunnels_added} cross-wing link(s)")
             except Exception as exc:

mempalace/general_extractor.py

@@ -159,7 +159,6 @@ EMOTION_MARKERS = [
     r"i need",
     r"never told anyone",
     r"nobody knows",
-    r"\*[^*]+\*",
 ]
 
 ALL_MARKERS = {

mempalace/hallways.py

@@ -199,6 +199,7 @@ def compute_hallways_for_wing(
     wing: str,
     col=None,
     min_count: int = 2,
+    config=None,
 ) -> list[dict]:
     """Compute entity-pair hallways for one wing.
 
@@ -229,6 +230,10 @@ def compute_hallways_for_wing(
             hallway between two entities. Default 2 — single co-occurrences
             are noise (entities mentioned together once in one drawer);
             two or more is a real signal. Clamped to ``>=1``.
+        config: Optional ``MempalaceConfig`` selecting the palace-scoped
+            hallway sidecar. Callers using an explicit palace path must pass
+            the matching config so derived graph state cannot leak into the
+            default palace.
 
     Returns:
         List of hallway dicts created for this wing. Records for other
@@ -310,7 +315,7 @@ def compute_hallways_for_wing(
     #    across recomputes. Without this preservation, every mine wipes
     #    the connection weights accumulated through use — defeating the
     #    living-connection layer entirely.
-    existing = _load_hallways()
+    existing = _load_hallways(config)
     existing_dynamics_lookup: dict = {}
     for h in existing:
         if h.get("wing") != wing:
@@ -360,7 +365,7 @@ def compute_hallways_for_wing(
 
     # 4. Persist — preserve other-wing records, replace this wing's records.
     preserved_other_wings = [h for h in existing if h.get("wing") != wing]
-    _save_hallways(preserved_other_wings + created)
+    _save_hallways(preserved_other_wings + created, config)
 
     return created
 
@@ -370,19 +375,19 @@ def compute_hallways_for_wing(
 # ─────────────────────────────────────────────────────────────────────────────
 
 
-def list_hallways(wing: Optional[str] = None) -> list[dict]:
+def list_hallways(wing: Optional[str] = None, config=None) -> list[dict]:
     """List hallway records. Filter by ``wing`` if specified."""
-    all_hallways = _load_hallways()
+    all_hallways = _load_hallways(config)
     if wing is None:
         return list(all_hallways)
     return [h for h in all_hallways if h.get("wing") == wing]
 
 
-def delete_hallway(hallway_id: str) -> bool:
+def delete_hallway(hallway_id: str, config=None) -> bool:
     """Remove one hallway record by id. Returns True if a record was removed."""
-    hallways = _load_hallways()
+    hallways = _load_hallways(config)
     filtered = [h for h in hallways if h.get("id") != hallway_id]
     if len(filtered) == len(hallways):
         return False
-    _save_hallways(filtered)
+    _save_hallways(filtered, config)
     return True

mempalace/hlc.py

@@ -0,0 +1,94 @@
+"""
+hlc.py — Hybrid Logical Clock for RFC 004 op ordering
+
+Total order across replicas without trusting wall clocks: physical
+milliseconds + a logical counter + the replica id as final tiebreak.
+Rendered as a fixed-width, lexicographically sortable string so SQLite TEXT
+comparison IS the causal comparison:
+
+    <unix_ms:13 decimal digits>-<counter:6 hex digits>-<replica_id>
+
+e.g. ``1783038849123-000000-rep_ab12cd34ef56``.
+
+Semantics (standard HLC):
+- ``tick()`` for a local op: physical time if it advanced, else previous
+  ms with counter+1 — never goes backwards, even against clock regression.
+- ``observe(remote)`` on receiving a remote op: the clock absorbs the
+  remote's instant so later local ops sort after everything already seen.
+
+Cursor semantics stay LOCAL (rowid arrival order) per RFC 004 A.4 — a tail
+consumer must see late-arriving remote ops even though their HLC is older.
+HLC is the *display/merge* order; arrival is the *delivery* order.
+"""
+
+import re
+import threading
+import time
+
+_HLC_RE = re.compile(r"^(\d{13})-([0-9a-f]{6})-(.+)$")
+
+MAX_COUNTER = 0xFFFFFF
+MAX_FUTURE_DRIFT_MS = 60_000
+
+
+def parse(hlc: str):
+    """Return (ms, counter, replica_id) or raise ValueError."""
+    m = _HLC_RE.match(hlc)
+    if not m:
+        raise ValueError(f"not a valid HLC string: {hlc!r}")
+    return int(m.group(1)), int(m.group(2), 16), m.group(3)
+
+
+def render(ms: int, counter: int, replica_id: str) -> str:
+    return f"{ms:013d}-{counter:06x}-{replica_id}"
+
+
+class HybridLogicalClock:
+    """Thread-safe HLC bound to one replica id.
+
+    ``last`` may be seeded from persisted state (the newest hlc in the
+    op-log) so monotonicity survives process restarts.
+    """
+
+    def __init__(self, replica_id: str, last: str = None, now_ms=None):
+        self.replica_id = replica_id
+        self._now_ms = now_ms or (lambda: int(time.time() * 1000))
+        self._lock = threading.Lock()
+        if last:
+            ms, counter, _ = parse(last)
+            if self._is_within_future_drift(ms):
+                self._ms, self._counter = ms, counter
+            else:
+                self._ms, self._counter = 0, 0
+        else:
+            self._ms, self._counter = 0, 0
+
+    def _is_within_future_drift(self, ms: int) -> bool:
+        return ms <= self._now_ms() + MAX_FUTURE_DRIFT_MS
+
+    def tick(self) -> str:
+        """Stamp a new local op; strictly greater than everything seen."""
+        with self._lock:
+            now = self._now_ms()
+            if now > self._ms:
+                self._ms, self._counter = now, 0
+            elif self._counter < MAX_COUNTER:
+                self._counter += 1
+            else:
+                # Counter exhaustion within one ms is pathological; step
+                # the physical component rather than wrap and reorder.
+                self._ms += 1
+                self._counter = 0
+            return render(self._ms, self._counter, self.replica_id)
+
+    def observe(self, remote_hlc: str) -> None:
+        """Absorb a remote op's instant so future ticks sort after it."""
+        try:
+            remote_ms, remote_counter, _ = parse(remote_hlc)
+        except ValueError:
+            return  # never let a malformed remote stamp wedge the clock
+        if not self._is_within_future_drift(remote_ms):
+            return
+        with self._lock:
+            if remote_ms > self._ms or (remote_ms == self._ms and remote_counter > self._counter):
+                self._ms, self._counter = remote_ms, remote_counter

mempalace/hooks_cli.py

@@ -7,6 +7,9 @@ Supported harnesses: claude-code, codex (extensible to cursor, gemini, etc.)
 """
 
 import hashlib
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
 import json
 import os
 import re
@@ -18,6 +21,13 @@ from pathlib import Path
 from typing import Optional
 
 from mempalace.config import MempalaceConfig
+from mempalace.write_routing import (
+    ResolvedWriteRoutingPolicy,
+    WriteRoutingDecision,
+    WriteRoutingError,
+    WriteRoutingPolicy,
+    choose_write_route,
+)
 
 SAVE_INTERVAL = 15
 STATE_DIR = Path.home() / ".mempalace" / "hook_state"
@@ -510,6 +520,11 @@ def _spawn_mine(cmd: list) -> None:
 
 
 def _hooks_daemon_enabled() -> bool:
+    """Legacy compatibility helper for the pre-policy hook setting.
+
+    New hook write paths use ``resolve_write_routing("hooks")``. This
+    helper remains for callers/tests that still inspect ``hooks.daemon``.
+    """
     try:
         return MempalaceConfig().hook_use_daemon is True
     except Exception:
@@ -527,11 +542,9 @@ def _daemon_mine_dedupe_key(source: str, mode: str) -> str:
 def _daemon_available() -> bool:
     """True iff a daemon is already running for the configured palace.
 
-    This is a fast localhost health check, not a spawn: the 500ms hook budget
-    forbids auto-starting a python subprocess from a hook (cold start is
-    ~15s). Daemon mode for hooks requires the user to have started the daemon
-    explicitly via `mempalace daemon start`; when it isn't up, hooks fall back
-    to the existing direct (in-process / spawn) path instead of blocking.
+    This is a fast localhost health check, not a spawn: the hook time budget
+    forbids cold-starting a long-lived daemon. ``prefer`` may fall back to the
+    direct path when this returns false; ``require`` must block the write.
     """
     from .daemon import HOOK_PROBE_TIMEOUT, get_client_if_running
 
@@ -544,6 +557,143 @@ def _daemon_available() -> bool:
         return False
 
 
+@dataclass(frozen=True)
+class HookWriteRouting:
+    """One hook invocation's resolved routing state."""
+
+    decision: Optional[WriteRoutingDecision]
+    source: str
+    error: Optional[str] = None
+
+    @property
+    def use_daemon(self) -> bool:
+        return self.decision is not None and self.decision.use_daemon
+
+    @property
+    def blocked(self) -> bool:
+        return self.error is not None or (self.decision is not None and self.decision.blocked)
+
+    @property
+    def notice(self) -> str:
+        if self.error is not None:
+            return (
+                "MemPalace hook writes were skipped because write-routing "
+                f"configuration is invalid: {self.error}. No direct ChromaDB "
+                "fallback was attempted."
+            )
+        if self.blocked:
+            return (
+                "MemPalace hook writes were skipped because routing is set to "
+                "'require' but the local daemon is unavailable. Start it with "
+                "`mempalace daemon start`; no direct ChromaDB fallback was attempted."
+            )
+        return ""
+
+
+_HOOK_WRITE_ROUTING_CONTEXT = ContextVar(
+    "mempalace_hook_write_routing",
+    default=None,
+)
+
+
+def _resolve_configured_hook_policy() -> ResolvedWriteRoutingPolicy:
+    """Resolve the new policy, with legacy-object compatibility."""
+
+    config = MempalaceConfig()
+    resolver = getattr(config, "resolve_write_routing", None)
+    if callable(resolver):
+        resolved = resolver("hooks")
+        if isinstance(resolved, ResolvedWriteRoutingPolicy):
+            return resolved
+
+    # Compatibility for older/custom config objects and existing tests that
+    # expose only the pre-policy ``hook_use_daemon`` property.
+    policy = (
+        WriteRoutingPolicy.PREFER
+        if getattr(config, "hook_use_daemon", False) is True
+        else WriteRoutingPolicy.DIRECT
+    )
+    return ResolvedWriteRoutingPolicy(
+        policy=policy,
+        source="legacy hook_use_daemon",
+    )
+
+
+def _compute_hook_write_routing() -> HookWriteRouting:
+    """Resolve hook policy and probe daemon liveness at most once."""
+
+    try:
+        resolved = _resolve_configured_hook_policy()
+    except WriteRoutingError as exc:
+        routing = HookWriteRouting(
+            decision=None,
+            source="configuration-error",
+            error=str(exc),
+        )
+        _log(routing.notice)
+        return routing
+    except Exception as exc:
+        # Preserve the historical save-on-config-read-failure behavior. An
+        # explicitly invalid routing value raises WriteRoutingError above and
+        # fails closed; an unrelated config I/O/runtime failure falls back to
+        # direct so a final checkpoint is not silently lost.
+        _log(f"WARNING: could not resolve hook write routing: {exc}; defaulting to direct")
+        resolved = ResolvedWriteRoutingPolicy(
+            policy=WriteRoutingPolicy.DIRECT,
+            source="config-unavailable fallback",
+        )
+
+    daemon_available = False
+    if resolved.policy is not WriteRoutingPolicy.DIRECT:
+        daemon_available = _daemon_available()
+
+    decision = choose_write_route(
+        resolved.policy,
+        daemon_available=daemon_available,
+        daemon_can_start=False,
+    )
+    routing = HookWriteRouting(
+        decision=decision,
+        source=resolved.source,
+    )
+
+    if decision.policy is not WriteRoutingPolicy.DIRECT:
+        _log(
+            "Hook write routing: "
+            f"policy={decision.policy.value} source={resolved.source} "
+            f"target={decision.target.value} reason={decision.reason}"
+        )
+
+    return routing
+
+
+def _current_hook_write_routing() -> HookWriteRouting:
+    routing = _HOOK_WRITE_ROUTING_CONTEXT.get()
+    if routing is not None:
+        return routing
+    return _compute_hook_write_routing()
+
+
+@contextmanager
+def _hook_write_routing_context():
+    """Share one policy resolution and one daemon probe across a hook fire."""
+
+    routing = _compute_hook_write_routing()
+    token = _HOOK_WRITE_ROUTING_CONTEXT.set(routing)
+    try:
+        yield routing
+    finally:
+        _HOOK_WRITE_ROUTING_CONTEXT.reset(token)
+
+
+def _log_hook_write_blocked(routing: HookWriteRouting, operation: str) -> None:
+    _log(f"{routing.notice} Operation skipped: {operation}.")
+
+
+def _blocked_hook_output(routing: HookWriteRouting) -> dict:
+    return {"systemMessage": routing.notice}
+
+
 def _submit_daemon_job(
     kind: str,
     payload: dict,
@@ -573,9 +723,33 @@ def _submit_daemon_job(
         wait=wait,
         auto_start=False,
         timeout=timeout,
+        # A job refused the palace lock is deferred, not failed (#2014), so it
+        # is never terminal while the holder lives. The waiting callers below
+        # wait on purpose, but a parked job cannot reach the state they wait
+        # for: they would burn the whole timeout and then report a failure that
+        # did not happen. Take the parked job back instead; the daemon still
+        # runs it once the lock frees.
+        stop_on_lock_deferral=True,
     )
 
 
+def _job_deferred_by_lock(job: dict) -> bool:
+    """True when the daemon parked this job behind the palace write lock.
+
+    Imported lazily like ``submit_job`` above: only callers that actually reach
+    the daemon pay for the module, and by this point it is already loaded.
+    """
+    from .daemon import job_deferred_by_lock
+
+    return job_deferred_by_lock(job)
+
+
+def _lock_deferral_reason(job: dict) -> str:
+    """Operator-facing reason a job is parked, for the hook log."""
+    reason = (job.get("error") or {}).get("message") or "the palace write lock is held"
+    return f"{reason} (job {job.get('id')} stays queued and runs when the holder exits)"
+
+
 def _maybe_auto_ingest():
     """Background-mine MEMPAL_DIR (project files) if set.
 
@@ -592,9 +766,15 @@ def _maybe_auto_ingest():
     targets = _get_mine_targets()
     if not targets:
         return
+
+    routing = _current_hook_write_routing()
+    if routing.blocked:
+        _log_hook_write_blocked(routing, "project auto-ingest")
+        return
+
     for mine_dir, mode in targets:
         try:
-            if _hooks_daemon_enabled() and _daemon_available():
+            if routing.use_daemon:
                 try:
                     _submit_daemon_job(
                         "mine",
@@ -626,11 +806,17 @@ def _mine_sync():
     targets = _get_mine_targets()
     if not targets:
         return
+
+    routing = _current_hook_write_routing()
+    if routing.blocked:
+        _log_hook_write_blocked(routing, "synchronous project mine")
+        return
+
     STATE_DIR.mkdir(parents=True, exist_ok=True)
     log_path = STATE_DIR / "hook.log"
     for mine_dir, mode in targets:
         try:
-            if _hooks_daemon_enabled() and _daemon_available():
+            if routing.use_daemon:
                 try:
                     job = _submit_daemon_job(
                         "mine",
@@ -640,7 +826,12 @@ def _mine_sync():
                         timeout=60,
                     )
                     result = job.get("result") or {}
-                    if job.get("state") != "succeeded" or not result.get("success", True):
+                    if _job_deferred_by_lock(job):
+                        # Parked behind the palace lock, not failed: the daemon
+                        # runs it once the holder exits. Saying "failed" here
+                        # would be the false report #2014 is about.
+                        _log(f"Daemon sync mine deferred: {_lock_deferral_reason(job)}")
+                    elif job.get("state") != "succeeded" or not result.get("success", True):
                         _log(f"Daemon sync mine failed: {result.get('error', job.get('error'))}")
                 except Exception as exc:
                     # Daemon accepted context — don't fall back (would double-mine).
@@ -660,6 +851,11 @@ def _mine_sync():
                     stdout=log_f,
                     stderr=log_f,
                     timeout=60,
+                    # Windows: hide the conhost window this sync mine would
+                    # otherwise flash on every fire. Mirrors the async paths
+                    # (_spawn_mine / _desktop_toast) via _detached_popen_kwargs().
+                    # 0 is a no-op off-Windows.
+                    creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
                 )
         except (OSError, subprocess.TimeoutExpired):
             pass
@@ -768,12 +964,24 @@ def _save_diary_direct(
     the agent wrote to, so project-derived wings stay discoverable.
 
     Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure.
+    A daemon lock deferral also returns {"count": 0}: nothing is filed yet, but
+    the entry is queued and the daemon files it once the holder exits, so the
+    checkpoint marker is deliberately not advanced.
     """
     messages = _extract_recent_messages(transcript_path)
     if not messages:
         _log("No recent messages to save")
         return {"count": 0}
 
+    routing = _current_hook_write_routing()
+    if routing.blocked:
+        _log_hook_write_blocked(routing, "diary checkpoint")
+        return {
+            "count": 0,
+            "routing_blocked": True,
+            "routing_message": routing.notice,
+        }
+
     themes = _extract_themes(messages)
 
     # Build a compressed diary entry from recent conversation
@@ -785,7 +993,7 @@ def _save_diary_direct(
     )
 
     try:
-        if _hooks_daemon_enabled() and _daemon_available():
+        if routing.use_daemon:
             try:
                 job = _submit_daemon_job(
                     "diary_write",
@@ -817,6 +1025,12 @@ def _save_diary_direct(
                 if toast:
                     _desktop_toast(f"Checkpoint saved - {len(messages)} messages archived")
                 return {"count": len(messages), "themes": themes}
+            if _job_deferred_by_lock(job):
+                # Queued behind the palace lock: the entry is held and the daemon
+                # files it once the holder exits. Not a failure, and not a reason
+                # to re-file it here -- that would duplicate verbatim content.
+                _log(f"Daemon diary checkpoint deferred: {_lock_deferral_reason(job)}")
+                return {"count": 0}
             _log(f"Daemon diary checkpoint failed: {result.get('error', job.get('error'))}")
             return {"count": 0}
 
@@ -851,8 +1065,13 @@ def _save_diary_direct(
 
 def _ingest_transcript(transcript_path: str):
     """Mine a Claude Code session transcript into the palace as a conversation."""
-    path = Path(transcript_path).expanduser()
-    if not path.is_file() or path.stat().st_size < 100:
+    path = _validate_transcript_path(transcript_path)
+    if path is None:
+        return
+    try:
+        if not path.is_file() or path.stat().st_size < 100:
+            return
+    except OSError:
         return
 
     try:
@@ -860,18 +1079,23 @@ def _ingest_transcript(transcript_path: str):
     except Exception:
         return
 
+    routing = _current_hook_write_routing()
+    if routing.blocked:
+        _log_hook_write_blocked(routing, "transcript ingest")
+        return
+
     try:
-        if _hooks_daemon_enabled() and _daemon_available():
+        if routing.use_daemon:
             try:
                 _submit_daemon_job(
                     "mine",
                     {
-                        "source": str(path.parent),
+                        "source": str(path),
                         "mode": "convos",
                         "wing": "sessions",
                         "agent": "mempalace",
                     },
-                    dedupe_key=_daemon_mine_dedupe_key(str(path.parent), "convos"),
+                    dedupe_key=_daemon_mine_dedupe_key(str(path), "convos"),
                     wait=False,
                 )
                 _log(f"Transcript ingest submitted to daemon: {path.name}")
@@ -889,7 +1113,7 @@ def _ingest_transcript(transcript_path: str):
                 "-m",
                 "mempalace",
                 "mine",
-                str(path.parent),
+                str(path),
                 "--mode",
                 "convos",
                 "--wing",
@@ -951,6 +1175,26 @@ _ENCODED_PARENT_PREFIXES = (
 )
 
 
+def _safe_wing_slug(name: str) -> str:
+    """Normalize a project directory name into a wing slug ``sanitize_name`` accepts.
+
+    Builds on the historical space/hyphen handling: map characters outside
+    ``sanitize_name``'s set to ``_`` while keeping ``.`` and ``'`` so existing wings
+    for names like ``my.app`` are preserved (renaming a wing would orphan diary
+    entries already filed under it), collapse ``..`` which the validator rejects as
+    path traversal, trim edge separators it won't accept, and cap the length so
+    ``wing_<slug>`` stays within ``sanitize_name``'s 128-character limit. Without
+    this a folder containing e.g. a leading ``+`` produced ``wing_+project``, which
+    ``sanitize_name`` rejects — silently breaking diary auto-save. Falls back to
+    ``sessions`` when a name reduces to nothing (e.g. ``+``).
+    """
+    slug = name.lower().replace(" ", "_").replace("-", "_")
+    slug = re.sub(r"[^\w.']+", "_", slug)
+    slug = re.sub(r"\.{2,}", ".", slug)
+    slug = slug[:120].strip("_.'")
+    return slug or "sessions"
+
+
 def _wing_from_jsonl_cwd(transcript_path: str) -> Optional[str]:
     """Read ``cwd`` from the first JSONL line that records it.
 
@@ -982,10 +1226,15 @@ def _wing_from_jsonl_cwd(transcript_path: str) -> Optional[str]:
                 cwd_norm = cwd.replace("\\", "/").rstrip("/")
                 if not cwd_norm:
                     continue
+                # A cwd inside "<project>/.claude/worktrees/<wt>" (a git
+                # worktree) belongs to <project>, not the ephemeral worktree
+                # directory -- otherwise every worktree spawns its own wing.
+                _wt_marker = "/.claude/worktrees/"
+                if _wt_marker in cwd_norm:
+                    cwd_norm = cwd_norm.split(_wt_marker, 1)[0]
                 project = cwd_norm.rsplit("/", 1)[-1]
                 if project:
-                    slug = project.lower().replace(" ", "_").replace("-", "_")
-                    return f"wing_{slug}"
+                    return f"wing_{_safe_wing_slug(project)}"
     except OSError:
         pass
     return None
@@ -1042,15 +1291,12 @@ def _wing_from_transcript_path(transcript_path: str) -> str:
             if encoded.startswith(prefix):
                 encoded = encoded[len(prefix) :]
                 break
-        project = encoded.lower().replace(" ", "_").replace("-", "_")
-        if project:
-            return f"wing_{project}"
+        return f"wing_{_safe_wing_slug(encoded)}"
 
     # 3. Legacy — explicit -Projects-<name> segment
     match = re.search(r"-Projects-([^/]+?)(?:/|$)", normalized)
     if match:
-        project = match.group(1).lower().replace(" ", "_").replace("-", "_")
-        return f"wing_{project}"
+        return f"wing_{_safe_wing_slug(match.group(1))}"
 
     # 4. Default
     return "wing_sessions"
@@ -1106,64 +1352,70 @@ def hook_stop(data: dict, harness: str):
     _log(f"Session {session_id}: {exchange_count} exchanges, {since_last} since last save")
 
     if since_last >= SAVE_INTERVAL and exchange_count > 0:
-        _log(f"TRIGGERING SAVE at exchange {exchange_count}")
+        with _hook_write_routing_context() as routing:
+            if routing.blocked:
+                _log_hook_write_blocked(routing, "stop-hook checkpoint")
+                _output(_blocked_hook_output(routing))
+                return
 
-        # Read hook settings from config
-        try:
-            config = MempalaceConfig()
-            silent = config.hook_silent_save
-            toast = config.hook_desktop_toast
-        except Exception:
-            silent = True
-            toast = False
+            _log(f"TRIGGERING SAVE at exchange {exchange_count}")
 
-        project_wing = _wing_from_transcript_path(transcript_path)
+            # Read hook settings from config
+            try:
+                config = MempalaceConfig()
+                silent = config.hook_silent_save
+                toast = config.hook_desktop_toast
+            except Exception:
+                silent = True
+                toast = False
 
-        if silent:
-            # Save directly via Python API — systemMessage renders in terminal
-            result = {"count": 0}
-            if transcript_path:
-                result = _save_diary_direct(
-                    transcript_path,
-                    session_id,
-                    wing=project_wing,
-                    toast=toast,
-                    agent_name=_diary_agent_for_harness(harness),
-                )
-                _ingest_transcript(transcript_path)
-            _maybe_auto_ingest()
-            # Only advance save marker after successful save
-            count = result.get("count", 0)
-            if count > 0:
+            project_wing = _wing_from_transcript_path(transcript_path)
+
+            if silent:
+                # Save directly via Python API — systemMessage renders in terminal
+                result = {"count": 0}
+                if transcript_path:
+                    result = _save_diary_direct(
+                        transcript_path,
+                        session_id,
+                        wing=project_wing,
+                        toast=toast,
+                        agent_name=_diary_agent_for_harness(harness),
+                    )
+                    _ingest_transcript(transcript_path)
+                _maybe_auto_ingest()
+                # Only advance save marker after successful save
+                count = result.get("count", 0)
+                if count > 0:
+                    try:
+                        last_save_file.write_text(str(exchange_count), encoding="utf-8")
+                    except OSError:
+                        pass
+                    themes = result.get("themes", [])
+                    if themes:
+                        tag = " \u2014 " + ", ".join(themes)
+                    else:
+                        tag = ""
+                    _output(
+                        {
+                            "systemMessage": f"\u2726 {count} memories woven into the palace{tag}",
+                        }
+                    )
+                else:
+                    _output({})
+            else:
+                # Legacy: block and ask Claude to save via MCP tools.
+                # Marker advances before confirmed save — best-effort; if Claude
+                # fails to save, the checkpoint is lost but won't retry endlessly.
                 try:
                     last_save_file.write_text(str(exchange_count), encoding="utf-8")
                 except OSError:
                     pass
-                themes = result.get("themes", [])
-                if themes:
-                    tag = " \u2014 " + ", ".join(themes)
-                else:
-                    tag = ""
-                _output(
-                    {
-                        "systemMessage": f"\u2726 {count} memories woven into the palace{tag}",
-                    }
-                )
-            else:
-                _output({})
-        else:
-            # Legacy: block and ask Claude to save via MCP tools.
-            # Marker advances before confirmed save — best-effort; if Claude
-            # fails to save, the checkpoint is lost but won't retry endlessly.
-            try:
-                last_save_file.write_text(str(exchange_count), encoding="utf-8")
-            except OSError:
-                pass
-            if transcript_path:
-                _ingest_transcript(transcript_path)
-            _maybe_auto_ingest()
-            reason = STOP_BLOCK_REASON + f" Write diary entry to wing={project_wing}."
-            _output({"decision": "block", "reason": reason})
+                if transcript_path:
+                    _ingest_transcript(transcript_path)
+                _maybe_auto_ingest()
+                reason = STOP_BLOCK_REASON + f" Write diary entry to wing={project_wing}."
+                _output({"decision": "block", "reason": reason})
     else:
         _output({})
 
@@ -1181,6 +1433,14 @@ def hook_session_start(data: dict, harness: str):
     # Initialize session state directory
     STATE_DIR.mkdir(parents=True, exist_ok=True)
 
+    # Surface a required-daemon problem at session start instead of waiting
+    # until the first save is due. Hooks still never cold-start the daemon.
+    with _hook_write_routing_context() as routing:
+        if routing.blocked:
+            _log_hook_write_blocked(routing, "session-start readiness check")
+            _output(_blocked_hook_output(routing))
+            return
+
     # Pass through — no blocking on session start
     _output({})
 
@@ -1279,16 +1539,22 @@ def hook_session_end(data: dict, harness: str):
         # short-circuit + upsert). ``reason`` is intentionally not branched on:
         # every clean-exit reason (incl. ``/clear`` / ``resume``) warrants the
         # flush. Order matches ``hook_stop``.
-        if valid_transcript:
-            _save_diary_direct(
-                valid_transcript,
-                session_id,
-                wing=_wing_from_transcript_path(valid_transcript),
-                toast=toast,
-                agent_name=_diary_agent_for_harness(harness),
-            )
-            _ingest_transcript(valid_transcript)
-        _maybe_auto_ingest()
+        with _hook_write_routing_context() as routing:
+            if routing.blocked:
+                _log_hook_write_blocked(routing, "session-end flush")
+                _output(_blocked_hook_output(routing))
+                return
+
+            if valid_transcript:
+                _save_diary_direct(
+                    valid_transcript,
+                    session_id,
+                    wing=_wing_from_transcript_path(valid_transcript),
+                    toast=toast,
+                    agent_name=_diary_agent_for_harness(harness),
+                )
+                _ingest_transcript(valid_transcript)
+            _maybe_auto_ingest()
 
         _output({})
     finally:
@@ -1315,14 +1581,20 @@ def hook_precompact(data: dict, harness: str):
 
     _log(f"PRE-COMPACT triggered for session {session_id}")
 
-    # Capture tool output via our normalize path before compaction loses it
-    if transcript_path:
-        _ingest_transcript(transcript_path)
+    with _hook_write_routing_context() as routing:
+        if routing.blocked:
+            _log_hook_write_blocked(routing, "precompact flush")
+            _output(_blocked_hook_output(routing))
+            return
 
-    # Mine MEMPAL_DIR synchronously so project data lands before
-    # compaction proceeds. Transcript convos were already kicked off
-    # above via _ingest_transcript.
-    _mine_sync()
+        # Capture tool output via our normalize path before compaction loses it
+        if transcript_path:
+            _ingest_transcript(transcript_path)
+
+        # Mine MEMPAL_DIR synchronously so project data lands before
+        # compaction proceeds. Transcript convos were already kicked off
+        # above via _ingest_transcript.
+        _mine_sync()
 
     _output({})
 

mempalace/i18n/__init__.py

@@ -97,6 +97,35 @@ def get_regex() -> dict:
     return _strings.get("regex", {})
 
 
+def get_stopwords(lang: Optional[str] = None) -> set:
+    """Return BM25 stop words for ``lang`` as a lowercase set.
+
+    When ``lang`` is given, loads that locale's regex section directly and
+    does not touch global state. When omitted, reads from the currently
+    loaded language. Returns an empty set if the locale is unknown or its
+    ``regex.stop_words`` field is missing.
+
+    ``regex.stop_words`` is stored as a space-separated string in each
+    locale JSON so non-ASCII particles (e.g. Japanese ``は/が/を``) stay
+    authorable without JSON-array escaping overhead.
+    """
+    if lang is not None:
+        canonical = _canonical_lang(lang)
+        if canonical is None:
+            return set()
+        lang_file = _LANG_DIR / f"{canonical}.json"
+        try:
+            data = json.loads(lang_file.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            return set()
+        sw_str = data.get("regex", {}).get("stop_words", "")
+    else:
+        sw_str = get_regex().get("stop_words", "")
+    if not isinstance(sw_str, str):
+        return set()
+    return {w.lower() for w in sw_str.split() if w}
+
+
 def _load_entity_section(lang: str) -> dict:
     """Load the raw entity section for one language. Returns {} if missing."""
     canonical = _canonical_lang(lang)

mempalace/i18n/ja.json

@@ -37,7 +37,7 @@
   },
   "regex": {
     "topic_pattern": "[\\u30A0-\\u30FF]{3,}|[\\u4E00-\\u9FFF]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
-    "stop_words": "は が を に で と も の へ から まで より した します している されて です ます ました こと もの ため それ これ その この あの ない なく ある いる する",
+    "stop_words": "から まで より した します している されて です ます ました こと もの ため それ これ その この あの ない なく ある いる する",
     "quote_pattern": "「([^」]{10,100})」",
     "action_pattern": "(構築|修正|追加|削除|確認|作成|実装|修復|書き直し|テスト|検証|更新|設定|起動|停止)(?:し|した|して|する|します)"
   }

mempalace/i18n/zh-CN.json

@@ -37,7 +37,7 @@
   },
   "regex": {
     "topic_pattern": "[\\u4E00-\\u9FFF]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
-    "stop_words": "的 了 在 是 我 有 和 就 不 人 都 一 一个 上 也 很 到 说 要 去 你 会 着 没有 看 好 自己 这 那 她 他 它 们 但是 因为 所以 如果 虽然 然后 或者 而且",
+    "stop_words": "一个 没有 自己 但是 因为 所以 如果 虽然 然后 或者 而且",
     "quote_pattern": "\\u201C([^\\u201D]{10,100})\\u201D|\"([^\"]{10,200})\"",
     "action_pattern": "(构建|修复|添加|删除|确认|创建|实现|修理|编写|测试|验证|更新|配置|启动|停止)(?:了|完成|成功)"
   },

mempalace/ids.py

@@ -108,6 +108,27 @@ def make_convo_sentinel_id(source_file: str, extract_mode: str) -> str:
     return f"_reg_{_delimited_sha256((source_file, extract_mode), _HASH_TRUNC_DRAWER)}"
 
 
+def make_exchange_drawer_id(
+    wing: str, room: str, source_file: str, filed_at: str, content: str
+) -> str:
+    """Drawer ID for a single verbatim conversation exchange.
+
+    Used by live agent integrations (e.g. Hermes) and their backfills via
+    ``convo_miner.file_conversation_exchange``. Hashes the FULL content,
+    not a prefix — prefix hashing collided on common openings ("User: hi
+    can you help me with…") and ChromaDB's upsert silently overwrote the
+    earlier drawer. ``filed_at`` is included so genuinely repeated
+    exchanges stay distinct drawers (verbatim always — repetition is
+    signal, not noise).
+
+    Hash input is ``f"{source_file}|{filed_at}|{content}"``.
+    """
+    return (
+        f"drawer_{wing}_{room}_"
+        f"{_delimited_sha256((source_file, filed_at, content), _HASH_TRUNC_DRAWER)}"
+    )
+
+
 def make_triple_id(
     sub_id: str, predicate: str, obj_id: str, valid_from: str, recorded_at: str
 ) -> str:

mempalace/knowledge_graph.py

@@ -39,7 +39,7 @@ import json
 import os
 import sqlite3
 import threading
-from datetime import date, datetime
+from datetime import date, datetime, timezone
 from pathlib import Path
 from typing import Optional
 from .config import sanitize_iso_temporal
@@ -113,6 +113,14 @@ def _temporal_filter_sql(as_of: str) -> tuple[str, list[str]]:
 
     This keeps legacy date-only facts working when callers query with
     canonical UTC datetimes such as '2026-05-06T15:00:00Z'.
+
+    The upper bound is *strict* (``valid_to > as_of``): a fact whose
+    ``valid_to`` equals the query instant has already ended at that instant,
+    so the interval is treated as half-open ``[valid_from, valid_to)``. This
+    is what lets a fact and its successor share a boundary instant without an
+    as-of query returning both. Date-only ``valid_to`` still expands to the
+    end of that day (``T23:59:59Z``), so a standalone date-only fact stays
+    valid through its whole final day exactly as before.
     """
 
     as_of_key = _temporal_start_key(as_of)
@@ -121,7 +129,7 @@ def _temporal_filter_sql(as_of: str) -> tuple[str, list[str]]:
 
     return (
         f" AND (t.valid_from IS NULL OR {valid_from_expr} <= ?) "
-        f"AND (t.valid_to IS NULL OR {valid_to_expr} >= ?)",
+        f"AND (t.valid_to IS NULL OR {valid_to_expr} > ?)",
         [as_of_key, as_of_key],
     )
 
@@ -359,6 +367,125 @@ class KnowledgeGraph:
                     (ended, sub_id, pred, obj_id),
                 )
 
+    def supersede(
+        self,
+        subject: str,
+        predicate: str,
+        old_obj: str,
+        new_obj: str,
+        at: str = None,
+        confidence: float = 1.0,
+        source_closet: str = None,
+        source_file: str = None,
+        source_drawer_id: str = None,
+        adapter_name: str = None,
+    ):
+        """Atomically replace one fact with another at a single shared boundary.
+
+        Closes the currently-open ``(subject, predicate, old_obj)`` triple with
+        ``valid_to = at`` and opens ``(subject, predicate, new_obj)`` with
+        ``valid_from = at`` in one transaction, at a single shared instant.
+        Paired with the half-open upper bound in ``_temporal_filter_sql``, an
+        as-of query at that instant returns only the successor.
+
+        This is the primitive for a value change. Hand-rolling a handover as
+        ``invalidate(ended=D)`` + ``add_triple(valid_from=D)`` with date-only
+        ``D`` leaves two facts sharing the whole day ``D`` (``valid_to`` expands
+        to ``T23:59:59Z`` while ``valid_from`` expands to ``T00:00:00Z``), so an
+        as-of query on ``D`` returns both. ``supersede`` avoids this by writing
+        one identical precise instant to both sides.
+
+        ``at`` defaults to the current UTC instant. A date-only ``at`` is
+        normalized to ``<date>T00:00:00Z`` so both sides carry the same precise
+        value rather than the asymmetric whole-day expansion.
+
+        Returns the new triple's id. If no open ``old_obj`` triple exists the
+        successor is still opened, so ``supersede`` degrades to ``add_triple``.
+        """
+        if at is None:
+            boundary = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        elif _is_date_only_temporal(at):
+            boundary = f"{at}T00:00:00Z"
+        else:
+            boundary = at
+        boundary = sanitize_iso_temporal(boundary, "at")
+
+        sub_id = self._entity_id(subject)
+        old_id = self._entity_id(old_obj)
+        new_id = self._entity_id(new_obj)
+        pred = predicate.lower().replace(" ", "_")
+
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                # Only create entities we actually open a fact for. old_obj is
+                # matched by id in the UPDATE below whether or not its row
+                # exists, so inserting it would just orphan an entity when no
+                # open old fact is present (the degrade-to-add path).
+                for name, eid in ((subject, sub_id), (new_obj, new_id)):
+                    conn.execute(
+                        "INSERT OR IGNORE INTO entities (id, name) VALUES (?, ?)",
+                        (eid, name),
+                    )
+
+                # Reject a boundary that precedes the old fact's start — an
+                # inverted interval would be invisible to every KG query.
+                rows = conn.execute(
+                    "SELECT valid_from FROM triples "
+                    "WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
+                    (sub_id, pred, old_id),
+                ).fetchall()
+                for row in rows:
+                    valid_from = row["valid_from"]
+                    if valid_from is not None and _temporal_end_key(boundary) < _temporal_start_key(
+                        valid_from
+                    ):
+                        raise ValueError(
+                            f"at={boundary!r} is before valid_from={valid_from!r}; "
+                            "an inverted interval would be invisible to every KG query"
+                        )
+
+                # Close the open old fact at the shared boundary.
+                conn.execute(
+                    "UPDATE triples SET valid_to=? "
+                    "WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
+                    (boundary, sub_id, pred, old_id),
+                )
+
+                # Open the successor at the same instant (idempotent if already open).
+                existing = conn.execute(
+                    "SELECT id FROM triples "
+                    "WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
+                    (sub_id, pred, new_id),
+                ).fetchone()
+                if existing:
+                    return existing["id"]
+
+                triple_id = make_triple_id(
+                    sub_id, pred, new_id, boundary, datetime.now().isoformat()
+                )
+                conn.execute(
+                    """INSERT INTO triples (
+                        id, subject, predicate, object, valid_from, valid_to,
+                        confidence, source_closet, source_file,
+                        source_drawer_id, adapter_name
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                    (
+                        triple_id,
+                        sub_id,
+                        pred,
+                        new_id,
+                        boundary,
+                        None,
+                        confidence,
+                        source_closet,
+                        source_file,
+                        source_drawer_id,
+                        adapter_name,
+                    ),
+                )
+                return triple_id
+
     # ── Query operations ──────────────────────────────────────────────────
 
     def query_entity(self, name: str, as_of: str = None, direction: str = "outgoing"):
@@ -503,6 +630,66 @@ class KnowledgeGraph:
 
     # ── Stats ─────────────────────────────────────────────────────────────
 
+    # -- Replication (RFC 004 step 1: read-replica snapshot) ---------------
+
+    _REPLICATION_TABLES = {
+        "entities": ("id", "name", "type", "properties", "created_at"),
+        "triples": (
+            "id",
+            "subject",
+            "predicate",
+            "object",
+            "valid_from",
+            "valid_to",
+            "confidence",
+            "source_closet",
+            "source_file",
+            "source_drawer_id",
+            "adapter_name",
+            "extracted_at",
+        ),
+    }
+
+    def dump_rows(self, table: str, after_rowid: int = 0, limit: int = 500) -> list:
+        """Page KG rows in rowid order for snapshot replication.
+
+        Rows are returned verbatim with a ``_rowid`` pagination cursor.
+        rowid order is deterministic, so pages never skip under concurrent
+        appends (updates in earlier pages are caught by the next full pass).
+        """
+        columns = self._REPLICATION_TABLES.get(table)
+        if columns is None:
+            raise ValueError(f"table must be one of {sorted(self._REPLICATION_TABLES)}")
+        with self._lock:
+            conn = self._conn()
+            rows = conn.execute(
+                f"SELECT rowid, {', '.join(columns)} FROM {table} "
+                "WHERE rowid > ? ORDER BY rowid ASC LIMIT ?",
+                (int(after_rowid), max(1, min(int(limit), 1000))),
+            ).fetchall()
+        return [dict(row) | {"_rowid": row["rowid"]} for row in rows]
+
+    def apply_row(self, table: str, row: dict) -> None:
+        """Fold one replicated KG row in, keyed by id (INSERT OR REPLACE).
+
+        REPLACE makes invalidations (valid_to updates) and entity edits
+        converge on re-pull; rows are never deleted by replication.
+        """
+        columns = self._REPLICATION_TABLES.get(table)
+        if columns is None:
+            raise ValueError(f"table must be one of {sorted(self._REPLICATION_TABLES)}")
+        if not row.get("id"):
+            raise ValueError("replicated row is missing 'id'")
+        values = [row.get(col) for col in columns]
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                conn.execute(
+                    f"INSERT OR REPLACE INTO {table} ({', '.join(columns)}) "
+                    f"VALUES ({', '.join('?' for _ in columns)})",
+                    values,
+                )
+
     def stats(self):
         with self._lock:
             conn = self._conn()

mempalace/layers.py

@@ -126,12 +126,21 @@ class Layer1:
         if not docs:
             return "## L1 — No memories yet."
 
-        # Score each drawer: prefer high importance, recent filing
+        # Score each drawer: prefer high importance, then most-recent filing.
+        # NOTE: the ingest pipeline (miner, convo_miner, diary, add_drawer)
+        # records provenance metadata — wing/room/source/chunk/filed_at — but
+        # never an evaluative importance/weight field. So `importance` is
+        # absent on virtually every drawer and ties at the default, which used
+        # to collapse the sort to insertion order (oldest first). `filed_at`
+        # is present on every drawer, so it is the *effective* ordering signal:
+        # newest first. This keeps importance as the primary key for the day a
+        # scoring pass populates it, while making the "recent filing" half of
+        # the promise true today with data we already have.
         scored = []
         for doc, meta in zip(docs, metas):
             meta = meta or {}
             doc = doc or ""
-            importance = 3
+            importance = 3.0
             # Try multiple metadata keys that might carry weight info
             for key in ("importance", "emotional_weight", "weight"):
                 val = meta.get(key)
@@ -141,11 +150,15 @@ class Layer1:
                     except (ValueError, TypeError):
                         pass
                     break
-            scored.append((importance, meta, doc))
+            # filed_at is an ISO-8601 string; ISO strings sort lexicographically
+            # in chronological order. Coerce to str so a missing/odd value sorts
+            # oldest rather than raising during the comparison.
+            recency = str(meta.get("filed_at") or "")
+            scored.append((importance, recency, meta, doc))
 
-        # Sort by importance descending, take top N
-        scored.sort(key=lambda x: x[0], reverse=True)
-        top = scored[: self.MAX_DRAWERS]
+        # Sort by importance desc, then recency (filed_at) desc; take top N.
+        scored.sort(key=lambda x: (x[0], x[1]), reverse=True)
+        top = [(imp, meta, doc) for imp, _recency, meta, doc in scored[: self.MAX_DRAWERS]]
 
         # Group by room for readability
         by_room = defaultdict(list)
@@ -306,6 +319,9 @@ class Layer3:
             lines.append(f"      {snippet}")
             if source:
                 lines.append(f"      src: {source}")
+            authored = (meta.get("authored_at") or "")[:10]
+            if authored:
+                lines.append(f"      authored: {authored}")
 
         return "\n".join(lines)
 
@@ -453,7 +469,7 @@ if __name__ == "__main__":
     import json
 
     def usage():
-        print("layers.py — 4-Layer Memory Stack")
+        print("layers.py -- 4-Layer Memory Stack")
         print()
         print("Usage:")
         print("  python layers.py wake-up              Show L0 + L1")

mempalace/llm_client.py

@@ -323,7 +323,7 @@ class OpenAICompatProvider(LLMProvider):
         base = base.removesuffix("/chat/completions").removesuffix("/v1")
         try:
             req = Request(f"{base}/v1/models")
-            if self.api_key:
+            if self.api_key and (self.api_key_source != "env" or not self.is_external_service):
                 req.add_header("Authorization", f"Bearer {self.api_key}")
             with urlopen(req, timeout=5):
                 pass

mempalace/logstream.py

@@ -0,0 +1,968 @@
+"""
+logstream.py — Agent coordination event log for MemPalace (RFC 003)
+===================================================================
+
+A small append-only event layer stored in the active palace directory as
+``logstream.sqlite3``. Agents use it to delegate work, wait for replies,
+and exchange exact patch/file artifacts through the shared MemPalace hub
+without a human relaying messages between machines.
+
+Design constraints (RFC 003):
+
+- No Chroma dependency, no vector index open — plain SQLite only.
+- Append-only: events are immutable; corrections are new events that
+  reference prior events (see :meth:`Logstream.ack_event`).
+- Exact payloads: event bodies and artifact content are stored verbatim.
+- Safe under concurrent HTTP requests (WAL + per-instance lock, same
+  pattern as ``knowledge_graph.py``).
+- Explicit size limits with clear errors, never silent truncation.
+
+Usage:
+    from mempalace.logstream import Logstream
+
+    ls = Logstream(db_path="/path/to/palace/logstream.sqlite3")
+    evt = ls.append_event(
+        type="task.request", stream="project/mempalace", room="delegation",
+        from_agent="mac-codex", to_agent="windows-codex",
+        correlation_id="task_123", body="Please fix search echo ranking.",
+    )
+    reply = ls.wait_events(correlation_id="task_123", type="patch.ready",
+                           timeout_ms=300_000)
+"""
+
+import json
+import random
+import re
+import secrets
+import sqlite3
+import threading
+import time
+from datetime import datetime, timezone
+from hashlib import sha256
+from pathlib import Path
+from typing import Optional
+
+from .config import sanitize_iso_temporal, strip_lone_surrogates
+
+LOGSTREAM_DB_FILENAME = "logstream.sqlite3"
+
+# Size limits (RFC 003 suggested defaults). Byte lengths of the UTF-8
+# encoding, checked with explicit errors — payloads are never truncated.
+DEFAULT_MAX_BODY_BYTES = 256 * 1024
+DEFAULT_MAX_ARTIFACT_BYTES = 4 * 1024 * 1024
+MAX_METADATA_BYTES = 64 * 1024
+
+# Wait limits: polling loop inside one MCP request, 250-1000 ms with jitter.
+MAX_WAIT_TIMEOUT_MS = 300_000
+_POLL_BASE_S = 0.25
+_POLL_MAX_S = 1.0
+
+MAX_LIST_LIMIT = 500
+DEFAULT_LIST_LIMIT = 50
+
+EVENT_STATUSES = frozenset(
+    {"open", "claimed", "ready", "applied", "blocked", "failed", "superseded"}
+)
+ARTIFACT_KINDS = frozenset({"patch", "file", "log", "json", "note"})
+ACK_EVENT_TYPE = "event.ack"
+
+# Event types are dotted lowercase identifiers (task.request, patch.ready).
+# Routing fields must stay structured — agents should never have to infer
+# them from prose — so the character set is enforced, not just length.
+_EVENT_TYPE_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$")
+
+_MAX_ROUTING_LENGTH = 256
+
+
+def _utc_now_iso() -> str:
+    """Canonical UTC timestamp, second precision: YYYY-MM-DDTHH:MM:SSZ.
+
+    Matches the canonical temporal shape accepted by
+    ``config.sanitize_iso_temporal`` so ``since_created_at`` filters
+    compare like against like. Ordering ties within one second are
+    resolved by the SQLite rowid (``seq``), not the timestamp.
+    """
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _new_id(prefix: str) -> str:
+    """Server-generated id: readable UTC stamp + random suffix.
+
+    Uniqueness comes from the random suffix; ordering guarantees come
+    from the rowid, so clock skew between writers cannot reorder or
+    collide events.
+    """
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S")
+    return f"{prefix}_{stamp}_{secrets.token_hex(6)}"
+
+
+def _sanitize_routing(value, field_name: str, required: bool = True) -> Optional[str]:
+    """Validate a short routing field (stream, room, agent, correlation_id).
+
+    Streams may contain ``/`` (``project/mempalace``), so this is looser
+    than ``config.sanitize_name`` — but null bytes, control characters,
+    and over-length values are still rejected.
+    """
+    if value is None or value == "":
+        if required:
+            raise ValueError(f"{field_name} must be a non-empty string")
+        return None
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{field_name} must be a non-empty string")
+    value = strip_lone_surrogates(value.strip())
+    if len(value) > _MAX_ROUTING_LENGTH:
+        raise ValueError(f"{field_name} exceeds maximum length of {_MAX_ROUTING_LENGTH} characters")
+    if any(ord(ch) < 0x20 or ch == "\x7f" for ch in value):
+        raise ValueError(f"{field_name} contains control characters")
+    return value
+
+
+def _sanitize_event_type(value) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("type must be a non-empty string")
+    value = value.strip()
+    if not _EVENT_TYPE_RE.match(value):
+        raise ValueError(
+            f"type={value!r} is not a valid event type "
+            "(lowercase letters, digits, '.', '_', '-'; max 64 chars)"
+        )
+    return value
+
+
+def _sanitize_status(value) -> Optional[str]:
+    if value is None or value == "":
+        return None
+    if not isinstance(value, str) or value not in EVENT_STATUSES:
+        allowed = ", ".join(sorted(EVENT_STATUSES))
+        raise ValueError(f"status={value!r} is not one of: {allowed}")
+    return value
+
+
+def _sanitize_body(value, max_bytes: int, field_name: str = "body") -> str:
+    """Validate verbatim payload text. Empty is allowed; ``None`` becomes ''."""
+    if value is None:
+        return ""
+    if not isinstance(value, str):
+        raise ValueError(f"{field_name} must be a string")
+    if "\x00" in value:
+        raise ValueError(f"{field_name} contains null bytes")
+    value = strip_lone_surrogates(value)
+    size = len(value.encode("utf-8"))
+    if size > max_bytes:
+        raise ValueError(f"{field_name} is {size} bytes; maximum is {max_bytes} bytes")
+    return value
+
+
+def _sanitize_metadata(value) -> str:
+    """Validate optional metadata dict and return its canonical JSON text."""
+    if value is None:
+        return "{}"
+    if not isinstance(value, dict):
+        raise ValueError("metadata must be an object")
+    try:
+        encoded = json.dumps(value, ensure_ascii=False, sort_keys=True)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"metadata is not JSON-serializable: {exc}") from None
+    if len(encoded.encode("utf-8")) > MAX_METADATA_BYTES:
+        raise ValueError(f"metadata exceeds maximum size of {MAX_METADATA_BYTES} bytes")
+    return encoded
+
+
+class Logstream:
+    """Durable append-only coordination log (events + artifacts).
+
+    Storage and threading mirror ``KnowledgeGraph``: one SQLite file in
+    WAL mode, a per-instance lock around writes, ``check_same_thread=False``
+    so the MCP HTTP server can call from worker threads.
+    """
+
+    def __init__(
+        self,
+        db_path: str,
+        max_body_bytes: int = DEFAULT_MAX_BODY_BYTES,
+        max_artifact_bytes: int = DEFAULT_MAX_ARTIFACT_BYTES,
+        replica_id: str = None,
+    ):
+        from .replica import get_replica_id
+
+        self.db_path = str(db_path)
+        self.max_body_bytes = max_body_bytes
+        self.max_artifact_bytes = max_artifact_bytes
+        db_parent = Path(self.db_path).parent
+        db_parent.mkdir(parents=True, exist_ok=True)
+        try:
+            db_parent.chmod(0o700)
+        except (OSError, NotImplementedError):
+            pass
+        # RFC 004 provenance: every op is stamped with the replica that authored it.
+        # replica.json lives in the palace dir (the db's parent).
+        self.replica_id = replica_id or get_replica_id(str(db_parent))
+        self._connection = None
+        self._lock = threading.Lock()
+        self._init_db()
+
+    def _init_db(self):
+        from .hlc import HybridLogicalClock
+
+        conn = self._conn()
+        conn.executescript("""
+            PRAGMA journal_mode=WAL;
+
+            CREATE TABLE IF NOT EXISTS events (
+                id TEXT PRIMARY KEY,
+                type TEXT NOT NULL,
+                stream TEXT NOT NULL,
+                room TEXT NOT NULL,
+                from_agent TEXT NOT NULL,
+                to_agent TEXT,
+                correlation_id TEXT,
+                branch TEXT,
+                base_commit TEXT,
+                status TEXT,
+                body TEXT NOT NULL DEFAULT '',
+                created_at TEXT NOT NULL,
+                metadata_json TEXT NOT NULL DEFAULT '{}',
+                origin_replica TEXT,
+                origin_seq INTEGER,
+                hlc TEXT
+            );
+
+            CREATE INDEX IF NOT EXISTS events_stream_created_idx
+                ON events(stream, created_at);
+            CREATE INDEX IF NOT EXISTS events_correlation_idx
+                ON events(correlation_id, created_at);
+            CREATE INDEX IF NOT EXISTS events_to_agent_idx
+                ON events(to_agent, created_at);
+            CREATE INDEX IF NOT EXISTS events_type_idx
+                ON events(type, created_at);
+
+            CREATE TABLE IF NOT EXISTS artifacts (
+                id TEXT PRIMARY KEY,
+                kind TEXT NOT NULL,
+                sha256 TEXT NOT NULL,
+                size_bytes INTEGER NOT NULL,
+                content TEXT NOT NULL,
+                created_by TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                metadata_json TEXT NOT NULL DEFAULT '{}',
+                origin_replica TEXT
+            );
+
+            CREATE INDEX IF NOT EXISTS artifacts_sha256_idx ON artifacts(sha256);
+
+            CREATE TABLE IF NOT EXISTS event_artifacts (
+                event_id TEXT NOT NULL,
+                artifact_id TEXT NOT NULL,
+                PRIMARY KEY (event_id, artifact_id)
+            );
+        """)
+        self._migrate_replication_schema(conn)
+        conn.commit()
+        # Seed the HLC from the newest stamp in the log so monotonicity
+        # survives restarts (RFC 004 op envelope).
+        row = conn.execute("SELECT max(hlc) AS last FROM events").fetchone()
+        self._clock = HybridLogicalClock(self.replica_id, last=row["last"] if row else None)
+
+    def _migrate_replication_schema(self, conn):
+        """RFC 004 step 0: add provenance columns to pre-replication logs.
+
+        Fresh databases get the columns from the canonical CREATE TABLE; this
+        path upgrades logs created before step 0. Backfill stamps existing
+        rows as authored by THIS replica (they were: pre-replication, only
+        one copy existed) with origin_seq = rowid and an HLC derived from
+        created_at + rowid, preserving their original total order.
+        """
+        existing = {row["name"] for row in conn.execute("PRAGMA table_info(events)")}
+        if "origin_replica" not in existing:
+            conn.execute("ALTER TABLE events ADD COLUMN origin_replica TEXT")
+        if "origin_seq" not in existing:
+            conn.execute("ALTER TABLE events ADD COLUMN origin_seq INTEGER")
+        if "hlc" not in existing:
+            conn.execute("ALTER TABLE events ADD COLUMN hlc TEXT")
+        art_existing = {row["name"] for row in conn.execute("PRAGMA table_info(artifacts)")}
+        if "origin_replica" not in art_existing:
+            conn.execute("ALTER TABLE artifacts ADD COLUMN origin_replica TEXT")
+
+        from .hlc import render as hlc_render
+
+        rows = conn.execute(
+            "SELECT rowid, created_at FROM events WHERE origin_replica IS NULL"
+        ).fetchall()
+        for row in rows:
+            try:
+                ms = int(
+                    datetime.strptime(row["created_at"], "%Y-%m-%dT%H:%M:%SZ")
+                    .replace(tzinfo=timezone.utc)
+                    .timestamp()
+                    * 1000
+                )
+            except ValueError:
+                ms = 0
+            conn.execute(
+                "UPDATE events SET origin_replica = ?, origin_seq = rowid, hlc = ? WHERE rowid = ?",
+                (
+                    self.replica_id,
+                    hlc_render(ms, min(row["rowid"], 0xFFFFFF), self.replica_id),
+                    row["rowid"],
+                ),
+            )
+        conn.execute(
+            "UPDATE artifacts SET origin_replica = ? WHERE origin_replica IS NULL",
+            (self.replica_id,),
+        )
+        conn.execute(
+            "CREATE UNIQUE INDEX IF NOT EXISTS events_origin_seq_idx "
+            "ON events(origin_replica, origin_seq)"
+        )
+        conn.execute("CREATE INDEX IF NOT EXISTS events_hlc_idx ON events(hlc)")
+
+    def _conn(self):
+        if self._connection is None:
+            self._connection = sqlite3.connect(self.db_path, timeout=10, check_same_thread=False)
+            self._connection.execute("PRAGMA journal_mode=WAL")
+            self._connection.row_factory = sqlite3.Row
+        return self._connection
+
+    def close(self):
+        with self._lock:
+            if self._connection is not None:
+                self._connection.close()
+                self._connection = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.close()
+        return False
+
+    # ── Row shaping ───────────────────────────────────────────────────────
+
+    @staticmethod
+    def _event_dict(row, artifact_ids: Optional[list] = None) -> dict:
+        try:
+            metadata = json.loads(row["metadata_json"])
+        except (ValueError, TypeError):
+            metadata = {}
+        return {
+            "id": row["id"],
+            "seq": row["rowid"],
+            # RFC 004 additive fields — committed compat surface. `seq` stays
+            # the LOCAL arrival cursor; `origin_seq` is the author's counter;
+            # `hlc` is the cross-replica display/merge order.
+            "origin_replica": row["origin_replica"],
+            "origin_seq": row["origin_seq"],
+            "hlc": row["hlc"],
+            "type": row["type"],
+            "stream": row["stream"],
+            "room": row["room"],
+            "from_agent": row["from_agent"],
+            "to_agent": row["to_agent"],
+            "correlation_id": row["correlation_id"],
+            "branch": row["branch"],
+            "base_commit": row["base_commit"],
+            "status": row["status"],
+            "artifact_ids": artifact_ids or [],
+            "body": row["body"],
+            "created_at": row["created_at"],
+            "metadata": metadata,
+        }
+
+    def _attach_artifact_ids(self, conn, events: list[dict]) -> list[dict]:
+        if not events:
+            return events
+        by_id = {e["id"]: e for e in events}
+        placeholders = ",".join("?" for _ in by_id)
+        rows = conn.execute(
+            f"SELECT event_id, artifact_id FROM event_artifacts "
+            f"WHERE event_id IN ({placeholders}) ORDER BY artifact_id",
+            list(by_id),
+        ).fetchall()
+        for row in rows:
+            by_id[row["event_id"]]["artifact_ids"].append(row["artifact_id"])
+        return events
+
+    # ── Write operations (append-only) ────────────────────────────────────
+
+    def append_event(
+        self,
+        type: str,
+        stream: str,
+        room: str,
+        from_agent: str,
+        to_agent: str = None,
+        correlation_id: str = None,
+        branch: str = None,
+        base_commit: str = None,
+        status: str = None,
+        body: str = "",
+        metadata: dict = None,
+        artifact_ids: list = None,
+    ) -> dict:
+        """Append one immutable event. Returns the stored event dict.
+
+        ``artifact_ids`` must reference already-stored artifacts; unknown
+        ids are rejected so readers never see a dangling reference.
+        """
+        type = _sanitize_event_type(type)
+        stream = _sanitize_routing(stream, "stream")
+        room = _sanitize_routing(room, "room")
+        from_agent = _sanitize_routing(from_agent, "from_agent")
+        to_agent = _sanitize_routing(to_agent, "to_agent", required=False)
+        correlation_id = _sanitize_routing(correlation_id, "correlation_id", required=False)
+        branch = _sanitize_routing(branch, "branch", required=False)
+        base_commit = _sanitize_routing(base_commit, "base_commit", required=False)
+        status = _sanitize_status(status)
+        body = _sanitize_body(body, self.max_body_bytes)
+        metadata_json = _sanitize_metadata(metadata)
+
+        if artifact_ids is None:
+            artifact_ids = []
+        if not isinstance(artifact_ids, list) or not all(
+            isinstance(a, str) and a for a in artifact_ids
+        ):
+            raise ValueError("artifact_ids must be a list of artifact id strings")
+        artifact_ids = list(dict.fromkeys(artifact_ids))  # dedup, keep order
+
+        event_id = _new_id("evt")
+        created_at = _utc_now_iso()
+        hlc = self._clock.tick()
+
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                for artifact_id in artifact_ids:
+                    found = conn.execute(
+                        "SELECT 1 FROM artifacts WHERE id = ?", (artifact_id,)
+                    ).fetchone()
+                    if not found:
+                        raise ValueError(
+                            f"artifact_ids references unknown artifact {artifact_id!r}"
+                        )
+                cursor = conn.execute(
+                    "INSERT INTO events (id, type, stream, room, from_agent, to_agent,"
+                    " correlation_id, branch, base_commit, status, body, created_at,"
+                    " metadata_json, origin_replica, hlc)"
+                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        event_id,
+                        type,
+                        stream,
+                        room,
+                        from_agent,
+                        to_agent,
+                        correlation_id,
+                        branch,
+                        base_commit,
+                        status,
+                        body,
+                        created_at,
+                        metadata_json,
+                        self.replica_id,
+                        hlc,
+                    ),
+                )
+                # Locally-authored ops use the local rowid as their per-origin
+                # counter — strictly monotonic, gap-free enough for cursors.
+                conn.execute(
+                    "UPDATE events SET origin_seq = rowid WHERE rowid = ?",
+                    (cursor.lastrowid,),
+                )
+                for artifact_id in artifact_ids:
+                    conn.execute(
+                        "INSERT INTO event_artifacts (event_id, artifact_id) VALUES (?, ?)",
+                        (event_id, artifact_id),
+                    )
+            row = conn.execute("SELECT rowid, * FROM events WHERE id = ?", (event_id,)).fetchone()
+        return self._event_dict(row, artifact_ids=list(artifact_ids))
+
+    @staticmethod
+    def _patch_content_warnings(kind: str, content: str) -> list[str]:
+        """Advisory checks for ``kind=patch`` content. Never mutates content.
+
+        Verbatim storage means we store exactly what the producer sent —
+        but a diff that ``git apply`` will reject as corrupt is a broken
+        handoff, so warn at store time where the producer can still fix it.
+        Found in the wild during the first RFC 003 dogfood: a patch stored
+        without its final trailing newline truncates the last hunk line.
+        """
+        if kind != "patch":
+            return []
+        warnings = []
+        if not content.endswith("\n"):
+            warnings.append(
+                "patch content does not end with a trailing newline; "
+                "git apply will reject the final hunk as corrupt — "
+                "store the diff byte-exactly including its trailing newline"
+            )
+        if "\r" in content:
+            warnings.append(
+                "patch content contains carriage returns (CRLF line endings); "
+                "git apply often rejects CRLF diffs — generate the diff with "
+                "LF endings"
+            )
+        return warnings
+
+    def put_artifact(
+        self,
+        kind: str,
+        content: str,
+        created_by: str,
+        metadata: dict = None,
+    ) -> dict:
+        """Store exact artifact content (v1: UTF-8 text only).
+
+        Returns the artifact record without echoing ``content`` back —
+        callers already hold the content; readers use :meth:`get_artifact`.
+        For ``kind=patch``, a ``warnings`` list is included when the diff
+        looks unappliable (missing trailing newline, CRLF endings); the
+        content itself is still stored verbatim.
+        """
+        if not isinstance(kind, str) or kind not in ARTIFACT_KINDS:
+            allowed = ", ".join(sorted(ARTIFACT_KINDS))
+            raise ValueError(f"kind={kind!r} is not one of: {allowed}")
+        created_by = _sanitize_routing(created_by, "created_by")
+        if not isinstance(content, str) or not content:
+            raise ValueError("content must be a non-empty string")
+        if "\x00" in content:
+            raise ValueError("content contains null bytes")
+        content = strip_lone_surrogates(content)
+        raw = content.encode("utf-8")
+        if len(raw) > self.max_artifact_bytes:
+            raise ValueError(
+                f"content is {len(raw)} bytes; maximum is {self.max_artifact_bytes} bytes"
+            )
+        metadata_json = _sanitize_metadata(metadata)
+
+        artifact_id = _new_id("art")
+        created_at = _utc_now_iso()
+        digest = sha256(raw).hexdigest()
+
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                conn.execute(
+                    "INSERT INTO artifacts (id, kind, sha256, size_bytes, content,"
+                    " created_by, created_at, metadata_json, origin_replica)"
+                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        artifact_id,
+                        kind,
+                        digest,
+                        len(raw),
+                        content,
+                        created_by,
+                        created_at,
+                        metadata_json,
+                        self.replica_id,
+                    ),
+                )
+        record = {
+            "id": artifact_id,
+            "kind": kind,
+            "sha256": digest,
+            "size_bytes": len(raw),
+            "created_by": created_by,
+            "created_at": created_at,
+            "origin_replica": self.replica_id,
+        }
+        warnings = self._patch_content_warnings(kind, content)
+        if warnings:
+            record["warnings"] = warnings
+        return record
+
+    def ack_event(
+        self,
+        event_id: str,
+        from_agent: str,
+        status: str = None,
+        body: str = "",
+    ) -> dict:
+        """Append an ``event.ack`` referencing a prior event.
+
+        The target event is never mutated. The ack copies the target's
+        stream/room, copies its ``correlation_id`` (falling back to the
+        target's id so request/ack stay tied together), and routes back
+        to the target's ``from_agent``.
+        """
+        event_id = _sanitize_routing(event_id, "event_id")
+        with self._lock:
+            conn = self._conn()
+            target = conn.execute(
+                "SELECT rowid, * FROM events WHERE id = ?", (event_id,)
+            ).fetchone()
+        if target is None:
+            raise ValueError(f"event {event_id!r} not found")
+
+        return self.append_event(
+            type=ACK_EVENT_TYPE,
+            stream=target["stream"],
+            room=target["room"],
+            from_agent=from_agent,
+            to_agent=target["from_agent"],
+            correlation_id=target["correlation_id"] or target["id"],
+            status=status,
+            body=body,
+            metadata={"ack_of": event_id},
+        )
+
+    def submit_patch(
+        self,
+        content: str,
+        from_agent: str,
+        stream: str,
+        room: str = "patches",
+        to_agent: str = None,
+        correlation_id: str = None,
+        branch: str = None,
+        base_commit: str = None,
+        body: str = "",
+        metadata: dict = None,
+    ) -> dict:
+        """Convenience wrapper: store a patch artifact + ``patch.ready`` event.
+
+        Both writes go through the validated single-writer paths; the event
+        insert re-checks the artifact exists, so readers never observe a
+        ``patch.ready`` event with a dangling artifact id.
+        """
+        artifact = self.put_artifact(
+            kind="patch",
+            content=content,
+            created_by=from_agent,
+            metadata=metadata,
+        )
+        event = self.append_event(
+            type="patch.ready",
+            stream=stream,
+            room=room,
+            from_agent=from_agent,
+            to_agent=to_agent,
+            correlation_id=correlation_id,
+            branch=branch,
+            base_commit=base_commit,
+            status="ready",
+            body=body,
+            metadata=metadata,
+            artifact_ids=[artifact["id"]],
+        )
+        return {"artifact": artifact, "event": event}
+
+    # ── Read operations ───────────────────────────────────────────────────
+
+    def get_artifact(self, artifact_id: str) -> Optional[dict]:
+        """Fetch one artifact by id, exact content included. None if missing."""
+        artifact_id = _sanitize_routing(artifact_id, "artifact_id")
+        with self._lock:
+            conn = self._conn()
+            row = conn.execute("SELECT * FROM artifacts WHERE id = ?", (artifact_id,)).fetchone()
+        if row is None:
+            return None
+        try:
+            metadata = json.loads(row["metadata_json"])
+        except (ValueError, TypeError):
+            metadata = {}
+        return {
+            "id": row["id"],
+            "kind": row["kind"],
+            "sha256": row["sha256"],
+            "size_bytes": row["size_bytes"],
+            "content": row["content"],
+            "created_by": row["created_by"],
+            "created_at": row["created_at"],
+            "origin_replica": row["origin_replica"],
+            "metadata": metadata,
+        }
+
+    def list_events(
+        self,
+        stream: str = None,
+        room: str = None,
+        type: str = None,
+        to_agent: str = None,
+        from_agent: str = None,
+        correlation_id: str = None,
+        status: str = None,
+        since_event_id: str = None,
+        since_created_at: str = None,
+        limit: int = DEFAULT_LIST_LIMIT,
+    ) -> list[dict]:
+        """List events matching structured filters, oldest first.
+
+        Cursor semantics:
+
+        - ``since_event_id`` is the precise cursor: strictly after that
+          event in append order (rowid), regardless of timestamp ties.
+        - ``since_created_at`` is inclusive (``>=``) so second-granularity
+          timestamps never skip events; callers dedup by ``id``.
+        - ``to_agent`` also matches broadcast events (``to_agent='*'``).
+        """
+        stream = _sanitize_routing(stream, "stream", required=False)
+        room = _sanitize_routing(room, "room", required=False)
+        if type not in (None, ""):
+            type = _sanitize_event_type(type)
+        else:
+            type = None
+        to_agent = _sanitize_routing(to_agent, "to_agent", required=False)
+        from_agent = _sanitize_routing(from_agent, "from_agent", required=False)
+        correlation_id = _sanitize_routing(correlation_id, "correlation_id", required=False)
+        status = _sanitize_status(status)
+        since_event_id = _sanitize_routing(since_event_id, "since_event_id", required=False)
+        since_created_at = sanitize_iso_temporal(since_created_at, "since_created_at") or None
+        if not isinstance(limit, int) or limit < 1:
+            raise ValueError("limit must be a positive integer")
+        limit = min(limit, MAX_LIST_LIMIT)
+
+        where = []
+        params = []
+        for column, value in (
+            ("stream", stream),
+            ("room", room),
+            ("type", type),
+            ("from_agent", from_agent),
+            ("correlation_id", correlation_id),
+            ("status", status),
+        ):
+            if value is not None:
+                where.append(f"{column} = ?")
+                params.append(value)
+        if to_agent is not None:
+            where.append("(to_agent = ? OR to_agent = '*')")
+            params.append(to_agent)
+        if since_created_at is not None:
+            where.append("created_at >= ?")
+            params.append(since_created_at)
+
+        with self._lock:
+            conn = self._conn()
+            if since_event_id is not None:
+                anchor = conn.execute(
+                    "SELECT rowid FROM events WHERE id = ?", (since_event_id,)
+                ).fetchone()
+                if anchor is None:
+                    raise ValueError(f"since_event_id {since_event_id!r} not found")
+                where.append("rowid > ?")
+                params.append(anchor["rowid"])
+
+            sql = "SELECT rowid, * FROM events"
+            if where:
+                sql += " WHERE " + " AND ".join(where)
+            sql += " ORDER BY rowid ASC LIMIT ?"
+            params.append(limit)
+            rows = conn.execute(sql, params).fetchall()
+            events = [self._event_dict(row) for row in rows]
+            return self._attach_artifact_ids(conn, events)
+
+    def latest_event_id(self) -> Optional[str]:
+        """Id of the newest event, or None on an empty log.
+
+        Live-tail consumers (the SSE stream) capture this at connect time
+        as their starting cursor so they receive only post-connect events.
+        """
+        with self._lock:
+            conn = self._conn()
+            row = conn.execute("SELECT id FROM events ORDER BY rowid DESC LIMIT 1").fetchone()
+        return row["id"] if row is not None else None
+
+    def wait_events(
+        self,
+        timeout_ms: int = 60_000,
+        poll_interval_s: float = None,
+        **filters,
+    ) -> dict:
+        """Block until at least one matching event exists or timeout expires.
+
+        v1 implementation per RFC 003: a polling loop inside the request,
+        sleeping 250-1000 ms with jitter. Timeouts are clamped to
+        ``MAX_WAIT_TIMEOUT_MS`` and return ``{"timed_out": True,
+        "events": []}`` rather than raising.
+
+        ``filters`` accepts the same keyword filters as :meth:`list_events`.
+        ``poll_interval_s`` pins the sleep (tests); default is jittered.
+        """
+        if not isinstance(timeout_ms, (int, float)) or timeout_ms < 0:
+            raise ValueError("timeout_ms must be a non-negative number")
+        timeout_ms = min(timeout_ms, MAX_WAIT_TIMEOUT_MS)
+        deadline = time.monotonic() + timeout_ms / 1000.0
+
+        attempt = 0
+        while True:
+            events = self.list_events(**filters)
+            if events:
+                return {"timed_out": False, "events": events}
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                return {"timed_out": True, "events": []}
+            if poll_interval_s is not None:
+                delay = poll_interval_s
+            else:
+                base = min(_POLL_MAX_S, _POLL_BASE_S * (1.5**attempt))
+                delay = base * (0.75 + random.random() * 0.25)
+            time.sleep(min(delay, remaining))
+            attempt += 1
+
+    # ── Replication (RFC 004 step 0: logstream multi-master) ─────────────
+
+    def version_vector(self) -> dict:
+        """{origin_replica: highest origin_seq applied locally}.
+
+        The complete description of this replica's knowledge — peers diff
+        their vectors to compute exactly which op ranges are missing.
+        """
+        with self._lock:
+            conn = self._conn()
+            rows = conn.execute(
+                "SELECT origin_replica, max(origin_seq) AS top FROM events "
+                "WHERE origin_replica IS NOT NULL GROUP BY origin_replica"
+            ).fetchall()
+        return {row["origin_replica"]: row["top"] for row in rows}
+
+    def list_ops(self, origin: str, after_seq: int = 0, limit: int = 500) -> list[dict]:
+        """Events authored by ``origin`` with origin_seq > after_seq, in
+        author order. The anti-entropy pull unit."""
+        origin = _sanitize_routing(origin, "origin")
+        if not isinstance(after_seq, int) or after_seq < 0:
+            raise ValueError("after_seq must be a non-negative integer")
+        if not isinstance(limit, int) or limit < 1:
+            raise ValueError("limit must be a positive integer")
+        limit = min(limit, MAX_LIST_LIMIT)
+        with self._lock:
+            conn = self._conn()
+            rows = conn.execute(
+                "SELECT rowid, * FROM events WHERE origin_replica = ? AND origin_seq > ? "
+                "ORDER BY origin_seq ASC LIMIT ?",
+                (origin, after_seq, limit),
+            ).fetchall()
+            events = [self._event_dict(row) for row in rows]
+            return self._attach_artifact_ids(conn, events)
+
+    def apply_remote_event(self, event: dict) -> bool:
+        """Fold one remote op into the local log, idempotently.
+
+        Verbatim rule: the event is stored exactly as authored (id,
+        created_at, hlc, origin stamps untouched); only the local rowid —
+        the arrival cursor — is ours. Referenced artifacts must already be
+        applied (sync pulls artifacts first) so readers never see a dangling
+        id, same invariant as append_event. Returns True if inserted, False
+        if we already had it. Raises ValueError on malformed input or a
+        missing artifact.
+        """
+        for key in (
+            "id",
+            "type",
+            "stream",
+            "room",
+            "from_agent",
+            "created_at",
+            "origin_replica",
+            "origin_seq",
+            "hlc",
+        ):
+            if not event.get(key):
+                raise ValueError(f"remote event is missing required field {key!r}")
+        if event["origin_replica"] == self.replica_id:
+            return False  # our own op echoed back — by definition already present
+        artifact_ids = list(dict.fromkeys(event.get("artifact_ids") or []))
+        metadata_json = _sanitize_metadata(event.get("metadata"))
+
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                dup = conn.execute(
+                    "SELECT 1 FROM events WHERE id = ? OR (origin_replica = ? AND origin_seq = ?)",
+                    (event["id"], event["origin_replica"], event["origin_seq"]),
+                ).fetchone()
+                if dup:
+                    return False
+                for artifact_id in artifact_ids:
+                    found = conn.execute(
+                        "SELECT 1 FROM artifacts WHERE id = ?", (artifact_id,)
+                    ).fetchone()
+                    if not found:
+                        raise ValueError(
+                            f"remote event {event['id']!r} references artifact "
+                            f"{artifact_id!r} not yet applied locally"
+                        )
+                conn.execute(
+                    "INSERT INTO events (id, type, stream, room, from_agent, to_agent,"
+                    " correlation_id, branch, base_commit, status, body, created_at,"
+                    " metadata_json, origin_replica, origin_seq, hlc)"
+                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        event["id"],
+                        event["type"],
+                        event["stream"],
+                        event["room"],
+                        event["from_agent"],
+                        event.get("to_agent"),
+                        event.get("correlation_id"),
+                        event.get("branch"),
+                        event.get("base_commit"),
+                        event.get("status"),
+                        event.get("body") or "",
+                        event["created_at"],
+                        metadata_json,
+                        event["origin_replica"],
+                        event["origin_seq"],
+                        event["hlc"],
+                    ),
+                )
+                for artifact_id in artifact_ids:
+                    conn.execute(
+                        "INSERT OR IGNORE INTO event_artifacts (event_id, artifact_id)"
+                        " VALUES (?, ?)",
+                        (event["id"], artifact_id),
+                    )
+        # Absorb the remote instant so our next local op sorts after it.
+        self._clock.observe(event["hlc"])
+        return True
+
+    def has_artifact(self, artifact_id: str) -> bool:
+        """Cheap existence probe (no content transfer) for the sync engine."""
+        artifact_id = _sanitize_routing(artifact_id, "artifact_id")
+        with self._lock:
+            conn = self._conn()
+            row = conn.execute("SELECT 1 FROM artifacts WHERE id = ?", (artifact_id,)).fetchone()
+        return row is not None
+
+    def apply_remote_artifact(self, artifact: dict) -> bool:
+        """Fold one remote artifact in, idempotently, verifying its hash.
+
+        Content is verbatim; the sha256 must match or the artifact is
+        rejected — a corrupt transfer must never enter the store.
+        """
+        for key in ("id", "kind", "sha256", "content", "created_by", "created_at"):
+            if not artifact.get(key):
+                raise ValueError(f"remote artifact is missing required field {key!r}")
+        content = artifact["content"]
+        digest = sha256(content.encode("utf-8")).hexdigest()
+        if digest != artifact["sha256"]:
+            raise ValueError(
+                f"remote artifact {artifact['id']!r} failed hash verification "
+                f"(claimed {artifact['sha256'][:16]}..., computed {digest[:16]}...)"
+            )
+        metadata_json = _sanitize_metadata(artifact.get("metadata"))
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                dup = conn.execute(
+                    "SELECT 1 FROM artifacts WHERE id = ?", (artifact["id"],)
+                ).fetchone()
+                if dup:
+                    return False
+                conn.execute(
+                    "INSERT INTO artifacts (id, kind, sha256, size_bytes, content,"
+                    " created_by, created_at, metadata_json, origin_replica)"
+                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        artifact["id"],
+                        artifact["kind"],
+                        artifact["sha256"],
+                        len(content.encode("utf-8")),
+                        content,
+                        artifact["created_by"],
+                        artifact["created_at"],
+                        metadata_json,
+                        artifact.get("origin_replica"),
+                    ),
+                )
+        return True

mempalace/logsync.py

@@ -0,0 +1,136 @@
+"""
+logsync.py — Anti-entropy sync engine for the logstream (RFC 004 step 0)
+
+Pull-based convergence between palace replicas: diff version vectors, pull
+missing per-origin op ranges (artifacts first, so referenced ids never
+dangle), fold with the idempotent apply primitives. Each replica pulls from
+its peers; two replicas pulling from each other converge without any push
+path or coordinator.
+
+Transport for the pilot rides the RFC 004 transport seam's `request` shape over
+plain HTTPS (Tailscale or any mutually-reachable network) using the peer's
+hub bearer token. Peers are configured in ``peers.json`` in the palace dir:
+
+    {
+      "peers": [
+        {"name": "windows", "url": "https://host.example.ts.net", "token": "..."}
+      ]
+    }
+
+This module lives ABOVE the RFC 004 transport seam (mempalace/transport.py):
+peers come from the transport's membership snapshot and every wire call goes
+through its ``request``. Swapping the link (HTTPS bearer today, MeshGuard
+next) never touches the sync logic here.
+"""
+
+import logging
+
+from .transport import (
+    PEERS_FILENAME,  # noqa: F401 — re-exported for existing importers
+    TransportError,
+    get_transport,
+    http_request,
+    load_peers,
+)
+
+logger = logging.getLogger("mempalace.logsync")
+
+_PULL_BATCH = 500
+
+# Historical names, kept so existing importers (replica_sync, CLI, tests)
+# keep working: the error class moved below the seam with the wire code.
+SyncPeerError = TransportError
+__all__ = ["PEERS_FILENAME", "SyncPeerError", "load_peers", "sync_all", "sync_with_peer"]
+
+
+def _peer_get(base_url: str, token: str, path: str, params: dict = None) -> dict:
+    """Compat shim for raw url+token callers (replica_sync, CLI --peer)."""
+    return http_request(base_url, token, path, params)
+
+
+def sync_with_peer(ls, url: str, token: str = "") -> dict:
+    """One anti-entropy round against one peer. Returns pull stats.
+
+    Never partially applies an event: artifacts are fetched and folded
+    before the event that references them, and every apply is idempotent,
+    so a crash mid-round just means the next round re-pulls the tail.
+    """
+    remote = _peer_get(url, token, "/sync/version_vector")
+    remote_vector = remote.get("version_vector") or {}
+    remote_replica = remote.get("replica_id", "?")
+    local_vector = ls.version_vector()
+
+    pulled_events = 0
+    pulled_artifacts = 0
+    per_origin = {}
+
+    for origin, remote_top in remote_vector.items():
+        if origin == ls.replica_id:
+            continue  # nobody knows more of our own ops than we do
+        after = local_vector.get(origin, 0)
+        origin_pulled = 0
+        while after < remote_top:
+            batch = (
+                _peer_get(
+                    url,
+                    token,
+                    "/sync/ops",
+                    {"origin": origin, "after": after, "limit": _PULL_BATCH},
+                ).get("events")
+                or []
+            )
+            if not batch:
+                break  # peer's vector promised more than it served; stop cleanly
+            for event in batch:
+                for artifact_id in event.get("artifact_ids") or []:
+                    if ls.has_artifact(artifact_id):
+                        continue
+                    artifact = _peer_get(url, token, "/sync/artifact", {"id": artifact_id}).get(
+                        "artifact"
+                    )
+                    if not artifact:
+                        raise SyncPeerError(
+                            f"{url}: peer served event {event.get('id')!r} but not "
+                            f"its artifact {artifact_id!r}"
+                        )
+                    if ls.apply_remote_artifact(artifact):
+                        pulled_artifacts += 1
+                if ls.apply_remote_event(event):
+                    pulled_events += 1
+                    origin_pulled += 1
+                after = max(after, event.get("origin_seq") or 0)
+        if origin_pulled:
+            per_origin[origin] = origin_pulled
+
+    return {
+        "peer_url": url,
+        "peer_replica": remote_replica,
+        "pulled_events": pulled_events,
+        "pulled_artifacts": pulled_artifacts,
+        "per_origin": per_origin,
+        # The peer's vector at round start — consumers (the /sync/peers
+        # estate endpoint, PalaceMind's mesh view) compute drift from it.
+        "remote_version_vector": remote_vector,
+        # Node-profile advertisement riding the same fetch: the peer's own
+        # self-description plus profiles it relays for origins it knows
+        # (absent from pre-profile peers — consumers treat None as unknown).
+        "remote_profile": remote.get("profile"),
+        "remote_profiles": remote.get("profiles") or {},
+    }
+
+
+def sync_all(ls, palace_path: str, transport=None) -> list[dict]:
+    """One round against every peer in the transport's membership snapshot;
+    per-peer errors are reported, never raised — one dead peer must not
+    block the others (R1: only convergence waits)."""
+    transport = transport or get_transport(palace_path)
+    results = []
+    for peer in transport.peers():
+        name = peer.get("name") or peer.get("url") or "?"
+        try:
+            stats = sync_with_peer(ls, peer["url"], peer.get("token", ""))
+            stats["peer_name"] = name
+            results.append(stats)
+        except (TransportError, ValueError) as exc:
+            results.append({"peer_name": name, "peer_url": peer.get("url"), "error": str(exc)})
+    return results

mempalace/migrate.py

@@ -29,7 +29,7 @@ from collections import defaultdict
 from contextlib import closing
 from datetime import datetime
 
-from .backups import prune_backups
+from .backups import copy_palace_dir, prune_backups
 from .config import MempalaceConfig
 
 
@@ -244,22 +244,43 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
     # A plain count() is not enough: some 0.6.x -> 1.5.x migrated collections
     # are readable but silently drop upsert/delete operations. In that state,
     # migrate must rebuild from SQLite instead of returning "No migration needed."
+    #
+    # Preflight HNSW divergence before touching the collection at all: the
+    # #1222 SIGSEGV/panic class crashes on count() against a diverged
+    # segment, and a native crash can't be caught by the except Exception
+    # below -- a diverged palace must never reach col.count() in the first
+    # place. The already-diverged case degrades to exactly the same
+    # SQLite-extraction path the except branch below already falls back to.
+    from .backends.chroma import hnsw_capacity_status
+
     try:
-        col = ChromaBackend().get_collection(palace_path, "mempalace_drawers")
-        count = col.count()
-
-        if collection_write_roundtrip_works(col):
-            print(f"\n Palace is already readable and writable by chromadb {target_version}.")
-            print(f" {count} drawers found. No migration needed.")
-            return True
-
-        print(
-            f"\n Palace is readable by chromadb {target_version}, but write/delete verification failed."
-        )
-        print(" Rebuilding from SQLite to restore native write/delete behavior...")
+        capacity_info = hnsw_capacity_status(palace_path, "mempalace_drawers")
     except Exception:
-        print(f"\n Palace is NOT readable by chromadb {target_version}.")
+        capacity_info = {}
+
+    if capacity_info.get("diverged"):
+        print(
+            f"\n Palace is NOT readable by chromadb {target_version}: HNSW index diverged from SQLite."
+        )
+        print(f" ({capacity_info.get('message', 'divergence detected')})")
         print(" Extracting from SQLite directly...")
+    else:
+        try:
+            col = ChromaBackend().get_collection(palace_path, "mempalace_drawers")
+            count = col.count()
+
+            if collection_write_roundtrip_works(col):
+                print(f"\n Palace is already readable and writable by chromadb {target_version}.")
+                print(f" {count} drawers found. No migration needed.")
+                return True
+
+            print(
+                f"\n Palace is readable by chromadb {target_version}, but write/delete verification failed."
+            )
+            print(" Rebuilding from SQLite to restore native write/delete behavior...")
+        except Exception:
+            print(f"\n Palace is NOT readable by chromadb {target_version}.")
+            print(" Extracting from SQLite directly...")
 
     # Extract all drawers via raw SQL
     drawers = extract_drawers_from_sqlite(db_path)
@@ -284,7 +305,7 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
             print(f"      ROOM: {room:30} {count:5}")
 
     if dry_run:
-        print("\n  DRY RUN — no changes made.")
+        print("\n  DRY RUN -- no changes made.")
         print(f"  Would migrate {len(drawers)} drawers.")
         return True
 
@@ -295,7 +316,7 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     backup_path = f"{palace_path}.pre-migrate.{timestamp}"
     print(f"\n  Backing up to {backup_path}...")
-    shutil.copytree(palace_path, backup_path)
+    copy_palace_dir(palace_path, backup_path, symlinks=True, log=print)
 
     # Enforce backup retention so repeated migrations cannot fill the disk
     # with full-palace copies. The backup we just created is the newest, so
@@ -549,7 +570,7 @@ def migrate_wing_names(palace_path: str, dry_run: bool = False, confirm: bool =
     topic_renames = _plan_topics_by_wing_renames()
 
     if not d_updates and not c_updates and not topic_renames:
-        print("  All wing names are already normalized — nothing to migrate.")
+        print("  All wing names are already normalized -- nothing to migrate.")
         return False
 
     print("\n  Wing-name migration plan:")
@@ -565,7 +586,7 @@ def migrate_wing_names(palace_path: str, dry_run: bool = False, confirm: bool =
         print(f"    topics_by_wing: {len(topic_renames)} key(s) re-keyed")
 
     if dry_run:
-        print("\n  DRY RUN — no changes made.\n")
+        print("\n  DRY RUN -- no changes made.\n")
         return True
 
     if not confirm:

mempalace/miner.py

@@ -14,6 +14,7 @@ import shlex
 import hashlib
 import fnmatch
 import logging
+import stat
 from pathlib import Path
 from datetime import datetime
 from collections import defaultdict
@@ -47,6 +48,38 @@ from .ids import ID_RECIPE, make_drawer_id_from_chunk
 
 logger = logging.getLogger("mempalace_mcp")
 
+
+def _path_within_root(path: Path, root: Path) -> bool:
+    try:
+        path.expanduser().resolve().relative_to(root.expanduser().resolve())
+        return True
+    except (OSError, ValueError):
+        return False
+
+
+def _read_text_no_follow(filepath: Path, root: Path) -> Optional[str]:
+    if not _path_within_root(filepath, root):
+        return None
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    fd = -1
+    try:
+        fd = os.open(filepath, flags)
+        st = os.fstat(fd)
+        if not stat.S_ISREG(st.st_mode) or st.st_size > MAX_FILE_SIZE:
+            return None
+        with os.fdopen(fd, "r", encoding="utf-8", errors="replace") as f:
+            fd = -1
+            return f.read()
+    except OSError:
+        return None
+    finally:
+        if fd != -1:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+
+
 PHP_EXTENSIONS = {
     # Compound Blade templates such as ``view.blade.php`` are covered by the
     # final ``.php`` suffix.
@@ -100,6 +133,8 @@ READABLE_EXTENSIONS = {
     ".csv",
     ".sql",
     ".toml",
+    ".tex",
+    ".bib",
     # C# / .NET
     ".cs",
     ".csproj",
@@ -209,20 +244,16 @@ class GitignoreMatcher:
         self.base_dir = base_dir
         self.rules = rules
 
-    @classmethod
-    def from_dir(cls, dir_path: Path):
-        gitignore_path = dir_path / ".gitignore"
-        if not gitignore_path.is_file():
-            return None
-
-        try:
-            lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines()
-        except Exception:
-            return None
+    @staticmethod
+    def _parse_rules(lines):
+        """Parse an iterable of raw pattern strings into a list of rule dicts.
 
+        Shared by :meth:`from_dir` and :meth:`from_patterns` so both paths
+        stay in sync with the gitignore parsing semantics.
+        """
         rules = []
         for raw_line in lines:
-            line = raw_line.strip()
+            line = str(raw_line).strip()
             if not line:
                 continue
 
@@ -254,12 +285,46 @@ class GitignoreMatcher:
                     "negated": negated,
                 }
             )
+        return rules
 
-        if not rules:
+    @classmethod
+    def from_dir(cls, dir_path: Path):
+        gitignore_path = dir_path / ".gitignore"
+        if not gitignore_path.is_file():
             return None
 
+        try:
+            lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines()
+        except Exception:
+            return None
+
+        rules = cls._parse_rules(lines)
+        if not rules:
+            return None
         return cls(dir_path, rules)
 
+    @classmethod
+    def from_patterns(cls, base_dir: Path, patterns):
+        """Create a matcher from an explicit list of gitignore-inspired pattern strings.
+
+        Supports a gitignore-inspired subset of pattern syntax for
+        ``exclude_patterns`` in ``mempalace.yaml``.  Patterns are parsed by
+        the same rule parser used for ``.gitignore`` files, so familiar
+        constructs (trailing ``/`` for dir-only, leading ``/`` for anchoring,
+        ``!`` negation, ``**`` globs) work as expected.  Full gitignore
+        semantics are not guaranteed for every edge case.
+
+        ``patterns`` must be a list of strings.  A single string is coerced
+        to a one-element list for convenience.  Non-string entries are
+        converted via ``str()``.
+        """
+        if isinstance(patterns, str):
+            patterns = [patterns]
+        rules = cls._parse_rules(patterns)
+        if not rules:
+            return None
+        return cls(base_dir, rules)
+
     def matches(self, path: Path, is_dir: bool = None):
         try:
             relative = path.relative_to(self.base_dir).as_posix().strip("/")
@@ -387,6 +452,29 @@ def is_force_included(path: Path, project_path: Path, include_paths: set) -> boo
     return False
 
 
+def _apply_exclude_patterns_to_prescanned_files(
+    files: list, project_path: Path, exclude_patterns: list, include_ignored: list
+) -> list:
+    """Filter a caller-provided (already-scanned) file list by exclude_patterns.
+
+    scan_project() applies exclude_patterns itself, but a caller that passes
+    its own pre-scanned ``files`` list bypasses that -- so callers reusing a
+    scan (e.g. the init double-scan) still need per-project exclusions
+    applied here. force_include paths are preserved, matching scan_project's
+    own precedence.
+    """
+    include_paths = normalize_include_paths(include_ignored)
+    exclude_matcher = GitignoreMatcher.from_patterns(project_path, exclude_patterns)
+    if exclude_matcher is None:
+        return files
+    return [
+        file_path
+        for file_path in files
+        if is_force_included(file_path, project_path, include_paths)
+        or exclude_matcher.matches(file_path, is_dir=False) is not True
+    ]
+
+
 # =============================================================================
 # CONFIG
 # =============================================================================
@@ -545,13 +633,19 @@ def chunk_text(
         raise ValueError(f"chunk_size must be a positive int, got {chunk_size!r}")
     if not isinstance(chunk_overlap, int) or chunk_overlap < 0:
         raise ValueError(f"chunk_overlap must be a non-negative int, got {chunk_overlap!r}")
-    if chunk_overlap >= chunk_size:
-        # ``start = end - chunk_overlap`` would not advance (or would go
-        # backward) when overlap >= size, producing an infinite loop on
-        # any non-empty input.
+    if chunk_overlap > chunk_size // 2:
+        # The windowing loop pulls ``end`` back to a boundary only when that
+        # boundary is past ``start + chunk_size // 2`` (the rfind guards below),
+        # so a pulled chunk always spans more than ``chunk_size // 2`` chars.
+        # ``start = end - chunk_overlap`` therefore advances only while
+        # ``chunk_overlap <= chunk_size // 2``; a larger overlap makes ``start``
+        # stall or move backward and the loop spins forever on short-line
+        # content (#2056). The largest safe value is ``chunk_size // 2`` (half,
+        # rounded down for odd sizes), which still advances and stays allowed.
         raise ValueError(
-            f"chunk_overlap ({chunk_overlap}) must be less than chunk_size "
-            f"({chunk_size}); equality or greater would loop forever"
+            f"chunk_overlap ({chunk_overlap}) must be at most chunk_size // 2 "
+            f"({chunk_size // 2}); a larger overlap can loop forever on "
+            f"short-line content (#2056)"
         )
     if not isinstance(min_chunk_size, int) or min_chunk_size < 0:
         raise ValueError(f"min_chunk_size must be a non-negative int, got {min_chunk_size!r}")
@@ -565,6 +659,28 @@ def chunk_text(
     start = 0
     chunk_index = 0
 
+    # Running newline tallies for the 1-indexed (line_start, line_end)
+    # locators emitted below. These replace the previous per-chunk
+    # ``content.count("\n", 0, pos)`` full-prefix rescans, which were O(pos)
+    # each and O(N*K) over K chunks: a 287 MB / ~433k-chunk file scanned
+    # ~1.2e14 bytes and ran for days, indistinguishable from a hang (#2054).
+    # ``start`` and ``end`` each advance monotonically for any
+    # ``chunk_overlap < chunk_size // 2`` (the default 800/100 config and
+    # every sane override), so counting only the newly-scanned span is O(N)
+    # total. Each position sequence keeps its own anchor; a backward step
+    # (a paragraph-boundary pull under a pathological near-``chunk_size``
+    # overlap, or a negative index) falls back to the exact full-prefix
+    # count. That fallback is defensive and does not fire on a terminating
+    # mine: the current windowing cannot send a filed chunk's ``start``
+    # backward without also entering a pre-existing infinite loop (overlap
+    # >= chunk_size // 2), so the else-branches read as uncovered. They keep
+    # every value byte-identical to the old form if the windowing ever gains
+    # a loop guard that admits backward steps.
+    _nl_before_start = 0
+    _start_anchor = 0
+    _nl_before_end = 0
+    _end_anchor = 0
+
     while start < len(content):
         end = min(start + chunk_size, len(content))
 
@@ -583,13 +699,25 @@ def chunk_text(
             # Tier 6a — 1-indexed line range in the stripped source.
             # Approximate locator (±1 at boundaries is fine for "jump to
             # roughly here"); exact-quote positioning is a future tier.
-            # Use the bounds form of ``str.count`` (counts on the original
-            # string with start/end limits) instead of slicing — slicing
-            # would allocate a new substring per chunk and produce O(N^2)
-            # work on a 500MB file with 50K chunks. Per PR #1579 review
-            # (gemini-code-assist, medium priority).
-            line_start = content.count("\n", 0, start) + 1
-            line_end = content.count("\n", 0, end) + 1
+            # ``str.count`` with bounds (not slicing) still avoids allocating
+            # a substring per chunk, the original PR #1579 review concern
+            # (gemini-code-assist, medium priority). The incremental anchors
+            # additionally avoid rescanning the whole prefix each time, the
+            # actual O(N*K) cost (#2054). Two anchors because ``start`` and
+            # ``end`` are distinct monotonic sequences; a backward step
+            # re-derives the value exactly from position 0.
+            if start >= _start_anchor:
+                _nl_before_start += content.count("\n", _start_anchor, start)
+                _start_anchor = start
+                line_start = _nl_before_start + 1
+            else:
+                line_start = content.count("\n", 0, start) + 1
+            if end >= _end_anchor:
+                _nl_before_end += content.count("\n", _end_anchor, end)
+                _end_anchor = end
+                line_end = _nl_before_end + 1
+            else:
+                line_end = content.count("\n", 0, end) + 1
             chunks.append(
                 {
                     "content": chunk,
@@ -1341,9 +1469,8 @@ def process_file(
     if not dry_run and file_already_mined(collection, source_file, check_mtime=True):
         return 0, "general", None
 
-    try:
-        content = filepath.read_text(encoding="utf-8", errors="replace")
-    except OSError:
+    content = _read_text_no_follow(filepath, project_path)
+    if content is None:
         return 0, "general", None
 
     content = content.strip()
@@ -1497,6 +1624,7 @@ def scan_project(
     project_dir: str,
     respect_gitignore: bool = True,
     include_ignored: list = None,
+    exclude_patterns: list = None,
 ) -> list:
     """Return list of all readable file paths under ``project_dir``.
 
@@ -1509,6 +1637,7 @@ def scan_project(
     active_matchers = []
     matcher_cache = {}
     include_paths = normalize_include_paths(include_ignored)
+    exclude_matcher = GitignoreMatcher.from_patterns(project_path, exclude_patterns or [])
 
     for root, dirs, filenames in os.walk(project_path):
         root_path = Path(root)
@@ -1536,6 +1665,13 @@ def scan_project(
                 if is_force_included(root_path / d, project_path, include_paths)
                 or not is_gitignored(root_path / d, active_matchers, is_dir=True)
             ]
+        if exclude_matcher:
+            dirs[:] = [
+                d
+                for d in dirs
+                if is_force_included(root_path / d, project_path, include_paths)
+                or exclude_matcher.matches(root_path / d, is_dir=True) is not True
+            ]
 
         for filename in filenames:
             filepath = root_path / filename
@@ -1549,6 +1685,15 @@ def scan_project(
             if respect_gitignore and active_matchers and not force_include:
                 if is_gitignored(filepath, active_matchers, is_dir=False):
                     continue
+            # Skip files matching project-level exclude_patterns (mempalace.yaml).
+            # force_include paths bypass this check so callers can selectively
+            # re-include files that would otherwise be excluded.
+            if (
+                not force_include
+                and exclude_matcher
+                and exclude_matcher.matches(filepath, is_dir=False)
+            ):
+                continue
             # Skip symlinks — prevents following links to /dev/urandom, etc.
             if filepath.is_symlink():
                 rel = filepath.relative_to(project_path).as_posix()
@@ -1557,11 +1702,30 @@ def scan_project(
                 except OSError:
                     pass
                 continue
-            # Skip files exceeding size limit
+            # Skip files exceeding size limit, or those whose stat() raises
+            # (permission denied, racing delete, broken symlink that survived
+            # the earlier is_symlink check). Both branches log to stderr to
+            # match the SKIP: (symlink) line above; silent drops at this
+            # gate were the original #923 complaint.
             try:
-                if filepath.stat().st_size > MAX_FILE_SIZE:
+                file_size = filepath.stat().st_size
+                if file_size > MAX_FILE_SIZE:
+                    print(
+                        f"  SKIP: {filepath.name} ({file_size / (1024 * 1024):.1f} MB)"
+                        f" exceeds {MAX_FILE_SIZE // (1024 * 1024)} MB limit",
+                        file=sys.stderr,
+                    )
                     continue
-            except OSError:
+            except OSError as exc:
+                # Prefer ``exc.strerror`` so the path isn't duplicated in the
+                # output (PermissionError stringifies to
+                # ``[Errno 13] Permission denied: '<path>'`` and the path is
+                # already in the SKIP prefix). Falls back to the default repr
+                # when strerror is unset.
+                print(
+                    f"  SKIP: {filepath.name} (stat error: {exc.strerror or exc})",
+                    file=sys.stderr,
+                )
                 continue
             files.append(filepath)
     return files
@@ -1653,13 +1817,20 @@ def _mine_impl(
 
     wing = wing_override or config["wing"]
     rooms = config.get("rooms", [{"name": "general", "description": "All project files"}])
+    exclude_patterns = config.get("exclude_patterns", [])
 
     if files is None:
         files = scan_project(
             project_dir,
             respect_gitignore=respect_gitignore,
             include_ignored=include_ignored,
+            exclude_patterns=exclude_patterns,
         )
+    elif exclude_patterns:
+        files = _apply_exclude_patterns_to_prescanned_files(
+            files, project_path, exclude_patterns, include_ignored
+        )
+
     from .embedding import describe_device
 
     print(f"\n{'=' * 55}")
@@ -1672,7 +1843,7 @@ def _mine_impl(
     print(f"  Palace:  {palace_path}")
     print(f"  Device:  {describe_device()}")
     if dry_run:
-        print("  DRY RUN — nothing will be filed")
+        print("  DRY RUN -- nothing will be filed")
     if not respect_gitignore:
         print("  .gitignore: DISABLED")
     if include_ignored:
@@ -1742,12 +1913,15 @@ def _mine_impl(
                     break
 
         if not dry_run:
+            from .config import MempalaceConfig
+
+            graph_config = MempalaceConfig(palace_path=palace_path)
             # Cross-wing topic tunnels: after every file in this wing has been
             # processed, link this wing to any other wing that shares a
             # confirmed TOPIC label. Out of scope for v1: manifest-dependency
             # overlap, per-topic allow/deny lists, search-result surfacing.
             try:
-                tunnels_added = _compute_topic_tunnels_for_wing(wing)
+                tunnels_added = _compute_topic_tunnels_for_wing(wing, config=graph_config)
                 if tunnels_added:
                     print(f"\n  Topic tunnels: +{tunnels_added} cross-wing link(s)")
             except Exception as e:
@@ -1763,7 +1937,9 @@ def _mine_impl(
             # must never fail a mine; it's a derived analytic, not load-bearing
             # for the drawer write that already committed above.
             try:
-                hallways_created = compute_hallways_for_wing(wing, col=collection)
+                hallways_created = compute_hallways_for_wing(
+                    wing, col=collection, config=graph_config
+                )
                 if hallways_created:
                     print(f"\n  Hallways: +{len(hallways_created)} within-wing entity link(s)")
             except Exception as e:
@@ -1779,7 +1955,7 @@ def _mine_impl(
             # ``kind="entity"`` / ``kind="topic"``. Same fault-tolerance
             # pattern: never fail a mine over a derived analytic.
             try:
-                entity_tunnels_added = _compute_entity_tunnels_for_wing(wing)
+                entity_tunnels_added = _compute_entity_tunnels_for_wing(wing, config=graph_config)
                 if entity_tunnels_added:
                     print(f"\n  Entity tunnels: +{entity_tunnels_added} cross-wing entity link(s)")
             except Exception as e:
@@ -1903,7 +2079,7 @@ def _cleanup_mine_pid_file() -> None:
         pass
 
 
-def _compute_topic_tunnels_for_wing(wing: str) -> int:
+def _compute_topic_tunnels_for_wing(wing: str, config=None) -> int:
     """Drop tunnels between ``wing`` and every other wing that shares
     confirmed topics, honoring the ``topic_tunnel_min_count`` config knob.
 
@@ -1916,13 +2092,13 @@ def _compute_topic_tunnels_for_wing(wing: str) -> int:
     topics_map = get_topics_by_wing()
     if not topics_map or wing not in topics_map:
         return 0
-    cfg = MempalaceConfig()
+    cfg = config or MempalaceConfig()
     min_count = cfg.topic_tunnel_min_count
-    created = topic_tunnels_for_wing(wing, topics_map, min_count=min_count)
+    created = topic_tunnels_for_wing(wing, topics_map, min_count=min_count, config=cfg)
     return len(created)
 
 
-def _compute_entity_tunnels_for_wing(wing: str) -> int:
+def _compute_entity_tunnels_for_wing(wing: str, config=None) -> int:
     """Drop tunnels between ``wing`` and every other wing that shares an
     entity via the within-wing hallway primitive.
 
@@ -1940,10 +2116,10 @@ def _compute_entity_tunnels_for_wing(wing: str) -> int:
     from .hallways import list_hallways
     from .palace_graph import entity_tunnels_for_wing
 
-    hallways = list_hallways()
+    hallways = list_hallways(config=config)
     if not hallways:
         return 0
-    created = entity_tunnels_for_wing(wing, hallways)
+    created = entity_tunnels_for_wing(wing, hallways, config=config)
     return len(created)
 
 
@@ -1962,7 +2138,7 @@ def status(palace_path: str):
     un-bootstrapped collection, or an unexpected schema); the fallback also
     emits the state-specific guidance for absent/empty palaces.
     """
-    from .backends.chroma import _sqlite_wing_room_counts
+    from .backends.chroma import _sqlite_wing_room_counts, hnsw_capacity_status
 
     counts = _sqlite_wing_room_counts(palace_path, "mempalace_drawers")
     if counts is not None:
@@ -1974,6 +2150,17 @@ def status(palace_path: str):
     if col is None:
         return
 
+    # Preflight HNSW divergence before falling back to the ChromaDB client
+    # path: count() on a diverged segment can hit the #1222 SIGSEGV/panic
+    # class, which a try/except around count() cannot catch. This fallback
+    # only runs when the direct sqlite read above was unavailable, so it's
+    # the one place in this function that still touches count() directly.
+    capacity_info = hnsw_capacity_status(palace_path, "mempalace_drawers")
+    if capacity_info.get("diverged"):
+        print(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+        print("  Run `mempalace repair --mode from-sqlite --archive-existing` first.")
+        return
+
     # Count by wing and room — paginate to avoid SQLite "too many SQL
     # variables" error on large palaces (see #802, #850).
     total = col.count()
@@ -1996,7 +2183,7 @@ def status(palace_path: str):
 def _print_status(total: int, wing_rooms: dict[str, dict[str, int]]) -> None:
     """Render the wing/room histogram shared by both status code paths."""
     print(f"\n{'=' * 55}")
-    print(f"  MemPalace Status — {total} drawers")
+    print(f"  MemPalace Status -- {total} drawers")
     print(f"{'=' * 55}\n")
     for wing, rooms in sorted(wing_rooms.items()):
         print(f"  WING: {wing}")

mempalace/normalize.py

@@ -5,7 +5,8 @@ normalize.py — Convert any chat export format to MemPalace transcript format.
 Supported:
     - Plain text with > markers (pass through)
     - Claude.ai JSON export
-    - ChatGPT conversations.json
+    - ChatGPT conversations.json (a single conversation, or the top-level
+      array of them that a real data export ships)
     - Claude Code JSONL (with tool_use/tool_result block capture)
     - OpenAI Codex CLI JSONL
     - Gemini CLI JSONL (~/.gemini/tmp/<project_hash>/chats/session-*.jsonl)
@@ -21,6 +22,7 @@ No API key. No internet. Everything local.
 import json
 import os
 import re
+import stat
 from pathlib import Path
 from typing import Optional
 
@@ -113,22 +115,41 @@ def strip_noise(text: str) -> str:
     return text.strip()
 
 
+def _read_transcript_file(filepath: str) -> str:
+    """Read a transcript source file with the same safety checks normalize()
+    and normalize_conversations() both need: no symlinks, regular files only,
+    size-capped, BOM-tolerant.
+    """
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    if os.path.islink(filepath):
+        raise IOError(f"Could not read {filepath}: symlinked files are skipped")
+    fd = -1
+    try:
+        fd = os.open(filepath, flags)
+        file_stat = os.fstat(fd)
+        if not stat.S_ISREG(file_stat.st_mode):
+            raise IOError(f"Could not read {filepath}: not a regular file")
+        if file_stat.st_size > 500 * 1024 * 1024:  # 500 MB safety limit
+            raise IOError(f"File too large ({file_stat.st_size // (1024 * 1024)} MB): {filepath}")
+        with os.fdopen(fd, "r", encoding="utf-8-sig", errors="replace") as f:
+            fd = -1
+            return f.read()
+    except OSError as e:
+        raise IOError(f"Could not read {filepath}: {e}") from e
+    finally:
+        if fd != -1:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+
+
 def normalize(filepath: str) -> str:
     """
     Load a file and normalize to transcript format if it's a chat export.
     Plain text files pass through unchanged.
     """
-    try:
-        file_size = os.path.getsize(filepath)
-    except OSError as e:
-        raise IOError(f"Could not read {filepath}: {e}") from e
-    if file_size > 500 * 1024 * 1024:  # 500 MB safety limit
-        raise IOError(f"File too large ({file_size // (1024 * 1024)} MB): {filepath}")
-    try:
-        with open(filepath, "r", encoding="utf-8-sig", errors="replace") as f:
-            content = f.read()
-    except OSError as e:
-        raise IOError(f"Could not read {filepath}: {e}") from e
+    content = _read_transcript_file(filepath)
 
     if not content.strip():
         return content
@@ -150,40 +171,96 @@ def normalize(filepath: str) -> str:
     return content
 
 
+def normalize_conversations(filepath: str) -> list:
+    """Like normalize(), but keeps each conversation in a bundle export as a
+    separate string instead of joining them into one.
+
+    A Claude.ai privacy export packs every conversation into a single JSON
+    file, and normalize() joins them with "\\n\\n".join(...) into one blob.
+    That collapses conversation boundaries, so content-hash dedup keyed on
+    the whole file breaks the moment the bundle is re-exported with one new
+    conversation added — the file-level hash changes even though none of
+    the existing conversations did. This returns the pieces un-joined so
+    callers can hash and dedup per conversation instead.
+
+    A ChatGPT data export is a bundle for the same reason: its
+    ``conversations.json`` is an array of conversations, so it splits per
+    conversation too.
+
+    Non-bundle formats (a single Claude Code session, plain text, ...)
+    always normalize to one conversation, so this returns a one-element
+    list for those — identical dedup granularity to before.
+    """
+    content = _read_transcript_file(filepath)
+
+    if not content.strip():
+        return []
+
+    lines = content.split("\n")
+    if sum(1 for line in lines if line.strip().startswith(">")) >= 3:
+        return [content]
+
+    ext = Path(filepath).suffix.lower()
+    if ext in (".json", ".jsonl") or content.strip()[:1] in ("{", "["):
+        split = _try_normalize_json_split(content)
+        if split:
+            return split
+
+    return [content]
+
+
 def _try_normalize_json(content: str) -> Optional[str]:
-    """Try all known JSON chat schemas."""
+    """Try all known JSON chat schemas, joining a multi-conversation bundle
+    into one string. See ``_try_normalize_json_split`` for the unjoined form.
+    """
+    split = _try_normalize_json_split(content)
+    if split is None:
+        return None
+    return "\n\n".join(split)
+
+
+def _try_normalize_json_split(content: str) -> Optional[list]:
+    """Try all known JSON chat schemas, returning each conversation found as
+    a separate list entry (bundle formats) or a single-element list.
+    """
 
     normalized = _try_claude_code_jsonl(content)
     if normalized:
-        return normalized
+        return [normalized]
 
     normalized = _try_codex_jsonl(content)
     if normalized:
-        return normalized
+        return [normalized]
 
     normalized = _try_gemini_jsonl(content)
     if normalized:
-        return normalized
+        return [normalized]
 
     normalized = _try_pi_jsonl(content)
     if normalized:
-        return normalized
+        return [normalized]
 
     try:
         data = json.loads(content)
     except json.JSONDecodeError:
         return None
 
-    for parser in (
-        _try_gemini_json,
-        _try_claude_ai_json,
-        _try_chatgpt_json,
-        _try_continue_json,
-        _try_slack_json,
-    ):
+    normalized = _try_gemini_json(data)
+    if normalized:
+        return [normalized]
+
+    split = _try_claude_ai_json_split(data)
+    if split:
+        return split
+
+    split = _try_chatgpt_export_json_split(data)
+    if split:
+        return split
+
+    for parser in (_try_chatgpt_json, _try_continue_json, _try_slack_json):
         normalized = parser(data)
         if normalized:
-            return normalized
+            return [normalized]
 
     return None
 
@@ -497,6 +574,16 @@ def _try_gemini_json(data) -> Optional[str]:
 
 def _try_claude_ai_json(data) -> Optional[str]:
     """Claude.ai JSON export: flat messages list or privacy export with chat_messages."""
+    split = _try_claude_ai_json_split(data)
+    if split is None:
+        return None
+    return "\n\n".join(split)
+
+
+def _try_claude_ai_json_split(data) -> Optional[list]:
+    """Same as ``_try_claude_ai_json`` but keeps each conversation in a
+    privacy export as its own list entry instead of joining them.
+    """
     if isinstance(data, dict):
         data = data.get("messages", data.get("chat_messages", []))
     if not isinstance(data, list):
@@ -514,13 +601,13 @@ def _try_claude_ai_json(data) -> Optional[str]:
             if len(messages) >= 2:
                 transcripts.append(_messages_to_transcript(messages))
         if transcripts:
-            return "\n\n".join(transcripts)
+            return transcripts
         return None
 
     # Flat messages list
     messages = _collect_claude_messages(data)
     if len(messages) >= 2:
-        return _messages_to_transcript(messages)
+        return [_messages_to_transcript(messages)]
     return None
 
 
@@ -545,8 +632,14 @@ def _collect_claude_messages(items) -> list:
 
 
 def _try_chatgpt_json(data) -> Optional[str]:
-    """ChatGPT conversations.json with mapping tree."""
-    if not isinstance(data, dict) or "mapping" not in data:
+    """ChatGPT conversations.json with mapping tree.
+
+    Every nested shape is type-checked rather than assumed: this parser is
+    reached from ``_try_chatgpt_export_json_split`` for each element of any
+    top-level JSON array, so it must return None on unrelated payloads that
+    merely carry a ``mapping`` key instead of raising.
+    """
+    if not isinstance(data, dict) or not isinstance(data.get("mapping"), dict):
         return None
     mapping = data["mapping"]
     messages = []
@@ -554,6 +647,8 @@ def _try_chatgpt_json(data) -> Optional[str]:
     root_id = None
     fallback_root = None
     for node_id, node in mapping.items():
+        if not isinstance(node, dict):
+            continue
         if node.get("parent") is None:
             if node.get("message") is None:
                 root_id = node_id
@@ -567,24 +662,65 @@ def _try_chatgpt_json(data) -> Optional[str]:
         visited = set()
         while current_id and current_id not in visited:
             visited.add(current_id)
-            node = mapping.get(current_id, {})
+            node = mapping.get(current_id)
+            if not isinstance(node, dict):
+                break
             msg = node.get("message")
-            if msg:
-                role = msg.get("author", {}).get("role", "")
+            if isinstance(msg, dict):
+                author = msg.get("author")
+                role = author.get("role", "") if isinstance(author, dict) else ""
                 content = msg.get("content", {})
-                parts = content.get("parts", []) if isinstance(content, dict) else []
+                parts = content.get("parts") if isinstance(content, dict) else None
+                if not isinstance(parts, list):
+                    parts = []
                 text = " ".join(str(p) for p in parts if isinstance(p, str) and p).strip()
                 if role == "user" and text:
                     messages.append(("user", text))
                 elif role == "assistant" and text:
                     messages.append(("assistant", text))
-            children = node.get("children", [])
-            current_id = children[0] if children else None
+            children = node.get("children")
+            next_id = children[0] if isinstance(children, list) and children else None
+            # Node ids index a dict and a visited set, so anything unhashable
+            # (a nested child object rather than an id) ends the walk.
+            current_id = next_id if isinstance(next_id, str) else None
     if len(messages) >= 2:
         return _messages_to_transcript(messages)
     return None
 
 
+def _try_chatgpt_export_json_split(data) -> Optional[list]:
+    """ChatGPT data export: top-level array of conversation objects.
+
+    The ``conversations.json`` OpenAI ships is an *array*, while
+    ``_try_chatgpt_json`` handles the single conversation object inside it.
+    Without this the whole export falls through to the plain-text path and is
+    chunked as raw JSON: the drawers hold serialized structure sliced at
+    arbitrary offsets, and every speaker turn is gone.
+
+    Each conversation is kept as its own segment rather than concatenated, so
+    per-conversation dedup survives a re-export (see ``normalize_conversations``);
+    the joined form is reached through ``_try_normalize_json``.
+
+    Runs after ``_try_gemini_json`` and ``_try_claude_ai_json_split`` and before
+    the ``_try_chatgpt_json``/``_try_continue_json``/``_try_slack_json`` loop.
+    That position is safe in both directions: Gemini requires a ``role="model"``
+    entry and Claude.ai requires ``chat_messages``/``messages`` on the first
+    element, neither of which a ChatGPT conversation object has, while Slack
+    entries carry no ``mapping`` and Continue.dev sessions are not arrays at
+    all, so this parser declines them and they fall through unchanged.
+    """
+    if not isinstance(data, list):
+        return None
+
+    transcripts = []
+    for convo in data:
+        transcript = _try_chatgpt_json(convo)
+        if transcript:
+            transcripts.append(transcript)
+    # None, not [], so an array of other JSON still reaches the later parsers.
+    return transcripts or None
+
+
 def _try_slack_json(data) -> Optional[str]:
     """
     Slack channel export: [{"type": "message", "user": "...", "text": "..."}]

mempalace/onboarding.py

@@ -57,7 +57,7 @@ DEFAULT_WINGS = {
 
 
 def _hr():
-    print(f"\n{'─' * 58}")
+    print(f"\n{'-' * 58}")
 
 
 def _header(text):
@@ -121,9 +121,9 @@ def _ask_mode() -> str:
 """)
     print("  How are you using MemPalace?")
     print()
-    print("    [1]  Work     — notes, projects, clients, colleagues, decisions")
-    print("    [2]  Personal — diary, family, health, relationships, reflections")
-    print("    [3]  Both     — personal and professional mixed")
+    print("    [1]  Work     -- notes, projects, clients, colleagues, decisions")
+    print("    [2]  Personal -- diary, family, health, relationships, reflections")
+    print("    [3]  Both     -- personal and professional mixed")
     print()
 
     while True:
@@ -430,7 +430,7 @@ def run_onboarding(
             print()
             if _yn("  Add any of these to your registry?"):
                 for e in detected:
-                    ans = input(f"    {e['name']} — (p)erson, (s)kip? ").strip().lower()
+                    ans = input(f"    {e['name']} -- (p)erson, (s)kip? ").strip().lower()
                     if ans == "p":
                         rel = input(f"    Relationship/role for {e['name']}? ").strip()
                         ctx = (
@@ -439,7 +439,7 @@ def run_onboarding(
                             else (
                                 "work"
                                 if mode == "work"
-                                else input("    Context — (p)ersonal or (w)ork? ")
+                                else input("    Context -- (p)ersonal or (w)ork? ")
                                 .strip()
                                 .lower()
                                 .replace("w", "work")

mempalace/palace.py

@@ -11,6 +11,7 @@ import os
 import re
 import sys
 import threading
+import time
 from typing import Optional
 
 from .backends import (
@@ -26,7 +27,11 @@ from .backends import (
     resolve_backend_for_palace,
 )
 from .backends.embedding_wrapper import EmbeddingCollection
-from .entity_detector import _apply_known_systems_prepass, _get_coca_filter
+from .entity_detector import (
+    _apply_known_systems_prepass,
+    _collapse_long_ascii_runs,
+    _get_coca_filter,
+)
 
 logger = logging.getLogger("mempalace_mcp")
 
@@ -76,6 +81,23 @@ NORMALIZE_VERSION = 2
 _VALIDATED_IDENTITY: set = set()
 
 
+def clear_validated_embedder_identity(palace_path: Optional[str] = None) -> None:
+    """Drop cached embedder-identity verdicts so the next open re-checks.
+
+    Read-only opens of an empty collection can mark a key as validated without
+    recording identity on disk (``create=False``). When MCP later promotes that
+    reader to a writable owner, the writable open must re-run enforcement so
+    the first drawers still get labelled with the active model.
+    """
+    if palace_path is None:
+        _VALIDATED_IDENTITY.clear()
+        return
+    palace_key = str(palace_path)
+    stale = [key for key in _VALIDATED_IDENTITY if key and key[0] == palace_key]
+    for key in stale:
+        _VALIDATED_IDENTITY.discard(key)
+
+
 def _enforce_embedder_identity(collection, palace_path, collection_name, *, create) -> None:
     """Check (and, for a brand-new collection, record) embedder identity (RFC 001).
 
@@ -138,10 +160,29 @@ def _enforce_embedder_identity(collection, palace_path, collection_name, *, crea
         return
 
     if state == "unknown" and stored is None:
+        # Preflight HNSW divergence before touching count(): this is the
+        # universal chokepoint every tool passes through via
+        # get_collection(), and count() on a diverged segment can raise
+        # chromadb's rust-level pyo3_runtime.PanicException or hard-segfault
+        # (#1222) -- neither of which the except Exception below can catch,
+        # since a native crash takes the whole process down regardless of
+        # any Python try/except. A diverged palace must never reach count()
+        # here; this bookkeeping-only identity check simply skips itself
+        # (count treated as unknown, matching the except-Exception fallback
+        # already below) rather than risk the read.
         try:
-            count = collection.count()
+            from .backends.chroma import hnsw_capacity_status
+
+            diverged = hnsw_capacity_status(str(palace_path), str(collection_name)).get("diverged")
         except Exception:
+            diverged = False
+        if diverged:
             count = None
+        else:
+            try:
+                count = collection.count()
+            except Exception:
+                count = None
         if count == 0:
             if create:
                 try:
@@ -165,10 +206,15 @@ def get_collection(
     collection_name: Optional[str] = None,
     create: bool = True,
     backend: Optional[str] = None,
+    read_only: bool = False,
     _skip_identity_check: bool = False,
 ):
     """Get the palace collection through the backend layer.
 
+    ``read_only=True`` asks local backends to open storage without schema
+    initialization, migrations, or metadata writes. Backends that support a
+    genuine read-only mode receive it through the backend ``options`` mapping.
+
     ``_skip_identity_check`` bypasses the embedder-identity enforcement so the
     ``set-embedder`` override path can open a palace whose recorded model
     differs from the current one (the very state it exists to repair).
@@ -179,20 +225,52 @@ def get_collection(
         collection_name = get_configured_collection_name()
     backend_obj = get_backend_for_palace(palace_path, explicit=backend)
     palace_ref = PalaceRef(id=palace_path, local_path=palace_path)
+    backend_options = {"read_only": True} if read_only else None
+    preferred_kwargs = {
+        "palace": palace_ref,
+        "collection_name": collection_name,
+        "create": create,
+    }
+    if backend_options is not None:
+        preferred_kwargs["options"] = backend_options
     try:
-        collection = backend_obj.get_collection(
-            palace=palace_ref,
-            collection_name=collection_name,
-            create=create,
-        )
+        collection = backend_obj.get_collection(**preferred_kwargs)
     except TypeError as exc:
-        if "unexpected keyword argument 'palace'" not in str(exc):
+        msg = str(exc)
+        # Plugin backends may still use the pre-options signature. Drop
+        # ``options`` first so read_only degrades gracefully instead of
+        # hard-failing TypeError on third-party entry points.
+        if backend_options is not None and "options" in msg:
+            preferred_kwargs.pop("options", None)
+            try:
+                collection = backend_obj.get_collection(**preferred_kwargs)
+            except TypeError as nested:
+                if "unexpected keyword argument 'palace'" not in str(nested):
+                    raise
+                collection = backend_obj.get_collection(
+                    palace_path,
+                    collection_name=collection_name,
+                    create=create,
+                )
+        elif "unexpected keyword argument 'palace'" not in msg:
             raise
-        collection = backend_obj.get_collection(
-            palace_path,
-            collection_name=collection_name,
-            create=create,
-        )
+        else:
+            legacy_kwargs = {
+                "collection_name": collection_name,
+                "create": create,
+            }
+            if backend_options is not None:
+                legacy_kwargs["options"] = backend_options
+            try:
+                collection = backend_obj.get_collection(palace_path, **legacy_kwargs)
+            except TypeError as nested:
+                if backend_options is None or "options" not in str(nested):
+                    raise
+                collection = backend_obj.get_collection(
+                    palace_path,
+                    collection_name=collection_name,
+                    create=create,
+                )
     if "requires_explicit_embeddings" in getattr(backend_obj, "capabilities", frozenset()):
         collection = EmbeddingCollection(collection)
     if not _skip_identity_check:
@@ -332,6 +410,29 @@ def resolve_backend_name(palace_path: str, explicit: Optional[str] = None) -> st
     return selected
 
 
+_MULTI_PROCESS_WRITER_BACKENDS = frozenset({"pgvector", "qdrant"})
+
+
+def backend_requires_single_writer(backend_name: str) -> bool:
+    """Return whether a backend needs one process-lifetime writer owner.
+
+    Local file-backed backends cannot safely coordinate independent long-lived
+    clients by serializing only individual calls: each process may retain
+    SQLite/WAL, FTS, or vector-index state across operations. Unknown and
+    plugin backends are treated conservatively. Only backends whose storage
+    service is explicitly responsible for cross-process concurrency opt out.
+    """
+    normalized = backend_name.strip().lower()
+    if normalized == "milvus":
+        # Only embedded Milvus Lite is local single-writer storage. A remote
+        # Milvus server or Zilliz Cloud coordinates concurrent clients itself.
+        from .backends.milvus import milvus_uri_is_server
+        from .config import MempalaceConfig
+
+        return not milvus_uri_is_server(MempalaceConfig().milvus_uri)
+    return normalized not in _MULTI_PROCESS_WRITER_BACKENDS
+
+
 def get_backend_for_palace(palace_path: str, explicit: Optional[str] = None):
     """Return the resolved backend instance for ``palace_path``."""
     return get_backend(resolve_backend_name(palace_path, explicit=explicit))
@@ -530,9 +631,12 @@ def _candidate_entity_words(text: str) -> list:
             except re.error:
                 continue
         _CANDIDATE_RX_CACHE = rxs
+    # Defuse ReDoS on long ASCII blobs before matching (#2063); see
+    # entity_detector._collapse_long_ascii_runs.
+    stripped = _collapse_long_ascii_runs(text)
     words = []
     for rx in _CANDIDATE_RX_CACHE:
-        words.extend(rx.findall(text))
+        words.extend(rx.findall(stripped))
     return words
 
 
@@ -725,6 +829,7 @@ def mine_lock(source_file: str):
     Prevents multiple agents from mining the same file simultaneously,
     which causes duplicate drawers when the delete+insert cycle interleaves.
     """
+    _maybe_reap_stale_mine_locks()
     lock_path = _mine_lock_path(source_file)
     lf = _acquire_mine_lock_file(lock_path)
     try:
@@ -904,6 +1009,92 @@ def _cleanup_mine_lock_file(lock_path: str) -> None:
             lf.close()
 
 
+def reap_stale_mine_locks(*, min_age_seconds: int = 3600) -> tuple[int, int]:
+    """Best-effort garbage collection for orphaned per-source-file mine locks.
+
+    ``_cleanup_mine_lock_file`` reclaims a lock file correctly on the happy
+    path (see its docstring) — but only for the *specific* lock a
+    :func:`mine_lock` context manager just released. A process that dies
+    before reaching its own ``finally`` block (killed, crashed, force-quit,
+    host reboot) never runs that cleanup, and nothing else in this codebase
+    later revisits that lock file. Locks in ``~/.mempalace/locks/`` can
+    accumulate unboundedly over time as a result — one long-lived
+    installation was found with 5,636 stale entries, the oldest several
+    months old, none held by any live process (confirmed via ``lsof``).
+
+    This reuses :func:`_cleanup_mine_lock_file` itself for the actual
+    removal — same nonblocking-flock-reacquire safety mechanism, same
+    Windows/POSIX handling, no duplicated locking logic. A lock is only
+    ever removed after *this* process re-acquires it, so anything
+    genuinely held by a live process is left untouched regardless of
+    ``min_age_seconds``. ``min_age_seconds`` is a courtesy throttle only —
+    it avoids racing a lock that was *just* released and may still be
+    mid-rendezvous with a waiter on the same pathname; it is not a
+    substitute for the flock check, which is what actually makes removal
+    safe.
+
+    Skips ``mine_palace_*.lock`` files — those belong to the newer
+    palace-level :func:`mine_palace_lock` and have their own
+    lifecycle/holder tracking; this targets only the per-source-file locks
+    :func:`mine_lock` creates via :func:`_mine_lock_path`.
+
+    Returns ``(reaped, skipped)`` counts, for logging/testing — callers
+    don't need to act on them.
+    """
+    lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks")
+    try:
+        entries = os.listdir(lock_dir)
+    except OSError:
+        return 0, 0
+
+    now = time.time()
+    reaped = 0
+    skipped = 0
+    for name in entries:
+        if not name.endswith(".lock") or name.startswith("mine_palace_"):
+            continue
+        lock_path = os.path.join(lock_dir, name)
+        try:
+            if now - os.path.getmtime(lock_path) < min_age_seconds:
+                continue
+        except OSError:
+            continue
+        _cleanup_mine_lock_file(lock_path)
+        if os.path.exists(lock_path):
+            skipped += 1
+        else:
+            reaped += 1
+    return reaped, skipped
+
+
+_LOCK_REAP_INTERVAL_SECONDS = 900  # 15 minutes between opportunistic sweeps
+
+
+def _maybe_reap_stale_mine_locks() -> None:
+    """Throttled, opportunistic call site for :func:`reap_stale_mine_locks`.
+
+    Runs at most once per ``_LOCK_REAP_INTERVAL_SECONDS``, piggybacking on
+    the natural cadence of mine operations rather than requiring a
+    background thread, a scheduled task, or any new CLI surface. Failures
+    are swallowed — lock maintenance must never be allowed to break an
+    actual mine.
+    """
+    lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks")
+    marker = os.path.join(lock_dir, ".last_reap")
+    try:
+        if (
+            os.path.exists(marker)
+            and time.time() - os.path.getmtime(marker) < _LOCK_REAP_INTERVAL_SECONDS
+        ):
+            return
+        os.makedirs(lock_dir, exist_ok=True)
+        open(marker, "a").close()
+        os.utime(marker, None)
+        reap_stale_mine_locks()
+    except Exception:
+        logger.debug("Opportunistic mine-lock reap failed", exc_info=True)
+
+
 class MineAlreadyRunning(RuntimeError):
     """Raised when another `mempalace mine` already holds the per-palace lock."""
 
@@ -928,12 +1119,19 @@ def _validate_palace_fts5_after_mine(palace_path: str) -> None:
     Reuses the same primitive that `cmd_repair` already runs as preflight so the
     operator sees the same recovery banner regardless of which command surfaces
     the bug.
+
+    An isolated FTS5 inverted-index corruption (the common case after a
+    killed-mid-write mine, #1596) is auto-healed in place first via the same
+    `maybe_autoheal_fts5_index` rebuild `cmd_repair` already runs as its own
+    preflight step — so a normal `mine` self-heals the recoverable case
+    instead of forcing the operator to run `mempalace repair` by hand for a
+    derived index that regenerates from intact content.
     """
     if resolve_backend_name(palace_path) != "chroma":
         return
 
     # Defer-import: keeps the repair module graph out of mine's hot import path.
-    from .repair import _close_chroma_handles, sqlite_integrity_errors
+    from .repair import _close_chroma_handles, maybe_autoheal_fts5_index, sqlite_integrity_errors
 
     # Pass the live singleton so the writer's cached PersistentClient actually
     # gets closed and WAL flushes before the read-only sqlite3 re-open.
@@ -943,49 +1141,89 @@ def _validate_palace_fts5_after_mine(palace_path: str) -> None:
     _close_chroma_handles(palace_path, backend=_DEFAULT_BACKEND)
 
     errors = sqlite_integrity_errors(palace_path)
+    if errors:
+        # progress=logger.info, not the default print: this runs inside the
+        # MCP server process too (mcp_server.tool_mine -> miner.mine), where
+        # stdout is the JSON-RPC transport -- a stray print() here would
+        # corrupt the protocol stream and crash the connection.
+        errors = maybe_autoheal_fts5_index(palace_path, errors, progress=logger.info)
     if errors:
         raise MineValidationError(palace_path, errors)
 
 
-# Per-thread record of palaces this thread already holds the lock for. Used by
-# `mine_palace_lock` to short-circuit re-entrant acquisition from the same
-# thread (e.g. miner.mine() acquires the outer lock then calls
+# Process-wide record of palaces this PROCESS already holds the lock for. Used
+# by `mine_palace_lock` to short-circuit re-entrant acquisition from the same
+# process (e.g. miner.mine() acquires the outer lock then calls
 # ChromaCollection.upsert which now also tries to acquire). Without this guard
 # the inner call would block on its own outer flock (Linux fcntl locks are per
-# open file description, so a same-thread second open of the lock file is a
-# distinct lock and self-deadlocks).
+# open file description, so a second open of the lock file from the same process
+# is a distinct lock and self-conflicts / EWOULDBLOCKs).
 #
-# The holder set is tagged with ``pid`` so that a forked child does NOT
-# inherit re-entrant credit from its parent: the OS-level flock IS NOT
-# inherited as a "we hold it" semantically — the child must reacquire — but
-# Python's ``threading.local`` IS inherited across fork. The pid check
-# clears stale state so a forked child correctly hits the fcntl path.
-_palace_lock_holders = threading.local()
+# This MUST be process-wide, not thread-local: the MCP HTTP transport
+# (ThreadingHTTPServer) acquires the long-lived writer-lease on one thread
+# (`mcp_server._acquire_mcp_writer_lock`) but dispatches each write request on a
+# different worker thread. A thread-local guard makes those handlers fail to see
+# the process-held lease, re-acquire the flock, and self-conflict
+# ("palace ... is held by PID <self>"). flock is per-process and HTTP writes are
+# serialized by `_HTTP_REQUEST_LOCK`, so the process is the correct re-entrancy
+# boundary.
+#
+# The holder set is tagged with ``pid`` so that a forked child does NOT inherit
+# re-entrant credit from its parent: the OS-level flock IS NOT inherited as a
+# "we hold it" semantically — the child must reacquire. The pid check clears
+# stale state so a forked child correctly hits the fcntl path. Access is guarded
+# by ``_palace_lock_guard`` because the set is now shared across threads.
+#
+# Fork safety: ``_palace_lock_guard`` is a real ``threading.Lock``, so a child
+# forked while another thread held it would inherit it locked (the holder thread
+# does not exist in the child) and deadlock on the next acquire. An at-fork
+# handler (registered below) replaces the guard with a fresh unlocked lock and
+# clears state in the child, which must reacquire the flock anyway.
+_palace_lock_guard = threading.Lock()
+_palace_lock_pid = None
+_palace_lock_keys = set()
 
 
-def _holder_state():
-    """Return the per-thread (pid, keys) record, refreshing after fork."""
-    keys = getattr(_palace_lock_holders, "keys", None)
-    pid = getattr(_palace_lock_holders, "pid", None)
+def _reset_palace_lock_state_after_fork() -> None:
+    """Reset lock state in a forked child to avoid an inherited-locked deadlock."""
+    global _palace_lock_guard, _palace_lock_pid, _palace_lock_keys
+    _palace_lock_guard = threading.Lock()
+    _palace_lock_keys = set()
+    _palace_lock_pid = os.getpid()
+
+
+# Availability: Unix (no-op elsewhere — Windows has no fork()).
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=_reset_palace_lock_state_after_fork)
+
+
+def _holder_keys_locked():
+    """Return the process-wide held-key set, refreshing after fork.
+
+    Caller MUST hold ``_palace_lock_guard``.
+    """
+    global _palace_lock_pid, _palace_lock_keys
     current_pid = os.getpid()
-    if keys is None or pid != current_pid:
-        keys = set()
-        _palace_lock_holders.keys = keys
-        _palace_lock_holders.pid = current_pid
-    return keys
+    if _palace_lock_pid != current_pid:
+        _palace_lock_keys = set()
+        _palace_lock_pid = current_pid
+    return _palace_lock_keys
 
 
-def _held_by_this_thread(lock_key: str) -> bool:
-    """Return True if this thread already holds ``mine_palace_lock`` for ``lock_key``."""
-    return lock_key in _holder_state()
+def _held_by_this_process(lock_key: str) -> bool:
+    """Return True if this process already holds ``mine_palace_lock`` for ``lock_key``."""
+    with _palace_lock_guard:
+        return lock_key in _holder_keys_locked()
 
 
 def _mark_held(lock_key: str) -> None:
-    _holder_state().add(lock_key)
+    with _palace_lock_guard:
+        _holder_keys_locked().add(lock_key)
 
 
 def _mark_released(lock_key: str) -> None:
-    _holder_state().discard(lock_key)
+    with _palace_lock_guard:
+        _holder_keys_locked().discard(lock_key)
 
 
 def _format_lock_holder(content: str) -> str:
@@ -1064,11 +1302,13 @@ def mine_palace_lock(palace_path: str):
     raise MineAlreadyRunning so the caller can exit cleanly instead of
     piling up as a waiting worker.
 
-    Re-entrant: if the current thread already holds the lock for the same
+    Re-entrant: if the current process already holds the lock for the same
     palace, the context manager passes through without re-acquiring. This
     lets ChromaCollection write methods (which acquire the lock themselves
     to protect MCP/direct callers) compose with miner.mine() (which holds
-    the outer lock for the entire mine pipeline) without self-deadlock.
+    the outer lock for the entire mine pipeline) without self-deadlock, and
+    lets the threaded MCP HTTP transport write from a worker thread while the
+    long-lived writer-lease is held on another thread of the same process.
     """
     lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks")
     os.makedirs(lock_dir, exist_ok=True)
@@ -1077,8 +1317,8 @@ def mine_palace_lock(palace_path: str):
     palace_key = hashlib.sha256(lock_key_source.encode()).hexdigest()[:16]
     lock_path = os.path.join(lock_dir, f"mine_palace_{palace_key}.lock")
 
-    if _held_by_this_thread(palace_key):
-        # Same thread already holds the lock for this palace — pass through.
+    if _held_by_this_process(palace_key):
+        # This process already holds the lock for this palace — pass through.
         yield
         return
 
@@ -1129,8 +1369,14 @@ def mine_palace_lock(palace_path: str):
                 ) from exc
         # Record our own identity for any later contender's diagnostic message.
         _write_lock_holder(lf)
-        _mark_held(palace_key)
+        # Mark the hold from inside the try so it always pairs with
+        # _mark_released. If it sat before the try, an async exception
+        # (a SIGINT/KeyboardInterrupt) landing in the gap would orphan
+        # palace_key in the holder set while the outer finally frees the
+        # flock, so the in-memory hold would outlive the OS lock and a later
+        # re-entrant acquire would pass through and write without the flock.
         try:
+            _mark_held(palace_key)
             yield
         finally:
             _mark_released(palace_key)
@@ -1159,10 +1405,26 @@ mine_global_lock = mine_palace_lock
 
 
 def _metadata_matches_extract_mode(meta: dict, extract_mode: Optional[str]) -> bool:
+    """Scope a drawer to a convo-miner extraction mode.
+
+    A missing ``extract_mode`` is treated as a legacy exchange-mode row
+    ONLY when the drawer is otherwise convo_miner's own (no ``ingest_mode``
+    at all -- pre-``ingest_mode``-schema convo drawers -- or the convo
+    miner's own ``"convos"`` tag). A drawer from a different producer that
+    never set ``extract_mode`` because it was never meant to carry one --
+    e.g. the sweeper's ``ingest_mode="sweep"`` rows -- must not match: the
+    legacy-compat rule otherwise scoops every sweeper drawer for a shared
+    transcript into convo_miner's default "exchange" purge/idempotency
+    scope and silently deletes them on the very next re-mine (#104).
+    """
     if extract_mode is None:
         return True
     stored_mode = meta.get("extract_mode")
-    return stored_mode == extract_mode or (extract_mode == "exchange" and stored_mode is None)
+    if stored_mode == extract_mode:
+        return True
+    if stored_mode is not None:
+        return False
+    return extract_mode == "exchange" and meta.get("ingest_mode") in (None, "convos")
 
 
 def file_already_mined(
@@ -1179,9 +1441,15 @@ def file_already_mined(
         schema (triggers silent rebuild after a normalization upgrade)
       - `check_mtime=True` and the file's mtime differs from the stored one
 
-    When check_mtime=True (used by project miner), also re-mines on content
-    change. When check_mtime=False (used by convo miner), transcripts are
-    assumed immutable, so only the version gate triggers a rebuild.
+    When check_mtime=True (used by the project miner, and by the convo
+    miner's in-lock recheck), also re-mines on content change. Conversation
+    transcripts are NOT assumed immutable: a Claude Code session keeps
+    appending to its own file while active, and /compact or /clear can
+    rewrite one in place. The convo miner's bulk skip-check uses
+    prefetch_mined_set()'s stored mtimes instead of calling this function
+    per file (same mtime-aware decision, without the O(n) per-file query
+    cost); this function's check_mtime=True path remains its per-file,
+    lock-held race-condition recheck.
 
     When extract_mode is set (used by convo miner), idempotency is scoped to
     that extraction mode so exchange-mode and general-mode drawers can coexist
@@ -1238,42 +1506,25 @@ def file_already_mined(
         return False
 
 
-def bulk_check_mined(collection) -> dict[str, float]:
-    """Pre-fetch source_file/source_mtime pairs for all documents in the collection.
+def prefetch_mined_set(
+    collection, extract_mode: Optional[str] = None
+) -> dict[str, Optional[float]]:
+    """Pre-fetch source_file -> stored source_mtime for files already mined
+    at the current NORMALIZE_VERSION, in one bulk pass instead of one
+    ChromaDB query per file.
 
-    Returns a dict mapping source_file -> source_mtime (as float) for every
-    document that has both fields.  Callers can check membership and compare
-    mtimes locally instead of issuing one ChromaDB query per file.
-
-    Fetches the full collection in paginated batches (like palace_graph.py)
-    since a WHERE-IN filter on thousands of paths is not supported by ChromaDB.
-    """
-    mined: dict[str, float] = {}
-    try:
-        total = collection.count()
-        offset = 0
-        while offset < total:
-            batch = collection.get(limit=1000, offset=offset, include=["metadatas"])
-            for meta in batch["metadatas"]:
-                src = meta.get("source_file")
-                mtime = meta.get("source_mtime")
-                if src and mtime is not None:
-                    mined[src] = float(mtime)
-            if not batch["ids"]:
-                break
-            offset += len(batch["ids"])
-    except Exception:
-        logger.warning("bulk_check_mined: partial fetch, %d files loaded", len(mined))
-    return mined
-
-
-def prefetch_mined_set(collection, extract_mode: Optional[str] = None) -> set[str]:
-    """Pre-fetch the set of source_files already mined at the current NORMALIZE_VERSION.
-
-    Mirrors file_already_mined()'s version-gate semantics (check_mtime=False
-    branch) but in one bulk pass instead of one ChromaDB query per file.
-    Returns a set of source_file paths whose stored drawers are at or above
-    NORMALIZE_VERSION; callers do `if path in result_set: skip`.
+    Return type is a dict rather than a bare set so callers get mtime
+    awareness "for free": conversation transcripts are not immutable once
+    mined (a Claude Code session keeps appending to the same file while
+    active, and /compact or /clear can rewrite one in place), so "we've
+    seen this source_file before" is not suffient to skip it -- the caller
+    must also confirm its current on-disk mtime still matches what was
+    stored. `if src in mined_set` still means the same thing as the old
+    set-based return (dict `in` checks keys); a caller that wants staleness
+    detection reads `mined_set[src]` and compares against
+    os.path.getmtime(src) itself. `None` means either no mtime was ever
+    stored (drawers written before this field existed) or getmtime failed
+    when the drawer was written -- both should be treated as stale.
 
     When extract_mode is set, mirrors file_already_mined(..., extract_mode=...)
     so conversation mines skip per extraction mode rather than per source file.
@@ -1283,7 +1534,7 @@ def prefetch_mined_set(collection, extract_mode: Optional[str] = None) -> set[st
     palace, making a 2000-file sweep take >1h of pure skip-checking. This
     helper drops that to a single paginated scan plus O(1) lookups.
     """
-    mined: set[str] = set()
+    mined: dict[str, Optional[float]] = {}
     try:
         total = collection.count()
         offset = 0
@@ -1299,10 +1550,69 @@ def prefetch_mined_set(collection, extract_mode: Optional[str] = None) -> set[st
                 # Same default as file_already_mined: missing version == 1
                 version = meta.get("normalize_version", 1)
                 if version >= NORMALIZE_VERSION:
-                    mined.add(src)
+                    stored_mtime = meta.get("source_mtime")
+                    mined[src] = float(stored_mtime) if stored_mtime is not None else None
             if not batch["ids"]:
                 break
             offset += len(batch["ids"])
     except Exception:
         logger.warning("prefetch_mined_set: partial fetch, %d files loaded", len(mined))
     return mined
+
+
+def prefetch_content_hashes(
+    collection, extract_mode: Optional[str] = None
+) -> dict[tuple[str, str], str]:
+    """Pre-fetch (wing, content_hash) -> source_file for drawers already
+    filed at the current NORMALIZE_VERSION, in one bulk pass.
+
+    Repeated exports from Claude/ChatGPT land under a new filename each run
+    (timestamped bundle, regenerated slug, etc.) even when the conversation
+    itself hasn't changed. `prefetch_mined_set` only recognizes a file as
+    already-mined by its exact path, so the same conversation re-exported
+    under a new path always looked "new" and got re-mined as a duplicate
+    drawer. This does the same bulk scan but keyed on the SHA-256 of the
+    normalized transcript text, so the convo miner can recognize "this exact
+    conversation is already filed under a different path" and skip it.
+
+    Keyed by (wing, content_hash) rather than content_hash alone — mining
+    the same transcript into a second wing is a deliberate re-file, not a
+    duplicate, and should produce real drawers in that wing rather than
+    just the registry sentinel.
+
+    A drawer's ``content_hash`` metadata may hold several comma-joined
+    SHA-256 hashes: a privacy-export bundle normalizes to one conversation
+    per drawer set, but the hash is computed per conversation so that a
+    re-export with one new conversation added doesn't change the hash of
+    the ones that didn't. Only the first source_file seen for a given
+    (wing, hash) pair is kept — good enough to detect and skip a repeat,
+    the point is not to track every alias.
+    """
+    hashes: dict[tuple[str, str], str] = {}
+    try:
+        total = collection.count()
+        offset = 0
+        while offset < total:
+            batch = collection.get(limit=1000, offset=offset, include=["metadatas"])
+            for meta in batch["metadatas"]:
+                meta = meta or {}
+                content_hash_field = meta.get("content_hash")
+                src = meta.get("source_file")
+                wing = meta.get("wing")
+                if not content_hash_field or not src or not wing:
+                    continue
+                if not _metadata_matches_extract_mode(meta, extract_mode):
+                    continue
+                version = meta.get("normalize_version", 1)
+                if version < NORMALIZE_VERSION:
+                    continue
+                for content_hash in content_hash_field.split(","):
+                    key = (wing, content_hash)
+                    if content_hash and key not in hashes:
+                        hashes[key] = src
+            if not batch["ids"]:
+                break
+            offset += len(batch["ids"])
+    except Exception:
+        logger.warning("prefetch_content_hashes: partial fetch, %d hashes loaded", len(hashes))
+    return hashes

mempalace/palace_graph.py

@@ -494,6 +494,7 @@ def create_tunnel(
     source_drawer_id: str = None,
     target_drawer_id: str = None,
     kind: str = "explicit",
+    config=None,
 ):
     """Create an explicit (symmetric) tunnel between two locations in the palace.
 
@@ -520,6 +521,9 @@ def create_tunnel(
             topical link where rooms are synthetic ``topic:<name>``
             identifiers). Preserved on the stored dict so readers can
             distinguish real-room traversals from topic connections.
+        config: Optional ``MempalaceConfig`` selecting the palace and its
+            tunnel sidecar. Explicit-path callers must pass the matching
+            config instead of falling back to the ambient default palace.
 
     Returns:
         The stored tunnel dict.
@@ -545,7 +549,7 @@ def create_tunnel(
     # mempalace.yaml from disk; before this change the helpers each
     # instantiated their own, triggering several redundant disk reads per
     # create_tunnel call (flagged by gemini-code-assist on #1469).
-    config = MempalaceConfig()
+    config = config or MempalaceConfig()
 
     # Validate room existence for explicit tunnels only. Use the verbatim wing
     # slugs here so #1504's hyphen-preserving write path remains intact.
@@ -742,6 +746,7 @@ def compute_topic_tunnels(
     topics_by_wing: dict,
     min_count: int = 1,
     label_prefix: str = "shared topic",
+    config=None,
 ) -> list[dict]:
     """Create tunnels for every pair of wings that share >= ``min_count`` topics.
 
@@ -817,6 +822,7 @@ def compute_topic_tunnels(
                     target_room=room,
                     label=f"{label_prefix}: {topic_name}",
                     kind="topic",
+                    config=config,
                 )
                 created.append(tunnel)
     return created
@@ -827,6 +833,7 @@ def topic_tunnels_for_wing(
     topics_by_wing: dict,
     min_count: int = 1,
     label_prefix: str = "shared topic",
+    config=None,
 ) -> list[dict]:
     """Compute topic tunnels involving a single wing.
 
@@ -871,6 +878,7 @@ def topic_tunnels_for_wing(
                 slice_map,
                 min_count=min_count,
                 label_prefix=label_prefix,
+                config=config,
             )
         )
     return created
@@ -880,6 +888,7 @@ def entity_tunnels_for_wing(
     wing: str,
     hallways: list,
     label_prefix: str = "shared entity",
+    config=None,
 ) -> list:
     """Compute entity tunnels involving a single wing.
 
@@ -946,6 +955,7 @@ def entity_tunnels_for_wing(
                 target_room=room,
                 label=f"{label_prefix}: {entity}",
                 kind="entity",
+                config=config,
             )
             created.append(tunnel)
     return created

mempalace/replica.py

@@ -0,0 +1,64 @@
+"""
+replica.py — Per-palace replica identity (RFC 004 transport seam / provenance)
+
+Every palace replica has one stable ``ReplicaId``, stamped as
+``origin_replica`` into every op it authors. For the step-0 pilot the id is
+minted locally on first use and persisted in ``replica.json`` inside the
+palace directory. Existing 12-hex pilot ids remain valid, but new ids use
+128 bits of entropy. When the transport layer lands (MeshGuard), the mesh
+Ed25519 identity supersedes it via an alias op — RFC 004 Appendix A.4:
+"a rename is just another provenance fact."
+
+The id never syncs and never rotates silently; it names the seat (this
+machine's copy of the palace), not the model or the agent.
+"""
+
+import json
+import os
+import re
+import secrets
+from pathlib import Path
+
+REPLICA_FILENAME = "replica.json"
+
+_REPLICA_ID_RE = re.compile(r"^rep_(?:[0-9a-f]{12}|[0-9a-f]{32})$")
+
+
+def _mint() -> str:
+    return f"rep_{secrets.token_hex(16)}"
+
+
+def get_replica_id(palace_path: str) -> str:
+    """Return this palace's stable replica id, minting it on first use.
+
+    The file is written atomically (tmp + rename) so a crash mid-mint can
+    never leave a half-written identity; a corrupt or foreign-shaped file
+    fails loudly rather than silently minting a second identity — two ids
+    for one replica would fork its op-log provenance.
+    """
+    path = Path(os.path.expanduser(palace_path)) / REPLICA_FILENAME
+    if path.exists():
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            replica_id = data["replica_id"]
+        except (ValueError, KeyError, TypeError) as exc:
+            raise ValueError(
+                f"{path} is corrupt ({exc}); refusing to mint a second replica "
+                "identity for this palace — restore or delete the file explicitly"
+            ) from None
+        if not isinstance(replica_id, str) or not _REPLICA_ID_RE.match(replica_id):
+            raise ValueError(
+                f"{path} holds an invalid replica_id {replica_id!r}; refusing to "
+                "mint a second identity — restore or delete the file explicitly"
+            )
+        return replica_id
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    replica_id = _mint()
+    tmp = path.with_suffix(".json.tmp")
+    tmp.write_text(
+        json.dumps({"replica_id": replica_id, "minted_at_note": "RFC 004 step 0"}, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    os.replace(tmp, path)
+    return replica_id

mempalace/room_detector_local.py

@@ -232,14 +232,14 @@ def detect_rooms_from_files(project_dir: str) -> list:
 
 def print_proposed_structure(project_name: str, rooms: list, total_files: int, source: str):
     print(f"\n{'=' * 55}")
-    print("  MemPalace Init — Local setup")
+    print("  MemPalace Init -- Local setup")
     print(f"{'=' * 55}")
     print(f"\n  WING: {project_name}")
     print(f"  ({total_files} files found, rooms detected from {source})\n")
     for room in rooms:
         print(f"    ROOM: {room['name']}")
         print(f"          {room['description']}")
-    print(f"\n{'─' * 55}")
+    print(f"\n{'-' * 55}")
 
 
 def get_user_approval(rooms: list) -> list:
@@ -259,7 +259,7 @@ def get_user_approval(rooms: list) -> list:
     if choice == "edit":
         print("\n  Current rooms:")
         for i, room in enumerate(rooms):
-            print(f"    {i + 1}. {room['name']} — {room['description']}")
+            print(f"    {i + 1}. {room['name']} -- {room['description']}")
         remove = input("\n  Room numbers to REMOVE (comma-separated, or enter to skip): ").strip()
         if remove:
             to_remove = {int(x.strip()) - 1 for x in remove.split(",") if x.strip().isdigit()}

mempalace/server_registry.py

@@ -0,0 +1,171 @@
+"""Per-palace discovery registry for the MemPalace HTTP MCP hub.
+
+``mempalace serve`` (and ``mempalace-mcp --transport http``) is meant to be
+the single long-lived writer for a shared palace: once it performs its first
+mutating tool call it holds the per-palace MCP writer lease (#1818) for its
+whole lifetime, and every other process is refused palace writes. That is
+correct for agents — they talk to the hub — but the background save hooks
+and the plain CLI still spawn short-lived ``mempalace mine`` processes,
+which would be refused while a hub is up and silently stop transcript
+capture on the hub machine.
+
+This module gives those local processes a way to find the hub instead of
+fighting it: the HTTP transport records ``{pid, host, port, scheme,
+read_only}`` next to the per-palace bearer token
+(``~/.mempalace/server/<key>/``), and callers use
+:func:`read_live_serverinfo` to decide "forward this write over HTTP" vs
+"no hub — do the write directly".
+
+The registry is local-machine only by design: it lives under the user's
+home, is keyed by the canonical palace path, and a record is trusted only
+while the recorded pid is still alive — a crashed hub leaves a stale file
+that every reader ignores and the next hub overwrites.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Bind-address wildcards: a hub bound to "all interfaces" is dialed via
+# loopback by local forwarders.
+_WILDCARD_HOSTS = {"0.0.0.0", "::", "[::]"}
+
+
+def _canonical(palace_path: str) -> str:
+    return os.path.abspath(os.path.realpath(os.path.expanduser(palace_path)))
+
+
+def server_state_dir(palace_path: str) -> Path:
+    """Per-palace state directory shared by the token and the serverinfo.
+
+    Keyed by the canonical palace path so one hub per palace reuses a stable
+    directory across restarts. Must stay in sync with the token location used
+    by ``mempalace serve`` (see ``cli._server_token_path``, which delegates
+    here).
+    """
+    key = hashlib.sha256(os.path.normcase(_canonical(palace_path)).encode("utf-8")).hexdigest()[:24]
+    return Path.home() / ".mempalace" / "server" / key
+
+
+def server_token_path(palace_path: str) -> Path:
+    return server_state_dir(palace_path) / "token"
+
+
+def serverinfo_path(palace_path: str) -> Path:
+    return server_state_dir(palace_path) / "serverinfo.json"
+
+
+def write_serverinfo(palace_path: str, *, host: str, port: int, scheme: str, read_only: bool):
+    """Record this process as the palace's HTTP hub. Returns the file path.
+
+    0600 like the token: the record itself is not secret, but the directory
+    convention is "private to the user" and there is no reason to relax it.
+    """
+    path = serverinfo_path(palace_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        os.chmod(str(path.parent), 0o700)
+    except OSError:
+        pass
+    payload = {
+        "pid": os.getpid(),
+        "host": host,
+        "port": int(port),
+        "scheme": scheme,
+        "read_only": bool(read_only),
+        "palace_path": _canonical(palace_path),
+    }
+    fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        json.dump(payload, fh)
+        fh.write("\n")
+    return path
+
+
+def clear_serverinfo(palace_path: str) -> None:
+    """Remove this process's serverinfo record, if it is still ours.
+
+    Guarded on the recorded pid so a slow atexit from an old hub cannot
+    delete the record a newer hub just wrote for the same palace.
+    """
+    path = serverinfo_path(palace_path)
+    try:
+        recorded = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return
+    if recorded.get("pid") != os.getpid():
+        return
+    try:
+        path.unlink()
+    except OSError:
+        logger.debug("serverinfo cleanup failed for %s", path, exc_info=True)
+
+
+def _pid_alive(pid) -> bool:
+    if not isinstance(pid, int) or pid <= 0:
+        return False
+    if os.name == "nt":  # pragma: no cover - exercised on Windows CI only
+        import ctypes
+
+        PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
+        handle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
+        if not handle:
+            return False
+        ctypes.windll.kernel32.CloseHandle(handle)
+        return True
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except OSError:
+        return False
+    return True
+
+
+def read_live_serverinfo(palace_path: str):
+    """Return the hub record for this palace, or None.
+
+    None when no record exists, the record is unreadable, or the recorded
+    pid is no longer alive (crashed hub — stale file, ignore it).
+    """
+    path = serverinfo_path(palace_path)
+    try:
+        info = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    if not isinstance(info, dict):
+        return None
+    if not _pid_alive(info.get("pid")):
+        return None
+    if not isinstance(info.get("port"), int) or not info.get("host"):
+        return None
+    return info
+
+
+def client_base_url(info: dict) -> str:
+    """Dial address for a local client, from a serverinfo record.
+
+    A wildcard bind ("all interfaces") is dialed via loopback — the record
+    is only ever read on the hub's own machine.
+    """
+    host = str(info.get("host", "")).strip()
+    if host.lower() in _WILDCARD_HOSTS:
+        host = "127.0.0.1"
+    scheme = info.get("scheme") or "http"
+    return f"{scheme}://{host}:{info['port']}"
+
+
+def load_server_token(palace_path: str) -> str:
+    """The palace's hub bearer token, or "" when none was ever generated."""
+    try:
+        return server_token_path(palace_path).read_text(encoding="utf-8").strip()
+    except OSError:
+        return ""

mempalace/service.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import contextlib
 import io
+import functools
 import os
 import sys
 from typing import Any
@@ -18,6 +19,36 @@ from .config import MempalaceConfig
 
 _EXPLICIT_BACKEND_ENV = "MEMPALACE_BACKEND_EXPLICIT"
 _PALACE_PATH_ENV = "MEMPALACE_PALACE_PATH"
+
+
+def _restores_palace_env(fn):
+    """Restore ``MEMPALACE_PALACE_PATH`` after the call that stamped it.
+
+    Each ``run_*`` entrypoint sets the variable so downstream config resolution sees THIS
+    call's palace. In the daemon that costs nothing — one call, one process. In any process
+    that makes two calls it leaks: ``config.py`` reads this variable with priority OVER the
+    config file, so the first call's palace silently becomes the second caller's, and a
+    tmpdir palace that has since been removed reads back as ``PalaceNotFoundError`` from an
+    unrelated code path.
+
+    ``daemon.py`` already saves and restores exactly these keys around its own dispatch.
+    This gives the service entrypoints the same discipline.
+    """
+
+    @functools.wraps(fn)
+    def _wrapped(*args, **kwargs):
+        previous = os.environ.get(_PALACE_PATH_ENV)
+        try:
+            return fn(*args, **kwargs)
+        finally:
+            if previous is None:
+                os.environ.pop(_PALACE_PATH_ENV, None)
+            else:
+                os.environ[_PALACE_PATH_ENV] = previous
+
+    return _wrapped
+
+
 _BACKEND_ENV = "MEMPALACE_BACKEND"
 # Env vars a job may mutate via _apply_backend / palace_path injection. They are
 # snapshotted per job and restored afterward so a job that switches the backend
@@ -55,6 +86,7 @@ WRITE_TOOLS = frozenset(
     {
         "mempalace_add_drawer",
         "mempalace_checkpoint",
+        "mempalace_delete_by_source",
         "mempalace_delete_drawer",
         "mempalace_update_drawer",
         "mempalace_diary_write",
@@ -139,6 +171,7 @@ def execute_job(kind: str, payload: dict[str, Any]) -> dict[str, Any]:
     return result
 
 
+@_restores_palace_env
 def run_mine(payload: dict[str, Any]) -> dict[str, Any]:
     """Run the same mine operation as the CLI, without daemon transport concerns."""
     palace_path = os.path.abspath(
@@ -148,21 +181,32 @@ def run_mine(payload: dict[str, Any]) -> dict[str, Any]:
     _apply_backend(payload.get("backend"))
 
     source = payload.get("source") or payload.get("dir")
+    source_adapter = payload.get("source_adapter")
     mode = payload.get("mode") or "projects"
     wing = payload.get("wing")
     agent = payload.get("agent") or "mempalace"
     limit = int(payload.get("limit") or 0)
     dry_run = bool(payload.get("dry_run"))
 
-    if payload.get("redetect_origin"):
+    if payload.get("redetect_origin") and not source_adapter:
         from .cli import _run_pass_zero
 
         _run_pass_zero(project_dir=source, palace_dir=palace_path, llm_provider=None)
 
+    from .daemon import LOCK_REFUSAL_ERROR_CLASS
     from .palace import MineAlreadyRunning, MineValidationError
 
     try:
-        if mode == "convos":
+        if source_adapter:
+            from .cli import mine_source_adapter
+
+            mine_source_adapter(
+                source_name=source_adapter,
+                source_path=source,
+                palace_path=palace_path,
+                dry_run=dry_run,
+            )
+        elif mode == "convos":
             from .convo_miner import mine_convos
 
             mine_convos(
@@ -206,7 +250,7 @@ def run_mine(payload: dict[str, Any]) -> dict[str, Any]:
         return {
             "success": False,
             "error": str(exc),
-            "error_class": "LockHeldByOtherProcess",
+            "error_class": LOCK_REFUSAL_ERROR_CLASS,
             "exit_code": 1,
         }
     except MineValidationError as exc:
@@ -227,9 +271,17 @@ def run_mine(payload: dict[str, Any]) -> dict[str, Any]:
     except Exception as exc:
         return {"success": False, "error": f"mine failed: {exc}", "exit_code": 1}
 
-    return {"success": True, "kind": "mine", "mode": mode, "dry_run": dry_run, "exit_code": 0}
+    result_mode = "source" if source_adapter else mode
+    return {
+        "success": True,
+        "kind": "mine",
+        "mode": result_mode,
+        "dry_run": dry_run,
+        "exit_code": 0,
+    }
 
 
+@_restores_palace_env
 def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
     """Run sync and render the same operator-facing summary shape as the CLI."""
     palace_path = os.path.abspath(
@@ -239,6 +291,7 @@ def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
     _apply_backend(payload.get("backend"))
 
     from .backends import detect_backend_for_path
+    from .daemon import LOCK_REFUSAL_ERROR_CLASS
     from .palace import MineAlreadyRunning, _backend_artifact_label, resolve_backend_name
 
     if not os.path.isdir(palace_path):
@@ -270,7 +323,7 @@ def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
     dry_run = bool(payload.get("dry_run", True))
 
     print(f"\n{'=' * 55}")
-    print("  MemPalace Sync — Gitignore-aware drawer prune")
+    print("  MemPalace Sync -- Gitignore-aware drawer prune")
     print(f"{'=' * 55}")
     print(f"  Palace:   {palace_path}")
     if payload.get("wing"):
@@ -298,7 +351,7 @@ def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
         return {
             "success": False,
             "error": str(exc),
-            "error_class": "LockHeldByOtherProcess",
+            "error_class": LOCK_REFUSAL_ERROR_CLASS,
             "exit_code": 1,
         }
     except ValueError as exc:
@@ -334,6 +387,7 @@ def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
     return {"success": True, "report": report, "exit_code": 0}
 
 
+@_restores_palace_env
 def run_diary_write(payload: dict[str, Any]) -> dict[str, Any]:
     palace_path = payload.get("palace_path")
     if palace_path:

mempalace/split_mega_files.py

@@ -285,7 +285,7 @@ def main():
         return
 
     print(f"\n{'=' * 60}")
-    print(f"  Mega-file splitter — {'DRY RUN' if args.dry_run else 'SPLITTING'}")
+    print(f"  Mega-file splitter -- {'DRY RUN' if args.dry_run else 'SPLITTING'}")
     print(f"{'=' * 60}")
     print(f"  Source:      {src_dir}")
     print(f"  Output:      {output_dir or 'same dir as source'}")
@@ -307,9 +307,9 @@ def main():
 
     print(f"{'-' * 60}")
     if args.dry_run:
-        print(f"  DRY RUN — would create {total_written} files from {len(mega_files)} mega-files")
+        print(f"  DRY RUN -- would create {total_written} files from {len(mega_files)} mega-files")
     else:
-        print(f"  Done — created {total_written} files from {len(mega_files)} mega-files")
+        print(f"  Done -- created {total_written} files from {len(mega_files)} mega-files")
     print()
 
 

mempalace/sweeper.py

@@ -230,28 +230,48 @@ def sweep(jsonl_path: str, palace_path: str, source_label: Optional[str] = None)
         nonlocal drawers_added, drawers_already_present
         if not batch_ids:
             return
-        # Pre-flight: which IDs in this batch are already present?
-        # Upsert is idempotent on data but counts as "added" would lie;
-        # this pre-query makes the metric honest (Copilot PR 998 review).
+        # Collapse repeated deterministic IDs before backend calls.
+        # Rewritten transcripts can contain the same message UUID more than
+        # once, and Chroma rejects duplicate IDs in one get or upsert request.
+        # Reassigning a dict key preserves its first position but replaces its
+        # value, so the final occurrence supplies the document and metadata.
+        deduped: dict[str, tuple[str, dict]] = {}
+        for drawer_id, document, metadata in zip(
+            batch_ids,
+            batch_docs,
+            batch_metas,
+        ):
+            deduped[drawer_id] = (document, metadata)
+
+        unique_ids = list(deduped)
+        unique_docs = [deduped[drawer_id][0] for drawer_id in unique_ids]
+        unique_metas = [deduped[drawer_id][1] for drawer_id in unique_ids]
+
+        # Pre-flight: which unique IDs are already present?
         try:
-            existing = collection.get(ids=list(batch_ids), include=[])
-            # Chroma returns a dict; typed backends return GetResult — the
-            # compat shim makes ``.get("ids")`` work on both.
+            existing = collection.get(
+                ids=unique_ids,
+                include=[],
+            )
+            # Chroma returns a dict; typed backends return GetResult - the
+            # compatibility shim makes .get("ids") work on both.
             present = set(existing.get("ids") or [])
         except Exception as exc:
             logger.warning(
                 "sweeper: existence pre-check failed (%s); "
-                "counting all batch rows as new (metric may over-count on reruns).",
+                "counting all unique batch rows as new "
+                "(metric may over-count on reruns).",
                 exc,
             )
             present = set()
-        new_count = sum(1 for rid in batch_ids if rid not in present)
-        already_count = len(batch_ids) - new_count
+
+        new_count = sum(1 for drawer_id in unique_ids if drawer_id not in present)
+        already_count = len(unique_ids) - new_count
 
         collection.upsert(
-            ids=batch_ids,
-            documents=batch_docs,
-            metadatas=batch_metas,
+            ids=unique_ids,
+            documents=unique_docs,
+            metadatas=unique_metas,
         )
         drawers_added += new_count
         drawers_already_present += already_count

mempalace/transport.py

@@ -0,0 +1,169 @@
+"""
+transport.py — The RFC 004 transport seam (Layer 1).
+
+Layer 2 (anti-entropy sync of the logstream and, in step 2a, the memory
+op-log) is allowed to see peers ONLY through this interface. Everything
+about networks, addressing, keys, and which machines are awake lives
+below the seam; nothing about merge semantics leaks down into it. The
+full contract, from the RFC:
+
+    self_id()                    stable ReplicaId of this node
+    peers()                      membership snapshot
+    request(peer, path, params)  one authenticated round-trip
+    open_stream(peer, path)      long-lived tail, resumable by cursor
+    on_presence_change(cb)       liveness deltas (R5)
+    on_inbound(path, handler)    serve anti-entropy pulls
+
+v1 ships the pull half — ``self_id``/``peers``/``request`` — because that
+is all the shipping sync engine consumes. The push half raises
+``NotImplementedError`` here and lands with the MeshGuard binding, where
+SWIM membership supplies presence, the Ed25519 node identity IS the
+ReplicaId (provenance and authentication as one fact), and membership in
+the mesh replaces bearer tokens as the only ACL.
+
+Two implementations are planned:
+
+- :class:`HttpsBearerTransport` (this module, shipping): tailnet HTTPS
+  with per-hub bearer tokens from ``peers.json``. Address discovery and
+  channel encryption come from the tailnet; authorization is an
+  app-layer token that a human must relay out-of-band for every edge —
+  the N-squared-token model the MeshGuard transport retires.
+- ``MeshGuardTransport`` (next): binds the meshguard daemon over its FFI.
+  Selected via ``MEMPALACE_TRANSPORT=meshguard`` once it exists; the
+  factory below is the single swap point, so Layer 2 never changes.
+"""
+
+import json
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Optional
+
+PEERS_FILENAME = "peers.json"
+
+_HTTP_TIMEOUT_S = 30
+
+
+class TransportError(Exception):
+    """A peer was unreachable or answered outside the protocol."""
+
+
+def _http_timeout_s() -> float:
+    """Per-request timeout for peer HTTP calls, env-tunable.
+
+    The default suits logstream syncs (small pages), but a deep-offset
+    snapshot page — Chroma's get(offset=N) scans N rows — behind a TLS
+    proxy can legitimately exceed 30s on a large palace (first hit in
+    production: a 31k-drawer bootstrap timing out at offset 27500). Set
+    MEMPALACE_SYNC_HTTP_TIMEOUT to raise it for bootstrap pulls.
+    """
+    try:
+        return float(os.environ.get("MEMPALACE_SYNC_HTTP_TIMEOUT", "") or _HTTP_TIMEOUT_S)
+    except ValueError:
+        return float(_HTTP_TIMEOUT_S)
+
+
+def load_peers(palace_path: str) -> list[dict]:
+    """Read peers.json; [] when absent. Malformed files fail loudly.
+
+    peers.json is TRANSPORT configuration — addressing plus app-layer
+    authorization — which is why it lives on this side of the seam.
+    """
+    path = Path(os.path.expanduser(palace_path)) / PEERS_FILENAME
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        peers = data["peers"]
+    except (ValueError, KeyError, TypeError) as exc:
+        raise ValueError(f"{path} is malformed: {exc}") from None
+    if not isinstance(peers, list):
+        raise ValueError(f"{path}: 'peers' must be a list")
+    for peer in peers:
+        if not isinstance(peer, dict) or not peer.get("url"):
+            raise ValueError(f"{path}: every peer needs at least a 'url'")
+    return peers
+
+
+def http_request(base_url: str, token: str, path: str, params: dict = None) -> dict:
+    """The HTTPS wire primitive: one bearer-authenticated GET, JSON back."""
+    url = base_url.rstrip("/") + path
+    if params:
+        url += "?" + urllib.parse.urlencode(params)
+    request = urllib.request.Request(url)
+    if token:
+        request.add_header("Authorization", f"Bearer {token}")
+    try:
+        with urllib.request.urlopen(request, timeout=_http_timeout_s()) as response:
+            return json.loads(response.read().decode("utf-8"))
+    except (urllib.error.URLError, TimeoutError, ValueError) as exc:
+        raise TransportError(f"{url}: {exc}") from None
+
+
+class Transport(ABC):
+    """The seam. Layer 2 holds one of these and nothing else network-shaped."""
+
+    @abstractmethod
+    def self_id(self) -> str:
+        """Stable ReplicaId of this node."""
+
+    @abstractmethod
+    def peers(self) -> list[dict]:
+        """Current membership snapshot. Each peer dict carries at least
+        ``name``; transport-specific addressing fields are opaque to
+        Layer 2, which only ever hands the dict back to :meth:`request`."""
+
+    @abstractmethod
+    def request(self, peer: dict, path: str, params: Optional[dict] = None) -> dict:
+        """One authenticated round-trip to ``peer``. Raises TransportError."""
+
+    # ── Push half — defined by the contract, lands with MeshGuard ────────
+
+    def open_stream(self, peer: dict, path: str):
+        raise NotImplementedError("open_stream lands with the MeshGuard transport binding")
+
+    def on_presence_change(self, callback):
+        raise NotImplementedError("presence lands with the MeshGuard transport binding")
+
+    def on_inbound(self, path: str, handler):
+        raise NotImplementedError("inbound serving lands with the MeshGuard transport binding")
+
+
+class HttpsBearerTransport(Transport):
+    """Shipping transport: tailnet HTTPS + peers.json bearer tokens."""
+
+    def __init__(self, palace_path: str):
+        self._palace_path = palace_path
+
+    def self_id(self) -> str:
+        from .replica import get_replica_id
+
+        return get_replica_id(self._palace_path)
+
+    def peers(self) -> list[dict]:
+        return load_peers(self._palace_path)
+
+    def request(self, peer: dict, path: str, params: Optional[dict] = None) -> dict:
+        return http_request(peer["url"], peer.get("token", ""), path, params)
+
+
+def get_transport(palace_path: str) -> Transport:
+    """The single swap point (MEMPALACE_TRANSPORT; default https).
+
+    ``meshguard`` is reserved: selecting it before the binding lands
+    fails loudly rather than silently degrading to bearer tokens —
+    a user who asked for mesh-identity auth must never unknowingly
+    run on the token model.
+    """
+    selected = (os.environ.get("MEMPALACE_TRANSPORT") or "https").strip().lower()
+    if selected == "https":
+        return HttpsBearerTransport(palace_path)
+    if selected == "meshguard":
+        raise NotImplementedError(
+            "MEMPALACE_TRANSPORT=meshguard is reserved for the MeshGuard binding "
+            "(RFC 004 transport seam); it has not shipped yet"
+        )
+    raise ValueError(f"unknown MEMPALACE_TRANSPORT {selected!r} (expected: https, meshguard)")

mempalace/version.py

@@ -1,3 +1,3 @@
 """Single source of truth for the MemPalace package version."""
 
-__version__ = "3.5.0"
+__version__ = "3.7.0"

mempalace/write_routing.py

@@ -0,0 +1,210 @@
+"""Shared daemon-routing policy for MemPalace write callers.
+
+The policy is deliberately transport-agnostic. Hook and CLI consumers decide
+whether a daemon is already available and whether they are allowed to start
+one; this module turns those facts plus a policy into one explicit route.
+
+This module changes no caller defaults by itself. Hook and CLI adoption are
+separate follow-up PRs.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Iterable
+
+
+class WriteRoutingError(ValueError):
+    """Raised when a write-routing policy is invalid or cannot be applied."""
+
+
+class WriteRoutingPolicy(str, Enum):
+    """User-selected policy for routine write operations."""
+
+    DIRECT = "direct"
+    PREFER = "prefer"
+    REQUIRE = "require"
+
+
+class WriteRoutingTarget(str, Enum):
+    """Concrete route selected for one write operation."""
+
+    DIRECT = "direct"
+    DAEMON = "daemon"
+    BLOCKED = "blocked"
+
+
+@dataclass(frozen=True)
+class RoutingPolicyCandidate:
+    """One precedence-ordered source for a routing policy."""
+
+    source: str
+    value: Any
+    legacy_boolean: bool = False
+
+
+@dataclass(frozen=True)
+class ResolvedWriteRoutingPolicy:
+    """A normalized policy plus the source that selected it."""
+
+    policy: WriteRoutingPolicy
+    source: str
+
+
+@dataclass(frozen=True)
+class WriteRoutingDecision:
+    """Concrete routing decision for a single operation."""
+
+    policy: WriteRoutingPolicy
+    target: WriteRoutingTarget
+    auto_start_daemon: bool
+    reason: str
+
+    @property
+    def use_daemon(self) -> bool:
+        """Whether this operation should use the daemon."""
+        return self.target is WriteRoutingTarget.DAEMON
+
+    @property
+    def blocked(self) -> bool:
+        """Whether the operation must stop instead of writing directly."""
+        return self.target is WriteRoutingTarget.BLOCKED
+
+
+_LEGACY_TRUE = {"1", "true", "yes", "on", "daemon"}
+_LEGACY_FALSE = {"0", "false", "no", "off"}
+
+
+def parse_write_routing_policy(
+    value: Any,
+    *,
+    legacy_boolean: bool = False,
+) -> WriteRoutingPolicy:
+    """Normalize one routing-policy value.
+
+    New policy settings accept only ``direct``, ``prefer``, and ``require``.
+
+    Legacy boolean settings additionally map truthy values to ``prefer`` and
+    falsy values to ``direct`` so existing ``hooks.daemon`` configurations
+    retain their historical behavior.
+    """
+
+    if isinstance(value, WriteRoutingPolicy):
+        return value
+
+    if legacy_boolean and isinstance(value, bool):
+        return WriteRoutingPolicy.PREFER if value else WriteRoutingPolicy.DIRECT
+
+    if legacy_boolean and isinstance(value, int) and not isinstance(value, bool):
+        if value == 1:
+            return WriteRoutingPolicy.PREFER
+        if value == 0:
+            return WriteRoutingPolicy.DIRECT
+
+    if not isinstance(value, str):
+        raise WriteRoutingError("write routing policy must be one of: direct, prefer, require")
+
+    normalized = value.strip().lower()
+
+    try:
+        return WriteRoutingPolicy(normalized)
+    except ValueError:
+        pass
+
+    if legacy_boolean:
+        if normalized in _LEGACY_TRUE:
+            return WriteRoutingPolicy.PREFER
+        if normalized in _LEGACY_FALSE:
+            return WriteRoutingPolicy.DIRECT
+
+    raise WriteRoutingError(
+        f"invalid write routing policy {value!r}; expected direct, prefer, or require"
+    )
+
+
+def resolve_write_routing_policy(
+    candidates: Iterable[RoutingPolicyCandidate],
+    *,
+    default: WriteRoutingPolicy = WriteRoutingPolicy.DIRECT,
+) -> ResolvedWriteRoutingPolicy:
+    """Return the first configured policy from ordered policy sources."""
+
+    for candidate in candidates:
+        if candidate.value is None:
+            continue
+
+        try:
+            policy = parse_write_routing_policy(
+                candidate.value,
+                legacy_boolean=candidate.legacy_boolean,
+            )
+        except WriteRoutingError as exc:
+            raise WriteRoutingError(f"{candidate.source}: {exc}") from exc
+
+        return ResolvedWriteRoutingPolicy(
+            policy=policy,
+            source=candidate.source,
+        )
+
+    return ResolvedWriteRoutingPolicy(
+        policy=default,
+        source="default",
+    )
+
+
+def choose_write_route(
+    policy: WriteRoutingPolicy,
+    *,
+    daemon_available: bool,
+    daemon_can_start: bool,
+) -> WriteRoutingDecision:
+    """Choose direct, daemon, or blocked for one routine write.
+
+    ``daemon_can_start`` is normally false for latency-sensitive hooks and
+    true for interactive CLI commands.
+
+    The key safety guarantee is that ``require`` never degrades to a direct
+    write when the daemon is unavailable.
+    """
+
+    policy = parse_write_routing_policy(policy)
+
+    if policy is WriteRoutingPolicy.DIRECT:
+        return WriteRoutingDecision(
+            policy=policy,
+            target=WriteRoutingTarget.DIRECT,
+            auto_start_daemon=False,
+            reason="policy-direct",
+        )
+
+    if daemon_available:
+        return WriteRoutingDecision(
+            policy=policy,
+            target=WriteRoutingTarget.DAEMON,
+            auto_start_daemon=False,
+            reason="daemon-available",
+        )
+
+    if daemon_can_start:
+        return WriteRoutingDecision(
+            policy=policy,
+            target=WriteRoutingTarget.DAEMON,
+            auto_start_daemon=True,
+            reason="daemon-auto-start",
+        )
+
+    if policy is WriteRoutingPolicy.PREFER:
+        return WriteRoutingDecision(
+            policy=policy,
+            target=WriteRoutingTarget.DIRECT,
+            auto_start_daemon=False,
+            reason="daemon-unavailable-fallback",
+        )
+
+    return WriteRoutingDecision(
+        policy=policy,
+        target=WriteRoutingTarget.BLOCKED,
+        auto_start_daemon=False,
+        reason="daemon-required-unavailable",
+    )

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mempalace"
-version = "3.5.0"
+version = "3.7.0"
 description = "Give your AI a memory — mine projects and conversations into a searchable palace. No API key required."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -60,6 +60,7 @@ mempalace-mcp = "mempalace.mcp_server:main"
 
 [project.entry-points."mempalace.backends"]
 chroma = "mempalace.backends.chroma:ChromaBackend"
+milvus = "mempalace.backends.milvus:MilvusBackend"
 pgvector = "mempalace.backends.pgvector:PgVectorBackend"
 qdrant = "mempalace.backends.qdrant:QdrantBackend"
 sqlite_exact = "mempalace.backends.sqlite_exact:SQLiteExactBackend"
@@ -78,7 +79,7 @@ dev = [
     # (wired only into the test-windows job in ci.yml, scoped to the specific
     # error). Local/Linux/macOS runs never rerun, so real failures stay loud.
     "pytest-rerunfailures>=12.0",
-    "ruff==0.15.18",
+    "ruff==0.16.1",
     "psutil>=5.9",
     # Property-based testing — generates hundreds of random inputs per
     # test to find counterexamples the hand-written positive tests miss.
@@ -99,6 +100,13 @@ dev = [
     "mypy>=1.0",
 ]
 spellcheck = ["autocorrect>=2.0"]
+# Opt-in Milvus backend. Defaults to per-palace Milvus Lite
+# (`<palace>/milvus.db`); set MEMPALACE_MILVUS_URI and
+# MEMPALACE_MILVUS_TOKEN to point at Milvus server or Zilliz Cloud.
+milvus = [
+    "pymilvus>=3.0.0",
+    "milvus-lite>=3.0; python_version >= '3.10' and sys_platform != 'win32'",
+]
 # Opt-in Postgres + pgvector backend. Only the psycopg driver is needed on the
 # client; the server must have the `vector` extension available. Selected via
 # MEMPALACE_BACKEND=pgvector / --backend pgvector; never required for the
@@ -139,7 +147,7 @@ dev = [
     # (wired only into the test-windows job in ci.yml, scoped to the specific
     # error). Local/Linux/macOS runs never rerun, so real failures stay loud.
     "pytest-rerunfailures>=12.0",
-    "ruff==0.15.18",
+    "ruff==0.16.1",
     "psutil>=5.9",
     "hypothesis>=6.0",
     "pre-commit>=3.0",
@@ -156,7 +164,11 @@ packages = ["mempalace"]
 [tool.ruff]
 line-length = 100
 target-version = "py39"
-extend-exclude = ["benchmarks"]
+# "*.md": ruff 0.16 began formatting Python inside markdown fences, which
+# pulls ~96 docs into scope and reflows hand-aligned example code (the
+# column-aligned field comments in docs/rfcs/002 are deliberate). Docs are
+# authored prose, not linted source — keep the formatter on real modules.
+extend-exclude = ["benchmarks", "*.md"]
 
 [tool.ruff.lint]
 select = ["E", "F", "W", "C901"]

scripts/backfill_authored_at.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Backfill ``authored_at`` onto existing conversation drawers.
+
+New mines stamp ``authored_at`` automatically (see ``convo_miner._extract_authored_at``),
+but drawers mined before that change only have ``filed_at`` (ingest time). Re-mining does
+NOT fix them: the scanner skips files already mined at the current ``NORMALIZE_VERSION``.
+
+This migration updates the affected drawers IN PLACE — metadata only, embeddings are left
+untouched, so there is no re-embedding cost. It is idempotent (drawers already correct are
+skipped) and safe to re-run. It only touches ``ingest_mode == "convos"`` drawers; markdown
+drawers have no per-line timestamps and keep their ``filed_at`` fallback.
+
+Drawers whose source transcript is no longer on disk are left as-is (they keep falling back
+to ``filed_at``), so point ``--sessions`` at the directories that still hold your ``.jsonl``
+transcripts (e.g. ``~/.claude`` and ``~/.codex``).
+
+Usage (dry-run prints what would change; pass --apply to write):
+
+    python scripts/backfill_authored_at.py \
+        --palace ~/.mempalace/palace \
+        --sessions ~/.claude --sessions ~/.codex [--apply]
+
+In Docker (the MCP image), mount the volume and your session dirs read-only:
+
+    docker run --rm \
+      -v mempalace-data:/data \
+      -v ~/.claude:/sessions/claude:ro -v ~/.codex:/sessions/codex:ro \
+      -v "$PWD/scripts/backfill_authored_at.py:/tmp/backfill.py:ro" \
+      --entrypoint /app/.venv/bin/python mempalace:local \
+      /tmp/backfill.py --palace /data/.mempalace/palace \
+        --sessions /sessions/claude --sessions /sessions/codex --apply
+"""
+
+import argparse
+import glob
+import os
+
+import chromadb
+
+from mempalace.convo_miner import _extract_authored_at
+
+COLLECTION = "mempalace_drawers"
+PAGE = 2000
+BATCH = 1000
+
+
+def _index_sessions(session_dirs):
+    """Map ``basename.jsonl -> realpath`` for every transcript under the given dirs."""
+    index = {}
+    for root in session_dirs:
+        for f in glob.glob(os.path.join(os.path.expanduser(root), "**", "*.jsonl"), recursive=True):
+            index.setdefault(os.path.basename(f), f)
+    return index
+
+
+def backfill_authored_at(collection, session_dirs, apply=False):
+    """Stamp ``authored_at`` on convos drawers from their source transcript timestamps.
+
+    Returns a stats dict: ``scanned``, ``updated``, ``resolved_files``, ``unresolved_files``.
+    """
+    index = _index_sessions(session_dirs)
+    cache = {}
+    unresolved = set()
+    pending_ids, pending_metas = [], []
+    scanned = updated = 0
+
+    def flush():
+        nonlocal pending_ids, pending_metas, updated
+        if pending_ids and apply:
+            collection.update(ids=pending_ids, metadatas=pending_metas)
+        updated += len(pending_ids)
+        pending_ids, pending_metas = [], []
+
+    offset = 0
+    while True:
+        res = collection.get(
+            where={"ingest_mode": "convos"}, include=["metadatas"], limit=PAGE, offset=offset
+        )
+        ids = res["ids"]
+        if not ids:
+            break
+        for drawer_id, meta in zip(ids, res["metadatas"]):
+            scanned += 1
+            basename = os.path.basename(meta.get("source_file") or "")
+            if basename in cache:
+                authored = cache[basename]
+            else:
+                path = index.get(basename)
+                authored = _extract_authored_at(path) if path else None
+                cache[basename] = authored
+                if path is None and basename:
+                    unresolved.add(basename)
+            if authored and meta.get("authored_at") != authored:
+                new_meta = dict(meta)
+                new_meta["authored_at"] = authored
+                pending_ids.append(drawer_id)
+                pending_metas.append(new_meta)
+                if len(pending_ids) >= BATCH:
+                    flush()
+        offset += len(ids)
+    flush()
+    return {
+        "scanned": scanned,
+        "updated": updated,
+        "resolved_files": sum(1 for v in cache.values() if v),
+        "unresolved_files": len(unresolved),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--palace", required=True, help="Path to the ChromaDB palace dir")
+    parser.add_argument(
+        "--sessions",
+        action="append",
+        default=[],
+        required=True,
+        help="Directory holding .jsonl transcripts (repeatable)",
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Write changes (default is a dry run that only reports counts)",
+    )
+    args = parser.parse_args()
+
+    client = chromadb.PersistentClient(path=os.path.expanduser(args.palace))
+    collection = client.get_collection(COLLECTION)
+    stats = backfill_authored_at(collection, args.sessions, apply=args.apply)
+    mode = "APPLIED" if args.apply else "DRY-RUN (use --apply to write)"
+    print(
+        f"{mode}: scanned={stats['scanned']} updated={stats['updated']} "
+        f"resolved_files={stats['resolved_files']} unresolved_files={stats['unresolved_files']}"
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/docker-smoke.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+#
+# Smoke-test a built MemPalace image and the shipped Compose files.
+#
+#   scripts/docker-smoke.sh [IMAGE]        # default image: mempalace:smoke
+#
+# CI builds the container images but, until this existed, never ran one and
+# never parsed a Compose file — so two defects that break the very first
+# documented command shipped anyway: a `docker-compose.yml` Compose refused to
+# load (#2188), and an embedding-vector bug that aborted `mine` on the first
+# drawer (#2187). A build that succeeds proves the image *compiles*; it says
+# nothing about whether the thing inside it works.
+#
+# So this exercises the paths the README actually tells users to run, and
+# asserts on returned content rather than exit codes alone — the `mine` crash
+# was a non-zero exit, but the verbatim read-back is what proves the palace
+# really holds the text.
+#
+# Requires: docker (with the compose plugin) and python3.
+
+set -euo pipefail
+
+IMAGE="${1:-mempalace:smoke}"
+VOLUME="mempalace-smoke-$$"
+WORKDIR="$(mktemp -d)"
+
+# The image runs as uid 1000, which will not match the host user on Linux, and
+# bind mounts carry host ownership through unchanged (Docker Desktop's uid
+# mapping hides this on macOS). `mktemp -d` gives 0700, so the container could
+# not even stat inside /work. Model an ordinary project checkout instead: 0755
+# dir, 0644 files — which is what makes the README's `-v /path/to/project:/work`
+# work for a normal repo.
+chmod 0755 "$WORKDIR"
+
+# A sentence we can assert on verbatim. Storing user words exactly is the
+# project's core promise, so the read-back check is the real assertion here.
+NEEDLE="eleven round trips to render one screen"
+
+cleanup() {
+    docker volume rm -f "$VOLUME" >/dev/null 2>&1 || true
+    rm -rf "$WORKDIR"
+}
+trap cleanup EXIT
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+# Dump captured output on failure. Lines are clipped because the failure this
+# most often reports — a rejected embedding batch — puts a whole 384-dim vector
+# repr on a single line, which buries the actual message in the CI log.
+dump() {
+    echo "--- last 40 lines (clipped to 500 cols) ---" >&2
+    printf '%s\n' "$1" | tail -40 | cut -c1-500 >&2
+    echo "-------------------------------------------" >&2
+}
+
+echo "== 1/5  Compose files parse =="
+# A key with only comments under it parses as null and Compose rejects the
+# whole file. `config` is the cheapest way to catch that class of defect.
+docker compose -f docker-compose.yml config --quiet \
+    || fail "docker-compose.yml is not a valid Compose file"
+echo "  docker-compose.yml ok"
+
+# The server file interpolates a mandatory token; a dummy satisfies the
+# `:?` guard so the rest of the file still gets validated.
+MEMPALACE_MCP_HTTP_TOKEN=smoke-token \
+    docker compose -f deploy/docker-compose.server.yml config --quiet \
+    || fail "deploy/docker-compose.server.yml is not a valid Compose file"
+echo "  deploy/docker-compose.server.yml ok"
+
+echo "== 2/5  CLI entrypoint dispatch =="
+# docker-entrypoint.sh routes `cli`/`mcp` keywords and forwards anything else
+# to the CLI. Both forms are documented, so both are checked.
+docker run --rm -v "$VOLUME:/data" "$IMAGE" cli --version >/dev/null \
+    || fail "'cli --version' failed"
+docker run --rm -v "$VOLUME:/data" "$IMAGE" --version >/dev/null \
+    || fail "bare '--version' passthrough failed"
+echo "  'cli ...' and bare passthrough both dispatch"
+
+echo "== 3/5  mine a mounted directory =="
+cat > "$WORKDIR/notes.md" <<EOF
+# Architecture decision
+
+We switched from REST to GraphQL because the mobile client was making
+$NEEDLE. We settled on persisted queries.
+EOF
+chmod 0644 "$WORKDIR/notes.md"
+
+# First run also downloads the ~80 MB embedding model into the volume, so this
+# doubles as a check that a cold container can reach and cache it.
+mine_out="$(docker run --rm -v "$VOLUME:/data" -v "$WORKDIR:/work" "$IMAGE" mine /work 2>&1)" \
+    || { dump "$mine_out"; fail "'mine /work' exited non-zero"; }
+
+echo "$mine_out" | grep -q "Drawers filed: 1" \
+    || { dump "$mine_out"; fail "expected 'Drawers filed: 1'"; }
+echo "  filed 1 drawer"
+
+echo "== 4/5  search it back from a separate container =="
+# A new container against the same volume: proves the palace persisted and
+# that the stored text is returned verbatim, not summarised.
+search_out="$(docker run --rm -v "$VOLUME:/data" "$IMAGE" search "why did we move off REST" 2>&1)" \
+    || { dump "$search_out"; fail "'search' exited non-zero"; }
+
+echo "$search_out" | grep -qF "$NEEDLE" \
+    || { dump "$search_out"; fail "search did not return the drawer verbatim"; }
+echo "  drawer returned verbatim"
+
+echo "== 5/5  MCP server over stdio =="
+# The README's MCP client config runs the image with -i and speaks JSON-RPC on
+# stdin/stdout. Drive a real handshake and one tool call.
+mcp_out="$(printf '%s\n' \
+    '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"smoke","version":"0"}}}' \
+    '{"jsonrpc":"2.0","method":"notifications/initialized"}' \
+    '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' \
+    '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"mempalace_search","arguments":{"query":"GraphQL"}}}' \
+    | docker run -i --rm -v "$VOLUME:/data" "$IMAGE" mcp 2>/dev/null)"
+
+# The transcript goes to a file and the parser reads it via argv: the heredoc
+# already occupies this command's stdin.
+printf '%s\n' "$mcp_out" > "$WORKDIR/mcp.jsonl"
+
+NEEDLE="$NEEDLE" python3 - "$WORKDIR/mcp.jsonl" <<'PY'
+import json, os, sys
+
+needle = os.environ["NEEDLE"]
+seen = {}
+with open(sys.argv[1], encoding="utf-8") as fh:
+    for line in fh:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            msg = json.loads(line)
+        except ValueError:
+            continue  # non-JSON banner lines are not our concern here
+        if "id" in msg:
+            seen[msg["id"]] = msg
+
+
+def result(rid, what):
+    msg = seen.get(rid)
+    if msg is None:
+        sys.exit(f"FAIL: no JSON-RPC response for {what} (id={rid})")
+    if "error" in msg:
+        sys.exit(f"FAIL: {what} returned an error: {msg['error']}")
+    return msg["result"]
+
+
+name = result(1, "initialize")["serverInfo"]["name"]
+if name != "mempalace":
+    sys.exit(f"FAIL: initialize reported serverInfo.name={name!r}")
+print(f"  initialize ok (server: {name})")
+
+tools = result(2, "tools/list")["tools"]
+if not tools:
+    sys.exit("FAIL: tools/list returned no tools")
+print(f"  tools/list ok ({len(tools)} tools)")
+
+call = result(3, "tools/call")
+if call.get("isError"):
+    sys.exit(f"FAIL: mempalace_search reported isError: {call}")
+text = "".join(part.get("text", "") for part in call.get("content", []))
+if needle not in text:
+    sys.exit("FAIL: mempalace_search did not return the drawer verbatim")
+print("  mempalace_search ok (drawer returned verbatim)")
+PY
+
+echo
+echo "docker smoke: all checks passed ($IMAGE)"

scripts/mempalace_repair_encoding.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""Repair legacy Windows mojibake in a MemPalace collection."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from mempalace.config import MempalaceConfig
+from mempalace.encoding_repair import (
+    repair_collection,
+    restore_collection,
+)
+from mempalace.palace import (
+    get_collection,
+    mine_palace_lock,
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Conservatively repair high-confidence UTF-8 mojibake "
+            "in legacy MemPalace drawers. Dry-run is the default."
+        )
+    )
+    parser.add_argument(
+        "--palace",
+        help=("Palace path; defaults to the configured palace."),
+    )
+    parser.add_argument(
+        "--collection",
+        help=("Collection name; defaults to the configured collection."),
+    )
+    parser.add_argument(
+        "--page-size",
+        type=int,
+        default=500,
+        help=("Rows scanned or restored per page (default: 500)."),
+    )
+    parser.add_argument(
+        "--preview-chars",
+        type=int,
+        default=180,
+        help=("Maximum characters shown in each before/after preview."),
+    )
+
+    action = parser.add_mutually_exclusive_group()
+    action.add_argument(
+        "--apply",
+        action="store_true",
+        help=(
+            "Write repairs after creating a private JSONL backup. "
+            "Without this flag, the command is read-only."
+        ),
+    )
+    action.add_argument(
+        "--restore-backup",
+        metavar="PATH",
+        help=("Restore original documents from a prior repair backup."),
+    )
+
+    parser.add_argument(
+        "--backup",
+        metavar="PATH",
+        help=(
+            "Backup destination used with --apply. Defaults to a "
+            "timestamped file beside the palace. Existing files "
+            "are never overwritten."
+        ),
+    )
+    return parser
+
+
+def _default_backup_path(
+    palace_path: str,
+) -> Path:
+    palace = Path(palace_path).expanduser().resolve()
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+
+    return palace.parent / (f"{palace.name}.encoding-repair-{timestamp}.jsonl")
+
+
+def _preview(
+    text: str,
+    limit: int,
+) -> str:
+    compact = text.replace(
+        "\r",
+        "\\r",
+    ).replace(
+        "\n",
+        "\\n",
+    )
+
+    if len(compact) <= limit:
+        return compact
+
+    return (
+        compact[
+            : max(
+                0,
+                limit - 1,
+            )
+        ]
+        + "…"
+    )
+
+
+def _print_change(
+    drawer_id: str,
+    before: str,
+    after: str,
+    *,
+    preview_chars: int,
+) -> None:
+    print()
+    print(f"Drawer: {drawer_id}")
+    print(
+        "  before: "
+        + _preview(
+            before,
+            preview_chars,
+        )
+    )
+    print(
+        "  after:  "
+        + _preview(
+            after,
+            preview_chars,
+        )
+    )
+
+
+def _reconfigure_stdio_utf8_on_windows() -> None:
+    """Decode stdio as UTF-8 on Windows for the encoding-repair CLI.
+
+    Thin wrapper around the shared helper in ``mempalace._stdio``, matching
+    ``cli.py`` and ``fact_checker.py``. stdout/stderr override to ``replace``
+    because every proposed change prints a before/after preview of verbatim
+    drawer text -- under the legacy console codepage this tool is written for,
+    ``strict`` raises on the mojibake lead bytes themselves and aborts the run
+    before a single drawer is repaired.
+    """
+    from mempalace._stdio import reconfigure_stdio_utf8_on_windows
+
+    reconfigure_stdio_utf8_on_windows(stdout_errors="replace", stderr_errors="replace")
+
+
+def main() -> int:
+    _reconfigure_stdio_utf8_on_windows()
+
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if args.page_size < 1:
+        parser.error("--page-size must be at least 1")
+
+    if args.preview_chars < 40:
+        parser.error("--preview-chars must be at least 40")
+
+    if args.backup and not args.apply:
+        parser.error("--backup requires --apply")
+
+    config = MempalaceConfig()
+    palace_path = args.palace or config.palace_path
+    collection_name = args.collection or getattr(
+        config,
+        "collection_name",
+        "mempalace_drawers",
+    )
+
+    if args.restore_backup:
+        with mine_palace_lock(palace_path):
+            collection = get_collection(
+                palace_path,
+                collection_name=(collection_name),
+                create=False,
+            )
+            report = restore_collection(
+                collection,
+                args.restore_backup,
+                batch_size=(args.page_size),
+            )
+
+        print("Mode: RESTORE")
+        print(f"Backup records validated: {report['validated']}")
+        print(f"Documents restored: {report['restored']}")
+        return 0
+
+    backup_path = None
+
+    if args.apply:
+        backup_path = Path(args.backup) if args.backup else _default_backup_path(palace_path)
+
+    def show_change(
+        drawer_id: str,
+        before: str,
+        after: str,
+    ) -> None:
+        _print_change(
+            drawer_id,
+            before,
+            after,
+            preview_chars=(args.preview_chars),
+        )
+
+    try:
+        if args.apply:
+            with mine_palace_lock(palace_path):
+                collection = get_collection(
+                    palace_path,
+                    collection_name=(collection_name),
+                    create=False,
+                )
+                report = repair_collection(
+                    collection,
+                    apply=True,
+                    page_size=(args.page_size),
+                    backup_path=(backup_path),
+                    on_change=show_change,
+                )
+        else:
+            collection = get_collection(
+                palace_path,
+                collection_name=(collection_name),
+                create=False,
+            )
+            report = repair_collection(
+                collection,
+                apply=False,
+                page_size=(args.page_size),
+                on_change=show_change,
+            )
+    except FileExistsError as exc:
+        print(
+            f"ERROR: backup file already exists; refusing to overwrite it: {exc.filename}",
+            file=sys.stderr,
+        )
+        return 2
+
+    print()
+    print("Mode: " + ("APPLY" if args.apply else "DRY RUN"))
+    print(f"Rows scanned: {report['scanned']}")
+    print(f"Documents needing repair: {report['changed']}")
+    print(f"Documents updated: {report['updated']}")
+
+    if report["backup_path"]:
+        print(f"Original-document backup: {report['backup_path']}")
+
+    if not args.apply and report["changed"]:
+        print()
+        print(
+            "Review every change above, then run again with "
+            "--apply. A private, non-overwriting backup will "
+            "be written before any document is updated."
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())