Merge pull request #2220 from MemPalace/fix/windows-hybrid-search-empty-results

fix(tests): harden hybrid search against empty Windows Chroma reads

.agents/plugins/marketplace.json

@@ -8,11 +8,11 @@
       "name": "mempalace",
       "source": {
         "source": "local",
-        "path": "./.codex-plugin"
+        "path": "./"
       },
       "policy": {
         "installation": "AVAILABLE",
-        "authentication": "NONE"
+        "authentication": "ON_INSTALL"
       },
       "category": "Coding"
     }

.claude-plugin/marketplace.json

@@ -9,7 +9,7 @@
       "name": "mempalace",
       "source": "./.claude-plugin",
       "description": "AI memory system — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, guided setup.",
-      "version": "3.6.0",
+      "version": "3.7.0",
       "author": {
         "name": "milla-jovovich"
       }

.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "mempalace",
-  "version": "3.6.0",
+  "version": "3.7.0",
   "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.",
   "author": {
     "name": "milla-jovovich"

.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "mempalace",
-  "version": "3.6.0",
+  "version": "3.7.0",
   "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.",
   "author": {
     "name": "milla-jovovich"
@@ -18,12 +18,7 @@
     "search"
   ],
   "skills": "./skills/",
-  "hooks": "./hooks.json",
-  "mcpServers": {
-    "mempalace": {
-      "command": "mempalace-mcp"
-    }
-  },
+  "mcpServers": "./.mcp.json",
   "interface": {
     "displayName": "MemPalace",
     "shortDescription": "AI memory system for Codex",

.github/workflows/ci.yml

@@ -30,13 +30,18 @@ jobs:
           python-version: "3.13"
           cache: 'pip'
       - run: pip install -e ".[dev]"
-      # ChromaDB's rust HNSW core intermittently fails compaction on Windows
-      # ("Failed to apply logs to the hnsw segment writer") regardless of our
-      # code — a long-standing, non-reproducible-on-Linux/macOS flake. Retry
-      # ONLY that specific transient error (via --only-rerun) so real,
+      # ChromaDB's rust HNSW core intermittently fails compaction or reader
+      # initialization on Windows ("Failed to apply logs to the hnsw segment
+      # writer" / "Error creating hnsw segment reader: Nothing found on disk")
+      # regardless of our code — long-standing, non-reproducible-on-Linux/macOS
+      # flakes. Retry ONLY those specific transient errors (via --only-rerun) so real,
       # deterministic failures still fail on the first run. Linux/macOS jobs
       # deliberately run with no reruns so genuine regressions surface there.
-      - run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10 --reruns 2 --reruns-delay 5 --only-rerun "Failed to apply logs to the hnsw segment writer"
+      - run: >-
+          python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace
+          --cov-report=term-missing --cov-fail-under=80 --durations=10 --reruns 2
+          --reruns-delay 5 --only-rerun
+          "Failed to apply logs to the hnsw segment writer|Error creating hnsw segment reader: Nothing found on disk"
 
   test-macos:
     runs-on: macos-latest
@@ -59,6 +64,10 @@ jobs:
       # Keep this pin identical to the `ruff==` pin in pyproject.toml
       # ([project.optional-dependencies].dev and [dependency-groups].dev)
       # so CI and `pip install -e ".[dev]"` format/lint identically.
-      - run: pip install "ruff==0.15.14"
+      # This drifted once (CI on 0.15.14 against pyproject's 0.15.20) and the
+      # drift was invisible: the lint job never installs from pyproject, so a
+      # dependabot bump to the pin there passes CI without the new version
+      # ever running. test_ruff_pins_match asserts the two stay equal.
+      - run: pip install "ruff==0.16.1"
       - run: ruff check .
       - run: ruff format --check .

.github/workflows/docker-publish.yml

@@ -16,8 +16,49 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
+  # Run the image before anyone can publish it. A green `build` only proves the
+  # Dockerfile compiles — it never started a container, so defects that break
+  # the first documented command shipped anyway (#2187, #2188). This builds
+  # amd64 natively, loads it into the local daemon, and exercises the README
+  # paths: CLI mine + verbatim search across two containers, a real MCP stdio
+  # handshake, and `compose config` on both shipped Compose files.
+  smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      # amd64 only and `load: true`: the smoke run needs a real image in the
+      # local daemon, and buildx cannot load a multi-arch manifest. Shares the
+      # `build` job's gha cache scope, so this is mostly a cache hit.
+      - name: Build image for smoke test
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64
+          push: false
+          load: true
+          tags: mempalace:smoke
+          # Read the publish job's cache too (its amd64 layers are identical),
+          # but write to a private scope so an amd64-only export never lands on
+          # top of that job's multi-arch one. Same split as `scope=gpu` below.
+          cache-from: |
+            type=gha
+            type=gha,scope=smoke
+          cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=smoke' || '' }}
+
+      # Reaches Chroma's S3 once to fetch the ~80 MB embedding model, so it is
+      # network-dependent; that download is itself part of a user's first run.
+      - name: Smoke test
+        run: ./scripts/docker-smoke.sh mempalace:smoke
+
   build:
     runs-on: ubuntu-latest
+    # Never publish an image the smoke test has not cleared.
+    needs: smoke
     permissions:
       contents: read
       packages: write

.gitignore

@@ -2,6 +2,7 @@
 dist/
 build/
 __pycache__/
+.hypothesis/
 *.pyc
 .pytest_cache/
 mempal.yaml

.mcp.json

@@ -0,0 +1,7 @@
+{
+  "mcpServers": {
+    "mempalace": {
+      "command": "mempalace-mcp"
+    }
+  }
+}

CHANGELOG.md

@@ -10,6 +10,61 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ---
 
+## [3.7.0] — 2026-08-11
+
+### Features
+
+- **Embeddings via any OpenAI-compatible `/v1/embeddings` endpoint.** New `embedding_model: "openai-compat"` option computes embeddings on a server (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or a self-hosted endpoint) instead of a local ONNX model — useful for larger/multilingual embedders such as Qwen3-Embedding, or GPU offload. New `OpenAICompatEmbeddingFunction` in [`mempalace/embedding.py`](mempalace/embedding.py) speaks the standard `/v1/embeddings` protocol over stdlib `urllib` (no new dependency), batches requests, re-sorts the response by `index`, and L2-normalizes for the cosine collection. Endpoint settings are resolved by `MempalaceConfig` as a single source of truth — `embedding_api_url` / `embedding_api_model` / `embedding_api_key` in `config.json`, each overridable via the matching `MEMPALACE_EMBEDDING_API_*` env var. The embedding function's `name()` encodes the model id so changing it forces `mempalace repair rebuild-index` (different vector space). Mirrors the existing `openai-compat` LLM provider naming; stays local when the endpoint is on your machine/LAN. (#1559)
+- **`mempalace_search` / `mempalace search` — `since`/`before` date window.** Semantic search is poor at temporal queries ("what did we discuss this week?" scores ~0.35 even when matching drawers exist), so the search surfaces now accept the same `[since, before)` window `list_drawers` gained in #1128: inclusive/exclusive ISO bounds compared wall-clock against each drawer's `filed_at`, undated drawers excluded while a bound is active. The window applies on every candidate path (vector, `candidate_strategy="union"`, and the BM25-only fallback); the vector candidate pool widens under an active window (ChromaDB cannot range-compare string metadata server-side), and a full pool is flagged via `date_filter_pool_truncated` instead of passing silently. Shared parsing lives in the new `mempalace.date_window` module, also backing the `list_drawers` filter. (#463)
+- **Hermes agent memory provider (core).** In-package Hermes `MemoryProvider` files live turns through the shared `file_conversation_exchange()` path (same metadata as convo mining), with a background worker so the agent loop never blocks on Chroma. Wake-up L1 uses the long-lived collection only — a second PersistentClient is not opened, avoiding local SQLite races. Install/backfill/docs remain a stacked follow-up. (#1915, #2215)
+- **Explicit RFC 002 source adapters on `mempalace mine`.** `mempalace mine <source> --source <adapter>` resolves adapters through the registry, holds the palace writer lease for the full ingest, and keeps dry-runs inert (no real backend/KG open). Legacy `--mode` paths are unchanged. (#2068, #2062)
+- **MCP refuses writes when the served library is no longer the one that started.** Detects mempalace (and chromadb when that backend is active) version drift or uninstall after upgrade without restarting the server; opt-out via `MEMPALACE_MCP_ALLOW_STALE_LIBRARY`. (#2081, #899)
+- **Hook write-routing through the daemon.** Background hook saves and mines can honor the shared write-routing policy so multi-session setups serialize mutations through one local owner instead of racing the palace. (#2030, #1963)
+
+### Performance
+
+- **EmbeddingGemma groups documents by size before sub-batching.** The tokenizer pads every row of a sub-batch to the longest sequence in it, so arrival order decided the bill: one long verbatim message dragged a whole sub-batch up to its own length. Measured over 43,157 `sweep` drawers from 160 Claude Code transcripts, padded token slots drop 39.7% and the quadratic attention term 45.0%. Vectors move by at most one float32 ULP (1.2e-07 absolute, cosine 0.99999992), which is reduction-order rounding and not a change of meaning. Applies to `embedding_model: embeddinggemma` only; the default MiniLM embedder pads to a fixed width and was never affected. (#2104)
+- **HNSW capacity probes are cached** and invalidated by palace file signature, so repeated MCP status/taxonomy paths no longer re-scan native segment files on every call. (#2051, #1471)
+- **`chunk_text` line numbering is O(N)** via incremental tallies, fixing multi-second hangs on large sources. (#2054, #2055)
+
+### Bug Fixes
+
+- **Orphaned per-source-file mine locks are reaped instead of accumulating forever.** `_cleanup_mine_lock_file` reclaims a lock correctly on the happy path, but only for the specific lock its own `mine_lock` context manager just released — a process killed abruptly (SIGKILL, force-quit, host crash) never reaches that cleanup, and nothing else revisited the file afterward. One long-lived installation was found with 5,636 stale entries in `~/.mempalace/locks/`, the oldest several months old, none held by any live process. `mine_lock` now opportunistically reaps locks older than an hour via the same nonblocking-flock-reacquire safety check `_cleanup_mine_lock_file` already uses, throttled to once per 15 minutes so it costs nothing on the common path. `mine_palace_*.lock` (the newer per-palace lock) is untouched — it has its own lifecycle and holder tracking.
+- **Windows MCP stdout capture falls back cleanly** when fd-level redirection is unavailable, and fails closed if protocol stdout cannot be restored after a successful redirect. (#2211, #2210)
+- **Claude Code `subagents/` transcripts are skipped by default** when mining conversations (98%+ noise on typical workspaces); `--include-subagents` opts back in. (#1330, #1217)
+- **Worktree transcript cwd no longer mints a throwaway wing** — wing is derived from the project root above `.claude/worktrees/`. (#2206)
+- **Markdown emphasis/bold no longer scores as emotional content** in the general extractor. (#2199, #2197)
+- **Encoding hardening on Windows locales:** pin `encoding=utf-8` on config/dialect opens; safer mojibake repair that does not destroy clean Portuguese/Vietnamese/Turkish prose; repair-encoding CLI reconfigures stdio like other entry points; remaining non-ASCII CLI symbols replaced for GBK consoles. (#2098, #2208, #2194, #1104, #1034, #2193)
+- **`rebuild_index` holds the palace writer lease** for the full snapshot→rebuild/swap cycle so concurrent writers cannot recreate HNSW divergence mid-repair. (#2195)
+- **`MEMPALACE_PALACE_PATH` is restored after each service entrypoint** so multi-call processes do not leak the first call's palace into the second. (#2192)
+- **systemd unit uses `Restart=always`** and documents `MEMPALACE_MCP_IDLE_HOURS=0` so the idle watchdog does not leave a dedicated server down after a clean exit. (#2203, #2204)
+- **Mining works again on the default Chroma backend.** Once Chroma began declaring `requires_explicit_embeddings`, every write started routing through `EmbeddingCollection`, whose `_embed_texts` built rows with `list(ndarray)` — that unpacks into `np.float32` *scalars*, which chromadb rejects outright (`Expected embeddings to be a list of floats or ints, a list of lists, a numpy array, or a list of numpy arrays`). `mine`, and every other write against a default palace, aborted on the first drawer. Vectors now convert to real Python floats. The suite was structurally blind to this: conftest's autouse embedding fixture replaces `_embed_texts` itself for every module outside `test_embedding` / `test_embeddinggemma`, so the defective function was never executed under test — the regression tests therefore live in `test_embedding.py`, where that stub does not apply. (#2187)
+- **ChatGPT data exports are parsed instead of stored as raw JSON.** A real `conversations.json` is a top-level array of conversations, which no parser claimed, so `mine --mode convos` chunked the raw JSON and lost every speaker turn while reporting success. Each conversation now normalizes to its own transcript, as Claude.ai privacy exports already do, so per-conversation dedup survives a re-export. The ChatGPT parser also type-checks its nested shapes, so an unrelated array carrying a `mapping` key is declined instead of raising. (#2160)
+- **Local backends enforce process-lifetime single-writer ownership.** File-backed and unknown backends require one writer owner for the full process lifetime (daemon holds the lease until workers exit; writable MCP HTTP acquires ownership before bind and refuses startup when blocked). Read-only MCP may coexist; `sqlite_exact` opens genuine query-only/immutable readers; remote Milvus/Zilliz remain multi-process. Addresses multi-writer SQLite/WAL corruption from MCP HTTP + daemon + mine topologies. (#2079, #2045)
+- **Chroma HNSW write defaults match chromadb** (`batch_size=100` / `sync_threshold=1000`) instead of the old 2/2 bloat guard that rewrote segments thousands of times on large mines. (#2107, #2106)
+- **Repair and recovery are safer under contention.** `repair --mode from-sqlite` takes the mine-lock before archiving; rebuilds preserve a verified temp collection when the live swap fails; sparse drawers with zero `embedding_metadata` rows are no longer dropped; truncated ID pagination fails loud instead of pretending success. (#2109, #2086, #2087)
+- **`repair --mode from-sqlite --dry-run` is a true preview.** It no longer archives or re-embeds; it prints per-collection would-be counts from SQLite ground truth and exits without touching the palace. Unreadable counts fail closed instead of inventing zeros. (#2133, #2095, #1654)
+- **`repair --dry-run` is a true preview in the default (legacy) mode too.** That path ignored the flag entirely and ran the real rebuild — deleting any existing `<palace>.backup`, copying the palace over it, and re-filing the drawers collection. It now prints a read-only plan and exits without opening a chromadb client, which is itself a write to `chroma.sqlite3`. The plan names the live-collection delete the rebuild performs, warns when an existing backup would be destroyed, and reports the truncation guard as disabled when `--confirm-truncation-ok` is set. An isolated FTS5 inverted-index error is reported as auto-healable instead of raising the manual-recovery abort a real run never reaches, unreadable counts fail closed with a non-zero exit, and the `--dry-run` help no longer claims to be `--mode max-seq-id` only. (#2144)
+- **`repair` and `migrate` survive a socket or a named pipe in the palace directory.** Both take a whole-directory `shutil.copytree` backup before they overwrite the palace, and `copytree` cannot duplicate a Unix domain socket left beside `chroma.sqlite3`, nor a named pipe: it copied everything else and then raised `shutil.Error`, so the command died at the backup step before any rebuild ran, and re-running only repeated it. Neither entry carries palace data, so both are now skipped and named in the output while the backup completes. Device nodes are skipped too, because the copy dereferences them instead of failing on them. Every other copy failure still aborts the command, including a symlink whose target cannot be read: no errno separates a deleted target from one on a volume that is not mounted, so an entry that may have held data is left for the copy to fail on. (#2207)
+- **HNSW divergence is preflighted before remaining `col.count()` crash sites** across mine, dedup, migrate, repair, and palace helpers. (#2093)
+- **Re-mine and conversation ingest no longer lose or duplicate drawers.** Content-hash dedup prevents duplicate LLM conversation drawers; sweeper drawers are excluded from convo extract-mode purge scope and failed purges abort; search returns round-trippable `drawer_id` values for `get_drawer`. (#2050, #2125, #2089, #2090, #2044, #2080)
+- **MCP and daemon lifecycle harden multi-agent use.** Read-only mode refuses config and checkpoint-ack tools that rewrite host state; stdio MCP exits on stdin EOF/broken pipe so orphaned sessions release locks; daemon jobs refused the palace lock are deferred instead of failed permanently. (#2126, #2103, #2101, #2072, #2029, #2014)
+- **Entity-candidate extraction no longer hangs on long ASCII runs** (base64, minified blobs) while preserving CJK/non-ASCII text. (#2127, #2065, #2063)
+- **Mining windowing rejects `chunk_overlap` above half the chunk size**, stopping infinite chunk_text loops. (#2056, #2058)
+- **`docker-compose.yml` is valid again.** The `environment:` key was declared with only comments beneath it, which YAML parses as null, so Compose rejected the whole file (`services.mcp.environment must be a mapping`) — every documented Compose command failed before starting. The key is commented out along with its examples, which now use mapping syntax so uncommenting them yields a valid block. (#2188)
+
+### Documentation
+
+- Operator write-routing / single-writer recovery notes in `docs/write-routing-policy.md`. (#2079)
+- Remote-server guide wording for read-only tools that change host state. (#2126)
+- **The README's Docker section leads with the published image.** It previously documented only `docker build`, even though `ghcr.io/mempalace/mempalace` ships multi-arch — and a clone builds `develop`, not the release, so a build and a pull could differ silently. It now covers what the omissions actually cost: the MCP client config mounts a transcripts directory (without one the server starts and every mine finds nothing), the first embedding call downloads ~80 MB into `/data` and looks like a hang, bind mounts keep host ownership against the image's uid 1000 so a `0700` directory fails with a bare `PermissionError` on Linux (and `--user` is the wrong fix — `/data` is mode 700 owned by that uid), mining sources can be mounted read-only, and the GPU image is x86_64-only. (#2196)
+
+### Internal
+
+- **The Docker workflow runs the image before publishing it.** It previously built both images without ever starting a container or parsing a Compose file, so a green run only proved the Dockerfile compiled — which is how two defects that break the first documented command shipped past it. `scripts/docker-smoke.sh` now validates both Compose files, checks entrypoint dispatch, mines a mounted directory, asserts the drawer reads back verbatim from a separate container, and drives a real MCP stdio handshake; publication is gated on it. The script runs the same way locally: `scripts/docker-smoke.sh <image>`. (#2189)
+
+---
+
 ## [3.6.0] — 2026-07-14
 
 ### Features
@@ -630,7 +685,8 @@ Initial public release.
 
 ---
 
-[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.6.0...HEAD
+[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.7.0...HEAD
+[3.7.0]: https://github.com/MemPalace/mempalace/compare/v3.6.0...v3.7.0
 [3.6.0]: https://github.com/MemPalace/mempalace/compare/v3.5.0...v3.6.0
 [3.5.0]: https://github.com/MemPalace/mempalace/compare/v3.4.1...v3.5.0
 [3.4.1]: https://github.com/MemPalace/mempalace/compare/v3.4.0...v3.4.1

CONTRIBUTING.md

@@ -119,7 +119,7 @@ If you're planning a significant change, open an issue first to discuss the appr
 
 - **Discord**: [Join us](https://discord.com/invite/ycTQQCu6kn)
 - **Issues**: Bug reports and feature requests welcome
-- **Discussions**: For questions and ideas
+- **Questions and ideas**: Please use Discord or open an issue while GitHub Discussions are not enabled for this repo
 
 ## License
 

README.md

@@ -72,39 +72,83 @@ pip install mempalace
 ### Docker
 
 A container image is also available for running the MCP server or the CLI
-without a local Python toolchain. Everything persists under `/data` (palace,
-config, and the cached embedding model), so mount a volume there.
+without a local Python toolchain. Multi-arch (amd64 + arm64), so it runs
+natively on Apple Silicon:
 
 ```bash
-# Build the image (CPU; bundles the `extract` + `spellcheck` extras)
-docker build -t mempalace .
-
-# MCP server over stdio — note the `-i` flag (JSON-RPC needs stdin)
-docker run -i --rm -v mempalace-data:/data mempalace
-
-# Run any CLI command instead (mount the host directory you want to mine)
-docker run --rm -v mempalace-data:/data -v /path/to/project:/work mempalace mine /work
-docker run --rm -v mempalace-data:/data mempalace search "why GraphQL"
+docker pull ghcr.io/mempalace/mempalace:latest
 ```
 
-Wire it into an MCP client (e.g. Claude Code) as a stdio server:
+Everything persists under `/data` — palace, config, and the cached embedding
+model — so mount a volume there and reuse it across runs:
+
+```bash
+# MCP server over stdio — note the `-i` flag (JSON-RPC needs stdin)
+docker run -i --rm -v mempalace-data:/data ghcr.io/mempalace/mempalace
+
+# Run any CLI command instead. The container only sees what you mount, so
+# mount the directory you want to mine — read-only is enough, mining never
+# writes to the source.
+docker run --rm -v mempalace-data:/data -v /path/to/project:/work:ro \
+  ghcr.io/mempalace/mempalace mine /work
+docker run --rm -v mempalace-data:/data ghcr.io/mempalace/mempalace search "why GraphQL"
+```
+
+The first command that needs embeddings downloads the model into `/data`
+(~80 MB for the default `minilm`, ~300 MB for `embeddinggemma`). It is a
+one-off as long as the volume persists, but it does mean the first call is
+slow and needs network — worth knowing before assuming a hung container.
+
+Wire it into an MCP client (e.g. Claude Code) as a stdio server. Mount
+anything you want the server to be able to mine — it cannot reach your
+transcripts otherwise:
 
 ```json
 {
   "mcpServers": {
     "mempalace": {
       "command": "docker",
-      "args": ["run", "-i", "--rm", "-v", "mempalace-data:/data", "mempalace"]
+      "args": [
+        "run", "-i", "--rm",
+        "-v", "mempalace-data:/data",
+        "-v", "/absolute/path/to/.claude/projects:/transcripts:ro",
+        "ghcr.io/mempalace/mempalace"
+      ]
     }
   }
 }
 ```
 
-`docker compose run --rm mcp` works too (see `docker-compose.yml`). For
-CUDA-accelerated embeddings, build the GPU variant with
-`docker build -f Dockerfile.gpu -t mempalace:gpu .` and run it with
-`--gpus all`. Customise the bundled extras at build time, e.g.
-`docker build --build-arg EXTRAS="extract,spellcheck" -t mempalace .`.
+Use a real absolute path there — `~` and `$HOME` are not expanded by every
+MCP client. Paths are container paths from then on: mine `/transcripts`, not
+`~/.claude/projects`.
+
+**Mount permissions on Linux.** The image runs as uid 1000 and bind mounts
+keep their host ownership, so a mounted directory has to be readable by that
+uid — an ordinary `0755` checkout is fine, a `0700` directory is not, and the
+failure surfaces as `PermissionError: [Errno 13]` rather than anything about
+Docker. Docker Desktop maps uids on macOS and Windows, so this only bites on
+Linux. Do **not** work around it with `--user`: `/data` is owned by uid 1000
+inside the image, so another uid cannot write the palace at all.
+
+`docker compose run --rm mcp` works too (see `docker-compose.yml`), and
+`deploy/docker-compose.server.yml` stands up the team server. To build the
+image yourself instead of pulling — required for the GPU variant, which is not
+published:
+
+```bash
+docker build -t mempalace .                                  # CPU
+docker build --build-arg EXTRAS="extract,spellcheck" -t mempalace .
+docker build -f Dockerfile.gpu -t mempalace:gpu .            # CUDA; run with --gpus all
+```
+
+The GPU image is x86_64-only: `onnxruntime-gpu` publishes no aarch64 Linux
+wheels, so that last build fails on an ARM host (including Apple Silicon) with
+a dependency-resolution error rather than an obvious one.
+
+Note that a build from a clone uses whatever branch you checked out; `develop`
+is the default branch, so pull the published image if you want the released
+version.
 
 ## Storage backends
 
@@ -209,8 +253,9 @@ Usage and tool reference:
 
 ## MCP server
 
-36 MCP tools cover palace reads/writes, knowledge-graph operations,
-cross-wing navigation, drawer management, and agent diaries. Installation
+44 MCP tools cover palace reads/writes, knowledge-graph operations,
+cross-wing navigation, drawer management, agent diaries, and agent
+coordination (logstream events + artifact handoffs). Installation
 and the full tool list:
 [mempalaceofficial.com/reference/mcp-tools](https://mempalaceofficial.com/reference/mcp-tools.html).
 
@@ -248,6 +293,7 @@ verbatim drawer per user/assistant message, idempotent and resume-safe.
 - Python 3.9+
 - A vector-store backend (ChromaDB by default)
 - ~300 MB disk for the embedding model. Onboarding (`python -m mempalace.onboarding`) offers `embeddinggemma-300m` (multilingual, 100+ languages, recommended) or `all-MiniLM-L6-v2` (English-only, ~30 MB). See the docstring at [`mempalace/embedding.py`](mempalace/embedding.py) for details and migration notes.
+- Optional — compute embeddings on a server instead of locally. Set `embedding_model: "openai-compat"` in `~/.mempalace/config.json` together with `embedding_api_url` / `embedding_api_model` (and `embedding_api_key` if the server needs auth) to use any OpenAI-compatible `/v1/embeddings` endpoint — LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or a self-hosted server (e.g. a larger multilingual or GPU-served embedder). Each key is overridable via the matching `MEMPALACE_EMBEDDING_API_*` env var. When the endpoint is on your machine or LAN, no content leaves your network. Switching to it requires `mempalace repair rebuild-index` (different vector space).
 
 No API key is required for the core benchmark path.
 
@@ -269,7 +315,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
 MIT — see [LICENSE](LICENSE).
 
 <!-- Link Definitions -->
-[version-shield]: https://img.shields.io/badge/version-3.6.0-4dc9f6?style=flat-square&labelColor=0a0e14
+[version-shield]: https://img.shields.io/badge/version-3.7.0-4dc9f6?style=flat-square&labelColor=0a0e14
 [release-link]: https://github.com/MemPalace/mempalace/releases
 [python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8
 [python-link]: https://www.python.org/

benchmarks/locomo_bench.py

@@ -665,7 +665,7 @@ def run_benchmark(
                 palace_cache = json.load(f)
             print(f"  Palace cache: {len(palace_cache)} room assignments loaded")
 
-    rerank_label = f" + LLM re-rank ({llm_model.split('-')[1]})" if llm_rerank_enabled else ""
+    rerank_label = f" + LLM re-rank ({llm_model})" if llm_rerank_enabled else ""
 
     print(f"\n{'=' * 60}")
     print("  MemPal × LoCoMo Benchmark")

benchmarks/longmemeval_bench.py

@@ -3015,7 +3015,7 @@ def run_benchmark(
 
         if unique_sessions and api_key and not skip_precompute:
             print(
-                f"  Diary ingest: pre-computing {len(unique_sessions)} sessions with {llm_model.split('-')[1]}..."
+                f"  Diary ingest: pre-computing {len(unique_sessions)} sessions with {llm_model}..."
             )
             done = 0
             cache_path = Path(diary_cache_file) if diary_cache_file else None
@@ -3051,9 +3051,8 @@ def run_benchmark(
     print(f"  Data:        {Path(data_file).name}")
     print(f"  Questions:   {len(data)}")
     print(f"  Granularity: {granularity}")
-    model_short = llm_model.split("-")[1] if "-" in llm_model else llm_model
-    rerank_label = f" + LLM re-rank ({model_short})" if llm_rerank_enabled else ""
-    diary_label = f" [diary ingest: {model_short}]" if mode == "diary" else ""
+    rerank_label = f" + LLM re-rank ({llm_model})" if llm_rerank_enabled else ""
+    diary_label = f" [diary ingest: {llm_model}]" if mode == "diary" else ""
     print(f"  Mode:        {mode}{diary_label}{rerank_label}")
     print(f"{'─' * 60}\n")
 

deploy/mempalace-server.service

@@ -24,7 +24,10 @@ User=mempalace
 Group=mempalace
 EnvironmentFile=/etc/mempalace/server.env
 ExecStart=mempalace serve --host 0.0.0.0 --port 8765
-Restart=on-failure
+# The idle watchdog exits with status 0, which on-failure reads as an
+# intentional stop, so the server would stay down. server.env disables the
+# watchdog; Restart=always covers a deployment that has not set it.
+Restart=always
 RestartSec=2
 
 # --- Hardening ---------------------------------------------------------------

deploy/server.env.example

@@ -23,6 +23,13 @@ MEMPALACE_EMBEDDING_DEVICE=auto
 # MEMPALACE_MCP_TLS_CERT=/etc/mempalace/tls/cert.pem
 # MEMPALACE_MCP_TLS_KEY=/etc/mempalace/tls/key.pem
 
+# --- Server lifecycle -----------------------------------------------------------
+# The MCP server exits by itself after this many idle hours (default 8) to free
+# the ChromaDB/HNSW file handles that per-session stdio servers leave behind.
+# A dedicated always-on server does not need that, so 0 disables it. Note that
+# /healthz traffic does not count as activity; only MCP requests reset the timer.
+MEMPALACE_MCP_IDLE_HOURS=0
+
 # --- Palace location (systemd / bare-metal) -------------------------------------
 # In Docker the palace lives on the mempalace-data volume by default.
 # MEMPALACE_PALACE_PATH=/var/lib/mempalace/palace

docker-compose.yml

@@ -21,12 +21,15 @@ services:
     tty: false
     volumes:
       - mempalace-data:/data
-    environment:
-      # Everything defaults correctly from HOME=/data (palace ->
-      # /data/.mempalace/palace, config -> /data/.mempalace, model cache ->
-      # /data/.cache). Override here only for non-default locations, e.g.:
-      # - MEMPALACE_EMBEDDING_MODEL=embeddinggemma   # multilingual (default: minilm)
-      # - MEMPALACE_PALACE_PATH=/data/custom/palace
+    # Everything defaults correctly from HOME=/data (palace ->
+    # /data/.mempalace/palace, config -> /data/.mempalace, model cache ->
+    # /data/.cache), so no environment block is needed. Uncomment the key
+    # *together with* an entry under it for non-default locations — a bare
+    # `environment:` with only comments beneath parses as null, and Compose
+    # rejects the whole file with "services.mcp.environment must be a mapping".
+    # environment:
+    #   MEMPALACE_EMBEDDING_MODEL: embeddinggemma   # multilingual (default: minilm)
+    #   MEMPALACE_PALACE_PATH: /data/custom/palace
 
 volumes:
   mempalace-data:

docs/hook-write-routing.md

@@ -0,0 +1,112 @@
+# Hook write routing
+
+Hook-triggered writes use the shared write-routing policy introduced for the
+Tier 3 rollout tracked in #1963.
+
+## Scope
+
+This applies to every routine write initiated by the Python hook layer:
+
+- Stop-hook diary checkpoints;
+- transcript/conversation ingest;
+- project auto-ingest;
+- SessionEnd final flushes;
+- PreCompact transcript ingest;
+- PreCompact synchronous project mining.
+
+It does not change CLI write routing. CLI adoption is a separate follow-up.
+
+## Policy behavior
+
+### `direct`
+
+Hooks use the existing direct in-process or subprocess paths.
+
+The daemon is not probed.
+
+### `prefer`
+
+Hooks use the daemon when it is already healthy.
+
+If the daemon is unavailable, hooks retain the historical direct fallback.
+
+### `require`
+
+Hooks use the daemon when it is already healthy.
+
+If the daemon is unavailable:
+
+- no in-process ChromaDB write runs;
+- no direct `mempalace mine` subprocess is started;
+- no daemon is cold-started from the hook;
+- the hook log records the skipped operation;
+- the hook returns a visible `systemMessage`;
+- the Stop save marker is not advanced, allowing a later retry.
+
+## Why hooks do not start the daemon
+
+Hooks operate under strict latency budgets. Starting a Python daemon and its
+storage dependencies from a Stop or SessionEnd hook can exceed that budget.
+
+A supervised installation using `require` must start the daemon earlier, for
+example at login, plugin initialization, or session setup:
+
+    mempalace daemon start
+
+SessionStart performs a fast health probe in `require` mode and warns early if
+the required daemon is unavailable.
+
+## One decision per hook event
+
+A Stop or SessionEnd event may perform several writes:
+
+1. diary checkpoint;
+2. transcript ingest;
+3. project auto-ingest.
+
+The route is resolved once and stored in a context-local value for the whole
+write burst. This avoids repeated health probes and prevents different writes
+from selecting inconsistent routes during the same event.
+
+## Submission ambiguity
+
+Once a daemon submission is attempted, an error never triggers direct
+fallback. The daemon may have accepted the job before the client observed the
+failure; retrying directly could duplicate content.
+
+## Invalid configuration
+
+An explicitly invalid routing policy fails closed: hook writes are blocked and
+no direct ChromaDB fallback is attempted.
+
+An unrelated configuration read/runtime failure preserves the historical
+direct-save behavior so a final checkpoint is not lost because of an
+independent configuration failure.
+
+## Backward compatibility
+
+The default policy remains `direct`.
+
+Legacy settings remain supported through the shared policy resolver:
+
+- `MEMPALACE_HOOKS_DAEMON=true` maps to `prefer`;
+- `hooks.daemon: true` maps to `prefer`;
+- false values map to `direct`.
+
+## Configuration examples
+
+Prefer the daemon but permit direct fallback:
+
+    MEMPALACE_HOOK_WRITE_ROUTING=prefer
+
+Require the daemon and prohibit direct writers:
+
+    MEMPALACE_HOOK_WRITE_ROUTING=require
+
+Configuration file:
+
+    {
+      "write_routing": {
+        "hooks": "require"
+      }
+    }

docs/rfcs/001-storage-backend-plugin-spec.md

@@ -0,0 +1,670 @@
+# RFC 001 — Storage Backend Plugin Specification
+
+- **Status:** Accepted (2026-06-07)
+- **Tracking issue:** [#737](https://github.com/MemPalace/mempalace/issues/737)
+- **Supersedes:** The informal seam introduced by [#413](https://github.com/MemPalace/mempalace/pull/413)
+- **Related:** [#266](https://github.com/MemPalace/mempalace/issues/266), [#574](https://github.com/MemPalace/mempalace/pull/574), [#643](https://github.com/MemPalace/mempalace/pull/643), [#665](https://github.com/MemPalace/mempalace/pull/665), [#697](https://github.com/MemPalace/mempalace/pull/697), [#700](https://github.com/MemPalace/mempalace/pull/700), [#381](https://github.com/MemPalace/mempalace/pull/381), [#1679](https://github.com/MemPalace/mempalace/pull/1679)
+- **Spec version:** `1.0`
+
+> **Implementation status (2026-08-11).** The §1–2 contract surface (`PalaceRef`,
+> typed results, `BaseBackend` / `BaseCollection`, capability tokens, and most of
+> the §10 seam cleanup) landed ahead of this merge via [#1679](https://github.com/MemPalace/mempalace/pull/1679),
+> which ships in-tree `pgvector`, `qdrant`, and `sqlite_exact` backends. The three
+> areas originally deferred as follow-ups have since landed: embedder-identity
+> (§1.5 / §5, [#1731](https://github.com/MemPalace/mempalace/pull/1731) /
+> [#1734](https://github.com/MemPalace/mempalace/pull/1734)), maintenance hooks
+> (§7.3, [#1732](https://github.com/MemPalace/mempalace/pull/1732)), and searcher
+> metric-awareness (§10, [#1727](https://github.com/MemPalace/mempalace/pull/1727)).
+> Accepting this RFC pins the contract; remaining gaps are evolutionary, not
+> blocking.
+
+## Summary
+
+A formal contract for MemPalace storage backends so third parties can ship `pip install mempalace-<name>` packages that drop into the core without patches. The spec defines the collection interface, the backend lifecycle, registration via Python entry points, configuration shape, a required test contract, and a migration path between backends.
+
+It also sets up MemPalace to run as a long-lived daemon that manages many palaces, where different palaces may route to different backends.
+
+## Motivation
+
+Six backend PRs are currently in flight. Each one solves the same problem six different ways — different method signatures, different registration mechanisms, different embedder ownership, incompatible where-clause dialects, no shared test suite. The ad-hoc `BaseCollection` ABC merged in #413 was deliberately minimal and deferred every non-obvious decision. This RFC closes the open decisions so backend authors can build to a stable contract.
+
+## Goals
+
+1. A backend ships as a standalone Python package; installing it is sufficient to use it.
+2. All callers in MemPalace core go through the collection interface. No direct `chromadb` imports outside `mempalace/backends/chroma.py`.
+3. Backends are interchangeable: every backend passes the same shared test suite, and `mempalace migrate` supports lossless movement between them when source/target capabilities allow, with explicit re-embedding as the fallback (§8.2).
+4. The model scales from single-user local (one backend, one palace, no config) to a daemon serving many palaces with heterogeneous backends.
+5. Chroma's current dict-shaped return values are not the long-term contract. Typed results are spec v1.
+
+## Non-goals
+
+- Defining the embedder pipeline in detail. The embedder is a separate contract this spec depends on but does not specify.
+- Defining the sync subsystem. This spec only declares the capability flag and the minimal hook a sync subsystem will read.
+- Specifying wire protocol for a future networked daemon. That is a separate RFC.
+
+---
+
+## 1. Collection contract
+
+### 1.1 Required methods
+
+All backends implement `BaseCollection` with kwargs-only signatures:
+
+```python
+class BaseCollection(ABC):
+    @abstractmethod
+    def add(
+        self,
+        *,
+        documents: list[str],
+        ids: list[str],
+        metadatas: list[dict] | None = None,
+        embeddings: list[list[float]] | None = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def upsert(
+        self,
+        *,
+        documents: list[str],
+        ids: list[str],
+        metadatas: list[dict] | None = None,
+        embeddings: list[list[float]] | None = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def query(
+        self,
+        *,
+        query_texts: list[str] | None = None,
+        query_embeddings: list[list[float]] | None = None,
+        n_results: int = 10,
+        where: dict | None = None,
+        where_document: dict | None = None,
+        include: list[str] | None = None,
+    ) -> QueryResult: ...
+
+    @abstractmethod
+    def get(
+        self,
+        *,
+        ids: list[str] | None = None,
+        where: dict | None = None,
+        where_document: dict | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
+        include: list[str] | None = None,
+    ) -> GetResult: ...
+
+    @abstractmethod
+    def delete(
+        self,
+        *,
+        ids: list[str] | None = None,
+        where: dict | None = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def count(self) -> int: ...
+```
+
+### 1.2 Optional methods (default implementations on the ABC)
+
+```python
+def estimated_count(self) -> int:
+    return self.count()
+
+def close(self) -> None:
+    return None
+
+def health(self) -> HealthStatus:
+    return HealthStatus.ok()
+
+def update(
+    self,
+    *,
+    ids: list[str],
+    documents: list[str] | None = None,
+    metadatas: list[dict] | None = None,
+    embeddings: list[list[float]] | None = None,
+) -> None:
+    """Partial update of existing rows. At least one of documents/metadatas/embeddings must be non-None.
+
+    Default implementation: get(ids=...), merge the provided fields, upsert. Non-atomic
+    and does two round-trips. Backends advertising `supports_update` MUST override with
+    an atomic, single-round-trip implementation.
+    """
+    ...  # default impl in the ABC
+```
+
+Backends with cheap approximate counters override `estimated_count`. Backends that hold connections must override `close`. Backends with native partial-update primitives (Postgres `UPDATE`, Lance `merge_insert`) override `update` and advertise `supports_update`; the token signals "atomic + single round-trip," not "supports partial updates at all" — the default implementation already supports them, just non-atomically.
+
+### 1.3 Typed results (replaces Chroma dict shape)
+
+```python
+@dataclass(frozen=True)
+class QueryResult:
+    ids: list[list[str]]                              # outer = queries, inner = hits
+    documents: list[list[str]]
+    metadatas: list[list[dict]]
+    distances: list[list[float]]
+    embeddings: list[list[list[float]]] | None = None
+
+@dataclass(frozen=True)
+class GetResult:
+    ids: list[str]
+    documents: list[str]
+    metadatas: list[dict]
+    embeddings: list[list[float]] | None = None
+```
+
+On empty results: return a result object with empty inner lists, never raise. Specifically, an empty query returns `QueryResult(ids=[[]], documents=[[]], metadatas=[[]], distances=[[]])` — the outer dimension is the number of query vectors issued; the inner dimension is hits per query and may be zero.
+
+`include` controls which fields are populated. Fields not in `include` are populated with empty lists of the correct outer shape; they are never `None` (except `embeddings`, which is `None` when not requested).
+
+### 1.4 Where-clause dialect
+
+**Required operators:** `$eq`, `$ne`, `$in`, `$nin`, `$and`, `$or`, `$contains`.
+
+Backends that do not support full-text natively MUST still implement `$contains` via payload string match — correctness is required; performance is not. `supports_contains_fast` (§2.1) is the only performance floor the spec promises. Without it, callers and benchmarks MUST assume `$contains` is O(n). This is an intentional split: `$contains` is a correctness requirement, `contains_fast` is the performance boundary, and the gap between scan and indexed FTS is too large for the spec to paper over.
+
+**Unknown operators:** backends MUST raise `UnsupportedFilterError`. Silent dropping is forbidden — it produces incorrect results.
+
+**Optional operators:** `$gt`, `$gte`, `$lt`, `$lte`. Backends either implement them or reject with `UnsupportedFilterError`. Advertised via capabilities.
+
+### 1.5 Embeddings
+
+#### Signature compliance (all backends)
+
+All backends MUST accept a pre-computed `embeddings=` argument on `add` / `upsert` without raising. This is signature compliance only — it does not guarantee the vectors are persisted (see passthrough below). Capability token: `supports_embeddings_in`.
+
+Backends MUST NOT hardcode embedding models or dimensions. Model selection is the embedder's responsibility (§4).
+
+#### Passthrough vs re-embed (separate guarantee)
+
+Accepting the argument is not the same as honoring it. Two distinct semantics, distinguished by capability:
+
+- **`supports_embeddings_passthrough`** — when `embeddings=` is provided, the backend MUST persist those vectors as-is and MUST NOT re-embed from text. This is the stronger guarantee lossless migration depends on.
+- **No `supports_embeddings_passthrough`** — the backend always re-embeds from text at write time. Provided `embeddings=` is accepted (signature compliance) but discarded. Migration *to* such a backend is re-embedding, not lossless transfer.
+
+`supports_migration_export` (source-side bulk read) MUST be paired with `supports_embeddings_passthrough` (target-side lossless write) for a migration to be labeled lossless. The `mempalace migrate` CLI refuses to run between backends where the target lacks `supports_embeddings_passthrough` unless `--accept-re-embed` is passed, which records re-embedding in the target palace's migration log.
+
+#### Dimension check (all backends, required)
+
+Backends MUST validate embedding dimension on first write to a new collection and on open of an existing collection, and MUST raise `DimensionMismatchError` on mismatch. Silent acceptance of mismatched dimensions produces unrecoverable corruption.
+
+#### Model identity check (all backends, three-state)
+
+Dimension matching is necessary but not sufficient. Swapping to a different model that happens to share a dimension (e.g., both 384-d) silently degrades retrieval without tripping `DimensionMismatchError`. Backends MUST persist `embedder.model_name` alongside the collection on first write and MUST check it on subsequent open. Three outcomes:
+
+| State | Condition | Required behavior |
+|---|---|---|
+| `known_match` | Stored name equals current `embedder.model_name` | Proceed normally. |
+| `known_mismatch` | Stored name exists and differs from current | Raise `EmbedderIdentityMismatchError`. Override only via explicit CLI `--force-model-swap`, which writes the swap to the palace's migration log and updates the stored identity. |
+| `unknown` | No model name recorded (legacy collection, pre-v1 palace) | Do not hard-fail — emit a `EmbedderIdentityUnknownWarning` on first open. The resolved identity is recorded on the next successful write, reindex, or migration, transitioning the palace to `known_match` going forward. CLI exposes `mempalace palace set-embedder --model NAME` for explicit resolution. |
+
+The `unknown` state exists because existing palaces from #413 and earlier have no recorded identity; hard-failing them on upgrade would be hostile. Once recorded, subsequent opens are strict.
+
+An injected embedder that exposes no usable `model_name` (empty or `None`) resolves to `unknown` rather than being a hard error — the backend persists no identity, emits `EmbedderIdentityUnknownWarning`, and records identity on the first open against an embedder that *does* report a name. A nameless embedder is therefore a degraded-but-valid mode, not a rejection; operators promote it with `mempalace palace set-embedder --model NAME`.
+
+> **Follow-up dependency.** The `MUST persist embedder.model_name` rule above is only satisfiable once the `Embedder` protocol (§5) is a normative contract — a backend cannot persist an identity it is never handed. §5 now pins the minimal protocol (`model_name`, `dimension`, `embed`) as sufficient for this section; the embedder-identity enforcement is tracked as follow-up implementation work (the current in-tree backends defer it). See §5.
+
+#### `server_embedder` backends are not exempt
+
+A backend advertising `server_embedder` (§2.1) provides its own embedder and MAY ignore the `embedder=` kwarg passed to `get_collection`. That does **not** exempt it from the dimension and identity rules above. Such backends MUST:
+
+- Expose an effective `model_name: str` and `dimension: int` describing the embedder actually in use (via `BaseCollection.effective_embedder_identity() -> EmbedderIdentity`).
+- Persist that effective identity on first write and validate it on open, per the three-state rules above.
+- Raise `DimensionMismatchError` and `EmbedderIdentityMismatchError` on conflicts between the effective identity and any injected `embedder` (if one was passed) or between the stored identity and the current effective identity.
+
+`server_embedder` documents where the embedding happens; it never suspends the safety contract. A backend that cannot report its effective embedder identity does not qualify for the `server_embedder` capability.
+
+---
+
+## 2. Backend contract
+
+### 2.1 Identity and capabilities
+
+```python
+class BaseBackend(ABC):
+    name: ClassVar[str]                       # "chroma", "postgres", "qdrant", ...
+    spec_version: ClassVar[str] = "1.0"       # which spec version this backend targets
+    capabilities: ClassVar[frozenset[str]]
+    distance_metric: ClassVar[str] = "cosine" # "cosine" | "l2" | "ip" (inner product)
+```
+
+`distance_metric` declares the space the backend's `distances` are reported in.
+It is **not** a capability token (it is a single value, not a boolean), so it is a
+class attribute. Core search code MUST convert a backend's reported distance to a
+similarity using this declaration rather than assuming cosine — see §10, which
+adds `searcher.py` to the cleanup precisely because `_hybrid_rank` currently
+hard-codes `max(0, 1 - distance)` (cosine-only). All in-tree backends are `cosine`
+today, so the assumption is latent, not yet wrong; the declaration makes a
+non-cosine backend (e.g. a dot-product store) correct rather than silently
+mis-ranked.
+
+Defined capability tokens (v1):
+
+| Token | Meaning |
+|---|---|
+| `supports_embeddings_in` | Accepts pre-computed `embeddings=` without raising (signature compliance; MUST be true for all backends) |
+| `supports_embeddings_passthrough` | Persists provided `embeddings=` as-is without re-embedding (required for lossless migration target) |
+| `supports_embeddings_out` | Returns embeddings when `include=["embeddings"]` is requested |
+| `supports_estimated_count` | `estimated_count()` is meaningfully cheaper than `count()` |
+| `supports_update` | `update()` is atomic and single-round-trip (vs the ABC default of get+merge+upsert) |
+| `supports_metadata_filters` | Implements the required where-clause subset (§1.4) |
+| `supports_range_filters` | Implements `$gt` / `$gte` / `$lt` / `$lte` |
+| `supports_contains_fast` | `$contains` is indexed (vs scan-based) |
+| `supports_server_side_indexes` | Exposes index creation / maintenance to operators |
+| `supports_migration_export` | Implements a bulk read path suitable for `mempalace migrate` |
+| `supports_change_feed` | Exposes `changes_since(cursor)` for the sync subsystem |
+| `supports_sync` | Implies `supports_change_feed` plus idempotent upserts under conflicts |
+| `requires_external_service` | Needs a running server (e.g., Postgres, hosted Qdrant) |
+| `local_mode` | Persists to `palace.local_path` |
+| `server_mode` | Connects to an external server; `palace.namespace` is used |
+| `server_embedder` | Backend provides its own embedder (may ignore injected one) |
+| `supports_namespace_isolation` | Enforces `PalaceRef.namespace` as a hard isolation boundary (§4.4). Multi-tenant deployments MAY rely on it for tenant isolation; backends without it MUST NOT be relied on for that. |
+
+A backend may advertise both `local_mode` and `server_mode` (e.g., Chroma with either `PersistentClient` or `HttpClient`).
+
+Capability tokens are free-form strings, not an enum — third-party backends may declare novel capabilities for their ecosystem. Core MemPalace only inspects the tokens listed above.
+
+### 2.2 Palace references
+
+A backend serves palaces, not raw filesystem paths. This is the central change from #413.
+
+```python
+@dataclass(frozen=True)
+class PalaceRef:
+    id: str                          # stable identity, used as cache key
+    local_path: str | None = None    # filesystem root, if this palace is local
+    namespace: str | None = None     # server-side namespace/prefix, if applicable
+```
+
+Rules:
+- `id` is always present. It is the key the backend uses to cache open handles.
+- Local-only backends read `local_path`. If `local_path is None` they raise `PalaceNotFoundError`.
+- Server-only backends read `namespace`. If `namespace is None` they derive one deterministically from `id`.
+- Mixed-mode backends may use both (e.g., a local cache alongside a server store).
+
+### 2.3 Methods
+
+```python
+class BaseBackend(ABC):
+    @abstractmethod
+    def get_collection(
+        self,
+        *,
+        palace: PalaceRef,
+        collection_name: str,
+        create: bool,
+        embedder: Embedder | None = None,
+        options: dict | None = None,
+    ) -> BaseCollection: ...
+
+    def close_palace(self, palace: PalaceRef) -> None:
+        """Evict a single palace's cached handles. Default: no-op."""
+        return None
+
+    def close(self) -> None:
+        """Shut down the entire backend instance. Default: no-op."""
+        return None
+
+    def health(self, palace: PalaceRef | None = None) -> HealthStatus:
+        """Return health. With palace=None, probe the backend itself."""
+        return HealthStatus.ok()
+```
+
+### 2.4 Semantics of `create`
+
+- `create=False` on a nonexistent palace MUST raise `PalaceNotFoundError` (subclass of `FileNotFoundError` for backwards compatibility with the #413 seam).
+- `create=True` MUST be idempotent — calling it repeatedly with the same arguments produces the same state and does not corrupt existing data.
+- `create=True` on local backends creates the directory with `0700` permissions (matches the existing Chroma behavior).
+
+**Multiple collections per palace.** `get_collection` is keyed by `collection_name`, and a palace MAY hold more than one collection. Backends MUST support N collections per palace, addressed by distinct `collection_name` values, with the §2.5 isolation guarantee applying per `(palace.id, collection_name)`. The "palace" is not 1:1 with a collection: production already splits verbatim drawers from short, query-term-saturated session-recovery checkpoints into sibling collections (`mempalace_drawers` vs `mempalace_session_recovery`) so the latter can't dominate vector top-N. Backends like Postgres (schema/table naming) and Qdrant (collection naming) handle this trivially; a backend author MUST NOT assume one collection per palace and design themselves into a corner. No signature changes — this is already implicit in `collection_name`; it is stated here so it is a contract fact, not a convention.
+
+### 2.5 Concurrency
+
+A backend instance is long-lived and serves many palaces. Backends MUST be thread-safe for concurrent `get_collection` calls across different `PalaceRef.id` values. Collection handles for the same `(palace.id, collection_name)` MAY be cached internally and returned on subsequent calls.
+
+Backends MAY assume a single thread accesses a given `BaseCollection` instance at a time. MemPalace core serializes access per palace; backend authors are not required to make individual collections thread-safe.
+
+### 2.6 Lifecycle
+
+1. `__init__`: lightweight. No I/O, no network connections. A backend instance may be constructed and never used.
+2. First call to `get_collection`: may open connections, create schemas, etc. All I/O is lazy.
+3. `close_palace(palace)`: releases cached handles for one palace. Safe to call on a palace that was never opened.
+4. `close()`: releases all resources. After `close()`, further calls MUST raise `BackendClosedError`.
+
+There is no explicit `connect()` — it is always implicit and lazy, matching current Chroma behavior.
+
+---
+
+## 3. Registration and discovery
+
+### 3.1 Entry points (primary mechanism)
+
+Third-party backends ship as installable packages:
+
+```toml
+# pyproject.toml of mempalace-postgres
+[project.entry-points."mempalace.backends"]
+postgres = "mempalace_postgres:PostgresBackend"
+```
+
+MemPalace discovers backends at process start via `importlib.metadata.entry_points(group="mempalace.backends")`. No patches to the core are required.
+
+### 3.2 In-tree registry (secondary)
+
+For tests and local development:
+
+```python
+from mempalace.backends.registry import register
+
+register("my-experimental-backend", MyBackend)
+```
+
+Entry-point discovery and explicit `register()` populate the same registry. Explicit registration wins on name conflict.
+
+### 3.3 Selection priority
+
+When resolving a palace's backend, priority (highest first):
+
+1. Explicit `backend=` kwarg to `Palace(...)` or CLI `--backend`
+2. Per-palace `backend` key in config (see §4)
+3. `MEMPALACE_BACKEND` environment variable
+4. Auto-detect from on-disk artifacts: `chroma.sqlite3` → `chroma`, `*.lance` → `lance`, etc. Backends declare detection hints via an optional `BaseBackend.detect(path: str) -> bool` classmethod.
+5. Default: `chroma`.
+
+**Auto-detection is strictly a migration/upgrade compatibility path, not a general selection mechanism.** It exists so existing palaces from v3.x keep opening without forced config migration. For *new* palaces, explicit configuration or CLI flag always wins — creating a palace without a resolved backend from (1)–(3) falls through to default (5), never to detection (4). Auto-detection fires only when a local path is presented AND no earlier rule has chosen a backend AND the path already contains backend-identifiable artifacts.
+
+Note the interaction with a globally-set `MEMPALACE_BACKEND`: rule (3) sits above detection (4), so a user who exports `MEMPALACE_BACKEND=postgres` and then opens a palace containing on-disk Chroma artifacts gets postgres — the env var wins and detection is skipped. That is intended (explicit configuration overrides detection), but it means **setting `MEMPALACE_BACKEND` globally overrides existing-palace auto-detection; users opening pre-existing palaces of mixed backends should leave it unset** and rely on per-palace config or detection.
+
+---
+
+## 4. Configuration
+
+### 4.1 Shape
+
+```json
+{
+  "backends": {
+    "chroma": { "type": "chroma" },
+    "pg_prod": {
+      "type": "postgres",
+      "dsn": "postgresql://...",
+      "pool_size": 10
+    }
+  },
+  "palaces": {
+    "work": {
+      "backend": "pg_prod",
+      "namespace": "work"
+    },
+    "personal": {
+      "backend": "chroma",
+      "local_path": "~/.mempalace/personal"
+    }
+  },
+  "embedder": {
+    "type": "onnx",
+    "model": "all-MiniLM-L6-v2"
+  }
+}
+```
+
+Single-user local mode: all of this is optional. The absence of a config file yields one Chroma backend, one palace at the default path, with the default embedder.
+
+### 4.2 Environment variables
+
+- `MEMPALACE_BACKEND` — shortcut for the default backend type when there is no config.
+- `MEMPALACE_<NAME>_*` — per-backend secrets and connection info (e.g., `MEMPALACE_POSTGRES_DSN`, `MEMPALACE_QDRANT_URL`, `MEMPALACE_QDRANT_API_KEY`).
+- `<NAME>` is the backend's **type** name (the `type` field in §4.1 — `postgres`, `qdrant`), uppercased, not the per-instance config key. So the `pg_prod` instance in §4.1 reads `MEMPALACE_POSTGRES_*`, not `MEMPALACE_PG_PROD_*` — instances of the same type share one env namespace, and connection-specific values that differ per instance (distinct DSNs) belong in the config file's per-backend block, not in env. Hyphens in a type name are normalized to underscores for the env prefix (`my-backend` → `MEMPALACE_MY_BACKEND_*`).
+- Secrets MUST be readable from env vars; config files are for structure, env vars for credentials.
+
+### 4.3 Backend-specific options
+
+The `options` kwarg to `get_collection` is a free-form dict. Each backend documents its accepted keys. Unknown keys MUST be ignored (forward compatibility), but the backend MAY log a warning.
+
+### 4.4 Multi-tenancy (absorbs #697)
+
+Per-tenant collection-name prefixing is not a backend concern. It is handled by the resolver layer above backends: `PalaceRef.namespace` carries the tenant identifier. The `collection_prefix` concept from #697 dissolves into this model.
+
+**Isolation contract.** `PalaceRef.id` is the *required* isolation key for every backend: within a single backend instance, a record written for one `id` MUST NOT be returned, modified, or deleted by an operation issued for a different `id`. Cross-palace access is a spec violation. This is the non-negotiable blanket MUST; multi-tenant deployments may cite it as their primary partition boundary even when they do not use `namespace`.
+
+`namespace` is *additional* partitioning. A backend that advertises `supports_namespace_isolation` (§2.1) MUST extend the same guarantee to namespaces:
+
+> A record written under one `namespace` MUST NOT be returned, modified, or deleted by an operation issued under a different `namespace` within the same backend instance. Cross-namespace access is a spec violation, not a caller misconfiguration.
+
+This is what hosted multi-tenant deployments cite as the basis for *namespace*-level tenant isolation. Authorization (which namespaces a given request may touch) stays on the deployment side; the backend's job is to guarantee no bleed *within* the instance once the namespace is fixed.
+
+Making namespace isolation a declared capability rather than a blanket `MUST` is deliberate: path-rooted local backends (e.g. `chroma`, `sqlite_exact`) already isolate by `local_path` / `id`, and forcing them to re-implement a second axis would be ceremony with no security gain.
+
+**Conformance arms.** Self-attestation is not enough. The isolation suite (`tests/_backend_conformance.py` / `assert_partition_isolation`) has two distinct arms:
+
+1. **Cross-`id` isolation** — required of every backend. Two `PalaceRef`s that differ only in `id` MUST NOT see each other's records.
+2. **Same-`id` / different-`namespace` isolation** — required of every backend that advertises `supports_namespace_isolation`. Two refs that share `id` (and, when applicable, `local_path`) but differ in `namespace` MUST NOT see each other's records. Distinct from arm 1; a backend that only passes arm 1 does not get to claim the capability.
+
+**No silent drop.** A backend that does **not** advertise `supports_namespace_isolation` MUST NOT silently accept and ignore a populated `namespace` — that is a latent cross-namespace leak, same spirit as `UnsupportedFilterError` for unknown operators. Such backends MUST either raise (e.g. `UnsupportedCapabilityError`) when `PalaceRef.namespace` is non-`None`, or honor the namespace and advertise the capability. Callers targeting path-rooted backends MUST leave `namespace` as `None`.
+
+---
+
+## 5. Embedder contract (minimal, normative here)
+
+§1.5 makes persisting and checking `embedder.model_name` a hard `MUST`. A backend cannot satisfy that against an embedder that has no identity to read, so the minimal protocol below is **normative for this spec** — not deferred. A fuller Embedder RFC (batching, async, pooling, multi-vector) is tracked separately, but it is *additive*: §1.5 conformance depends only on the three members here.
+
+```python
+class Embedder(Protocol):
+    model_name: str          # stable identity persisted and checked per §1.5
+    dimension: int           # validated per §1.5 dimension check
+    def embed(self, texts: list[str]) -> list[list[float]]: ...
+```
+
+Backends receive an `Embedder` via `get_collection(embedder=...)`. Backends with the `server_embedder` capability MAY ignore the injected embedder but MUST still expose an effective `model_name` / `dimension` (§1.5). An embedder whose `model_name` is empty or `None` is handled as the §1.5 `unknown` state, not a hard error.
+
+> **Follow-up.** The full Embedder RFC is the only external contract §1.5 leans on; it is tracked as a hard, blocking dependency of the §1.5 *implementation* (not of this spec's acceptance — the minimal protocol above closes the contract gap). Tracking issue: see §13. The current in-tree backends defer embedder-identity enforcement until that work lands.
+
+---
+
+## 6. Sync (capability declaration only)
+
+The sync subsystem is out of scope for this spec. What this spec defines:
+
+- `supports_sync` capability flag (§2.1) — a backend advertising it agrees to implement idempotent upserts under conflict and to expose change data.
+- Optional method on `BaseCollection`:
+  ```python
+  def changes_since(self, cursor: SyncCursor) -> Iterator[Change]: ...
+  ```
+- Backends without `supports_change_feed` / `supports_sync` are rejected by the sync subsystem at bind time.
+
+Local single-user deployments never load the sync subsystem; non-sync-capable backends cost them nothing.
+
+---
+
+## 7. Testing contract
+
+### 7.1 The abstract suite
+
+MemPalace ships `mempalace.backends.testing.AbstractBackendContractSuite` — a pytest mixin. Every backend package ships a concrete subclass:
+
+```python
+from mempalace.backends.testing import AbstractBackendContractSuite
+
+class TestPostgresBackend(AbstractBackendContractSuite):
+    @pytest.fixture
+    def backend(self, tmp_path):
+        return PostgresBackend(dsn=os.environ["TEST_PG_DSN"])
+```
+
+The suite covers:
+- Round-trip for every required method
+- Empty-result shape (outer dimension preserved, inner lists empty)
+- `create=False` on missing palace raises `PalaceNotFoundError`
+- `create=True` is idempotent
+- Full required where-clause subset including `$contains`
+- Unknown operator raises `UnsupportedFilterError`
+- Dimension-mismatch detection
+- Unicode text and unicode IDs
+- Large batch writes (10k+ items)
+- Delete-then-query consistency
+- `close()` releases handles and further calls raise `BackendClosedError`
+- Concurrent `get_collection` across different palaces is safe
+- Isolation arm 1: cross-`PalaceRef.id` (every backend)
+- Isolation arm 2: same-`id` / different-`namespace` (only when `supports_namespace_isolation` is advertised; self-attestation without this arm is a conformance failure)
+- Non-advertising backends raise on a populated `namespace` rather than silently accepting it (§4.4)
+
+### 7.2 Parametrized core suite
+
+The existing MemPalace test suite is parametrized over all registered backends when `MEMPALACE_TEST_ALL_BACKENDS=1` is set in the environment. This is the "strongest parity claim" — if a backend passes the full core suite, it is drop-in compatible. This is expensive; local development defaults to Chroma only, CI runs all backends on a scheduled job.
+
+### 7.3 Benchmark methodology hooks
+
+Backend-to-backend comparisons are meaningless without accounting for per-backend maintenance state. Postgres with stale planner stats behaves very differently from Postgres post-`VACUUM ANALYZE`; HNSW-based stores behave differently before and after index compaction.
+
+Backends MAY implement `maintenance_state()` returning a structured dict describing the current state (e.g., `{"autovacuum_age_seconds": 42, "last_analyze": "...", "index_build_complete": true}`), and `run_maintenance(kind: str)` to trigger supported kinds. Both are optional.
+
+Supported maintenance kinds MUST be advertised via a class-level frozenset:
+
+```python
+class BaseBackend(ABC):
+    maintenance_kinds: ClassVar[frozenset[str]] = frozenset()
+```
+
+The spec reserves the kind names `"analyze"` (update planner/query statistics), `"compact"` (reclaim space, rewrite storage), and `"reindex"` (rebuild secondary indexes). Backends MAY add their own kinds; the reserved names MUST mean what the spec says if advertised. A backend that has no analogue for a reserved kind MUST omit it from `maintenance_kinds` rather than declaring it as a no-op — otherwise a benchmark harness sees `"analyze"` advertised and assumes it did what the spec says when the implementation did nothing.
+
+`run_maintenance(kind)` MUST raise `UnsupportedMaintenanceKindError` when called with a kind not in `maintenance_kinds`. Advertising a kind without implementing it is a conformance failure.
+
+**`run_maintenance` is observable, not fire-and-forget.** This resolves the §12 open question, and it is driven by a production failure: on a lazy-index path, multiple daemon writers crossing the index-build threshold in the same window each issued the build, stacked an `ACCESS EXCLUSIVE` lock, and blocked writes for the whole build (fixed in-backend with a session-level advisory lock). A pure fire-and-forget call reproduces that race — if the call cannot report "already running," concurrent callers re-trigger the build. Therefore:
+
+- `run_maintenance(kind)` MUST be safe to call concurrently. A backend MUST serialize same-kind maintenance internally (advisory lock, build flag, or equivalent) so a second caller does not start a duplicate operation.
+- It returns a structured `MaintenanceResult` (e.g. `{"kind": str, "status": "ran" | "already_running" | "noop", "stats": {...}}`) rather than `None`. `stats` is free-form per kind (rows analyzed, bytes reclaimed, fragments merged). `already_running` is how a concurrent caller learns it must not re-trigger.
+
+This makes the maintenance hook the operator-safe path the lazy-index concurrency wedge requires, and gives the benchmark harness (above) machine-readable phase data.
+
+The benchmark harness under [benchmarks/](../../benchmarks/) records `maintenance_state()` alongside every latency/recall measurement it publishes. Published numbers MUST include three phases: immediately after bulk load, after the backend's native background maintenance has caught up, and after `run_maintenance(kind)` has been called for each kind in `maintenance_kinds`. Harnesses rely on this advertisement to decide what to call — they MUST NOT assume kind names. This prevents comparing an un-`ANALYZE`d Postgres to a settled Chroma and calling the former slow.
+
+### 7.4 ID stability for non-string-ID backends
+
+Backends requiring UUID IDs (Qdrant) use a canonical namespace:
+
+```python
+NAMESPACE_MEMPALACE = uuid.UUID("c06c3fc7-5c14-4dc4-84c2-24a5f72d8dc1")
+backend_id = uuid.uuid5(NAMESPACE_MEMPALACE, original_id)
+```
+
+The namespace UUID is fixed at spec v1 adoption and recorded here — once, for all time. This value is the one the in-tree `qdrant` backend already shipped with in #1679, so existing Qdrant palaces' point IDs already derive from it; promoting the shipped constant to canonical (rather than minting a fresh one) avoids re-deriving any deployed palace's IDs. New UUID-ID backends MUST use this exact namespace. This resolves the #700 vs #381 divergence.
+
+---
+
+## 8. Migration
+
+### 8.1 The CLI
+
+```
+mempalace migrate --palace PATH --from chroma --to postgres
+mempalace migrate --all --to lance
+```
+
+Implementation is backend-agnostic: reads from source via `BaseCollection.get(include=["documents", "metadatas", "embeddings"])`, writes to target via `BaseCollection.upsert(...)` with the original embeddings. No backend-specific migration code.
+
+### 8.2 Lossless vs re-embed
+
+Migration is labeled **lossless** only when:
+
+- The source advertises `supports_migration_export` (bulk read includes embeddings), AND
+- The target advertises `supports_embeddings_passthrough` (persists provided embeddings as-is), AND
+- Source and target agree on `embedder.model_name` (or `--force-model-swap` is explicit).
+
+If the target lacks `supports_embeddings_passthrough`, `mempalace migrate` refuses to run. Passing `--accept-re-embed` overrides — the migration proceeds but re-embeds from document text at write time, and the migration record labels the result as re-embedded rather than lossless. Retrieval quality may shift.
+
+A backend that persists the exact float32 vector in its own durable store (rather than re-embedding from text on read) satisfies **both** sides of the lossless pairing: it qualifies for `supports_migration_export` as a source (it can hand back the verbatim vector it stored) and for `supports_embeddings_passthrough` as a target (it persists provided vectors as-is). Such "rank-from-stored-vectors" backends are first-class migration endpoints in both directions — the pairing in this section is about the two *capabilities* being present, not about any particular index structure. An exact-vector store therefore makes `chroma → <store>` and `<store> → chroma` lossless under §8 in both directions, given model-identity agreement.
+
+### 8.3 Safety
+
+- Source is never modified. Migration is read-only against the source backend.
+- Target palace must not already exist unless `--overwrite` is passed.
+- A successful migration writes a `.mempalace-migration.json` record into the target palace containing: source backend name, source path/ref, timestamp, row count, `lossless: true|false`, source and target `embedder.model_name`, and whether `--force-model-swap` or `--accept-re-embed` was used.
+
+### 8.4 Verification
+
+After migration, run `mempalace verify --palace PATH --against SOURCE_PATH --source-backend chroma`. This samples N rows and confirms round-trip parity (ids match, documents match, embedding cosine similarity ≥ 0.999 when the migration was lossless; a looser document-overlap check when re-embedded).
+
+---
+
+## 9. Versioning and compatibility
+
+- `BaseBackend.spec_version` declares which spec version a backend implements.
+- MemPalace refuses to load a backend declaring a different major version. The failure is loud and names the mismatch, e.g.:
+  ```python
+  raise BackendVersionMismatchError(
+      f"backend {name!r} targets spec {backend.spec_version!r}; "
+      f"this MemPalace implements major version {CORE_MAJOR!r}. "
+      f"Install a build of {name!r} that targets spec {CORE_MAJOR}.x."
+  )
+  ```
+- Minor versions are additive (new optional methods, new capability tokens). Backends declaring an older minor continue to work.
+- This is spec v1.0.
+
+---
+
+## 10. Cleanup prerequisite (mostly landed via #1679)
+
+**Status update (2026-06-07).** The bulk of this cleanup landed in [#1679](https://github.com/MemPalace/mempalace/pull/1679): the direct `chromadb` *client* imports across `repair.py`, `dedup.py`, `cli.py`, `mcp_server.py`, and `migrate.py` were routed through `BaseCollection`, and the dict-to-typed-result migration (§1.3) shipped. What remains is narrow:
+
+- **Residual exception-class imports.** `mcp_server.py` and `repair.py` still `from chromadb.errors import NotFoundError` to catch Chroma's not-found error. These are catch-site couplings, not client construction; they should resolve to a backend-neutral `PalaceNotFoundError` / collection-not-found exception from the contract (§2.4) so non-Chroma palaces raise the same type.
+- **`mcp_server._get_client()` caching.** Caches a `PersistentClient` at module scope and invalidates it on `chroma.sqlite3` inode or mtime changes (merged via [#757](https://github.com/MemPalace/mempalace/pull/757)). Both the cache and the stat-based freshness check are Chroma-specific. They should migrate into `ChromaBackend.get_collection()` (§2.5, handle caching) and `ChromaBackend.close_palace()` (§2.6, explicit flush) — other backends do not have a single on-disk SQLite file to stat. The `mempalace_reconnect` MCP tool then becomes a thin wrapper around `backend.close_palace(palace_ref)`.
+
+**`searcher.py` is the highest-leakage module still coupled, and #1679 did not cover it** (flagged by two backend authors — kostadis-ntnx on this RFC, jphein on #1679). Two concrete couplings:
+
+- `_hybrid_rank()` hard-codes `vec_sim = max(0, 1 - distance)`, i.e. it assumes cosine distance ∈ [0, 2]. For a backend whose metric isn't cosine (dot-product, L2) the ranking is silently wrong. Fix: make the conversion metric-aware off the backend-declared `distance_metric` (§2.1) rather than hard-coding cosine. All in-tree backends are cosine today, so this is latent — but it is a contract gap, not a coincidence to keep relying on.
+- The BM25 survival fallback opens `chroma.sqlite3` directly and queries Chroma's FTS5 `embedding_fulltext_search` shadow table. On any non-Chroma palace this path is dead. Fix: route lexical fallback through `BaseCollection` (§1.4 `$contains` / a lexical capability), or gate it on a backend-declared capability so backends whose index self-heals never request it.
+
+This `searcher.py` work is tracked as follow-up implementation (see §13) — it is the remaining piece that promotes a non-Chroma backend from "stores and retrieves" to a first-class search peer.
+
+---
+
+## 11. Impact on in-flight PRs
+
+**Update (2026-06-07).** The first wave of in-tree backends — `pgvector`, `qdrant`, `sqlite_exact` — landed via [#1679](https://github.com/MemPalace/mempalace/pull/1679) implementing the §1–2 contract surface directly, ahead of this RFC's merge. That reshapes the table below: the community PRs now rebase against both the spec *and* the shipped in-tree backends, and the prediction that `collection_prefix` would dissolve into `PalaceRef.namespace` (#697) held.
+
+| PR | Status | Effort to align |
+|---|---|---|
+| [#574](https://github.com/MemPalace/mempalace/pull/574) LanceDB | Open | Closest to final shape. Needs `PalaceRef` and typed results (now both shipped in-tree — rebase against the merged ABC). |
+| [#665](https://github.com/MemPalace/mempalace/pull/665) Postgres (`pg_sorted_heap`) | Open | The in-tree `pgvector` backend (#1679) now occupies the basic Postgres slot; #665 rebases as the optional `pg_sorted_heap` performance variant. Decouple embedder; adopt the merged `PalaceRef`. Its session-level advisory-lock fix for the lazy-index concurrency wedge is the production basis for §7.3's observable `run_maintenance`. |
+| [#700](https://github.com/MemPalace/mempalace/pull/700) Qdrant | Open | Largely superseded by the in-tree `qdrant` backend (#1679), which already uses the §7.4 canonical namespace. Reconcile or close in favor of the merged backend. |
+| [#381](https://github.com/MemPalace/mempalace/pull/381) Qdrant (older) | Open | Same as #700; subclass the merged `BaseCollection` rather than a bare `Protocol`, or close as superseded. |
+| [#643](https://github.com/MemPalace/mempalace/pull/643) PalaceStore | Closed | POC; the parametrized-test approach it explored became the standard (§7.2). |
+| [#697](https://github.com/MemPalace/mempalace/pull/697) Chroma HttpClient + prefix | Closed | `collection_prefix` dissolved into `PalaceRef.namespace` + `supports_namespace_isolation` (§4.4), as predicted. |
+
+---
+
+## 12. Resolved decisions
+
+The three questions that were open at draft are resolved for v1, informed by the backend-author reviews:
+
+- **`changes_since` accepts a collection filter — yes.** It takes an optional `collection_name` so a sync caller can request changes for one collection (LanceDB and others already track changes per table; filtering by collection is natural). Additive on the §6 optional hook.
+- **Per-palace capabilities — no; capabilities stay static/class-level.** Making `supports_*` palace-dependent turns every capability check into "maybe — depends," pushing state tracking onto every caller. A backend that builds indexes lazily either guarantees the index exists when the capability is needed (and advertises it) or doesn't advertise it. The one isolation-by-deployment case (`supports_contains_fast` where an FTS index may or may not exist) is handled by the backend either guaranteeing the floor or omitting the token — not by per-palace variance. (Note: `supports_namespace_isolation` is still a static, class-level token — it declares what the backend *enforces*, not a per-palace fact.)
+- **`run_maintenance(kind)` returns a structured, observable result — resolved in §7.3.** It is not fire-and-forget: it returns `MaintenanceResult` and must serialize concurrent same-kind calls (the production lazy-index concurrency wedge is why). See §7.3.
+
+No remaining blockers. Genuinely additive future items (new optional methods, new tokens) land under the minor-version rule (§9).
+
+---
+
+## 13. Rollout
+
+The original draft sequenced cleanup → spec → ChromaBackend → in-flight rebase → migrate CLI. In practice the contract surface and the first backends shipped together in [#1679](https://github.com/MemPalace/mempalace/pull/1679) ahead of this merge, so the remaining sequence is what's left, not the whole list.
+
+**Done (via #1679):**
+1. ✅ §10 seam cleanup (client imports routed through `BaseCollection`; typed-result migration §1.3).
+2. ✅ `BaseBackend` / `BaseCollection`, `PalaceRef`, capability tokens, entry-point/registry discovery.
+3. ✅ In-tree `pgvector`, `qdrant`, `sqlite_exact` backends + `tests/test_backend_conformance.py`.
+4. ✅ Canonical §7.4 namespace pinned to the shipped value.
+
+**Remaining (tracked as follow-up issues):**
+5. **Embedder-identity contract (§1.5 / §5)** — [#1724](https://github.com/MemPalace/mempalace/issues/1724). Land the minimal `Embedder` protocol as a real injected dependency; persist + check `model_name` per the three-state model. *Hard dependency for §1.5 conformance.*
+6. **Maintenance hooks (§7.3)** — [#1725](https://github.com/MemPalace/mempalace/issues/1725). Implement `maintenance_kinds` / `maintenance_state()` / observable `run_maintenance()` in the in-tree backends; wire the §7.3 advisory-lock serialization (`pg_sorted_heap` / #665 has the reference fix).
+7. **`searcher.py` metric-awareness + lexical fallback (§10)** — [#1726](https://github.com/MemPalace/mempalace/issues/1726). Make `_hybrid_rank` read `distance_metric`; route the BM25 fallback through `BaseCollection` instead of `chroma.sqlite3`.
+8. Residual §10 items: backend-neutral not-found exception; migrate `mcp_server._get_client()` caching into `ChromaBackend`.
+9. Finish the `mempalace migrate` CLI (§8) and the parametrized core suite gate (§7.2, `MEMPALACE_TEST_ALL_BACKENDS=1` in CI).
+10. Rebase / reconcile the remaining community backend PRs (§11) against the merged contract.
+11. Update [ROADMAP.md](../../ROADMAP.md) with spec v1.0 adoption under v4.0.0-alpha.

docs/rfcs/003-agent-logstream-coordination.md

@@ -0,0 +1,505 @@
+# RFC 003: Agent Logstream Coordination
+
+Status: Implemented (phases 1-5). Phase 5 SSE shipped as GET /logstream/stream on the hub HTTP transport (bearer-authenticated, event_list filter set, since_event_id/Last-Event-ID resume, 15s heartbeats, bounded clients); logstream tools additionally dispatch outside the global HTTP request lock so long-polls cannot starve the hub. Server-side cursor storage and drawer compaction remain future work.
+Owner: Claude Fable 5
+Created: 2026-07-01
+Branch: `feat/shared-brain-dogfood`
+
+## Summary
+
+MemPalace should become a local-first coordination substrate for agents, not only a long-term recall system. Agents running on different machines should be able to delegate work, wait for replies, exchange patches or other artifacts, and subscribe to realtime coordination messages through the shared MemPalace hub.
+
+This RFC proposes **MemPalace Logstream**: a small append-only event layer served by the existing MemPalace MCP HTTP hub. It preserves the core MemPalace promise of durable, local, exact storage while adding coordination primitives agents can use without Igor manually relaying messages between them.
+
+## Motivation
+
+The shared-brain dogfood exposed a real workflow gap:
+
+1. Mac Codex wrote a delegation packet into `shared_agent_brain/delegation`.
+2. Windows Codex found it, did the work, and wrote back status.
+3. Windows Codex forgot to push the code.
+4. Mac Codex could read the status but could not receive the actual patch through a formal channel.
+
+The memory layer worked. The coordination layer was implicit and fragile.
+
+We need a first-class way for agents to:
+
+- create durable work packets;
+- subscribe or wait for matching replies;
+- submit patch/file artifacts before they are committed;
+- acknowledge, apply, reject, or supersede handoffs;
+- keep enough structured metadata to avoid ambiguity across machines, branches, and worktrees.
+
+## Non-Goals
+
+- Do not build a cloud service.
+- Do not require external APIs, hosted queues, Redis, Postgres, or SaaS infra.
+- Do not replace Git for final source control.
+- Do not make agents apply patches silently without explicit caller intent.
+- Do not weaken verbatim drawer storage. Event payloads and artifacts must remain exact.
+- Do not add telemetry or phone-home behavior.
+
+## Design Principles
+
+- **Local-first**: the event bus runs inside the existing MemPalace hub and is reachable over loopback, LAN, or tailnet according to the user's deployment.
+- **Append-only by default**: events are immutable. Corrections are new events that reference prior events.
+- **Exact payloads**: event bodies and artifacts are stored verbatim.
+- **Structured envelopes**: agents should not infer core routing fields from prose.
+- **Durable before realtime**: every realtime message must also be recoverable after reconnect.
+- **Small useful v1**: favor long-poll and polling over a large streaming subsystem if that ships faster.
+- **No hidden authority**: patch application and destructive actions must be explicit local operations.
+
+## Core Concepts
+
+### Event
+
+An event is a structured coordination message. It has routing metadata plus an optional verbatim body.
+
+Required fields:
+
+- `id`: stable event id, generated by the server.
+- `type`: event type, such as `task.request`, `task.reply`, `patch.ready`, `patch.applied`.
+- `stream`: logical stream name. Suggested format: `project/<project-name>` or `shared_agent_brain`.
+- `room`: sub-channel inside a stream, such as `delegation`, `patches`, `reviews`, `status`.
+- `from_agent`: writer identity, supplied by caller.
+- `created_at`: server timestamp.
+
+Common optional fields:
+
+- `to_agent`: target agent or `*`.
+- `correlation_id`: task or conversation id tying request/reply events together.
+- `branch`: Git branch, when relevant.
+- `base_commit`: Git commit the work started from.
+- `status`: `open`, `claimed`, `ready`, `applied`, `blocked`, `failed`, `superseded`.
+- `artifact_ids`: patch/file artifact references.
+- `body`: verbatim human-readable content.
+
+### Artifact
+
+An artifact is exact content attached to an event. Examples:
+
+- unified diff patches;
+- generated files;
+- test logs;
+- benchmark reports;
+- screenshots encoded by reference, not necessarily inline;
+- structured JSON payloads.
+
+Artifacts must have:
+
+- `id`;
+- `kind`: `patch`, `file`, `log`, `json`, `note`;
+- `sha256`;
+- `size_bytes`;
+- `content`;
+- `created_at`;
+- `created_by`.
+
+### Cursor
+
+Each agent may maintain a cursor per stream so it can ask, "what happened since I last checked?"
+
+Cursor state can be server-side later. For v1, it is acceptable for the client to pass `since_event_id` or `since_created_at`.
+
+## Event Envelope
+
+Example event:
+
+```json
+{
+  "id": "evt_01J...",
+  "type": "patch.ready",
+  "stream": "project/mempalace",
+  "room": "patches",
+  "from_agent": "windows-codex",
+  "to_agent": "mac-codex",
+  "correlation_id": "task_01J...",
+  "branch": "feat/shared-brain-dogfood",
+  "base_commit": "2668053",
+  "status": "ready",
+  "artifact_ids": ["art_01J..."],
+  "body": "Search ranking patch is ready. Tests passed on Windows.",
+  "created_at": "2026-07-01T20:00:00Z"
+}
+```
+
+Example patch artifact:
+
+```json
+{
+  "id": "art_01J...",
+  "kind": "patch",
+  "sha256": "abc123...",
+  "size_bytes": 18422,
+  "created_by": "windows-codex",
+  "created_at": "2026-07-01T20:00:00Z",
+  "content": "diff --git a/mempalace/searcher.py b/mempalace/searcher.py\n..."
+}
+```
+
+## Storage Model
+
+Use SQLite in the active palace directory:
+
+- `logstream.sqlite3`
+
+Tables:
+
+```sql
+CREATE TABLE events (
+  id TEXT PRIMARY KEY,
+  type TEXT NOT NULL,
+  stream TEXT NOT NULL,
+  room TEXT NOT NULL,
+  from_agent TEXT NOT NULL,
+  to_agent TEXT,
+  correlation_id TEXT,
+  branch TEXT,
+  base_commit TEXT,
+  status TEXT,
+  body TEXT NOT NULL DEFAULT '',
+  created_at TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX events_stream_created_idx ON events(stream, created_at);
+CREATE INDEX events_correlation_idx ON events(correlation_id, created_at);
+CREATE INDEX events_to_agent_idx ON events(to_agent, created_at);
+CREATE INDEX events_type_idx ON events(type, created_at);
+
+CREATE TABLE artifacts (
+  id TEXT PRIMARY KEY,
+  kind TEXT NOT NULL,
+  sha256 TEXT NOT NULL,
+  size_bytes INTEGER NOT NULL,
+  content TEXT NOT NULL,
+  created_by TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX artifacts_sha256_idx ON artifacts(sha256);
+
+CREATE TABLE event_artifacts (
+  event_id TEXT NOT NULL,
+  artifact_id TEXT NOT NULL,
+  PRIMARY KEY (event_id, artifact_id)
+);
+```
+
+Rationale:
+
+- Keep coordination separate from Chroma/vector search for reliability and latency.
+- Avoid forcing realtime coordination through semantic search.
+- Make append/list/wait cheap and deterministic.
+- Optionally mirror summaries into normal drawers later for recall, but do not depend on it.
+
+## MCP Tool Surface
+
+### `mempalace_event_append`
+
+Append one event.
+
+Input:
+
+```json
+{
+  "type": "task.request",
+  "stream": "project/mempalace",
+  "room": "delegation",
+  "from_agent": "mac-codex",
+  "to_agent": "windows-codex",
+  "correlation_id": "task_...",
+  "branch": "feat/shared-brain-dogfood",
+  "base_commit": "2668053",
+  "status": "open",
+  "body": "Please fix search echo ranking.",
+  "metadata": {}
+}
+```
+
+Returns:
+
+```json
+{
+  "success": true,
+  "event": { "...": "..." }
+}
+```
+
+### `mempalace_event_list`
+
+List events with structured filters.
+
+Filters:
+
+- `stream`
+- `room`
+- `type`
+- `to_agent`
+- `from_agent`
+- `correlation_id`
+- `since_event_id`
+- `since_created_at`
+- `limit`
+
+Default limit: 50.
+
+### `mempalace_event_wait`
+
+Block until a matching event exists or timeout expires.
+
+Input:
+
+```json
+{
+  "stream": "project/mempalace",
+  "to_agent": "mac-codex",
+  "correlation_id": "task_...",
+  "type": "patch.ready",
+  "since_event_id": "evt_...",
+  "timeout_ms": 60000
+}
+```
+
+Implementation for v1:
+
+- simple polling loop inside the MCP request;
+- sleep interval 250-1000 ms with jitter;
+- max timeout 5 minutes;
+- return `{ "timed_out": true, "events": [] }` rather than error on timeout.
+
+`mempalace_event_wait` remains the polling MCP tool. Clients that can hold an
+HTTP stream should use `GET /logstream/stream` SSE for live tailing.
+
+### `mempalace_event_ack`
+
+Append an acknowledgement event.
+
+Input:
+
+```json
+{
+  "event_id": "evt_...",
+  "from_agent": "mac-codex",
+  "status": "applied",
+  "body": "Patch applied and tests passed."
+}
+```
+
+This should create a new `event.ack` event with `correlation_id` copied from the target event.
+
+### `mempalace_artifact_put`
+
+Store exact artifact content.
+
+Input:
+
+```json
+{
+  "kind": "patch",
+  "created_by": "windows-codex",
+  "content": "diff --git ...",
+  "metadata": {
+    "branch": "feat/shared-brain-dogfood",
+    "base_commit": "2668053"
+  }
+}
+```
+
+Returns `id`, `sha256`, `size_bytes`.
+
+### `mempalace_artifact_get`
+
+Fetch an artifact by id. Returns exact content and metadata.
+
+### `mempalace_patch_submit`
+
+Convenience wrapper:
+
+1. store patch artifact;
+2. append `patch.ready` event referencing the artifact;
+3. return both ids.
+
+This is optional for v1 if `artifact_put` + `event_append` are enough.
+
+## HTTP Endpoint Surface
+
+MCP tools are sufficient for v1. Add HTTP streaming later.
+
+Future endpoints:
+
+- `GET /logstream/events?stream=...&since=...`
+- `GET /logstream/stream?stream=...` using Server-Sent Events.
+
+Do not add unauthenticated logstream endpoints. They expose work metadata and patch contents.
+
+## Agent Workflow Examples
+
+### Delegation With Wait
+
+Mac Codex:
+
+1. `mempalace_event_append(type=task.request, to_agent=windows-codex, correlation_id=task_123)`
+2. `mempalace_event_wait(correlation_id=task_123, type=patch.ready, to_agent=mac-codex, timeout_ms=300000)`
+3. `mempalace_artifact_get(artifact_id)`
+4. Apply patch locally.
+5. Run tests.
+6. `mempalace_event_ack(status=applied)`.
+
+Windows Codex:
+
+1. `mempalace_event_wait(to_agent=windows-codex, type=task.request, timeout_ms=300000)`
+2. Do work.
+3. `mempalace_patch_submit(type=patch.ready, to_agent=mac-codex, correlation_id=task_123)`.
+
+### Status-Only Handoff
+
+If an agent cannot produce a patch, it can still append `task.reply` or `task.blocked` with exact notes and evidence.
+
+## Implementation Plan
+
+### Phase 1: Durable Logstream Core
+
+Add:
+
+- `mempalace/logstream.py`
+- SQLite schema creation/migration inside the active palace dir.
+- Pure Python API:
+  - `append_event(...)`
+  - `list_events(...)`
+  - `wait_events(...)`
+  - `put_artifact(...)`
+  - `get_artifact(...)`
+  - `ack_event(...)`
+
+Requirements:
+
+- no Chroma dependency;
+- no vector index open;
+- safe under concurrent HTTP requests;
+- append-only events;
+- validate/sanitize string fields with existing config helpers where appropriate;
+- cap body/artifact sizes with explicit errors.
+
+Suggested limits:
+
+- event body: 256 KiB default;
+- artifact content: 4 MiB default;
+- wait timeout: 300 seconds max.
+
+### Phase 2: MCP Tools
+
+Add tools in `mempalace/mcp_server.py`:
+
+- `mempalace_event_append`
+- `mempalace_event_list`
+- `mempalace_event_wait`
+- `mempalace_event_ack`
+- `mempalace_artifact_put`
+- `mempalace_artifact_get`
+- optionally `mempalace_patch_submit`
+
+Read-only mode:
+
+- `event_list`, `event_wait`, and `artifact_get` are allowed.
+- `event_append`, `event_ack`, `artifact_put`, and `patch_submit` are mutating tools and must be hidden/refused in read-only mode.
+
+SQLite integrity gate:
+
+- Logstream should not depend on Chroma SQLite integrity.
+- However, if the active palace path is unsafe/missing, fail clearly.
+
+### Phase 3: CLI Utilities
+
+Add:
+
+- `mempalace logstream append`
+- `mempalace logstream list`
+- `mempalace logstream wait`
+- `mempalace artifact put`
+- `mempalace artifact get`
+
+Keep CLI output scriptable with `--json`.
+
+### Phase 4: Docs and Skills
+
+Docs:
+
+- `website/concepts/agent-logstream.md`
+- `website/reference/mcp-tools.md` entries.
+- Update `website/guide/remote-server.md` to mention agent coordination over the hub.
+
+Agent skills/prompts:
+
+- Update MemPalace recall/search skills to mention logstream for active coordination.
+- Add a brief "when delegating work, use event_append/event_wait" protocol.
+
+### Phase 5: Realtime Stream
+
+Only after Phase 1-4 are solid:
+
+- add SSE endpoint;
+- per-agent cursor helpers;
+- maybe background compaction of old event payloads into normal drawers for long-term recall.
+
+## Test Plan
+
+Unit tests:
+
+- schema initializes in empty palace dir;
+- append/list round trip;
+- filters by stream, room, type, to_agent, correlation_id;
+- wait returns immediately when event already exists;
+- wait times out cleanly;
+- artifact put/get preserves exact bytes/text;
+- artifact hash is stable;
+- ack creates new event and does not mutate target event;
+- read-only mode hides/refuses mutating tools;
+- size limits reject oversized event bodies/artifacts.
+
+Integration tests:
+
+- two threads: one waits, one appends, waiter returns matching event;
+- patch submit stores artifact and event atomically enough that readers never see a dangling artifact id;
+- MCP HTTP server can append from one client and wait/list from another;
+- no auth headers/request bodies appear in `/statusz`.
+
+Dogfood test:
+
+1. Mac agent creates task request for Windows.
+2. Windows agent waits for it.
+3. Windows agent submits patch artifact.
+4. Mac agent waits, gets patch, verifies hash, applies patch manually, runs tests.
+5. Mac agent acknowledges `applied`.
+
+## Open Questions
+
+- Should logstream events also be mirrored into Chroma drawers for semantic recall, or should structured search remain the primary access path?
+- Should artifact content support binary data in v1, or should v1 be UTF-8 text only?
+- Should agent identities be free-form strings initially, or tied to configured local agent profiles?
+- Should `event_wait` be allowed over stdio MCP clients that may have shorter host timeouts?
+- Should events have retention/archival controls, or stay forever by default like memory?
+
+## Claude Fable 5 Instructions
+
+Please implement Phase 1 and Phase 2 first. Keep the patch narrow and avoid unrelated refactors.
+
+Start by creating `mempalace/logstream.py` with the durable SQLite core and focused tests. Then expose MCP tools in `mempalace/mcp_server.py` with read-only and mutating-tool semantics. Only add CLI/docs after the core and MCP tools are green.
+
+Before finishing:
+
+- run focused logstream tests;
+- run relevant MCP server tests;
+- run `ruff check` on touched Python files;
+- write a MemPalace delegation reply with the implementation status, changed files, and exact verification commands.
+
+## Implementation Notes
+
+Windows dogfood verification by `windows-codex` on 2026-07-02:
+
+- Branch/base: `feat/shared-brain-dogfood` at `1ff3125`.
+- OS: Microsoft Windows 11 Pro Insider Preview 10.0.29576 build 29576 64-bit.
+- Python: 3.12.11 via `uv run python`.
+- Command: `uv run pytest tests/test_logstream.py tests/test_mcp_logstream.py tests/test_cli_logstream.py -q`.
+- Result: 72 passed in 2.93s.

docs/rfcs/004-replicated-palace.md

@@ -0,0 +1,630 @@
+# RFC 004: The Replicated Palace
+
+Status: Draft complete — all sections drafted (storage: mac-claude; transport/lifecycle/appendix: windows-claude); awaiting Igor's review
+
+Shipped so far: step 0 (logstream multi-master replication) and step 1 (memory read replicas — snapshot pull, local fold, distributed embedding), plus the transport seam and the estate endpoint. Step 2a (the memory op-log, anti-entropy, fold/promote) and the v4 content-pure id migration + write-flip are designed here but not yet landed on develop — they carry a palace migration and are staged for a later release.
+Owners: mac-claude (storage layers, §6–§9), windows-claude (transport & lifecycle, §5 and Appendix A), decided by Igor
+Created: 2026-07-02
+Branch: `feat/shared-brain-dogfood`
+Prior art: RFC 003 (logstream), the `rfc004_replicated_palace_position` correlation thread (position debate, verbatim in the logstream)
+
+## Summary
+
+Each human has ONE palace — an extension of their brain — replicated in full
+across every machine they own. Agents always talk to the MemPalace service on
+`127.0.0.1`; services converge with each other over an encrypted mesh. The
+hub stops being a dependency and becomes a role (rendezvous, and the home of
+*shared* palaces). The design is judged against offline operation as the
+default posture, not as an edge case.
+
+One sentence: **N equal replicas of the facts, each with locally-derived
+indexes and a local writer, converging through provenance-stamped ops over
+the mesh — with origin as the home-of-record for source-bound maintenance.**
+
+## Motivation
+
+The shared-brain dogfood proved the hub topology works — and watched it fail:
+when the hub machine slept, every other machine lost recall, capture, and
+coordination simultaneously. A brain does not stop remembering because
+another brain is asleep. Concretely, today a remote machine without the hub
+has **no palace at all**.
+
+The mission statement ("memory is identity") implies the requirement
+directly: you do not rent your identity, and you do not park it on a single
+machine.
+
+## Requirements
+
+The availability invariant, and the offline requirements distilled from the
+fleet (logstream correlation `rfc004_replicated_palace_position`):
+
+- **R0 — Mission invariants hold everywhere**: verbatim always, local-first,
+  zero external API for core operations, hooks < 500 ms, startup injection
+  < 100 ms. A design that meets availability by adding a network round-trip
+  to recall fails R0.
+- **R1 — Availability invariant**: recall reads and capture writes never
+  block on the network; only convergence may wait.
+- **R2 — Task freshness**: delegations to offline agents must not execute
+  stale on rejoin (optional `expires_at` on `task.request`; re-check the
+  correlation for `superseded` before acting on an old claim).
+- **R3 — Partition claims**: duplicate task claims across replicas resolve
+  deterministically post-merge (earliest HLC wins; the loser yields with
+  `superseded`). Append-only makes double work safe-but-wasteful, never
+  corrupting.
+- **R4 — Offline capture**: hooks write to the local replica unconditionally;
+  organization-op conflicts merge LWW-by-HLC and the merge is **surfaced**
+  to the user, never silent.
+- **R5 — Presence**: per-agent last-seen derived from log activity (plus
+  device-level liveness from the transport layer) so requesters route around
+  dead machines instead of burning `event_wait` timeouts.
+- **R6 — Rejoin via snapshot + tail**: op-log compaction and snapshot
+  bootstrap are v1 requirements; a months-offline replica cannot replay
+  history.
+- **R7 — Lost-device threat model**: N replicas put the whole brain on every
+  device; encryption at rest per replica is mandatory, and the mesh must
+  support membership revocation for a lost machine.
+- **R8 — Replication is not backup**: tombstones propagate; snapshot backups
+  remain a separate concern.
+
+## Non-Goals
+
+- **Replacing the hub for shared palaces.** Federation (team palace,
+  `shared_agent_brain`) is a different problem: a shared organ legitimately
+  has a home and benefits from strong consistency. This RFC covers the
+  *personal* palace; federation keeps the RFC 003 hub model.
+- **Cloud as a system of record** — constitutionally excluded. The only
+  admissible cloud role is an optional end-to-end-encrypted blob courier for
+  op-sync when no two personal machines are online simultaneously, plus
+  encrypted offsite snapshots. Zero knowledge; never queryable server-side.
+- **Thin/partial replicas (phone-class devices) in v1.** They remain remote
+  clients of a nearby full replica; partial replication is future work.
+- **Multi-user merge and authorization semantics.** One human, N devices.
+  Trust boundary = palace boundary. Mesh membership is full-replica trust,
+  not family/team scoped read or write permission.
+
+## Architecture Overview
+
+```
+  Machine A (mac)                Machine B (windows)          Machine C (laptop)
+  ┌─────────────────────┐        ┌─────────────────────┐      ┌──────────────────┐
+  │ agents → 127.0.0.1  │        │ agents → 127.0.0.1  │      │ agents → local   │
+  │ ┌─────────────────┐ │  ops   │ ┌─────────────────┐ │ ops  │ ┌──────────────┐ │
+  │ │ mempalace svc   │◀┼───────▶│ │ mempalace svc   │◀┼─────▶│ │ mempalace svc│ │
+  │ │  op-log (SoT)   │ │  mesh  │ │  op-log (SoT)   │ │ mesh │ │  op-log (SoT)│ │
+  │ │  derived index  │ │        │ │  derived index  │ │      │ │ derived index│ │
+  │ └─────────────────┘ │        │ └─────────────────┘ │      │ └──────────────┘ │
+  └─────────────────────┘        └─────────────────────┘      └──────────────────┘
+         encrypted mesh transport (Layer 1) · anti-entropy op sync (Layer 2)
+```
+
+Three layers, separable by design:
+
+1. **Transport (Layer 1)** — encrypted peer connectivity, membership, and
+   device identity between the machines. Owner: windows-claude (§5).
+2. **Sync (Layer 2)** — a canonical append-only op-log per replica, merged by
+   union with small domain-specific semantics. Owner: mac-claude (§6).
+3. **Derived state (Layer 3)** — vector indexes, embeddings, caches: rebuilt
+   locally per device, never synced. Owner: mac-claude (§7).
+
+The decisive property: **sync the facts, derive the senses.** Ops are
+kilobytes; HNSW graphs are gigabytes. Every machine remembers everything;
+each machine senses with its own hardware.
+
+## Alternatives Considered
+
+| Dimension | Replicated mesh | Self-hosted central server | Cloud backend |
+|---|---|---|---|
+| Offline / partition | Full function, converge later | Dead when server/link down | Dead without internet |
+| Recall latency | Local, sub-ms–ms | LAN ms / tailnet 10–30 ms | 50–150 ms+ |
+| Privacy | Never leaves your devices | Your hardware, one exposed box | Provider sees plaintext or E2EE cripples search |
+| Durability | N live replicas (+R8 backups) | One box | Best-in-class |
+| Ops burden on user | ~zero if software earns it | Forever (patching, TLS, backups) | ~zero |
+| Engineering complexity | High, paid once by us | ~zero (exists today) | Zero for user, trust cost |
+| Consistency | Eventual + merge semantics | Strong, trivially | Strong |
+| Teams / sharing | Wrong tool | Natural | Natural |
+| Exit / lock-in | SQLite files you hold | Files you hold | Provider's mercy |
+
+Verdicts: the mesh is the only option satisfying R0+R1 for the personal
+palace (a cloud round-trip spends the entire hook budget on network; central
+fails the availability invariant we watched fail in production). The central
+server remains the *correct* model for shared organs (federation). Cloud is
+admissible only as the E2EE courier of the Non-Goals section.
+
+## §5. Layer 1: Transport
+
+Layer 1 exists so Layer 2 never has to think about networks, keys, or which
+machines are awake. It carries ops between replicas and answers one question
+for the layers above: *which of my peers can I reach right now, and are they
+who they claim to be.* Nothing about merge semantics lives here; nothing
+about sockets leaks up.
+
+### 5.1 The seam (what Layer 2 is allowed to assume)
+
+Layer 2 sees peers only through this interface — the entire contract:
+
+```
+Transport {
+  self(): ReplicaId                      // stable, = mesh node identity
+  peers(): Map<ReplicaId, Presence>      // membership + liveness snapshot
+  onPresenceChange(cb)                   // SWIM/heartbeat deltas → R5
+
+  request(peer, path, body): Response    // one authenticated round-trip
+  openStream(peer, path): EventStream    // long-lived, resumable by cursor
+  onInbound(path, handler)               // serve anti-entropy pulls
+}
+```
+
+Two channel shapes cover every need: **request** for anti-entropy pulls
+(`GET /sync/ops?origin=X&after=N`, §6.2) and **stream** for push-notify of
+new ops (the RFC 003 SSE surface, re-pointed peer-to-peer). Both are
+mutually authenticated — a `request` whose caller identity isn't a current,
+non-revoked member is refused at Layer 1, before Layer 2 sees a byte. That
+refusal is the replication ACL (R7) and it is the *only* authorization Layer
+2 relies on: an op that arrived is an op from a trusted replica.
+
+This is deliberately a v1 device-trust boundary, not a user/role permission
+model. Admitting a replica means admitting it to the entire personal palace,
+verbatim content included. Delegated, family, team, or other partial-access
+workflows need a separate palace-level capability layer; they must not be
+modeled as "just add this device to the replication mesh."
+
+`ReplicaId` is the transport's node identity — no separate replica registry
+to drift. Under MeshGuard it is the Ed25519 public key; that same value is
+the `origin_replica` stamped into every op (§8), so provenance and
+authentication are one fact, established once at the transport and trusted
+everywhere above.
+
+### 5.2 Peer addressing & rendezvous
+
+Replicas are named by identity, never by address — a laptop's IP changes
+between café and home; its `ReplicaId` does not. Address discovery is the
+transport's job:
+
+- **MeshGuard / Tailscale**: the mesh's own coordination plane maps identity
+  → current endpoint(s); we never hardcode IPs. A replica "moves" and peers
+  re-resolve transparently.
+- **Bare LAN** (no mesh): mDNS/`_mempalace._udp` discovery within a
+  broadcast domain, identity verified by the membership key (§5.5) — an
+  address hint is never a trust grant.
+- **No global directory.** Rendezvous is peer-to-peer within the mesh
+  membership set. The old hub becomes *a* rendezvous helper for personal
+  replicas (and stays the home of shared palaces per Non-Goals), never a
+  required broker: two personal machines on the same LAN converge with the
+  hub asleep.
+
+### 5.3 NAT traversal posture
+
+Convergence must survive both machines being behind NAT on different
+networks (R1 across the internet, not just the LAN):
+
+1. **Direct** when a route exists (same LAN, or one side reachable).
+2. **Hole-punched P2P** via the mesh coordination plane (WireGuard-style for
+   MeshGuard/Tailscale) — the common cross-network case, still zero data
+   through any third party.
+3. **Relay** only when hole-punching fails: encrypted frames pass through a
+   mesh relay that cannot read them (payloads are already E2E-encrypted at
+   Layer 1). This is transport relay, categorically distinct from the
+   constitutionally-excluded cloud-as-SoR (Non-Goals) — the relay sees
+   ciphertext, never palace content, never queries.
+4. **Never-simultaneously-online** is a real fleet state (desktop by day,
+   laptop by night). Two escape hatches, both opt-in: LAN sync when they do
+   overlap, and the Non-Goals E2EE blob courier (a replica pushes an
+   encrypted op-bundle the other pulls later). The courier is a §5 consumer
+   of the same op ranges, not a new path.
+
+### 5.4 Connection lifecycle
+
+- **Discover → authenticate → sync-on-connect → tail.** On establishing a
+  link a replica exchanges version vectors (§6), pulls the deltas each side
+  is missing, *then* subscribes to the peer's live op stream — snapshot then
+  tail, the pattern windows-codex asked us to name (identical to the viewer's
+  `event_list` cursor → SSE `Last-Event-ID` bootstrap, RFC 003). No live-only
+  connect: the gap between snapshot and stream-open is closed by cursor, so
+  no op is ever missed on reconnect.
+- **Backpressure & retry** mirror the logstream tail already shipped:
+  bounded concurrent streams, `503 + Retry-After` under load, exponential
+  reconnect with the version vector as the resume cursor. A flapping link
+  degrades to periodic anti-entropy pulls, never to lost ops.
+- **Idempotent by construction.** Re-delivering an op is a no-op (union
+  merge, §6.2), so the lifecycle can be as dumb and as retry-happy as it
+  likes — correctness lives in Layer 2, not in careful delivery.
+
+### 5.5 Replica join / leave / revoke ceremony
+
+Membership *is* trust; there is no other authorization in the system.
+
+- **Join** (adding a new personal device): the new replica generates its
+  Ed25519 identity locally (private key never leaves it). An existing
+  authenticated replica admits it to the mesh membership set — a physical-
+  presence / existing-device action by the human, not a password. First sync
+  is a full snapshot + tail (§6, R6). Because organization syncs as ops, the
+  new device reproduces the *same* palace, not a re-clustered one.
+- **Leave** (graceful): a replica can announce departure; peers keep its
+  historical ops forever (they are provenance) but stop expecting presence.
+- **Revoke** (lost/stolen device — R7): the human revokes a `ReplicaId` from
+  any surviving replica; the removal propagates as a membership op and every
+  peer refuses further connections from that identity. Revocation cannot
+  reach into the lost device — hence R7's mandatory at-rest encryption
+  (mac-claude, §10): revocation stops the *network*; encryption protects the
+  *disk already gone*. Ops that identity authored before revocation remain
+  valid history (revoking a device is not disavowing its memories); only its
+  future write access is severed.
+
+### 5.6 Key custody
+
+- **Device key**: Ed25519 private key per replica, generated on-device,
+  non-exportable, OS keystore where available (Keychain / DPAPI / kernel
+  keyring). It is the node's whole identity — losing it = re-join as a new
+  replica; it is never synced.
+- **Membership authority**: which identities are trusted is itself
+  op-carried state (a membership OR-set with revocation tombstones), signed
+  by an admitting device — so "who is in the mesh" converges like everything
+  else and survives any single machine's loss.
+- **At-rest data key** (the palace on disk, R7) is a *separate* concern
+  owned by mac-claude in §10; §5 owns only the identity/membership keys that
+  gate the wire. The two never mix: a compromised transport key exposes no
+  plaintext, a stolen disk exposes no network.
+
+### 5.7 Transport fallback matrix
+
+Per-link, best available wins; the seam (§5.1) makes the choice invisible
+to Layer 2.
+
+| Situation | Transport | Notes |
+|---|---|---|
+| Same LAN, mesh up | MeshGuard direct | Lowest latency; hub not involved |
+| Cross-network, both online | MeshGuard hole-punched (Tailscale bridge if MeshGuard not yet integrated) | E2E, P2P |
+| Hole-punch fails | Encrypted mesh relay | Ciphertext only; distinct from cloud-SoR |
+| Mesh unavailable, same LAN | Bare LAN + mDNS, membership-key auth | Degraded discovery, full trust |
+| Never simultaneously online | E2EE blob courier (Non-Goals) | Async op-bundle exchange |
+| MeshGuard pre-integration | Tailscale (today's dogfood transport) | Ship Sequencing step 0–1 before MeshGuard lands |
+
+MeshGuard is the target; **Tailscale is the shipping fallback that unblocks
+Sequencing steps 0–1 today** (it already carries this very logstream). The
+seam guarantees swapping in MeshGuard later touches no Layer 2 code.
+
+MeshGuard pre-integration checklist (gate, not blocker): the 2026-06-06
+review's H1 (inner-source-IP spoofing on the userspace plane) is remediated
+with regression tests (meshguard PR #101, RX cryptokey-routing check in
+`decryptTransport`); all Criticals/Highs fixed across 17 hardening commits.
+Before MeshGuard becomes the default link: (a) a trust-path sweep of
+post-review commits, (b) an FFI-consumer pass where the daemon binds it.
+Until both pass, Tailscale carries production and MeshGuard rides behind the
+seam in test.
+
+### 5.8 Failure detection → presence (R5)
+
+Presence has two sources; the transport fuses them so the layers above ask
+one question:
+
+- **Device liveness** (Layer 1): SWIM-style failure detection from the mesh
+  (MeshGuard membership; heartbeat pings on bare LAN). Answers "is the
+  machine reachable *now*," sub-second, and drives reconnect.
+- **Agent liveness** (derived, Layer 2): per-agent last-seen from op/log
+  activity (RFC 003), answering "when did this seat last do anything."
+
+`Presence = { reachable: bool (device), lastSeen: hlc (agent), replicaId }`.
+Requesters use it to route around dead machines instead of burning
+`event_wait` timeouts, and it is what the PalaceMind viewer renders
+(Appendix A). This is also the substrate for **R2 (task freshness)** and
+**R3 (partition claims)** on the wire:
+
+- **R2**: a `task.request` with `expires_at` is still *delivered* to an
+  offline agent (ops never drop), but presence tells the requester the agent
+  was unreachable across the gap, so the etiquette check — "re-read the
+  correlation for `superseded`/expiry before acting on a claim older than
+  its transport gap" — has the data it needs. Layer 1 supplies the gap;
+  Layer 2 supplies the rule.
+- **R3**: partitions *cause* duplicate claims (two replicas each admit a
+  claim while unable to see each other). Layer 1's job is to make partitions
+  observable (presence shows the split) and healable (anti-entropy on
+  rejoin); the deterministic resolution — earliest-HLC wins, loser yields
+  `superseded` — is mac-claude's merge rule (§6.2). Presence makes the
+  window small; the merge rule makes the outcome safe.
+
+## §6. Layer 2: The Canonical Op-Log
+
+### 6.1 The op envelope
+
+Every mutation of the palace becomes an immutable op:
+
+```json
+{
+  "op_id": "op_<origin>_<counter>",
+  "origin_replica": "<transport identity of the writing machine>",
+  "author_agent": "mac-claude",
+  "hlc": "0189f3a2-0007-mac",
+  "authored_at": "2026-07-02T21:14:09Z",
+  "kind": "drawer.add | drawer.revise | drawer.tombstone | org.file | org.move | org.tunnel.add | org.tunnel.remove | kg.assert | kg.close | kg.entity.upsert | registry.entity.upsert | event.append | artifact.put | ...",
+  "payload": { "...kind-specific, verbatim content inline or by sha256..." }
+}
+```
+
+- `hlc` is a hybrid logical clock (physical ms + logical counter + replica
+  tiebreak): total order across replicas without clock trust.
+- Per-origin logs are strictly ordered by a local counter; a replica's state
+  is a **version vector** {origin → highest counter applied}.
+- Storage: `oplog.sqlite3` in the palace dir, append-only, WAL — the
+  logstream pattern (RFC 003) generalized; that pattern is production-proven.
+
+### 6.2 Merge semantics (complete list — nothing else exists)
+
+| State | Op kinds | Merge rule |
+|---|---|---|
+| Drawer content | add / revise / tombstone | Grow-only set of content-addressed revisions; head = latest by HLC; tombstone hides, never deletes (verbatim survives) |
+| Artifacts | artifact.put | True G-set, union by sha256 — conflicts impossible |
+| Organization | org.file / org.move / org.tunnel.* | LWW-by-HLC register per drawer (placement) / OR-set (tunnels); merges surfaced to user (R4) |
+| Knowledge graph | kg.assert / kg.close / kg.entity.upsert | Assert = G-set; close = interval-close (idempotent, min valid_to wins); entity upsert = LWW-by-HLC |
+| Registry | registry.entity.upsert | LWW-by-HLC per entity key (replaces whole-file JSON write) |
+| Logstream | event.append | Append-only union; cross-replica order by HLC; per-origin `seq` preserved; consumer contract additive (`origin_replica`, `hlc` are new fields) |
+| Diary | drawer.add in diary rooms | Same as drawer content (already append-only) |
+
+`org.tunnel.remove` is the remove half of the tunnel OR-set: it hides the edge
+from the current organization view, not from history. If MemPalace adopts a
+first-class dormant-tunnel product state, ship it as an explicit state op
+(`org.tunnel.set_state(active|dormant)` or equivalent) rather than overloading
+remove with dormancy semantics.
+
+Anti-entropy: peers exchange version vectors and pull missing per-origin
+ranges (`GET /sync/ops?origin=X&after=N`), push-notified over the existing
+SSE channel. No broker, no framework: automerge/yjs are document-CRDTs
+(wrong shape, heavy deps); cr-sqlite is a native extension whose generic
+table-CRDTs know nothing of id purity or verbatim; file-level sync of live
+SQLite corrupts. The merge logic above is ~hundreds of lines we fully own.
+
+Future domain-specific state needs the same explicit treatment before it
+enters Layer 2. Closet/card semantics, succession edges, contradiction
+intervals, or other semantic structures are not automatically drawer
+placement registers. Each adopted state must name its own op kinds, conflict
+surface, and any merge-exempt or intentionally non-LWW transitions; otherwise
+replicas can silently erase meaning while still "converging."
+
+### 6.3 Id purity (prerequisite, not footnote)
+
+Verified against the code (2026-07-02): today's drawer identities are NOT
+content-addressed. Miner drawers hash `(source_file, chunk_index)` — re-mining
+rewrites content in place under the same id; MCP drawers hash
+`(wing, room, content)` — organization lives inside identity;
+`tool_update_drawer` mutates in place; dedup deletes.
+
+**v4 identity recipe**: `drawer_<hash(content)>` — identity is the verbatim
+content alone. Organization (wing/room) becomes op-carried metadata
+(`org.file`); location provenance (`source_file`, `chunk_index`) becomes
+plain metadata; revision chains link content-addressed revisions. `ids.py`
+already versions recipes (`ID_RECIPE` v1→v3 precedent) and drawers carry
+`id_recipe` metadata, so migration is an audited rewrite with a legacy-id
+alias table for inbound references (tunnels, KG `source_drawer_id`).
+
+### 6.4 Mutable-state inventory → op mapping (verified, with file refs)
+
+| Today (mutable) | Where | Becomes |
+|---|---|---|
+| `tool_update_drawer` in-place update/upsert | mcp_server.py | `drawer.revise` (new content-addressed revision) |
+| Miner re-mine upsert over same id | miner.py:1336,1478 | `drawer.revise` at origin replica only (§8) |
+| `delete_drawer` / `delete_by_source` / dedup batch delete | dedup.py:127 | `drawer.tombstone` (hide, never destroy) |
+| Entity registry whole-file `json.dumps` | entity_registry.py:328 | `registry.entity.upsert` op stream |
+| `hallways.json` whole-file rewrite | hallways.py:140 | `org.tunnel.add/remove` OR-set ops |
+| KG `invalidate` UPDATE of valid_to; entities INSERT OR REPLACE | knowledge_graph.py | `kg.close` interval op; `kg.entity.upsert` |
+| repair / migrate / dedup wholesale rewrites | repair.py, migrate.py | replica-local maintenance of derived state (never synced) |
+
+Hardest today, cleanest after: the two whole-file JSONs (currently pure
+last-writer-wins with silent loss) gain real merge semantics for free.
+
+## §7. Layer 3: Derived State
+
+The vector store stops being the system of record — that single change
+dissolves the fleet-level writer lease, the stdio proxy's reason to exist,
+and the cross-machine integrity gates. Chroma/Qdrant/pgvector/sqlite_exact
+become **fold-and-index consumers** of the op-log, each rebuilt or
+incrementally folded locally.
+
+- Precedent already in-repo: `repair --mode from-sqlite` rebuilds the vector
+  index from content; embeddings are already treated as re-derivable.
+- Embedder identity stays per-replica (RFC 001): pin one model fleet-wide or
+  accept per-device vector spaces — legal because queries execute locally.
+- **The lease demotes, it does not dissolve**: per-replica single-writer
+  over local index state remains (HNSW physics); what disappears is
+  cross-machine write arbitration.
+- Organization is NOT derived state. Filing decisions sync as ops (§6.2):
+  the method of loci means the layout IS the memory; two replicas clustering
+  differently would give the user two different palaces.
+
+## §8. Provenance & Source-Bound Maintenance
+
+Every op carries `(origin_replica, author_agent, hlc, authored_at)` —
+simultaneously the sync unit, the conflict tiebreak, the audit trail, and
+the answer to "which machine/agent did this memory come from."
+
+Local references are replica-local: a drawer mined from `P:\...` on Windows
+references a path that exists only there. The memory replicates everywhere;
+**source-bound maintenance does not**: re-mining after file edits,
+`repair` against origin files, and `delete_by_source` execute only at the
+origin replica (other replicas receive the resulting ops). The mesh must
+know: every memory lives everywhere, but its umbilical cord attaches to one
+machine.
+
+## §9. Sequencing
+
+0. **Logstream multi-master (pilot)** — already append-only; add
+   `origin_replica` + `hlc` (additive to the viewer contract), per-origin
+   logs, HLC ordering, artifact union by sha256. Smallest surface; fixes the
+   pain that motivated everything (coordination dies with the hub).
+1. **Read replicas for memory** — content snapshot + op tail; indexes
+   derived locally (never copy `chroma.sqlite3` — that replicates Chroma's
+   fragility to N machines). Cheap availability for recall; builds the R6
+   snapshot machinery.
+2. **Canonical op-log + v4 id migration** — §6 in full; backends demoted to
+   derived consumers. Decided 2026-07-02 (Igor): ships as **2a**
+   (drawers + KG ops — the waiting customers: mining promotion and the
+   multi-writer foundation) followed by **2b** (registry + hallways/tunnels
+   op conversion); superseded revisions are kept **forever** (verbatim
+   maximalism — search surfaces head revisions only; no GC path exists);
+   the v4 migration runs **staged on a palace copy first**, validated, then
+   live with a timestamped backup and a brief read-only window; the mac
+   origin runs a **dual-write shadow period** (Chroma writes + op emission,
+   divergence detectable) before cutover. Step 3's write-flip on remote
+   replicas begins only **after the local-capture promotion validates**
+   end-to-end. Two commitments this step MUST honor:
+   - **Local-capture promotion.** A step-1 replica may mine machine-local
+     data (projects, conversations) into its own palace before step 2
+     exists — such drawers carry no `replica_origin` stamp, so read-replica
+     reconciliation cannot touch them. Step 2 ships a one-time promotion
+     pass: every unstamped local drawer becomes a `drawer.add` op under the
+     replica's identity and flows to the whole mesh. Capture-now is
+     forward-compatible by contract, not by luck; nothing mined early is
+     ever re-mined or lost.
+   - **v4 ids are the cross-machine dedup.** Content-pure identity means
+     the same content mined on two machines yields the same drawer id; the
+     grow-only merge collapses duplicates into one drawer with multiple
+     provenance records. The migration is the dedup mechanism — no separate
+     dedup pass across origins.
+3. **Full multi-writer** — every replica captures locally, all converge.
+
+Each step ships value alone; none blocks on Layer 1 choice (seam, §5).
+
+## §10. Security — shared ownership
+
+Threat model: N replicas mean the entire brain — every drawer, verbatim —
+sits on every device. Two independent exposures follow, and the design keeps
+them independent: the **wire** (in transit between replicas) and the **disk**
+(at rest on each replica). A compromise of one must not yield the other.
+
+### 10.1 Transport-side (windows-claude)
+
+- **In transit**: every op crosses the mesh E2E-encrypted (MeshGuard /
+  WireGuard-class); relays and couriers (§5.3) see ciphertext only. No
+  palace bytes ever traverse a third party in the clear — the constitutional
+  no-cloud-SoR line holds even when a relay is used.
+- **Membership as the only ACL**: an op is authorized iff it arrived over a
+  channel authenticated to a current, non-revoked `ReplicaId` (§5.1). There
+  is no per-op signature check in Layer 2 — trust is established once at the
+  transport and is total above it. Membership itself is convergent,
+  revocation-aware state (§5.6).
+- **Revocation ceremony** (R7, network half): revoke a lost device's
+  identity from any survivor; the tombstone propagates and every peer
+  refuses it thereafter. This severs *future* access only — see 10.3.
+- **Identity-key custody**: per-device Ed25519 key, on-device, non-exportable
+  (§5.6). Compromising it grants mesh access (mitigated by revocation) but,
+  by construction, decrypts nothing at rest.
+
+### 10.2 At-rest side (mac-claude)
+
+Two tiers, both grounded in one rule: **data keys are per-replica and never
+traverse the mesh.** Ops arrive over the wire (10.1), are held decrypted
+only in memory, and are written under the receiving replica's own key.
+Compromising one device's disk therefore never yields a key that opens any
+other device — the same independence 10.3 demands between wire and disk
+holds between disks.
+
+- **Tier 0 (baseline, v1-mandatory to document and detect)**: full-disk
+  encryption — FileVault / BitLocker / LUKS. Zero code, protects the
+  powered-off stolen device, and is the only tier that also covers derived
+  indexes we do not control internally (Chroma's own SQLite holds verbatim
+  documents). Setup surfaces a loud warning when the palace directory lives
+  on an unencrypted volume.
+- **Tier 1 (target): per-palace data key over the canonical stores.** A
+  symmetric data key encrypts `oplog.sqlite3`, the content store, the
+  logstream, and the KG (SQLCipher-style page encryption; AEAD, hardware-
+  accelerated — page crypto on hot recall paths fits the R0 budgets). The
+  data key is wrapped by the OS keystore (macOS Keychain/Secure Enclave,
+  Windows DPAPI/TPM, Linux libsecret/TPM), never stored in the palace
+  directory, never synced, and is distinct from the 10.1 identity key by
+  construction — neither derives from the other.
+- **Derived indexes under Tier 1**: rebuildable by definition (§7), so they
+  get the cheaper policy — rely on Tier 0, or (paranoid profile) treat them
+  as ephemeral: discarded on lock, refolded from the encrypted op-log on
+  unlock. Never the canonical stores' key.
+- **Rotation is replica-local**: re-encrypt local files under a new data
+  key and re-wrap; no mesh coordination, no ops emitted, because keys are
+  not replicated state.
+- **R7 composition**: a lost powered-off device presents Tier 0 + Tier 1 to
+  the attacker; the revocation ceremony (10.1) has already severed its
+  future sync regardless of whether the disk ever yields.
+
+### 10.3 The seam between them
+
+Revocation stops the network; encryption protects the disk that is already
+gone. A stolen, powered-off device is defended only by 10.2; a live device
+still on the mesh is defended by 10.2 (unlocked-disk exposure) *and* 10.1
+(revocation cuts its future sync). Neither half covers the other — which is
+why both are mandatory, not alternatives.
+
+Explicitly: **replication ≠ backup (R8)**. Tombstones propagate, so a
+fat-fingered mass delete replicates faithfully to every device; encrypted
+offsite snapshots remain a separate mechanism, out of this layer's scope.
+
+## Open Questions
+
+- HLC skew bounds and how loudly to surface clock anomalies.
+- Op-log compaction policy (R6): checkpoint cadence for snapshot bootstrap.
+  (Content-revision retention is settled: superseded revisions are kept
+  forever; compaction only concerns op-replay bootstrap cost, never
+  content.)
+- Does the E2EE cloud courier ship in v1 or wait for demand?
+- Partial replicas for phone-class devices (deferred; Non-Goals).
+- Federation bridge: can a personal replica project shadow wings into a
+  team hub palace mechanically, or is that manual today?
+
+## Appendix A: PalaceMind
+
+The desktop app is where replication stops being infrastructure and becomes
+something the human can see and trust. Its guiding principle is the one
+already shipping in the connected-palace and Agents surfaces (ADR-0028):
+**honest status over hidden magic.** Four consumer surfaces.
+
+### A.1 Replica status (generalizing Live/Polling)
+
+The Agents viewer already renders a `Live · seq N` vs `Polling` honesty flag
+off the logstream. That generalizes directly to the fleet: a compact
+**estate health** panel — one row per known replica showing `reachable`
+(device presence, §5.8), last-seen, and **version-vector drift** ("this
+device is 14 ops behind mac"). Drift is the replication analogue of the
+Polling flag: it never lies about how converged you are. The Wings-page
+estate map (already an outer orbit for the connected palace) becomes the
+natural home — replicas as nodes, edges dimming when a peer goes
+unreachable, so "the house breathes when the fleet works" is literal.
+
+### A.2 Merge surfacing (R4 — the load-bearing UX)
+
+R4 forbids silent organization merges: when two replicas filed the same
+drawer differently during a partition and LWW-by-HLC picks a winner, the
+loser must be *shown*, not swallowed. PalaceMind owns that moment:
+
+- A quiet, dismissible notice — "This capture was filed in *Meds* on your
+  laptop and *Billing* here; kept *Billing* (more recent). Move it?" — with
+  one-click accept-the-other. Never a modal, never blocking; the calm
+  enterprise register, matching the assistant-offline notice we just shipped.
+- A **Reconciliation** view listing recent auto-merges with their HLC
+  reasoning and undo, so a power user can audit what convergence decided.
+  The op-log makes this free: every merge is two ops and a rule.
+
+### A.3 Presence rendering (R5)
+
+Presence (§5.8) surfaces wherever a human waits on another machine: the
+Agents inbox marks a delegate's device offline so the user doesn't expect a
+reply from a sleeping laptop; agent pickers show a liveness dot; the
+open-loops panel flags a thread blocked on an unreachable replica distinctly
+from one merely awaiting a claim. The rule: never make the user infer from
+silence what the system already knows.
+
+### A.4 Multi-origin logstream
+
+The viewer must not fracture when ops carry `origin_replica` + `hlc`
+(additive fields, per §6.2 — the contract stays compatible):
+
+- **Order by HLC**, not per-origin `seq`, once events span replicas; keep
+  `seq` as the per-origin resume cursor.
+- **Dedupe by `op_id`/event id** across origins (the same op can arrive via
+  two peers) — the merge is idempotent, the UI must be too.
+- **Origin as provenance**, alongside the existing agent identity: "filed by
+  mac-claude on *laptop*." This also lands windows-codex's identity-alias
+  request cleanly — a rename is just another provenance fact the log already
+  carries; render "windows-claude (formerly claude-fable-5-windows)" from
+  the correction op, thread the identities together in filters, keep raw
+  `from_agent`/`origin_replica` per event for audit.
+
+Consumer contract, restated for implementers: bootstrap is always
+**snapshot (`event_list`/`/sync` cursor) → set version vector → tail
+(SSE `Last-Event-ID`)**; never live-only connect (the gap loses ops). This
+is the same pattern the Agents page already uses and the one windows-codex
+asked us to name canonically.

docs/rfcs/005-agent-identity-routing.md

@@ -0,0 +1,272 @@
+# RFC 005: Agent Identity & Routing
+
+Status: Draft — for Igor's review
+Owner: mac-claude (backend/hub owner; identity is palace/fleet infrastructure, not a meshguard concern)
+Created: 2026-07-04
+Branch: `feat/shared-brain-dogfood`
+Prior art: RFC 003 (logstream — the routing surface this refines), RFC 004 (the replicated palace — provides the stable replica identity a host label derives from), the `rfc005_agent_identity_routing` correlation thread
+
+## Summary
+
+An agent's coordination identity is the triple **`host:agent:project`** — the
+granularity at which a working actor actually shares filesystem, local config,
+and knowledge. Two chat sessions open in the same project on the same machine
+are the **same** actor and carry the **same** identity; process/session/PID is
+event *metadata*, never part of who you are. Routing stays exactly as RFC 003
+defines it — exact match plus `*` broadcast over an opaque string — so this RFC
+adds a naming convention and one durable renderer change, not a new matcher.
+
+One sentence: **identity = `host:agent:project`, minted once by the block that
+already renders it, routed by the string equality the logstream already has.**
+
+## Motivation
+
+The shared brain identifies each agent by a flat name — `mac-claude`,
+`windows-claude`, `blade-claude`. That name is not chosen by the agent; it is
+**rendered into** the agent's instructions by a managed block MemPalace writes
+to `~/.claude/CLAUDE.md`:
+
+```
+<!-- mempalace-shared-brain:start -->
+## MemPalace shared brain
+You share a MemPalace hub with other agents. Your agent identity is
+windows-claude — use it as from_agent/created_by ...
+<!-- mempalace-shared-brain:end -->
+```
+
+The flat name conflates two things that have now come apart in the live fleet:
+
+1. **Two sessions, one name.** The windows box ran two concurrent `claude`
+   sessions — one on the palace/mesh track, one on meshguard — both rendering
+   `from_agent=windows-claude`. Their inboxes, their claims, and their diary
+   entries interleaved under one identity with no way to tell "which window."
+   A `status=claimed` from one window looked, to the other, like *someone else*
+   had claimed it — or worse, like *itself* had, with no way to be sure.
+
+2. **No project seam.** Recall, delegation, and the diary all key off the flat
+   name, so a meshguard session and a mempalace session on the same box share
+   one knowledge wing. Recall for the meshguard actor surfaces mempalace facts
+   and vice-versa; the routing name carries no notion of *what work this is*.
+
+Igor's framing resolves both: identity is the tuple at which knowledge scope,
+workspace, and local configuration are actually shared. Same box + same project
+= same actor, however many windows are open. Different project on the same box
+= a different actor with its own knowledge seam.
+
+## Requirements
+
+- **I1 — Identity is `host:agent:project`.** The three components are the axes
+  along which real sharing happens: `host` (same machine → same filesystem,
+  same daemon, same local config), `agent` (which assistant/runtime), `project`
+  (which workspace → which knowledge scope).
+- **I2 — Same-project sessions share one identity.** A second window in the
+  same project on the same box inherits the same fs, config, and knowledge — it
+  *is* the same actor. Identity must not fragment per session, per window, or
+  per PID.
+- **I3 — Session/PID is metadata, never identity.** Which process holds a port,
+  whose PID to signal on cleanup — that belongs in event `metadata`, not in
+  `from_agent`/`to_agent`.
+- **I4 — Claims are held by the identity, not the session.** A `status=claimed`
+  by *any* session of an identity is a claim by that identity; a second session
+  of the same identity must treat its own identity's open claim as a mutex
+  ("don't work what you've already claimed").
+- **I5 — No new routing primitive required.** The tuple must route on the
+  logstream exactly as flat names do today (RFC 003: `to_agent` exact match, or
+  `to_agent='*'` broadcast). Hierarchical/glob routing is explicitly deferred
+  (see Decision 1).
+- **I6 — Minted at the source, not hand-edited.** The identity is *rendered*
+  into each box's instructions; the durable fix changes the renderer so every
+  box re-derives its tuple on the next sync. No per-box hand-editing as the
+  steady state.
+- **I7 — Backward compatible.** Existing flat names (`mac-claude`, …) remain
+  valid and keep routing. Migration to the tuple is incremental and only forced
+  where a real collision exists.
+
+## Non-Goals
+
+- **A lease/lock server.** Claim safety is achieved by append-only events and a
+  deterministic post-hoc tiebreak (Decision 2), consistent with RFC 004's R3.
+  Nothing here introduces a coordinator or a mutex service.
+- **A new matcher.** Prefix/glob routing (`windows:claude:*`, `*:*:meshguard`)
+  is a plausible future affordance but is **not** adopted here (Decision 1).
+- **Cross-user identity.** One human, N devices, per RFC 004. The tuple
+  distinguishes *this* human's actors, not multiple humans.
+- **Changing how `from_agent`/`to_agent` are stored or validated.** Colons
+  already pass `_sanitize_routing`; the tuple is a legal value today.
+
+## The Identity Tuple
+
+```
+host : agent : project
+```
+
+- **`host`** — a stable, human-readable label for the machine. It derives from
+  the replica identity RFC 004 already establishes (each replica has a durable
+  id), rendered to a friendly label (`mac`, `blade`, `windows`) rather than an
+  opaque hash. One host = one replica = one local daemon.
+- **`agent`** — the assistant/runtime family (`claude`, `codex`, `hermes`, …).
+  This is the existing suffix of today's flat names, lifted out intact.
+- **`project`** — the workspace the session is operating in, derived from the
+  session's working directory (its project/repo name). This is the new axis and
+  the one that carries the knowledge seam.
+
+Today's flat names are exactly `host-agent` with the `project` axis missing.
+That is why migration is cheap: `mac-claude` → `mac:claude:<project>` is an
+append, not a rename, and a single-project agent may stay flat until a real
+collision appears.
+
+**Delimiter.** The colon is deliberate: it is already legal in routing fields
+(`_sanitize_routing` rejects only control characters, null bytes, and
+over-length values), it does not collide with the stream delimiter `/`
+(`project/mempalace`), and it reads unambiguously. The tuple is treated by the
+hub as one opaque string — the colons are a *convention for humans and
+renderers*, not something the matcher parses (see Decision 1).
+
+## Same-Session Equivalence (I2)
+
+Two sessions with the same `host:agent:project` are one actor. Concretely:
+
+- They watch the **same inbox** (`to_agent=<tuple>`, which also catches `*`).
+- They write with the **same `from_agent`**.
+- They share **one knowledge wing and one diary** (see Knowledge Partition).
+- A task claimed by one is claimed by the identity (I4).
+
+The second window is not a new participant to be tracked; it is the same
+participant with a second process. When the fleet needs to know *which process*
+(e.g. which one holds a hardcoded port, which PID to signal), that detail rides
+in event `metadata` — `{"pid": …, "session": …}` — and never in the identity.
+
+## Claim Safety Under a Shared Identity (I4, Decision 2)
+
+A shared identity introduces one race: two sessions of the same identity (or
+two replicas of the same identity, post-partition per RFC 004 R3) claim the
+same task before either sees the other's claim. Resolution, no lease server:
+
+1. **Natural mutex.** Before claiming, a session lists open claims for its own
+   identity on the correlation. If its identity already holds an open claim, it
+   does not double-claim — it either joins or waits. This alone removes the
+   common case (a second window picking up work the first already took).
+2. **Deterministic tiebreak for the residual window.** If two claims land
+   before either is visible, the claim with the **lowest HLC wins**; the loser
+   backs off and re-acks `status=superseded`. This is the RFC 004 R3 rule
+   (earliest-HLC-wins), reused verbatim — append-only makes the wasted work
+   safe, never corrupting.
+
+No new event type is required: `event.ack` with `status=claimed` /
+`status=superseded` already expresses both steps.
+
+## Knowledge Partition (I2, bonus)
+
+Keying the palace wing and the diary by the full tuple partitions recall along
+the same seam routing already uses: a `…:meshguard` actor's recall stops
+surfacing `…:mempalace` facts. Today a single-agent box interleaves both
+projects in one wing (e.g. `wing_windows-claude`); the tuple gives each project
+its own wing without a new mechanism — it is just a longer wing key.
+
+This is a **bonus, not a v1 requirement** for routing: an agent may adopt the
+tuple for `from_agent`/`to_agent` first (fixing the coordination collision) and
+partition its knowledge wing later. The two are independent adoptions.
+
+## Decisions
+
+Two choices live in the backend/hub and are resolved here so downstream
+consumers (PalaceMind's viewer, the renderer, other agents) can build against a
+fixed contract.
+
+### Decision 1 — Matcher: **no change.** Route the tuple as an opaque string.
+
+`to_agent` matching stays exactly as RFC 003 defines it:
+
+```sql
+(to_agent = ? OR to_agent = '*')
+```
+
+The tuple `host:agent:project` is matched by **string equality**; `*` remains
+the only wildcard, meaning fleet-wide broadcast. Hierarchical routing —
+`windows:claude:*` for "any agent on this box", `*:*:meshguard` for "whoever
+owns meshguard" — is **not** adopted in this RFC.
+
+*Rationale.* The collision that motivated this RFC is solved entirely by
+minting distinct identities (Decision below on order + the renderer fix); it
+does not require the hub to *understand* the tuple. A glob/prefix matcher is a
+real index-and-correctness surface (partial-match semantics, index design,
+interaction with `*` broadcast) and should be its own proposal driven by a
+concrete need — "route to whoever owns project X regardless of host" — that the
+fleet has not yet hit. Keeping the matcher on exact-plus-broadcast means this
+RFC ships as a **convention + one renderer change**, with zero risk to the
+routing hot path. If hierarchical routing is later wanted, it is additive: a new
+optional match mode, not a migration.
+
+### Decision 2 — Order: **`host:agent:project`** (host-first).
+
+*Rationale.* Host-first matches how the fleet's names already read
+(`mac-*`, `windows-*`, `blade-*` are host-then-agent), so migration is a pure
+suffix append with no reordering. It favors the "any agent on this box" reading,
+which aligns with the operational reality that a host owns one daemon and one
+filesystem. Project-first would favor "who owns project X across hosts" — the
+arguably more common *coordination* query — but that query is exactly the one
+Decision 1 declines to make routable for now, so optimizing the string order
+for it buys nothing today. If Decision 1 is ever revisited and cross-host
+project routing becomes a first-class need, the order can be revisited with it;
+until then, host-first is the lower-friction choice.
+
+## The Durable Fix: mint the identity at the renderer (I6)
+
+The identity is hardcoded because MemPalace **renders** it. The one-place fix is
+to change the renderer/installer that writes the
+`<!-- mempalace-shared-brain:start … end -->` block so it emits the tuple
+instead of the flat name:
+
+- **`host`** — derived from the local replica identity (RFC 004), mapped to a
+  friendly label.
+- **`agent`** — the runtime family, as today.
+- **`project`** — derived from the session's workspace (the project/repo the
+  block is being rendered for).
+
+Then every box re-renders to its tuple on the next sync — no hand-editing. A
+hand-edited stopgap on a colliding box (deriving `host:agent:<project>` locally)
+is legitimate as a bridge and is simply superseded — or overwritten — by the
+renderer change, which is the intended outcome.
+
+**Compatibility.** The renderer must not thrash existing single-project boxes:
+where an agent has exactly one project and no collision, rendering may keep the
+flat `host-agent` form (I7) and only expand to the tuple when a second project
+or a collision appears on that box. The block's managed markers make the
+rewrite safe and idempotent — same rules as today.
+
+## Sequencing
+
+This RFC is **deferred behind the RFC 004 write-flip**: it touches no storage or
+merge semantics, so it does not gate the flip, and the flip's cutover work takes
+priority. Once that lands:
+
+1. **Convention adopted** (docs-only): agents may begin using
+   `host:agent:project` as `from_agent`/`to_agent`. Routes today, no code
+   change (colons already valid; matcher unchanged).
+2. **Renderer change** (the durable fix): the shared-brain block emits the tuple
+   per the rules above. Reviewed on the windows box first, since it has the only
+   live two-session collision.
+3. **Knowledge partition** (optional, later): wing/diary keyed by the tuple, so
+   recall partitions on the same seam. Independent of steps 1–2.
+
+## Open Questions
+
+- **`project` derivation edge cases.** A session with no clear workspace (a
+  bare shell, a home-directory session) has no natural `project`. Proposal: fall
+  back to the flat `host-agent` form (I7) rather than invent a sentinel project.
+- **Friendly host labels.** The replica→label mapping (`mac`, `blade`,
+  `windows`) needs a stable source of truth. Proposal: derive from the
+  replica record RFC 004 already maintains; a label collision across two of the
+  user's machines is resolved by the user at join time, once.
+- **Whether the renderer should ever *contract* a tuple back to flat** when a
+  project is abandoned. Leaning no — an identity that has appeared on the
+  logstream should be stable — but the managed block technically could.
+
+## Relationship to RFC 004
+
+RFC 004 gives this RFC its `host` axis for free: a replicated palace already has
+a durable per-replica identity, and one host is one replica is one local
+daemon. RFC 004's R3 (deterministic partition-claim resolution, earliest-HLC
+wins) is the exact rule Decision 2's claim-safety tiebreak reuses. This RFC does
+not alter RFC 004's storage, op-log, or merge design in any way — it refines the
+RFC 003 coordination surface that rides on top.

docs/write-routing-policy.md

@@ -109,11 +109,48 @@ Invalid values fail with a source-specific error rather than silently falling
 back. This is important because silently turning a misspelled `require` into a
 direct write would violate the safety purpose of the policy.
 
+## Local backend single-writer safety
+
+File-backed backends such as `chroma`, `sqlite_exact`, and Milvus Lite support
+exactly one writable process per palace. Serializing individual calls is not
+enough because each long-lived process can retain SQLite/WAL, FTS, or vector
+index state between calls.
+
+- A writable daemon owns the palace writer lease for its full lifetime.
+- Writable MCP HTTP acquires that lease before binding, holds it through the
+  full serving lifetime, and releases it after active requests stop.
+- MCP stdio opens `sqlite_exact` read-only until it acquires the writer lease.
+  It may therefore coexist for reads; mutating tools refuse while another
+  process owns the lease and reopen writable storage after that owner exits.
+- Read-only MCP HTTP may coexist with the writer.
+- Read-only `sqlite_exact` clients use an immutable connection for a clean
+  checkpointed database, or `mode=ro` when an active writer's complete WAL
+  sidecar pair must remain visible. Both paths enable `query_only` and skip
+  schema, WAL, FTS, migration, and metadata initialization.
+- Direct CLI and hook writes must not run beside a writable daemon or MCP HTTP
+  owner. Route them through the daemon with `require` when the daemon owns the
+  palace.
+- Direct `sqlite_exact` collection mutations contend for the same palace lease,
+  and full LLM closet regeneration owns it before opening collections or
+  calling the configured model.
+
+`MEMPALACE_MCP_ALLOW_PEER_WRITER` cannot bypass this protection for local
+file-backed or unknown plugin backends. It is retained only for explicitly
+remote service backends (`qdrant`, `pgvector`, and Milvus server/Zilliz Cloud)
+that coordinate concurrent clients themselves. Milvus Lite remains protected
+as local file-backed storage.
+
+Do not delete or unlink a live palace lock to recover ownership. Stop the
+owning process cleanly; the operating system releases its lock automatically.
+If corruption is suspected, back up the palace and run integrity/repair
+operations offline, with no writable service running.
+
 ## Follow-up PRs
 
-PR 2 will apply the policy to hook-triggered writes.
+Hook-triggered writes now consume this policy; see
+`docs/hook-write-routing.md`.
 
-PR 3 will apply the policy to routine CLI writes.
+The remaining rollout PR will apply the policy to routine CLI writes.
 
 Maintenance operations such as repair, migration, and index rebuild are not
 ordinary routed writes. They require a separate exclusive-maintenance policy.

examples/mx3_public_shim_embeddings_rerank.md

@@ -0,0 +1,92 @@
+# MX3 Public Shim Embeddings and Rerank
+
+MemPalace can optionally route Chroma embeddings and post-retrieval rerank through the public
+MX3 shim helpers.
+
+This path is local by default. It sends the live search query and candidate hits to the configured
+shim endpoint, so remote endpoints are blocked unless you explicitly opt in, and any remote
+endpoint must use HTTPS.
+
+Public repo:
+
+- `https://github.com/grtninja/mx3-public-shim`
+
+## Hardware required
+
+This setup requires a MemryX MX3 accelerator in an M.2 slot behind the configured public-shim
+endpoint.
+
+## Install
+
+```bash
+pip install "mx3-public-shim[mempalace] @ git+https://github.com/grtninja/mx3-public-shim.git"
+```
+
+## Validate before enabling
+
+Use this path only with a hardware-backed public-shim deployment.
+
+```bash
+python -m mx3_public_shim.doctor
+```
+
+MemPalace rejects the explicit `cpu_reference` fallback, but you should still validate the
+configured endpoint before enabling it:
+
+- the runtime must expose an embeddings provider
+- the selected provider must not be `cpu_reference`
+- the endpoint should be the local public shim unless you intentionally opt in to a remote HTTPS
+  target
+
+On Windows, a healthy doctor report normally shows:
+
+- `mx3_linux` unavailable because the direct official Python runtime is Linux-only
+- `openai_compat` available and selected for `embeddings`
+- `openai_compat` available and selected for `chat`
+
+That means MemPalace is using the local public-shim OpenAI-compatible surface while the required
+hardware lives behind that endpoint.
+
+## Enable embeddings + rerank in MemPalace
+
+```bash
+export MEMPALACE_EMBEDDING_BACKEND=mx3_public_shim
+export MEMPALACE_SEARCH_RERANKER=mx3_public_shim
+export MEMPALACE_MX3_PUBLIC_SHIM_BASE_URL=http://127.0.0.1:9000/v1
+export MEMPALACE_MX3_PUBLIC_SHIM_EMBEDDING_MODEL=text-embedding-nomic-embed-text-v1.5
+export MEMPALACE_MX3_PUBLIC_SHIM_PROVIDER_ORDER=mx3_linux,openai_compat
+export MEMPALACE_MX3_PUBLIC_SHIM_REQUIRE_HARDWARE=1
+```
+
+On Windows, the direct `mx3_linux` provider is not expected to resolve; the normal path is a local
+public-shim OpenAI-compatible endpoint that is itself backed by the required hardware.
+
+If your endpoint requires authentication, also set:
+
+```bash
+export MEMPALACE_MX3_PUBLIC_SHIM_API_KEY=...
+```
+
+`MEMPALACE_EMBEDDING_BACKEND` and `MEMPALACE_SEARCH_RERANKER` can be enabled independently. Use
+both when you want MX3-backed Chroma embeddings plus post-retrieval rerank.
+
+If you intentionally point `MEMPALACE_MX3_PUBLIC_SHIM_BASE_URL` at a non-localhost endpoint, also
+set:
+
+```bash
+export MEMPALACE_MX3_PUBLIC_SHIM_ALLOW_REMOTE=1
+```
+
+## Optional example script
+
+The companion example below demonstrates the narrow Chroma seam directly:
+
+```bash
+python examples/mx3_public_shim_embeddings_rerank.py
+```
+
+Then search normally:
+
+```bash
+mempalace search "auth decisions"
+```

examples/mx3_public_shim_embeddings_rerank.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Example: query a local public-shim embeddings endpoint and pass vectors into Chroma."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from urllib import request
+
+try:
+    from chromadb import PersistentClient
+except ImportError as exc:  # pragma: no cover
+    raise SystemExit("Install chromadb before running this example.") from exc
+
+
+BASE_URL = os.environ.get("MEMPALACE_MX3_PUBLIC_SHIM_BASE_URL", "http://127.0.0.1:9000/v1")
+EMBEDDINGS_URL = BASE_URL.rstrip("/") + "/embeddings"
+EMBED_MODEL = os.environ.get(
+    "MEMPALACE_MX3_PUBLIC_SHIM_EMBEDDING_MODEL",
+    "text-embedding-nomic-embed-text-v1.5",
+)
+
+
+def fetch_embeddings(texts: list[str]) -> list[list[float]]:
+    payload = json.dumps({"model": EMBED_MODEL, "input": texts}).encode("utf-8")
+    headers = {"Content-Type": "application/json"}
+    api_key = os.environ.get("MEMPALACE_MX3_PUBLIC_SHIM_API_KEY")
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    req = request.Request(
+        EMBEDDINGS_URL,
+        data=payload,
+        headers=headers,
+        method="POST",
+    )
+    with request.urlopen(req, timeout=20) as response:
+        body = json.loads(response.read().decode("utf-8"))
+    data = body.get("data", [])
+    if len(data) != len(texts):
+        raise RuntimeError("Embedding response count did not match input count")
+    return [row["embedding"] for row in data]
+
+
+def main() -> None:
+    documents = [
+        "A local retrieval queue keeps notes that still need verification.",
+        "The workstation stores a short MX3 setup checklist for repeatable bring-up.",
+        "This note is about watering tomatoes in the backyard.",
+    ]
+    metadatas = [
+        {"wing": "verification"},
+        {"wing": "operations"},
+        {"wing": "personal"},
+    ]
+    query = "How does the system track notes that still need verification?"
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        client = PersistentClient(path=str(Path(tmpdir)))
+        collection = client.get_or_create_collection("mempalace_drawers")
+        doc_vectors = fetch_embeddings(documents)
+        collection.add(
+            ids=["1", "2", "3"],
+            documents=documents,
+            embeddings=doc_vectors,
+            metadatas=metadatas,
+        )
+        query_vector = fetch_embeddings([query])[0]
+        result = collection.query(query_embeddings=[query_vector], n_results=3)
+        print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

integrations/openclaw/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: mempalace
 description: "MemPalace — Local AI memory with 96.6% recall. Semantic search, temporal knowledge graph, palace architecture (wings/rooms/drawers). Free, no cloud, no API keys."
-version: 3.6.0
+version: 3.7.0
 homepage: https://github.com/MemPalace/mempalace
 user-invocable: true
 metadata:

integrations/shared/coordination-protocol.md

@@ -0,0 +1,125 @@
+# MemPalace Shared-Brain Coordination Protocol
+
+The canonical protocol for agents sharing one MemPalace hub — memory
+discipline plus the logstream coordination layer (RFC 003). Like
+`recall-protocol.md`, this file is the single source of truth: skills,
+rules, and system prompts should link here or copy the System-Prompt
+Snippet below verbatim, so the protocol never drifts per-agent.
+
+## The two layers
+
+A shared palace gives every agent two distinct channels. Do not mix them:
+
+- **Memory (drawers, KG, diary)** — durable knowledge worth recalling
+  later. Searched semantically. Follow
+  [`recall-protocol.md`](recall-protocol.md).
+- **Coordination (logstream events + artifacts)** — active work moving
+  between agents *right now*: delegations, replies, patches, acks.
+  Filtered structurally, never searched semantically.
+
+Rule of thumb: if another agent should **act** on it, it is an event.
+If a future session should **know** it, it is a drawer. A concluded
+delegation usually produces both: the events carried the work; a drawer
+records the outcome.
+
+## Identity
+
+Every agent uses one stable `from_agent` identity, formatted
+`<machine>-<harness>` (e.g. `mac-claude`, `windows-codex`,
+`aero-opencode`). Never impersonate another agent; never rotate names —
+the event trail is only auditable if identities are stable.
+
+## Delegating work (requester)
+
+1. Generate a `correlation_id` for the task: `task_<short-description>`
+   plus enough entropy to be unique (e.g. `task_fix_ranking_7f3a`).
+2. `mempalace_event_append` with `type=task.request`, `stream=project/<name>`,
+   `room=delegation`, `to_agent=<worker>`, `status=open`, and a body that
+   states the goal, the branch, the base commit, and the definition of done.
+3. Wait for the reply: `mempalace_event_wait` with the `correlation_id`
+   and `to_agent=<you>`. Waits cap at 5 minutes — loop, passing
+   `since_event_id` of the last event you saw.
+4. When a `patch.ready` arrives: `mempalace_artifact_get`, verify the
+   `sha256`, apply locally, run the stated verification.
+5. Always close the loop with `mempalace_event_ack` — `status=applied`
+   on success, `status=failed` with verbatim evidence on failure.
+
+## Receiving work (worker)
+
+1. Poll or wait for `type=task.request`, `to_agent=<you>` (plus `*`
+   broadcasts are matched automatically).
+2. Claim it: `mempalace_event_ack` with `status=claimed` so no other
+   agent duplicates the work.
+3. Do the work on the stated branch/commit.
+4. Deliver through the formal channel — `mempalace_patch_submit` with the
+   diff, `correlation_id`, `branch`, and `base_commit`. **Pushing a
+   branch is not a handoff**; the event is. If you also pushed, say so
+   in the body.
+5. If blocked or unable to produce a patch, still reply:
+   `type=task.reply` with `status=blocked` or `failed` and verbatim
+   notes. Silence is the only unrecoverable failure.
+
+## Hard rules
+
+- **Never apply a patch silently.** Fetching an artifact is free;
+  applying it is an explicit local decision, stated to the user.
+- **Verify hashes.** An artifact's `sha256` must match its content
+  before you act on it.
+- **Append-only.** Never try to edit or delete events; supersede with a
+  new event (`status=superseded`) referencing the old one.
+- **Exact payloads.** Bodies and artifacts are verbatim — no summaries
+  of diffs, no truncated logs. If it is too big, store it as an
+  artifact and reference it.
+- **Close every loop.** Every `task.request` you claimed ends in an
+  `applied`, `failed`, or `blocked` — no dangling `open` tasks.
+- **File the outcome.** When a delegation concludes, write one drawer
+  (`mempalace_add_drawer`) recording what was decided/learned, so the
+  result is searchable without replaying the event trail.
+
+## System-Prompt Snippet
+
+Copy this block into an agent's system prompt / custom instructions.
+Replace `<AGENT_ID>` with the agent's stable identity.
+
+```text
+## MemPalace shared brain
+
+You share a MemPalace hub with other agents. Your agent identity is
+<AGENT_ID> — use it as from_agent/created_by in every MemPalace call.
+
+Memory (recall + writing):
+- Before answering about past work, decisions, people, or projects,
+  search the palace (mempalace_search; mempalace_kg_query for
+  relational/temporal facts). Quote results verbatim — never paraphrase
+  stored content. If the palace has nothing, say so; don't guess.
+- File durable outcomes (decisions, conclusions, learned facts) with
+  mempalace_add_drawer. When a fact changes: mempalace_kg_invalidate the
+  old fact, then mempalace_kg_add the new one. Don't file secrets or
+  tokens.
+
+Coordination (logstream):
+- Check your inbox when starting work and before long tasks:
+  mempalace_event_list with to_agent=<AGENT_ID> (new since your last
+  seen event id).
+- To delegate: mempalace_event_append (type=task.request, stream=
+  project/<name>, room=delegation, correlation_id=task_..., status=open,
+  body = goal + branch + base commit + definition of done), then
+  mempalace_event_wait on that correlation_id for the reply.
+- When you accept a task: ack it with status=claimed. Deliver code as a
+  patch via mempalace_patch_submit (never just push a branch and go
+  silent). If blocked, reply with status=blocked and verbatim notes.
+- When you receive a patch: mempalace_artifact_get, verify sha256,
+  apply only with explicit user-visible intent, run the stated tests,
+  then mempalace_event_ack with status=applied or failed.
+- Events are append-only and verbatim. Close every loop — no task you
+  touched stays open without an applied/failed/blocked ack.
+```
+
+## See also
+
+- [`recall-protocol.md`](recall-protocol.md) — the search-before-answer
+  memory protocol this composes with.
+- [Agent Logstream concepts](../../website/concepts/agent-logstream.md) —
+  event/artifact model and the full tool reference.
+- RFC 003 (`docs/rfcs/003-agent-logstream-coordination.md`) — design
+  rationale and storage model.

integrations/shared/recall-protocol.md

@@ -116,5 +116,10 @@ added through the MCP server and diary entries, which have no source file
 
 - [`integrations/openclaw/SKILL.md`](../openclaw/SKILL.md) — the original
   full-protocol skill this is distilled from.
+- [`coordination-protocol.md`](coordination-protocol.md) — the shared-brain
+  companion protocol: when agents delegate work to each other over the
+  hub, they use the logstream (`mempalace_event_append` /
+  `mempalace_event_wait`), not drawers. Recall answers questions;
+  the logstream moves work.
 - MemPalace design principles (verbatim, local-first, never summarize):
   <https://github.com/MemPalace/mempalace>

mempalace/backends/base.py

@@ -123,10 +123,13 @@ class PalaceRef:
         access is a spec violation.
 
     Backends that do not advertise ``supports_namespace_isolation`` (e.g.
-    ``sqlite_exact``, whose isolation is the on-disk path alone) MAY ignore
-    ``namespace`` entirely; callers MUST NOT rely on it for tenant isolation
-    on such backends. Any conforming backend can self-check both guarantees by
-    running the shared assertions in ``tests/_backend_conformance.py``.
+    path-rooted ``chroma`` / ``sqlite_exact``) MUST NOT silently accept and
+    ignore a populated ``namespace`` — they MUST raise
+    :class:`UnsupportedCapabilityError` (same spirit as
+    :class:`UnsupportedFilterError`). Callers targeting those backends MUST
+    leave ``namespace`` as ``None``. Isolation conformance lives in
+    ``tests/_backend_conformance.py`` (cross-id arm for every backend;
+    same-id / different-namespace arm for advertisers only).
     """
 
     id: str
@@ -604,8 +607,9 @@ class BaseBackend(ABC):
     ``PalaceRef.namespace`` (multi-tenant / hosted deployments) MUST advertise
     the ``supports_namespace_isolation`` capability token; doing so is a
     promise to satisfy the cross-namespace guarantee and to pass the namespace
-    arm of the conformance suite. Backends without the token MAY ignore
-    ``namespace``.
+    arm of the conformance suite. Backends without the token MUST raise
+    :class:`UnsupportedCapabilityError` when ``PalaceRef.namespace`` is
+    non-``None`` rather than silently accept-and-ignore (RFC 001 §4.4).
     """
 
     name: ClassVar[str]
@@ -626,6 +630,21 @@ class BaseBackend(ABC):
     #: for anything not listed here.
     maintenance_kinds: ClassVar[frozenset[str]] = frozenset()
 
+    def require_namespace_support(self, palace: PalaceRef) -> None:
+        """Raise if ``palace.namespace`` is set but this backend does not isolate by it.
+
+        Call at the start of ``get_collection`` (and any other entry that
+        accepts a :class:`PalaceRef`) so non-advertising backends never
+        silently drop a tenant namespace (RFC 001 §4.4).
+        """
+        if palace.namespace is None:
+            return
+        if "supports_namespace_isolation" not in self.capabilities:
+            raise UnsupportedCapabilityError(
+                f"{type(self).name} does not advertise supports_namespace_isolation; "
+                f"leave PalaceRef.namespace as None (got {palace.namespace!r})"
+            )
+
     @abstractmethod
     def get_collection(
         self,

mempalace/backends/chroma.py

@@ -160,34 +160,101 @@ def _hnsw_payload_appears_sane(seg_dir: str) -> bool:
     return ratio is None or ratio <= _HNSW_LINK_TO_DATA_MAX_RATIO
 
 
-# HNSW batch/sync thresholds applied at collection creation.
+# HNSW batch/sync thresholds applied at collection creation — chromadb's own
+# documented defaults (chromadb/api/configuration.py:263-273,
+# chromadb/api/collection_configuration.py:451-454,
+# chromadb/segment/impl/vector/hnsw_params.py:79-80, and
+# https://docs.trychroma.com/docs/collections/configure).
 #
-# chromadb's Rust HNSW segment writes index_metadata.pickle and
-# link_lists.bin only when internal counters cross both thresholds
-# (batch_size gates _apply_batch; sync_threshold gates _persist).
-# Records below both thresholds stay in memory and are lost on exit.
+# Both ran at 2 to answer #1579 (a sub-threshold mine left index_metadata.pickle
+# absent, and quarantine_stale_hnsw renamed the segment away). Three findings on
+# chromadb 1.5.9 (PersistentClient / Rust bindings, single writer) retire that:
 #
-# Previously 50k/50k to work around link_lists.bin sparse-file bloat
-# in pre-1.5.x Python chromadb (#344).  chromadb >=1.5.4 Rust bindings
-# (the minimum mempalace supports) do not exhibit that bloat; verified
-# at batch_size=2 with 20k records: link_lists.bin = 171 KB, no
-# sparse-file inflation.
+#   * 2 SITS OUTSIDE CHROMA'S OWN DECLARED VALID RANGE. hnsw_params.py:21-22
+#     validates both knobs as `isinstance(p, int) and p > 2`. Not an inequality
+#     between the two — a floor on each.
 #
-# The 50k guard caused #1579: mines under 50k drawers never triggered
-# _persist(), leaving index_metadata.pickle absent and link_lists.bin
-# empty.  quarantine_stale_hnsw then renamed the segment on every cold
-# open after a 300s mtime gap, accumulating .drift-* directories.
+#   * IT COSTS WRITE AMPLIFICATION, measured. Bytes written (/proc/self/io) for
+#     one mine of N records, 64-dim, num_threads=1, identical corpus and seed:
 #
-# Lowered to 2 (empirical Rust-side minimum for chromadb >=1.5.4; the
-# Rust bindings reject 1 with InvalidArgumentError) so any mine of 2+
-# drawers triggers a natural persist.  Existing palaces created under
-# the old 50k guard keep those thresholds in their collection metadata
-# until the user runs repair --mode from-sqlite --archive-existing.
-_HNSW_BLOAT_GUARD = {
-    "hnsw:batch_size": 2,
-    "hnsw:sync_threshold": 2,
+#         N        2/2        100/1000     ratio
+#         10,000    47.1 MB    28.1 MB     1.67x
+#         20,000   127.4 MB    60.0 MB     2.12x
+#         40,000   390.3 MB   134.0 MB     2.91x
+#
+#     Two runs at N=20,000 reproduced within 0.1%. sync_threshold dominates:
+#     3/3 measured identical to 2/2, and 1000/1000 identical to 100/1000. The
+#     ratio grows with collection size, so the cost is worst on the largest
+#     palaces.
+#
+#   * IT BUYS NOTHING. A 5-record collection at 100/1000 — far below the
+#     threshold, so no persist ever fires — reads back whole from a FRESH
+#     PROCESS (count and vector query both answer). On that artifact
+#     link_lists.bin is 0 bytes with no index_metadata.pickle, which is #1579's
+#     trigger shape exactly, and quarantine_stale_hnsw() creates no drift dir:
+#     _segment_appears_healthy reads it as never-persisted rather than as a torn
+#     persist.
+#
+# NOT claimed here: that a small sync_threshold is a known chroma failure mode.
+# No such report was found in chroma's issues or docs, and chroma's own guidance
+# runs the other way (raise sync_threshold for bulk inserts). The Python
+# persist path that #1579 and chroma#6975 describe does not execute under the
+# Rust bindings at all — hnswlib is not a dependency of this line.
+#
+# A palace keeps whatever thresholds it was created under;
+# `repair --mode from-sqlite --archive-existing` re-creates it under these.
+_HNSW_WRITE_DEFAULTS = {
+    "hnsw:batch_size": 100,
+    "hnsw:sync_threshold": 1000,
 }
 
+
+def _hnsw_creation_metadata(options: Optional[dict]) -> dict:
+    """Build the ``metadata=`` dict for a fresh collection from caller options.
+
+    Centralizes the HNSW knobs so a multi-collection palace can tune each
+    collection at creation while the base keeps every config value in the
+    ``collection_metadata`` table where the divergence guard, the cosine-space
+    detector (``ChromaCollection.distance_metric``), and ``_read_sync_threshold``
+    already read it. The legacy ``metadata=`` keys are kept deliberately: the
+    modern ``configuration=`` API stores the same parameters in
+    ``configuration_json`` instead, leaving ``collection.metadata`` empty and
+    silently blinding all of that existing tooling.
+
+    Caller option -> chromadb metadata key:
+
+    * ``hnsw_space``       -> ``hnsw:space``        (default ``"cosine"``)
+    * ``num_threads``      -> ``hnsw:num_threads``  (default 1; serializes inserts)
+    * ``ef_construction``  -> ``hnsw:construction_ef``
+    * ``max_neighbors``    -> ``hnsw:M``
+    * ``sync_threshold``   -> ``hnsw:sync_threshold``
+    * ``batch_size``       -> ``hnsw:batch_size``
+
+    ``sync_threshold``/``batch_size`` default to chromadb's own values
+    (:data:`_HNSW_WRITE_DEFAULTS`), which amortize the index flush across a
+    mine. A caller tunes them per collection; a caller writing far fewer
+    records than the threshold still keeps them (the Rust writer holds the
+    sub-threshold tail durable — see :data:`_HNSW_WRITE_DEFAULTS`).
+    ``ef_construction``/``max_neighbors`` are omitted when the caller does not
+    set them, so chromadb applies its own defaults.
+    """
+    opts = options if isinstance(options, dict) else {}
+    md: dict[str, Any] = {
+        "hnsw:space": opts.get("hnsw_space", "cosine"),
+        "hnsw:num_threads": int(opts.get("num_threads", 1)),
+        **_HNSW_WRITE_DEFAULTS,
+    }
+    if "ef_construction" in opts and opts["ef_construction"] is not None:
+        md["hnsw:construction_ef"] = int(opts["ef_construction"])
+    if "max_neighbors" in opts and opts["max_neighbors"] is not None:
+        md["hnsw:M"] = int(opts["max_neighbors"])
+    if "sync_threshold" in opts and opts["sync_threshold"] is not None:
+        md["hnsw:sync_threshold"] = int(opts["sync_threshold"])
+    if "batch_size" in opts and opts["batch_size"] is not None:
+        md["hnsw:batch_size"] = int(opts["batch_size"])
+    return md
+
+
 # Below this size, data_level0.bin is too small for a meaningful HNSW graph.
 # Used by _hnsw_link_lists_is_usable_for_payload (empty link_lists is fine
 # when data is trivially small) and _missing_dimensionality_appears_recoverable
@@ -633,9 +700,10 @@ def _hnsw_element_count(palace_path: str, segment_id: str) -> Optional[int]:
 # read the collection metadata (older palaces missing the row, sqlite
 # unreadable). 2000 = 2 × chromadb's default sync_threshold of 1000.
 #
-# Why dynamic: legacy palaces may still carry ``sync_threshold = 50_000``
-# (the pre-#1579 guard), so flush-lag can grow up to 50K on those palaces.
-# New palaces use sync_threshold=2 (#1579) and flush almost immediately.
+# Why dynamic: a palace carries whatever ``sync_threshold`` it was created
+# under, and flush-lag grows to that threshold before a persist fires — up
+# to 50K on a palace created under an old large guard, and 2 on one created
+# under the small guard that #1308 traces back to.
 # A fixed 2000 floor would flag actively-written legacy palaces as
 # DIVERGED the moment their queue exceeded 10% of sqlite_count, even
 # though chromadb is behaving correctly. The floor must scale with the
@@ -727,6 +795,112 @@ def _hnsw_metadata_age_seconds(palace_path: str, segment_id: str) -> Optional[fl
         return None
 
 
+# Probe verdicts keyed by ``(palace_path, collection_name)``; each value is
+# ``(fingerprint, segment_id, status, probed_at)``. Written as one tuple
+# assignment, so a concurrent reader either sees the previous entry or the new
+# one, never a half-updated pair. Two threads that miss together simply both
+# run the probe and the last writer wins — a benign race, and cheaper than
+# serializing every reader behind a lock.
+_capacity_cache: dict[tuple[str, str], tuple[tuple, Optional[str], dict, float]] = {}
+
+# Bumped by every :func:`reset_hnsw_capacity_cache`. A probe reads it before
+# running and refuses to store its verdict if it changed meanwhile, so a probe
+# already in flight when a reset lands cannot repopulate the entry the reset
+# was meant to discard (the ``tool_reconnect`` race).
+_capacity_cache_generation = 0
+
+# A long-lived server watches one palace and a handful of collections, so the
+# map stays tiny; the bound only exists so a process that walks many palaces
+# (benchmarks, batch tooling) cannot grow it without limit.
+_CAPACITY_CACHE_MAX_ENTRIES = 32
+
+# Ceiling on how long one verdict may be reused, as a backstop for filesystems
+# whose timestamps are too coarse to notice a quick rewrite: FAT32 stores mtime
+# at 2 s granularity, exFAT at 10 ms, and a write that lands inside existing
+# sqlite pages need not change the file size either. On ext4/APFS/NTFS the
+# signature already catches every write, so this ceiling never fires in
+# practice. It bounds the worst case; it is not the freshness mechanism.
+_CAPACITY_CACHE_MAX_AGE_SECONDS = 10.0
+
+
+def _stat_signature(path: str) -> tuple[int, int, int]:
+    """Return ``(inode, mtime_ns, size)`` for ``path``, all zeros when absent.
+
+    Catches every exception, not just ``OSError``: a palace path carrying an
+    embedded null byte makes ``os.stat`` raise ``ValueError``, and
+    :func:`hnsw_capacity_status` promises never to raise. An unreadable path
+    simply yields the "absent" signature and the probe reports ``unknown``.
+    """
+    try:
+        st = os.stat(path)
+    except Exception:
+        return (0, 0, 0)
+    return (st.st_ino, st.st_mtime_ns, st.st_size)
+
+
+def _db_family_signature(palace_path: str) -> tuple:
+    """Signature of the sqlite files whose contents the probe depends on.
+
+    chromadb 1.5.x leaves ``chroma.sqlite3`` in ``journal_mode=delete``, so the
+    ``-wal`` sidecar usually does not exist and stat'ing it is one cheap miss.
+    It is covered anyway because the journal mode belongs to the database
+    rather than to this code: under WAL a writer appends rows the probe would
+    count while the main file's own mtime stays put, and the verdict would
+    otherwise be reused against data it never saw.
+
+    ``-shm`` is deliberately excluded. It is the WAL index in shared memory,
+    and sqlite restamps it every time a connection opens the database — even
+    read-only. Including it would make the probe invalidate its own cache on
+    every call, so on a WAL-mode palace the cache would never hit.
+    """
+    db_path = os.path.join(palace_path, "chroma.sqlite3")
+    return (
+        _stat_signature(db_path),
+        _stat_signature(db_path + "-wal"),
+    )
+
+
+def _pickle_signature(palace_path: str, segment_id: Optional[str]) -> tuple[int, int, int]:
+    """Signature of the segment's ``index_metadata.pickle``."""
+    if not segment_id:
+        return (0, 0, 0)
+    return _stat_signature(os.path.join(palace_path, segment_id, "index_metadata.pickle"))
+
+
+def _segment_id_safe(palace_path: str, collection_name: str) -> Optional[str]:
+    """``_vector_segment_id`` that never raises, for the pre-probe signature."""
+    try:
+        return _vector_segment_id(palace_path, collection_name)
+    except Exception:
+        return None
+
+
+def _capacity_fingerprint(palace_path: str, segment_id: Optional[str]) -> tuple:
+    """Signature over every file the probe reads: the sqlite family + the pickle.
+
+    Both halves must be captured for the same ``segment_id`` so a rewrite of
+    ``index_metadata.pickle`` is caught. The probe reads that pickle partway
+    through, then makes two more sqlite calls, so a signature taken only after
+    the probe returned would record a mid-probe pickle rewrite as "unchanged"
+    while the verdict still reflected the pre-write file (#1471 review).
+    """
+    return (_db_family_signature(palace_path), _pickle_signature(palace_path, segment_id))
+
+
+def reset_hnsw_capacity_cache() -> None:
+    """Forget every cached capacity verdict.
+
+    The signature check already picks up on-disk changes on its own; this is
+    for callers that drop all cached palace state at once (``tool_reconnect``,
+    ``_force_chroma_cache_reset``) and for tests that want a probe to run
+    unconditionally. Bumps the generation so a probe already running cannot
+    re-store the entry this call just dropped.
+    """
+    global _capacity_cache_generation
+    _capacity_cache_generation += 1
+    _capacity_cache.clear()
+
+
 def hnsw_capacity_status(palace_path: str, collection_name: str = "mempalace_drawers") -> dict:
     """Compare sqlite embedding count against HNSW element count.
 
@@ -747,7 +921,70 @@ def hnsw_capacity_status(palace_path: str, collection_name: str = "mempalace_dra
     * ``message``          — human-readable summary
 
     Never raises — a probe that throws would defeat the point.
+
+    A fully-measured verdict is cached per ``(palace_path, collection_name)``
+    and reused while every file the probe reads is unchanged on disk (#1471).
+    Each call otherwise costs a ``COUNT(*)`` over the embeddings table and a
+    full unpickle of the segment metadata — the two dominant costs — plus a few
+    small sqlite reads, on a path every search, duplicate check and status call
+    runs through. A verdict the probe could not fully measure (``sqlite_count``
+    is ``None`` from a locked database, or there is no palace yet) is returned
+    but never cached, so a transient failure cannot pin a false reading.
+
+    Freshness comes from an ``(inode, mtime_ns, size)`` signature rather than a
+    wall-clock TTL, so an external writer — ``mempalace repair``, a peer mine,
+    another process — invalidates the verdict as soon as it touches the files,
+    instead of leaving the #1222 guard blind for a fixed window.
+    ``_CAPACITY_CACHE_MAX_AGE_SECONDS`` caps how long one verdict may be
+    reused, but only as a backstop for filesystems with coarse timestamps; the
+    signature is what makes the verdict fresh.
+
+    Unlike :meth:`ChromaBackend._client`, which tolerates a 0.01 s mtime
+    epsilon to avoid rebuilding an expensive client, this compares exactly:
+    re-running the probe costs milliseconds, whereas serving one stale verdict
+    can route a query into a diverged segment.
     """
+    key = (palace_path, collection_name)
+    cached = _capacity_cache.get(key)
+    if cached is not None:
+        fingerprint, cached_segment, status, probed_at = cached
+        if time.monotonic() - probed_at <= _CAPACITY_CACHE_MAX_AGE_SECONDS:
+            if _capacity_fingerprint(palace_path, cached_segment) == fingerprint:
+                return dict(status)
+
+    generation = _capacity_cache_generation
+    # Snapshot the files the probe is about to read, before it reads them, and
+    # again after — using the segment id the probe itself resolved. Caching
+    # only when both snapshots agree makes an external write during the probe
+    # (sqlite OR the pickle) fall through uncached rather than pin a verdict
+    # the disk no longer supports.
+    before = _capacity_fingerprint(palace_path, _segment_id_safe(palace_path, collection_name))
+    out = _hnsw_capacity_status_uncached(palace_path, collection_name)
+    segment_id = out.get("segment_id")
+    after = _capacity_fingerprint(palace_path, segment_id)
+    cacheable = (
+        before == after
+        # A None sqlite_count means the probe could not read the database
+        # (transient lock/error), not a real "unknown" — pinning it would go
+        # blind for the whole ceiling. A None segment id has no pickle path to
+        # watch, so its fingerprint can never notice a first flush.
+        and out.get("sqlite_count") is not None
+        and segment_id is not None
+        # A reset that landed while this probe ran already dropped the entry
+        # it was told to; do not resurrect it.
+        and generation == _capacity_cache_generation
+    )
+    if cacheable:
+        if len(_capacity_cache) >= _CAPACITY_CACHE_MAX_ENTRIES:
+            _capacity_cache.clear()
+        _capacity_cache[key] = (after, segment_id, dict(out), time.monotonic())
+    return out
+
+
+def _hnsw_capacity_status_uncached(
+    palace_path: str, collection_name: str = "mempalace_drawers"
+) -> dict:
+    """Run the capacity probe, bypassing the cache. See :func:`hnsw_capacity_status`."""
     out: dict[str, Any] = {
         "segment_id": None,
         "sqlite_count": None,
@@ -1916,6 +2153,7 @@ class ChromaBackend(BaseBackend):
     name = "chroma"
     capabilities = frozenset(
         {
+            "requires_explicit_embeddings",
             "supports_embeddings_in",
             "supports_embeddings_passthrough",
             "supports_embeddings_out",
@@ -2157,6 +2395,7 @@ class ChromaBackend(BaseBackend):
           — still used by callers not yet migrated.
         """
         palace_ref, collection_name, create, options = _normalize_get_collection_args(args, kwargs)
+        self.require_namespace_support(palace_ref)
 
         palace_path = palace_ref.local_path
         if palace_path is None:
@@ -2173,9 +2412,6 @@ class ChromaBackend(BaseBackend):
                 pass
 
         client = self._client(palace_path)
-        hnsw_space = "cosine"
-        if options and isinstance(options, dict):
-            hnsw_space = options.get("hnsw_space", hnsw_space)
 
         ef = self._resolve_embedding_function()
         ef_kwargs = {"embedding_function": ef} if ef is not None else {}
@@ -2186,11 +2422,7 @@ class ChromaBackend(BaseBackend):
             except _ChromaNotFoundError:
                 collection = client.create_collection(
                     collection_name,
-                    metadata={
-                        "hnsw:space": hnsw_space,
-                        "hnsw:num_threads": 1,
-                        **_HNSW_BLOAT_GUARD,
-                    },
+                    metadata=_hnsw_creation_metadata(options),
                     **ef_kwargs,
                 )
             except ValueError as e:
@@ -2280,11 +2512,7 @@ class ChromaBackend(BaseBackend):
         ef_kwargs = {"embedding_function": ef} if ef is not None else {}
         collection = self._client(palace_path).create_collection(
             collection_name,
-            metadata={
-                "hnsw:space": hnsw_space,
-                "hnsw:num_threads": 1,
-                **_HNSW_BLOAT_GUARD,
-            },
+            metadata=_hnsw_creation_metadata({"hnsw_space": hnsw_space}),
             **ef_kwargs,
         )
         return ChromaCollection(collection, palace_path=palace_path)

mempalace/backends/embedding_wrapper.py

@@ -8,14 +8,26 @@ from .base import BaseCollection
 
 
 def _embed_texts(texts: list[str]) -> list[list[float]]:
-    """Embed ``texts`` with the configured local embedding function."""
+    """Embed ``texts`` with the configured local embedding function.
+
+    Embedding functions return ``list[np.ndarray]`` (float32). ``list(arr)``
+    would unpack that into ``np.float32`` *scalars*, which ChromaDB's
+    ``normalize_embeddings`` rejects outright ("Expected embeddings to be a
+    list of floats or ints..."), so every write through this wrapper must
+    convert to real Python floats. ``.tolist()`` does that in C; the
+    ``float(x)`` branch covers embedders that already hand back plain
+    sequences.
+    """
     if not texts:
         return []
     from ..embedding import get_embedding_function
 
     ef = get_embedding_function()
     vectors = ef(input=texts)
-    return [list(v) for v in vectors]
+    return [
+        v.tolist() if hasattr(v, "tolist") else [float(x) for x in v]  # numpy | plain sequence
+        for v in vectors
+    ]
 
 
 def _as_list(value):

mempalace/backends/milvus.py

@@ -76,8 +76,11 @@ def _utcnow() -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
-def _is_server_uri(uri: str) -> bool:
-    normalized = uri.lower()
+def milvus_uri_is_server(uri: Optional[str]) -> bool:
+    """Return whether ``uri`` targets service-managed Milvus storage."""
+    if not uri:
+        return False
+    normalized = uri.strip().lower()
     return normalized.startswith(("http://", "https://", "tcp://", "grpc://"))
 
 
@@ -358,7 +361,7 @@ class MilvusCollection(BaseCollection):
         if score is None:
             return 1.0
         value = float(score)
-        if self._config.uri and _is_server_uri(self._config.uri):
+        if milvus_uri_is_server(self._config.uri):
             return 1.0 - max(-1.0, min(1.0, value))
         return value
 
@@ -976,7 +979,7 @@ class MilvusBackend(BaseBackend):
         collection_name: str,
         config: _MilvusConfig,
     ) -> str:
-        if config.uri and not _is_server_uri(config.uri) and not config.namespace:
+        if config.uri and not milvus_uri_is_server(config.uri) and not config.namespace:
             return _slug(collection_name)
         prefix = self._remote_collection_prefix(palace=palace, config=config)
         return f"{prefix}_{_slug(collection_name)}"
@@ -1059,7 +1062,7 @@ class MilvusBackend(BaseBackend):
             client = self._clients.get(config)
             if client is not None:
                 return client
-            if config.uri and not _is_server_uri(config.uri):
+            if config.uri and not milvus_uri_is_server(config.uri):
                 parent = os.path.dirname(os.path.abspath(config.uri)) or "."
                 os.makedirs(parent, exist_ok=True)
                 try:
@@ -1089,7 +1092,7 @@ class MilvusBackend(BaseBackend):
                     pass
             marker_path = self._marker_path(palace.local_path)
             local_db_exists = bool(
-                config.uri and not _is_server_uri(config.uri) and os.path.exists(config.uri)
+                config.uri and not milvus_uri_is_server(config.uri) and os.path.exists(config.uri)
             )
             if os.path.isfile(marker_path):
                 self._validate_marker_target(palace, config)

mempalace/backends/pgvector.py

@@ -54,6 +54,7 @@ from .base import (
     PalaceNotFoundError,
     PalaceRef,
     QueryResult,
+    UnsupportedCapabilityError,
     UnsupportedFilterError,
     _IncludeSpec,
 )
@@ -727,6 +728,29 @@ class _PgVectorClient:
         rows = self._execute(f"SELECT count(*) FROM {_quote_identifier(table)}", fetch=True)
         return int(rows[0][0]) if rows and rows[0] else 0
 
+    def facet_counts(
+        self,
+        table: str,
+        *,
+        field: str,
+        where: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        qi = _quote_identifier(table)
+        params: list = [field]
+        where_sql = _where_to_sql(where, params) if where else "TRUE"
+        sql = (
+            f"SELECT metadata->>%s AS k, count(*) AS c "
+            f"FROM {qi} WHERE {where_sql} "
+            f"GROUP BY 1 ORDER BY 2 DESC, 1 LIMIT %s"
+        )
+        params.append(int(limit))
+        rows = self._execute(sql, params, fetch=True)
+        # Exclude NULL bucket: matches Qdrant's facet semantics; mcp_server
+        # reconciles missing values into "unknown" via the count diff
+        # (count(*) - sum(facet_values)).
+        return {str(row[0]): int(row[1]) for row in rows if row[0] is not None}
+
     def drop_table(self, table: str) -> None:
         self._execute(f"DROP TABLE IF EXISTS {_quote_identifier(table)}")
 
@@ -1194,6 +1218,26 @@ class PgVectorCollection(BaseCollection):
             return 0
         return self._client.count_rows(self._table)
 
+    def facet_counts(
+        self,
+        field: str,
+        where: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        self._ensure_open()
+        # Validate the filter before the existence short-circuit so an
+        # unsupported local-only filter raises even on an unmaterialized
+        # collection — matches the order used by get()/lexical_search() and
+        # qdrant.facet_counts (PR #1868 review).
+        _validate_where(where)
+        if _requires_local_filter(where):
+            raise UnsupportedCapabilityError("facet_counts does not support local-only filters")
+        if not self._table_exists():
+            if self._marker_exists():
+                raise CollectionNotInitializedError(self._collection_name)
+            return {}
+        return self._client.facet_counts(self._table, field=field, where=where, limit=limit)
+
     def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None):
         _validate_where(where)
         pushdown = None if _requires_local_filter(where) else where
@@ -1291,6 +1335,7 @@ class PgVectorBackend(BaseBackend):
             "supports_embeddings_out",
             "supports_metadata_filters",
             "supports_lexical_search",
+            "supports_metadata_facets",
             "supports_namespace_isolation",
             "supports_server_side_indexes",
             "server_mode",

mempalace/backends/sqlite_exact.py

@@ -15,12 +15,14 @@ import re
 import sqlite3
 import threading
 from datetime import datetime, timezone
+from pathlib import Path
 from typing import Any, Optional
 
 import numpy as np
 
 from .base import (
     BackendClosedError,
+    BackendError,
     BaseBackend,
     BaseCollection,
     CollectionNotInitializedError,
@@ -247,9 +249,23 @@ def _validate_write_batch(
 
 
 class _SQLiteExactHandle:
-    def __init__(self, conn: sqlite3.Connection, lock: threading.RLock):
+    def __init__(
+        self,
+        conn: sqlite3.Connection,
+        lock: threading.RLock,
+        palace_path: str,
+        *,
+        read_only: bool = False,
+        immutable: bool = False,
+    ):
         self.conn = conn
         self.lock = lock
+        self.palace_path = palace_path
+        self.read_only = read_only
+        # True when opened with ``immutable=1`` because no WAL existed at connect
+        # time. A later writer can create WAL sidecars that this connection will
+        # never see, so the backend must reopen once those files appear.
+        self.immutable = immutable
         self.closed = False
 
 
@@ -264,8 +280,27 @@ class SQLiteExactCollection(BaseCollection):
             raise BackendClosedError("SQLiteExactCollection has been closed")
 
     @contextlib.contextmanager
-    def _cursor(self):
+    def _write_lock(self):
+        """Serialize this handle before taking process-wide writer ownership.
+
+        ``mine_palace_lock`` grants cross-thread re-entrant access whenever
+        this process already owns the palace. Taking it before ``handle.lock``
+        lets a waiting thread consume that re-entrant credit, outlive the
+        thread that owns the OS lease, and then mutate after the lease has been
+        released. The handle mutex must therefore be the outer context.
+        """
+        # Late import avoids a palace.py -> backend -> palace.py cycle.
+        from ..palace import mine_palace_lock
+
         with self._handle.lock:
+            self._ensure_open()
+            with mine_palace_lock(self._handle.palace_path):
+                yield
+
+    @contextlib.contextmanager
+    def _cursor(self, *, write: bool = False):
+        serialization = self._write_lock() if write else self._handle.lock
+        with serialization:
             self._ensure_open()
             cur = self._handle.conn.cursor()
             try:
@@ -345,7 +380,7 @@ class SQLiteExactCollection(BaseCollection):
     def set_embedder_identity(self, identity) -> None:
         if not identity or not identity.model_name:
             return
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             cur.execute(
                 "INSERT INTO meta(key, value) VALUES (?, ?) "
                 "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
@@ -375,7 +410,7 @@ class SQLiteExactCollection(BaseCollection):
             raise ValueError("sqlite_exact requires explicit embeddings")
         metadatas = metadatas or [{} for _ in ids]
         now = _utcnow()
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             collection_id = self._collection_id(cur)
             prepared = []
             for doc_id, doc, meta, emb in zip(ids, documents, metadatas, embeddings):
@@ -413,7 +448,7 @@ class SQLiteExactCollection(BaseCollection):
             raise ValueError("sqlite_exact requires explicit embeddings")
         metadatas = metadatas or [{} for _ in ids]
         now = _utcnow()
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             collection_id = self._collection_id(cur)
             prepared = []
             for doc_id, doc, meta, emb in zip(ids, documents, metadatas, embeddings):
@@ -457,7 +492,7 @@ class SQLiteExactCollection(BaseCollection):
         ):
             if value is not None and len(value) != n:
                 raise ValueError(f"{label} length {len(value)} does not match ids length {n}")
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             collection_id = self._collection_id(cur)
             updates = []
             for idx, doc_id in enumerate(ids):
@@ -653,7 +688,7 @@ class SQLiteExactCollection(BaseCollection):
         )
 
     def delete(self, *, ids=None, where=None):
-        with self._cursor() as cur:
+        with self._cursor(write=True) as cur:
             collection_id = self._collection_id(cur)
             if ids is None:
                 rows = self._rows(cur, where=where)
@@ -797,17 +832,16 @@ class SQLiteExactCollection(BaseCollection):
             )
         if kind == "analyze":
             # Refresh planner stats. Concurrent runs serialize on the handle lock.
-            with self._cursor() as cur:
+            with self._cursor(write=True) as cur:
                 cur.execute("ANALYZE")
             return MaintenanceResult(kind="analyze", status="ran")
 
         # compact → VACUUM. It cannot run inside a transaction, so flip the
-        # connection to autocommit for the duration. The handle lock serializes
-        # concurrent runs in-process; SQLite's own write lock serializes across
-        # processes.
+        # connection to autocommit for the duration. _write_lock takes the
+        # handle mutex before the palace lease so a waiting thread cannot
+        # retain stale process-reentrant ownership after another thread exits.
         before = self.maintenance_state()
-        with self._handle.lock:
-            self._ensure_open()
+        with self._write_lock():
             conn = self._handle.conn
             prev_isolation = conn.isolation_level
             try:
@@ -848,6 +882,7 @@ class SQLiteExactBackend(BaseBackend):
 
     def __init__(self):
         self._clients: dict[str, _SQLiteExactHandle] = {}
+        self._read_only_clients: dict[str, _SQLiteExactHandle] = {}
         self._clients_lock = threading.RLock()
         self._closed = False
 
@@ -855,9 +890,64 @@ class SQLiteExactBackend(BaseBackend):
     def _db_path(palace_path: str) -> str:
         return os.path.join(palace_path, _DB_FILENAME)
 
-    def _connect(self, palace_path: str, create: bool):
+    @staticmethod
+    def _wal_sidecar_state(db_path: str) -> tuple[bool, bool]:
+        return (
+            os.path.isfile(f"{db_path}-wal"),
+            os.path.isfile(f"{db_path}-shm"),
+        )
+
+    @staticmethod
+    def _connect_read_only(db_path: str) -> tuple[sqlite3.Connection, bool]:
+        """Open without creating WAL files while preserving an active WAL.
+
+        Returns ``(connection, immutable)``. ``immutable`` is True when the
+        database was clean (no WAL) and was opened with ``immutable=1``.
+        """
+        wal_exists, shm_exists = SQLiteExactBackend._wal_sidecar_state(db_path)
+        if wal_exists != shm_exists:
+            raise BackendError(
+                "sqlite_exact read-only open found an incomplete WAL sidecar set; "
+                "open the palace after its writer exits cleanly or restore both "
+                "the -wal and -shm files"
+            )
+
+        db_uri = Path(db_path).resolve().as_uri()
+        if wal_exists:
+            # An active writer's uncheckpointed rows live in the WAL. With both
+            # sidecars already present, mode=ro can read them without creating
+            # filesystem state, including on a read-only mount.
+            db_uri = f"{db_uri}?mode=ro"
+            immutable = False
+        else:
+            # A clean WAL-mode database would otherwise make SQLite create new
+            # -wal/-shm files while connecting. Immutable mode is safe here
+            # only until a writer creates sidecars this connection would miss.
+            db_uri = f"{db_uri}?mode=ro&immutable=1"
+            immutable = True
+        return sqlite3.connect(db_uri, uri=True, check_same_thread=False), immutable
+
+    def _retire_read_only_handle(self, palace_path: str, handle: _SQLiteExactHandle) -> None:
+        """Drop a cached read-only handle so the next open re-evaluates WAL state."""
+        self._read_only_clients.pop(palace_path, None)
+        with handle.lock:
+            if handle.closed:
+                return
+            handle.closed = True
+            try:
+                handle.conn.close()
+            except Exception:
+                logger.debug(
+                    "Failed to close stale immutable sqlite_exact reader for %s",
+                    palace_path,
+                    exc_info=True,
+                )
+
+    def _connect(self, palace_path: str, create: bool, *, read_only: bool = False):
         if self._closed:
             raise BackendClosedError("SQLiteExactBackend has been closed")
+        if create and read_only:
+            raise ValueError("sqlite_exact read-only connections cannot create a palace")
         db_path = self._db_path(palace_path)
         if not create and not os.path.isfile(db_path):
             raise PalaceNotFoundError(db_path)
@@ -877,20 +967,52 @@ class SQLiteExactBackend(BaseBackend):
         with self._clients_lock:
             if self._closed:
                 raise BackendClosedError("SQLiteExactBackend has been closed")
-            cached = self._clients.get(palace_path)
+            clients = self._read_only_clients if read_only else self._clients
+            cached = clients.get(palace_path)
             if cached is not None and not cached.closed:
-                return cached
-            conn = sqlite3.connect(db_path, check_same_thread=False)
+                if read_only and cached.immutable:
+                    # An immutable snapshot freezes the clean-database view.
+                    # Reopen only when the complete WAL sidecar pair is present
+                    # (active writer). A partial pair is a transient mid-open
+                    # state — keep the immutable handle rather than forcing a
+                    # reconnect that would raise on the incomplete set.
+                    wal_exists, shm_exists = self._wal_sidecar_state(db_path)
+                    if wal_exists and shm_exists:
+                        self._retire_read_only_handle(palace_path, cached)
+                        cached = None
+                if cached is not None:
+                    return cached
+            if read_only:
+                conn, immutable = self._connect_read_only(db_path)
+            else:
+                conn = sqlite3.connect(db_path, check_same_thread=False)
+                immutable = False
             try:
                 conn.row_factory = sqlite3.Row
                 lock = threading.RLock()
-                handle = _SQLiteExactHandle(conn, lock)
+                handle = _SQLiteExactHandle(
+                    conn,
+                    lock,
+                    palace_path,
+                    read_only=read_only,
+                    immutable=immutable,
+                )
                 with handle.lock:
-                    self._init_schema(conn)
+                    if read_only:
+                        # ``mode=ro`` prevents filesystem writes. ``query_only``
+                        # adds a second connection-local guard so a future
+                        # refactor cannot accidentally introduce a temp/schema
+                        # write through a read-only MCP collection.
+                        conn.execute("PRAGMA query_only=ON")
+                    else:
+                        from ..palace import mine_palace_lock
+
+                        with mine_palace_lock(palace_path):
+                            self._init_schema(conn)
             except BaseException:
                 conn.close()
                 raise
-            self._clients[palace_path] = handle
+            clients[palace_path] = handle
             return handle
 
     def _init_schema(self, conn: sqlite3.Connection) -> None:
@@ -955,13 +1077,14 @@ class SQLiteExactBackend(BaseBackend):
         *args,
         **kwargs,
     ) -> SQLiteExactCollection:
-        palace, collection_name, create = self._normalize_args(args, kwargs)
+        palace, collection_name, create, read_only = self._normalize_args(args, kwargs)
+        self.require_namespace_support(palace)
         palace_path = palace.local_path
         if palace_path is None:
             raise PalaceNotFoundError("SQLiteExactBackend requires PalaceRef.local_path")
         if not create and not os.path.isdir(palace_path):
             raise PalaceNotFoundError(palace_path)
-        handle = self._connect(palace_path, create=create)
+        handle = self._connect(palace_path, create=create, read_only=read_only)
         with handle.lock:
             row = handle.conn.execute(
                 "SELECT id FROM collections WHERE name = ?",
@@ -970,11 +1093,14 @@ class SQLiteExactBackend(BaseBackend):
             if row is None:
                 if not create:
                     raise CollectionNotInitializedError(collection_name)
-                handle.conn.execute(
-                    "INSERT INTO collections(name, created_at) VALUES (?, ?)",
-                    (collection_name, _utcnow()),
-                )
-                handle.conn.commit()
+                from ..palace import mine_palace_lock
+
+                with mine_palace_lock(palace_path):
+                    handle.conn.execute(
+                        "INSERT INTO collections(name, created_at) VALUES (?, ?)",
+                        (collection_name, _utcnow()),
+                    )
+                    handle.conn.commit()
         return SQLiteExactCollection(handle, collection_name)
 
     @staticmethod
@@ -985,10 +1111,11 @@ class SQLiteExactBackend(BaseBackend):
                 raise TypeError("palace= must be a PalaceRef instance")
             collection_name = kwargs.pop("collection_name")
             create = bool(kwargs.pop("create", False))
-            kwargs.pop("options", None)
+            options = kwargs.pop("options", None) or {}
+            read_only = bool(options.get("read_only", False))
             if args or kwargs:
                 raise TypeError("unexpected arguments to get_collection")
-            return palace, collection_name, create
+            return palace, collection_name, create, read_only
         if args:
             palace_path = args[0]
             rest = list(args[1:])
@@ -996,18 +1123,32 @@ class SQLiteExactBackend(BaseBackend):
             if collection_name is None:
                 raise TypeError("collection_name is required")
             create = kwargs.pop("create", False)
+            options = kwargs.pop("options", None) or {}
+            read_only = bool(options.get("read_only", False))
             if rest:
                 create = rest.pop(0)
             if rest or kwargs:
                 raise TypeError("unexpected arguments to get_collection")
-            return PalaceRef(id=palace_path, local_path=palace_path), collection_name, bool(create)
+            return (
+                PalaceRef(id=palace_path, local_path=palace_path),
+                collection_name,
+                bool(create),
+                read_only,
+            )
         if "palace_path" in kwargs:
             palace_path = kwargs.pop("palace_path")
             collection_name = kwargs.pop("collection_name")
             create = bool(kwargs.pop("create", False))
+            options = kwargs.pop("options", None) or {}
+            read_only = bool(options.get("read_only", False))
             if kwargs:
                 raise TypeError("unexpected arguments to get_collection")
-            return PalaceRef(id=palace_path, local_path=palace_path), collection_name, create
+            return (
+                PalaceRef(id=palace_path, local_path=palace_path),
+                collection_name,
+                create,
+                read_only,
+            )
         raise TypeError("get_collection requires palace= or a positional palace_path")
 
     def close_palace(self, palace: PalaceRef | str) -> None:
@@ -1015,11 +1156,15 @@ class SQLiteExactBackend(BaseBackend):
         if path is None:
             return
         with self._clients_lock:
-            cached = self._clients.pop(path, None)
-        if cached is not None:
-            with cached.lock:
-                cached.closed = True
-                cached.conn.close()
+            cached_handles = [
+                self._clients.pop(path, None),
+                self._read_only_clients.pop(path, None),
+            ]
+        for cached in cached_handles:
+            if cached is not None:
+                with cached.lock:
+                    cached.closed = True
+                    cached.conn.close()
 
     def close(self) -> None:
         # Flip _closed under the registry lock so a concurrent _connect either
@@ -1028,8 +1173,9 @@ class SQLiteExactBackend(BaseBackend):
         # Unlocked readers of _closed elsewhere are advisory fast-fails; the
         # locked recheck in _connect is the authoritative gate.
         with self._clients_lock:
-            handles = list(self._clients.values())
+            handles = list(self._clients.values()) + list(self._read_only_clients.values())
             self._clients.clear()
+            self._read_only_clients.clear()
             self._closed = True
         for handle in handles:
             with handle.lock:

mempalace/backups.py

@@ -1,4 +1,4 @@
-"""Retention pruning for timestamped palace backups.
+"""Writing and pruning palace backups.
 
 ``mempalace migrate`` and ``mempalace repair max-seq-id`` each write a fresh,
 timestamped backup every time they run and historically never deleted the old
@@ -9,11 +9,222 @@ filling the disk. This module prunes the backup set down to a bounded count
 after each new backup is written.
 
 The retention count comes from ``MempalaceConfig.max_backups`` (default 10).
+
+``copy_palace_dir`` is the other half: the whole-directory copy that
+``mempalace repair`` in its default mode and ``mempalace migrate`` take before
+they overwrite a live palace. The other backup paths copy a single file and do
+not use it.
 """
 
 import glob
 import os
 import shutil
+import stat
+
+
+# Names for the file types a backup copy deliberately leaves behind. A socket
+# or a named pipe makes ``shutil.copytree`` record an error for that entry and
+# raise at the end. Device nodes do not, but dereferencing one copies the
+# DEVICE instead of palace data, and a link to ``/dev/zero`` is copied without
+# bound, so they are left out for the opposite reason: the copy would succeed
+# at the wrong thing.
+_UNCOPYABLE_FILE_TYPES = (
+    (stat.S_ISSOCK, "socket"),
+    (stat.S_ISFIFO, "named pipe"),
+    (stat.S_ISCHR, "character device"),
+    (stat.S_ISBLK, "block device"),
+)
+
+_UNNAMED_FILE_TYPE = "not a regular file or directory"
+
+
+def _file_type_label(mode):
+    """Name the file type in ``mode``, falling back to a generic phrase.
+
+    The fallback covers a file type this table has no name for. Nothing on
+    Linux reaches it, because ``stat`` there reports ``S_IFDOOR``,
+    ``S_IFPORT`` and ``S_IFWHT`` as 0. macOS does define ``S_IFWHT``, a
+    union-mount whiteout, which is precisely what the fallback exists for:
+    an entry no copy can carry is skipped with an honest reason rather than
+    read as copyable.
+    """
+    for is_type, label in _UNCOPYABLE_FILE_TYPES:
+        if is_type(mode):
+            return label
+    return _UNNAMED_FILE_TYPE
+
+
+def _uncopyable_reason(path, *, follow_symlinks):
+    """Name why ``path`` cannot be copied into a backup, or return ``None``.
+
+    This is an allowlist, matching how the rest of the package treats
+    directory entries: only regular files and directories are copyable, and
+    everything else is named and left out.
+
+    Args:
+        path: The directory entry to classify.
+        follow_symlinks: Mirrors the copy's own link handling. ``True`` when
+            the copy dereferences links, so the TARGET's file type decides;
+            ``False`` when it recreates them as links, in which case any
+            symlink is fine and its target is never read.
+
+    Returns:
+        A short human-readable reason, or ``None`` both when the entry is
+        copyable and when its type could not be established.
+
+    Those two ``None`` cases are deliberately the same answer: only a file
+    type this could read names a reason. An entry whose type it could not
+    read is left for the copy to attempt and, if it really is broken, to
+    fail on.
+
+    A failed ``stat`` therefore never becomes a reason to skip. It says the
+    entry cannot be resolved right now, which is not the same as holding no
+    data, and no errno separates the two: a symlink into a volume that is
+    not mounted fails exactly like one whose target was deleted, and on
+    Windows an unmapped drive letter and an unreachable network share both
+    arrive as ``ENOENT`` as well.
+    """
+    try:
+        st = os.lstat(path)
+    except OSError:
+        return None
+
+    if stat.S_ISLNK(st.st_mode):
+        if not follow_symlinks:
+            return None
+        try:
+            st = os.stat(path)
+        except OSError:
+            return None
+
+    if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode):
+        return None
+    return _file_type_label(st.st_mode)
+
+
+def _report_skipped(skipped, src, log):
+    """Tell the operator what the copy left out.
+
+    Every step that can fail is guarded on its own, because this also runs
+    after the copy has failed: both callers pass ``print``, so one line the
+    terminal will not take must cost neither the copy's own outcome nor the
+    lines after it.
+    """
+    noun = "entry" if len(skipped) == 1 else "entries"
+    lines = [f"  Backup: skipped {len(skipped)} {noun} that cannot be copied:"]
+    for path, reason in skipped:
+        try:
+            name = os.path.relpath(path, src)
+        except (OSError, ValueError):
+            # On Windows ``relpath`` rejects paths on different drives; for a
+            # relative ``src`` it also reaches ``os.getcwd()``, which can fail
+            # on a deleted directory. Neither costs us the entry's name.
+            name = path
+        lines.append(f"    {name} ({reason})")
+
+    for line in lines:
+        try:
+            log(line)
+        except UnicodeError:
+            # An stdout that cannot encode this entry's name. Dropping the
+            # line would leave a header whose count disagrees with the list
+            # under it, so the name is escaped and retried before it is
+            # given up on.
+            try:
+                log(line.encode("ascii", "backslashreplace").decode("ascii"))
+            except (OSError, UnicodeError):
+                continue
+        except OSError:
+            # A write that the device refused. A short report is buffered
+            # whole, so this arrives only once the report is long enough to
+            # flush partway through, and then the header is already out.
+            # Anything else is a caller passing something that is not a
+            # working ``log``, which should surface rather than be swallowed.
+            continue
+
+
+def copy_palace_dir(src, dst, *, symlinks=False, log=None):
+    """Copy a palace directory to ``dst``, skipping entries no copy can carry.
+
+    ``shutil.copytree`` finishes the copy but raises ``shutil.Error`` at the
+    end when a directory entry is not something it knows how to duplicate: a
+    Unix domain socket, or a named pipe. The caller has to treat that as a
+    failed backup, so the command died before the rebuild the backup was
+    guarding (#2207). Such entries are runtime artifacts of whatever process
+    created them and never hold palace data, so a backup without them is
+    still a complete backup of the palace.
+
+    Args:
+        src: Palace directory to copy.
+        dst: Destination path. Must not already exist.
+        symlinks: Passed to ``shutil.copytree``. ``True`` recreates symlinks
+            as symlinks, ``False`` copies what they point at.
+        log: Optional callable (e.g. ``print``) for human-readable progress.
+
+    Returns:
+        The list of ``(path, reason)`` pairs that were skipped: sorted within
+        each directory, in the order the copy visited the directories. Like
+        ``prune_backups``, this both returns what it did and logs it.
+
+    Every other copy failure still raises out of ``shutil.copytree``. A caller
+    about to overwrite the live palace must still stop when its safety copy
+    did not come out whole, so this narrows what the copy attempts rather
+    than swallowing what it reports.
+
+    ``shutil.copytree`` hands the callback names rather than the ``os.scandir``
+    entries it already holds, so classifying costs one extra ``os.lstat`` per
+    directory entry, and it classifies a whole directory before copying any of
+    it. An entry whose type changes inside that window is handled by the
+    earlier reading, which cuts both ways: one that became a socket still
+    aborts the copy, and one that was a socket and became a regular file is
+    skipped with its contents. Nothing in MemPalace replaces an entry that
+    way, and the skipped name is printed either way.
+    """
+    skipped = []
+    # ``shutil.copytree`` classifies the top directory before it creates the
+    # destination, so a copy that never starts still produces a skip list.
+    # Reporting one would tell the operator what a nonexistent backup is
+    # missing, so both ways that happens are excluded below: a destination
+    # already occupied (this snapshot), and a destination that could not be
+    # created at all.
+    dst_existed = os.path.lexists(dst)
+
+    def _ignore(directory, names):
+        ignored = set()
+        found = []
+        for name in names:
+            path = os.path.join(directory, name)
+            reason = _uncopyable_reason(path, follow_symlinks=not symlinks)
+            if reason is not None:
+                ignored.add(name)
+                found.append((path, reason))
+        skipped.extend(sorted(found))
+        return ignored
+
+    def _report():
+        if skipped and log and not dst_existed and os.path.isdir(dst):
+            _report_skipped(skipped, src, log)
+
+    try:
+        shutil.copytree(src, dst, symlinks=symlinks, ignore=_ignore)
+    except BaseException:
+        # Reported for a failed copy too, including an interrupted one: a
+        # half-written backup is exactly when the operator needs to know what
+        # was left out of it. But the copy's own failure is what the caller
+        # has to diagnose, so a report that fails on top of it is dropped
+        # rather than raised in its place.
+        try:
+            _report()
+        except Exception:
+            pass
+        raise
+
+    # Not suppressed here: with the backup complete and nothing destructive
+    # done yet, a ``log`` that cannot be called at all is a caller's bug worth
+    # hearing about. The terminal failures inside ``_report_skipped`` are
+    # absorbed on both paths.
+    _report()
+    return skipped
 
 
 def prune_backups(pattern, max_backups, *, log=None):

mempalace/closet_llm.py

@@ -36,6 +36,7 @@ No vendor lock-in. No hidden dependency on any specific provider. Zero deps
 added to pyproject — uses stdlib urllib.
 """
 
+import contextlib
 import json
 import os
 import re
@@ -51,6 +52,7 @@ from .palace import (
     get_closets_collection,
     get_collection,
     mine_lock,
+    mine_palace_lock,
     purge_file_closets,
     upsert_closet_lines,
 )
@@ -225,8 +227,31 @@ def regenerate_closets(
         print("or pass --endpoint / --model / --key on the CLI.")
         return {"error": "missing-config", "missing": missing}
 
-    drawers_col = get_collection(palace_path, create=False)
-    closets_col = get_closets_collection(palace_path)
+    # A full regeneration is one long-lived read/LLM/purge/upsert operation.
+    # Acquire ownership before opening either collection or calling the LLM so
+    # a conflicting local writer fails immediately. The palace lock is
+    # process-wide and re-entrant, so daemon/MCP-owned calls compose safely.
+    lease = contextlib.nullcontext() if dry_run else mine_palace_lock(palace_path)
+    with lease:
+        return _regenerate_closets_owned(
+            palace_path,
+            wing=wing,
+            sample=sample,
+            dry_run=dry_run,
+            cfg=cfg,
+        )
+
+
+def _regenerate_closets_owned(
+    palace_path,
+    *,
+    wing,
+    sample,
+    dry_run,
+    cfg: LLMConfig,
+):
+    drawers_col = get_collection(palace_path, create=False, read_only=dry_run)
+    closets_col = None if dry_run else get_closets_collection(palace_path)
 
     total = drawers_col.count()
     if total == 0:
@@ -303,6 +328,7 @@ def regenerate_closets(
         # otherwise a regex closet rebuild mid-regenerate races with our
         # purge+upsert cycle and leaves mixed regex/LLM lines.
         with mine_lock(source):
+            assert closets_col is not None
             purge_file_closets(closets_col, source)
             upsert_closet_lines(
                 closets_col,

mempalace/config.py

@@ -374,9 +374,9 @@ class MempalaceConfig:
 
         if self._config_file.exists():
             try:
-                with open(self._config_file, "r") as f:
+                with open(self._config_file, "r", encoding="utf-8") as f:
                     self._file_config = json.load(f)
-            except (json.JSONDecodeError, OSError):
+            except (json.JSONDecodeError, UnicodeDecodeError, OSError):
                 self._file_config = {}
 
     @property
@@ -550,9 +550,9 @@ class MempalaceConfig:
         """Mapping of name variants to canonical names."""
         if self._people_map_file.exists():
             try:
-                with open(self._people_map_file, "r") as f:
+                with open(self._people_map_file, "r", encoding="utf-8") as f:
                     return json.load(f)
-            except (json.JSONDecodeError, OSError):
+            except (json.JSONDecodeError, UnicodeDecodeError, OSError):
                 pass
         return self._file_config.get("people_map", {})
 
@@ -622,7 +622,8 @@ class MempalaceConfig:
 
         Enforces the invariants the miner relies on:
           * ``chunk_size >= 1``
-          * ``0 <= chunk_overlap < chunk_size`` — equality would loop forever
+          * ``0 <= chunk_overlap <= chunk_size // 2``. A larger overlap can
+            loop the miner forever on short-line content (#2056)
           * ``min_chunk_size <= chunk_size`` — otherwise no chunk is ever
             large enough to file, and ingest silently produces 0 drawers
 
@@ -635,12 +636,14 @@ class MempalaceConfig:
             "min_chunk_size", DEFAULT_MIN_CHUNK_SIZE, minimum=0
         )
 
-        if chunk_overlap >= chunk_size:
-            chunk_overlap = (
-                DEFAULT_CHUNK_OVERLAP
-                if DEFAULT_CHUNK_OVERLAP < chunk_size
-                else max(0, chunk_size - 1)
-            )
+        if chunk_overlap > chunk_size // 2:
+            # Overlap past half the chunk size can hang miner.chunk_text's
+            # windowing loop on short-line content (#2056): a boundary pull can
+            # shrink a chunk below chunk_overlap, so
+            # ``start = end - chunk_overlap`` stops advancing. Repair to the
+            # default when it is still at most half, else clamp to the largest
+            # safe overlap.
+            chunk_overlap = min(DEFAULT_CHUNK_OVERLAP, chunk_size // 2)
 
         if min_chunk_size > chunk_size:
             min_chunk_size = (
@@ -656,7 +659,7 @@ class MempalaceConfig:
 
     @property
     def chunk_overlap(self) -> int:
-        """Overlap between adjacent chunks (validated, ``< chunk_size``)."""
+        """Overlap between adjacent chunks (validated, ``<= chunk_size // 2``)."""
         return self._validated_chunk_config()[1]
 
     @property
@@ -745,7 +748,10 @@ class MempalaceConfig:
 
         Values: ``"minilm"`` (ChromaDB's all-MiniLM-L6-v2 — English-only),
         ``"embeddinggemma"`` (multilingual, 100+ languages, default for
-        new installs since onboarding writes the choice). Read from env
+        new installs since onboarding writes the choice), or
+        ``"openai-compat"`` (embeddings served by an OpenAI-compatible
+        ``/v1/embeddings`` endpoint — see ``embedding_api_url`` /
+        ``embedding_api_model`` / ``embedding_api_key``). Read from env
         ``MEMPALACE_EMBEDDING_MODEL`` first, then ``embedding_model`` in
         ``config.json``, then ``"minilm"`` as a back-compat fallback for
         palaces created before onboarding asked the question.
@@ -829,6 +835,52 @@ class MempalaceConfig:
         except (OSError, NotImplementedError):
             pass
 
+    def _resolve_str_setting(self, env_var: str, config_key: str):
+        """Resolve a string setting: env var > ``config.json`` > ``None``.
+
+        Whitespace-only values are treated as unset, so a blank env var or a
+        hand-edited empty config key doesn't mask the value below it. Unlike
+        ``embedding_model`` the result is not lower-cased — URLs, model ids,
+        and API keys are case-sensitive.
+        """
+        env_val = os.environ.get(env_var)
+        if env_val and env_val.strip():
+            return env_val.strip()
+        cfg_val = self._file_config.get(config_key)
+        if isinstance(cfg_val, str) and cfg_val.strip():
+            return cfg_val.strip()
+        return None
+
+    @property
+    def embedding_api_url(self):
+        """Base URL of the OpenAI-compatible ``/v1/embeddings`` endpoint.
+
+        Used only when ``embedding_model == "openai-compat"``. Resolved from
+        env ``MEMPALACE_EMBEDDING_API_URL`` first, then ``embedding_api_url``
+        in ``config.json``; ``None`` when unset. Accepts a bare host, a
+        ``…/v1`` base, or a full endpoint URL.
+        """
+        return self._resolve_str_setting("MEMPALACE_EMBEDDING_API_URL", "embedding_api_url")
+
+    @property
+    def embedding_api_model(self):
+        """Server-side model id for the ``openai-compat`` embeddings endpoint.
+
+        Resolved from env ``MEMPALACE_EMBEDDING_API_MODEL`` first, then
+        ``embedding_api_model`` in ``config.json``; ``None`` when unset.
+        """
+        return self._resolve_str_setting("MEMPALACE_EMBEDDING_API_MODEL", "embedding_api_model")
+
+    @property
+    def embedding_api_key(self):
+        """Optional bearer token / API key for the embeddings endpoint.
+
+        Resolved from env ``MEMPALACE_EMBEDDING_API_KEY`` first, then
+        ``embedding_api_key`` in ``config.json``; ``None`` when unset (for
+        local endpoints that need no auth).
+        """
+        return self._resolve_str_setting("MEMPALACE_EMBEDDING_API_KEY", "embedding_api_key")
+
     @property
     def topic_tunnel_min_count(self):
         """Minimum number of overlapping confirmed topics required to create
@@ -885,6 +937,43 @@ class MempalaceConfig:
         )
         return DEFAULT_MAX_BACKUPS if coerced is None else coerced
 
+    @property
+    def lang_explicit(self):
+        """Primary language code when explicitly configured, else ``None``.
+
+        Resolution order: ``MEMPALACE_LANG`` / ``MEMPAL_LANG`` env var, then
+        ``config.json["lang"]``. Returns ``None`` if neither is set. Use this
+        when a caller needs to know whether the user has opted in to locale
+        behaviour (e.g. to avoid silently changing search scoring for palaces
+        that have never set a language).
+        """
+        env_val = os.environ.get("MEMPALACE_LANG") or os.environ.get("MEMPAL_LANG")
+        if env_val and env_val.strip():
+            return env_val.strip()
+        cfg = self._file_config.get("lang")
+        if isinstance(cfg, str) and cfg.strip():
+            return cfg.strip()
+        return None
+
+    @property
+    def lang(self):
+        """Primary language code for localized output and display.
+
+        Resolution order: ``lang_explicit`` (env or config.json), first entry
+        of ``entity_languages``, then ``"en"``. Always returns a non-empty
+        string so callers that need a language for display purposes never
+        have to handle ``None``. Code paths that must not silently change
+        behaviour for unconfigured palaces should read ``lang_explicit``
+        instead.
+        """
+        explicit = self.lang_explicit
+        if explicit:
+            return explicit
+        entity_langs = self.entity_languages
+        if entity_langs:
+            return entity_langs[0]
+        return "en"
+
     @property
     def hook_silent_save(self):
         """Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""
@@ -1038,7 +1127,7 @@ class MempalaceConfig:
                 "topic_wings": DEFAULT_TOPIC_WINGS,
                 "hall_keywords": DEFAULT_HALL_KEYWORDS,
             }
-            with open(self._config_file, "w") as f:
+            with open(self._config_file, "w", encoding="utf-8") as f:
                 json.dump(default_config, f, indent=2)
             # Restrict config file to owner read/write only
             try:
@@ -1054,7 +1143,7 @@ class MempalaceConfig:
             people_map: Dict mapping name variants to canonical names.
         """
         self._config_dir.mkdir(parents=True, exist_ok=True)
-        with open(self._people_map_file, "w") as f:
+        with open(self._people_map_file, "w", encoding="utf-8") as f:
             json.dump(people_map, f, indent=2)
         try:
             self._people_map_file.chmod(0o600)

mempalace/convo_miner.py

@@ -11,6 +11,7 @@ Same palace as project mining. Different ingest strategy.
 import os
 import sys
 import json
+import hashlib
 import logging
 import stat
 from pathlib import Path
@@ -18,9 +19,15 @@ from datetime import datetime
 from collections import defaultdict
 from typing import Optional
 
+from .backends import PalaceNotFoundError
 from .collision_scan import assert_no_collisions
-from .ids import ID_RECIPE, make_convo_drawer_id, make_convo_sentinel_id
-from .normalize import normalize
+from .ids import (
+    ID_RECIPE,
+    make_convo_drawer_id,
+    make_convo_sentinel_id,
+    make_exchange_drawer_id,
+)
+from .normalize import normalize_conversations
 from .entities import entities_metadata
 from .palace import (
     NORMALIZE_VERSION,
@@ -31,6 +38,7 @@ from .palace import (
     get_collection,
     mine_lock,
     mine_palace_lock,
+    prefetch_content_hashes,
     prefetch_mined_set,
 )
 
@@ -57,6 +65,82 @@ def _detect_hall_cached(content: str) -> str:
     return max(scores, key=scores.get) if scores else "general"
 
 
+def file_conversation_exchange(
+    collection,
+    *,
+    wing: str,
+    room: str,
+    text: str,
+    source_file: str,
+    agent: str,
+    authored_at: Optional[str] = None,
+    extra_metadata: Optional[dict] = None,
+) -> Optional[str]:
+    """File one verbatim conversation exchange as a single drawer.
+
+    Canonical write path for live agent integrations (e.g. Hermes) and
+    their backfills — both must route here so routing, normalization,
+    and metadata conventions stay identical between live and historical
+    ingest. Builds the same metadata the convo miner writes so hallway
+    traversal, entity search, and since/before date filters see
+    integration drawers exactly like mined ones.
+
+    ``wing`` and ``room`` are validated with the same ``sanitize_name``
+    rules the MCP write tools apply, but a failed name falls back
+    (``wing_general`` / ``conversations``) instead of erroring: this
+    path files *live* turns, and dropping a turn over a config typo
+    would break the verbatim / 100%-recall promise. The fallback is
+    logged at warning level so the misconfiguration is visible.
+
+    ``extra_metadata`` lets callers append integration-specific fields
+    (e.g. ``source`` / ``session_id``); keys that collide with the
+    canonical fields are ignored, so it cannot be used to overwrite or
+    drop them. Returns the drawer id, or None when ``text`` is empty
+    after stripping.
+    """
+    from .config import sanitize_name
+
+    text = (text or "").strip()
+    if not text:
+        return None
+    try:
+        wing = sanitize_name(wing, "wing")
+    except ValueError:
+        logger.warning(
+            "file_conversation_exchange: invalid wing %r — filing under wing_general", wing
+        )
+        wing = "wing_general"
+    try:
+        room = sanitize_name(room, "room")
+    except ValueError:
+        logger.warning(
+            "file_conversation_exchange: invalid room %r — filing under conversations", room
+        )
+        room = "conversations"
+    filed_at = datetime.now().isoformat()
+    drawer_id = make_exchange_drawer_id(wing, room, source_file, filed_at, text)
+    metadata = {
+        "wing": wing,
+        "room": room,
+        "hall": _detect_hall_cached(text),
+        "source_file": source_file,
+        "chunk_index": 0,
+        "added_by": agent,
+        "filed_at": filed_at,
+        "entities": entities_metadata(text),
+        "authored_at": authored_at if authored_at is not None else filed_at,
+        "ingest_mode": "convos",
+        "extract_mode": "exchange",
+        "normalize_version": NORMALIZE_VERSION,
+        "id_recipe": ID_RECIPE,
+    }
+    if extra_metadata:
+        for key, value in extra_metadata.items():
+            metadata.setdefault(key, value)
+    collection.upsert(ids=[drawer_id], documents=[text], metadatas=[metadata])
+    return drawer_id
+
+
 # File types that might contain conversations
 CONVO_EXTENSIONS = {
     ".txt",
@@ -117,7 +201,14 @@ def _is_regular_source_file(filepath: Path, root: Path) -> bool:
                 pass
 
 
-def _register_file(collection, source_file: str, wing: str, agent: str, extract_mode: str):
+def _register_file(
+    collection,
+    source_file: str,
+    wing: str,
+    agent: str,
+    extract_mode: str,
+    content_hash: Optional[str] = None,
+):
     """Write a sentinel so file_already_mined() returns True for 0-chunk files.
 
     Without this, files that normalize to nothing or produce zero chunks are
@@ -128,6 +219,11 @@ def _register_file(collection, source_file: str, wing: str, agent: str, extract_
     grows past the min-chunk-size floor (e.g. a short session that gets
     extended) is correctly detected as changed on the next mine instead of
     being skipped forever by this sentinel.
+
+    Also used to register a file recognized as a content-duplicate of an
+    already-mined transcript under a different path (see
+    ``prefetch_content_hashes``) — stamping it here means the next run skips
+    it via the cheap mtime check instead of re-normalizing and re-hashing it.
     """
     try:
         source_mtime = os.path.getmtime(source_file)
@@ -147,6 +243,8 @@ def _register_file(collection, source_file: str, wing: str, agent: str, extract_
     }
     if source_mtime is not None:
         meta["source_mtime"] = source_mtime
+    if content_hash is not None:
+        meta["content_hash"] = content_hash
     collection.upsert(
         documents=[f"[registry] {source_file}"],
         ids=[sentinel_id],
@@ -395,17 +493,37 @@ def detect_convo_room(content: str) -> str:
 # =============================================================================
 
 
-def scan_convos(convo_dir: str) -> list:
+def scan_convos(convo_dir: str, include_subagents: bool = False) -> list:
     """Find all potential conversation files.
 
     Skips symlinks and oversized files. Each skipped symlink is logged to
     ``sys.stderr`` with a ``  SKIP: <relative-path> (symlink)`` line so the
     caller can tell why an apparent conversation directory yielded no files.
+
+    By default, directories named ``subagents`` are skipped: Claude Code
+    records Explore/Plan/Grep subagent transcripts there, and on typical
+    workspaces they outnumber main session files by one to two orders of
+    magnitude. Pass ``include_subagents=True`` to mine them anyway.
+
+    The match is case-insensitive on the directory name only (``subagents``
+    or ``Subagents``), so directories like ``mysubagents`` or
+    ``subagentsbackup`` are not affected.
     """
-    convo_path = Path(convo_dir).expanduser().resolve()
+    # A direct conversation file is a valid source. For a file, feed only
+    # its basename through the existing directory validation loop.
+    requested_path = Path(convo_dir).expanduser()
+    single_file = requested_path.is_file()
+    convo_path = (requested_path.parent if single_file else requested_path).resolve()
+    scan_entries = (
+        [(str(convo_path), [], [requested_path.name])] if single_file else os.walk(convo_path)
+    )
     files = []
-    for root, dirs, filenames in os.walk(convo_path):
-        dirs[:] = [d for d in dirs if d not in CONVO_SKIP_DIRS]
+    for root, dirs, filenames in scan_entries:
+        dirs[:] = [
+            d
+            for d in dirs
+            if d not in CONVO_SKIP_DIRS and (include_subagents or d.lower() != "subagents")
+        ]
         for filename in filenames:
             if filename.endswith(".meta.json"):
                 continue
@@ -487,7 +605,15 @@ def _extract_authored_at(filepath):
 
 
 def _file_chunks_locked(
-    collection, source_file, chunks, wing, room, agent, extract_mode, authored_at=None
+    collection,
+    source_file,
+    chunks,
+    wing,
+    room,
+    agent,
+    extract_mode,
+    authored_at=None,
+    content_hash=None,
 ):
     """Lock the source file, purge stale drawers, and upsert fresh chunks.
 
@@ -514,12 +640,28 @@ def _file_chunks_locked(
         # (file_already_mined() returned False for pre-v2 drawers) and on a
         # changed/grown transcript (mtime differs) — clean them out so the
         # source doesn't end up with mixed old/new drawers.
+        #
+        # A failed purge must abort this file's mine attempt rather than
+        # fall through to upsert: proceeding on top of an unpurged (or
+        # partially purged) set produces duplicate/stale drawers under
+        # mixed schema versions, with no operator-visible signal beyond a
+        # debug log (#105 — convo_miner's own instance of the same swallow
+        # already fixed for miner.py at #23). Returning here leaves the old
+        # drawers' stored mtime untouched, so the next mine still sees a
+        # mismatch and retries.
         try:
             delete_ids = _source_file_delete_ids(collection, source_file, extract_mode)
             if delete_ids:
                 collection.delete(ids=delete_ids)
-        except Exception:
+        except Exception as exc:
+            print(
+                f"  ! [skip] stale-drawer purge failed for {source_file!r} "
+                f"({exc!r}); leaving existing drawers untouched, will retry "
+                f"on the next mine",
+                file=sys.stderr,
+            )
             logger.debug("Stale-drawer purge failed for %s", source_file, exc_info=True)
+            return 0, room_counts_delta, True
 
         # Batch chunks into bounded upserts so large transcripts keep most of
         # the embedding speedup without one huge Chroma/SQLite request. Keep
@@ -560,6 +702,12 @@ def _file_chunks_locked(
                 }
                 if source_mtime is not None:
                     meta["source_mtime"] = source_mtime
+                # Stamp content_hash only on chunk 0 so multi-conversation
+                # privacy-export hashes are not O(N²)-duplicated across every
+                # chunk row. ``prefetch_content_hashes`` still finds them —
+                # it scans all drawers and splits comma-joined hash fields.
+                if content_hash is not None and chunk.get("chunk_index", 0) == 0:
+                    meta["content_hash"] = content_hash
                 batch_metas.append(meta)
             assert_no_collisions(list(zip(batch_ids, batch_metas)), collection)
             try:
@@ -604,6 +752,32 @@ def _is_ai_tool_path(path: Path) -> bool:
     return False
 
 
+def _split_new_and_duplicate_conversations(
+    conversations: list,
+    wing: str,
+    source_file: str,
+    mined_content_hashes: dict,
+) -> tuple:
+    """Hash each conversation and split them into (new, duplicate) lists.
+
+    A conversation is a duplicate when its hash is already registered under
+    a *different* source_file in the same wing — mining the same transcript
+    into a second wing is a deliberate re-file, not a repeat, so the lookup
+    is scoped to (wing, hash). Returns ([(hash, text), ...] new, [(hash,
+    dup_source_file), ...] duplicates).
+    """
+    new_items = []
+    duplicates = []
+    for conversation in conversations:
+        content_hash = hashlib.sha256(conversation.strip().encode("utf-8")).hexdigest()
+        dup_source = mined_content_hashes.get((wing, content_hash))
+        if dup_source is None or dup_source == source_file:
+            new_items.append((content_hash, conversation))
+        else:
+            duplicates.append((content_hash, dup_source))
+    return new_items, duplicates
+
+
 def _is_unchanged_since_last_mine(source_file: str, mined_mtimes: dict) -> bool:
     """True iff source_file was mined at the current schema AND its on-disk
     mtime still matches what was stored -- the mtime-aware replacement for
@@ -659,12 +833,16 @@ def mine_convos(
     limit: int = 0,
     dry_run: bool = False,
     extract_mode: str = "exchange",
+    include_subagents: bool = False,
 ):
     """Mine a directory of conversation files into the palace.
 
     extract_mode:
         "exchange" — default exchange-pair chunking (Q+A = one unit)
         "general"  — general extractor: decisions, preferences, milestones, problems, emotions
+    include_subagents:
+        False (default) — skip Claude Code ``subagents/`` directories
+        True            — also mine subagent transcripts
 
     The real work is in :func:`_mine_convos_impl`; this wrapper holds the
     per-palace flock around it so two concurrent ``mempalace mine --mode
@@ -691,6 +869,7 @@ def mine_convos(
             limit=limit,
             dry_run=dry_run,
             extract_mode=extract_mode,
+            include_subagents=include_subagents,
         )
 
     with mine_palace_lock(palace_path):
@@ -702,6 +881,7 @@ def mine_convos(
             limit=limit,
             dry_run=dry_run,
             extract_mode=extract_mode,
+            include_subagents=include_subagents,
         )
 
 
@@ -721,6 +901,64 @@ def _compute_hallways_for_wing_safe(wing, collection, drawers_filed, config=None
         print(f"  (hallways skipped: {exc})")
 
 
+def _normalize_convo_conversations(
+    filepath: Path,
+    source_file: str,
+    cfg_min_chunk_size: int,
+    collection,
+    wing: str,
+    agent: str,
+    extract_mode: str,
+    dry_run: bool,
+) -> Optional[list]:
+    """Normalize a transcript file into its individual conversations,
+    registering it as filed when there's nothing worth mining. Returns None
+    when the caller should skip the file (normalize failed, or normalized
+    content is too short to chunk).
+
+    Kept as separate conversations rather than joined into one string so
+    dedup can hash and skip per conversation — a Claude.ai privacy export
+    bundles every conversation into a single file, and hashing the joined
+    bundle means one new conversation added to a re-export changes the
+    whole-file hash and hides the conversations that didn't change.
+    """
+    try:
+        conversations = [c for c in normalize_conversations(str(filepath)) if c]
+    except (OSError, ValueError):
+        if not dry_run:
+            _register_file(collection, source_file, wing, agent, extract_mode)
+        return None
+
+    total_len = sum(len(c.strip()) for c in conversations)
+    if not conversations or total_len < cfg_min_chunk_size:
+        if not dry_run:
+            _register_file(collection, source_file, wing, agent, extract_mode)
+        return None
+
+    return conversations
+
+
+def _open_convo_collection(
+    palace_path: str,
+    *,
+    dry_run: bool,
+):
+    """Open the conversation collection without creating it during dry-run."""
+    if not dry_run:
+        return get_collection(palace_path)
+
+    try:
+        return get_collection(
+            palace_path,
+            create=False,
+            read_only=True,
+        )
+    except PalaceNotFoundError:
+        # A missing palace or uninitialized collection represents empty
+        # prior state to a dry-run. Do not create either one.
+        return None
+
+
 def _mine_convos_impl(
     convo_dir: str,
     palace_path: str,
@@ -729,6 +967,7 @@ def _mine_convos_impl(
     limit: int = 0,
     dry_run: bool = False,
     extract_mode: str = "exchange",
+    include_subagents: bool = False,
 ):
     from .config import MempalaceConfig
 
@@ -748,10 +987,10 @@ def _mine_convos_impl(
     convo_path = Path(convo_dir).expanduser().resolve()
     wing = _resolve_wing(convo_path, wing)
 
-    files = scan_convos(convo_dir)
+    files = scan_convos(convo_dir, include_subagents=include_subagents)
 
     print(f"\n{'=' * 55}")
-    print("  MemPalace Mine — Conversations")
+    print("  MemPalace Mine -- Conversations")
     print(f"{'=' * 55}")
     print(f"  Wing:    {wing}")
     print(f"  Source:  {convo_path}")
@@ -759,10 +998,13 @@ def _mine_convos_impl(
     print(f"  Files:   {len(files)}{limit_suffix}")
     print(f"  Palace:  {palace_path}")
     if dry_run:
-        print("  DRY RUN — nothing will be filed")
+        print("  DRY RUN -- nothing will be filed")
     print(f"{'-' * 55}\n")
 
-    collection = get_collection(palace_path) if not dry_run else None
+    collection = _open_convo_collection(
+        palace_path,
+        dry_run=dry_run,
+    )
 
     # Bulk pre-fetch already-mined source_file -> stored mtime in one
     # paginated pass instead of `len(files)` separate WHERE-source_file
@@ -771,7 +1013,17 @@ def _mine_convos_impl(
     # prefetch_mined_set() does the same decisions in a single scan; loop
     # body becomes an O(1) dict lookup + a cheap local mtime comparison.
     mined_mtimes: dict = (
-        prefetch_mined_set(collection, extract_mode=extract_mode) if not dry_run else {}
+        prefetch_mined_set(collection, extract_mode=extract_mode) if collection is not None else {}
+    )
+    # content_hash -> source_file for transcripts already filed. Repeated
+    # exports from Claude/ChatGPT commonly land under a new filename each
+    # run even when the conversation itself is unchanged, so the
+    # source_file-keyed skip above ("mined_mtimes") never recognizes them —
+    # this catches the same conversation reappearing at a new path.
+    mined_content_hashes: dict = (
+        prefetch_content_hashes(collection, extract_mode=extract_mode)
+        if collection is not None
+        else {}
     )
 
     total_drawers = 0
@@ -792,7 +1044,7 @@ def _mine_convos_impl(
         # Falling through re-mines: _file_chunks_locked purges this
         # source_file's stale drawers before inserting fresh ones, so this
         # never leaves duplicates behind.
-        if not dry_run and _is_unchanged_since_last_mine(source_file, mined_mtimes):
+        if _is_unchanged_since_last_mine(source_file, mined_mtimes):
             files_skipped += 1
             continue
 
@@ -800,19 +1052,43 @@ def _mine_convos_impl(
             files_skipped += 1
             continue
 
-        # Normalize format
-        try:
-            content = normalize(str(filepath))
-        except (OSError, ValueError):
-            if not dry_run:
-                _register_file(collection, source_file, wing, agent, extract_mode)
+        conversations = _normalize_convo_conversations(
+            filepath,
+            source_file,
+            cfg_min_chunk_size,
+            collection,
+            wing,
+            agent,
+            extract_mode,
+            dry_run,
+        )
+        if conversations is None:
             continue
 
-        if not content or len(content.strip()) < cfg_min_chunk_size:
+        # Hash and dedup per conversation, not per file: a Claude/ChatGPT
+        # privacy export bundles every conversation into one file, so a
+        # re-export that adds one new conversation changes the whole-file
+        # hash and would hide the conversations that didn't change if we
+        # hashed the joined bundle. Conversations whose hash is already
+        # filed under a different source_file in this wing are dropped;
+        # the rest are re-joined and mined as usual.
+        new_items, duplicates = _split_new_and_duplicate_conversations(
+            conversations, wing, source_file, mined_content_hashes
+        )
+        if not new_items:
             if not dry_run:
                 _register_file(collection, source_file, wing, agent, extract_mode)
+            dup_source = duplicates[0][1]
+            print(
+                f"  = [{i:4}/{len(files)}] {filepath.name[:50]:50} "
+                f"duplicate of {Path(dup_source).name}"
+            )
+            files_skipped += 1
             continue
 
+        content = "\n\n".join(text for _, text in new_items)
+        content_hash = ",".join(h for h, _ in new_items)
+
         # Chunk — either exchange pairs or general extraction
         if extract_mode == "general":
             from .general_extractor import extract_memories
@@ -843,9 +1119,9 @@ def _mine_convos_impl(
 
                 type_counts = Counter(c.get("memory_type", "general") for c in chunks)
                 types_str = ", ".join(f"{t}:{n}" for t, n in type_counts.most_common())
-                print(f"    [DRY RUN] {filepath.name} → {len(chunks)} memories ({types_str})")
+                print(f"    [DRY RUN] {filepath.name} -> {len(chunks)} memories ({types_str})")
             else:
-                print(f"    [DRY RUN] {filepath.name} → room:{room} ({len(chunks)} drawers)")
+                print(f"    [DRY RUN] {filepath.name} -> room:{room} ({len(chunks)} drawers)")
             total_drawers += len(chunks)
             # Track room counts
             if extract_mode == "general":
@@ -872,6 +1148,7 @@ def _mine_convos_impl(
             agent,
             extract_mode,
             authored_at=_extract_authored_at(filepath),
+            content_hash=content_hash,
         )
         if skipped:
             files_skipped += 1
@@ -879,6 +1156,8 @@ def _mine_convos_impl(
         for r, n in room_delta.items():
             room_counts[r] += n
 
+        for h, _ in new_items:
+            mined_content_hashes[(wing, h)] = source_file
         total_drawers += drawers_added
         files_mined += 1
         print(f"  + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}")

mempalace/daemon.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 import argparse
 import contextlib
 import json
+import math
 import os
 import secrets
 import sqlite3
@@ -28,6 +29,12 @@ from urllib import request as urlrequest
 from urllib.parse import parse_qs, urlparse
 
 from .config import MempalaceConfig
+from .palace import (
+    MineAlreadyRunning,
+    backend_requires_single_writer,
+    mine_palace_lock,
+    resolve_backend_name,
+)
 
 HOST = "127.0.0.1"
 STATE_ROOT_ENV = "MEMPALACE_DAEMON_STATE_ROOT"
@@ -39,6 +46,39 @@ DEFAULT_WAIT_TIMEOUT = 60.0 * 60.0
 HOOK_PROBE_TIMEOUT = 0.5
 TERMINAL_STATES = {"succeeded", "failed", "cancelled"}
 MAX_ATTEMPTS = 3
+# ``error_class`` marking a job refused the per-palace write lock. The lock
+# wraps the palace write itself, so a refusal means no drawer was filed and
+# re-running cannot duplicate palace content. Distinct from a crash
+# mid-execution, whose outcome is unknown -- see ``QueueStore.defer``.
+LOCK_REFUSAL_ERROR_CLASS = "LockHeldByOtherProcess"
+_DEFAULT_LOCK_BACKOFF_SECONDS = 60.0
+
+
+def _lock_defer_backoff_seconds() -> float:
+    """Seconds a job refused the lock cools off before the worker re-claims it.
+
+    The holder keeps the lock for as long as its write runs, which can be a long
+    mine, so re-claiming at once would spin without progress. Anything that is
+    not a positive, finite number yields the default: a typo must not crash the
+    daemon at import; ``0`` or a negative value would make the cooldown expire
+    instantly and spin the worker against a lock it cannot take; and ``inf``
+    would park the job until the daemon restarts, which is the dropped work
+    #2014 is about.
+    """
+    try:
+        value = float(
+            os.environ.get("MEMPALACE_DAEMON_LOCK_BACKOFF_SECONDS", "")
+            or _DEFAULT_LOCK_BACKOFF_SECONDS
+        )
+    except ValueError:
+        return _DEFAULT_LOCK_BACKOFF_SECONDS
+    if not math.isfinite(value) or value <= 0:
+        return _DEFAULT_LOCK_BACKOFF_SECONDS
+    return value
+
+
+# Override via env for operators; tests patch the module attribute directly.
+LOCK_DEFER_BACKOFF_SECONDS = _lock_defer_backoff_seconds()
 MAX_BODY_BYTES = 1 << 20  # 1 MiB cap on request bodies (auth-gated DoS guard)
 SHUTDOWN_DRAIN_SECONDS = 10.0
 # Terminal jobs are kept for diagnostics then pruned so the queue DB (which
@@ -411,36 +451,67 @@ class QueueStore:
             row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
             return self._row_to_job(row)
 
-    def claim_next(self) -> Job | None:
+    def claim_next(self, *, exclude: set[str] | None = None) -> Job | None:
         # Atomic across processes: the UPDATE only fires if the row is still
         # 'queued'. If two daemon processes SELECT the same row, the first to
         # UPDATE it flips state to 'running' (rowcount=1); the second's UPDATE
         # matches 0 rows (WHERE state='queued' is now false) and we re-loop
         # instead of double-executing the job. The in-process RLock does not
         # protect against a second OS process — this guard does.
+        #
+        # ``exclude`` skips jobs still cooling off after a lock refusal, so the
+        # ordering below cannot hand back the same refused job forever while
+        # newer work waits behind it. The cooldown is the worker's, held in
+        # memory rather than the schema (which has no migrations): a restart
+        # just retries such a job at once, costing one refusal, never the work.
+        #
+        # The filter runs in Python rather than as ``id NOT IN (?, ?, ...)``:
+        # that binds one host parameter per cooling job, and
+        # SQLITE_MAX_VARIABLE_NUMBER defaults to 999 below SQLite 3.32 (we
+        # support Python 3.9, whose bundled SQLite can predate it). Over the cap
+        # the query raises into the worker's catch-all, which retries a second
+        # later against a set that only ages out on the cooldown, so the queue
+        # would stall in cooldown-long fits rather than drain. Fetching
+        # ``len(exclude) + 1`` rows costs one parameter and is enough by
+        # pigeonhole: at most ``len(exclude)`` of those rows can be excluded, so
+        # the first one that survives the filter is the row a plain LIMIT 1 would
+        # have returned.
+        #
+        # ``error_json`` is cleared because it describes the claim that just
+        # ended: ``defer`` parks the refusal reason on a queued job, and a stale
+        # one would outlive its cause -- reporting a lock refusal for a job that
+        # later crashed, and leaking into recover_running's dead-letter (which
+        # COALESCEs it) in place of ``MaxAttemptsExceeded``.
+        exclude = exclude or set()
         with self._lock, self._connect() as conn:
-            row = conn.execute(
+            # Only the ids: the rows being skipped are here to be discarded, and
+            # jobs carry verbatim payloads, so selecting * would materialise a
+            # cooling job's user content just to read its id past it.
+            candidates = conn.execute(
                 """
-                SELECT * FROM jobs
+                SELECT id FROM jobs
                 WHERE state = 'queued'
                 ORDER BY priority DESC, created_at ASC
-                LIMIT 1
-                """
-            ).fetchone()
-            if row is None:
+                LIMIT ?
+                """,
+                (len(exclude) + 1,),
+            ).fetchall()
+            job_id = next((r["id"] for r in candidates if r["id"] not in exclude), None)
+            if job_id is None:
                 return None
             cur = conn.execute(
                 """
                 UPDATE jobs
-                SET state = 'running', started_at = ?, attempts = attempts + 1
+                SET state = 'running', started_at = ?, attempts = attempts + 1,
+                    error_json = NULL
                 WHERE id = ? AND state = 'queued'
                 """,
-                (_now(), row["id"]),
+                (_now(), job_id),
             )
             if cur.rowcount != 1:
                 # Lost the race to another process — nothing to run this iteration.
                 return None
-            claimed = conn.execute("SELECT * FROM jobs WHERE id = ?", (row["id"],)).fetchone()
+            claimed = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
             return self._row_to_job(claimed)
 
     def finish(
@@ -477,6 +548,56 @@ class QueueStore:
             row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
             return self._row_to_job(row)
 
+    def defer(
+        self,
+        job_id: str,
+        *,
+        claimed_started_at: str | None = None,
+        error: dict[str, Any] | None = None,
+    ) -> Job:
+        """Return a job refused the palace write lock to ``queued``, unspent.
+
+        ``mine_palace_lock`` wraps the palace write itself, so a refusal means
+        no drawer was filed and re-running cannot duplicate palace content.
+        That is what separates it from the failure ``MAX_ATTEMPTS`` guards -- a
+        daemon that died mid-execution, whose outcome is unknown and whose blind
+        retry would re-file verbatim content. Undoing ``claim_next``'s increment
+        keeps a palace that stays locked from spending the retry budget of work
+        that never landed.
+
+        ``error`` records why the job went back to the queue, so a job parked
+        behind a lock is distinguishable from one merely awaiting its turn.
+        ``claim_next`` clears it on the next claim, so the reason never outlives
+        the claim it describes.
+
+        The ``state = 'running'`` guard mirrors ``finish(only_if_running=True)``:
+        if shutdown already cancelled this job, deferring must not resurrect it.
+        ``claimed_started_at`` narrows the update further, to the claim that was
+        actually refused: if the row was re-queued and re-claimed in the window
+        (recover_running on another daemon start), a late defer must not throw
+        away that newer claim by re-queuing -- and refunding -- work it does not
+        own. ``None`` skips the check.
+        """
+        with self._lock, self._connect() as conn:
+            conn.execute(
+                """
+                UPDATE jobs
+                SET state = 'queued', started_at = NULL,
+                    attempts = MAX(attempts - 1, 0),
+                    error_json = ?
+                WHERE id = ? AND state = 'running'
+                  AND (? IS NULL OR started_at = ?)
+                """,
+                (
+                    json.dumps(error or {}, ensure_ascii=False) if error else None,
+                    job_id,
+                    claimed_started_at,
+                    claimed_started_at,
+                ),
+            )
+            row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
+            return self._row_to_job(row)
+
     def get(self, job_id: str) -> Job:
         with self._lock, self._connect() as conn:
             row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
@@ -523,6 +644,24 @@ class QueueStore:
         )
 
 
+def job_deferred_by_lock(job: dict[str, Any]) -> bool:
+    """True when a job's last claim was refused the palace lock.
+
+    Such a job is deferred, not failed (#2014): it went back to the queue and
+    runs once the lock frees. That makes it non-terminal, so a caller blocking
+    until ``TERMINAL_STATES`` would wait out the holder -- which can be a
+    long-lived session. Interactive callers use this to report the parked job
+    instead of stranding the terminal.
+
+    Keyed on the reason ``defer`` records, which ``claim_next`` clears on the
+    next claim: the answer is about the claim that just ended, not a live probe
+    of the lock (the holder may already have exited during the backoff).
+    """
+    if job.get("state") != "queued":
+        return False
+    return (job.get("error") or {}).get("error_class") == LOCK_REFUSAL_ERROR_CLASS
+
+
 def job_to_dict(job: Job, *, include_payload: bool = True) -> dict[str, Any]:
     out = {
         "id": job.id,
@@ -551,6 +690,9 @@ class DaemonRuntime:
         self.worker_wake = threading.Event()
         self.active_job_id: str | None = None
         self.worker_thread: threading.Thread | None = None
+        # job_id -> monotonic deadline before which a lock-refused job is not
+        # re-claimed (#2014). Worker-owned; only _worker_loop touches it.
+        self._deferred_until: dict[str, float] = {}
 
     def start_worker(self) -> threading.Thread:
         self.store.recover_running()
@@ -579,12 +721,37 @@ class DaemonRuntime:
         except Exception:  # noqa: BLE001 - a finish failure must not kill the worker
             pass
 
+    def _safe_defer(
+        self, job_id: str, *, claimed_started_at: str | None = None, error: dict | None = None
+    ) -> None:
+        try:
+            self.store.defer(job_id, claimed_started_at=claimed_started_at, error=error)
+        except Exception:  # noqa: BLE001 - a defer failure must not kill the worker
+            # The job stays 'running', so recover_running normally re-queues it
+            # on the next start and the work is still held -- mirroring
+            # _safe_finish's swallow. (recover_running does not refund the
+            # attempt, so repeated defer failures across MAX_ATTEMPTS restarts
+            # would eventually dead-letter it.)
+            pass
+
+    def _cooling_job_ids(self) -> set[str]:
+        """Ids of jobs still inside their post-refusal cooldown.
+
+        Pruned on every read, so the map tracks only jobs the worker is actually
+        holding back and cannot grow without bound.
+        """
+        now = time.monotonic()
+        self._deferred_until = {
+            job_id: until for job_id, until in self._deferred_until.items() if until > now
+        }
+        return set(self._deferred_until)
+
     def _worker_loop(self) -> None:
         from .service import execute_job
 
         while not self.shutdown_event.is_set():
             try:
-                job = self.store.claim_next()
+                job = self.store.claim_next(exclude=self._cooling_job_ids())
             except Exception:  # noqa: BLE001 - sqlite/disk errors must not kill the worker
                 self.shutdown_event.wait(1.0)
                 continue
@@ -602,6 +769,32 @@ class DaemonRuntime:
                     payload["backend"] = self.backend
                 result = execute_job(job.kind, payload)
                 ok = bool(result.get("success", True))
+                if not ok and result.get("error_class") == LOCK_REFUSAL_ERROR_CLASS:
+                    # Refused the palace write lock: no drawer was filed, so this
+                    # is a deferral, not a failure (#2014). Dead-lettering it
+                    # here would silently drop the work -- only hook-submitted
+                    # kinds get re-emitted, and the hook is not a retry
+                    # mechanism.
+                    self._safe_defer(
+                        job.id,
+                        claimed_started_at=job.started_at,
+                        error={
+                            "error_class": LOCK_REFUSAL_ERROR_CLASS,
+                            "message": result.get(
+                                "error", "palace write lock held by another process"
+                            ),
+                        },
+                    )
+                    # Cool the job off in the map instead of sleeping here. This
+                    # is the only worker, and the holder may outlive any wait we
+                    # could pick -- it keeps the lock until its write finishes,
+                    # and that write can be a long mine. Blocking would stall
+                    # every unrelated job behind a lock that has nothing to do
+                    # with them, and a job merely queued behind this one would
+                    # never be claimed, so it would never get the refusal marker
+                    # its caller waits on: the #2014 hang wearing a different hat.
+                    self._deferred_until[job.id] = time.monotonic() + LOCK_DEFER_BACKOFF_SECONDS
+                    continue
                 state = "succeeded" if ok else "failed"
                 error = None if ok else {"message": result.get("error", "job failed")}
                 self._safe_finish(job.id, state=state, result=result, error=error)
@@ -633,6 +826,49 @@ def _json_response(handler: BaseHTTPRequestHandler, status: int, payload: dict[s
     handler.close_connection = True
 
 
+def _restore_server_process_state(previous_env: dict[str, str | None], previous_umask: int) -> None:
+    for key, value in previous_env.items():
+        if value is None:
+            os.environ.pop(key, None)
+        else:
+            os.environ[key] = value
+    os.umask(previous_umask)
+
+
+def _close_writer_lease_after_worker(
+    writer_lease: contextlib.ExitStack,
+    worker: threading.Thread,
+) -> None:
+    """Retain writer ownership until a timed-out daemon worker really exits.
+
+    The normal daemon process may terminate first, in which case the operating
+    system releases the file lock. When ``run_server`` is embedded in a larger
+    process, this reaper prevents the server thread from exposing the palace to
+    another writer while the old worker is still finishing a mutation.
+    """
+
+    def _wait_and_close() -> None:
+        worker.join()
+        writer_lease.close()
+
+    threading.Thread(
+        target=_wait_and_close,
+        name="mempalace-daemon-writer-lease-reaper",
+        daemon=True,
+    ).start()
+
+
+def _close_or_defer_writer_lease(
+    writer_lease: contextlib.ExitStack,
+    runtime: "DaemonRuntime | None",
+) -> None:
+    worker = runtime.worker_thread if runtime is not None else None
+    if worker is not None and worker.is_alive():
+        _close_writer_lease_after_worker(writer_lease, worker)
+    else:
+        writer_lease.close()
+
+
 def run_server(palace_path: str, *, backend: str | None = None, port: int = 0) -> None:
     palace_path = canonical_palace_path(palace_path)
     previous_env = {
@@ -651,8 +887,31 @@ def run_server(palace_path: str, *, backend: str | None = None, port: int = 0) -
     # QueueStore (its _init_db opens the DB in WAL mode) — not only once the HTTP
     # server starts. Restored in the finally at the end of run_server.
     prev_umask = os.umask(0o077)
-    token = ensure_token(palace_path)
-    runtime = DaemonRuntime(palace_path, backend=backend)
+    runtime = None
+    writer_lease = contextlib.ExitStack()
+    try:
+        resolved_backend = resolve_backend_name(palace_path, explicit=backend)
+        if backend_requires_single_writer(resolved_backend):
+            try:
+                writer_lease.enter_context(mine_palace_lock(palace_path))
+            except MineAlreadyRunning as exc:
+                raise DaemonError(
+                    "writable daemon startup refused: another writer owns "
+                    f"local backend {resolved_backend!r} for {palace_path!r}; "
+                    "stop the existing writable MCP/direct/daemon owner, or "
+                    "route all writes through that owner"
+                ) from exc
+
+        token = ensure_token(palace_path)
+        # Backend resolution above is only the ownership decision. Preserve
+        # the caller's explicit/implicit distinction in queued payloads:
+        # DaemonRuntime historically injects a backend only when one was
+        # explicitly selected.
+        runtime = DaemonRuntime(palace_path, backend=backend)
+    except BaseException:
+        writer_lease.close()
+        _restore_server_process_state(previous_env, prev_umask)
+        raise
 
     class _Handler(BaseHTTPRequestHandler):
         protocol_version = "HTTP/1.1"
@@ -803,7 +1062,8 @@ def run_server(palace_path: str, *, backend: str | None = None, port: int = 0) -
             finally:
                 _drain_and_cleanup(runtime, palace_path, previous_env)
     finally:
-        os.umask(prev_umask)
+        _close_or_defer_writer_lease(writer_lease, runtime)
+        _restore_server_process_state(previous_env, prev_umask)
 
 
 def _drain_and_cleanup(
@@ -935,12 +1195,24 @@ class DaemonClient:
     def list_jobs(self, limit: int = 20) -> list[dict[str, Any]]:
         return self.request("GET", f"/jobs?limit={int(limit)}")["jobs"]
 
-    def wait(self, job_id: str, *, timeout: float = DEFAULT_WAIT_TIMEOUT) -> dict[str, Any]:
+    def wait(
+        self,
+        job_id: str,
+        *,
+        timeout: float = DEFAULT_WAIT_TIMEOUT,
+        stop_on_lock_deferral: bool = False,
+    ) -> dict[str, Any]:
         deadline = time.monotonic() + timeout
         while True:
             job = self.get_job(job_id)
             if job["state"] in TERMINAL_STATES:
                 return job
+            # A job parked behind the palace lock never becomes terminal on its
+            # own, so an interactive caller must be able to stop here instead of
+            # waiting out the holder (#2014). Background callers keep the old
+            # behaviour and simply wait.
+            if stop_on_lock_deferral and job_deferred_by_lock(job):
+                return job
             if time.monotonic() >= deadline:
                 raise DaemonError(f"timed out waiting for job {job_id}")
             time.sleep(0.2)
@@ -1120,6 +1392,7 @@ def submit_job(
     wait: bool = True,
     auto_start: bool = False,
     timeout: float = DEFAULT_WAIT_TIMEOUT,
+    stop_on_lock_deferral: bool = False,
 ) -> dict[str, Any]:
     # Strictly opt-in: callers that want the daemon auto-started must say so
     # explicitly (the CLI --daemon path passes auto_start=True). The default
@@ -1133,7 +1406,7 @@ def submit_job(
     job = client.submit(kind, payload, dedupe_key=dedupe_key, priority=priority)
     if not wait:
         return job
-    return client.wait(job["id"], timeout=timeout)
+    return client.wait(job["id"], timeout=timeout, stop_on_lock_deferral=stop_on_lock_deferral)
 
 
 def stop_daemon(palace_path: str) -> bool:

mempalace/date_window.py

@@ -0,0 +1,107 @@
+"""Shared date-window parsing for read-path filters.
+
+``list_drawers`` (#1128) and ``search`` (#463) accept the same optional
+``since``/``before`` bounds and compare them against drawer ``filed_at``
+metadata. The parsing and window semantics live here — a side-effect-free
+module — so ``searcher`` can use them without importing ``mcp_server``
+(whose import installs MCP stdio protection) and ``mcp_server`` keeps its
+existing behavior by aliasing these functions.
+
+Semantics (issue spec, #1128):
+* ``since`` is inclusive, ``before`` is exclusive: ``[since, before)``.
+* Comparison is wall-clock and timezone-naive. ``filed_at`` values are
+  written as naive local ISO strings (``datetime.now().isoformat()``)
+  almost everywhere; the one aware writer (``diary_ingest``, UTC) is
+  compared on its wall-clock fields after the offset is dropped.
+* A drawer whose ``filed_at`` is missing or unparseable is EXCLUDED
+  whenever a bound is active — a date-filtered result must never
+  silently include rows of unknown age.
+"""
+
+from datetime import datetime
+from typing import Optional
+
+
+def parse_date_bound(value: Optional[str] = None, field_name: str = "date") -> Optional[datetime]:
+    """Parse an optional ISO-8601 date/datetime filter bound.
+
+    Accepts a date (``"2026-04-01"``), a naive timestamp
+    (``"2026-04-01T09:30:00"``), or one carrying a ``Z``/``+HH:MM`` offset.
+    Returns a naive ``datetime`` for wall-clock
+    comparison against drawer ``filed_at`` values, which are stored as naive
+    local ISO strings (``datetime.now().isoformat()``). Any timezone offset on
+    the input is dropped so an aware bound never raises a ``TypeError`` against
+    a naive ``filed_at``. Comparison is therefore wall-clock, which is what the
+    local-first single-machine model wants; an offset bound is matched on its
+    wall-clock fields, not its absolute instant, so a bound whose offset differs
+    from the zone ``filed_at`` was recorded in is matched by clock time.
+    The accepted grammar is a date, an ISO timestamp (optionally fractional),
+    and an optional ``Z``/``±HH:MM`` offset; other ISO 8601 forms (basic format,
+    week dates) are outside the contract and are rejected on the Python 3.9 floor
+    even where a newer ``fromisoformat`` would accept them.
+    Blank / whitespace-only means "no filter" (``None``).
+    Raises ``ValueError`` on an unparseable value so the caller can surface a
+    clear error, mirroring the wing/room sanitizers.
+    """
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise ValueError(f"{field_name} must be an ISO date string")
+    value = value.strip()
+    if not value:
+        return None
+    # datetime.fromisoformat before Python 3.11 rejects a trailing "Z" (Zulu),
+    # and appending "+00:00" would break a date-only value on 3.9/3.10
+    # ("2026-04-01+00:00" is rejected there). Any offset is dropped below for
+    # wall-clock comparison anyway, so just strip a trailing Z/z; both date and
+    # date-time Zulu inputs then parse on the 3.9 floor.
+    iso = value[:-1] if value.endswith(("Z", "z")) else value
+    try:
+        parsed = datetime.fromisoformat(iso)
+    except ValueError as exc:
+        raise ValueError(
+            f"{field_name} must be an ISO date string "
+            f"(e.g. '2026-04-01' or '2026-04-01T09:30:00'), got {value!r}"
+        ) from exc
+    if parsed.tzinfo is not None:
+        parsed = parsed.replace(tzinfo=None)
+    return parsed
+
+
+def parse_window(since: Optional[str] = None, before: Optional[str] = None):
+    """Parse a ``[since, before)`` pair, rejecting an inverted window.
+
+    Returns ``(since_dt, before_dt)`` — either side ``None`` when absent.
+    Raises ``ValueError`` (naming the offending field or the inversion) so
+    callers surface the same message everywhere a window is accepted.
+    """
+    since_dt = parse_date_bound(since, "since")
+    before_dt = parse_date_bound(before, "before")
+    if since_dt is not None and before_dt is not None and since_dt >= before_dt:
+        raise ValueError(f"since ({since!r}) must be earlier than before ({before!r})")
+    return since_dt, before_dt
+
+
+def filed_at_in_window(
+    filed_at, since_dt: Optional[datetime], before_dt: Optional[datetime]
+) -> bool:
+    """True if a drawer's ``filed_at`` falls in ``[since, before)``.
+
+    ``since`` is inclusive and ``before`` is exclusive, matching the issue spec.
+    Parsing (``Z``/offset normalization, tz drop) is delegated to
+    ``parse_date_bound`` so a bound and a ``filed_at`` are compared
+    identically. A drawer whose ``filed_at`` is missing or unparseable cannot
+    be confirmed in-window, so it is EXCLUDED whenever a bound is active — a
+    date-filtered listing must never silently include rows of unknown age.
+    """
+    try:
+        filed_dt = parse_date_bound(filed_at, "filed_at")
+    except ValueError:
+        return False
+    if filed_dt is None:
+        return False
+    if since_dt is not None and filed_dt < since_dt:
+        return False
+    if before_dt is not None and filed_dt >= before_dt:
+        return False
+    return True

mempalace/dedup.py

@@ -50,12 +50,29 @@ def _get_palace_path():
         return os.path.join(os.path.expanduser("~"), ".mempalace", "palace")
 
 
-def get_source_groups(col, min_count=MIN_DRAWERS_TO_CHECK, source_pattern=None, wing=None):
+def get_source_groups(
+    col, min_count=MIN_DRAWERS_TO_CHECK, source_pattern=None, wing=None, palace_path=None
+):
     """Group drawers by source_file, return groups with min_count+ entries.
 
     If wing is specified, only considers drawers in that wing. This catches
     cross-wing duplicates when the same source was mined into multiple wings.
+
+    ``palace_path``, when passed, preflights HNSW divergence before count():
+    a diverged segment can hit the #1222 SIGSEGV/panic class, which a
+    try/except around count() cannot catch. Omitted by existing callers
+    that don't have a palace_path handy (e.g. tests) -- the check is simply
+    skipped in that case, matching this function's pre-existing behavior.
     """
+    if palace_path is not None:
+        from .backends.chroma import hnsw_capacity_status
+
+        capacity_info = hnsw_capacity_status(palace_path, COLLECTION_NAME)
+        if capacity_info.get("diverged"):
+            print(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+            print("  Run `mempalace repair --mode from-sqlite --archive-existing` first.")
+            return {}
+
     total = col.count()
     groups = defaultdict(list)
 
@@ -134,7 +151,7 @@ def show_stats(palace_path=None):
     palace_path = palace_path or _get_palace_path()
     col = get_collection(palace_path, COLLECTION_NAME)
 
-    groups = get_source_groups(col)
+    groups = get_source_groups(col, palace_path=palace_path)
 
     total_drawers = sum(len(ids) for ids in groups.values())
     print(f"\n  Sources with {MIN_DRAWERS_TO_CHECK}+ drawers: {len(groups)}")
@@ -166,15 +183,28 @@ def dedup_palace(
 
     col = get_collection(palace_path, COLLECTION_NAME)
 
+    # Preflight HNSW divergence before this function's own count() print --
+    # get_source_groups's palace_path guard (added alongside this one) only
+    # covers its own internal count(), not this earlier one. count() on a
+    # diverged segment can hard-crash the process (#1222); a try/except
+    # cannot catch that, so it must never be reached at all when diverged.
+    from .backends.chroma import hnsw_capacity_status
+
+    capacity_info = hnsw_capacity_status(palace_path, COLLECTION_NAME)
+    if capacity_info.get("diverged"):
+        print(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+        print("  Run `mempalace repair --mode from-sqlite --archive-existing` first.")
+        return
+
     print(f"  Palace: {palace_path}")
     print(f"  Drawers: {col.count():,}")
     print(f"  Threshold: {threshold}")
     print(f"  Mode: {'DRY RUN' if dry_run else 'LIVE'}")
-    print(f"{'─' * 55}")
+    print(f"{'-' * 55}")
 
     if wing:
         print(f"  Wing: {wing}")
-    groups = get_source_groups(col, min_count, source_pattern, wing=wing)
+    groups = get_source_groups(col, min_count, source_pattern, wing=wing, palace_path=palace_path)
     print(f"\n  Sources to check: {len(groups)}")
 
     t0 = time.time()
@@ -197,7 +227,7 @@ def dedup_palace(
 
     elapsed = time.time() - t0
 
-    print(f"\n{'─' * 55}")
+    print(f"\n{'-' * 55}")
     print(f"  Done in {elapsed:.1f}s")
     print(
         f"  Drawers: {total_kept + total_deleted:,} → {total_kept:,}  (-{total_deleted:,} removed)"

mempalace/dialect.py

@@ -357,8 +357,11 @@ class Dialect:
             "skip_names": ["Gandalf", "Sherlock"]
         }
         """
-        with open(config_path, "r") as f:
-            config = json.load(f)
+        try:
+            with open(config_path, "r", encoding="utf-8") as f:
+                config = json.load(f)
+        except UnicodeDecodeError as exc:
+            raise ValueError(f"{config_path} is not valid UTF-8 — re-save it as UTF-8") from exc
         return cls(
             entities=config.get("entities", {}),
             skip_names=config.get("skip_names", []),
@@ -381,7 +384,7 @@ class Dialect:
             "entities": canonical,
             "skip_names": self.skip_names,
         }
-        with open(config_path, "w") as f:
+        with open(config_path, "w", encoding="utf-8") as f:
             json.dump(config, f, indent=2)
 
     # === ENCODING (entity/emotion primitives) ===
@@ -776,11 +779,11 @@ class Dialect:
 
     def compress_file(self, zettel_json_path: str, output_path: str = None) -> str:
         """Read a zettel JSON file and compress it to AAAK Dialect."""
-        with open(zettel_json_path, "r") as f:
+        with open(zettel_json_path, "r", encoding="utf-8") as f:
             data = json.load(f)
         dialect = self.encode_file(data)
         if output_path:
-            with open(output_path, "w") as f:
+            with open(output_path, "w", encoding="utf-8") as f:
                 f.write(dialect)
         return dialect
 
@@ -790,14 +793,14 @@ class Dialect:
         for fname in sorted(os.listdir(zettel_dir)):
             if fname.endswith(".json"):
                 fpath = os.path.join(zettel_dir, fname)
-                with open(fpath, "r") as f:
+                with open(fpath, "r", encoding="utf-8") as f:
                     data = json.load(f)
                 dialect = self.encode_file(data)
                 all_dialect.append(dialect)
                 all_dialect.append("---")
         combined = "\n".join(all_dialect)
         if output_path:
-            with open(output_path, "w") as f:
+            with open(output_path, "w", encoding="utf-8") as f:
                 f.write(combined)
         return combined
 
@@ -824,7 +827,7 @@ class Dialect:
             if not fname.endswith(".json"):
                 continue
             fpath = os.path.join(zettel_dir, fname)
-            with open(fpath, "r") as f:
+            with open(fpath, "r", encoding="utf-8") as f:
                 data = json.load(f)
 
             file_num = fname.replace("file_", "").replace(".json", "")
@@ -846,7 +849,7 @@ class Dialect:
             if not fname.endswith(".json"):
                 continue
             fpath = os.path.join(zettel_dir, fname)
-            with open(fpath, "r") as f:
+            with open(fpath, "r", encoding="utf-8") as f:
                 data = json.load(f)
             for t in data.get("tunnels", []):
                 all_tunnels.append(t)
@@ -918,7 +921,7 @@ class Dialect:
         result = "\n".join(lines)
 
         if output_path:
-            with open(output_path, "w") as f:
+            with open(output_path, "w", encoding="utf-8") as f:
                 f.write(result)
 
         return result
@@ -1029,7 +1032,7 @@ if __name__ == "__main__":
             "skip_names": [],
         }
         out_path = "entities.json"
-        with open(out_path, "w") as f:
+        with open(out_path, "w", encoding="utf-8") as f:
             json.dump(example, f, indent=2)
         print(f"Created example config: {out_path}")
         print("Edit this file with your own entity mappings, then use --config entities.json")
@@ -1052,7 +1055,7 @@ if __name__ == "__main__":
         print(result)
 
     elif args[0] == "--stats":
-        with open(args[1], "r") as f:
+        with open(args[1], "r", encoding="utf-8") as f:
             data = json.load(f)
         json_str = json.dumps(data, indent=2)
         encoded = dialect.encode_file(data)
@@ -1060,7 +1063,7 @@ if __name__ == "__main__":
         print("=== COMPRESSION STATS ===")
         print(f"JSON:     ~{stats['original_tokens_est']:,} tokens (est)")
         print(f"AAAK:     ~{stats['summary_tokens_est']:,} tokens (est)")
-        print(f"Ratio:    {stats['size_ratio']}x (lossy — information is lost)")
+        print(f"Ratio:    {stats['size_ratio']}x (lossy -- information is lost)")
         print()
         print("=== AAAK DIALECT OUTPUT ===")
         print(encoded)

mempalace/embedding.py

@@ -1,10 +1,12 @@
 """Embedding function factory with hardware acceleration.
 
-Returns a ChromaDB-compatible embedding function bound to a user-selected
-ONNX Runtime execution provider.
+Returns a ChromaDB-compatible embedding function — either a local ONNX model
+bound to a user-selected ONNX Runtime execution provider, or an
+OpenAI-compatible HTTP ``/v1/embeddings`` endpoint.
 
-Two embedding models are available, selected via ``MEMPALACE_EMBEDDING_MODEL``
-or ``embedding_model`` in ``~/.mempalace/config.json``:
+Three embedding-model options are available, selected via
+``MEMPALACE_EMBEDDING_MODEL`` or ``embedding_model`` in
+``~/.mempalace/config.json``:
 
 * ``minilm`` (default) — ``all-MiniLM-L6-v2``, 384-dim, English-only training.
   ChromaDB's default; what every existing palace was built with.
@@ -15,6 +17,16 @@ or ``embedding_model`` in ``~/.mempalace/config.json``:
   model is lazy-downloaded from HuggingFace on first use. Switching models
   on an existing palace requires ``mempalace repair rebuild-index``
   (different vector space).
+* ``openai-compat`` — embeddings served by any OpenAI-compatible
+  ``/v1/embeddings`` endpoint (LM Studio, llama.cpp, vLLM, Ollama's OpenAI
+  shim, or a self-hosted server) instead of a local ONNX model. Useful for
+  larger / multilingual embedders (e.g. Qwen3-Embedding) or GPU offload.
+  Endpoint settings are read from ``config.json`` as ``embedding_api_url`` /
+  ``embedding_api_model`` / ``embedding_api_key`` (each overridable via the
+  matching ``MEMPALACE_EMBEDDING_API_*`` env var). Vectors are L2-normalized
+  for the cosine collection; the dimension is whatever the server returns, so
+  switching to/from this backend also requires ``mempalace repair
+  rebuild-index``. Stays local when the endpoint is on your machine/LAN.
 
 Supported devices (env ``MEMPALACE_EMBEDDING_DEVICE`` or ``embedding_device``
 in ``~/.mempalace/config.json``):
@@ -31,11 +43,14 @@ rather than hard-failing — mining must still work on a laptop without CUDA.
 
 from __future__ import annotations
 
+import hashlib
 import logging
 import os
 import threading
 from typing import Optional
 
+from .version import __version__
+
 logger = logging.getLogger(__name__)
 
 _PROVIDER_MAP = {
@@ -89,7 +104,7 @@ def _resolve_providers(device: str) -> tuple[list, str]:
     requested = _PROVIDER_MAP.get(device)
     if requested is None:
         if device not in _WARNED:
-            logger.warning("Unknown embedding_device %r — falling back to cpu", device)
+            logger.warning("Unknown embedding_device %r -- falling back to cpu", device)
             _WARNED.add(device)
         return (["CPUExecutionProvider"], "cpu")
 
@@ -216,10 +231,49 @@ _EMBEDDINGGEMMA_MAX_LEN = 2048
 # matches the internal batch size of chromadb's ONNXMiniLM_L6_V2, whose
 # chunked _forward survives the same call sites. embeddinggemma's
 # sentence_embedding output is attention-masked, so sub-batch padding
-# does not change any row's vector.
+# does not change any row's vector. __call__ decides which documents share
+# a sub-batch by size rather than by arrival order (#2104), because the run
+# is priced on that padded length and not on the document count.
 _EMBEDDINGGEMMA_BATCH_SIZE = 32
 
 
+def _sanitize_embeddinggemma_input_ids(tokenizer, input_ids, np):
+    """Replace tokenizer-only IDs that the text ONNX model cannot embed."""
+    model_vocab_size = tokenizer.get_vocab_size(with_added_tokens=False)
+    out_of_range = (input_ids < 0) | (input_ids >= model_vocab_size)
+
+    if not np.any(out_of_range):
+        return input_ids
+
+    unknown_token_id = tokenizer.token_to_id("<unk>")
+    if unknown_token_id is None or not 0 <= unknown_token_id < model_vocab_size:
+        raise RuntimeError(
+            "EmbeddingGemma tokenizer produced token IDs outside the ONNX "
+            "text vocabulary, but no valid <unk> token is available"
+        )
+
+    invalid_ids = sorted({int(token_id) for token_id in input_ids[out_of_range]})
+    warning_key = (
+        "embeddinggemma-out-of-range-token-ids",
+        model_vocab_size,
+        tuple(invalid_ids),
+    )
+
+    if warning_key not in _WARNED:
+        logger.warning(
+            "EmbeddingGemma tokenizer produced token IDs outside the ONNX "
+            "text vocabulary (size=%d): %s; remapping to <unk> (%d)",
+            model_vocab_size,
+            invalid_ids,
+            unknown_token_id,
+        )
+        _WARNED.add(warning_key)
+
+    sanitized = input_ids.copy()
+    sanitized[out_of_range] = unknown_token_id
+    return sanitized
+
+
 class EmbeddinggemmaONNX:
     """ChromaDB-compatible EF using embeddinggemma-300m ONNX (q8, MRL→384d).
 
@@ -319,6 +373,33 @@ class EmbeddinggemmaONNX:
             self._session = session
 
     def __call__(self, input: str | list[str] | None) -> list[list[float]]:  # noqa: A002 — ChromaDB EF protocol
+        """Embed ``input``, returning one vector per document in input order.
+
+        Documents are grouped by size before the sub-batch split. The
+        tokenizer pads every row of a sub-batch to the longest sequence in
+        it, and attention cost per layer is batch x heads x length^2, so one
+        long document drags a whole sub-batch up to its own length. Without
+        grouping the bill is set by arrival order: a verbatim transcript
+        whose long tool results sit between one-line replies pays the long
+        length for nearly every row (#2104).
+
+        An input that fits a single sub-batch is left in arrival order: every
+        row pads to the same width either way, so the keys would buy nothing
+        on the one-document search path.
+
+        Regrouping does not change what a row means. The model's
+        ``sentence_embedding`` output is attention-masked, so padding never
+        enters a row's values; what does move is float32 rounding, because a
+        different padded width changes the reduction order inside the GEMMs.
+        Measured against the same documents embedded in arrival order, that
+        residual peaks at one float32 ULP (1.2e-07 absolute, cosine
+        0.99999992).
+
+        The key is UTF-8 byte length rather than character count: this model
+        is multilingual, and bytes per token vary far less across scripts
+        than characters per token do. The sort is stable, so equal-size
+        documents keep arrival order and the split stays reproducible.
+        """
         if isinstance(input, str):
             # A bare string would be iterated character by character below,
             # silently producing one garbage vector per character.
@@ -330,16 +411,28 @@ class EmbeddinggemmaONNX:
             return []
         self._lazy_load()
         np = self._np
-        embeddings: list[list[float]] = []
-        # Tokenize and run per sub-batch, not over the whole input: padding
-        # is to the longest sequence in the sub-batch, and the ONNX runtime
-        # only ever holds batch_size rows of attention buffers at a time
-        # (#1770).
-        for start in range(0, len(input), self._batch_size):
-            chunk = input[start : start + self._batch_size]
-            texts = [_EMBEDDINGGEMMA_PREFIX + t for t in chunk]
+        # One sub-batch pads identically whatever the order, so the sort is
+        # only worth its keys once the input splits into several.
+        order: range | list[int] = range(len(input))
+        if len(input) > self._batch_size:
+            order = sorted(range(len(input)), key=lambda i: len(input[i].encode("utf-8")))
+        # Row i is filled by the sub-batch that carries document i. ``order``
+        # is a permutation of every index, so no placeholder survives; callers
+        # (ChromaDB included) zip the result against their ids positionally.
+        embeddings: list[list[float] | None] = [None] * len(input)
+        # Tokenize and run per sub-batch, not over the whole input: the ONNX
+        # runtime only ever holds batch_size rows of attention buffers at a
+        # time (#1770).
+        for start in range(0, len(order), self._batch_size):
+            idxs = order[start : start + self._batch_size]
+            texts = [_EMBEDDINGGEMMA_PREFIX + input[i] for i in idxs]
             encs = self._tokenizer.encode_batch(texts)
             input_ids = np.asarray([e.ids for e in encs], dtype=np.int64)
+            input_ids = _sanitize_embeddinggemma_input_ids(
+                self._tokenizer,
+                input_ids,
+                np,
+            )
             attention_mask = np.asarray([e.attention_mask for e in encs], dtype=np.int64)
             outputs = self._session.run(
                 None, {"input_ids": input_ids, "attention_mask": attention_mask}
@@ -348,7 +441,16 @@ class EmbeddinggemmaONNX:
             # L2-normalize so cosine similarity == dot product (matches what the
             # MTEB methodology assumes; ChromaDB's distance is configured for it).
             norms = np.linalg.norm(sent_emb, axis=1, keepdims=True) + 1e-12
-            embeddings.extend((sent_emb / norms).tolist())
+            rows = (sent_emb / norms).tolist()
+            if len(rows) != len(idxs):
+                # zip would truncate silently and leave a None in the result,
+                # which only surfaces far downstream in the caller's array
+                # conversion. Fail on the sub-batch that came back short.
+                raise RuntimeError(
+                    f"embeddinggemma returned {len(rows)} rows for a {len(idxs)}-document sub-batch"
+                )
+            for row_index, row in zip(idxs, rows):
+                embeddings[row_index] = row
         return embeddings
 
     def embed_query(self, input: list[str]) -> list[list[float]]:  # noqa: A002 — ChromaDB EF protocol
@@ -360,6 +462,165 @@ class EmbeddinggemmaONNX:
         return self(input)
 
 
+# ── OpenAI-compatible embedding API ──────────────────────────────────────
+# Fetch embeddings from an OpenAI-compatible ``/v1/embeddings`` server
+# (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or any compatible
+# endpoint) instead of running a model locally. Selected by
+# ``embedding_model == "openai-compat"``. Connection settings (URL, model,
+# optional key) are resolved by :class:`~mempalace.config.MempalaceConfig`
+# as the single source of truth — see ``embedding_api_url`` /
+# ``embedding_api_model`` / ``embedding_api_key`` (each env-overridable).
+_EF_API_BATCH = 64
+_EF_API_TIMEOUT = 120
+
+
+class EmbeddingAPIError(RuntimeError):
+    """Raised when the embedding API is unreachable or returns an invalid body.
+
+    Module-specific subclass mirroring ``llm_client.LLMError`` so callers can
+    distinguish embedding-endpoint failures; subclasses ``RuntimeError`` so
+    existing ``except RuntimeError`` paths still catch it.
+    """
+
+
+class OpenAICompatEmbeddingFunction:
+    """ChromaDB-compatible EF backed by an OpenAI-compatible ``/v1/embeddings``
+    endpoint (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, etc.).
+
+    Selected via ``embedding_model == "openai-compat"``. Vectors are produced
+    server-side and fetched over HTTP, which changes the vector space — so
+    ``name()`` encodes the model id: ChromaDB persists the EF name on the
+    collection and rejects mismatched reads, the signal to run ``mempalace
+    repair rebuild-index`` after changing model/endpoint. stdlib ``urllib``
+    only, no new dependency.
+    """
+
+    def __init__(self, base_url: str, model: str, api_key: Optional[str] = None):
+        self._url = self._resolve_url(base_url)
+        self._model = model
+        self._api_key = api_key
+
+    @staticmethod
+    def _resolve_url(base_url: str) -> str:
+        """Accept a base host, a ``/v1`` base, or a full endpoint URL.
+
+        Mirrors ``llm_client.OpenAICompatProvider._resolve_url`` so both sides
+        treat an ``http://host:port`` endpoint the same way.
+        """
+        url = base_url.rstrip("/")
+        if url.endswith("/embeddings"):
+            return url
+        if url.endswith("/v1"):
+            return f"{url}/embeddings"
+        return f"{url}/v1/embeddings"
+
+    def name(self) -> str:
+        # Encode the model so switching it changes the persisted EF identity
+        # and forces a rebuild_index (vectors from a different model/space are
+        # not interchangeable). ChromaDB compares this on every read.
+        return f"openai_compat_emb_{self._model}".replace("/", "_")
+
+    def embed_query(self, input):  # noqa: A002 — ChromaDB EF protocol uses `input`
+        # ChromaDB 1.5 dispatches query embedding through embed_query (add uses
+        # __call__). Mirror the EmbeddingFunction protocol default: same path.
+        return self(input)
+
+    def __call__(self, input):  # noqa: A002 — ChromaDB EF protocol uses `input`
+        import http.client
+        import json
+        from urllib.error import HTTPError, URLError
+        from urllib.request import Request, urlopen
+
+        headers = {
+            "Content-Type": "application/json",
+            # Some hosted (Cloudflare-fronted) endpoints 403 the default
+            # ``Python-urllib`` User-Agent — send our own (see issue #1570).
+            "User-Agent": f"mempalace/{__version__}",
+        }
+        if self._api_key:
+            headers["Authorization"] = f"Bearer {self._api_key}"
+
+        out: list = []
+        texts = list(input)
+        for start in range(0, len(texts), _EF_API_BATCH):
+            batch = texts[start : start + _EF_API_BATCH]
+            # encoding_format=float is explicit so a server that defaults to
+            # base64 doesn't hand back strings we'd mis-parse as vectors.
+            payload = {"model": self._model, "input": batch, "encoding_format": "float"}
+            req = Request(self._url, data=json.dumps(payload).encode("utf-8"), headers=headers)
+            try:
+                with urlopen(req, timeout=_EF_API_TIMEOUT) as resp:
+                    data = json.loads(resp.read())
+            # ValueError covers an invalid/missing URL scheme and json.JSONDecodeError;
+            # http.client.HTTPException covers low-level protocol faults (BadStatusLine,
+            # IncompleteRead) common with local/overloaded servers.
+            except (HTTPError, URLError, OSError, http.client.HTTPException, ValueError) as e:
+                raise EmbeddingAPIError(
+                    f"Embedding API request to {self._url} failed: {e}. Check that the "
+                    f"server is reachable and MEMPALACE_EMBEDDING_API_URL / embedding_api_url "
+                    f"is correct."
+                ) from e
+            out.extend(self._vectors_from_response(data, len(batch)))
+        return out
+
+    def _vectors_from_response(self, data, n: int) -> list:
+        """Validate one ``/v1/embeddings`` response and return L2-normed vectors.
+
+        Guards every way a non-conformant server could corrupt the store
+        silently: a missing/short ``data`` array, response ``index`` values
+        that aren't the contiguous ``0..n-1`` batch positions (sorting then
+        zipping positionally would otherwise misalign vectors with texts), and
+        malformed / ragged / base64 embedding payloads. All failures raise
+        :class:`EmbeddingAPIError` naming the endpoint rather than a cryptic
+        numpy error — a silent wrong result would break the 100%-recall promise.
+        """
+        import numpy as np
+
+        if not isinstance(data, dict):
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned a non-object response: {data}"
+            )
+        rows = data.get("data")
+        if not isinstance(rows, list):
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned no 'data' array: {data.get('error', data)}"
+            )
+        if len(rows) != n:
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned {len(rows)} embeddings for {n} inputs"
+            )
+        # The endpoint may return rows out of order — sort by index, then
+        # require the indices to be exactly 0..n-1 so positional alignment is
+        # provably correct (a server using absolute or duplicate indices would
+        # otherwise pass the count check yet map vectors to the wrong texts).
+        try:
+            rows = sorted(rows, key=lambda d: d.get("index", -1))
+            indices = [r.get("index") for r in rows]
+        except AttributeError as e:
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned non-object rows: {e}"
+            ) from e
+        if indices != list(range(n)):
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned non-contiguous or duplicate "
+                f"'index' values; cannot align embeddings with inputs"
+            )
+        try:
+            arr = np.asarray([r["embedding"] for r in rows], dtype=np.float32)
+        except (KeyError, TypeError, ValueError) as e:
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned malformed embeddings: {e}"
+            ) from e
+        if arr.ndim != 2:
+            raise EmbeddingAPIError(
+                f"Embedding API at {self._url} returned non-vector embeddings (shape {arr.shape})"
+            )
+        # L2-normalize so cosine == dot product (collection uses
+        # hnsw:space=cosine), matching EmbeddinggemmaONNX above.
+        norms = np.linalg.norm(arr, axis=1, keepdims=True) + 1e-12
+        return (arr / norms).tolist()
+
+
 def get_embedding_function(device: Optional[str] = None, model: Optional[str] = None):
     """Return a cached embedding function for the requested device + model.
 
@@ -377,6 +638,41 @@ def get_embedding_function(device: Optional[str] = None, model: Optional[str] =
         if model is None:
             model = cfg.embedding_model
 
+    # OpenAI-compatible embedding API: bypasses local ONNX entirely. Checked
+    # before device→provider resolution since it needs no hardware accelerator.
+    if model == "openai-compat":
+        from .config import MempalaceConfig
+
+        cfg = MempalaceConfig()
+        url = cfg.embedding_api_url
+        if not url:
+            raise ValueError(
+                "embedding_model='openai-compat' requires an endpoint — set "
+                "embedding_api_url in ~/.mempalace/config.json or the "
+                "MEMPALACE_EMBEDDING_API_URL env var (e.g. http://host:port)"
+            )
+        api_model = cfg.embedding_api_model
+        if not api_model:
+            raise ValueError(
+                "embedding_model='openai-compat' requires a model — set "
+                "embedding_api_model in ~/.mempalace/config.json or the "
+                "MEMPALACE_EMBEDDING_API_MODEL env var"
+            )
+        api_key = cfg.embedding_api_key
+        # Include a fingerprint of the key (never the raw secret) so a token
+        # rotation busts the cache in long-lived processes (e.g. MCP server).
+        key_fp = hashlib.sha256((api_key or "").encode("utf-8")).hexdigest()[:16]
+        cache_key = ("openai-compat", url, api_model, key_fp)
+        cached = _EF_CACHE.get(cache_key)
+        if cached is not None:
+            return cached
+        ef = OpenAICompatEmbeddingFunction(base_url=url, model=api_model, api_key=api_key)
+        _EF_CACHE[cache_key] = ef
+        logger.info(
+            "Embedding function initialized (openai-compat url=%s model=%s)", url, api_model
+        )
+        return ef
+
     providers, effective = _resolve_providers(device)
     cache_key = (model, tuple(providers))
     cached = _EF_CACHE.get(cache_key)  # lock-free fast path; dict.get is GIL-atomic
@@ -406,15 +702,21 @@ def get_embedding_function(device: Optional[str] = None, model: Optional[str] =
 
 
 def describe_device(device: Optional[str] = None) -> str:
-    """Return a short human-readable label for the resolved device.
+    """Return a short human-readable label for the resolved embedding backend.
 
-    Used by the miner CLI header so users can see at a glance whether GPU
-    acceleration actually engaged.
+    Used by the miner CLI header / MCP status so users can see at a glance
+    whether GPU acceleration engaged — or, for the ``openai-compat`` backend,
+    that embeddings are served by a remote endpoint rather than local hardware
+    (in which case the ``embedding_device`` accelerator label is irrelevant).
     """
     if device is None:
         from .config import MempalaceConfig
 
-        device = MempalaceConfig().embedding_device
+        cfg = MempalaceConfig()
+        if cfg.embedding_model == "openai-compat":
+            url = cfg.embedding_api_url
+            return f"openai-compat ({url})" if url else "openai-compat"
+        device = cfg.embedding_device
     _, effective = _resolve_providers(device)
     return effective
 

mempalace/encoding_repair.py

@@ -0,0 +1,624 @@
+"""Safely repair high-confidence UTF-8 mojibake in MemPalace drawers."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import unicodedata
+from pathlib import Path
+from typing import Callable, Iterator, Optional, TextIO, Union
+
+_BACKUP_FORMAT = "mempalace-encoding-repair"
+_BACKUP_VERSION = 1
+_UNDEFINED_CP1252_BYTES = frozenset(
+    {
+        0x81,
+        0x8D,
+        0x8F,
+        0x90,
+        0x9D,
+    }
+)
+
+
+def _cp1252_character(
+    byte_value: int,
+) -> str:
+    """Map a legacy byte to the character stored in old palaces."""
+    if byte_value in _UNDEFINED_CP1252_BYTES:
+        # Pre-3.1 Windows reads could preserve these five undefined
+        # Windows-1252 byte values as their corresponding invisible
+        # C1 control code points.
+        return chr(byte_value)
+
+    return bytes([byte_value]).decode("cp1252")
+
+
+def _encode_mojibake_candidate(
+    text: str,
+) -> bytes:
+    """Recover original bytes, including undefined CP1252 values."""
+    raw = bytearray()
+
+    for character in text:
+        codepoint = ord(character)
+
+        if codepoint in _UNDEFINED_CP1252_BYTES:
+            raw.append(codepoint)
+        else:
+            raw.extend(character.encode("cp1252"))
+
+    return bytes(raw)
+
+
+_CONTINUATION_CHARS = "".join(
+    _cp1252_character(byte_value)
+    for byte_value in range(
+        0x80,
+        0xC0,
+    )
+)
+_CONTINUATION_CLASS = re.escape(_CONTINUATION_CHARS)
+
+# These are the characteristic visible lead characters produced by
+# common UTF-8-as-Windows-1252 corruption:
+#
+# C2/C3 -> Â/Ã
+# E2    -> â
+# F0    -> ð
+# EF    -> ï
+#
+# C4/C5 -> Ä/Å are deliberately excluded because strings such as Ų
+# can be legitimate scientific text.
+_HIGH_CONFIDENCE_RUN = re.compile(
+    rf"(?:"
+    rf"[ÂÃ][{_CONTINUATION_CLASS}]"
+    rf"|â[{_CONTINUATION_CLASS}]{{2}}"
+    rf"|ð[{_CONTINUATION_CLASS}]{{3}}"
+    rf"|ï[{_CONTINUATION_CLASS}]{{2}}"
+    rf")+"
+)
+
+# A single mojibake unit, so a chained run can be told apart from a lone
+# two-character window.
+_MOJIBAKE_UNIT = re.compile(
+    rf"[ÂÃ][{_CONTINUATION_CLASS}]"
+    rf"|â[{_CONTINUATION_CLASS}]{{2}}"
+    rf"|ð[{_CONTINUATION_CLASS}]{{3}}"
+    rf"|ï[{_CONTINUATION_CLASS}]{{2}}"
+)
+
+# Continuation bytes whose CP1252 character is typographic punctuation that
+# legitimately follows a letter in prose.
+#
+# The [ÂÃ] alternative above is only TWO characters wide, and Portuguese,
+# Vietnamese and Turkish really do end all-caps words in Ã/Â — IRMÃ, MAÇÃ,
+# MANHÃ, AMANHÃ, BÃO, NHÃ, HÂLÂ, IMÂ. Prose then puts a closing quote,
+# guillemet, ellipsis or dash straight after, and every one of those lives in
+# the continuation class. "«MAÇû." and "“IRMÔ" therefore match a shape that is
+# also perfectly clean text, and repairing them collapses two characters into
+# one — silently, and reported as a success.
+#
+# This is the same argument that already excluded Ä/Å as lead characters, applied
+# to the continuation side. The wider â/ð/ï windows are 3-4 characters and no
+# clean text produces those shapes, so they keep the full class.
+_PROSE_PUNCTUATION = frozenset(
+    _cp1252_character(byte_value)
+    for byte_value in (
+        0x82,  # ‚ single low-9 quotation mark
+        0x84,  # „ double low-9 quotation mark
+        0x85,  # … horizontal ellipsis
+        0x8B,  # ‹ single left-pointing angle quotation mark
+        0x91,  # ' left single quotation mark
+        0x92,  # ' right single quotation mark
+        0x93,  # " left double quotation mark
+        0x94,  # " right double quotation mark
+        0x95,  # • bullet
+        0x96,  # – en dash
+        0x97,  # — em dash
+        0x9B,  # › single right-pointing angle quotation mark
+        0xAB,  # « left-pointing double angle quotation mark
+        0xBB,  # » right-pointing double angle quotation mark
+    )
+)
+
+
+def _is_text_character(
+    character: str,
+) -> bool:
+    """Reject decode products that are invisible control characters.
+
+    Â followed by 0x80-0x9F decodes to exactly the C1 control block, so
+    "İMÂ… edildi." becomes "İM\\u0085 edildi." — visible text swapped for an
+    invisible control and reported as a repair. Natural text contains no C1
+    controls, so refusing these costs nothing.
+
+    Cf (format) stays allowed: U+FEFF is the legitimate product of the ""
+    BOM run and U+200D joins emoji sequences.
+    """
+    if character in "\t\n\r":
+        return True
+
+    return unicodedata.category(character) not in {
+        "Cc",
+        "Cs",
+        "Cn",
+    }
+
+
+def _is_ambiguous_window(
+    candidate: str,
+) -> bool:
+    """True for the one shape clean prose can also produce.
+
+    A lone [ÂÃ] followed by prose punctuation. Chained units ("ação") and the
+    3-4 character â/ð/ï windows cannot occur in clean text, so they stay
+    unconditional.
+    """
+    units = _MOJIBAKE_UNIT.findall(candidate) or [candidate]
+
+    if len(units) > 1 or units[0][0] not in "ÂÃ":
+        return False
+
+    return units[0][1] in _PROSE_PUNCTUATION
+
+
+def _preceded_by_lowercase(
+    text: str,
+    start: int,
+) -> bool:
+    """Evidence that the run is mojibake rather than clean prose.
+
+    Mojibaked letters sit INSIDE a word, so a lowercase letter runs straight into
+    the lead: "coûte", "Noël", "català»". The false-positive family is the
+    exact opposite — an ALL-CAPS word whose final letter genuinely is Ã/Â, as in
+    IRMÔ, MAÇû, MANHÃ…, HÂL», IM». Clean prose does not put a lowercase
+    letter directly before an uppercase Ã/Â.
+
+    Deliberately local. Inferring "this drawer is mojibake" from a run elsewhere
+    in the text destroys clean prose in a MIXED drawer, and drawers are mixed by
+    construction because the miner concatenates several sources into one.
+    """
+    return start > 0 and text[start - 1].isalpha() and text[start - 1].islower()
+
+
+def _decode_high_confidence_run(
+    match: re.Match,
+    text: str,
+) -> str:
+    candidate = match.group(0)
+
+    try:
+        decoded = _encode_mojibake_candidate(candidate).decode("utf-8")
+    except (
+        UnicodeEncodeError,
+        UnicodeDecodeError,
+    ):
+        return candidate
+
+    if not decoded or not all(_is_text_character(character) for character in decoded):
+        return candidate
+
+    # An ambiguous window is repaired only on positive local evidence. With no
+    # evidence the text is returned untouched: a missed repair costs nothing
+    # because the drawer is unchanged and the tool can be re-run, while a wrong
+    # repair destroys good text irrecoverably.
+    if _is_ambiguous_window(candidate) and not _preceded_by_lowercase(text, match.start()):
+        return candidate
+
+    return decoded
+
+
+def repair_mojibake_once(text: str) -> str:
+    """Repair one layer of high-confidence UTF-8-as-CP1252 mojibake."""
+    return _HIGH_CONFIDENCE_RUN.sub(
+        lambda match: _decode_high_confidence_run(match, text),
+        text,
+    )
+
+
+def repair_mojibake(
+    text: str,
+    *,
+    max_passes: int = 3,
+) -> str:
+    """Repair repeated high-confidence mojibake layers until stable."""
+    if max_passes < 1:
+        raise ValueError("max_passes must be at least 1")
+
+    current = text
+
+    for _ in range(max_passes):
+        repaired = repair_mojibake_once(current)
+
+        if repaired == current:
+            break
+
+        current = repaired
+
+    return current
+
+
+def _result_field(result, name: str):
+    if isinstance(result, dict):
+        return result.get(name)
+
+    return getattr(result, name, None)
+
+
+def _collection_name(
+    collection,
+) -> Optional[str]:
+    """Resolve a collection name through MemPalace backend wrappers."""
+
+    def resolve_name(candidate) -> Optional[str]:
+        name = getattr(
+            candidate,
+            "name",
+            None,
+        )
+
+        if callable(name):
+            try:
+                name = name()
+            except TypeError:
+                name = None
+
+        return str(name) if name else None
+
+    direct_name = resolve_name(collection)
+    if direct_name:
+        return direct_name
+
+    # ChromaCollection already provides this resolver for its wrapped
+    # chromadb collection.
+    resolver = getattr(
+        collection,
+        "_collection_name",
+        None,
+    )
+
+    if callable(resolver):
+        try:
+            resolved = resolver()
+        except (
+            AttributeError,
+            TypeError,
+        ):
+            resolved = None
+
+        if resolved:
+            return str(resolved)
+
+    # Defensive fallback for thin wrappers that expose only their inner
+    # collection object.
+    inner = getattr(
+        collection,
+        "_collection",
+        None,
+    )
+
+    if inner is not None and inner is not collection:
+        return resolve_name(inner)
+
+    return None
+
+
+def _open_private_backup(
+    path: Path,
+) -> TextIO:
+    path.parent.mkdir(
+        parents=True,
+        exist_ok=True,
+    )
+
+    descriptor = os.open(
+        str(path),
+        os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+        0o600,
+    )
+
+    try:
+        try:
+            os.chmod(path, 0o600)
+        except (
+            OSError,
+            NotImplementedError,
+        ):
+            pass
+
+        return os.fdopen(
+            descriptor,
+            "w",
+            encoding="utf-8",
+            newline="\n",
+        )
+    except Exception:
+        os.close(descriptor)
+        raise
+
+
+def _write_json_line(
+    handle: TextIO,
+    value: dict,
+) -> None:
+    handle.write(
+        json.dumps(
+            value,
+            ensure_ascii=True,
+            sort_keys=True,
+        )
+        + "\n"
+    )
+
+
+def _write_backup_header(
+    handle: TextIO,
+    collection,
+) -> None:
+    _write_json_line(
+        handle,
+        {
+            "format": _BACKUP_FORMAT,
+            "version": _BACKUP_VERSION,
+            "collection": _collection_name(collection),
+        },
+    )
+
+
+def _read_backup_header(
+    path: Path,
+) -> dict:
+    try:
+        with path.open(
+            "r",
+            encoding="utf-8",
+        ) as handle:
+            header = json.loads(handle.readline())
+    except OSError as exc:
+        raise ValueError(f"could not read repair backup: {path}") from exc
+    except (
+        json.JSONDecodeError,
+        TypeError,
+    ) as exc:
+        raise ValueError("repair backup has an invalid header") from exc
+
+    if not isinstance(
+        header,
+        dict,
+    ) or (header.get("format") != _BACKUP_FORMAT or header.get("version") != _BACKUP_VERSION):
+        raise ValueError("unsupported repair backup format")
+
+    return header
+
+
+def _iter_backup_records(
+    path: Path,
+) -> Iterator[tuple[str, str]]:
+    _read_backup_header(path)
+
+    with path.open(
+        "r",
+        encoding="utf-8",
+    ) as handle:
+        # Skip the validated header.
+        handle.readline()
+
+        for line_number, line in enumerate(
+            handle,
+            start=2,
+        ):
+            if not line.strip():
+                continue
+
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"invalid backup JSON at line {line_number}") from exc
+
+            drawer_id = record.get("id") if isinstance(record, dict) else None
+            document = record.get("original_document") if isinstance(record, dict) else None
+
+            if not isinstance(
+                drawer_id,
+                str,
+            ) or not isinstance(
+                document,
+                str,
+            ):
+                raise ValueError(f"invalid repair backup record at line {line_number}")
+
+            yield drawer_id, document
+
+
+def repair_collection(
+    collection,
+    *,
+    apply: bool = False,
+    page_size: int = 500,
+    backup_path: Optional[Union[str, Path]] = None,
+    on_change: Optional[Callable[[str, str, str], None]] = None,
+) -> dict:
+    """Scan a collection and optionally repair high-confidence mojibake."""
+    if page_size < 1:
+        raise ValueError("page_size must be at least 1")
+
+    if apply and backup_path is None:
+        raise ValueError("backup_path is required when apply=True")
+
+    backup = Path(backup_path) if backup_path is not None else None
+    backup_handle: Optional[TextIO] = None
+    backup_used: Optional[str] = None
+
+    scanned = 0
+    changed = 0
+    updated = 0
+    offset = 0
+
+    try:
+        while True:
+            page = collection.get(
+                limit=page_size,
+                offset=offset,
+                include=["documents"],
+            )
+            ids = list(
+                _result_field(
+                    page,
+                    "ids",
+                )
+                or []
+            )
+            documents = list(
+                _result_field(
+                    page,
+                    "documents",
+                )
+                or []
+            )
+
+            if len(ids) != len(documents):
+                raise RuntimeError("collection returned misaligned ids and documents")
+
+            if not ids:
+                break
+
+            page_changes = []
+
+            for drawer_id, document in zip(
+                ids,
+                documents,
+            ):
+                scanned += 1
+
+                if not isinstance(
+                    document,
+                    str,
+                ):
+                    continue
+
+                repaired = repair_mojibake(document)
+
+                if repaired == document:
+                    continue
+
+                item = (
+                    str(drawer_id),
+                    document,
+                    repaired,
+                )
+                page_changes.append(item)
+                changed += 1
+
+                if on_change is not None:
+                    on_change(*item)
+
+            if apply and page_changes:
+                if backup_handle is None:
+                    assert backup is not None
+
+                    backup_handle = _open_private_backup(backup)
+                    _write_backup_header(
+                        backup_handle,
+                        collection,
+                    )
+                    backup_used = str(backup)
+
+                for (
+                    drawer_id,
+                    original,
+                    _repaired,
+                ) in page_changes:
+                    _write_json_line(
+                        backup_handle,
+                        {
+                            "id": drawer_id,
+                            "original_document": (original),
+                        },
+                    )
+
+                # Originals must reach durable storage before their
+                # live collection rows are overwritten.
+                backup_handle.flush()
+                os.fsync(backup_handle.fileno())
+
+                collection.update(
+                    ids=[item[0] for item in page_changes],
+                    documents=[item[2] for item in page_changes],
+                )
+                updated += len(page_changes)
+
+            offset += len(ids)
+
+            if len(ids) < page_size:
+                break
+    finally:
+        if backup_handle is not None:
+            backup_handle.close()
+
+    return {
+        "scanned": scanned,
+        "changed": changed,
+        "updated": updated,
+        "backup_path": backup_used,
+    }
+
+
+def restore_collection(
+    collection,
+    backup_path: Union[str, Path],
+    *,
+    batch_size: int = 500,
+) -> dict:
+    """Restore original documents from an encoding-repair backup."""
+    if batch_size < 1:
+        raise ValueError("batch_size must be at least 1")
+
+    path = Path(backup_path)
+    header = _read_backup_header(path)
+
+    backup_collection = header.get("collection")
+    target_collection = _collection_name(collection)
+
+    if backup_collection and target_collection and backup_collection != target_collection:
+        raise ValueError(
+            f"backup belongs to collection {backup_collection!r}, not {target_collection!r}"
+        )
+
+    # Validate the complete file before performing the first restore write.
+    validated = sum(1 for _record in _iter_backup_records(path))
+
+    restored = 0
+    batch_ids = []
+    batch_documents = []
+
+    for (
+        drawer_id,
+        document,
+    ) in _iter_backup_records(path):
+        batch_ids.append(drawer_id)
+        batch_documents.append(document)
+
+        if len(batch_ids) < batch_size:
+            continue
+
+        collection.update(
+            ids=batch_ids,
+            documents=batch_documents,
+        )
+        restored += len(batch_ids)
+        batch_ids = []
+        batch_documents = []
+
+    if batch_ids:
+        collection.update(
+            ids=batch_ids,
+            documents=batch_documents,
+        )
+        restored += len(batch_ids)
+
+    return {
+        "validated": validated,
+        "restored": restored,
+    }

mempalace/entity_detector.py

@@ -272,6 +272,34 @@ SKIP_FILENAMES = {
 # ==================== CANDIDATE EXTRACTION ====================
 
 
+# Entity-candidate matching is ReDoS-prone: the English candidate pattern's
+# nested alternation (i18n/en.json) backtracks catastrophically on a long
+# unbroken run of printable ASCII (base64, minified JS, hashes, data URIs),
+# pinning a mine on one ~5000-char window for hours (#2063). Such a run is never
+# a name, so it is collapsed to a space before single-word matching. Scope:
+#   * ASCII-only ([!-~]): non-ASCII scripts (CJK, Cyrillic, Devanagari,
+#     accented Latin) are deliberately left untouched — a CJK paragraph is one
+#     unbroken run with no ASCII whitespace, and those locales' candidate
+#     patterns are bounded and never backtrack, so collapsing them would
+#     silently destroy their entity detection.
+#   * The threshold sits above the 20-char cap of the simple-name pattern
+#     ([A-Z][a-z]{1,19}), so no real single-word name is dropped. A CamelCase
+#     code identifier glued inside a long ASCII run (a path/URL/dotted name) is
+#     dropped as noise; whitespace-delimited proper nouns are unaffected.
+_MAX_CANDIDATE_TOKEN_LEN = 24
+_LONG_ASCII_RUN_RX = re.compile(rf"[!-~]{{{_MAX_CANDIDATE_TOKEN_LEN},}}")
+
+
+def _collapse_long_ascii_runs(text: str) -> str:
+    """Collapse a long unbroken run of printable ASCII to a single space (#2063).
+
+    Applied only before single-word candidate matching — the whitespace-delimited
+    multi-word patterns cannot backtrack on such runs. See ``_LONG_ASCII_RUN_RX``
+    for scope and rationale.
+    """
+    return _LONG_ASCII_RUN_RX.sub(" ", text)
+
+
 def extract_candidates(text: str, languages=("en",)) -> dict:
     """
     Extract all capitalized proper noun candidates from text.
@@ -296,13 +324,15 @@ def extract_candidates(text: str, languages=("en",)) -> dict:
     for compound, n in compound_counts.items():
         counts[compound] += n
 
-    # Single-word candidates — one pre-wrapped pattern per language
+    # Single-word candidates — one pre-wrapped pattern per language.
+    # Collapse long ASCII blobs first (base64/minified) to defuse ReDoS (#2063).
+    candidate_text = _collapse_long_ascii_runs(working_text)
     for wrapped_pat in patterns["candidate_patterns"]:
         try:
             rx = re.compile(wrapped_pat)
         except re.error:
             continue
-        for word in rx.findall(working_text):
+        for word in rx.findall(candidate_text):
             wl = word.lower()
             if wl in stopwords:
                 continue
@@ -718,7 +748,7 @@ def _print_entity_list(entities: list, label: str):
         print("    (none detected)")
         return
     for i, e in enumerate(entities):
-        confidence_bar = "●" * int(e["confidence"] * 5) + "○" * (5 - int(e["confidence"] * 5))
+        confidence_bar = "#" * int(e["confidence"] * 5) + "." * (5 - int(e["confidence"] * 5))
         signals_str = ", ".join(e["signals"][:2]) if e["signals"] else ""
         print(f"    {i + 1:2}. {e['name']:20} [{confidence_bar}] {signals_str}")
 
@@ -738,7 +768,7 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     Pass yes=True to auto-accept all detected entities without prompting.
     """
     print(f"\n{'=' * 58}")
-    print("  MemPalace — Entity Detection")
+    print("  MemPalace -- Entity Detection")
     print(f"{'=' * 58}")
     print("\n  Scanned your files. Here's what we found:\n")
 
@@ -768,7 +798,7 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
             "topics": confirmed_topics,
         }
 
-    print(f"\n{'─' * 58}")
+    print(f"\n{'-' * 58}")
     print("  Options:")
     print("    [enter]  Accept all")
     print("    [edit]   Remove wrong entries or reclassify uncertain")
@@ -783,9 +813,9 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     if choice == "edit":
         # Handle uncertain first
         if detected["uncertain"]:
-            print("\n  Uncertain entities — classify each:")
+            print("\n  Uncertain entities -- classify each:")
             for e in detected["uncertain"]:
-                ans = input(f"    {e['name']} — (p)erson, (r)project, or (s)kip? ").strip().lower()
+                ans = input(f"    {e['name']} -- (p)erson, (r)project, or (s)kip? ").strip().lower()
                 if ans == "p":
                     confirmed_people.append(e["name"])
                 elif ans == "r":

mempalace/exporter.py

@@ -84,7 +84,7 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -
     total = col.count()
 
     if total == 0:
-        print("  Palace is empty — nothing to export.")
+        print("  Palace is empty -- nothing to export.")
         return {"wings": 0, "rooms": 0, "drawers": 0}
 
     _reject_symlink(output_dir, "output_dir")

mempalace/format_miner.py

@@ -362,7 +362,7 @@ def extract_text(
         logger.info("skip:unreadable (file gone after scan) %s", p)
         return None, ExtractionStatus.SKIP_UNREADABLE
     except OSError as exc:
-        logger.info("skip:unreadable %s — %s", p, exc)
+        logger.info("skip:unreadable %s -- %s", p, exc)
         return None, ExtractionStatus.SKIP_UNREADABLE
 
     # Fringe Case 5 — empty file. Skip silently.
@@ -433,9 +433,9 @@ def extract_text(
         # Fringe Case 4 vs Case 10: encrypted vs generic crash, by message.
         msg = str(exc)
         if _ENCRYPTED_PATTERNS.search(msg):
-            logger.info("skip:encrypted %s — %s", p, msg[:120])
+            logger.info("skip:encrypted %s -- %s", p, msg[:120])
             return None, ExtractionStatus.SKIP_ENCRYPTED
-        logger.warning("skip:extraction_error %s — %s: %s", p, type(exc).__name__, msg[:200])
+        logger.warning("skip:extraction_error %s -- %s: %s", p, type(exc).__name__, msg[:200])
         return None, ExtractionStatus.SKIP_EXTRACTION_ERROR
 
     # Either transformer can legitimately return None / empty (malformed
@@ -443,7 +443,7 @@ def extract_text(
     # so the caller knows to skip rather than file an empty drawer.
     if not text:
         transformer = "striprtf" if is_rtf else "markitdown"
-        logger.info("skip:extraction_error %s — %s returned None/empty", p, transformer)
+        logger.info("skip:extraction_error %s -- %s returned None/empty", p, transformer)
         return None, ExtractionStatus.SKIP_EXTRACTION_ERROR
 
     return text, ExtractionStatus.OK
@@ -803,7 +803,7 @@ def mine_formats(
         files = scan_formats(format_path)
 
         print(f"\n{'=' * 55}")
-        print("  MemPalace Mine — Format extraction")
+        print("  MemPalace Mine -- Format extraction")
         print(f"{'=' * 55}")
         print(f"  Wing:    {wing}")
         print(f"  Source:  {format_path}")
@@ -811,7 +811,7 @@ def mine_formats(
         print(f"  Files:   {len(files)}{limit_suffix}")
         print(f"  Palace:  {palace_path}")
         if dry_run:
-            print("  DRY RUN — nothing will be filed")
+            print("  DRY RUN -- nothing will be filed")
         print(f"{'-' * 55}\n")
 
         collection = get_collection(palace_path) if not dry_run else None
@@ -876,7 +876,7 @@ def mine_formats(
                 files_with_text += 1
 
                 if dry_run:
-                    print(f"    [DRY RUN] {filepath.name} → {len(chunks)} drawers")
+                    print(f"    [DRY RUN] {filepath.name} -> {len(chunks)} drawers")
                     total_drawers += len(chunks)
                     files_mined += 1
                     if limit > 0 and files_mined >= limit:

mempalace/general_extractor.py

@@ -159,7 +159,6 @@ EMOTION_MARKERS = [
     r"i need",
     r"never told anyone",
     r"nobody knows",
-    r"\*[^*]+\*",
 ]
 
 ALL_MARKERS = {

mempalace/hlc.py

@@ -0,0 +1,94 @@
+"""
+hlc.py — Hybrid Logical Clock for RFC 004 op ordering
+
+Total order across replicas without trusting wall clocks: physical
+milliseconds + a logical counter + the replica id as final tiebreak.
+Rendered as a fixed-width, lexicographically sortable string so SQLite TEXT
+comparison IS the causal comparison:
+
+    <unix_ms:13 decimal digits>-<counter:6 hex digits>-<replica_id>
+
+e.g. ``1783038849123-000000-rep_ab12cd34ef56``.
+
+Semantics (standard HLC):
+- ``tick()`` for a local op: physical time if it advanced, else previous
+  ms with counter+1 — never goes backwards, even against clock regression.
+- ``observe(remote)`` on receiving a remote op: the clock absorbs the
+  remote's instant so later local ops sort after everything already seen.
+
+Cursor semantics stay LOCAL (rowid arrival order) per RFC 004 A.4 — a tail
+consumer must see late-arriving remote ops even though their HLC is older.
+HLC is the *display/merge* order; arrival is the *delivery* order.
+"""
+
+import re
+import threading
+import time
+
+_HLC_RE = re.compile(r"^(\d{13})-([0-9a-f]{6})-(.+)$")
+
+MAX_COUNTER = 0xFFFFFF
+MAX_FUTURE_DRIFT_MS = 60_000
+
+
+def parse(hlc: str):
+    """Return (ms, counter, replica_id) or raise ValueError."""
+    m = _HLC_RE.match(hlc)
+    if not m:
+        raise ValueError(f"not a valid HLC string: {hlc!r}")
+    return int(m.group(1)), int(m.group(2), 16), m.group(3)
+
+
+def render(ms: int, counter: int, replica_id: str) -> str:
+    return f"{ms:013d}-{counter:06x}-{replica_id}"
+
+
+class HybridLogicalClock:
+    """Thread-safe HLC bound to one replica id.
+
+    ``last`` may be seeded from persisted state (the newest hlc in the
+    op-log) so monotonicity survives process restarts.
+    """
+
+    def __init__(self, replica_id: str, last: str = None, now_ms=None):
+        self.replica_id = replica_id
+        self._now_ms = now_ms or (lambda: int(time.time() * 1000))
+        self._lock = threading.Lock()
+        if last:
+            ms, counter, _ = parse(last)
+            if self._is_within_future_drift(ms):
+                self._ms, self._counter = ms, counter
+            else:
+                self._ms, self._counter = 0, 0
+        else:
+            self._ms, self._counter = 0, 0
+
+    def _is_within_future_drift(self, ms: int) -> bool:
+        return ms <= self._now_ms() + MAX_FUTURE_DRIFT_MS
+
+    def tick(self) -> str:
+        """Stamp a new local op; strictly greater than everything seen."""
+        with self._lock:
+            now = self._now_ms()
+            if now > self._ms:
+                self._ms, self._counter = now, 0
+            elif self._counter < MAX_COUNTER:
+                self._counter += 1
+            else:
+                # Counter exhaustion within one ms is pathological; step
+                # the physical component rather than wrap and reorder.
+                self._ms += 1
+                self._counter = 0
+            return render(self._ms, self._counter, self.replica_id)
+
+    def observe(self, remote_hlc: str) -> None:
+        """Absorb a remote op's instant so future ticks sort after it."""
+        try:
+            remote_ms, remote_counter, _ = parse(remote_hlc)
+        except ValueError:
+            return  # never let a malformed remote stamp wedge the clock
+        if not self._is_within_future_drift(remote_ms):
+            return
+        with self._lock:
+            if remote_ms > self._ms or (remote_ms == self._ms and remote_counter > self._counter):
+                self._ms, self._counter = remote_ms, remote_counter

mempalace/hooks_cli.py

@@ -7,6 +7,9 @@ Supported harnesses: claude-code, codex (extensible to cursor, gemini, etc.)
 """
 
 import hashlib
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
 import json
 import os
 import re
@@ -18,6 +21,13 @@ from pathlib import Path
 from typing import Optional
 
 from mempalace.config import MempalaceConfig
+from mempalace.write_routing import (
+    ResolvedWriteRoutingPolicy,
+    WriteRoutingDecision,
+    WriteRoutingError,
+    WriteRoutingPolicy,
+    choose_write_route,
+)
 
 SAVE_INTERVAL = 15
 STATE_DIR = Path.home() / ".mempalace" / "hook_state"
@@ -510,6 +520,11 @@ def _spawn_mine(cmd: list) -> None:
 
 
 def _hooks_daemon_enabled() -> bool:
+    """Legacy compatibility helper for the pre-policy hook setting.
+
+    New hook write paths use ``resolve_write_routing("hooks")``. This
+    helper remains for callers/tests that still inspect ``hooks.daemon``.
+    """
     try:
         return MempalaceConfig().hook_use_daemon is True
     except Exception:
@@ -527,11 +542,9 @@ def _daemon_mine_dedupe_key(source: str, mode: str) -> str:
 def _daemon_available() -> bool:
     """True iff a daemon is already running for the configured palace.
 
-    This is a fast localhost health check, not a spawn: the 500ms hook budget
-    forbids auto-starting a python subprocess from a hook (cold start is
-    ~15s). Daemon mode for hooks requires the user to have started the daemon
-    explicitly via `mempalace daemon start`; when it isn't up, hooks fall back
-    to the existing direct (in-process / spawn) path instead of blocking.
+    This is a fast localhost health check, not a spawn: the hook time budget
+    forbids cold-starting a long-lived daemon. ``prefer`` may fall back to the
+    direct path when this returns false; ``require`` must block the write.
     """
     from .daemon import HOOK_PROBE_TIMEOUT, get_client_if_running
 
@@ -544,6 +557,143 @@ def _daemon_available() -> bool:
         return False
 
 
+@dataclass(frozen=True)
+class HookWriteRouting:
+    """One hook invocation's resolved routing state."""
+
+    decision: Optional[WriteRoutingDecision]
+    source: str
+    error: Optional[str] = None
+
+    @property
+    def use_daemon(self) -> bool:
+        return self.decision is not None and self.decision.use_daemon
+
+    @property
+    def blocked(self) -> bool:
+        return self.error is not None or (self.decision is not None and self.decision.blocked)
+
+    @property
+    def notice(self) -> str:
+        if self.error is not None:
+            return (
+                "MemPalace hook writes were skipped because write-routing "
+                f"configuration is invalid: {self.error}. No direct ChromaDB "
+                "fallback was attempted."
+            )
+        if self.blocked:
+            return (
+                "MemPalace hook writes were skipped because routing is set to "
+                "'require' but the local daemon is unavailable. Start it with "
+                "`mempalace daemon start`; no direct ChromaDB fallback was attempted."
+            )
+        return ""
+
+
+_HOOK_WRITE_ROUTING_CONTEXT = ContextVar(
+    "mempalace_hook_write_routing",
+    default=None,
+)
+
+
+def _resolve_configured_hook_policy() -> ResolvedWriteRoutingPolicy:
+    """Resolve the new policy, with legacy-object compatibility."""
+
+    config = MempalaceConfig()
+    resolver = getattr(config, "resolve_write_routing", None)
+    if callable(resolver):
+        resolved = resolver("hooks")
+        if isinstance(resolved, ResolvedWriteRoutingPolicy):
+            return resolved
+
+    # Compatibility for older/custom config objects and existing tests that
+    # expose only the pre-policy ``hook_use_daemon`` property.
+    policy = (
+        WriteRoutingPolicy.PREFER
+        if getattr(config, "hook_use_daemon", False) is True
+        else WriteRoutingPolicy.DIRECT
+    )
+    return ResolvedWriteRoutingPolicy(
+        policy=policy,
+        source="legacy hook_use_daemon",
+    )
+
+
+def _compute_hook_write_routing() -> HookWriteRouting:
+    """Resolve hook policy and probe daemon liveness at most once."""
+
+    try:
+        resolved = _resolve_configured_hook_policy()
+    except WriteRoutingError as exc:
+        routing = HookWriteRouting(
+            decision=None,
+            source="configuration-error",
+            error=str(exc),
+        )
+        _log(routing.notice)
+        return routing
+    except Exception as exc:
+        # Preserve the historical save-on-config-read-failure behavior. An
+        # explicitly invalid routing value raises WriteRoutingError above and
+        # fails closed; an unrelated config I/O/runtime failure falls back to
+        # direct so a final checkpoint is not silently lost.
+        _log(f"WARNING: could not resolve hook write routing: {exc}; defaulting to direct")
+        resolved = ResolvedWriteRoutingPolicy(
+            policy=WriteRoutingPolicy.DIRECT,
+            source="config-unavailable fallback",
+        )
+
+    daemon_available = False
+    if resolved.policy is not WriteRoutingPolicy.DIRECT:
+        daemon_available = _daemon_available()
+
+    decision = choose_write_route(
+        resolved.policy,
+        daemon_available=daemon_available,
+        daemon_can_start=False,
+    )
+    routing = HookWriteRouting(
+        decision=decision,
+        source=resolved.source,
+    )
+
+    if decision.policy is not WriteRoutingPolicy.DIRECT:
+        _log(
+            "Hook write routing: "
+            f"policy={decision.policy.value} source={resolved.source} "
+            f"target={decision.target.value} reason={decision.reason}"
+        )
+
+    return routing
+
+
+def _current_hook_write_routing() -> HookWriteRouting:
+    routing = _HOOK_WRITE_ROUTING_CONTEXT.get()
+    if routing is not None:
+        return routing
+    return _compute_hook_write_routing()
+
+
+@contextmanager
+def _hook_write_routing_context():
+    """Share one policy resolution and one daemon probe across a hook fire."""
+
+    routing = _compute_hook_write_routing()
+    token = _HOOK_WRITE_ROUTING_CONTEXT.set(routing)
+    try:
+        yield routing
+    finally:
+        _HOOK_WRITE_ROUTING_CONTEXT.reset(token)
+
+
+def _log_hook_write_blocked(routing: HookWriteRouting, operation: str) -> None:
+    _log(f"{routing.notice} Operation skipped: {operation}.")
+
+
+def _blocked_hook_output(routing: HookWriteRouting) -> dict:
+    return {"systemMessage": routing.notice}
+
+
 def _submit_daemon_job(
     kind: str,
     payload: dict,
@@ -573,9 +723,33 @@ def _submit_daemon_job(
         wait=wait,
         auto_start=False,
         timeout=timeout,
+        # A job refused the palace lock is deferred, not failed (#2014), so it
+        # is never terminal while the holder lives. The waiting callers below
+        # wait on purpose, but a parked job cannot reach the state they wait
+        # for: they would burn the whole timeout and then report a failure that
+        # did not happen. Take the parked job back instead; the daemon still
+        # runs it once the lock frees.
+        stop_on_lock_deferral=True,
     )
 
 
+def _job_deferred_by_lock(job: dict) -> bool:
+    """True when the daemon parked this job behind the palace write lock.
+
+    Imported lazily like ``submit_job`` above: only callers that actually reach
+    the daemon pay for the module, and by this point it is already loaded.
+    """
+    from .daemon import job_deferred_by_lock
+
+    return job_deferred_by_lock(job)
+
+
+def _lock_deferral_reason(job: dict) -> str:
+    """Operator-facing reason a job is parked, for the hook log."""
+    reason = (job.get("error") or {}).get("message") or "the palace write lock is held"
+    return f"{reason} (job {job.get('id')} stays queued and runs when the holder exits)"
+
+
 def _maybe_auto_ingest():
     """Background-mine MEMPAL_DIR (project files) if set.
 
@@ -592,9 +766,15 @@ def _maybe_auto_ingest():
     targets = _get_mine_targets()
     if not targets:
         return
+
+    routing = _current_hook_write_routing()
+    if routing.blocked:
+        _log_hook_write_blocked(routing, "project auto-ingest")
+        return
+
     for mine_dir, mode in targets:
         try:
-            if _hooks_daemon_enabled() and _daemon_available():
+            if routing.use_daemon:
                 try:
                     _submit_daemon_job(
                         "mine",
@@ -626,11 +806,17 @@ def _mine_sync():
     targets = _get_mine_targets()
     if not targets:
         return
+
+    routing = _current_hook_write_routing()
+    if routing.blocked:
+        _log_hook_write_blocked(routing, "synchronous project mine")
+        return
+
     STATE_DIR.mkdir(parents=True, exist_ok=True)
     log_path = STATE_DIR / "hook.log"
     for mine_dir, mode in targets:
         try:
-            if _hooks_daemon_enabled() and _daemon_available():
+            if routing.use_daemon:
                 try:
                     job = _submit_daemon_job(
                         "mine",
@@ -640,7 +826,12 @@ def _mine_sync():
                         timeout=60,
                     )
                     result = job.get("result") or {}
-                    if job.get("state") != "succeeded" or not result.get("success", True):
+                    if _job_deferred_by_lock(job):
+                        # Parked behind the palace lock, not failed: the daemon
+                        # runs it once the holder exits. Saying "failed" here
+                        # would be the false report #2014 is about.
+                        _log(f"Daemon sync mine deferred: {_lock_deferral_reason(job)}")
+                    elif job.get("state") != "succeeded" or not result.get("success", True):
                         _log(f"Daemon sync mine failed: {result.get('error', job.get('error'))}")
                 except Exception as exc:
                     # Daemon accepted context — don't fall back (would double-mine).
@@ -773,12 +964,24 @@ def _save_diary_direct(
     the agent wrote to, so project-derived wings stay discoverable.
 
     Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure.
+    A daemon lock deferral also returns {"count": 0}: nothing is filed yet, but
+    the entry is queued and the daemon files it once the holder exits, so the
+    checkpoint marker is deliberately not advanced.
     """
     messages = _extract_recent_messages(transcript_path)
     if not messages:
         _log("No recent messages to save")
         return {"count": 0}
 
+    routing = _current_hook_write_routing()
+    if routing.blocked:
+        _log_hook_write_blocked(routing, "diary checkpoint")
+        return {
+            "count": 0,
+            "routing_blocked": True,
+            "routing_message": routing.notice,
+        }
+
     themes = _extract_themes(messages)
 
     # Build a compressed diary entry from recent conversation
@@ -790,7 +993,7 @@ def _save_diary_direct(
     )
 
     try:
-        if _hooks_daemon_enabled() and _daemon_available():
+        if routing.use_daemon:
             try:
                 job = _submit_daemon_job(
                     "diary_write",
@@ -822,6 +1025,12 @@ def _save_diary_direct(
                 if toast:
                     _desktop_toast(f"Checkpoint saved - {len(messages)} messages archived")
                 return {"count": len(messages), "themes": themes}
+            if _job_deferred_by_lock(job):
+                # Queued behind the palace lock: the entry is held and the daemon
+                # files it once the holder exits. Not a failure, and not a reason
+                # to re-file it here -- that would duplicate verbatim content.
+                _log(f"Daemon diary checkpoint deferred: {_lock_deferral_reason(job)}")
+                return {"count": 0}
             _log(f"Daemon diary checkpoint failed: {result.get('error', job.get('error'))}")
             return {"count": 0}
 
@@ -870,18 +1079,23 @@ def _ingest_transcript(transcript_path: str):
     except Exception:
         return
 
+    routing = _current_hook_write_routing()
+    if routing.blocked:
+        _log_hook_write_blocked(routing, "transcript ingest")
+        return
+
     try:
-        if _hooks_daemon_enabled() and _daemon_available():
+        if routing.use_daemon:
             try:
                 _submit_daemon_job(
                     "mine",
                     {
-                        "source": str(path.parent),
+                        "source": str(path),
                         "mode": "convos",
                         "wing": "sessions",
                         "agent": "mempalace",
                     },
-                    dedupe_key=_daemon_mine_dedupe_key(str(path.parent), "convos"),
+                    dedupe_key=_daemon_mine_dedupe_key(str(path), "convos"),
                     wait=False,
                 )
                 _log(f"Transcript ingest submitted to daemon: {path.name}")
@@ -899,7 +1113,7 @@ def _ingest_transcript(transcript_path: str):
                 "-m",
                 "mempalace",
                 "mine",
-                str(path.parent),
+                str(path),
                 "--mode",
                 "convos",
                 "--wing",
@@ -1012,6 +1226,12 @@ def _wing_from_jsonl_cwd(transcript_path: str) -> Optional[str]:
                 cwd_norm = cwd.replace("\\", "/").rstrip("/")
                 if not cwd_norm:
                     continue
+                # A cwd inside "<project>/.claude/worktrees/<wt>" (a git
+                # worktree) belongs to <project>, not the ephemeral worktree
+                # directory -- otherwise every worktree spawns its own wing.
+                _wt_marker = "/.claude/worktrees/"
+                if _wt_marker in cwd_norm:
+                    cwd_norm = cwd_norm.split(_wt_marker, 1)[0]
                 project = cwd_norm.rsplit("/", 1)[-1]
                 if project:
                     return f"wing_{_safe_wing_slug(project)}"
@@ -1132,64 +1352,70 @@ def hook_stop(data: dict, harness: str):
     _log(f"Session {session_id}: {exchange_count} exchanges, {since_last} since last save")
 
     if since_last >= SAVE_INTERVAL and exchange_count > 0:
-        _log(f"TRIGGERING SAVE at exchange {exchange_count}")
+        with _hook_write_routing_context() as routing:
+            if routing.blocked:
+                _log_hook_write_blocked(routing, "stop-hook checkpoint")
+                _output(_blocked_hook_output(routing))
+                return
 
-        # Read hook settings from config
-        try:
-            config = MempalaceConfig()
-            silent = config.hook_silent_save
-            toast = config.hook_desktop_toast
-        except Exception:
-            silent = True
-            toast = False
+            _log(f"TRIGGERING SAVE at exchange {exchange_count}")
 
-        project_wing = _wing_from_transcript_path(transcript_path)
+            # Read hook settings from config
+            try:
+                config = MempalaceConfig()
+                silent = config.hook_silent_save
+                toast = config.hook_desktop_toast
+            except Exception:
+                silent = True
+                toast = False
 
-        if silent:
-            # Save directly via Python API — systemMessage renders in terminal
-            result = {"count": 0}
-            if transcript_path:
-                result = _save_diary_direct(
-                    transcript_path,
-                    session_id,
-                    wing=project_wing,
-                    toast=toast,
-                    agent_name=_diary_agent_for_harness(harness),
-                )
-                _ingest_transcript(transcript_path)
-            _maybe_auto_ingest()
-            # Only advance save marker after successful save
-            count = result.get("count", 0)
-            if count > 0:
+            project_wing = _wing_from_transcript_path(transcript_path)
+
+            if silent:
+                # Save directly via Python API — systemMessage renders in terminal
+                result = {"count": 0}
+                if transcript_path:
+                    result = _save_diary_direct(
+                        transcript_path,
+                        session_id,
+                        wing=project_wing,
+                        toast=toast,
+                        agent_name=_diary_agent_for_harness(harness),
+                    )
+                    _ingest_transcript(transcript_path)
+                _maybe_auto_ingest()
+                # Only advance save marker after successful save
+                count = result.get("count", 0)
+                if count > 0:
+                    try:
+                        last_save_file.write_text(str(exchange_count), encoding="utf-8")
+                    except OSError:
+                        pass
+                    themes = result.get("themes", [])
+                    if themes:
+                        tag = " \u2014 " + ", ".join(themes)
+                    else:
+                        tag = ""
+                    _output(
+                        {
+                            "systemMessage": f"\u2726 {count} memories woven into the palace{tag}",
+                        }
+                    )
+                else:
+                    _output({})
+            else:
+                # Legacy: block and ask Claude to save via MCP tools.
+                # Marker advances before confirmed save — best-effort; if Claude
+                # fails to save, the checkpoint is lost but won't retry endlessly.
                 try:
                     last_save_file.write_text(str(exchange_count), encoding="utf-8")
                 except OSError:
                     pass
-                themes = result.get("themes", [])
-                if themes:
-                    tag = " \u2014 " + ", ".join(themes)
-                else:
-                    tag = ""
-                _output(
-                    {
-                        "systemMessage": f"\u2726 {count} memories woven into the palace{tag}",
-                    }
-                )
-            else:
-                _output({})
-        else:
-            # Legacy: block and ask Claude to save via MCP tools.
-            # Marker advances before confirmed save — best-effort; if Claude
-            # fails to save, the checkpoint is lost but won't retry endlessly.
-            try:
-                last_save_file.write_text(str(exchange_count), encoding="utf-8")
-            except OSError:
-                pass
-            if transcript_path:
-                _ingest_transcript(transcript_path)
-            _maybe_auto_ingest()
-            reason = STOP_BLOCK_REASON + f" Write diary entry to wing={project_wing}."
-            _output({"decision": "block", "reason": reason})
+                if transcript_path:
+                    _ingest_transcript(transcript_path)
+                _maybe_auto_ingest()
+                reason = STOP_BLOCK_REASON + f" Write diary entry to wing={project_wing}."
+                _output({"decision": "block", "reason": reason})
     else:
         _output({})
 
@@ -1207,6 +1433,14 @@ def hook_session_start(data: dict, harness: str):
     # Initialize session state directory
     STATE_DIR.mkdir(parents=True, exist_ok=True)
 
+    # Surface a required-daemon problem at session start instead of waiting
+    # until the first save is due. Hooks still never cold-start the daemon.
+    with _hook_write_routing_context() as routing:
+        if routing.blocked:
+            _log_hook_write_blocked(routing, "session-start readiness check")
+            _output(_blocked_hook_output(routing))
+            return
+
     # Pass through — no blocking on session start
     _output({})
 
@@ -1305,16 +1539,22 @@ def hook_session_end(data: dict, harness: str):
         # short-circuit + upsert). ``reason`` is intentionally not branched on:
         # every clean-exit reason (incl. ``/clear`` / ``resume``) warrants the
         # flush. Order matches ``hook_stop``.
-        if valid_transcript:
-            _save_diary_direct(
-                valid_transcript,
-                session_id,
-                wing=_wing_from_transcript_path(valid_transcript),
-                toast=toast,
-                agent_name=_diary_agent_for_harness(harness),
-            )
-            _ingest_transcript(valid_transcript)
-        _maybe_auto_ingest()
+        with _hook_write_routing_context() as routing:
+            if routing.blocked:
+                _log_hook_write_blocked(routing, "session-end flush")
+                _output(_blocked_hook_output(routing))
+                return
+
+            if valid_transcript:
+                _save_diary_direct(
+                    valid_transcript,
+                    session_id,
+                    wing=_wing_from_transcript_path(valid_transcript),
+                    toast=toast,
+                    agent_name=_diary_agent_for_harness(harness),
+                )
+                _ingest_transcript(valid_transcript)
+            _maybe_auto_ingest()
 
         _output({})
     finally:
@@ -1341,14 +1581,20 @@ def hook_precompact(data: dict, harness: str):
 
     _log(f"PRE-COMPACT triggered for session {session_id}")
 
-    # Capture tool output via our normalize path before compaction loses it
-    if transcript_path:
-        _ingest_transcript(transcript_path)
+    with _hook_write_routing_context() as routing:
+        if routing.blocked:
+            _log_hook_write_blocked(routing, "precompact flush")
+            _output(_blocked_hook_output(routing))
+            return
 
-    # Mine MEMPAL_DIR synchronously so project data lands before
-    # compaction proceeds. Transcript convos were already kicked off
-    # above via _ingest_transcript.
-    _mine_sync()
+        # Capture tool output via our normalize path before compaction loses it
+        if transcript_path:
+            _ingest_transcript(transcript_path)
+
+        # Mine MEMPAL_DIR synchronously so project data lands before
+        # compaction proceeds. Transcript convos were already kicked off
+        # above via _ingest_transcript.
+        _mine_sync()
 
     _output({})
 

mempalace/i18n/__init__.py

@@ -97,6 +97,35 @@ def get_regex() -> dict:
     return _strings.get("regex", {})
 
 
+def get_stopwords(lang: Optional[str] = None) -> set:
+    """Return BM25 stop words for ``lang`` as a lowercase set.
+
+    When ``lang`` is given, loads that locale's regex section directly and
+    does not touch global state. When omitted, reads from the currently
+    loaded language. Returns an empty set if the locale is unknown or its
+    ``regex.stop_words`` field is missing.
+
+    ``regex.stop_words`` is stored as a space-separated string in each
+    locale JSON so non-ASCII particles (e.g. Japanese ``は/が/を``) stay
+    authorable without JSON-array escaping overhead.
+    """
+    if lang is not None:
+        canonical = _canonical_lang(lang)
+        if canonical is None:
+            return set()
+        lang_file = _LANG_DIR / f"{canonical}.json"
+        try:
+            data = json.loads(lang_file.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            return set()
+        sw_str = data.get("regex", {}).get("stop_words", "")
+    else:
+        sw_str = get_regex().get("stop_words", "")
+    if not isinstance(sw_str, str):
+        return set()
+    return {w.lower() for w in sw_str.split() if w}
+
+
 def _load_entity_section(lang: str) -> dict:
     """Load the raw entity section for one language. Returns {} if missing."""
     canonical = _canonical_lang(lang)

mempalace/i18n/ja.json

@@ -37,7 +37,7 @@
   },
   "regex": {
     "topic_pattern": "[\\u30A0-\\u30FF]{3,}|[\\u4E00-\\u9FFF]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
-    "stop_words": "は が を に で と も の へ から まで より した します している されて です ます ました こと もの ため それ これ その この あの ない なく ある いる する",
+    "stop_words": "から まで より した します している されて です ます ました こと もの ため それ これ その この あの ない なく ある いる する",
     "quote_pattern": "「([^」]{10,100})」",
     "action_pattern": "(構築|修正|追加|削除|確認|作成|実装|修復|書き直し|テスト|検証|更新|設定|起動|停止)(?:し|した|して|する|します)"
   }

mempalace/i18n/zh-CN.json

@@ -37,7 +37,7 @@
   },
   "regex": {
     "topic_pattern": "[\\u4E00-\\u9FFF]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
-    "stop_words": "的 了 在 是 我 有 和 就 不 人 都 一 一个 上 也 很 到 说 要 去 你 会 着 没有 看 好 自己 这 那 她 他 它 们 但是 因为 所以 如果 虽然 然后 或者 而且",
+    "stop_words": "一个 没有 自己 但是 因为 所以 如果 虽然 然后 或者 而且",
     "quote_pattern": "\\u201C([^\\u201D]{10,100})\\u201D|\"([^\"]{10,200})\"",
     "action_pattern": "(构建|修复|添加|删除|确认|创建|实现|修理|编写|测试|验证|更新|配置|启动|停止)(?:了|完成|成功)"
   },

mempalace/ids.py

@@ -108,6 +108,27 @@ def make_convo_sentinel_id(source_file: str, extract_mode: str) -> str:
     return f"_reg_{_delimited_sha256((source_file, extract_mode), _HASH_TRUNC_DRAWER)}"
 
 
+def make_exchange_drawer_id(
+    wing: str, room: str, source_file: str, filed_at: str, content: str
+) -> str:
+    """Drawer ID for a single verbatim conversation exchange.
+
+    Used by live agent integrations (e.g. Hermes) and their backfills via
+    ``convo_miner.file_conversation_exchange``. Hashes the FULL content,
+    not a prefix — prefix hashing collided on common openings ("User: hi
+    can you help me with…") and ChromaDB's upsert silently overwrote the
+    earlier drawer. ``filed_at`` is included so genuinely repeated
+    exchanges stay distinct drawers (verbatim always — repetition is
+    signal, not noise).
+
+    Hash input is ``f"{source_file}|{filed_at}|{content}"``.
+    """
+    return (
+        f"drawer_{wing}_{room}_"
+        f"{_delimited_sha256((source_file, filed_at, content), _HASH_TRUNC_DRAWER)}"
+    )
+
+
 def make_triple_id(
     sub_id: str, predicate: str, obj_id: str, valid_from: str, recorded_at: str
 ) -> str:

mempalace/knowledge_graph.py

@@ -630,6 +630,66 @@ class KnowledgeGraph:
 
     # ── Stats ─────────────────────────────────────────────────────────────
 
+    # -- Replication (RFC 004 step 1: read-replica snapshot) ---------------
+
+    _REPLICATION_TABLES = {
+        "entities": ("id", "name", "type", "properties", "created_at"),
+        "triples": (
+            "id",
+            "subject",
+            "predicate",
+            "object",
+            "valid_from",
+            "valid_to",
+            "confidence",
+            "source_closet",
+            "source_file",
+            "source_drawer_id",
+            "adapter_name",
+            "extracted_at",
+        ),
+    }
+
+    def dump_rows(self, table: str, after_rowid: int = 0, limit: int = 500) -> list:
+        """Page KG rows in rowid order for snapshot replication.
+
+        Rows are returned verbatim with a ``_rowid`` pagination cursor.
+        rowid order is deterministic, so pages never skip under concurrent
+        appends (updates in earlier pages are caught by the next full pass).
+        """
+        columns = self._REPLICATION_TABLES.get(table)
+        if columns is None:
+            raise ValueError(f"table must be one of {sorted(self._REPLICATION_TABLES)}")
+        with self._lock:
+            conn = self._conn()
+            rows = conn.execute(
+                f"SELECT rowid, {', '.join(columns)} FROM {table} "
+                "WHERE rowid > ? ORDER BY rowid ASC LIMIT ?",
+                (int(after_rowid), max(1, min(int(limit), 1000))),
+            ).fetchall()
+        return [dict(row) | {"_rowid": row["rowid"]} for row in rows]
+
+    def apply_row(self, table: str, row: dict) -> None:
+        """Fold one replicated KG row in, keyed by id (INSERT OR REPLACE).
+
+        REPLACE makes invalidations (valid_to updates) and entity edits
+        converge on re-pull; rows are never deleted by replication.
+        """
+        columns = self._REPLICATION_TABLES.get(table)
+        if columns is None:
+            raise ValueError(f"table must be one of {sorted(self._REPLICATION_TABLES)}")
+        if not row.get("id"):
+            raise ValueError("replicated row is missing 'id'")
+        values = [row.get(col) for col in columns]
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                conn.execute(
+                    f"INSERT OR REPLACE INTO {table} ({', '.join(columns)}) "
+                    f"VALUES ({', '.join('?' for _ in columns)})",
+                    values,
+                )
+
     def stats(self):
         with self._lock:
             conn = self._conn()

mempalace/layers.py

@@ -469,7 +469,7 @@ if __name__ == "__main__":
     import json
 
     def usage():
-        print("layers.py — 4-Layer Memory Stack")
+        print("layers.py -- 4-Layer Memory Stack")
         print()
         print("Usage:")
         print("  python layers.py wake-up              Show L0 + L1")

mempalace/logstream.py

@@ -0,0 +1,968 @@
+"""
+logstream.py — Agent coordination event log for MemPalace (RFC 003)
+===================================================================
+
+A small append-only event layer stored in the active palace directory as
+``logstream.sqlite3``. Agents use it to delegate work, wait for replies,
+and exchange exact patch/file artifacts through the shared MemPalace hub
+without a human relaying messages between machines.
+
+Design constraints (RFC 003):
+
+- No Chroma dependency, no vector index open — plain SQLite only.
+- Append-only: events are immutable; corrections are new events that
+  reference prior events (see :meth:`Logstream.ack_event`).
+- Exact payloads: event bodies and artifact content are stored verbatim.
+- Safe under concurrent HTTP requests (WAL + per-instance lock, same
+  pattern as ``knowledge_graph.py``).
+- Explicit size limits with clear errors, never silent truncation.
+
+Usage:
+    from mempalace.logstream import Logstream
+
+    ls = Logstream(db_path="/path/to/palace/logstream.sqlite3")
+    evt = ls.append_event(
+        type="task.request", stream="project/mempalace", room="delegation",
+        from_agent="mac-codex", to_agent="windows-codex",
+        correlation_id="task_123", body="Please fix search echo ranking.",
+    )
+    reply = ls.wait_events(correlation_id="task_123", type="patch.ready",
+                           timeout_ms=300_000)
+"""
+
+import json
+import random
+import re
+import secrets
+import sqlite3
+import threading
+import time
+from datetime import datetime, timezone
+from hashlib import sha256
+from pathlib import Path
+from typing import Optional
+
+from .config import sanitize_iso_temporal, strip_lone_surrogates
+
+LOGSTREAM_DB_FILENAME = "logstream.sqlite3"
+
+# Size limits (RFC 003 suggested defaults). Byte lengths of the UTF-8
+# encoding, checked with explicit errors — payloads are never truncated.
+DEFAULT_MAX_BODY_BYTES = 256 * 1024
+DEFAULT_MAX_ARTIFACT_BYTES = 4 * 1024 * 1024
+MAX_METADATA_BYTES = 64 * 1024
+
+# Wait limits: polling loop inside one MCP request, 250-1000 ms with jitter.
+MAX_WAIT_TIMEOUT_MS = 300_000
+_POLL_BASE_S = 0.25
+_POLL_MAX_S = 1.0
+
+MAX_LIST_LIMIT = 500
+DEFAULT_LIST_LIMIT = 50
+
+EVENT_STATUSES = frozenset(
+    {"open", "claimed", "ready", "applied", "blocked", "failed", "superseded"}
+)
+ARTIFACT_KINDS = frozenset({"patch", "file", "log", "json", "note"})
+ACK_EVENT_TYPE = "event.ack"
+
+# Event types are dotted lowercase identifiers (task.request, patch.ready).
+# Routing fields must stay structured — agents should never have to infer
+# them from prose — so the character set is enforced, not just length.
+_EVENT_TYPE_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$")
+
+_MAX_ROUTING_LENGTH = 256
+
+
+def _utc_now_iso() -> str:
+    """Canonical UTC timestamp, second precision: YYYY-MM-DDTHH:MM:SSZ.
+
+    Matches the canonical temporal shape accepted by
+    ``config.sanitize_iso_temporal`` so ``since_created_at`` filters
+    compare like against like. Ordering ties within one second are
+    resolved by the SQLite rowid (``seq``), not the timestamp.
+    """
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _new_id(prefix: str) -> str:
+    """Server-generated id: readable UTC stamp + random suffix.
+
+    Uniqueness comes from the random suffix; ordering guarantees come
+    from the rowid, so clock skew between writers cannot reorder or
+    collide events.
+    """
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S")
+    return f"{prefix}_{stamp}_{secrets.token_hex(6)}"
+
+
+def _sanitize_routing(value, field_name: str, required: bool = True) -> Optional[str]:
+    """Validate a short routing field (stream, room, agent, correlation_id).
+
+    Streams may contain ``/`` (``project/mempalace``), so this is looser
+    than ``config.sanitize_name`` — but null bytes, control characters,
+    and over-length values are still rejected.
+    """
+    if value is None or value == "":
+        if required:
+            raise ValueError(f"{field_name} must be a non-empty string")
+        return None
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{field_name} must be a non-empty string")
+    value = strip_lone_surrogates(value.strip())
+    if len(value) > _MAX_ROUTING_LENGTH:
+        raise ValueError(f"{field_name} exceeds maximum length of {_MAX_ROUTING_LENGTH} characters")
+    if any(ord(ch) < 0x20 or ch == "\x7f" for ch in value):
+        raise ValueError(f"{field_name} contains control characters")
+    return value
+
+
+def _sanitize_event_type(value) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("type must be a non-empty string")
+    value = value.strip()
+    if not _EVENT_TYPE_RE.match(value):
+        raise ValueError(
+            f"type={value!r} is not a valid event type "
+            "(lowercase letters, digits, '.', '_', '-'; max 64 chars)"
+        )
+    return value
+
+
+def _sanitize_status(value) -> Optional[str]:
+    if value is None or value == "":
+        return None
+    if not isinstance(value, str) or value not in EVENT_STATUSES:
+        allowed = ", ".join(sorted(EVENT_STATUSES))
+        raise ValueError(f"status={value!r} is not one of: {allowed}")
+    return value
+
+
+def _sanitize_body(value, max_bytes: int, field_name: str = "body") -> str:
+    """Validate verbatim payload text. Empty is allowed; ``None`` becomes ''."""
+    if value is None:
+        return ""
+    if not isinstance(value, str):
+        raise ValueError(f"{field_name} must be a string")
+    if "\x00" in value:
+        raise ValueError(f"{field_name} contains null bytes")
+    value = strip_lone_surrogates(value)
+    size = len(value.encode("utf-8"))
+    if size > max_bytes:
+        raise ValueError(f"{field_name} is {size} bytes; maximum is {max_bytes} bytes")
+    return value
+
+
+def _sanitize_metadata(value) -> str:
+    """Validate optional metadata dict and return its canonical JSON text."""
+    if value is None:
+        return "{}"
+    if not isinstance(value, dict):
+        raise ValueError("metadata must be an object")
+    try:
+        encoded = json.dumps(value, ensure_ascii=False, sort_keys=True)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"metadata is not JSON-serializable: {exc}") from None
+    if len(encoded.encode("utf-8")) > MAX_METADATA_BYTES:
+        raise ValueError(f"metadata exceeds maximum size of {MAX_METADATA_BYTES} bytes")
+    return encoded
+
+
+class Logstream:
+    """Durable append-only coordination log (events + artifacts).
+
+    Storage and threading mirror ``KnowledgeGraph``: one SQLite file in
+    WAL mode, a per-instance lock around writes, ``check_same_thread=False``
+    so the MCP HTTP server can call from worker threads.
+    """
+
+    def __init__(
+        self,
+        db_path: str,
+        max_body_bytes: int = DEFAULT_MAX_BODY_BYTES,
+        max_artifact_bytes: int = DEFAULT_MAX_ARTIFACT_BYTES,
+        replica_id: str = None,
+    ):
+        from .replica import get_replica_id
+
+        self.db_path = str(db_path)
+        self.max_body_bytes = max_body_bytes
+        self.max_artifact_bytes = max_artifact_bytes
+        db_parent = Path(self.db_path).parent
+        db_parent.mkdir(parents=True, exist_ok=True)
+        try:
+            db_parent.chmod(0o700)
+        except (OSError, NotImplementedError):
+            pass
+        # RFC 004 provenance: every op is stamped with the replica that authored it.
+        # replica.json lives in the palace dir (the db's parent).
+        self.replica_id = replica_id or get_replica_id(str(db_parent))
+        self._connection = None
+        self._lock = threading.Lock()
+        self._init_db()
+
+    def _init_db(self):
+        from .hlc import HybridLogicalClock
+
+        conn = self._conn()
+        conn.executescript("""
+            PRAGMA journal_mode=WAL;
+
+            CREATE TABLE IF NOT EXISTS events (
+                id TEXT PRIMARY KEY,
+                type TEXT NOT NULL,
+                stream TEXT NOT NULL,
+                room TEXT NOT NULL,
+                from_agent TEXT NOT NULL,
+                to_agent TEXT,
+                correlation_id TEXT,
+                branch TEXT,
+                base_commit TEXT,
+                status TEXT,
+                body TEXT NOT NULL DEFAULT '',
+                created_at TEXT NOT NULL,
+                metadata_json TEXT NOT NULL DEFAULT '{}',
+                origin_replica TEXT,
+                origin_seq INTEGER,
+                hlc TEXT
+            );
+
+            CREATE INDEX IF NOT EXISTS events_stream_created_idx
+                ON events(stream, created_at);
+            CREATE INDEX IF NOT EXISTS events_correlation_idx
+                ON events(correlation_id, created_at);
+            CREATE INDEX IF NOT EXISTS events_to_agent_idx
+                ON events(to_agent, created_at);
+            CREATE INDEX IF NOT EXISTS events_type_idx
+                ON events(type, created_at);
+
+            CREATE TABLE IF NOT EXISTS artifacts (
+                id TEXT PRIMARY KEY,
+                kind TEXT NOT NULL,
+                sha256 TEXT NOT NULL,
+                size_bytes INTEGER NOT NULL,
+                content TEXT NOT NULL,
+                created_by TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                metadata_json TEXT NOT NULL DEFAULT '{}',
+                origin_replica TEXT
+            );
+
+            CREATE INDEX IF NOT EXISTS artifacts_sha256_idx ON artifacts(sha256);
+
+            CREATE TABLE IF NOT EXISTS event_artifacts (
+                event_id TEXT NOT NULL,
+                artifact_id TEXT NOT NULL,
+                PRIMARY KEY (event_id, artifact_id)
+            );
+        """)
+        self._migrate_replication_schema(conn)
+        conn.commit()
+        # Seed the HLC from the newest stamp in the log so monotonicity
+        # survives restarts (RFC 004 op envelope).
+        row = conn.execute("SELECT max(hlc) AS last FROM events").fetchone()
+        self._clock = HybridLogicalClock(self.replica_id, last=row["last"] if row else None)
+
+    def _migrate_replication_schema(self, conn):
+        """RFC 004 step 0: add provenance columns to pre-replication logs.
+
+        Fresh databases get the columns from the canonical CREATE TABLE; this
+        path upgrades logs created before step 0. Backfill stamps existing
+        rows as authored by THIS replica (they were: pre-replication, only
+        one copy existed) with origin_seq = rowid and an HLC derived from
+        created_at + rowid, preserving their original total order.
+        """
+        existing = {row["name"] for row in conn.execute("PRAGMA table_info(events)")}
+        if "origin_replica" not in existing:
+            conn.execute("ALTER TABLE events ADD COLUMN origin_replica TEXT")
+        if "origin_seq" not in existing:
+            conn.execute("ALTER TABLE events ADD COLUMN origin_seq INTEGER")
+        if "hlc" not in existing:
+            conn.execute("ALTER TABLE events ADD COLUMN hlc TEXT")
+        art_existing = {row["name"] for row in conn.execute("PRAGMA table_info(artifacts)")}
+        if "origin_replica" not in art_existing:
+            conn.execute("ALTER TABLE artifacts ADD COLUMN origin_replica TEXT")
+
+        from .hlc import render as hlc_render
+
+        rows = conn.execute(
+            "SELECT rowid, created_at FROM events WHERE origin_replica IS NULL"
+        ).fetchall()
+        for row in rows:
+            try:
+                ms = int(
+                    datetime.strptime(row["created_at"], "%Y-%m-%dT%H:%M:%SZ")
+                    .replace(tzinfo=timezone.utc)
+                    .timestamp()
+                    * 1000
+                )
+            except ValueError:
+                ms = 0
+            conn.execute(
+                "UPDATE events SET origin_replica = ?, origin_seq = rowid, hlc = ? WHERE rowid = ?",
+                (
+                    self.replica_id,
+                    hlc_render(ms, min(row["rowid"], 0xFFFFFF), self.replica_id),
+                    row["rowid"],
+                ),
+            )
+        conn.execute(
+            "UPDATE artifacts SET origin_replica = ? WHERE origin_replica IS NULL",
+            (self.replica_id,),
+        )
+        conn.execute(
+            "CREATE UNIQUE INDEX IF NOT EXISTS events_origin_seq_idx "
+            "ON events(origin_replica, origin_seq)"
+        )
+        conn.execute("CREATE INDEX IF NOT EXISTS events_hlc_idx ON events(hlc)")
+
+    def _conn(self):
+        if self._connection is None:
+            self._connection = sqlite3.connect(self.db_path, timeout=10, check_same_thread=False)
+            self._connection.execute("PRAGMA journal_mode=WAL")
+            self._connection.row_factory = sqlite3.Row
+        return self._connection
+
+    def close(self):
+        with self._lock:
+            if self._connection is not None:
+                self._connection.close()
+                self._connection = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.close()
+        return False
+
+    # ── Row shaping ───────────────────────────────────────────────────────
+
+    @staticmethod
+    def _event_dict(row, artifact_ids: Optional[list] = None) -> dict:
+        try:
+            metadata = json.loads(row["metadata_json"])
+        except (ValueError, TypeError):
+            metadata = {}
+        return {
+            "id": row["id"],
+            "seq": row["rowid"],
+            # RFC 004 additive fields — committed compat surface. `seq` stays
+            # the LOCAL arrival cursor; `origin_seq` is the author's counter;
+            # `hlc` is the cross-replica display/merge order.
+            "origin_replica": row["origin_replica"],
+            "origin_seq": row["origin_seq"],
+            "hlc": row["hlc"],
+            "type": row["type"],
+            "stream": row["stream"],
+            "room": row["room"],
+            "from_agent": row["from_agent"],
+            "to_agent": row["to_agent"],
+            "correlation_id": row["correlation_id"],
+            "branch": row["branch"],
+            "base_commit": row["base_commit"],
+            "status": row["status"],
+            "artifact_ids": artifact_ids or [],
+            "body": row["body"],
+            "created_at": row["created_at"],
+            "metadata": metadata,
+        }
+
+    def _attach_artifact_ids(self, conn, events: list[dict]) -> list[dict]:
+        if not events:
+            return events
+        by_id = {e["id"]: e for e in events}
+        placeholders = ",".join("?" for _ in by_id)
+        rows = conn.execute(
+            f"SELECT event_id, artifact_id FROM event_artifacts "
+            f"WHERE event_id IN ({placeholders}) ORDER BY artifact_id",
+            list(by_id),
+        ).fetchall()
+        for row in rows:
+            by_id[row["event_id"]]["artifact_ids"].append(row["artifact_id"])
+        return events
+
+    # ── Write operations (append-only) ────────────────────────────────────
+
+    def append_event(
+        self,
+        type: str,
+        stream: str,
+        room: str,
+        from_agent: str,
+        to_agent: str = None,
+        correlation_id: str = None,
+        branch: str = None,
+        base_commit: str = None,
+        status: str = None,
+        body: str = "",
+        metadata: dict = None,
+        artifact_ids: list = None,
+    ) -> dict:
+        """Append one immutable event. Returns the stored event dict.
+
+        ``artifact_ids`` must reference already-stored artifacts; unknown
+        ids are rejected so readers never see a dangling reference.
+        """
+        type = _sanitize_event_type(type)
+        stream = _sanitize_routing(stream, "stream")
+        room = _sanitize_routing(room, "room")
+        from_agent = _sanitize_routing(from_agent, "from_agent")
+        to_agent = _sanitize_routing(to_agent, "to_agent", required=False)
+        correlation_id = _sanitize_routing(correlation_id, "correlation_id", required=False)
+        branch = _sanitize_routing(branch, "branch", required=False)
+        base_commit = _sanitize_routing(base_commit, "base_commit", required=False)
+        status = _sanitize_status(status)
+        body = _sanitize_body(body, self.max_body_bytes)
+        metadata_json = _sanitize_metadata(metadata)
+
+        if artifact_ids is None:
+            artifact_ids = []
+        if not isinstance(artifact_ids, list) or not all(
+            isinstance(a, str) and a for a in artifact_ids
+        ):
+            raise ValueError("artifact_ids must be a list of artifact id strings")
+        artifact_ids = list(dict.fromkeys(artifact_ids))  # dedup, keep order
+
+        event_id = _new_id("evt")
+        created_at = _utc_now_iso()
+        hlc = self._clock.tick()
+
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                for artifact_id in artifact_ids:
+                    found = conn.execute(
+                        "SELECT 1 FROM artifacts WHERE id = ?", (artifact_id,)
+                    ).fetchone()
+                    if not found:
+                        raise ValueError(
+                            f"artifact_ids references unknown artifact {artifact_id!r}"
+                        )
+                cursor = conn.execute(
+                    "INSERT INTO events (id, type, stream, room, from_agent, to_agent,"
+                    " correlation_id, branch, base_commit, status, body, created_at,"
+                    " metadata_json, origin_replica, hlc)"
+                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        event_id,
+                        type,
+                        stream,
+                        room,
+                        from_agent,
+                        to_agent,
+                        correlation_id,
+                        branch,
+                        base_commit,
+                        status,
+                        body,
+                        created_at,
+                        metadata_json,
+                        self.replica_id,
+                        hlc,
+                    ),
+                )
+                # Locally-authored ops use the local rowid as their per-origin
+                # counter — strictly monotonic, gap-free enough for cursors.
+                conn.execute(
+                    "UPDATE events SET origin_seq = rowid WHERE rowid = ?",
+                    (cursor.lastrowid,),
+                )
+                for artifact_id in artifact_ids:
+                    conn.execute(
+                        "INSERT INTO event_artifacts (event_id, artifact_id) VALUES (?, ?)",
+                        (event_id, artifact_id),
+                    )
+            row = conn.execute("SELECT rowid, * FROM events WHERE id = ?", (event_id,)).fetchone()
+        return self._event_dict(row, artifact_ids=list(artifact_ids))
+
+    @staticmethod
+    def _patch_content_warnings(kind: str, content: str) -> list[str]:
+        """Advisory checks for ``kind=patch`` content. Never mutates content.
+
+        Verbatim storage means we store exactly what the producer sent —
+        but a diff that ``git apply`` will reject as corrupt is a broken
+        handoff, so warn at store time where the producer can still fix it.
+        Found in the wild during the first RFC 003 dogfood: a patch stored
+        without its final trailing newline truncates the last hunk line.
+        """
+        if kind != "patch":
+            return []
+        warnings = []
+        if not content.endswith("\n"):
+            warnings.append(
+                "patch content does not end with a trailing newline; "
+                "git apply will reject the final hunk as corrupt — "
+                "store the diff byte-exactly including its trailing newline"
+            )
+        if "\r" in content:
+            warnings.append(
+                "patch content contains carriage returns (CRLF line endings); "
+                "git apply often rejects CRLF diffs — generate the diff with "
+                "LF endings"
+            )
+        return warnings
+
+    def put_artifact(
+        self,
+        kind: str,
+        content: str,
+        created_by: str,
+        metadata: dict = None,
+    ) -> dict:
+        """Store exact artifact content (v1: UTF-8 text only).
+
+        Returns the artifact record without echoing ``content`` back —
+        callers already hold the content; readers use :meth:`get_artifact`.
+        For ``kind=patch``, a ``warnings`` list is included when the diff
+        looks unappliable (missing trailing newline, CRLF endings); the
+        content itself is still stored verbatim.
+        """
+        if not isinstance(kind, str) or kind not in ARTIFACT_KINDS:
+            allowed = ", ".join(sorted(ARTIFACT_KINDS))
+            raise ValueError(f"kind={kind!r} is not one of: {allowed}")
+        created_by = _sanitize_routing(created_by, "created_by")
+        if not isinstance(content, str) or not content:
+            raise ValueError("content must be a non-empty string")
+        if "\x00" in content:
+            raise ValueError("content contains null bytes")
+        content = strip_lone_surrogates(content)
+        raw = content.encode("utf-8")
+        if len(raw) > self.max_artifact_bytes:
+            raise ValueError(
+                f"content is {len(raw)} bytes; maximum is {self.max_artifact_bytes} bytes"
+            )
+        metadata_json = _sanitize_metadata(metadata)
+
+        artifact_id = _new_id("art")
+        created_at = _utc_now_iso()
+        digest = sha256(raw).hexdigest()
+
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                conn.execute(
+                    "INSERT INTO artifacts (id, kind, sha256, size_bytes, content,"
+                    " created_by, created_at, metadata_json, origin_replica)"
+                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        artifact_id,
+                        kind,
+                        digest,
+                        len(raw),
+                        content,
+                        created_by,
+                        created_at,
+                        metadata_json,
+                        self.replica_id,
+                    ),
+                )
+        record = {
+            "id": artifact_id,
+            "kind": kind,
+            "sha256": digest,
+            "size_bytes": len(raw),
+            "created_by": created_by,
+            "created_at": created_at,
+            "origin_replica": self.replica_id,
+        }
+        warnings = self._patch_content_warnings(kind, content)
+        if warnings:
+            record["warnings"] = warnings
+        return record
+
+    def ack_event(
+        self,
+        event_id: str,
+        from_agent: str,
+        status: str = None,
+        body: str = "",
+    ) -> dict:
+        """Append an ``event.ack`` referencing a prior event.
+
+        The target event is never mutated. The ack copies the target's
+        stream/room, copies its ``correlation_id`` (falling back to the
+        target's id so request/ack stay tied together), and routes back
+        to the target's ``from_agent``.
+        """
+        event_id = _sanitize_routing(event_id, "event_id")
+        with self._lock:
+            conn = self._conn()
+            target = conn.execute(
+                "SELECT rowid, * FROM events WHERE id = ?", (event_id,)
+            ).fetchone()
+        if target is None:
+            raise ValueError(f"event {event_id!r} not found")
+
+        return self.append_event(
+            type=ACK_EVENT_TYPE,
+            stream=target["stream"],
+            room=target["room"],
+            from_agent=from_agent,
+            to_agent=target["from_agent"],
+            correlation_id=target["correlation_id"] or target["id"],
+            status=status,
+            body=body,
+            metadata={"ack_of": event_id},
+        )
+
+    def submit_patch(
+        self,
+        content: str,
+        from_agent: str,
+        stream: str,
+        room: str = "patches",
+        to_agent: str = None,
+        correlation_id: str = None,
+        branch: str = None,
+        base_commit: str = None,
+        body: str = "",
+        metadata: dict = None,
+    ) -> dict:
+        """Convenience wrapper: store a patch artifact + ``patch.ready`` event.
+
+        Both writes go through the validated single-writer paths; the event
+        insert re-checks the artifact exists, so readers never observe a
+        ``patch.ready`` event with a dangling artifact id.
+        """
+        artifact = self.put_artifact(
+            kind="patch",
+            content=content,
+            created_by=from_agent,
+            metadata=metadata,
+        )
+        event = self.append_event(
+            type="patch.ready",
+            stream=stream,
+            room=room,
+            from_agent=from_agent,
+            to_agent=to_agent,
+            correlation_id=correlation_id,
+            branch=branch,
+            base_commit=base_commit,
+            status="ready",
+            body=body,
+            metadata=metadata,
+            artifact_ids=[artifact["id"]],
+        )
+        return {"artifact": artifact, "event": event}
+
+    # ── Read operations ───────────────────────────────────────────────────
+
+    def get_artifact(self, artifact_id: str) -> Optional[dict]:
+        """Fetch one artifact by id, exact content included. None if missing."""
+        artifact_id = _sanitize_routing(artifact_id, "artifact_id")
+        with self._lock:
+            conn = self._conn()
+            row = conn.execute("SELECT * FROM artifacts WHERE id = ?", (artifact_id,)).fetchone()
+        if row is None:
+            return None
+        try:
+            metadata = json.loads(row["metadata_json"])
+        except (ValueError, TypeError):
+            metadata = {}
+        return {
+            "id": row["id"],
+            "kind": row["kind"],
+            "sha256": row["sha256"],
+            "size_bytes": row["size_bytes"],
+            "content": row["content"],
+            "created_by": row["created_by"],
+            "created_at": row["created_at"],
+            "origin_replica": row["origin_replica"],
+            "metadata": metadata,
+        }
+
+    def list_events(
+        self,
+        stream: str = None,
+        room: str = None,
+        type: str = None,
+        to_agent: str = None,
+        from_agent: str = None,
+        correlation_id: str = None,
+        status: str = None,
+        since_event_id: str = None,
+        since_created_at: str = None,
+        limit: int = DEFAULT_LIST_LIMIT,
+    ) -> list[dict]:
+        """List events matching structured filters, oldest first.
+
+        Cursor semantics:
+
+        - ``since_event_id`` is the precise cursor: strictly after that
+          event in append order (rowid), regardless of timestamp ties.
+        - ``since_created_at`` is inclusive (``>=``) so second-granularity
+          timestamps never skip events; callers dedup by ``id``.
+        - ``to_agent`` also matches broadcast events (``to_agent='*'``).
+        """
+        stream = _sanitize_routing(stream, "stream", required=False)
+        room = _sanitize_routing(room, "room", required=False)
+        if type not in (None, ""):
+            type = _sanitize_event_type(type)
+        else:
+            type = None
+        to_agent = _sanitize_routing(to_agent, "to_agent", required=False)
+        from_agent = _sanitize_routing(from_agent, "from_agent", required=False)
+        correlation_id = _sanitize_routing(correlation_id, "correlation_id", required=False)
+        status = _sanitize_status(status)
+        since_event_id = _sanitize_routing(since_event_id, "since_event_id", required=False)
+        since_created_at = sanitize_iso_temporal(since_created_at, "since_created_at") or None
+        if not isinstance(limit, int) or limit < 1:
+            raise ValueError("limit must be a positive integer")
+        limit = min(limit, MAX_LIST_LIMIT)
+
+        where = []
+        params = []
+        for column, value in (
+            ("stream", stream),
+            ("room", room),
+            ("type", type),
+            ("from_agent", from_agent),
+            ("correlation_id", correlation_id),
+            ("status", status),
+        ):
+            if value is not None:
+                where.append(f"{column} = ?")
+                params.append(value)
+        if to_agent is not None:
+            where.append("(to_agent = ? OR to_agent = '*')")
+            params.append(to_agent)
+        if since_created_at is not None:
+            where.append("created_at >= ?")
+            params.append(since_created_at)
+
+        with self._lock:
+            conn = self._conn()
+            if since_event_id is not None:
+                anchor = conn.execute(
+                    "SELECT rowid FROM events WHERE id = ?", (since_event_id,)
+                ).fetchone()
+                if anchor is None:
+                    raise ValueError(f"since_event_id {since_event_id!r} not found")
+                where.append("rowid > ?")
+                params.append(anchor["rowid"])
+
+            sql = "SELECT rowid, * FROM events"
+            if where:
+                sql += " WHERE " + " AND ".join(where)
+            sql += " ORDER BY rowid ASC LIMIT ?"
+            params.append(limit)
+            rows = conn.execute(sql, params).fetchall()
+            events = [self._event_dict(row) for row in rows]
+            return self._attach_artifact_ids(conn, events)
+
+    def latest_event_id(self) -> Optional[str]:
+        """Id of the newest event, or None on an empty log.
+
+        Live-tail consumers (the SSE stream) capture this at connect time
+        as their starting cursor so they receive only post-connect events.
+        """
+        with self._lock:
+            conn = self._conn()
+            row = conn.execute("SELECT id FROM events ORDER BY rowid DESC LIMIT 1").fetchone()
+        return row["id"] if row is not None else None
+
+    def wait_events(
+        self,
+        timeout_ms: int = 60_000,
+        poll_interval_s: float = None,
+        **filters,
+    ) -> dict:
+        """Block until at least one matching event exists or timeout expires.
+
+        v1 implementation per RFC 003: a polling loop inside the request,
+        sleeping 250-1000 ms with jitter. Timeouts are clamped to
+        ``MAX_WAIT_TIMEOUT_MS`` and return ``{"timed_out": True,
+        "events": []}`` rather than raising.
+
+        ``filters`` accepts the same keyword filters as :meth:`list_events`.
+        ``poll_interval_s`` pins the sleep (tests); default is jittered.
+        """
+        if not isinstance(timeout_ms, (int, float)) or timeout_ms < 0:
+            raise ValueError("timeout_ms must be a non-negative number")
+        timeout_ms = min(timeout_ms, MAX_WAIT_TIMEOUT_MS)
+        deadline = time.monotonic() + timeout_ms / 1000.0
+
+        attempt = 0
+        while True:
+            events = self.list_events(**filters)
+            if events:
+                return {"timed_out": False, "events": events}
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                return {"timed_out": True, "events": []}
+            if poll_interval_s is not None:
+                delay = poll_interval_s
+            else:
+                base = min(_POLL_MAX_S, _POLL_BASE_S * (1.5**attempt))
+                delay = base * (0.75 + random.random() * 0.25)
+            time.sleep(min(delay, remaining))
+            attempt += 1
+
+    # ── Replication (RFC 004 step 0: logstream multi-master) ─────────────
+
+    def version_vector(self) -> dict:
+        """{origin_replica: highest origin_seq applied locally}.
+
+        The complete description of this replica's knowledge — peers diff
+        their vectors to compute exactly which op ranges are missing.
+        """
+        with self._lock:
+            conn = self._conn()
+            rows = conn.execute(
+                "SELECT origin_replica, max(origin_seq) AS top FROM events "
+                "WHERE origin_replica IS NOT NULL GROUP BY origin_replica"
+            ).fetchall()
+        return {row["origin_replica"]: row["top"] for row in rows}
+
+    def list_ops(self, origin: str, after_seq: int = 0, limit: int = 500) -> list[dict]:
+        """Events authored by ``origin`` with origin_seq > after_seq, in
+        author order. The anti-entropy pull unit."""
+        origin = _sanitize_routing(origin, "origin")
+        if not isinstance(after_seq, int) or after_seq < 0:
+            raise ValueError("after_seq must be a non-negative integer")
+        if not isinstance(limit, int) or limit < 1:
+            raise ValueError("limit must be a positive integer")
+        limit = min(limit, MAX_LIST_LIMIT)
+        with self._lock:
+            conn = self._conn()
+            rows = conn.execute(
+                "SELECT rowid, * FROM events WHERE origin_replica = ? AND origin_seq > ? "
+                "ORDER BY origin_seq ASC LIMIT ?",
+                (origin, after_seq, limit),
+            ).fetchall()
+            events = [self._event_dict(row) for row in rows]
+            return self._attach_artifact_ids(conn, events)
+
+    def apply_remote_event(self, event: dict) -> bool:
+        """Fold one remote op into the local log, idempotently.
+
+        Verbatim rule: the event is stored exactly as authored (id,
+        created_at, hlc, origin stamps untouched); only the local rowid —
+        the arrival cursor — is ours. Referenced artifacts must already be
+        applied (sync pulls artifacts first) so readers never see a dangling
+        id, same invariant as append_event. Returns True if inserted, False
+        if we already had it. Raises ValueError on malformed input or a
+        missing artifact.
+        """
+        for key in (
+            "id",
+            "type",
+            "stream",
+            "room",
+            "from_agent",
+            "created_at",
+            "origin_replica",
+            "origin_seq",
+            "hlc",
+        ):
+            if not event.get(key):
+                raise ValueError(f"remote event is missing required field {key!r}")
+        if event["origin_replica"] == self.replica_id:
+            return False  # our own op echoed back — by definition already present
+        artifact_ids = list(dict.fromkeys(event.get("artifact_ids") or []))
+        metadata_json = _sanitize_metadata(event.get("metadata"))
+
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                dup = conn.execute(
+                    "SELECT 1 FROM events WHERE id = ? OR (origin_replica = ? AND origin_seq = ?)",
+                    (event["id"], event["origin_replica"], event["origin_seq"]),
+                ).fetchone()
+                if dup:
+                    return False
+                for artifact_id in artifact_ids:
+                    found = conn.execute(
+                        "SELECT 1 FROM artifacts WHERE id = ?", (artifact_id,)
+                    ).fetchone()
+                    if not found:
+                        raise ValueError(
+                            f"remote event {event['id']!r} references artifact "
+                            f"{artifact_id!r} not yet applied locally"
+                        )
+                conn.execute(
+                    "INSERT INTO events (id, type, stream, room, from_agent, to_agent,"
+                    " correlation_id, branch, base_commit, status, body, created_at,"
+                    " metadata_json, origin_replica, origin_seq, hlc)"
+                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        event["id"],
+                        event["type"],
+                        event["stream"],
+                        event["room"],
+                        event["from_agent"],
+                        event.get("to_agent"),
+                        event.get("correlation_id"),
+                        event.get("branch"),
+                        event.get("base_commit"),
+                        event.get("status"),
+                        event.get("body") or "",
+                        event["created_at"],
+                        metadata_json,
+                        event["origin_replica"],
+                        event["origin_seq"],
+                        event["hlc"],
+                    ),
+                )
+                for artifact_id in artifact_ids:
+                    conn.execute(
+                        "INSERT OR IGNORE INTO event_artifacts (event_id, artifact_id)"
+                        " VALUES (?, ?)",
+                        (event["id"], artifact_id),
+                    )
+        # Absorb the remote instant so our next local op sorts after it.
+        self._clock.observe(event["hlc"])
+        return True
+
+    def has_artifact(self, artifact_id: str) -> bool:
+        """Cheap existence probe (no content transfer) for the sync engine."""
+        artifact_id = _sanitize_routing(artifact_id, "artifact_id")
+        with self._lock:
+            conn = self._conn()
+            row = conn.execute("SELECT 1 FROM artifacts WHERE id = ?", (artifact_id,)).fetchone()
+        return row is not None
+
+    def apply_remote_artifact(self, artifact: dict) -> bool:
+        """Fold one remote artifact in, idempotently, verifying its hash.
+
+        Content is verbatim; the sha256 must match or the artifact is
+        rejected — a corrupt transfer must never enter the store.
+        """
+        for key in ("id", "kind", "sha256", "content", "created_by", "created_at"):
+            if not artifact.get(key):
+                raise ValueError(f"remote artifact is missing required field {key!r}")
+        content = artifact["content"]
+        digest = sha256(content.encode("utf-8")).hexdigest()
+        if digest != artifact["sha256"]:
+            raise ValueError(
+                f"remote artifact {artifact['id']!r} failed hash verification "
+                f"(claimed {artifact['sha256'][:16]}..., computed {digest[:16]}...)"
+            )
+        metadata_json = _sanitize_metadata(artifact.get("metadata"))
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                dup = conn.execute(
+                    "SELECT 1 FROM artifacts WHERE id = ?", (artifact["id"],)
+                ).fetchone()
+                if dup:
+                    return False
+                conn.execute(
+                    "INSERT INTO artifacts (id, kind, sha256, size_bytes, content,"
+                    " created_by, created_at, metadata_json, origin_replica)"
+                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        artifact["id"],
+                        artifact["kind"],
+                        artifact["sha256"],
+                        len(content.encode("utf-8")),
+                        content,
+                        artifact["created_by"],
+                        artifact["created_at"],
+                        metadata_json,
+                        artifact.get("origin_replica"),
+                    ),
+                )
+        return True

mempalace/logsync.py

@@ -0,0 +1,136 @@
+"""
+logsync.py — Anti-entropy sync engine for the logstream (RFC 004 step 0)
+
+Pull-based convergence between palace replicas: diff version vectors, pull
+missing per-origin op ranges (artifacts first, so referenced ids never
+dangle), fold with the idempotent apply primitives. Each replica pulls from
+its peers; two replicas pulling from each other converge without any push
+path or coordinator.
+
+Transport for the pilot rides the RFC 004 transport seam's `request` shape over
+plain HTTPS (Tailscale or any mutually-reachable network) using the peer's
+hub bearer token. Peers are configured in ``peers.json`` in the palace dir:
+
+    {
+      "peers": [
+        {"name": "windows", "url": "https://host.example.ts.net", "token": "..."}
+      ]
+    }
+
+This module lives ABOVE the RFC 004 transport seam (mempalace/transport.py):
+peers come from the transport's membership snapshot and every wire call goes
+through its ``request``. Swapping the link (HTTPS bearer today, MeshGuard
+next) never touches the sync logic here.
+"""
+
+import logging
+
+from .transport import (
+    PEERS_FILENAME,  # noqa: F401 — re-exported for existing importers
+    TransportError,
+    get_transport,
+    http_request,
+    load_peers,
+)
+
+logger = logging.getLogger("mempalace.logsync")
+
+_PULL_BATCH = 500
+
+# Historical names, kept so existing importers (replica_sync, CLI, tests)
+# keep working: the error class moved below the seam with the wire code.
+SyncPeerError = TransportError
+__all__ = ["PEERS_FILENAME", "SyncPeerError", "load_peers", "sync_all", "sync_with_peer"]
+
+
+def _peer_get(base_url: str, token: str, path: str, params: dict = None) -> dict:
+    """Compat shim for raw url+token callers (replica_sync, CLI --peer)."""
+    return http_request(base_url, token, path, params)
+
+
+def sync_with_peer(ls, url: str, token: str = "") -> dict:
+    """One anti-entropy round against one peer. Returns pull stats.
+
+    Never partially applies an event: artifacts are fetched and folded
+    before the event that references them, and every apply is idempotent,
+    so a crash mid-round just means the next round re-pulls the tail.
+    """
+    remote = _peer_get(url, token, "/sync/version_vector")
+    remote_vector = remote.get("version_vector") or {}
+    remote_replica = remote.get("replica_id", "?")
+    local_vector = ls.version_vector()
+
+    pulled_events = 0
+    pulled_artifacts = 0
+    per_origin = {}
+
+    for origin, remote_top in remote_vector.items():
+        if origin == ls.replica_id:
+            continue  # nobody knows more of our own ops than we do
+        after = local_vector.get(origin, 0)
+        origin_pulled = 0
+        while after < remote_top:
+            batch = (
+                _peer_get(
+                    url,
+                    token,
+                    "/sync/ops",
+                    {"origin": origin, "after": after, "limit": _PULL_BATCH},
+                ).get("events")
+                or []
+            )
+            if not batch:
+                break  # peer's vector promised more than it served; stop cleanly
+            for event in batch:
+                for artifact_id in event.get("artifact_ids") or []:
+                    if ls.has_artifact(artifact_id):
+                        continue
+                    artifact = _peer_get(url, token, "/sync/artifact", {"id": artifact_id}).get(
+                        "artifact"
+                    )
+                    if not artifact:
+                        raise SyncPeerError(
+                            f"{url}: peer served event {event.get('id')!r} but not "
+                            f"its artifact {artifact_id!r}"
+                        )
+                    if ls.apply_remote_artifact(artifact):
+                        pulled_artifacts += 1
+                if ls.apply_remote_event(event):
+                    pulled_events += 1
+                    origin_pulled += 1
+                after = max(after, event.get("origin_seq") or 0)
+        if origin_pulled:
+            per_origin[origin] = origin_pulled
+
+    return {
+        "peer_url": url,
+        "peer_replica": remote_replica,
+        "pulled_events": pulled_events,
+        "pulled_artifacts": pulled_artifacts,
+        "per_origin": per_origin,
+        # The peer's vector at round start — consumers (the /sync/peers
+        # estate endpoint, PalaceMind's mesh view) compute drift from it.
+        "remote_version_vector": remote_vector,
+        # Node-profile advertisement riding the same fetch: the peer's own
+        # self-description plus profiles it relays for origins it knows
+        # (absent from pre-profile peers — consumers treat None as unknown).
+        "remote_profile": remote.get("profile"),
+        "remote_profiles": remote.get("profiles") or {},
+    }
+
+
+def sync_all(ls, palace_path: str, transport=None) -> list[dict]:
+    """One round against every peer in the transport's membership snapshot;
+    per-peer errors are reported, never raised — one dead peer must not
+    block the others (R1: only convergence waits)."""
+    transport = transport or get_transport(palace_path)
+    results = []
+    for peer in transport.peers():
+        name = peer.get("name") or peer.get("url") or "?"
+        try:
+            stats = sync_with_peer(ls, peer["url"], peer.get("token", ""))
+            stats["peer_name"] = name
+            results.append(stats)
+        except (TransportError, ValueError) as exc:
+            results.append({"peer_name": name, "peer_url": peer.get("url"), "error": str(exc)})
+    return results

mempalace/migrate.py

@@ -29,7 +29,7 @@ from collections import defaultdict
 from contextlib import closing
 from datetime import datetime
 
-from .backups import prune_backups
+from .backups import copy_palace_dir, prune_backups
 from .config import MempalaceConfig
 
 
@@ -244,22 +244,43 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
     # A plain count() is not enough: some 0.6.x -> 1.5.x migrated collections
     # are readable but silently drop upsert/delete operations. In that state,
     # migrate must rebuild from SQLite instead of returning "No migration needed."
+    #
+    # Preflight HNSW divergence before touching the collection at all: the
+    # #1222 SIGSEGV/panic class crashes on count() against a diverged
+    # segment, and a native crash can't be caught by the except Exception
+    # below -- a diverged palace must never reach col.count() in the first
+    # place. The already-diverged case degrades to exactly the same
+    # SQLite-extraction path the except branch below already falls back to.
+    from .backends.chroma import hnsw_capacity_status
+
     try:
-        col = ChromaBackend().get_collection(palace_path, "mempalace_drawers")
-        count = col.count()
-
-        if collection_write_roundtrip_works(col):
-            print(f"\n Palace is already readable and writable by chromadb {target_version}.")
-            print(f" {count} drawers found. No migration needed.")
-            return True
-
-        print(
-            f"\n Palace is readable by chromadb {target_version}, but write/delete verification failed."
-        )
-        print(" Rebuilding from SQLite to restore native write/delete behavior...")
+        capacity_info = hnsw_capacity_status(palace_path, "mempalace_drawers")
     except Exception:
-        print(f"\n Palace is NOT readable by chromadb {target_version}.")
+        capacity_info = {}
+
+    if capacity_info.get("diverged"):
+        print(
+            f"\n Palace is NOT readable by chromadb {target_version}: HNSW index diverged from SQLite."
+        )
+        print(f" ({capacity_info.get('message', 'divergence detected')})")
         print(" Extracting from SQLite directly...")
+    else:
+        try:
+            col = ChromaBackend().get_collection(palace_path, "mempalace_drawers")
+            count = col.count()
+
+            if collection_write_roundtrip_works(col):
+                print(f"\n Palace is already readable and writable by chromadb {target_version}.")
+                print(f" {count} drawers found. No migration needed.")
+                return True
+
+            print(
+                f"\n Palace is readable by chromadb {target_version}, but write/delete verification failed."
+            )
+            print(" Rebuilding from SQLite to restore native write/delete behavior...")
+        except Exception:
+            print(f"\n Palace is NOT readable by chromadb {target_version}.")
+            print(" Extracting from SQLite directly...")
 
     # Extract all drawers via raw SQL
     drawers = extract_drawers_from_sqlite(db_path)
@@ -284,7 +305,7 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
             print(f"      ROOM: {room:30} {count:5}")
 
     if dry_run:
-        print("\n  DRY RUN — no changes made.")
+        print("\n  DRY RUN -- no changes made.")
         print(f"  Would migrate {len(drawers)} drawers.")
         return True
 
@@ -295,7 +316,7 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     backup_path = f"{palace_path}.pre-migrate.{timestamp}"
     print(f"\n  Backing up to {backup_path}...")
-    shutil.copytree(palace_path, backup_path, symlinks=True)
+    copy_palace_dir(palace_path, backup_path, symlinks=True, log=print)
 
     # Enforce backup retention so repeated migrations cannot fill the disk
     # with full-palace copies. The backup we just created is the newest, so
@@ -549,7 +570,7 @@ def migrate_wing_names(palace_path: str, dry_run: bool = False, confirm: bool =
     topic_renames = _plan_topics_by_wing_renames()
 
     if not d_updates and not c_updates and not topic_renames:
-        print("  All wing names are already normalized — nothing to migrate.")
+        print("  All wing names are already normalized -- nothing to migrate.")
         return False
 
     print("\n  Wing-name migration plan:")
@@ -565,7 +586,7 @@ def migrate_wing_names(palace_path: str, dry_run: bool = False, confirm: bool =
         print(f"    topics_by_wing: {len(topic_renames)} key(s) re-keyed")
 
     if dry_run:
-        print("\n  DRY RUN — no changes made.\n")
+        print("\n  DRY RUN -- no changes made.\n")
         return True
 
     if not confirm:

mempalace/miner.py

@@ -633,13 +633,19 @@ def chunk_text(
         raise ValueError(f"chunk_size must be a positive int, got {chunk_size!r}")
     if not isinstance(chunk_overlap, int) or chunk_overlap < 0:
         raise ValueError(f"chunk_overlap must be a non-negative int, got {chunk_overlap!r}")
-    if chunk_overlap >= chunk_size:
-        # ``start = end - chunk_overlap`` would not advance (or would go
-        # backward) when overlap >= size, producing an infinite loop on
-        # any non-empty input.
+    if chunk_overlap > chunk_size // 2:
+        # The windowing loop pulls ``end`` back to a boundary only when that
+        # boundary is past ``start + chunk_size // 2`` (the rfind guards below),
+        # so a pulled chunk always spans more than ``chunk_size // 2`` chars.
+        # ``start = end - chunk_overlap`` therefore advances only while
+        # ``chunk_overlap <= chunk_size // 2``; a larger overlap makes ``start``
+        # stall or move backward and the loop spins forever on short-line
+        # content (#2056). The largest safe value is ``chunk_size // 2`` (half,
+        # rounded down for odd sizes), which still advances and stays allowed.
         raise ValueError(
-            f"chunk_overlap ({chunk_overlap}) must be less than chunk_size "
-            f"({chunk_size}); equality or greater would loop forever"
+            f"chunk_overlap ({chunk_overlap}) must be at most chunk_size // 2 "
+            f"({chunk_size // 2}); a larger overlap can loop forever on "
+            f"short-line content (#2056)"
         )
     if not isinstance(min_chunk_size, int) or min_chunk_size < 0:
         raise ValueError(f"min_chunk_size must be a non-negative int, got {min_chunk_size!r}")
@@ -653,6 +659,28 @@ def chunk_text(
     start = 0
     chunk_index = 0
 
+    # Running newline tallies for the 1-indexed (line_start, line_end)
+    # locators emitted below. These replace the previous per-chunk
+    # ``content.count("\n", 0, pos)`` full-prefix rescans, which were O(pos)
+    # each and O(N*K) over K chunks: a 287 MB / ~433k-chunk file scanned
+    # ~1.2e14 bytes and ran for days, indistinguishable from a hang (#2054).
+    # ``start`` and ``end`` each advance monotonically for any
+    # ``chunk_overlap < chunk_size // 2`` (the default 800/100 config and
+    # every sane override), so counting only the newly-scanned span is O(N)
+    # total. Each position sequence keeps its own anchor; a backward step
+    # (a paragraph-boundary pull under a pathological near-``chunk_size``
+    # overlap, or a negative index) falls back to the exact full-prefix
+    # count. That fallback is defensive and does not fire on a terminating
+    # mine: the current windowing cannot send a filed chunk's ``start``
+    # backward without also entering a pre-existing infinite loop (overlap
+    # >= chunk_size // 2), so the else-branches read as uncovered. They keep
+    # every value byte-identical to the old form if the windowing ever gains
+    # a loop guard that admits backward steps.
+    _nl_before_start = 0
+    _start_anchor = 0
+    _nl_before_end = 0
+    _end_anchor = 0
+
     while start < len(content):
         end = min(start + chunk_size, len(content))
 
@@ -671,13 +699,25 @@ def chunk_text(
             # Tier 6a — 1-indexed line range in the stripped source.
             # Approximate locator (±1 at boundaries is fine for "jump to
             # roughly here"); exact-quote positioning is a future tier.
-            # Use the bounds form of ``str.count`` (counts on the original
-            # string with start/end limits) instead of slicing — slicing
-            # would allocate a new substring per chunk and produce O(N^2)
-            # work on a 500MB file with 50K chunks. Per PR #1579 review
-            # (gemini-code-assist, medium priority).
-            line_start = content.count("\n", 0, start) + 1
-            line_end = content.count("\n", 0, end) + 1
+            # ``str.count`` with bounds (not slicing) still avoids allocating
+            # a substring per chunk, the original PR #1579 review concern
+            # (gemini-code-assist, medium priority). The incremental anchors
+            # additionally avoid rescanning the whole prefix each time, the
+            # actual O(N*K) cost (#2054). Two anchors because ``start`` and
+            # ``end`` are distinct monotonic sequences; a backward step
+            # re-derives the value exactly from position 0.
+            if start >= _start_anchor:
+                _nl_before_start += content.count("\n", _start_anchor, start)
+                _start_anchor = start
+                line_start = _nl_before_start + 1
+            else:
+                line_start = content.count("\n", 0, start) + 1
+            if end >= _end_anchor:
+                _nl_before_end += content.count("\n", _end_anchor, end)
+                _end_anchor = end
+                line_end = _nl_before_end + 1
+            else:
+                line_end = content.count("\n", 0, end) + 1
             chunks.append(
                 {
                     "content": chunk,
@@ -1803,7 +1843,7 @@ def _mine_impl(
     print(f"  Palace:  {palace_path}")
     print(f"  Device:  {describe_device()}")
     if dry_run:
-        print("  DRY RUN — nothing will be filed")
+        print("  DRY RUN -- nothing will be filed")
     if not respect_gitignore:
         print("  .gitignore: DISABLED")
     if include_ignored:
@@ -2098,7 +2138,7 @@ def status(palace_path: str):
     un-bootstrapped collection, or an unexpected schema); the fallback also
     emits the state-specific guidance for absent/empty palaces.
     """
-    from .backends.chroma import _sqlite_wing_room_counts
+    from .backends.chroma import _sqlite_wing_room_counts, hnsw_capacity_status
 
     counts = _sqlite_wing_room_counts(palace_path, "mempalace_drawers")
     if counts is not None:
@@ -2110,6 +2150,17 @@ def status(palace_path: str):
     if col is None:
         return
 
+    # Preflight HNSW divergence before falling back to the ChromaDB client
+    # path: count() on a diverged segment can hit the #1222 SIGSEGV/panic
+    # class, which a try/except around count() cannot catch. This fallback
+    # only runs when the direct sqlite read above was unavailable, so it's
+    # the one place in this function that still touches count() directly.
+    capacity_info = hnsw_capacity_status(palace_path, "mempalace_drawers")
+    if capacity_info.get("diverged"):
+        print(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+        print("  Run `mempalace repair --mode from-sqlite --archive-existing` first.")
+        return
+
     # Count by wing and room — paginate to avoid SQLite "too many SQL
     # variables" error on large palaces (see #802, #850).
     total = col.count()
@@ -2132,7 +2183,7 @@ def status(palace_path: str):
 def _print_status(total: int, wing_rooms: dict[str, dict[str, int]]) -> None:
     """Render the wing/room histogram shared by both status code paths."""
     print(f"\n{'=' * 55}")
-    print(f"  MemPalace Status — {total} drawers")
+    print(f"  MemPalace Status -- {total} drawers")
     print(f"{'=' * 55}\n")
     for wing, rooms in sorted(wing_rooms.items()):
         print(f"  WING: {wing}")

mempalace/normalize.py

@@ -5,7 +5,8 @@ normalize.py — Convert any chat export format to MemPalace transcript format.
 Supported:
     - Plain text with > markers (pass through)
     - Claude.ai JSON export
-    - ChatGPT conversations.json
+    - ChatGPT conversations.json (a single conversation, or the top-level
+      array of them that a real data export ships)
     - Claude Code JSONL (with tool_use/tool_result block capture)
     - OpenAI Codex CLI JSONL
     - Gemini CLI JSONL (~/.gemini/tmp/<project_hash>/chats/session-*.jsonl)
@@ -114,10 +115,10 @@ def strip_noise(text: str) -> str:
     return text.strip()
 
 
-def normalize(filepath: str) -> str:
-    """
-    Load a file and normalize to transcript format if it's a chat export.
-    Plain text files pass through unchanged.
+def _read_transcript_file(filepath: str) -> str:
+    """Read a transcript source file with the same safety checks normalize()
+    and normalize_conversations() both need: no symlinks, regular files only,
+    size-capped, BOM-tolerant.
     """
     flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
     if os.path.islink(filepath):
@@ -132,7 +133,7 @@ def normalize(filepath: str) -> str:
             raise IOError(f"File too large ({file_stat.st_size // (1024 * 1024)} MB): {filepath}")
         with os.fdopen(fd, "r", encoding="utf-8-sig", errors="replace") as f:
             fd = -1
-            content = f.read()
+            return f.read()
     except OSError as e:
         raise IOError(f"Could not read {filepath}: {e}") from e
     finally:
@@ -142,6 +143,14 @@ def normalize(filepath: str) -> str:
             except OSError:
                 pass
 
+
+def normalize(filepath: str) -> str:
+    """
+    Load a file and normalize to transcript format if it's a chat export.
+    Plain text files pass through unchanged.
+    """
+    content = _read_transcript_file(filepath)
+
     if not content.strip():
         return content
 
@@ -162,40 +171,96 @@ def normalize(filepath: str) -> str:
     return content
 
 
+def normalize_conversations(filepath: str) -> list:
+    """Like normalize(), but keeps each conversation in a bundle export as a
+    separate string instead of joining them into one.
+
+    A Claude.ai privacy export packs every conversation into a single JSON
+    file, and normalize() joins them with "\\n\\n".join(...) into one blob.
+    That collapses conversation boundaries, so content-hash dedup keyed on
+    the whole file breaks the moment the bundle is re-exported with one new
+    conversation added — the file-level hash changes even though none of
+    the existing conversations did. This returns the pieces un-joined so
+    callers can hash and dedup per conversation instead.
+
+    A ChatGPT data export is a bundle for the same reason: its
+    ``conversations.json`` is an array of conversations, so it splits per
+    conversation too.
+
+    Non-bundle formats (a single Claude Code session, plain text, ...)
+    always normalize to one conversation, so this returns a one-element
+    list for those — identical dedup granularity to before.
+    """
+    content = _read_transcript_file(filepath)
+
+    if not content.strip():
+        return []
+
+    lines = content.split("\n")
+    if sum(1 for line in lines if line.strip().startswith(">")) >= 3:
+        return [content]
+
+    ext = Path(filepath).suffix.lower()
+    if ext in (".json", ".jsonl") or content.strip()[:1] in ("{", "["):
+        split = _try_normalize_json_split(content)
+        if split:
+            return split
+
+    return [content]
+
+
 def _try_normalize_json(content: str) -> Optional[str]:
-    """Try all known JSON chat schemas."""
+    """Try all known JSON chat schemas, joining a multi-conversation bundle
+    into one string. See ``_try_normalize_json_split`` for the unjoined form.
+    """
+    split = _try_normalize_json_split(content)
+    if split is None:
+        return None
+    return "\n\n".join(split)
+
+
+def _try_normalize_json_split(content: str) -> Optional[list]:
+    """Try all known JSON chat schemas, returning each conversation found as
+    a separate list entry (bundle formats) or a single-element list.
+    """
 
     normalized = _try_claude_code_jsonl(content)
     if normalized:
-        return normalized
+        return [normalized]
 
     normalized = _try_codex_jsonl(content)
     if normalized:
-        return normalized
+        return [normalized]
 
     normalized = _try_gemini_jsonl(content)
     if normalized:
-        return normalized
+        return [normalized]
 
     normalized = _try_pi_jsonl(content)
     if normalized:
-        return normalized
+        return [normalized]
 
     try:
         data = json.loads(content)
     except json.JSONDecodeError:
         return None
 
-    for parser in (
-        _try_gemini_json,
-        _try_claude_ai_json,
-        _try_chatgpt_json,
-        _try_continue_json,
-        _try_slack_json,
-    ):
+    normalized = _try_gemini_json(data)
+    if normalized:
+        return [normalized]
+
+    split = _try_claude_ai_json_split(data)
+    if split:
+        return split
+
+    split = _try_chatgpt_export_json_split(data)
+    if split:
+        return split
+
+    for parser in (_try_chatgpt_json, _try_continue_json, _try_slack_json):
         normalized = parser(data)
         if normalized:
-            return normalized
+            return [normalized]
 
     return None
 
@@ -509,6 +574,16 @@ def _try_gemini_json(data) -> Optional[str]:
 
 def _try_claude_ai_json(data) -> Optional[str]:
     """Claude.ai JSON export: flat messages list or privacy export with chat_messages."""
+    split = _try_claude_ai_json_split(data)
+    if split is None:
+        return None
+    return "\n\n".join(split)
+
+
+def _try_claude_ai_json_split(data) -> Optional[list]:
+    """Same as ``_try_claude_ai_json`` but keeps each conversation in a
+    privacy export as its own list entry instead of joining them.
+    """
     if isinstance(data, dict):
         data = data.get("messages", data.get("chat_messages", []))
     if not isinstance(data, list):
@@ -526,13 +601,13 @@ def _try_claude_ai_json(data) -> Optional[str]:
             if len(messages) >= 2:
                 transcripts.append(_messages_to_transcript(messages))
         if transcripts:
-            return "\n\n".join(transcripts)
+            return transcripts
         return None
 
     # Flat messages list
     messages = _collect_claude_messages(data)
     if len(messages) >= 2:
-        return _messages_to_transcript(messages)
+        return [_messages_to_transcript(messages)]
     return None
 
 
@@ -557,8 +632,14 @@ def _collect_claude_messages(items) -> list:
 
 
 def _try_chatgpt_json(data) -> Optional[str]:
-    """ChatGPT conversations.json with mapping tree."""
-    if not isinstance(data, dict) or "mapping" not in data:
+    """ChatGPT conversations.json with mapping tree.
+
+    Every nested shape is type-checked rather than assumed: this parser is
+    reached from ``_try_chatgpt_export_json_split`` for each element of any
+    top-level JSON array, so it must return None on unrelated payloads that
+    merely carry a ``mapping`` key instead of raising.
+    """
+    if not isinstance(data, dict) or not isinstance(data.get("mapping"), dict):
         return None
     mapping = data["mapping"]
     messages = []
@@ -566,6 +647,8 @@ def _try_chatgpt_json(data) -> Optional[str]:
     root_id = None
     fallback_root = None
     for node_id, node in mapping.items():
+        if not isinstance(node, dict):
+            continue
         if node.get("parent") is None:
             if node.get("message") is None:
                 root_id = node_id
@@ -579,24 +662,65 @@ def _try_chatgpt_json(data) -> Optional[str]:
         visited = set()
         while current_id and current_id not in visited:
             visited.add(current_id)
-            node = mapping.get(current_id, {})
+            node = mapping.get(current_id)
+            if not isinstance(node, dict):
+                break
             msg = node.get("message")
-            if msg:
-                role = msg.get("author", {}).get("role", "")
+            if isinstance(msg, dict):
+                author = msg.get("author")
+                role = author.get("role", "") if isinstance(author, dict) else ""
                 content = msg.get("content", {})
-                parts = content.get("parts", []) if isinstance(content, dict) else []
+                parts = content.get("parts") if isinstance(content, dict) else None
+                if not isinstance(parts, list):
+                    parts = []
                 text = " ".join(str(p) for p in parts if isinstance(p, str) and p).strip()
                 if role == "user" and text:
                     messages.append(("user", text))
                 elif role == "assistant" and text:
                     messages.append(("assistant", text))
-            children = node.get("children", [])
-            current_id = children[0] if children else None
+            children = node.get("children")
+            next_id = children[0] if isinstance(children, list) and children else None
+            # Node ids index a dict and a visited set, so anything unhashable
+            # (a nested child object rather than an id) ends the walk.
+            current_id = next_id if isinstance(next_id, str) else None
     if len(messages) >= 2:
         return _messages_to_transcript(messages)
     return None
 
 
+def _try_chatgpt_export_json_split(data) -> Optional[list]:
+    """ChatGPT data export: top-level array of conversation objects.
+
+    The ``conversations.json`` OpenAI ships is an *array*, while
+    ``_try_chatgpt_json`` handles the single conversation object inside it.
+    Without this the whole export falls through to the plain-text path and is
+    chunked as raw JSON: the drawers hold serialized structure sliced at
+    arbitrary offsets, and every speaker turn is gone.
+
+    Each conversation is kept as its own segment rather than concatenated, so
+    per-conversation dedup survives a re-export (see ``normalize_conversations``);
+    the joined form is reached through ``_try_normalize_json``.
+
+    Runs after ``_try_gemini_json`` and ``_try_claude_ai_json_split`` and before
+    the ``_try_chatgpt_json``/``_try_continue_json``/``_try_slack_json`` loop.
+    That position is safe in both directions: Gemini requires a ``role="model"``
+    entry and Claude.ai requires ``chat_messages``/``messages`` on the first
+    element, neither of which a ChatGPT conversation object has, while Slack
+    entries carry no ``mapping`` and Continue.dev sessions are not arrays at
+    all, so this parser declines them and they fall through unchanged.
+    """
+    if not isinstance(data, list):
+        return None
+
+    transcripts = []
+    for convo in data:
+        transcript = _try_chatgpt_json(convo)
+        if transcript:
+            transcripts.append(transcript)
+    # None, not [], so an array of other JSON still reaches the later parsers.
+    return transcripts or None
+
+
 def _try_slack_json(data) -> Optional[str]:
     """
     Slack channel export: [{"type": "message", "user": "...", "text": "..."}]

mempalace/onboarding.py

@@ -57,7 +57,7 @@ DEFAULT_WINGS = {
 
 
 def _hr():
-    print(f"\n{'─' * 58}")
+    print(f"\n{'-' * 58}")
 
 
 def _header(text):
@@ -121,9 +121,9 @@ def _ask_mode() -> str:
 """)
     print("  How are you using MemPalace?")
     print()
-    print("    [1]  Work     — notes, projects, clients, colleagues, decisions")
-    print("    [2]  Personal — diary, family, health, relationships, reflections")
-    print("    [3]  Both     — personal and professional mixed")
+    print("    [1]  Work     -- notes, projects, clients, colleagues, decisions")
+    print("    [2]  Personal -- diary, family, health, relationships, reflections")
+    print("    [3]  Both     -- personal and professional mixed")
     print()
 
     while True:
@@ -430,7 +430,7 @@ def run_onboarding(
             print()
             if _yn("  Add any of these to your registry?"):
                 for e in detected:
-                    ans = input(f"    {e['name']} — (p)erson, (s)kip? ").strip().lower()
+                    ans = input(f"    {e['name']} -- (p)erson, (s)kip? ").strip().lower()
                     if ans == "p":
                         rel = input(f"    Relationship/role for {e['name']}? ").strip()
                         ctx = (
@@ -439,7 +439,7 @@ def run_onboarding(
                             else (
                                 "work"
                                 if mode == "work"
-                                else input("    Context — (p)ersonal or (w)ork? ")
+                                else input("    Context -- (p)ersonal or (w)ork? ")
                                 .strip()
                                 .lower()
                                 .replace("w", "work")

mempalace/palace.py

@@ -11,6 +11,7 @@ import os
 import re
 import sys
 import threading
+import time
 from typing import Optional
 
 from .backends import (
@@ -26,7 +27,11 @@ from .backends import (
     resolve_backend_for_palace,
 )
 from .backends.embedding_wrapper import EmbeddingCollection
-from .entity_detector import _apply_known_systems_prepass, _get_coca_filter
+from .entity_detector import (
+    _apply_known_systems_prepass,
+    _collapse_long_ascii_runs,
+    _get_coca_filter,
+)
 
 logger = logging.getLogger("mempalace_mcp")
 
@@ -76,6 +81,23 @@ NORMALIZE_VERSION = 2
 _VALIDATED_IDENTITY: set = set()
 
 
+def clear_validated_embedder_identity(palace_path: Optional[str] = None) -> None:
+    """Drop cached embedder-identity verdicts so the next open re-checks.
+
+    Read-only opens of an empty collection can mark a key as validated without
+    recording identity on disk (``create=False``). When MCP later promotes that
+    reader to a writable owner, the writable open must re-run enforcement so
+    the first drawers still get labelled with the active model.
+    """
+    if palace_path is None:
+        _VALIDATED_IDENTITY.clear()
+        return
+    palace_key = str(palace_path)
+    stale = [key for key in _VALIDATED_IDENTITY if key and key[0] == palace_key]
+    for key in stale:
+        _VALIDATED_IDENTITY.discard(key)
+
+
 def _enforce_embedder_identity(collection, palace_path, collection_name, *, create) -> None:
     """Check (and, for a brand-new collection, record) embedder identity (RFC 001).
 
@@ -138,10 +160,29 @@ def _enforce_embedder_identity(collection, palace_path, collection_name, *, crea
         return
 
     if state == "unknown" and stored is None:
+        # Preflight HNSW divergence before touching count(): this is the
+        # universal chokepoint every tool passes through via
+        # get_collection(), and count() on a diverged segment can raise
+        # chromadb's rust-level pyo3_runtime.PanicException or hard-segfault
+        # (#1222) -- neither of which the except Exception below can catch,
+        # since a native crash takes the whole process down regardless of
+        # any Python try/except. A diverged palace must never reach count()
+        # here; this bookkeeping-only identity check simply skips itself
+        # (count treated as unknown, matching the except-Exception fallback
+        # already below) rather than risk the read.
         try:
-            count = collection.count()
+            from .backends.chroma import hnsw_capacity_status
+
+            diverged = hnsw_capacity_status(str(palace_path), str(collection_name)).get("diverged")
         except Exception:
+            diverged = False
+        if diverged:
             count = None
+        else:
+            try:
+                count = collection.count()
+            except Exception:
+                count = None
         if count == 0:
             if create:
                 try:
@@ -165,10 +206,15 @@ def get_collection(
     collection_name: Optional[str] = None,
     create: bool = True,
     backend: Optional[str] = None,
+    read_only: bool = False,
     _skip_identity_check: bool = False,
 ):
     """Get the palace collection through the backend layer.
 
+    ``read_only=True`` asks local backends to open storage without schema
+    initialization, migrations, or metadata writes. Backends that support a
+    genuine read-only mode receive it through the backend ``options`` mapping.
+
     ``_skip_identity_check`` bypasses the embedder-identity enforcement so the
     ``set-embedder`` override path can open a palace whose recorded model
     differs from the current one (the very state it exists to repair).
@@ -179,20 +225,52 @@ def get_collection(
         collection_name = get_configured_collection_name()
     backend_obj = get_backend_for_palace(palace_path, explicit=backend)
     palace_ref = PalaceRef(id=palace_path, local_path=palace_path)
+    backend_options = {"read_only": True} if read_only else None
+    preferred_kwargs = {
+        "palace": palace_ref,
+        "collection_name": collection_name,
+        "create": create,
+    }
+    if backend_options is not None:
+        preferred_kwargs["options"] = backend_options
     try:
-        collection = backend_obj.get_collection(
-            palace=palace_ref,
-            collection_name=collection_name,
-            create=create,
-        )
+        collection = backend_obj.get_collection(**preferred_kwargs)
     except TypeError as exc:
-        if "unexpected keyword argument 'palace'" not in str(exc):
+        msg = str(exc)
+        # Plugin backends may still use the pre-options signature. Drop
+        # ``options`` first so read_only degrades gracefully instead of
+        # hard-failing TypeError on third-party entry points.
+        if backend_options is not None and "options" in msg:
+            preferred_kwargs.pop("options", None)
+            try:
+                collection = backend_obj.get_collection(**preferred_kwargs)
+            except TypeError as nested:
+                if "unexpected keyword argument 'palace'" not in str(nested):
+                    raise
+                collection = backend_obj.get_collection(
+                    palace_path,
+                    collection_name=collection_name,
+                    create=create,
+                )
+        elif "unexpected keyword argument 'palace'" not in msg:
             raise
-        collection = backend_obj.get_collection(
-            palace_path,
-            collection_name=collection_name,
-            create=create,
-        )
+        else:
+            legacy_kwargs = {
+                "collection_name": collection_name,
+                "create": create,
+            }
+            if backend_options is not None:
+                legacy_kwargs["options"] = backend_options
+            try:
+                collection = backend_obj.get_collection(palace_path, **legacy_kwargs)
+            except TypeError as nested:
+                if backend_options is None or "options" not in str(nested):
+                    raise
+                collection = backend_obj.get_collection(
+                    palace_path,
+                    collection_name=collection_name,
+                    create=create,
+                )
     if "requires_explicit_embeddings" in getattr(backend_obj, "capabilities", frozenset()):
         collection = EmbeddingCollection(collection)
     if not _skip_identity_check:
@@ -332,6 +410,29 @@ def resolve_backend_name(palace_path: str, explicit: Optional[str] = None) -> st
     return selected
 
 
+_MULTI_PROCESS_WRITER_BACKENDS = frozenset({"pgvector", "qdrant"})
+
+
+def backend_requires_single_writer(backend_name: str) -> bool:
+    """Return whether a backend needs one process-lifetime writer owner.
+
+    Local file-backed backends cannot safely coordinate independent long-lived
+    clients by serializing only individual calls: each process may retain
+    SQLite/WAL, FTS, or vector-index state across operations. Unknown and
+    plugin backends are treated conservatively. Only backends whose storage
+    service is explicitly responsible for cross-process concurrency opt out.
+    """
+    normalized = backend_name.strip().lower()
+    if normalized == "milvus":
+        # Only embedded Milvus Lite is local single-writer storage. A remote
+        # Milvus server or Zilliz Cloud coordinates concurrent clients itself.
+        from .backends.milvus import milvus_uri_is_server
+        from .config import MempalaceConfig
+
+        return not milvus_uri_is_server(MempalaceConfig().milvus_uri)
+    return normalized not in _MULTI_PROCESS_WRITER_BACKENDS
+
+
 def get_backend_for_palace(palace_path: str, explicit: Optional[str] = None):
     """Return the resolved backend instance for ``palace_path``."""
     return get_backend(resolve_backend_name(palace_path, explicit=explicit))
@@ -530,9 +631,12 @@ def _candidate_entity_words(text: str) -> list:
             except re.error:
                 continue
         _CANDIDATE_RX_CACHE = rxs
+    # Defuse ReDoS on long ASCII blobs before matching (#2063); see
+    # entity_detector._collapse_long_ascii_runs.
+    stripped = _collapse_long_ascii_runs(text)
     words = []
     for rx in _CANDIDATE_RX_CACHE:
-        words.extend(rx.findall(text))
+        words.extend(rx.findall(stripped))
     return words
 
 
@@ -725,6 +829,7 @@ def mine_lock(source_file: str):
     Prevents multiple agents from mining the same file simultaneously,
     which causes duplicate drawers when the delete+insert cycle interleaves.
     """
+    _maybe_reap_stale_mine_locks()
     lock_path = _mine_lock_path(source_file)
     lf = _acquire_mine_lock_file(lock_path)
     try:
@@ -904,6 +1009,92 @@ def _cleanup_mine_lock_file(lock_path: str) -> None:
             lf.close()
 
 
+def reap_stale_mine_locks(*, min_age_seconds: int = 3600) -> tuple[int, int]:
+    """Best-effort garbage collection for orphaned per-source-file mine locks.
+
+    ``_cleanup_mine_lock_file`` reclaims a lock file correctly on the happy
+    path (see its docstring) — but only for the *specific* lock a
+    :func:`mine_lock` context manager just released. A process that dies
+    before reaching its own ``finally`` block (killed, crashed, force-quit,
+    host reboot) never runs that cleanup, and nothing else in this codebase
+    later revisits that lock file. Locks in ``~/.mempalace/locks/`` can
+    accumulate unboundedly over time as a result — one long-lived
+    installation was found with 5,636 stale entries, the oldest several
+    months old, none held by any live process (confirmed via ``lsof``).
+
+    This reuses :func:`_cleanup_mine_lock_file` itself for the actual
+    removal — same nonblocking-flock-reacquire safety mechanism, same
+    Windows/POSIX handling, no duplicated locking logic. A lock is only
+    ever removed after *this* process re-acquires it, so anything
+    genuinely held by a live process is left untouched regardless of
+    ``min_age_seconds``. ``min_age_seconds`` is a courtesy throttle only —
+    it avoids racing a lock that was *just* released and may still be
+    mid-rendezvous with a waiter on the same pathname; it is not a
+    substitute for the flock check, which is what actually makes removal
+    safe.
+
+    Skips ``mine_palace_*.lock`` files — those belong to the newer
+    palace-level :func:`mine_palace_lock` and have their own
+    lifecycle/holder tracking; this targets only the per-source-file locks
+    :func:`mine_lock` creates via :func:`_mine_lock_path`.
+
+    Returns ``(reaped, skipped)`` counts, for logging/testing — callers
+    don't need to act on them.
+    """
+    lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks")
+    try:
+        entries = os.listdir(lock_dir)
+    except OSError:
+        return 0, 0
+
+    now = time.time()
+    reaped = 0
+    skipped = 0
+    for name in entries:
+        if not name.endswith(".lock") or name.startswith("mine_palace_"):
+            continue
+        lock_path = os.path.join(lock_dir, name)
+        try:
+            if now - os.path.getmtime(lock_path) < min_age_seconds:
+                continue
+        except OSError:
+            continue
+        _cleanup_mine_lock_file(lock_path)
+        if os.path.exists(lock_path):
+            skipped += 1
+        else:
+            reaped += 1
+    return reaped, skipped
+
+
+_LOCK_REAP_INTERVAL_SECONDS = 900  # 15 minutes between opportunistic sweeps
+
+
+def _maybe_reap_stale_mine_locks() -> None:
+    """Throttled, opportunistic call site for :func:`reap_stale_mine_locks`.
+
+    Runs at most once per ``_LOCK_REAP_INTERVAL_SECONDS``, piggybacking on
+    the natural cadence of mine operations rather than requiring a
+    background thread, a scheduled task, or any new CLI surface. Failures
+    are swallowed — lock maintenance must never be allowed to break an
+    actual mine.
+    """
+    lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks")
+    marker = os.path.join(lock_dir, ".last_reap")
+    try:
+        if (
+            os.path.exists(marker)
+            and time.time() - os.path.getmtime(marker) < _LOCK_REAP_INTERVAL_SECONDS
+        ):
+            return
+        os.makedirs(lock_dir, exist_ok=True)
+        open(marker, "a").close()
+        os.utime(marker, None)
+        reap_stale_mine_locks()
+    except Exception:
+        logger.debug("Opportunistic mine-lock reap failed", exc_info=True)
+
+
 class MineAlreadyRunning(RuntimeError):
     """Raised when another `mempalace mine` already holds the per-palace lock."""
 
@@ -1214,10 +1405,26 @@ mine_global_lock = mine_palace_lock
 
 
 def _metadata_matches_extract_mode(meta: dict, extract_mode: Optional[str]) -> bool:
+    """Scope a drawer to a convo-miner extraction mode.
+
+    A missing ``extract_mode`` is treated as a legacy exchange-mode row
+    ONLY when the drawer is otherwise convo_miner's own (no ``ingest_mode``
+    at all -- pre-``ingest_mode``-schema convo drawers -- or the convo
+    miner's own ``"convos"`` tag). A drawer from a different producer that
+    never set ``extract_mode`` because it was never meant to carry one --
+    e.g. the sweeper's ``ingest_mode="sweep"`` rows -- must not match: the
+    legacy-compat rule otherwise scoops every sweeper drawer for a shared
+    transcript into convo_miner's default "exchange" purge/idempotency
+    scope and silently deletes them on the very next re-mine (#104).
+    """
     if extract_mode is None:
         return True
     stored_mode = meta.get("extract_mode")
-    return stored_mode == extract_mode or (extract_mode == "exchange" and stored_mode is None)
+    if stored_mode == extract_mode:
+        return True
+    if stored_mode is not None:
+        return False
+    return extract_mode == "exchange" and meta.get("ingest_mode") in (None, "convos")
 
 
 def file_already_mined(
@@ -1351,3 +1558,61 @@ def prefetch_mined_set(
     except Exception:
         logger.warning("prefetch_mined_set: partial fetch, %d files loaded", len(mined))
     return mined
+
+
+def prefetch_content_hashes(
+    collection, extract_mode: Optional[str] = None
+) -> dict[tuple[str, str], str]:
+    """Pre-fetch (wing, content_hash) -> source_file for drawers already
+    filed at the current NORMALIZE_VERSION, in one bulk pass.
+
+    Repeated exports from Claude/ChatGPT land under a new filename each run
+    (timestamped bundle, regenerated slug, etc.) even when the conversation
+    itself hasn't changed. `prefetch_mined_set` only recognizes a file as
+    already-mined by its exact path, so the same conversation re-exported
+    under a new path always looked "new" and got re-mined as a duplicate
+    drawer. This does the same bulk scan but keyed on the SHA-256 of the
+    normalized transcript text, so the convo miner can recognize "this exact
+    conversation is already filed under a different path" and skip it.
+
+    Keyed by (wing, content_hash) rather than content_hash alone — mining
+    the same transcript into a second wing is a deliberate re-file, not a
+    duplicate, and should produce real drawers in that wing rather than
+    just the registry sentinel.
+
+    A drawer's ``content_hash`` metadata may hold several comma-joined
+    SHA-256 hashes: a privacy-export bundle normalizes to one conversation
+    per drawer set, but the hash is computed per conversation so that a
+    re-export with one new conversation added doesn't change the hash of
+    the ones that didn't. Only the first source_file seen for a given
+    (wing, hash) pair is kept — good enough to detect and skip a repeat,
+    the point is not to track every alias.
+    """
+    hashes: dict[tuple[str, str], str] = {}
+    try:
+        total = collection.count()
+        offset = 0
+        while offset < total:
+            batch = collection.get(limit=1000, offset=offset, include=["metadatas"])
+            for meta in batch["metadatas"]:
+                meta = meta or {}
+                content_hash_field = meta.get("content_hash")
+                src = meta.get("source_file")
+                wing = meta.get("wing")
+                if not content_hash_field or not src or not wing:
+                    continue
+                if not _metadata_matches_extract_mode(meta, extract_mode):
+                    continue
+                version = meta.get("normalize_version", 1)
+                if version < NORMALIZE_VERSION:
+                    continue
+                for content_hash in content_hash_field.split(","):
+                    key = (wing, content_hash)
+                    if content_hash and key not in hashes:
+                        hashes[key] = src
+            if not batch["ids"]:
+                break
+            offset += len(batch["ids"])
+    except Exception:
+        logger.warning("prefetch_content_hashes: partial fetch, %d hashes loaded", len(hashes))
+    return hashes

mempalace/repair.py

@@ -186,14 +186,70 @@ def _paginate_ids(col, where=None):
         except Exception:
             try:
                 r = col.get(where=where, include=[], limit=page)
-                new_ids = [i for i in r["ids"] if i not in set(ids)]
-                if not new_ids:
-                    break
-                ids.extend(new_ids)
-                offset += len(new_ids)
-                continue
-            except Exception:
+            except Exception as fallback_exc:
+                # Both the offset request AND the no-offset fallback failed.
+                # Whatever is in ``ids`` so far is a partial (or empty) prefix
+                # of the collection, not a complete listing. Returning it
+                # would make scan_palace/rebuild act on a truncated palace as
+                # though it were whole (or, on the first page, print "Nothing
+                # to scan." for a palace we never actually read). The whole
+                # point of this path is to fail loudly rather than silently
+                # truncate, so raise instead of breaking.
+                raise RuntimeError(
+                    f"_paginate_ids: both offset-based pagination and the "
+                    f"no-offset fallback failed after collecting {len(ids)} "
+                    f"ids. Refusing to return a silently truncated ID list -- "
+                    f"investigate the collection, or use a repair mode that "
+                    f"does not depend on offset paging (e.g. --mode "
+                    f"from-sqlite). Underlying error: {fallback_exc}"
+                ) from fallback_exc
+            new_ids = [i for i in r["ids"] if i not in set(ids)]
+            if not new_ids:
+                # Offset is broken and the no-offset fallback always
+                # re-fetches the same first `page` results, so it can never
+                # advance past that boundary. Landing exactly on that
+                # boundary (len(ids) >= page) is ambiguous from the fetched
+                # page alone: it could mean "collection has exactly `page`
+                # ids, genuinely complete" or "collection has more and we
+                # are truncating" -- those two states are indistinguishable
+                # without an authoritative count. Disambiguate against the
+                # collection's own count() (same pattern already used by
+                # _verify_collection_count in this file; shares its known
+                # native-crash-surface caveat on a corrupted collection,
+                # which is out of scope for this fix -- see the deferred
+                # HNSW-preflight-sweep work).
+                #
+                # ``col.count()`` is collection-wide and takes no ``where``
+                # argument, so it is only authoritative when this call is
+                # UNFILTERED. With a ``where`` filter, a global count > the
+                # collected rows does NOT prove the filtered set is truncated
+                # (the extra rows may belong to other wings), so we cannot use
+                # it as proof and treat filtered completeness as unknown
+                # rather than raising a false-positive truncation error.
+                # This raise is intentionally OUTSIDE the narrow try/except
+                # above so it propagates instead of being swallowed as a
+                # get()-failure.
+                if where is None and len(ids) >= page and len(r["ids"] or []) >= page:
+                    try:
+                        total = col.count()
+                    except Exception:
+                        total = None
+                    if total is None or total > len(ids):
+                        raise RuntimeError(
+                            f"_paginate_ids: offset-based pagination failed "
+                            f"and the no-offset fallback cannot advance past "
+                            f"the first {page} results ({len(ids)} collected, "
+                            f"collection reports "
+                            f"{total if total is not None else 'an unreadable'} "
+                            f"total). Refusing to return a silently truncated "
+                            f"ID list -- investigate the collection, or use a "
+                            f"repair mode that does not depend on offset "
+                            f"paging (e.g. --mode from-sqlite)."
+                        )
                 break
+            ids.extend(new_ids)
+            offset += len(new_ids)
+            continue
         n = len(r["ids"]) if r["ids"] else 0
         if n == 0:
             break
@@ -311,11 +367,71 @@ def _rebuild_collection_via_temp(
             pass
         return rebuilt
     except Exception as exc:
-        try:
-            _delete_collection_if_exists(backend, palace_path, temp_name)
-        except Exception:
-            pass
-        raise RebuildCollectionError(str(exc), live_replaced=live_replaced) from exc
+        if not live_replaced:
+            # The live collection was never touched -- the temp build is a
+            # discardable in-progress copy, safe to clean up on failure.
+            try:
+                _delete_collection_if_exists(backend, palace_path, temp_name)
+            except Exception:
+                pass
+            raise RebuildCollectionError(str(exc), live_replaced=live_replaced) from exc
+        # The live collection was already deleted (line 294) before this
+        # failure. `temp_name` is now the ONLY intact, verified copy of the
+        # data left on disk -- never delete it here. Point the operator
+        # at it instead of destroying the one thing that can still recover.
+        raise RebuildCollectionError(
+            f"{exc}. The live collection '{collection_name}' was already "
+            f"replaced and the re-upload into it failed. The fully-verified "
+            f"pre-swap copy survives under '{temp_name}' -- do NOT delete it. "
+            f"Recover by removing the broken '{collection_name}' collection "
+            f"and promoting '{temp_name}' in its place.",
+            live_replaced=live_replaced,
+        ) from exc
+
+
+def _promote_temp_collection(
+    backend,
+    palace_path: str,
+    temp_name: str,
+    collection_name: str,
+    expected: int,
+    batch_size: int,
+    progress=print,
+) -> int:
+    """Recover a failed live-swap by promoting the verified temp copy.
+
+    `_rebuild_collection_via_temp` fully verifies `temp_name` before it ever
+    touches the live collection. If the post-swap re-upload into a fresh live
+    collection then fails, the honest recovery is to copy directly from that
+    verified temp copy -- NOT to restore a pre-rebuild sqlite3 file backup,
+    whose on-disk HNSW segment directories were already destroyed by the
+    live-collection delete and would leave the palace referencing segment
+    UUIDs that no longer exist on disk.
+    """
+    temp_col = backend.get_collection(palace_path, temp_name)
+    ids, docs, metas = _extract_drawers(temp_col, expected, batch_size)
+    _delete_collection_if_exists(backend, palace_path, collection_name)
+    new_col = backend.create_collection(palace_path, collection_name)
+    promoted = 0
+    for i in range(0, len(ids), batch_size):
+        new_col.upsert(
+            documents=docs[i : i + batch_size],
+            ids=ids[i : i + batch_size],
+            metadatas=metas[i : i + batch_size],
+        )
+        promoted += len(ids[i : i + batch_size])
+        progress(f"  Promoted {promoted}/{expected} drawers from verified temp copy...")
+    _verify_collection_count(new_col, expected, "promoted temp collection")
+    # Promotion has already fully succeeded and verified at this point --
+    # cleaning up the now-redundant temp copy is best-effort, matching the
+    # identical post-success cleanup in _rebuild_collection_via_temp above.
+    # A failure here (e.g. a transient Windows file lock) must not turn a
+    # successful recovery into a reported failure.
+    try:
+        _delete_collection_if_exists(backend, palace_path, temp_name)
+    except Exception:
+        pass
+    return promoted
 
 
 def scan_palace(palace_path=None, only_wing=None, collection_name: Optional[str] = None):
@@ -331,6 +447,17 @@ def scan_palace(palace_path=None, only_wing=None, collection_name: Optional[str]
     print(f"\n  Palace: {palace_path}")
     print("  Loading...")
 
+    # Preflight HNSW divergence before opening the collection: count() on a
+    # diverged segment can hit the #1222 SIGSEGV/panic class, which a
+    # try/except around count() cannot catch (a native crash takes the
+    # whole process down). scan_palace is meant to find corruption, not
+    # crash on the exact corruption class it should be reporting.
+    capacity_info = hnsw_capacity_status(palace_path, collection_name)
+    if capacity_info.get("diverged"):
+        print(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+        print(index_read_recovery_guidance())
+        return set(), set()
+
     col = ChromaBackend().get_collection(palace_path, collection_name)
 
     where = {"wing": only_wing} if only_wing else None
@@ -404,7 +531,7 @@ def prune_corrupt(palace_path=None, confirm=False, collection_name: Optional[str
     bad_file = os.path.join(palace_path, "corrupt_ids.txt")
 
     if not os.path.exists(bad_file):
-        print("  No corrupt_ids.txt found — run scan first.")
+        print("  No corrupt_ids.txt found -- run scan first.")
         return
 
     with open(bad_file) as f:
@@ -412,10 +539,19 @@ def prune_corrupt(palace_path=None, confirm=False, collection_name: Optional[str
     print(f"  {len(bad_ids):,} corrupt IDs queued for deletion")
 
     if not confirm:
-        print("\n  DRY RUN — no deletions performed.")
+        print("\n  DRY RUN -- no deletions performed.")
         print("  Re-run with --confirm to actually delete.")
         return
 
+    # Preflight HNSW divergence before opening the collection — see
+    # scan_palace's identical guard for why this can't rely on except
+    # Exception around count() alone.
+    capacity_info = hnsw_capacity_status(palace_path, collection_name)
+    if capacity_info.get("diverged"):
+        print(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+        print(index_read_recovery_guidance())
+        return
+
     col = ChromaBackend().get_collection(palace_path, collection_name)
     before = col.count()
     print(f"  Collection size before: {before:,}")
@@ -441,7 +577,7 @@ def prune_corrupt(palace_path=None, confirm=False, collection_name: Optional[str
     after = col.count()
     print(f"\n  Deleted: {deleted:,}")
     print(f"  Failed:  {failed:,}")
-    print(f"  Collection size: {before:,} → {after:,}")
+    print(f"  Collection size: {before:,} -> {after:,}")
 
 
 # ChromaDB's ``collection.get()`` enforces an internal default ``limit``
@@ -994,7 +1130,7 @@ def rebuild_index(
         return
 
     progress(f"\n{'=' * 55}")
-    progress("  MemPalace Repair — Index Rebuild")
+    progress("  MemPalace Repair -- Index Rebuild")
     progress(f"{'=' * 55}\n")
     progress(f" Palace: {palace_path}")
 
@@ -1018,7 +1154,44 @@ def rebuild_index(
     if preflight is not None:
         return
 
-    backend = ChromaBackend()
+    # Preflight HNSW divergence before opening the collection: status()
+    # already has this same guard via hnsw_capacity_status (docstring above
+    # this module's status() references it as "the safe pattern"), but
+    # rebuild_index -- the legacy rebuild the CLI's rebuild-index subcommand
+    # dispatches straight to -- opens the collection and calls col.count()
+    # directly, wrapped only in except Exception, which cannot catch the
+    # #1222 SIGSEGV/panic class a diverged segment triggers.
+    capacity_info = hnsw_capacity_status(palace_path, collection_name)
+    if capacity_info.get("diverged"):
+        progress(f"\n  HNSW index is diverged: {capacity_info.get('message', '')}")
+        progress(index_read_recovery_guidance())
+        return
+
+    # Hold the palace writer lease for the complete snapshot -> rebuild/swap
+    # -> cleanup cycle. A writer landing after the snapshot but before the
+    # rebuilt collection becomes authoritative would otherwise be lost from
+    # the rebuilt index and recreate SQLite/HNSW divergence.
+    from .palace import mine_palace_lock
+
+    with mine_palace_lock(palace_path):
+        _rebuild_index_under_lease(
+            backend=ChromaBackend(),
+            palace_path=palace_path,
+            collection_name=collection_name,
+            confirm_truncation_ok=confirm_truncation_ok,
+            progress=progress,
+        )
+
+
+def _rebuild_index_under_lease(
+    *,
+    backend,
+    palace_path: str,
+    collection_name: str,
+    confirm_truncation_ok: bool,
+    progress: Callable[[str], None],
+):
+    """Run rebuild_index's snapshot/rebuild body under its writer lease."""
     try:
         col = backend.get_collection(palace_path, collection_name)
         total = col.count()
@@ -1078,18 +1251,37 @@ def rebuild_index(
     except RebuildCollectionError as e:
         progress(f"\n  ERROR during rebuild: {e}")
         progress("  Rebuild aborted before completion.")
-        if e.live_replaced and os.path.exists(backup_path):
-            progress(f"  Restoring from backup: {backup_path}")
+        if e.live_replaced:
+            # Restoring the pre-rebuild chroma.sqlite3 file here would be
+            # misleading: the live collection's on-disk HNSW segment
+            # directories were already destroyed by the delete that
+            # preceded this failure, so a sqlite-only restore leaves the
+            # palace referencing segment UUIDs that no longer exist.
+            # The verified good copy is the temp collection instead --
+            # promote it directly.
+            temp_name = f"{collection_name}__repair_tmp"
+            progress(f"  Attempting recovery: promoting verified copy from '{temp_name}'...")
             try:
                 _close_chroma_handles(palace_path, backend=backend)
-                _delete_collection_if_exists(backend, palace_path, collection_name)
-                _copy_file_no_follow(backup_path, sqlite_path, replace=True)
-                progress("  Backup restored. Palace is back to pre-repair state.")
-            except Exception as restore_error:
-                progress(f"  Backup restore failed: {restore_error}")
-                progress(f"  Manual restore required from: {backup_path}")
-        elif e.live_replaced:
-            progress("  No backup available. Re-mine from source files to recover.")
+                _promote_temp_collection(
+                    backend,
+                    palace_path,
+                    temp_name,
+                    collection_name,
+                    len(all_ids),
+                    batch_size,
+                    progress=progress,
+                )
+                progress(
+                    "  Recovery succeeded: live collection restored from the verified temp copy."
+                )
+            except Exception as promote_error:
+                progress(f"  Automatic recovery failed: {promote_error}")
+                progress(
+                    f"  The verified pre-swap copy still survives under '{temp_name}' -- "
+                    f"do NOT delete it. Recover manually by promoting it, or re-mine "
+                    f"from source files."
+                )
         else:
             print("  Live collection was not replaced; leaving the original palace untouched.")
         raise
@@ -1098,6 +1290,23 @@ def rebuild_index(
 
     print(f"\n  Repair complete. {filed} drawers rebuilt.")
     print("  HNSW index is now clean with cosine distance metric.")
+
+    # rebuild_index only ever touches collection_name (drawers by default).
+    # status() checks divergence for BOTH drawers and closets and recommends
+    # --mode from-sqlite (which rebuilds both via _recoverable_collections()),
+    # but a caller who ran this legacy rebuild instead would otherwise see
+    # an unqualified "Repair complete" even when closets remains diverged
+    # and just as capable of crashing reads via the same #1222 mechanism (#13).
+    if collection_name == _drawers_collection_name():
+        closets_info = hnsw_capacity_status(palace_path, CLOSETS_COLLECTION_NAME)
+        if closets_info.get("diverged"):
+            print(
+                f"\n  NOTE: the closets index is still diverged "
+                f"({closets_info.get('message', '')}).\n"
+                "  This rebuild only covers drawers. Run "
+                "`mempalace repair --mode from-sqlite --archive-existing`\n"
+                "  to rebuild closets too."
+            )
     print(f"\n{'=' * 55}\n")
 
 
@@ -1260,6 +1469,13 @@ def extract_via_sqlite(palace_path: str, collection_name: str) -> Iterator[tuple
     returned as the document; this matches how chromadb itself stores
     ``add(documents=...)``.
 
+    Driven from ``embeddings`` (LEFT JOIN ``embedding_metadata``), not
+    the other way around: an embedding with zero ``embedding_metadata``
+    rows — a sparse historical write with no ``chroma:document`` and no
+    other key, the same condition ``_extract_drawers`` already sanitizes
+    for the collection-layer path, see #1458 — must still be yielded
+    with an empty metadata dict, not silently excluded by the join.
+
     Silent on missing palace, missing ``chroma.sqlite3``, or unknown
     collection name — yields nothing. Callers that need to distinguish
     "empty collection" from "collection not present" should query
@@ -1289,15 +1505,21 @@ def extract_via_sqlite(palace_path: str, collection_name: str) -> Iterator[tuple
             """
             SELECT e.embedding_id, em.key, em.string_value, em.int_value,
                    em.float_value, em.bool_value
-            FROM embedding_metadata em
-            JOIN embeddings e ON em.id = e.id
+            FROM embeddings e
+            LEFT JOIN embedding_metadata em ON em.id = e.id
             WHERE e.segment_id = ?
-            ORDER BY em.id
+            ORDER BY e.id
             """,
             (segment_id,),
         ):
             if emb_id not in per_id:
                 order.append(emb_id)
+            if key is None:
+                # LEFT JOIN unmatched row: this embedding has zero
+                # embedding_metadata rows. `order`/`per_id` already
+                # account for it via the defaultdict below; nothing to
+                # merge for this row.
+                continue
             if sv is not None:
                 per_id[emb_id][key] = sv
             elif iv is not None:
@@ -1355,6 +1577,7 @@ def rebuild_from_sqlite(
     *,
     archive_existing_dest: bool = False,
     batch_size: int = 1000,
+    dry_run: bool = False,
 ) -> dict[str, int]:
     """Rebuild a palace by reading drawers from ``source_palace``'s
     ``chroma.sqlite3`` and upserting them into a fresh palace at
@@ -1387,6 +1610,17 @@ def rebuild_from_sqlite(
       instead. Used by the in-place CLI flow where ``--source`` defaults
       to the same path as ``--palace``.
 
+    ``dry_run`` (CLI: ``--dry-run``) previews the rebuild without making any
+    change: source validation runs as normal, then per-collection row counts
+    are read from the source SQLite and printed, and the function returns
+    those would-be counts *without* archiving the existing palace, taking the
+    mine-lock, creating collections, or re-embedding (#2095, #2133). Useful
+    before a multi-hour rebuild on a large palace. A dry run returns a
+    populated dict (one key per recoverable collection) so CLI callers treat
+    it as success; a validation refusal still returns ``{}`` exactly as a real
+    run would. If SQLite row counts cannot be read, the preview fails closed
+    with ``{}`` rather than inventing zeros.
+
     Returns a ``{collection_name: row_count}`` dict so callers (CLI,
     tests) can verify the per-collection rebuild count without parsing
     stdout. A successful rebuild always returns a dict with one key per
@@ -1436,7 +1670,7 @@ def rebuild_from_sqlite(
     in_place = source_palace == dest_palace
 
     print(f"\n{'=' * 55}")
-    print("  MemPalace Repair — Rebuild from SQLite")
+    print("  MemPalace Repair -- Rebuild from SQLite")
     print(f"{'=' * 55}\n")
     print(f"  Source: {source_palace}")
     print(f"  Dest:   {dest_palace}")
@@ -1471,11 +1705,221 @@ def rebuild_from_sqlite(
             )
             return {}
 
+    # --dry-run: validation has passed, so report what a real run would do
+    # and stop before the first irreversible step (mine-lock + archive).
+    # Counts come from ``sqlite_drawer_count`` — the same SQLite ground-truth
+    # helper repair uses elsewhere — so the preview matches the per-collection
+    # counts a real rebuild upserts. Reads the original ``source_palace``
+    # (not yet archived). Must never take the mine-lock or rename anything.
+    if dry_run:
+        return _preview_rebuild_from_sqlite(
+            source_palace=source_palace,
+            dest_palace=dest_palace,
+            in_place=in_place,
+        )
+
+    # Acquire the single-writer mine-lock BEFORE the archive/rename. The
+    # rebuild upserts into ``dest_palace`` through the same backend write
+    # path that takes ``mine_palace_lock`` per batch; if a daemon or a
+    # concurrent mine already holds it, those upserts used to fail *after*
+    # ``shutil.move`` had already stranded the existing palace aside
+    # (renamed to ``.pre-rebuild-…`` with no rebuilt replacement, leaving a
+    # partial/archived mess). Taking the lock up front makes contention
+    # fail CLEAN — no archive, no partial dest, the palace left untouched —
+    # and runs the whole rebuild as one writer (the inner per-batch
+    # acquires pass through re-entrantly on this thread).
+    from .palace import mine_palace_lock
+
+    with mine_palace_lock(dest_palace):
+        return _rebuild_from_sqlite_locked(
+            source_palace=source_palace,
+            dest_palace=dest_palace,
+            in_place=in_place,
+            batch_size=batch_size,
+        )
+
+
+def _print_unreadable_count_refusal(*, collection_name: str, palace_path: str) -> None:
+    """Refuse to preview a collection whose SQLite row count cannot be read.
+
+    Fail closed: inventing 0 would hide an unreadable source and make the
+    operator believe a real run would upsert nothing (review note on #1654 /
+    #2095). Shared by both previews so the wording cannot drift apart.
+    """
+    print(
+        f"\n  Cannot preview [{collection_name}]: SQLite row count is unreadable "
+        f"at {os.path.join(palace_path, 'chroma.sqlite3')}.\n"
+        "  Fix source readability (schema, lock, permissions) and re-run "
+        "--dry-run; refusing to invent zero counts."
+    )
+
+
+def _preview_rebuild_from_sqlite(
+    *,
+    source_palace: str,
+    dest_palace: str,
+    in_place: bool,
+) -> dict[str, int]:
+    """Read-only preview for :func:`rebuild_from_sqlite` (``dry_run=True``).
+
+    Never archives, locks, or writes. Returns ``{}`` if SQLite counts are
+    unreadable so a broken preview cannot look like a successful zero-row plan.
+    """
+    print("\n  DRY RUN -- no changes will be made.")
+    if in_place:
+        print(
+            f"  Would archive {dest_palace} → "
+            f"{dest_palace}.pre-rebuild-<timestamp>, then rebuild from the copy."
+        )
+    else:
+        print(f"  Would rebuild into {dest_palace} from {source_palace}.")
+
+    counts: dict[str, int] = {}
+    for cname in _recoverable_collections():
+        n = sqlite_drawer_count(source_palace, cname)
+        if n is None:
+            _print_unreadable_count_refusal(collection_name=cname, palace_path=source_palace)
+            return {}
+        counts[cname] = n
+        print(f"  [{cname}] would re-embed and upsert {n} rows")
+    print(
+        f"\n  Would rebuild {sum(counts.values())} total rows. Re-run without --dry-run to execute."
+    )
+    print(f"{'=' * 55}\n")
+    return counts
+
+
+def _preview_legacy_repair(
+    *,
+    palace_path: str,
+    collection_name: str,
+    confirm_truncation_ok: bool = False,
+) -> dict[str, int]:
+    """Read-only preview for the default (legacy) ``repair`` path (``dry_run=True``).
+
+    Never opens a chromadb client, takes a lock, or writes. Opening a client is
+    itself a write to ``chroma.sqlite3``, so the row count comes from the
+    read-only SQLite ground truth :func:`check_extraction_safety` already
+    trusts. That is a different source than the real run rebuilds from (it
+    re-files what the chromadb collection layer returns), so the plan below
+    states the ``#1208`` contingency rather than promising the number.
+
+    ``confirm_truncation_ok`` mirrors the real run's flag: it switches that
+    contingency off, so the preview has to say the guard is disabled rather
+    than promise an abort that would not happen.
+
+    Returns ``{}`` when the count is unreadable so a broken preview cannot look
+    like a valid plan (#1654, #2095, #2133).
+    """
+    print("\n  DRY RUN -- no changes will be made.")
+    n = sqlite_drawer_count(palace_path, collection_name)
+    if n is None:
+        _print_unreadable_count_refusal(collection_name=collection_name, palace_path=palace_path)
+        print(f"{'=' * 55}\n")
+        return {}
+
+    if n == 0:
+        # The real run stops at ``total == 0`` with "Nothing to repair.", or —
+        # when the collection is absent altogether — at the index-read error
+        # that points to --mode from-sqlite. Neither backs up nor rebuilds, so
+        # promising a backup and a VACUUM here would describe a run that does
+        # not happen.
+        print(
+            f"  [{collection_name}] chroma.sqlite3 holds no rows. A real run would report\n"
+            "  nothing to repair, or an index read error, and change nothing."
+        )
+        print(f"{'=' * 55}\n")
+        return {collection_name: 0}
+
+    backup_path = os.path.normpath(palace_path) + ".backup"
+    if confirm_truncation_ok:
+        print(
+            f"  [{collection_name}] chroma.sqlite3 holds {n} rows, and --confirm-truncation-ok\n"
+            "  is set, so the #1208 truncation guard is DISABLED. A real run would re-file\n"
+            f"  whatever the chromadb collection layer returns, even if that is fewer than {n}\n"
+            "  rows, and the difference would be destroyed. It would, in order:"
+        )
+    else:
+        print(
+            f"  [{collection_name}] chroma.sqlite3 holds {n} rows. A real run would extract them\n"
+            "  through the chromadb collection layer first and abort without changes if that\n"
+            f"  returns fewer than {n} (#1208 truncation guard). It would then, in order:"
+        )
+    if os.path.exists(backup_path):
+        print(f"    1. DELETE the existing backup at {backup_path} -- or refuse outright")
+        print("       if it is not a palace -- and copy the live palace in its place")
+    else:
+        print(f"    1. copy the palace directory to {backup_path}")
+    print(f"    2. DELETE the live '{collection_name}' collection and re-file the extracted rows")
+    print("       into a fresh one, staged and verified in a temp collection first")
+    print("    3. rebuild the FTS5 index and VACUUM chroma.sqlite3")
+    print("\n  Without --yes it would ask for confirmation before step 1.")
+    print("  Re-run without --dry-run to execute.")
+    print(f"{'=' * 55}\n")
+    return {collection_name: n}
+
+
+def resolve_repair_preflight_errors(
+    palace_path: str,
+    errors: list[str],
+    *,
+    dry_run: bool,
+    progress=print,
+) -> list[str]:
+    """Return the quick_check errors that still block a repair.
+
+    A real run heals an isolated malformed FTS5 inverted index in place and
+    carries on (#1596). ``--dry-run`` must not perform that write, so it
+    classifies the errors with the same :func:`_errors_are_isolated_fts5`
+    predicate the real path gates on: an isolated FTS5 error is reported and
+    cleared, anything broader still aborts. Without this a preview would print
+    the ABORT banner — offline ``sqlite3 .recover``, recreate the FTS5 table —
+    for a palace the tool repairs by itself.
+
+    The prediction is deliberately the optimistic branch, and it is stated as
+    an attempt rather than a promise: the real heal still returns the errors
+    unchanged when another process holds the mine lock, when the rebuild
+    raises, or when ``quick_check`` is still dirty afterwards. A dry run cannot
+    tell those apart without taking the lock and writing, which is exactly what
+    it must not do, so the wording names them instead.
+    """
+    if not errors:
+        return errors
+    if not dry_run:
+        return maybe_autoheal_fts5_index(palace_path, errors, progress=progress)
+    if _errors_are_isolated_fts5(errors):
+        progress(
+            "\n  DRY RUN — quick_check reports an isolated FTS5 inverted-index error.\n"
+            "  A real run would attempt an in-place rebuild of that index from the\n"
+            "  intact content table and continue if it succeeds; it aborts instead if\n"
+            "  another process holds the mine lock or the rebuild leaves quick_check\n"
+            "  dirty. This preview leaves the index untouched."
+        )
+        return []
+    return errors
+
+
+def _rebuild_from_sqlite_locked(
+    *,
+    source_palace: str,
+    dest_palace: str,
+    in_place: bool,
+    batch_size: int,
+) -> dict[str, int]:
+    """Body of :func:`rebuild_from_sqlite`, run while holding
+    ``mine_palace_lock(dest_palace)`` so the archive/rename and the
+    upserts form one atomic single-writer operation.
+
+    Split out so the lock acquired in :func:`rebuild_from_sqlite` wraps
+    every destructive step (the archive ``shutil.move`` below included).
+    Raising :class:`MineAlreadyRunning` from the wrapping ``with`` happens
+    before this body runs, so a held lock never reaches the archive.
+    """
     archive_path: Optional[str] = None
     if in_place:
         ts = datetime.now().strftime("%Y%m%d-%H%M%S")
         archive_path = f"{dest_palace}.pre-rebuild-{ts}"
-        print(f"  Archiving {dest_palace} → {archive_path}")
+        print(f"  Archiving {dest_palace} -> {archive_path}")
         # os.rename, NOT shutil.move. When any file inside the palace is
         # held open by another process (MCP server, a running mine, another
         # harness), renaming the directory fails atomically UP FRONT on
@@ -1501,7 +1945,6 @@ def rebuild_from_sqlite(
             )
             return {}
         source_palace = archive_path
-        src_db = os.path.join(source_palace, "chroma.sqlite3")
 
         # In-place only: drop chromadb's process-wide System registry so
         # the new client at dest_palace builds a fresh System. Without
@@ -1605,7 +2048,7 @@ def status(palace_path=None, collection_name: Optional[str] = None) -> dict:
     palace_path = palace_path or _get_palace_path()
     collection_name = collection_name or _drawers_collection_name()
     print(f"\n{'=' * 55}")
-    print("  MemPalace Repair — Status")
+    print("  MemPalace Repair -- Status")
     print(f"{'=' * 55}\n")
     print(f"  Palace: {palace_path}")
 
@@ -1822,7 +2265,7 @@ def repair_max_seq_id(
     }
 
     print(f"\n{'=' * 55}")
-    print("  MemPalace Repair — max_seq_id Un-poison")
+    print("  MemPalace Repair -- max_seq_id Un-poison")
     print(f"{'=' * 55}\n")
     print(f"  Palace:  {palace_path}")
     if segment:
@@ -1873,10 +2316,10 @@ def repair_max_seq_id(
     source = "sidecar" if from_sidecar else "heuristic (collection MAX)"
     print(f"    clean-value source   {source}")
     for seg_id, old_val, new_val in plan:
-        print(f"    {seg_id}  {old_val}  →  {new_val}")
+        print(f"    {seg_id}  {old_val}  ->  {new_val}")
 
     if dry_run:
-        print("\n  DRY RUN — no rows modified.\n" + "=" * 55 + "\n")
+        print("\n  DRY RUN -- no rows modified.\n" + "=" * 55 + "\n")
         return result
 
     if not plan:

mempalace/replica.py

@@ -0,0 +1,64 @@
+"""
+replica.py — Per-palace replica identity (RFC 004 transport seam / provenance)
+
+Every palace replica has one stable ``ReplicaId``, stamped as
+``origin_replica`` into every op it authors. For the step-0 pilot the id is
+minted locally on first use and persisted in ``replica.json`` inside the
+palace directory. Existing 12-hex pilot ids remain valid, but new ids use
+128 bits of entropy. When the transport layer lands (MeshGuard), the mesh
+Ed25519 identity supersedes it via an alias op — RFC 004 Appendix A.4:
+"a rename is just another provenance fact."
+
+The id never syncs and never rotates silently; it names the seat (this
+machine's copy of the palace), not the model or the agent.
+"""
+
+import json
+import os
+import re
+import secrets
+from pathlib import Path
+
+REPLICA_FILENAME = "replica.json"
+
+_REPLICA_ID_RE = re.compile(r"^rep_(?:[0-9a-f]{12}|[0-9a-f]{32})$")
+
+
+def _mint() -> str:
+    return f"rep_{secrets.token_hex(16)}"
+
+
+def get_replica_id(palace_path: str) -> str:
+    """Return this palace's stable replica id, minting it on first use.
+
+    The file is written atomically (tmp + rename) so a crash mid-mint can
+    never leave a half-written identity; a corrupt or foreign-shaped file
+    fails loudly rather than silently minting a second identity — two ids
+    for one replica would fork its op-log provenance.
+    """
+    path = Path(os.path.expanduser(palace_path)) / REPLICA_FILENAME
+    if path.exists():
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            replica_id = data["replica_id"]
+        except (ValueError, KeyError, TypeError) as exc:
+            raise ValueError(
+                f"{path} is corrupt ({exc}); refusing to mint a second replica "
+                "identity for this palace — restore or delete the file explicitly"
+            ) from None
+        if not isinstance(replica_id, str) or not _REPLICA_ID_RE.match(replica_id):
+            raise ValueError(
+                f"{path} holds an invalid replica_id {replica_id!r}; refusing to "
+                "mint a second identity — restore or delete the file explicitly"
+            )
+        return replica_id
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    replica_id = _mint()
+    tmp = path.with_suffix(".json.tmp")
+    tmp.write_text(
+        json.dumps({"replica_id": replica_id, "minted_at_note": "RFC 004 step 0"}, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    os.replace(tmp, path)
+    return replica_id

mempalace/room_detector_local.py

@@ -232,14 +232,14 @@ def detect_rooms_from_files(project_dir: str) -> list:
 
 def print_proposed_structure(project_name: str, rooms: list, total_files: int, source: str):
     print(f"\n{'=' * 55}")
-    print("  MemPalace Init — Local setup")
+    print("  MemPalace Init -- Local setup")
     print(f"{'=' * 55}")
     print(f"\n  WING: {project_name}")
     print(f"  ({total_files} files found, rooms detected from {source})\n")
     for room in rooms:
         print(f"    ROOM: {room['name']}")
         print(f"          {room['description']}")
-    print(f"\n{'─' * 55}")
+    print(f"\n{'-' * 55}")
 
 
 def get_user_approval(rooms: list) -> list:
@@ -259,7 +259,7 @@ def get_user_approval(rooms: list) -> list:
     if choice == "edit":
         print("\n  Current rooms:")
         for i, room in enumerate(rooms):
-            print(f"    {i + 1}. {room['name']} — {room['description']}")
+            print(f"    {i + 1}. {room['name']} -- {room['description']}")
         remove = input("\n  Room numbers to REMOVE (comma-separated, or enter to skip): ").strip()
         if remove:
             to_remove = {int(x.strip()) - 1 for x in remove.split(",") if x.strip().isdigit()}

mempalace/server_registry.py

@@ -0,0 +1,171 @@
+"""Per-palace discovery registry for the MemPalace HTTP MCP hub.
+
+``mempalace serve`` (and ``mempalace-mcp --transport http``) is meant to be
+the single long-lived writer for a shared palace: once it performs its first
+mutating tool call it holds the per-palace MCP writer lease (#1818) for its
+whole lifetime, and every other process is refused palace writes. That is
+correct for agents — they talk to the hub — but the background save hooks
+and the plain CLI still spawn short-lived ``mempalace mine`` processes,
+which would be refused while a hub is up and silently stop transcript
+capture on the hub machine.
+
+This module gives those local processes a way to find the hub instead of
+fighting it: the HTTP transport records ``{pid, host, port, scheme,
+read_only}`` next to the per-palace bearer token
+(``~/.mempalace/server/<key>/``), and callers use
+:func:`read_live_serverinfo` to decide "forward this write over HTTP" vs
+"no hub — do the write directly".
+
+The registry is local-machine only by design: it lives under the user's
+home, is keyed by the canonical palace path, and a record is trusted only
+while the recorded pid is still alive — a crashed hub leaves a stale file
+that every reader ignores and the next hub overwrites.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Bind-address wildcards: a hub bound to "all interfaces" is dialed via
+# loopback by local forwarders.
+_WILDCARD_HOSTS = {"0.0.0.0", "::", "[::]"}
+
+
+def _canonical(palace_path: str) -> str:
+    return os.path.abspath(os.path.realpath(os.path.expanduser(palace_path)))
+
+
+def server_state_dir(palace_path: str) -> Path:
+    """Per-palace state directory shared by the token and the serverinfo.
+
+    Keyed by the canonical palace path so one hub per palace reuses a stable
+    directory across restarts. Must stay in sync with the token location used
+    by ``mempalace serve`` (see ``cli._server_token_path``, which delegates
+    here).
+    """
+    key = hashlib.sha256(os.path.normcase(_canonical(palace_path)).encode("utf-8")).hexdigest()[:24]
+    return Path.home() / ".mempalace" / "server" / key
+
+
+def server_token_path(palace_path: str) -> Path:
+    return server_state_dir(palace_path) / "token"
+
+
+def serverinfo_path(palace_path: str) -> Path:
+    return server_state_dir(palace_path) / "serverinfo.json"
+
+
+def write_serverinfo(palace_path: str, *, host: str, port: int, scheme: str, read_only: bool):
+    """Record this process as the palace's HTTP hub. Returns the file path.
+
+    0600 like the token: the record itself is not secret, but the directory
+    convention is "private to the user" and there is no reason to relax it.
+    """
+    path = serverinfo_path(palace_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        os.chmod(str(path.parent), 0o700)
+    except OSError:
+        pass
+    payload = {
+        "pid": os.getpid(),
+        "host": host,
+        "port": int(port),
+        "scheme": scheme,
+        "read_only": bool(read_only),
+        "palace_path": _canonical(palace_path),
+    }
+    fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        json.dump(payload, fh)
+        fh.write("\n")
+    return path
+
+
+def clear_serverinfo(palace_path: str) -> None:
+    """Remove this process's serverinfo record, if it is still ours.
+
+    Guarded on the recorded pid so a slow atexit from an old hub cannot
+    delete the record a newer hub just wrote for the same palace.
+    """
+    path = serverinfo_path(palace_path)
+    try:
+        recorded = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return
+    if recorded.get("pid") != os.getpid():
+        return
+    try:
+        path.unlink()
+    except OSError:
+        logger.debug("serverinfo cleanup failed for %s", path, exc_info=True)
+
+
+def _pid_alive(pid) -> bool:
+    if not isinstance(pid, int) or pid <= 0:
+        return False
+    if os.name == "nt":  # pragma: no cover - exercised on Windows CI only
+        import ctypes
+
+        PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
+        handle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
+        if not handle:
+            return False
+        ctypes.windll.kernel32.CloseHandle(handle)
+        return True
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except OSError:
+        return False
+    return True
+
+
+def read_live_serverinfo(palace_path: str):
+    """Return the hub record for this palace, or None.
+
+    None when no record exists, the record is unreadable, or the recorded
+    pid is no longer alive (crashed hub — stale file, ignore it).
+    """
+    path = serverinfo_path(palace_path)
+    try:
+        info = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    if not isinstance(info, dict):
+        return None
+    if not _pid_alive(info.get("pid")):
+        return None
+    if not isinstance(info.get("port"), int) or not info.get("host"):
+        return None
+    return info
+
+
+def client_base_url(info: dict) -> str:
+    """Dial address for a local client, from a serverinfo record.
+
+    A wildcard bind ("all interfaces") is dialed via loopback — the record
+    is only ever read on the hub's own machine.
+    """
+    host = str(info.get("host", "")).strip()
+    if host.lower() in _WILDCARD_HOSTS:
+        host = "127.0.0.1"
+    scheme = info.get("scheme") or "http"
+    return f"{scheme}://{host}:{info['port']}"
+
+
+def load_server_token(palace_path: str) -> str:
+    """The palace's hub bearer token, or "" when none was ever generated."""
+    try:
+        return server_token_path(palace_path).read_text(encoding="utf-8").strip()
+    except OSError:
+        return ""

mempalace/service.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import contextlib
 import io
+import functools
 import os
 import sys
 from typing import Any
@@ -18,6 +19,36 @@ from .config import MempalaceConfig
 
 _EXPLICIT_BACKEND_ENV = "MEMPALACE_BACKEND_EXPLICIT"
 _PALACE_PATH_ENV = "MEMPALACE_PALACE_PATH"
+
+
+def _restores_palace_env(fn):
+    """Restore ``MEMPALACE_PALACE_PATH`` after the call that stamped it.
+
+    Each ``run_*`` entrypoint sets the variable so downstream config resolution sees THIS
+    call's palace. In the daemon that costs nothing — one call, one process. In any process
+    that makes two calls it leaks: ``config.py`` reads this variable with priority OVER the
+    config file, so the first call's palace silently becomes the second caller's, and a
+    tmpdir palace that has since been removed reads back as ``PalaceNotFoundError`` from an
+    unrelated code path.
+
+    ``daemon.py`` already saves and restores exactly these keys around its own dispatch.
+    This gives the service entrypoints the same discipline.
+    """
+
+    @functools.wraps(fn)
+    def _wrapped(*args, **kwargs):
+        previous = os.environ.get(_PALACE_PATH_ENV)
+        try:
+            return fn(*args, **kwargs)
+        finally:
+            if previous is None:
+                os.environ.pop(_PALACE_PATH_ENV, None)
+            else:
+                os.environ[_PALACE_PATH_ENV] = previous
+
+    return _wrapped
+
+
 _BACKEND_ENV = "MEMPALACE_BACKEND"
 # Env vars a job may mutate via _apply_backend / palace_path injection. They are
 # snapshotted per job and restored afterward so a job that switches the backend
@@ -140,6 +171,7 @@ def execute_job(kind: str, payload: dict[str, Any]) -> dict[str, Any]:
     return result
 
 
+@_restores_palace_env
 def run_mine(payload: dict[str, Any]) -> dict[str, Any]:
     """Run the same mine operation as the CLI, without daemon transport concerns."""
     palace_path = os.path.abspath(
@@ -149,21 +181,32 @@ def run_mine(payload: dict[str, Any]) -> dict[str, Any]:
     _apply_backend(payload.get("backend"))
 
     source = payload.get("source") or payload.get("dir")
+    source_adapter = payload.get("source_adapter")
     mode = payload.get("mode") or "projects"
     wing = payload.get("wing")
     agent = payload.get("agent") or "mempalace"
     limit = int(payload.get("limit") or 0)
     dry_run = bool(payload.get("dry_run"))
 
-    if payload.get("redetect_origin"):
+    if payload.get("redetect_origin") and not source_adapter:
         from .cli import _run_pass_zero
 
         _run_pass_zero(project_dir=source, palace_dir=palace_path, llm_provider=None)
 
+    from .daemon import LOCK_REFUSAL_ERROR_CLASS
     from .palace import MineAlreadyRunning, MineValidationError
 
     try:
-        if mode == "convos":
+        if source_adapter:
+            from .cli import mine_source_adapter
+
+            mine_source_adapter(
+                source_name=source_adapter,
+                source_path=source,
+                palace_path=palace_path,
+                dry_run=dry_run,
+            )
+        elif mode == "convos":
             from .convo_miner import mine_convos
 
             mine_convos(
@@ -207,7 +250,7 @@ def run_mine(payload: dict[str, Any]) -> dict[str, Any]:
         return {
             "success": False,
             "error": str(exc),
-            "error_class": "LockHeldByOtherProcess",
+            "error_class": LOCK_REFUSAL_ERROR_CLASS,
             "exit_code": 1,
         }
     except MineValidationError as exc:
@@ -228,9 +271,17 @@ def run_mine(payload: dict[str, Any]) -> dict[str, Any]:
     except Exception as exc:
         return {"success": False, "error": f"mine failed: {exc}", "exit_code": 1}
 
-    return {"success": True, "kind": "mine", "mode": mode, "dry_run": dry_run, "exit_code": 0}
+    result_mode = "source" if source_adapter else mode
+    return {
+        "success": True,
+        "kind": "mine",
+        "mode": result_mode,
+        "dry_run": dry_run,
+        "exit_code": 0,
+    }
 
 
+@_restores_palace_env
 def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
     """Run sync and render the same operator-facing summary shape as the CLI."""
     palace_path = os.path.abspath(
@@ -240,6 +291,7 @@ def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
     _apply_backend(payload.get("backend"))
 
     from .backends import detect_backend_for_path
+    from .daemon import LOCK_REFUSAL_ERROR_CLASS
     from .palace import MineAlreadyRunning, _backend_artifact_label, resolve_backend_name
 
     if not os.path.isdir(palace_path):
@@ -271,7 +323,7 @@ def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
     dry_run = bool(payload.get("dry_run", True))
 
     print(f"\n{'=' * 55}")
-    print("  MemPalace Sync — Gitignore-aware drawer prune")
+    print("  MemPalace Sync -- Gitignore-aware drawer prune")
     print(f"{'=' * 55}")
     print(f"  Palace:   {palace_path}")
     if payload.get("wing"):
@@ -299,7 +351,7 @@ def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
         return {
             "success": False,
             "error": str(exc),
-            "error_class": "LockHeldByOtherProcess",
+            "error_class": LOCK_REFUSAL_ERROR_CLASS,
             "exit_code": 1,
         }
     except ValueError as exc:
@@ -335,6 +387,7 @@ def run_sync(payload: dict[str, Any]) -> dict[str, Any]:
     return {"success": True, "report": report, "exit_code": 0}
 
 
+@_restores_palace_env
 def run_diary_write(payload: dict[str, Any]) -> dict[str, Any]:
     palace_path = payload.get("palace_path")
     if palace_path:

mempalace/split_mega_files.py

@@ -285,7 +285,7 @@ def main():
         return
 
     print(f"\n{'=' * 60}")
-    print(f"  Mega-file splitter — {'DRY RUN' if args.dry_run else 'SPLITTING'}")
+    print(f"  Mega-file splitter -- {'DRY RUN' if args.dry_run else 'SPLITTING'}")
     print(f"{'=' * 60}")
     print(f"  Source:      {src_dir}")
     print(f"  Output:      {output_dir or 'same dir as source'}")
@@ -307,9 +307,9 @@ def main():
 
     print(f"{'-' * 60}")
     if args.dry_run:
-        print(f"  DRY RUN — would create {total_written} files from {len(mega_files)} mega-files")
+        print(f"  DRY RUN -- would create {total_written} files from {len(mega_files)} mega-files")
     else:
-        print(f"  Done — created {total_written} files from {len(mega_files)} mega-files")
+        print(f"  Done -- created {total_written} files from {len(mega_files)} mega-files")
     print()
 
 

mempalace/sweeper.py

@@ -230,28 +230,48 @@ def sweep(jsonl_path: str, palace_path: str, source_label: Optional[str] = None)
         nonlocal drawers_added, drawers_already_present
         if not batch_ids:
             return
-        # Pre-flight: which IDs in this batch are already present?
-        # Upsert is idempotent on data but counts as "added" would lie;
-        # this pre-query makes the metric honest (Copilot PR 998 review).
+        # Collapse repeated deterministic IDs before backend calls.
+        # Rewritten transcripts can contain the same message UUID more than
+        # once, and Chroma rejects duplicate IDs in one get or upsert request.
+        # Reassigning a dict key preserves its first position but replaces its
+        # value, so the final occurrence supplies the document and metadata.
+        deduped: dict[str, tuple[str, dict]] = {}
+        for drawer_id, document, metadata in zip(
+            batch_ids,
+            batch_docs,
+            batch_metas,
+        ):
+            deduped[drawer_id] = (document, metadata)
+
+        unique_ids = list(deduped)
+        unique_docs = [deduped[drawer_id][0] for drawer_id in unique_ids]
+        unique_metas = [deduped[drawer_id][1] for drawer_id in unique_ids]
+
+        # Pre-flight: which unique IDs are already present?
         try:
-            existing = collection.get(ids=list(batch_ids), include=[])
-            # Chroma returns a dict; typed backends return GetResult — the
-            # compat shim makes ``.get("ids")`` work on both.
+            existing = collection.get(
+                ids=unique_ids,
+                include=[],
+            )
+            # Chroma returns a dict; typed backends return GetResult - the
+            # compatibility shim makes .get("ids") work on both.
             present = set(existing.get("ids") or [])
         except Exception as exc:
             logger.warning(
                 "sweeper: existence pre-check failed (%s); "
-                "counting all batch rows as new (metric may over-count on reruns).",
+                "counting all unique batch rows as new "
+                "(metric may over-count on reruns).",
                 exc,
             )
             present = set()
-        new_count = sum(1 for rid in batch_ids if rid not in present)
-        already_count = len(batch_ids) - new_count
+
+        new_count = sum(1 for drawer_id in unique_ids if drawer_id not in present)
+        already_count = len(unique_ids) - new_count
 
         collection.upsert(
-            ids=batch_ids,
-            documents=batch_docs,
-            metadatas=batch_metas,
+            ids=unique_ids,
+            documents=unique_docs,
+            metadatas=unique_metas,
         )
         drawers_added += new_count
         drawers_already_present += already_count

mempalace/transport.py

@@ -0,0 +1,169 @@
+"""
+transport.py — The RFC 004 transport seam (Layer 1).
+
+Layer 2 (anti-entropy sync of the logstream and, in step 2a, the memory
+op-log) is allowed to see peers ONLY through this interface. Everything
+about networks, addressing, keys, and which machines are awake lives
+below the seam; nothing about merge semantics leaks down into it. The
+full contract, from the RFC:
+
+    self_id()                    stable ReplicaId of this node
+    peers()                      membership snapshot
+    request(peer, path, params)  one authenticated round-trip
+    open_stream(peer, path)      long-lived tail, resumable by cursor
+    on_presence_change(cb)       liveness deltas (R5)
+    on_inbound(path, handler)    serve anti-entropy pulls
+
+v1 ships the pull half — ``self_id``/``peers``/``request`` — because that
+is all the shipping sync engine consumes. The push half raises
+``NotImplementedError`` here and lands with the MeshGuard binding, where
+SWIM membership supplies presence, the Ed25519 node identity IS the
+ReplicaId (provenance and authentication as one fact), and membership in
+the mesh replaces bearer tokens as the only ACL.
+
+Two implementations are planned:
+
+- :class:`HttpsBearerTransport` (this module, shipping): tailnet HTTPS
+  with per-hub bearer tokens from ``peers.json``. Address discovery and
+  channel encryption come from the tailnet; authorization is an
+  app-layer token that a human must relay out-of-band for every edge —
+  the N-squared-token model the MeshGuard transport retires.
+- ``MeshGuardTransport`` (next): binds the meshguard daemon over its FFI.
+  Selected via ``MEMPALACE_TRANSPORT=meshguard`` once it exists; the
+  factory below is the single swap point, so Layer 2 never changes.
+"""
+
+import json
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Optional
+
+PEERS_FILENAME = "peers.json"
+
+_HTTP_TIMEOUT_S = 30
+
+
+class TransportError(Exception):
+    """A peer was unreachable or answered outside the protocol."""
+
+
+def _http_timeout_s() -> float:
+    """Per-request timeout for peer HTTP calls, env-tunable.
+
+    The default suits logstream syncs (small pages), but a deep-offset
+    snapshot page — Chroma's get(offset=N) scans N rows — behind a TLS
+    proxy can legitimately exceed 30s on a large palace (first hit in
+    production: a 31k-drawer bootstrap timing out at offset 27500). Set
+    MEMPALACE_SYNC_HTTP_TIMEOUT to raise it for bootstrap pulls.
+    """
+    try:
+        return float(os.environ.get("MEMPALACE_SYNC_HTTP_TIMEOUT", "") or _HTTP_TIMEOUT_S)
+    except ValueError:
+        return float(_HTTP_TIMEOUT_S)
+
+
+def load_peers(palace_path: str) -> list[dict]:
+    """Read peers.json; [] when absent. Malformed files fail loudly.
+
+    peers.json is TRANSPORT configuration — addressing plus app-layer
+    authorization — which is why it lives on this side of the seam.
+    """
+    path = Path(os.path.expanduser(palace_path)) / PEERS_FILENAME
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        peers = data["peers"]
+    except (ValueError, KeyError, TypeError) as exc:
+        raise ValueError(f"{path} is malformed: {exc}") from None
+    if not isinstance(peers, list):
+        raise ValueError(f"{path}: 'peers' must be a list")
+    for peer in peers:
+        if not isinstance(peer, dict) or not peer.get("url"):
+            raise ValueError(f"{path}: every peer needs at least a 'url'")
+    return peers
+
+
+def http_request(base_url: str, token: str, path: str, params: dict = None) -> dict:
+    """The HTTPS wire primitive: one bearer-authenticated GET, JSON back."""
+    url = base_url.rstrip("/") + path
+    if params:
+        url += "?" + urllib.parse.urlencode(params)
+    request = urllib.request.Request(url)
+    if token:
+        request.add_header("Authorization", f"Bearer {token}")
+    try:
+        with urllib.request.urlopen(request, timeout=_http_timeout_s()) as response:
+            return json.loads(response.read().decode("utf-8"))
+    except (urllib.error.URLError, TimeoutError, ValueError) as exc:
+        raise TransportError(f"{url}: {exc}") from None
+
+
+class Transport(ABC):
+    """The seam. Layer 2 holds one of these and nothing else network-shaped."""
+
+    @abstractmethod
+    def self_id(self) -> str:
+        """Stable ReplicaId of this node."""
+
+    @abstractmethod
+    def peers(self) -> list[dict]:
+        """Current membership snapshot. Each peer dict carries at least
+        ``name``; transport-specific addressing fields are opaque to
+        Layer 2, which only ever hands the dict back to :meth:`request`."""
+
+    @abstractmethod
+    def request(self, peer: dict, path: str, params: Optional[dict] = None) -> dict:
+        """One authenticated round-trip to ``peer``. Raises TransportError."""
+
+    # ── Push half — defined by the contract, lands with MeshGuard ────────
+
+    def open_stream(self, peer: dict, path: str):
+        raise NotImplementedError("open_stream lands with the MeshGuard transport binding")
+
+    def on_presence_change(self, callback):
+        raise NotImplementedError("presence lands with the MeshGuard transport binding")
+
+    def on_inbound(self, path: str, handler):
+        raise NotImplementedError("inbound serving lands with the MeshGuard transport binding")
+
+
+class HttpsBearerTransport(Transport):
+    """Shipping transport: tailnet HTTPS + peers.json bearer tokens."""
+
+    def __init__(self, palace_path: str):
+        self._palace_path = palace_path
+
+    def self_id(self) -> str:
+        from .replica import get_replica_id
+
+        return get_replica_id(self._palace_path)
+
+    def peers(self) -> list[dict]:
+        return load_peers(self._palace_path)
+
+    def request(self, peer: dict, path: str, params: Optional[dict] = None) -> dict:
+        return http_request(peer["url"], peer.get("token", ""), path, params)
+
+
+def get_transport(palace_path: str) -> Transport:
+    """The single swap point (MEMPALACE_TRANSPORT; default https).
+
+    ``meshguard`` is reserved: selecting it before the binding lands
+    fails loudly rather than silently degrading to bearer tokens —
+    a user who asked for mesh-identity auth must never unknowingly
+    run on the token model.
+    """
+    selected = (os.environ.get("MEMPALACE_TRANSPORT") or "https").strip().lower()
+    if selected == "https":
+        return HttpsBearerTransport(palace_path)
+    if selected == "meshguard":
+        raise NotImplementedError(
+            "MEMPALACE_TRANSPORT=meshguard is reserved for the MeshGuard binding "
+            "(RFC 004 transport seam); it has not shipped yet"
+        )
+    raise ValueError(f"unknown MEMPALACE_TRANSPORT {selected!r} (expected: https, meshguard)")

mempalace/version.py

@@ -1,3 +1,3 @@
 """Single source of truth for the MemPalace package version."""
 
-__version__ = "3.6.0"
+__version__ = "3.7.0"

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mempalace"
-version = "3.6.0"
+version = "3.7.0"
 description = "Give your AI a memory — mine projects and conversations into a searchable palace. No API key required."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -79,7 +79,7 @@ dev = [
     # (wired only into the test-windows job in ci.yml, scoped to the specific
     # error). Local/Linux/macOS runs never rerun, so real failures stay loud.
     "pytest-rerunfailures>=12.0",
-    "ruff==0.15.20",
+    "ruff==0.16.1",
     "psutil>=5.9",
     # Property-based testing — generates hundreds of random inputs per
     # test to find counterexamples the hand-written positive tests miss.
@@ -147,7 +147,7 @@ dev = [
     # (wired only into the test-windows job in ci.yml, scoped to the specific
     # error). Local/Linux/macOS runs never rerun, so real failures stay loud.
     "pytest-rerunfailures>=12.0",
-    "ruff==0.15.20",
+    "ruff==0.16.1",
     "psutil>=5.9",
     "hypothesis>=6.0",
     "pre-commit>=3.0",
@@ -164,7 +164,11 @@ packages = ["mempalace"]
 [tool.ruff]
 line-length = 100
 target-version = "py39"
-extend-exclude = ["benchmarks"]
+# "*.md": ruff 0.16 began formatting Python inside markdown fences, which
+# pulls ~96 docs into scope and reflows hand-aligned example code (the
+# column-aligned field comments in docs/rfcs/002 are deliberate). Docs are
+# authored prose, not linted source — keep the formatter on real modules.
+extend-exclude = ["benchmarks", "*.md"]
 
 [tool.ruff.lint]
 select = ["E", "F", "W", "C901"]

scripts/docker-smoke.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+#
+# Smoke-test a built MemPalace image and the shipped Compose files.
+#
+#   scripts/docker-smoke.sh [IMAGE]        # default image: mempalace:smoke
+#
+# CI builds the container images but, until this existed, never ran one and
+# never parsed a Compose file — so two defects that break the very first
+# documented command shipped anyway: a `docker-compose.yml` Compose refused to
+# load (#2188), and an embedding-vector bug that aborted `mine` on the first
+# drawer (#2187). A build that succeeds proves the image *compiles*; it says
+# nothing about whether the thing inside it works.
+#
+# So this exercises the paths the README actually tells users to run, and
+# asserts on returned content rather than exit codes alone — the `mine` crash
+# was a non-zero exit, but the verbatim read-back is what proves the palace
+# really holds the text.
+#
+# Requires: docker (with the compose plugin) and python3.
+
+set -euo pipefail
+
+IMAGE="${1:-mempalace:smoke}"
+VOLUME="mempalace-smoke-$$"
+WORKDIR="$(mktemp -d)"
+
+# The image runs as uid 1000, which will not match the host user on Linux, and
+# bind mounts carry host ownership through unchanged (Docker Desktop's uid
+# mapping hides this on macOS). `mktemp -d` gives 0700, so the container could
+# not even stat inside /work. Model an ordinary project checkout instead: 0755
+# dir, 0644 files — which is what makes the README's `-v /path/to/project:/work`
+# work for a normal repo.
+chmod 0755 "$WORKDIR"
+
+# A sentence we can assert on verbatim. Storing user words exactly is the
+# project's core promise, so the read-back check is the real assertion here.
+NEEDLE="eleven round trips to render one screen"
+
+cleanup() {
+    docker volume rm -f "$VOLUME" >/dev/null 2>&1 || true
+    rm -rf "$WORKDIR"
+}
+trap cleanup EXIT
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+# Dump captured output on failure. Lines are clipped because the failure this
+# most often reports — a rejected embedding batch — puts a whole 384-dim vector
+# repr on a single line, which buries the actual message in the CI log.
+dump() {
+    echo "--- last 40 lines (clipped to 500 cols) ---" >&2
+    printf '%s\n' "$1" | tail -40 | cut -c1-500 >&2
+    echo "-------------------------------------------" >&2
+}
+
+echo "== 1/5  Compose files parse =="
+# A key with only comments under it parses as null and Compose rejects the
+# whole file. `config` is the cheapest way to catch that class of defect.
+docker compose -f docker-compose.yml config --quiet \
+    || fail "docker-compose.yml is not a valid Compose file"
+echo "  docker-compose.yml ok"
+
+# The server file interpolates a mandatory token; a dummy satisfies the
+# `:?` guard so the rest of the file still gets validated.
+MEMPALACE_MCP_HTTP_TOKEN=smoke-token \
+    docker compose -f deploy/docker-compose.server.yml config --quiet \
+    || fail "deploy/docker-compose.server.yml is not a valid Compose file"
+echo "  deploy/docker-compose.server.yml ok"
+
+echo "== 2/5  CLI entrypoint dispatch =="
+# docker-entrypoint.sh routes `cli`/`mcp` keywords and forwards anything else
+# to the CLI. Both forms are documented, so both are checked.
+docker run --rm -v "$VOLUME:/data" "$IMAGE" cli --version >/dev/null \
+    || fail "'cli --version' failed"
+docker run --rm -v "$VOLUME:/data" "$IMAGE" --version >/dev/null \
+    || fail "bare '--version' passthrough failed"
+echo "  'cli ...' and bare passthrough both dispatch"
+
+echo "== 3/5  mine a mounted directory =="
+cat > "$WORKDIR/notes.md" <<EOF
+# Architecture decision
+
+We switched from REST to GraphQL because the mobile client was making
+$NEEDLE. We settled on persisted queries.
+EOF
+chmod 0644 "$WORKDIR/notes.md"
+
+# First run also downloads the ~80 MB embedding model into the volume, so this
+# doubles as a check that a cold container can reach and cache it.
+mine_out="$(docker run --rm -v "$VOLUME:/data" -v "$WORKDIR:/work" "$IMAGE" mine /work 2>&1)" \
+    || { dump "$mine_out"; fail "'mine /work' exited non-zero"; }
+
+echo "$mine_out" | grep -q "Drawers filed: 1" \
+    || { dump "$mine_out"; fail "expected 'Drawers filed: 1'"; }
+echo "  filed 1 drawer"
+
+echo "== 4/5  search it back from a separate container =="
+# A new container against the same volume: proves the palace persisted and
+# that the stored text is returned verbatim, not summarised.
+search_out="$(docker run --rm -v "$VOLUME:/data" "$IMAGE" search "why did we move off REST" 2>&1)" \
+    || { dump "$search_out"; fail "'search' exited non-zero"; }
+
+echo "$search_out" | grep -qF "$NEEDLE" \
+    || { dump "$search_out"; fail "search did not return the drawer verbatim"; }
+echo "  drawer returned verbatim"
+
+echo "== 5/5  MCP server over stdio =="
+# The README's MCP client config runs the image with -i and speaks JSON-RPC on
+# stdin/stdout. Drive a real handshake and one tool call.
+mcp_out="$(printf '%s\n' \
+    '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"smoke","version":"0"}}}' \
+    '{"jsonrpc":"2.0","method":"notifications/initialized"}' \
+    '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' \
+    '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"mempalace_search","arguments":{"query":"GraphQL"}}}' \
+    | docker run -i --rm -v "$VOLUME:/data" "$IMAGE" mcp 2>/dev/null)"
+
+# The transcript goes to a file and the parser reads it via argv: the heredoc
+# already occupies this command's stdin.
+printf '%s\n' "$mcp_out" > "$WORKDIR/mcp.jsonl"
+
+NEEDLE="$NEEDLE" python3 - "$WORKDIR/mcp.jsonl" <<'PY'
+import json, os, sys
+
+needle = os.environ["NEEDLE"]
+seen = {}
+with open(sys.argv[1], encoding="utf-8") as fh:
+    for line in fh:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            msg = json.loads(line)
+        except ValueError:
+            continue  # non-JSON banner lines are not our concern here
+        if "id" in msg:
+            seen[msg["id"]] = msg
+
+
+def result(rid, what):
+    msg = seen.get(rid)
+    if msg is None:
+        sys.exit(f"FAIL: no JSON-RPC response for {what} (id={rid})")
+    if "error" in msg:
+        sys.exit(f"FAIL: {what} returned an error: {msg['error']}")
+    return msg["result"]
+
+
+name = result(1, "initialize")["serverInfo"]["name"]
+if name != "mempalace":
+    sys.exit(f"FAIL: initialize reported serverInfo.name={name!r}")
+print(f"  initialize ok (server: {name})")
+
+tools = result(2, "tools/list")["tools"]
+if not tools:
+    sys.exit("FAIL: tools/list returned no tools")
+print(f"  tools/list ok ({len(tools)} tools)")
+
+call = result(3, "tools/call")
+if call.get("isError"):
+    sys.exit(f"FAIL: mempalace_search reported isError: {call}")
+text = "".join(part.get("text", "") for part in call.get("content", []))
+if needle not in text:
+    sys.exit("FAIL: mempalace_search did not return the drawer verbatim")
+print("  mempalace_search ok (drawer returned verbatim)")
+PY
+
+echo
+echo "docker smoke: all checks passed ($IMAGE)"

scripts/mempalace_repair_encoding.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""Repair legacy Windows mojibake in a MemPalace collection."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from mempalace.config import MempalaceConfig
+from mempalace.encoding_repair import (
+    repair_collection,
+    restore_collection,
+)
+from mempalace.palace import (
+    get_collection,
+    mine_palace_lock,
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Conservatively repair high-confidence UTF-8 mojibake "
+            "in legacy MemPalace drawers. Dry-run is the default."
+        )
+    )
+    parser.add_argument(
+        "--palace",
+        help=("Palace path; defaults to the configured palace."),
+    )
+    parser.add_argument(
+        "--collection",
+        help=("Collection name; defaults to the configured collection."),
+    )
+    parser.add_argument(
+        "--page-size",
+        type=int,
+        default=500,
+        help=("Rows scanned or restored per page (default: 500)."),
+    )
+    parser.add_argument(
+        "--preview-chars",
+        type=int,
+        default=180,
+        help=("Maximum characters shown in each before/after preview."),
+    )
+
+    action = parser.add_mutually_exclusive_group()
+    action.add_argument(
+        "--apply",
+        action="store_true",
+        help=(
+            "Write repairs after creating a private JSONL backup. "
+            "Without this flag, the command is read-only."
+        ),
+    )
+    action.add_argument(
+        "--restore-backup",
+        metavar="PATH",
+        help=("Restore original documents from a prior repair backup."),
+    )
+
+    parser.add_argument(
+        "--backup",
+        metavar="PATH",
+        help=(
+            "Backup destination used with --apply. Defaults to a "
+            "timestamped file beside the palace. Existing files "
+            "are never overwritten."
+        ),
+    )
+    return parser
+
+
+def _default_backup_path(
+    palace_path: str,
+) -> Path:
+    palace = Path(palace_path).expanduser().resolve()
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+
+    return palace.parent / (f"{palace.name}.encoding-repair-{timestamp}.jsonl")
+
+
+def _preview(
+    text: str,
+    limit: int,
+) -> str:
+    compact = text.replace(
+        "\r",
+        "\\r",
+    ).replace(
+        "\n",
+        "\\n",
+    )
+
+    if len(compact) <= limit:
+        return compact
+
+    return (
+        compact[
+            : max(
+                0,
+                limit - 1,
+            )
+        ]
+        + "…"
+    )
+
+
+def _print_change(
+    drawer_id: str,
+    before: str,
+    after: str,
+    *,
+    preview_chars: int,
+) -> None:
+    print()
+    print(f"Drawer: {drawer_id}")
+    print(
+        "  before: "
+        + _preview(
+            before,
+            preview_chars,
+        )
+    )
+    print(
+        "  after:  "
+        + _preview(
+            after,
+            preview_chars,
+        )
+    )
+
+
+def _reconfigure_stdio_utf8_on_windows() -> None:
+    """Decode stdio as UTF-8 on Windows for the encoding-repair CLI.
+
+    Thin wrapper around the shared helper in ``mempalace._stdio``, matching
+    ``cli.py`` and ``fact_checker.py``. stdout/stderr override to ``replace``
+    because every proposed change prints a before/after preview of verbatim
+    drawer text -- under the legacy console codepage this tool is written for,
+    ``strict`` raises on the mojibake lead bytes themselves and aborts the run
+    before a single drawer is repaired.
+    """
+    from mempalace._stdio import reconfigure_stdio_utf8_on_windows
+
+    reconfigure_stdio_utf8_on_windows(stdout_errors="replace", stderr_errors="replace")
+
+
+def main() -> int:
+    _reconfigure_stdio_utf8_on_windows()
+
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if args.page_size < 1:
+        parser.error("--page-size must be at least 1")
+
+    if args.preview_chars < 40:
+        parser.error("--preview-chars must be at least 40")
+
+    if args.backup and not args.apply:
+        parser.error("--backup requires --apply")
+
+    config = MempalaceConfig()
+    palace_path = args.palace or config.palace_path
+    collection_name = args.collection or getattr(
+        config,
+        "collection_name",
+        "mempalace_drawers",
+    )
+
+    if args.restore_backup:
+        with mine_palace_lock(palace_path):
+            collection = get_collection(
+                palace_path,
+                collection_name=(collection_name),
+                create=False,
+            )
+            report = restore_collection(
+                collection,
+                args.restore_backup,
+                batch_size=(args.page_size),
+            )
+
+        print("Mode: RESTORE")
+        print(f"Backup records validated: {report['validated']}")
+        print(f"Documents restored: {report['restored']}")
+        return 0
+
+    backup_path = None
+
+    if args.apply:
+        backup_path = Path(args.backup) if args.backup else _default_backup_path(palace_path)
+
+    def show_change(
+        drawer_id: str,
+        before: str,
+        after: str,
+    ) -> None:
+        _print_change(
+            drawer_id,
+            before,
+            after,
+            preview_chars=(args.preview_chars),
+        )
+
+    try:
+        if args.apply:
+            with mine_palace_lock(palace_path):
+                collection = get_collection(
+                    palace_path,
+                    collection_name=(collection_name),
+                    create=False,
+                )
+                report = repair_collection(
+                    collection,
+                    apply=True,
+                    page_size=(args.page_size),
+                    backup_path=(backup_path),
+                    on_change=show_change,
+                )
+        else:
+            collection = get_collection(
+                palace_path,
+                collection_name=(collection_name),
+                create=False,
+            )
+            report = repair_collection(
+                collection,
+                apply=False,
+                page_size=(args.page_size),
+                on_change=show_change,
+            )
+    except FileExistsError as exc:
+        print(
+            f"ERROR: backup file already exists; refusing to overwrite it: {exc.filename}",
+            file=sys.stderr,
+        )
+        return 2
+
+    print()
+    print("Mode: " + ("APPLY" if args.apply else "DRY RUN"))
+    print(f"Rows scanned: {report['scanned']}")
+    print(f"Documents needing repair: {report['changed']}")
+    print(f"Documents updated: {report['updated']}")
+
+    if report["backup_path"]:
+        print(f"Original-document backup: {report['backup_path']}")
+
+    if not args.apply and report["changed"]:
+        print()
+        print(
+            "Review every change above, then run again with "
+            "--apply. A private, non-overwriting backup will "
+            "be written before any document is updated."
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

skills/mempalace-recall/SKILL.md

@@ -82,6 +82,13 @@ rule and the other integrations — lives in
 question — not a system prompt or pasted conversation) plus optional
 `wing` / `room` filters and `limit` (default 5).
 
+**Active coordination is not recall.** When delegating work to another
+agent on the shared hub — or waiting for its reply or patch — use the
+logstream tools (`mempalace_event_append`, `mempalace_event_wait`,
+`mempalace_patch_submit`, `mempalace_artifact_get`), not drawers or
+search. The canonical protocol lives in
+[`integrations/shared/coordination-protocol.md`](../../integrations/shared/coordination-protocol.md).
+
 ## Unhappy paths
 
 - **Empty results.** Say the palace has nothing on this; do not invent an

tests/conftest.py

@@ -11,6 +11,9 @@ instead of the real user profile.
 """
 
 import os
+import hashlib
+import math
+import re
 import shutil
 import tempfile
 
@@ -33,6 +36,61 @@ import pytest  # noqa: E402
 from mempalace.config import MempalaceConfig  # noqa: E402
 from mempalace.knowledge_graph import KnowledgeGraph  # noqa: E402
 
+_TEST_EMBED_DIM = 384
+_TEST_TOKEN_RE = re.compile(r"\w+", re.UNICODE)
+_REAL_EMBEDDING_TEST_MODULES = {
+    "test_embedding",
+    "test_embedding_api",
+    "test_embeddinggemma",
+}
+
+
+def _stable_test_embedding(text: str) -> list[float]:
+    """Small deterministic embedding for tests that do not test ONNX itself."""
+    vec = [0.0] * _TEST_EMBED_DIM
+    tokens = _TEST_TOKEN_RE.findall((text or "").lower())
+    if not tokens:
+        tokens = [""]
+    for token in tokens:
+        digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
+        vec[int.from_bytes(digest[:4], "little") % _TEST_EMBED_DIM] += 1.0
+    norm = math.sqrt(sum(v * v for v in vec)) or 1.0
+    return [v / norm for v in vec]
+
+
+class _StableTestEmbeddingFunction:
+    @staticmethod
+    def name() -> str:
+        return "default"
+
+    @staticmethod
+    def build_from_config(config):
+        _StableTestEmbeddingFunction.validate_config(config)
+        return _StableTestEmbeddingFunction()
+
+    @staticmethod
+    def validate_config(config) -> None:
+        return
+
+    def get_config(self) -> dict:
+        return {}
+
+    def is_legacy(self) -> bool:
+        return False
+
+    def default_space(self) -> str:
+        return "cosine"
+
+    def supported_spaces(self) -> list[str]:
+        return ["cosine", "l2", "ip"]
+
+    def embed_query(self, input):
+        return self(input=input)
+
+    def __call__(self, input):
+        return [_stable_test_embedding(str(text)) for text in list(input or [])]
+
+
 # Redirect ChromaDB's ONNX model cache back to the real user's cache so tests
 # don't re-download the 79 MB model on every run. The HOME redirect above
 # would otherwise point ONNXMiniLM_L6_V2.DOWNLOAD_PATH at the empty temp dir.
@@ -51,6 +109,39 @@ except ImportError:
     pass
 
 
+@pytest.fixture(autouse=True)
+def _stable_embedding_function_for_tests(request, monkeypatch):
+    """Keep ordinary tests off ChromaDB's native ONNX embedding path.
+
+    Module-sized Windows runs were crashing inside onnxruntime after many raw
+    Chroma add/query calls. The embedding-specific tests opt out below; every
+    other test gets a deterministic in-process EF so it still exercises vector
+    writes/search without loading native ONNX sessions.
+    """
+    module_name = getattr(getattr(request, "module", None), "__name__", "")
+    if module_name in _REAL_EMBEDDING_TEST_MODULES:
+        yield
+        return
+
+    ef = _StableTestEmbeddingFunction()
+
+    import mempalace.backends.chroma as chroma_mod
+    import mempalace.backends.embedding_wrapper as embedding_wrapper
+    import mempalace.embedding as embedding_mod
+    from chromadb.api.types import DefaultEmbeddingFunction
+
+    monkeypatch.setattr(DefaultEmbeddingFunction, "__call__", lambda self, input: ef(input=input))
+    monkeypatch.setattr(
+        DefaultEmbeddingFunction, "embed_query", lambda self, input: ef(input=input)
+    )
+    monkeypatch.setattr(embedding_mod, "get_embedding_function", lambda *_, **__: ef)
+    monkeypatch.setattr(
+        chroma_mod.ChromaBackend, "_resolve_embedding_function", staticmethod(lambda: ef)
+    )
+    monkeypatch.setattr(embedding_wrapper, "_embed_texts", lambda texts: ef(input=list(texts)))
+    yield
+
+
 @pytest.fixture(autouse=True)
 def _reset_mcp_cache():
     """Reset cached MCP state between tests without importing mcp_server.
@@ -101,10 +192,14 @@ def _reset_mcp_cache():
 
         try:
             # Reset the per-process quarantine gate so tests don't leak
-            # state through ChromaBackend._quarantined_paths.
-            from mempalace.backends.chroma import ChromaBackend
+            # state through ChromaBackend._quarantined_paths, and drop cached
+            # HNSW capacity verdicts (#1471) for the same reason — a test that
+            # reuses a palace path would otherwise inherit the previous test's
+            # verdict.
+            from mempalace.backends.chroma import ChromaBackend, reset_hnsw_capacity_cache
 
             ChromaBackend._quarantined_paths.clear()
+            reset_hnsw_capacity_cache()
         except (ImportError, AttributeError):
             pass
 

tests/test_backend_conformance.py

@@ -41,3 +41,20 @@ def test_local_backends_do_not_claim_namespace_isolation():
     advertise the namespace-isolation capability (RFC 001 isolation contract)."""
     assert "supports_namespace_isolation" not in ChromaBackend.capabilities
     assert "supports_namespace_isolation" not in SQLiteExactBackend.capabilities
+
+
+@pytest.mark.parametrize("backend_cls", _LOCAL_BACKENDS)
+def test_local_backends_reject_populated_namespace(backend_cls, tmp_path):
+    """RFC 001 section 4.4 no-silent-drop: non-advertising backends MUST raise when
+    PalaceRef.namespace is set rather than accept-and-ignore."""
+    from mempalace.backends.base import UnsupportedCapabilityError
+
+    backend = backend_cls()
+    try:
+        path = tmp_path / "palace"
+        path.mkdir()
+        ref = PalaceRef(id=str(path), local_path=str(path), namespace="tenant-a")
+        with pytest.raises(UnsupportedCapabilityError):
+            backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True)
+    finally:
+        backend.close()

tests/test_backends.py

@@ -509,6 +509,31 @@ def test_chroma_backend_create_true_creates_directory_and_collection(tmp_path):
     client.get_collection("mempalace_drawers")
 
 
+def test_palace_wrapper_embeds_for_chroma(tmp_path, monkeypatch):
+    """Normal Chroma callers should not rely on Chroma's internal ONNX embedder."""
+    import mempalace.backends.embedding_wrapper as embedding_wrapper
+    from mempalace.backends.embedding_wrapper import EmbeddingCollection
+    from mempalace.palace import get_collection
+
+    calls = []
+
+    def fake_embed(texts):
+        texts = list(texts)
+        calls.append(texts)
+        return [[float(len(text)), 1.0, 0.0, 0.0] for text in texts]
+
+    monkeypatch.setattr(embedding_wrapper, "_embed_texts", fake_embed)
+
+    col = get_collection(str(tmp_path), create=True, backend="chroma")
+    assert isinstance(col, EmbeddingCollection)
+
+    col.add(ids=["a"], documents=["alpha"], metadatas=[{"wing": "w"}])
+    result = col.query(query_texts=["alpha"], n_results=1)
+
+    assert result.ids == [["a"]]
+    assert calls == [["alpha"], ["alpha"]]
+
+
 def test_chroma_backend_creates_collection_with_cosine_distance(tmp_path):
     palace_path = tmp_path / "palace"
 
@@ -523,13 +548,12 @@ def test_chroma_backend_creates_collection_with_cosine_distance(tmp_path):
     assert col.metadata.get("hnsw:space") == "cosine"
 
 
-def test_chroma_backend_sets_hnsw_bloat_guard_on_creation(tmp_path):
+def test_chroma_backend_sets_hnsw_write_defaults_on_creation(tmp_path):
     """HNSW batch/sync thresholds must land on freshly-created collection metadata.
 
-    Low thresholds (2/2 per #1579) make chromadb's Rust HNSW segment
-    persist index_metadata and link_lists after any mine of 2+ drawers.
-    Asserting both keys land on the persisted metadata also covers the
-    #1161 "config silently dropped" concern at CI time.
+    That both keys land covers #1161, where the configuration was silently dropped.
+    The values are chromadb's own documented defaults, so a collection this backend
+    creates indexes on the same terms as one chromadb creates itself.
     """
     palace_path = tmp_path / "palace"
 
@@ -541,35 +565,44 @@ def test_chroma_backend_sets_hnsw_bloat_guard_on_creation(tmp_path):
 
     client = chromadb.PersistentClient(path=str(palace_path))
     col = client.get_collection("mempalace_drawers")
-    assert col.metadata.get("hnsw:batch_size") == 2
-    assert col.metadata.get("hnsw:sync_threshold") == 2
+    batch = col.metadata.get("hnsw:batch_size")
+    sync = col.metadata.get("hnsw:sync_threshold")
+    assert batch == 100
+    assert sync == 1000
+    assert batch <= sync, "chromadb permits batch_size <= sync_threshold"
 
 
-def test_chroma_backend_create_collection_sets_hnsw_bloat_guard(tmp_path):
-    """Same guard must apply via the legacy create_collection() path."""
+def test_chroma_backend_create_collection_sets_hnsw_write_defaults(tmp_path):
+    """The same defaults must apply via the legacy create_collection() path."""
     palace_path = tmp_path / "palace"
 
     ChromaBackend().create_collection(str(palace_path), "mempalace_drawers")
 
     client = chromadb.PersistentClient(path=str(palace_path))
     col = client.get_collection("mempalace_drawers")
-    assert col.metadata.get("hnsw:batch_size") == 2
-    assert col.metadata.get("hnsw:sync_threshold") == 2
+    assert col.metadata.get("hnsw:batch_size") == 100
+    assert col.metadata.get("hnsw:sync_threshold") == 1000
 
 
-def test_sub_threshold_mine_persists_hnsw_metadata(tmp_path):
-    """Regression for #1579: small mines must persist HNSW metadata.
+def test_sub_threshold_mine_survives_reopen_and_escapes_quarantine(tmp_path):
+    """Regression for #1579, asserted as the PROPERTY, not the mechanism.
 
-    _HNSW_BLOAT_GUARD sets batch_size=2 and sync_threshold=2 so that any
-    upsert of 2+ records crosses both thresholds, triggering chromadb's
-    _apply_batch and _persist.  Without this, index_metadata and link_lists
-    stay empty and quarantine_stale_hnsw renames the segment on cold open.
+    #1579 is a durability bug: a sub-threshold mine lost its drawers when
+    quarantine_stale_hnsw renamed the segment away on cold open. So assert what
+    the user needs — the drawers read back from a FRESH backend, and quarantine
+    leaves the segment alone.
+
+    Asserting the mechanism instead (index_metadata.pickle present after 3
+    records) only holds at sync_threshold=2, a value chromadb's own parameter
+    validation rejects. Under chromadb's documented thresholds the Rust writer
+    keeps the tail durable WITHOUT a pickle, so
+    the pickle is rightly absent here and asserting on it would fail a healthy
+    palace.
     """
     palace_path = str(tmp_path / "palace")
     backend = ChromaBackend()
     try:
         col = backend.get_collection(palace_path, "mempalace_drawers", create=True)
-
         col.upsert(
             ids=["a", "b", "c"],
             documents=["doc a", "doc b", "doc c"],
@@ -579,34 +612,34 @@ def test_sub_threshold_mine_persists_hnsw_metadata(tmp_path):
     finally:
         backend.close()
 
-    found_healthy_segment = False
-    for entry in (tmp_path / "palace").iterdir():
-        if not entry.is_dir() or entry.name.startswith("."):
-            continue
-        meta = entry / "index_metadata.pickle"
-        link = entry / "link_lists.bin"
-        data = entry / "data_level0.bin"
-        if data.exists() and data.stat().st_size > _HNSW_MISSING_METADATA_DATA_FLOOR:
-            assert meta.exists(), "index_metadata missing after sub-threshold upsert"
-            assert link.exists() and link.stat().st_size > 0, "link_lists empty"
-            assert _segment_appears_healthy(str(entry))
-            found_healthy_segment = True
-
-    assert found_healthy_segment, "no VECTOR segment with data found"
-
-    # stale_seconds=0.0 forces the stage-2 integrity gate (_segment_appears_healthy)
-    # to run on every segment regardless of mtime delta, proving the fix directly.
+    # Quarantine runs on cold open; stale_seconds=0.0 forces the integrity gate
+    # onto every segment regardless of mtime delta, so the gate itself is proven.
     moved = quarantine_stale_hnsw(palace_path, stale_seconds=0.0)
-    assert moved == [], f"quarantine fired on freshly-persisted segment: {moved}"
+    assert moved == [], f"quarantine fired on a healthy sub-threshold segment: {moved}"
+
+    for entry in (tmp_path / "palace").iterdir():
+        if entry.is_dir() and not entry.name.startswith("."):
+            assert _segment_appears_healthy(str(entry))
+
+    # The durability claim: a FRESH backend still finds every drawer, and the
+    # vector index still answers.
+    reopened = ChromaBackend()
+    try:
+        col = reopened.get_collection(palace_path, "mempalace_drawers")
+        assert col.count() == 3, "sub-threshold mine lost drawers across reopen"
+        hits = col.query(query_embeddings=[[0.1] * _TEST_EMBED_DIM], n_results=3)
+        assert len(hits["ids"][0]) == 3, "vector index empty after reopen"
+    finally:
+        reopened.close()
 
 
 def test_single_record_upsert_not_quarantined(tmp_path):
     """A single-record upsert must not trigger quarantine.
 
-    With batch_size=2 chromadb only persists HNSW metadata after the second
-    record.  A one-record segment has no index_metadata.pickle and no
-    link_lists.bin data; _segment_appears_healthy must treat that combination
-    as sub-threshold (never persisted), not as corruption.
+    A one-record segment sits below the HNSW thresholds, so chromadb never
+    persists: no index_metadata.pickle, no link_lists.bin data.
+    _segment_appears_healthy must read that combination as sub-threshold
+    (never persisted), not as corruption.
     """
     palace_path = str(tmp_path / "palace")
     backend = ChromaBackend()
@@ -657,7 +690,7 @@ def test_get_collection_create_true_preserves_existing_metadata(tmp_path):
     backend.get_collection(palace, collection_name="mempalace_drawers", create=True)
     col = backend.get_collection(palace, collection_name="mempalace_drawers", create=True)
     assert col._collection.metadata["hnsw:space"] == "cosine"
-    assert col._collection.metadata.get("hnsw:batch_size") == 2
+    assert col._collection.metadata.get("hnsw:batch_size") == 100
 
 
 def test_fix_blob_seq_ids_converts_blobs_to_integers(tmp_path):

tests/test_backups.py

@@ -1,16 +1,100 @@
-"""Tests for backup retention pruning (mempalace.backups.prune_backups).
+"""Tests for writing and pruning palace backups (mempalace.backups).
 
-These guard the fix for unbounded backup growth: ``mempalace migrate`` and
-``mempalace repair max-seq-id`` each drop a fresh full-size, timestamped copy
-every run, and used to never delete the old ones — a palace was found with
-hundreds of GB of stale backups beside a few hundred MB of live data.
+``prune_backups`` guards the fix for unbounded backup growth: ``mempalace
+migrate`` and ``mempalace repair max-seq-id`` each drop a fresh full-size,
+timestamped copy every run, and used to never delete the old ones. A palace
+was found with hundreds of GB of stale backups beside a few hundred MB of
+live data.
+
+``copy_palace_dir`` guards #2207: the whole-directory copy that ``repair``
+and ``migrate`` take before overwriting a live palace used to abort the whole
+command when the palace held a directory entry ``shutil.copytree`` cannot
+duplicate.
 """
 
+import errno
 import os
+import shutil
+import socket
+import stat
 
 import pytest
 
-from mempalace.backups import prune_backups
+from mempalace.backups import (
+    _file_type_label,
+    _uncopyable_reason,
+    copy_palace_dir,
+    prune_backups,
+)
+
+needs_unix_socket = pytest.mark.skipif(
+    os.name == "nt" or not hasattr(socket, "AF_UNIX"),
+    reason="Unix domain socket files are POSIX-only",
+)
+needs_fifo = pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="named pipes are POSIX-only")
+needs_unprivileged_posix = pytest.mark.skipif(
+    os.name == "nt" or (hasattr(os, "geteuid") and os.geteuid() == 0),
+    reason="directory permission bits gate neither root nor Windows",
+)
+needs_char_device = pytest.mark.skipif(
+    os.name == "nt" or not os.path.exists("/dev/null"),
+    reason="device nodes are POSIX-only",
+)
+
+
+def _symlink_or_skip(link, target):
+    """Create ``link`` pointing at ``target``, or skip if the platform refuses.
+
+    Windows without ``SeCreateSymbolicLinkPrivilege`` raises ``OSError``
+    before any product code runs. Per PR #1555 review (Igor), symlink tests
+    skip cleanly there rather than fail spuriously. Trying the syscall keeps
+    the symlink half of this module under test wherever it does work, which a
+    blanket ``os.name == "nt"`` skip would give up on.
+
+    ``NotImplementedError`` covers the restricted sandboxes ``test_exporter``
+    already documents. Otherwise only a permission refusal is turned into a
+    skip: ``EEXIST`` from a test that left the name behind, ``ENOENT`` from a
+    missing parent and ``ENOSPC`` are bugs in the test, and swallowing them
+    would delete this module's symlink coverage from a run that still
+    reported success.
+    """
+    try:
+        link.symlink_to(target)
+    except NotImplementedError as exc:
+        pytest.skip(f"symlinks are unavailable here: {exc}")
+    except OSError as exc:
+        if os.name != "nt" and exc.errno not in (errno.EPERM, errno.EACCES):
+            raise
+        pytest.skip(f"symlink creation not permitted for this user: {exc}")
+
+
+def _make_palace(parent, name="palace"):
+    """A palace directory holding the one file every caller cares about."""
+    path = parent / name
+    path.mkdir()
+    (path / "chroma.sqlite3").write_bytes(b"SQLite format 3\x00")
+    segment = path / "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+    segment.mkdir()
+    (segment / "data_level0.bin").write_bytes(b"\x00" * 32)
+    return path
+
+
+def _bind_socket(directory, name, monkeypatch):
+    """Bind a Unix socket inside ``directory`` and return its path.
+
+    Bound by relative name from inside the directory: an absolute pytest tmp
+    path can exceed the ~100-byte ``sun_path`` limit, which is tight on
+    macOS. ``monkeypatch.chdir`` restores the working directory afterwards.
+    """
+    monkeypatch.chdir(directory)
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    try:
+        sock.bind(name)
+    finally:
+        # The socket FILE outlives the process that bound it, which is
+        # exactly the state #2207 was reported in.
+        sock.close()
+    return os.path.join(str(directory), name)
 
 
 def _make_backup_dir(parent, name, mtime):
@@ -155,3 +239,621 @@ def test_prune_is_best_effort_on_delete_failure(tmp_path, monkeypatch):
     assert str(tmp_path / "b.1") not in removed
     assert (tmp_path / "b.1").exists()
     assert any("could not remove" in line for line in logs)
+
+
+# ---------------------------------------------------------------------------
+# copy_palace_dir (#2207)
+# ---------------------------------------------------------------------------
+
+
+def test_copy_palace_dir_copies_an_ordinary_palace(tmp_path):
+    palace = _make_palace(tmp_path)
+    dest = tmp_path / "palace.backup"
+
+    skipped = copy_palace_dir(str(palace), str(dest))
+
+    assert skipped == []
+    assert (dest / "chroma.sqlite3").read_bytes() == b"SQLite format 3\x00"
+    assert (dest / "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" / "data_level0.bin").is_file()
+
+
+def test_copy_palace_dir_logs_nothing_when_every_entry_copies(tmp_path):
+    palace = _make_palace(tmp_path)
+    logs = []
+
+    copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log=logs.append)
+
+    assert logs == []
+
+
+@needs_unix_socket
+def test_copy_palace_dir_skips_a_socket_and_still_copies_the_database(tmp_path, monkeypatch):
+    """The reported #2207 case: a leftover daemon socket beside chroma.sqlite3."""
+    palace = _make_palace(tmp_path)
+    sock_path = _bind_socket(palace, "mcp.sock", monkeypatch)
+    dest = tmp_path / "palace.backup"
+    logs = []
+
+    skipped = copy_palace_dir(str(palace), str(dest), log=logs.append)
+
+    assert skipped == [(sock_path, "socket")]
+    assert (dest / "chroma.sqlite3").read_bytes() == b"SQLite format 3\x00"
+    assert not os.path.lexists(dest / "mcp.sock")
+    # Compared line for line, and with one entry, so the singular header and
+    # the socket detail line are both pinned. A substring assertion accepts a
+    # plural that disagrees with the count and a detail line with anything
+    # appended to it.
+    assert logs == [
+        "  Backup: skipped 1 entry that cannot be copied:",
+        "    mcp.sock (socket)",
+    ]
+
+
+@needs_fifo
+def test_copy_palace_dir_skips_a_named_pipe(tmp_path):
+    palace = _make_palace(tmp_path)
+    fifo = palace / "mempalace.fifo"
+    os.mkfifo(fifo)
+    dest = tmp_path / "palace.backup"
+
+    skipped = copy_palace_dir(str(palace), str(dest))
+
+    assert skipped == [(str(fifo), "named pipe")]
+    assert (dest / "chroma.sqlite3").is_file()
+
+
+def test_copy_palace_dir_still_raises_on_a_symlink_whose_target_is_missing(tmp_path):
+    """A link that does not resolve is NOT proof that it carried no data.
+
+    A palace segment can be a link onto another volume, and a volume that is
+    not mounted right now fails ``os.stat`` with the same ``ENOENT`` as a
+    target that was deleted. On Windows an unmapped drive letter and an
+    unreachable network share arrive as ``ENOENT`` too. Skipping on that
+    would drop a whole subtree from the safety copy and let the rebuild run
+    over the live palace anyway, so the copy has to fail the way it does on
+    ``develop``.
+    """
+    palace = _make_palace(tmp_path)
+    link = palace / "chroma.sqlite3.prev"
+    _symlink_or_skip(link, palace / "gone-away.sqlite3")
+    dest = tmp_path / "palace.backup"
+
+    assert _uncopyable_reason(str(link), follow_symlinks=True) is None
+    with pytest.raises(shutil.Error):
+        copy_palace_dir(str(palace), str(dest))
+    assert not os.path.lexists(dest / "chroma.sqlite3.prev")
+
+
+@needs_unix_socket
+def test_copy_palace_dir_skips_a_symlink_pointing_at_a_socket(tmp_path, monkeypatch):
+    """The copy dereferences links by default, so the TARGET's type decides."""
+    palace = _make_palace(tmp_path)
+    sock_path = _bind_socket(palace, "daemon.sock", monkeypatch)
+    link = palace / "mcp.sock"
+    _symlink_or_skip(link, palace / "daemon.sock")
+    dest = tmp_path / "palace.backup"
+
+    skipped = copy_palace_dir(str(palace), str(dest))
+
+    # Sorted within the directory: "daemon.sock" before "mcp.sock".
+    assert skipped == [(sock_path, "socket"), (str(link), "socket")]
+    assert (dest / "chroma.sqlite3").is_file()
+
+
+def test_copy_palace_dir_follows_a_symlink_to_a_real_file(tmp_path):
+    """Link handling is unchanged from the plain ``copytree`` this replaced.
+
+    A resolvable link is dereferenced and its content lands in the backup,
+    which is what the bare call did. Pinned so the refactor cannot quietly
+    alter it, not as an argument that dereferencing is the right default.
+    """
+    palace = _make_palace(tmp_path)
+    (tmp_path / "outside.json").write_text("payload", encoding="utf-8")
+    _symlink_or_skip(palace / "tunnels.json", tmp_path / "outside.json")
+    dest = tmp_path / "palace.backup"
+
+    skipped = copy_palace_dir(str(palace), str(dest))
+
+    assert skipped == []
+    assert (dest / "tunnels.json").read_text(encoding="utf-8") == "payload"
+
+
+def test_copy_palace_dir_keeps_a_broken_symlink_when_symlinks_are_preserved(tmp_path):
+    """``migrate`` copies with ``symlinks=True``, so links are recreated as
+    links and their targets are never read."""
+    palace = _make_palace(tmp_path)
+    _symlink_or_skip(palace / "chroma.sqlite3.prev", palace / "gone-away.sqlite3")
+    dest = tmp_path / "palace.pre-migrate.1"
+
+    skipped = copy_palace_dir(str(palace), str(dest), symlinks=True)
+
+    assert skipped == []
+    assert (dest / "chroma.sqlite3.prev").is_symlink()
+
+
+@needs_unix_socket
+def test_copy_palace_dir_skips_a_socket_even_when_symlinks_are_preserved(tmp_path, monkeypatch):
+    palace = _make_palace(tmp_path)
+    sock_path = _bind_socket(palace, "mcp.sock", monkeypatch)
+    dest = tmp_path / "palace.pre-migrate.1"
+
+    skipped = copy_palace_dir(str(palace), str(dest), symlinks=True)
+
+    assert skipped == [(sock_path, "socket")]
+    assert (dest / "chroma.sqlite3").is_file()
+
+
+@needs_unix_socket
+def test_copy_palace_dir_keeps_a_link_to_a_socket_when_symlinks_are_preserved(
+    tmp_path, monkeypatch
+):
+    """The link's own type decides when the copy recreates links as links.
+
+    ``migrate`` never reads through a link, so a link that happens to point
+    at a socket is a link like any other and belongs in the backup. Judging
+    it by its target instead would drop it.
+    """
+    palace = _make_palace(tmp_path)
+    _bind_socket(palace, "daemon.sock", monkeypatch)
+    _symlink_or_skip(palace / "mcp.sock", palace / "daemon.sock")
+    dest = tmp_path / "palace.pre-migrate.1"
+
+    skipped = copy_palace_dir(str(palace), str(dest), symlinks=True)
+
+    assert [reason for _, reason in skipped] == ["socket"]
+    assert (dest / "mcp.sock").is_symlink()
+
+
+@needs_unprivileged_posix
+def test_copy_palace_dir_still_raises_on_a_copy_failure_it_cannot_classify(tmp_path):
+    """Only provably uncopyable entries are skipped.
+
+    Anything else must still abort the caller: the backup is the safety net
+    for a rebuild that is about to overwrite the live palace, so a copy that
+    did not come out whole has to be loud.
+    """
+    palace = _make_palace(tmp_path)
+    locked = palace / "locked-segment"
+    locked.mkdir()
+    (locked / "data_level0.bin").write_bytes(b"\x00")
+    locked.chmod(0o000)
+    try:
+        with pytest.raises(shutil.Error):
+            copy_palace_dir(str(palace), str(tmp_path / "palace.backup"))
+    finally:
+        locked.chmod(0o700)
+
+
+@needs_unix_socket
+def test_uncopyable_reason_names_a_socket(tmp_path, monkeypatch):
+    sock_path = _bind_socket(tmp_path, "mcp.sock", monkeypatch)
+    assert _uncopyable_reason(sock_path, follow_symlinks=True) == "socket"
+    assert _uncopyable_reason(sock_path, follow_symlinks=False) == "socket"
+
+
+def test_uncopyable_reason_passes_regular_files_and_directories(tmp_path):
+    regular = tmp_path / "chroma.sqlite3"
+    regular.write_bytes(b"SQLite format 3\x00")
+    directory = tmp_path / "segment"
+    directory.mkdir()
+
+    for path in (regular, directory):
+        assert _uncopyable_reason(str(path), follow_symlinks=True) is None
+        assert _uncopyable_reason(str(path), follow_symlinks=False) is None
+
+
+@needs_unix_socket
+def test_uncopyable_reason_judges_a_symlink_by_how_the_copy_treats_it(tmp_path, monkeypatch):
+    _bind_socket(tmp_path, "mcp.sock", monkeypatch)
+    link = tmp_path / "link-to-sock"
+    _symlink_or_skip(link, tmp_path / "mcp.sock")
+
+    # Dereferenced by the copy: the target's type decides.
+    assert _uncopyable_reason(str(link), follow_symlinks=True) == "socket"
+    # Recreated as a link by the copy: the target is never read.
+    assert _uncopyable_reason(str(link), follow_symlinks=False) is None
+
+
+def test_uncopyable_reason_never_skips_on_a_failed_stat(tmp_path):
+    """An entry whose type could not be read is left for the copy to attempt.
+
+    Naming a reason here would skip it silently. A failed ``stat`` says the
+    entry cannot be resolved right now, not that it holds no data, and no
+    errno separates a deleted target from one on a volume that is not
+    mounted, so ``None`` is the only answer that cannot lose data.
+    """
+    missing = tmp_path / "not-there"
+    assert _uncopyable_reason(str(missing), follow_symlinks=True) is None
+
+    link = tmp_path / "onto-a-missing-volume"
+    _symlink_or_skip(link, missing)
+    assert _uncopyable_reason(str(link), follow_symlinks=True) is None
+
+
+def test_file_type_label_names_what_it_knows_and_refuses_the_rest():
+    """The allowlist needs an answer for a type its table has no name for.
+
+    Nothing on Linux reaches the fallback: ``stat.S_IFDOOR`` and its siblings
+    read as ``0`` there. macOS does define ``S_IFWHT``, so a mode carrying it
+    would land here, though producing one needs a union mount that no CI
+    runner has. The label is therefore exercised directly, rather than left
+    unpinned behind a state no platform here can create.
+    """
+    assert _file_type_label(stat.S_IFSOCK) == "socket"
+    assert _file_type_label(stat.S_IFIFO) == "named pipe"
+    assert _file_type_label(stat.S_IFCHR) == "character device"
+    assert _file_type_label(stat.S_IFBLK) == "block device"
+    assert _file_type_label(stat.S_IFLNK) == "not a regular file or directory"
+
+
+@needs_unprivileged_posix
+def test_copy_palace_dir_still_raises_when_a_symlink_target_cannot_be_inspected(tmp_path):
+    """The target may be real palace data behind a directory we cannot enter.
+
+    Skipping it would drop live data from the safety copy and let the rebuild
+    proceed over the live palace, so the copy has to fail the way it does on
+    ``develop``.
+    """
+    palace = _make_palace(tmp_path)
+    vault = tmp_path / "vault"
+    vault.mkdir()
+    (vault / "shard.sqlite3").write_text("REAL PALACE DATA", encoding="utf-8")
+    _symlink_or_skip(palace / "shard.sqlite3", vault / "shard.sqlite3")
+    vault.chmod(0o000)
+    dest = tmp_path / "palace.backup"
+    try:
+        with pytest.raises(shutil.Error):
+            copy_palace_dir(str(palace), str(dest))
+    finally:
+        vault.chmod(0o700)
+    assert not (dest / "shard.sqlite3").exists()
+
+
+def test_copy_palace_dir_still_raises_on_a_symlink_cycle(tmp_path):
+    """A cycle resolves to nothing, and that is still not a reason to skip."""
+    palace = _make_palace(tmp_path)
+    _symlink_or_skip(palace / "loop_a", palace / "loop_b")
+    _symlink_or_skip(palace / "loop_b", palace / "loop_a")
+
+    with pytest.raises(shutil.Error):
+        copy_palace_dir(str(palace), str(tmp_path / "palace.backup"))
+
+
+def test_copy_palace_dir_still_raises_on_a_symlink_to_its_own_parent(tmp_path):
+    """The directory-cycle form, which bloats the copy rather than failing fast."""
+    palace = _make_palace(tmp_path)
+    _symlink_or_skip(palace / "self", palace)
+
+    with pytest.raises(shutil.Error):
+        copy_palace_dir(str(palace), str(tmp_path / "palace.backup"))
+
+
+@needs_char_device
+def test_copy_palace_dir_skips_a_symlink_to_a_character_device(tmp_path):
+    """Device nodes do not make the copy raise; they make it copy the wrong thing.
+
+    Dereferencing a link to ``/dev/null`` writes an empty regular file, and a
+    link to ``/dev/zero`` is copied without bound, so both are left out. The
+    entry is a link rather than a real node because creating one needs root.
+    """
+    palace = _make_palace(tmp_path)
+    link = palace / "devnull"
+    _symlink_or_skip(link, "/dev/null")
+    dest = tmp_path / "palace.backup"
+
+    skipped = copy_palace_dir(str(palace), str(dest))
+
+    assert skipped == [(str(link), "character device")]
+    assert not os.path.lexists(dest / "devnull")
+    assert (dest / "chroma.sqlite3").is_file()
+
+
+@needs_fifo
+@needs_unprivileged_posix
+def test_copy_palace_dir_reports_skips_even_when_the_copy_fails(tmp_path):
+    """The skip list is what the operator needs most when the copy died.
+
+    The copy is made to fail partway through rather than before it starts, so
+    this really is the half-written-backup case and does not lean on the
+    order in which ``shutil`` happens to call the ignore callback.
+    """
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "aaa.fifo")
+    locked = palace / "locked-segment"
+    locked.mkdir()
+    (locked / "data_level0.bin").write_bytes(b"\x00")
+    locked.chmod(0o000)
+    dest = tmp_path / "palace.backup"
+    logs = []
+
+    try:
+        with pytest.raises(shutil.Error):
+            copy_palace_dir(str(palace), str(dest), log=logs.append)
+    finally:
+        locked.chmod(0o700)
+
+    assert any("aaa.fifo (named pipe)" in line for line in logs)
+    assert (dest / "chroma.sqlite3").is_file()
+
+
+@needs_fifo
+@needs_unprivileged_posix
+def test_copy_palace_dir_report_never_replaces_the_copys_own_failure(tmp_path):
+    """A failing ``log`` must not become the error the caller diagnoses.
+
+    Both callers pass ``print``, so the report can fail on its own: a closed
+    pipe (``mempalace repair | head``) or an stdout that cannot encode a
+    skipped entry's name. Letting that out of the failure path would hand the
+    caller that error instead of the reason the backup did not come out whole.
+    """
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "leftover.fifo")
+    locked = palace / "locked-segment"
+    locked.mkdir()
+    (locked / "data_level0.bin").write_bytes(b"\x00")
+    locked.chmod(0o000)
+    dest = tmp_path / "palace.backup"
+
+    def log_that_dies(_line):
+        # Not one of the terminal failures the per-line guard absorbs, so
+        # this is the report escaping as far as it ever can.
+        raise RuntimeError("logging is broken")
+
+    try:
+        with pytest.raises(shutil.Error):
+            copy_palace_dir(str(palace), str(dest), log=log_that_dies)
+    finally:
+        locked.chmod(0o700)
+
+
+@needs_fifo
+@needs_unprivileged_posix
+def test_copy_palace_dir_says_nothing_when_the_destination_cannot_be_created(tmp_path):
+    """The other way the copy never starts: the destination's parent is closed.
+
+    ``os.path.lexists`` reads False there as well, so the before-and-after
+    snapshot alone would report a skip list for a backup that does not exist.
+    """
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "leftover.fifo")
+    closed = tmp_path / "closed"
+    closed.mkdir()
+    closed.chmod(0o500)  # readable, not writable
+    logs = []
+
+    try:
+        with pytest.raises(PermissionError):
+            copy_palace_dir(str(palace), str(closed / "palace.backup"), log=logs.append)
+    finally:
+        closed.chmod(0o700)
+
+    assert logs == []
+
+
+@needs_fifo
+def test_copy_palace_dir_surfaces_an_unusable_log_when_the_copy_succeeded(tmp_path):
+    """Best-effort covers the terminal, not a caller who passed no logger.
+
+    Suppressing this too would hide an integration bug behind a backup that
+    reports success. The backup is already complete when it surfaces, and
+    nothing destructive has run yet.
+    """
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "leftover.fifo")
+
+    with pytest.raises(TypeError):
+        copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log="not-a-callable")
+
+
+@needs_fifo
+@pytest.mark.parametrize("shape", ["directory", "regular file", "dangling symlink"])
+def test_copy_palace_dir_says_nothing_about_a_backup_it_never_started(tmp_path, shape):
+    """``copytree`` classifies the top directory before it creates the
+    destination, so a destination it refuses still yields a skip list. It
+    describes a backup that does not exist, whatever is sitting in the way."""
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "leftover.fifo")
+    dest = tmp_path / "palace.backup"
+    if shape == "directory":
+        dest.mkdir()
+    elif shape == "regular file":
+        dest.write_text("in the way", encoding="utf-8")
+    else:
+        _symlink_or_skip(dest, tmp_path / "gone-away")
+    logs = []
+
+    with pytest.raises(FileExistsError):
+        copy_palace_dir(str(palace), str(dest), log=logs.append)
+
+    assert logs == []
+
+
+@needs_fifo
+def test_copy_palace_dir_survives_a_failing_report_on_a_good_copy(tmp_path):
+    """The report is best-effort, like ``prune_backups``' own cleanup."""
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "leftover.fifo")
+    dest = tmp_path / "palace.backup"
+
+    def log_that_dies(_line):
+        raise BrokenPipeError(32, "Broken pipe")
+
+    skipped = copy_palace_dir(str(palace), str(dest), log=log_that_dies)
+
+    assert [reason for _, reason in skipped] == ["named pipe"]
+    assert (dest / "chroma.sqlite3").is_file()
+
+
+@needs_fifo
+def test_copy_palace_dir_reports_skips_as_palace_relative_lines(tmp_path):
+    """The whole report is pinned, not just fragments of it.
+
+    Substring assertions accept an absolute path, a wrong plural and a lost
+    indent alike, so the rendering is compared line for line instead.
+    """
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "aaa.fifo")
+    os.mkfifo(palace / "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" / "bbb.fifo")
+    logs = []
+
+    copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log=logs.append)
+
+    assert logs == [
+        "  Backup: skipped 2 entries that cannot be copied:",
+        "    aaa.fifo (named pipe)",
+        f"    aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee{os.sep}bbb.fifo (named pipe)",
+    ]
+
+
+@needs_fifo
+def test_copy_palace_dir_report_does_not_swallow_an_interrupt(tmp_path):
+    """Best-effort covers failures, not the operator pressing Ctrl-C."""
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "leftover.fifo")
+
+    def log_that_aborts(_line):
+        raise KeyboardInterrupt
+
+    with pytest.raises(KeyboardInterrupt):
+        copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log=log_that_aborts)
+
+
+@needs_fifo
+def test_copy_palace_dir_reports_skips_in_a_stable_order(tmp_path):
+    """Within one directory the order is sorted, not whatever ``scandir`` gave.
+
+    Across directories it stays the order the copy visited them in, which is
+    filesystem-dependent; only the within-a-directory half is a promise.
+    """
+    palace = _make_palace(tmp_path)
+    for name in ("zzz.fifo", "mmm.fifo", "aaa.fifo"):
+        os.mkfifo(palace / name)
+
+    skipped = copy_palace_dir(str(palace), str(tmp_path / "palace.backup"))
+
+    names = [os.path.basename(path) for path, _ in skipped]
+    assert names == ["aaa.fifo", "mmm.fifo", "zzz.fifo"]
+
+
+@needs_fifo
+def test_copy_palace_dir_keeps_directories_in_visit_order(tmp_path):
+    """Directories stay in the order the copy walked them, not sorted globally.
+
+    The nested entry's full path sorts before the top-level one, so sorting
+    the whole list at the end would swap them and break the promise the
+    docstring makes.
+    """
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "zzz.fifo")
+    os.mkfifo(palace / "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" / "bbb.fifo")
+    logs = []
+
+    copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log=logs.append)
+
+    assert logs == [
+        "  Backup: skipped 2 entries that cannot be copied:",
+        "    zzz.fifo (named pipe)",
+        f"    aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee{os.sep}bbb.fifo (named pipe)",
+    ]
+
+
+@needs_fifo
+def test_copy_palace_dir_reports_skips_when_the_copy_is_interrupted(tmp_path, monkeypatch):
+    """Ctrl-C during a long copy still tells the operator what was left out.
+
+    That is why the copy is wrapped in ``except BaseException`` rather than
+    ``except Exception``: the half-written backup is real, and what it lacks
+    is what the operator has to know before deciding what to do with it.
+    """
+    palace = _make_palace(tmp_path)
+    os.mkfifo(palace / "leftover.fifo")
+    logs = []
+
+    def interrupt(*_args, **_kwargs):
+        raise KeyboardInterrupt
+
+    monkeypatch.setattr("mempalace.backups.shutil.copystat", interrupt)
+
+    with pytest.raises(KeyboardInterrupt):
+        copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log=logs.append)
+
+    assert logs == [
+        "  Backup: skipped 1 entry that cannot be copied:",
+        "    leftover.fifo (named pipe)",
+    ]
+
+
+@needs_fifo
+def test_copy_palace_dir_names_an_entry_it_cannot_make_relative(tmp_path, monkeypatch):
+    """``os.path.relpath`` calls ``os.getcwd()`` and rejects other drives.
+
+    Neither is a reason to drop the entry from the report, so the absolute
+    path stands in for the palace-relative one.
+    """
+    palace = _make_palace(tmp_path)
+    fifo = palace / "leftover.fifo"
+    os.mkfifo(fifo)
+    logs = []
+
+    def no_relpath(*_args, **_kwargs):
+        raise ValueError("path is on mount 'C:', start on mount 'D:'")
+
+    monkeypatch.setattr("mempalace.backups.os.path.relpath", no_relpath)
+
+    copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log=logs.append)
+
+    assert logs == [
+        "  Backup: skipped 1 entry that cannot be copied:",
+        f"    {fifo} (named pipe)",
+    ]
+
+
+@needs_fifo
+def test_copy_palace_dir_escapes_a_name_the_terminal_cannot_encode(tmp_path):
+    """A name stdout cannot encode is escaped, not dropped.
+
+    The count in the header comes from the copy, so dropping the line would
+    leave the operator reading "skipped 3" above a list of two. Retried in
+    ASCII, the entry is still named.
+    """
+    palace = _make_palace(tmp_path)
+    for name in ("aaa.fifo", "mü.fifo", "zzz.fifo"):
+        os.mkfifo(palace / name)
+    logs = []
+
+    def log_that_cannot_encode_one(line):
+        # Stands in for an stdout whose encoding cannot carry the name.
+        if not line.isascii():
+            raise UnicodeEncodeError("ascii", line, 0, 1, "cannot encode")
+        logs.append(line)
+
+    copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log=log_that_cannot_encode_one)
+
+    assert logs == [
+        "  Backup: skipped 3 entries that cannot be copied:",
+        "    aaa.fifo (named pipe)",
+        "    m\\xfc.fifo (named pipe)",
+        "    zzz.fifo (named pipe)",
+    ]
+
+
+@needs_fifo
+def test_copy_palace_dir_report_survives_a_line_that_cannot_be_written(tmp_path):
+    """A per-line write failure must not cost the operator the other lines."""
+    palace = _make_palace(tmp_path)
+    for name in ("aaa.fifo", "mmm.fifo", "zzz.fifo"):
+        os.mkfifo(palace / name)
+    logs = []
+
+    def log_that_dies_once(line):
+        if "mmm.fifo" in line:
+            raise BrokenPipeError(32, "Broken pipe")
+        logs.append(line)
+
+    copy_palace_dir(str(palace), str(tmp_path / "palace.backup"), log=log_that_dies_once)
+
+    assert logs == [
+        "  Backup: skipped 3 entries that cannot be copied:",
+        "    aaa.fifo (named pipe)",
+        "    zzz.fifo (named pipe)",
+    ]

tests/test_cli.py

@@ -1,8 +1,10 @@
 """Tests for mempalace.cli — the main CLI dispatcher."""
 
 import argparse
+import errno
 import os
 import shlex
+import socket
 import sqlite3
 import subprocess
 import sys
@@ -122,7 +124,13 @@ def test_cmd_status_custom_palace(mock_config_cls):
 def test_cmd_search_calls_search(mock_config_cls):
     mock_config_cls.return_value.palace_path = "/fake/palace"
     args = argparse.Namespace(
-        palace=None, query="test query", wing="mywing", room="myroom", results=3
+        palace=None,
+        query="test query",
+        wing="mywing",
+        room="myroom",
+        results=3,
+        since="2026-04-01",
+        before=None,
     )
     with patch("mempalace.searcher.search") as mock_search:
         cmd_search(args)
@@ -132,13 +140,17 @@ def test_cmd_search_calls_search(mock_config_cls):
             wing="mywing",
             room="myroom",
             n_results=3,
+            since="2026-04-01",
+            before=None,
         )
 
 
 @patch("mempalace.cli.MempalaceConfig")
 def test_cmd_search_error_exits(mock_config_cls):
     mock_config_cls.return_value.palace_path = "/fake/palace"
-    args = argparse.Namespace(palace=None, query="q", wing=None, room=None, results=5)
+    args = argparse.Namespace(
+        palace=None, query="q", wing=None, room=None, results=5, since=None, before=None
+    )
     from mempalace.searcher import SearchError
 
     with patch("mempalace.searcher.search", side_effect=SearchError("fail")):
@@ -593,6 +605,7 @@ def test_cmd_mine_convos_mode(mock_config_cls):
         no_gitignore=False,
         include_ignored=[],
         extract="general",
+        include_subagents=False,
     )
     with patch("mempalace.convo_miner.mine_convos") as mock_mine:
         cmd_mine(args)
@@ -604,9 +617,32 @@ def test_cmd_mine_convos_mode(mock_config_cls):
             limit=10,
             dry_run=True,
             extract_mode="general",
+            include_subagents=False,
         )
 
 
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_mine_convos_mode_threads_include_subagents_flag(mock_config_cls):
+    mock_config_cls.return_value.palace_path = "/fake/palace"
+    args = argparse.Namespace(
+        dir="/chats",
+        palace=None,
+        mode="convos",
+        wing="mywing",
+        agent="me",
+        limit=10,
+        dry_run=True,
+        no_gitignore=False,
+        include_ignored=[],
+        extract="exchange",
+        include_subagents=True,
+    )
+    with patch("mempalace.convo_miner.mine_convos") as mock_mine:
+        cmd_mine(args)
+        kwargs = mock_mine.call_args.kwargs
+        assert kwargs["include_subagents"] is True
+
+
 @patch("mempalace.cli.MempalaceConfig")
 def test_cmd_mine_include_ignored_comma_split(mock_config_cls):
     mock_config_cls.return_value.palace_path = "/fake/palace"
@@ -664,6 +700,54 @@ def test_cmd_mine_daemon_background_submits_job(mock_config_cls, capsys):
     assert "job-1" in capsys.readouterr().out
 
 
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_mine_daemon_lock_deferral_reports_a_runnable_command(mock_config_cls, capsys):
+    """A foreground mine refused the palace lock must say so and hand back a
+    command that actually works. --palace is global, so it has to be echoed back
+    ahead of the subcommand: without it the suggestion silently lists the
+    default palace's queue instead of the one the job is parked in."""
+    mock_config_cls.return_value.palace_path = "/fake/palace"
+    parked = {
+        "id": "job-1",
+        "state": "queued",
+        "error": {
+            "error_class": "LockHeldByOtherProcess",
+            "message": "palace /my palace is held by PID 999",
+        },
+    }
+    args = argparse.Namespace(
+        dir="/src",
+        palace="/my palace",  # non-default, and with a space to pin the quoting
+        mode="projects",
+        wing=None,
+        agent="mempalace",
+        limit=0,
+        dry_run=False,
+        no_gitignore=False,
+        include_ignored=None,
+        extract="exchange",
+        daemon=True,
+        background=False,
+        backend=None,
+        global_backend=None,
+        max_chunks_per_file=None,
+        redetect_origin=False,
+    )
+    with patch("mempalace.daemon.submit_job", return_value=parked) as mock_submit:
+        with pytest.raises(SystemExit) as exc:
+            cmd_mine(args)
+
+    assert exc.value.code == 1
+    assert mock_submit.call_args.kwargs["stop_on_lock_deferral"] is True
+    err = capsys.readouterr().err
+    assert "is held by PID 999" in err  # the holder, not a generic failure
+    assert "daemon submission failed" not in err  # the submission did not fail
+    assert f"mempalace --palace {shlex.quote('/my palace')} daemon jobs" in err
+    # Not `daemon wait`: this branch exists because we would not wait out the
+    # holder, so it must not hand back a command that does exactly that.
+    assert "daemon wait" not in err
+
+
 @patch("mempalace.cli.MempalaceConfig")
 def test_cmd_mine_background_requires_daemon(mock_config_cls, capsys):
     mock_config_cls.return_value.palace_path = "/fake/palace"
@@ -1127,6 +1211,79 @@ def test_cmd_repair_success(mock_config_cls, tmp_path, capsys):
     mock_new_col.add.assert_not_called()
 
 
+@pytest.mark.skipif(
+    os.name == "nt" or not hasattr(socket, "AF_UNIX"),
+    reason="Unix domain socket files are POSIX-only",
+)
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_survives_a_socket_in_the_palace_directory(
+    mock_config_cls, tmp_path, monkeypatch, capsys
+):
+    """#2207: a leftover daemon socket aborted repair at the backup step.
+
+    ``shutil.copytree`` raises on a Unix domain socket, so the command died
+    with a traceback before ``_rebuild_collection_via_temp`` ever ran, and
+    re-running only repeated it.
+    """
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close()
+    # Bound by relative name: the absolute tmp path can exceed the sun_path
+    # limit, which is tight on macOS.
+    monkeypatch.chdir(palace_dir)
+    leftover = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    try:
+        leftover.bind("mcp.sock")
+    finally:
+        leftover.close()
+    # A link out of the palace, to pin that this backup still DEREFERENCES
+    # links. Preserving them instead would put a bare symlink in the backup,
+    # and the rebuild that follows can leave it pointing at nothing.
+    outside = tmp_path / "outside.json"
+    outside.write_text("tunnel payload", encoding="utf-8")
+    try:
+        (palace_dir / "tunnels.json").symlink_to(outside)
+    except OSError as exc:
+        # Only a permission refusal is a skip; anything else is a bug here.
+        if os.name != "nt" and exc.errno not in (errno.EPERM, errno.EACCES):
+            raise
+        pytest.skip(f"symlink creation not permitted for this user: {exc}")
+
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+    mock_config_cls.return_value.collection_name = "mempalace_drawers"
+    args = argparse.Namespace(palace=None, yes=True)
+    mock_col = MagicMock()
+    mock_col.count.return_value = 2
+    mock_col.get.return_value = {
+        "ids": ["id1", "id2"],
+        "documents": ["doc1", "doc2"],
+        "metadatas": [{"wing": "a"}, {"wing": "b"}],
+    }
+    mock_temp_col = MagicMock()
+    mock_temp_col.count.return_value = 2
+    mock_new_col = MagicMock()
+    mock_new_col.count.return_value = 2
+    mock_backend = _mock_backend_for(col=mock_col, new_col=mock_new_col)
+    mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col]
+
+    with patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend):
+        cmd_repair(args)
+
+    out = capsys.readouterr().out
+    assert "Repair complete" in out
+    assert "    mcp.sock (socket)" in out.splitlines()
+    backup_dir = tmp_path / "palace.backup"
+    assert (backup_dir / "chroma.sqlite3").is_file()
+    assert not (backup_dir / "mcp.sock").exists()
+    # The socket is skipped, never removed from the live palace.
+    assert (palace_dir / "mcp.sock").exists()
+    # Dereferenced, so the backup survives losing what the link pointed at.
+    backed_up_link = backup_dir / "tunnels.json"
+    assert not backed_up_link.is_symlink()
+    outside.unlink()
+    assert backed_up_link.read_text(encoding="utf-8") == "tunnel payload"
+
+
 @patch("mempalace.cli.MempalaceConfig")
 def test_cmd_repair_uses_configured_collection(mock_config_cls, tmp_path, capsys):
     palace_dir = tmp_path / "palace"
@@ -1166,8 +1323,199 @@ def test_cmd_repair_uses_configured_collection(mock_config_cls, tmp_path, capsys
     ]
 
 
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_default_mode_dry_run_writes_nothing(mock_config_cls, tmp_path, capsys):
+    """``repair --dry-run`` with no --mode must print a plan and leave the palace alone.
+
+    #2095 / #2133 fixed this for ``--mode from-sqlite``; the default (legacy)
+    path still ran the backup copy and the full rebuild.
+    """
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close()
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+    mock_config_cls.return_value.collection_name = "mempalace_drawers"
+    args = argparse.Namespace(palace=None, yes=True, dry_run=True)
+    mock_col = MagicMock()
+    mock_col.count.return_value = 2
+    # A concrete extract payload, so that if the dry-run guard ever regresses
+    # this test fails on its assertions instead of spinning in _extract_drawers.
+    mock_col.get.return_value = {
+        "ids": ["id1", "id2"],
+        "documents": ["doc1", "doc2"],
+        "metadatas": [{"wing": "a"}, {"wing": "b"}],
+    }
+    mock_backend = _mock_backend_for(col=mock_col)
+
+    with (
+        patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend),
+        patch("mempalace.repair.sqlite_drawer_count", return_value=2) as mock_count,
+        patch("mempalace.migrate.confirm_destructive_action") as mock_confirm,
+    ):
+        cmd_repair(args)
+
+    out = capsys.readouterr().out
+    assert "DRY RUN -- no changes will be made." in out
+    assert "holds 2 rows" in out
+    assert "Repair complete" not in out
+    # The count comes from read-only SQLite, never from a chromadb client:
+    # opening one is itself a write to chroma.sqlite3.
+    mock_count.assert_called_once_with(str(palace_dir), "mempalace_drawers")
+    mock_backend.get_collection.assert_not_called()
+    mock_col.get.assert_not_called()
+    mock_backend.create_collection.assert_not_called()
+    mock_backend.delete_collection.assert_not_called()
+    mock_confirm.assert_not_called()
+    assert not (tmp_path / "palace.backup").exists()
+
+
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_default_mode_dry_run_honours_custom_collection(
+    mock_config_cls, tmp_path, capsys
+):
+    """The preview must target the configured collection, not a hardcoded name."""
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close()
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+    mock_config_cls.return_value.collection_name = "custom_drawers"
+    args = argparse.Namespace(palace=None, yes=True, dry_run=True)
+
+    with (
+        patch("mempalace.backends.chroma.ChromaBackend", return_value=_mock_backend_for()),
+        patch("mempalace.repair.sqlite_drawer_count", return_value=7) as mock_count,
+    ):
+        cmd_repair(args)
+
+    assert "[custom_drawers]" in capsys.readouterr().out
+    mock_count.assert_called_once_with(str(palace_dir), "custom_drawers")
+
+
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_default_mode_dry_run_reports_healable_fts5_and_continues(
+    mock_config_cls, tmp_path, capsys
+):
+    """An isolated FTS5 error is auto-healed by a real run, so the preview must not abort.
+
+    Aborting here would hand the operator the manual ``sqlite3 .recover``
+    recipe for a palace ``maybe_autoheal_fts5_index`` repairs by itself (#1596).
+    """
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close()
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+    mock_config_cls.return_value.collection_name = "mempalace_drawers"
+    args = argparse.Namespace(palace=None, yes=True, dry_run=True)
+
+    with (
+        patch(
+            "mempalace.repair.sqlite_integrity_errors",
+            return_value=["malformed inverted index for FTS5 table x"],
+        ),
+        patch("mempalace.repair.maybe_autoheal_fts5_index") as mock_autoheal,
+        patch("mempalace.repair.sqlite_drawer_count", return_value=4),
+        patch("mempalace.backends.chroma.ChromaBackend", return_value=_mock_backend_for()),
+    ):
+        cmd_repair(args)
+
+    out = capsys.readouterr().out
+    mock_autoheal.assert_not_called()
+    assert "isolated FTS5 inverted-index error" in out
+    assert "ABORT" not in out
+    assert "holds 4 rows" in out
+
+
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_default_mode_dry_run_still_aborts_on_broad_corruption(
+    mock_config_cls, tmp_path, capsys
+):
+    """Corruption a real run cannot heal must still abort the preview with exit 1."""
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close()
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+    mock_config_cls.return_value.collection_name = "mempalace_drawers"
+    args = argparse.Namespace(palace=None, yes=True, dry_run=True)
+
+    with (
+        patch(
+            "mempalace.repair.sqlite_integrity_errors",
+            return_value=["*** in database main *** Page 42 is never used"],
+        ),
+        patch("mempalace.repair.maybe_autoheal_fts5_index") as mock_autoheal,
+        patch("mempalace.backends.chroma.ChromaBackend"),
+        pytest.raises(SystemExit) as excinfo,
+    ):
+        cmd_repair(args)
+
+    assert excinfo.value.code == 1
+    mock_autoheal.assert_not_called()
+    assert "ABORT" in capsys.readouterr().out
+
+
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_non_dry_run_still_autoheals_fts5(mock_config_cls, tmp_path, capsys):
+    """The real path must keep calling the autoheal — the patch restructured this branch."""
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close()
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+    mock_config_cls.return_value.collection_name = "mempalace_drawers"
+    args = argparse.Namespace(palace=None, yes=True)
+    mock_col = MagicMock()
+    mock_col.count.return_value = 0
+
+    with (
+        patch(
+            "mempalace.repair.sqlite_integrity_errors",
+            return_value=["malformed inverted index for FTS5 table x"],
+        ),
+        patch("mempalace.repair.maybe_autoheal_fts5_index", return_value=[]) as mock_autoheal,
+        patch(
+            "mempalace.backends.chroma.ChromaBackend",
+            return_value=_mock_backend_for(col=mock_col),
+        ),
+    ):
+        cmd_repair(args)
+
+    mock_autoheal.assert_called_once()
+    assert "Nothing to repair" in capsys.readouterr().out
+
+
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_default_mode_dry_run_refuses_to_invent_zero(mock_config_cls, tmp_path, capsys):
+    """An unreadable count must refuse AND exit non-zero, like the from-sqlite preview.
+
+    Otherwise ``repair --dry-run && repair --yes`` walks into the destructive
+    run after a preview that could not be produced.
+    """
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close()
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+    mock_config_cls.return_value.collection_name = "mempalace_drawers"
+    args = argparse.Namespace(palace=None, yes=True, dry_run=True)
+    mock_backend = _mock_backend_for(col=MagicMock())
+
+    with (
+        patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend),
+        patch("mempalace.repair.sqlite_drawer_count", return_value=None),
+        pytest.raises(SystemExit) as excinfo,
+    ):
+        cmd_repair(args)
+
+    out = capsys.readouterr().out
+    assert excinfo.value.code == 1
+    assert "Cannot preview [mempalace_drawers]" in out
+    assert "refusing to invent zero counts" in out
+    assert "would extract" not in out
+    mock_backend.get_collection.assert_not_called()
+
+
 @patch("mempalace.cli.MempalaceConfig")
 def test_cmd_repair_restores_backup_on_live_rebuild_failure(mock_config_cls, tmp_path, capsys):
+    """When the live swap fails after the delete, recovery must PROMOTE the
+    verified temp copy instead of restoring a sqlite-only file backup."""
     palace_dir = tmp_path / "palace"
     palace_dir.mkdir()
     sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close()
@@ -1183,21 +1531,30 @@ def test_cmd_repair_restores_backup_on_live_rebuild_failure(mock_config_cls, tmp
     }
     mock_temp_col = MagicMock()
     mock_temp_col.count.return_value = 2
+    mock_promoted_col = MagicMock()
+    mock_promoted_col.count.return_value = 2
     mock_backend = _mock_backend_for(col=mock_col)
-    mock_backend.create_collection.side_effect = [mock_temp_col, RuntimeError("live build failed")]
+    mock_backend.create_collection.side_effect = [
+        mock_temp_col,
+        RuntimeError("live build failed"),
+        mock_promoted_col,
+    ]
     with patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend):
         with pytest.raises(SystemExit) as excinfo:
             cmd_repair(args)
     out = capsys.readouterr().out
     assert excinfo.value.code == 1
     assert "Repair failed" in out
-    assert "restoring from backup" in out
+    assert "Attempting recovery: promoting verified copy" in out
+    assert "Recovery succeeded" in out
     mock_backend.close_palace.assert_called_once_with(str(palace_dir))
     assert mock_backend.delete_collection.call_args_list == [
-        call(str(palace_dir), "mempalace_drawers__repair_tmp"),
-        call(str(palace_dir), "mempalace_drawers"),
-        call(str(palace_dir), "mempalace_drawers__repair_tmp"),
+        call(str(palace_dir), "mempalace_drawers__repair_tmp"),  # pre-clean stale temp
+        call(str(palace_dir), "mempalace_drawers"),  # live delete before re-upload
+        call(str(palace_dir), "mempalace_drawers"),  # delete broken live before promotion
+        call(str(palace_dir), "mempalace_drawers__repair_tmp"),  # temp cleaned after promotion
     ]
+    assert mock_promoted_col.upsert.called
 
 
 def _repair_backend_mocks(mock_config_cls, palace_dir, create_collection_results=None):
@@ -1779,6 +2136,35 @@ def test_cmd_repair_from_sqlite_success_does_not_exit(mock_config_cls, tmp_path)
         cmd_repair(args)
 
 
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_from_sqlite_dry_run_passes_through_and_skips_confirm(mock_config_cls, tmp_path):
+    """``repair --mode from-sqlite --dry-run`` must forward dry_run and skip
+    the destructive confirmation prompt (#2095, #2133)."""
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+
+    args = argparse.Namespace(
+        palace=str(palace_dir),
+        mode="from-sqlite",
+        source=None,
+        archive_existing=False,
+        yes=False,
+        dry_run=True,
+    )
+    with (
+        patch(
+            "mempalace.repair.rebuild_from_sqlite",
+            return_value={"mempalace_drawers": 0, "mempalace_closets": 0},
+        ) as mock_rebuild,
+        patch("mempalace.migrate.confirm_destructive_action") as mock_confirm,
+    ):
+        cmd_repair(args)
+
+    mock_confirm.assert_not_called()
+    assert mock_rebuild.call_args.kwargs["dry_run"] is True
+
+
 @patch("mempalace.cli.MempalaceConfig")
 def test_cmd_repair_from_sqlite_cleanup_failure_exits_nonzero(mock_config_cls, tmp_path, capsys):
     from mempalace.repair import RebuildCleanupError
@@ -1821,6 +2207,7 @@ def test_cmd_repair_rebuild_index_alias_uses_sqlite_archive(mock_config_cls, tmp
         source=None,
         archive_existing=False,
         yes=True,
+        dry_run=False,
     )
     fake_counts = {"mempalace_drawers": 1, "mempalace_closets": 0}
     with patch("mempalace.repair.rebuild_from_sqlite", return_value=fake_counts) as rebuild:
@@ -1830,4 +2217,5 @@ def test_cmd_repair_rebuild_index_alias_uses_sqlite_archive(mock_config_cls, tmp
         source_palace=str(palace_dir),
         dest_palace=str(palace_dir),
         archive_existing_dest=True,
+        dry_run=False,
     )

tests/test_cli_logstream.py

@@ -0,0 +1,354 @@
+"""Tests for the RFC 003 logstream/artifact CLI commands.
+
+Covers cmd_logstream (append/list/wait/ack) and cmd_artifact (put/get):
+JSON and human output, exact-content stdout piping, timeout exit code,
+and error exits. Uses SimpleNamespace args like the rest of test_cli.py.
+"""
+
+import json
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+from mempalace.cli import cmd_artifact, cmd_logstream, main
+
+
+def _append_args(palace, **overrides):
+    fields = dict(
+        palace=palace,
+        logstream_action="append",
+        type="task.request",
+        stream="project/mempalace",
+        room="delegation",
+        from_agent="mac-fable",
+        to_agent="windows-codex",
+        correlation_id="task_cli",
+        branch=None,
+        base_commit=None,
+        status=None,
+        body="Please fix the thing.",
+        body_file=None,
+        metadata=None,
+        artifact_id=None,
+        json=True,
+    )
+    fields.update(overrides)
+    return SimpleNamespace(**fields)
+
+
+def _list_args(palace, **overrides):
+    fields = dict(
+        palace=palace,
+        logstream_action="list",
+        stream=None,
+        room=None,
+        type=None,
+        to_agent=None,
+        from_agent=None,
+        correlation_id=None,
+        status=None,
+        since_event_id=None,
+        since_created_at=None,
+        limit=50,
+        json=True,
+    )
+    fields.update(overrides)
+    return SimpleNamespace(**fields)
+
+
+def _wait_args(palace, **overrides):
+    args = _list_args(palace, **overrides)
+    args.logstream_action = "wait"
+    if not hasattr(args, "timeout_ms"):
+        args.timeout_ms = 100
+    return args
+
+
+def _put_args(palace, content, **overrides):
+    fields = dict(
+        palace=palace,
+        artifact_action="put",
+        kind="patch",
+        created_by="windows-codex",
+        content=content,
+        file=None,
+        metadata=None,
+        json=True,
+    )
+    fields.update(overrides)
+    return SimpleNamespace(**fields)
+
+
+class TestLogstreamCli:
+    def test_append_then_list_json_round_trip(self, palace_path, capsys):
+        cmd_logstream(_append_args(palace_path))
+        appended = json.loads(capsys.readouterr().out)
+        assert appended["id"].startswith("evt_")
+
+        cmd_logstream(_list_args(palace_path, correlation_id="task_cli"))
+        listed = json.loads(capsys.readouterr().out)
+        assert listed["count"] == 1
+        assert listed["events"][0]["id"] == appended["id"]
+        assert listed["events"][0]["body"] == "Please fix the thing."
+
+    def test_append_human_output(self, palace_path, capsys):
+        cmd_logstream(_append_args(palace_path, json=False))
+        out = capsys.readouterr().out
+        assert "Appended:" in out
+        assert "task.request" in out
+        assert "project/mempalace/delegation" in out
+        assert "mac-fable->windows-codex" in out
+
+    def test_append_invalid_status_exits_1(self, palace_path, capsys):
+        with pytest.raises(SystemExit) as exc:
+            cmd_logstream(_append_args(palace_path, status="bogus"))
+        assert exc.value.code == 1
+        assert "status" in json.loads(capsys.readouterr().out)["error"]
+
+    def test_append_invalid_metadata_exits_1(self, palace_path, capsys):
+        with pytest.raises(SystemExit) as exc:
+            cmd_logstream(_append_args(palace_path, metadata="not json"))
+        assert exc.value.code == 1
+        assert "metadata" in json.loads(capsys.readouterr().out)["error"]
+
+    def test_body_file_stdin(self, palace_path, capsys, monkeypatch):
+        monkeypatch.setattr(sys, "stdin", __import__("io").StringIO("body from stdin\n"))
+        cmd_logstream(_append_args(palace_path, body=None, body_file="-"))
+        appended = json.loads(capsys.readouterr().out)
+        assert appended["body"] == "body from stdin\n"
+
+    def test_wait_existing_event_returns_immediately(self, palace_path, capsys):
+        cmd_logstream(_append_args(palace_path))
+        capsys.readouterr()
+        cmd_logstream(_wait_args(palace_path, correlation_id="task_cli", timeout_ms=5000))
+        result = json.loads(capsys.readouterr().out)
+        assert result["timed_out"] is False
+        assert result["count"] == 1
+
+    def test_wait_timeout_exits_2(self, palace_path, capsys):
+        with pytest.raises(SystemExit) as exc:
+            cmd_logstream(_wait_args(palace_path, correlation_id="task_never", timeout_ms=100))
+        assert exc.value.code == 2
+        result = json.loads(capsys.readouterr().out)
+        assert result["timed_out"] is True
+
+    def test_ack_round_trip(self, palace_path, capsys):
+        cmd_logstream(_append_args(palace_path))
+        appended = json.loads(capsys.readouterr().out)
+        cmd_logstream(
+            SimpleNamespace(
+                palace=palace_path,
+                logstream_action="ack",
+                event_id=appended["id"],
+                from_agent="windows-codex",
+                status="applied",
+                body="Done.",
+                json=True,
+            )
+        )
+        ack = json.loads(capsys.readouterr().out)
+        assert ack["type"] == "event.ack"
+        assert ack["to_agent"] == "mac-fable"
+        assert ack["correlation_id"] == "task_cli"
+
+
+class TestArtifactCli:
+    PATCH = "diff --git a/x b/x\n+cli\n"
+
+    def test_put_then_get_stdout_is_exact(self, palace_path, capsys):
+        cmd_artifact(_put_args(palace_path, self.PATCH))
+        artifact = json.loads(capsys.readouterr().out)
+        assert artifact["id"].startswith("art_")
+
+        cmd_artifact(
+            SimpleNamespace(
+                palace=palace_path,
+                artifact_action="get",
+                artifact_id=artifact["id"],
+                out=None,
+                json=False,
+            )
+        )
+        # Exact content, nothing else — must survive `| git apply`.
+        assert capsys.readouterr().out == self.PATCH
+
+    def test_get_out_writes_file(self, palace_path, tmp_dir, capsys):
+        cmd_artifact(_put_args(palace_path, self.PATCH))
+        artifact = json.loads(capsys.readouterr().out)
+        out_path = f"{tmp_dir}/fetched.patch"
+        cmd_artifact(
+            SimpleNamespace(
+                palace=palace_path,
+                artifact_action="get",
+                artifact_id=artifact["id"],
+                out=out_path,
+                json=True,
+            )
+        )
+        meta = json.loads(capsys.readouterr().out)
+        assert "content" not in meta
+        assert meta["content_written_to"] == out_path
+        assert open(out_path, encoding="utf-8").read() == self.PATCH
+
+    def test_get_missing_exits_1(self, palace_path, capsys):
+        with pytest.raises(SystemExit) as exc:
+            cmd_artifact(
+                SimpleNamespace(
+                    palace=palace_path,
+                    artifact_action="get",
+                    artifact_id="art_nope",
+                    out=None,
+                    json=True,
+                )
+            )
+        assert exc.value.code == 1
+        assert "not found" in json.loads(capsys.readouterr().out)["error"]
+
+    def test_put_reads_stdin_by_default(self, palace_path, capsys, monkeypatch):
+        monkeypatch.setattr(sys, "stdin", __import__("io").StringIO(self.PATCH))
+        cmd_artifact(_put_args(palace_path, None))
+        artifact = json.loads(capsys.readouterr().out)
+        assert artifact["size_bytes"] == len(self.PATCH.encode("utf-8"))
+
+    def test_event_can_reference_cli_artifact(self, palace_path, capsys):
+        cmd_artifact(_put_args(palace_path, self.PATCH))
+        artifact = json.loads(capsys.readouterr().out)
+        cmd_logstream(_append_args(palace_path, type="patch.ready", artifact_id=[artifact["id"]]))
+        event = json.loads(capsys.readouterr().out)
+        assert event["artifact_ids"] == [artifact["id"]]
+
+
+class TestMainDispatch:
+    def test_main_dispatches_logstream_list(self, palace_path, capsys, monkeypatch):
+        monkeypatch.setattr(
+            sys,
+            "argv",
+            ["mempalace", "--palace", palace_path, "logstream", "list", "--json"],
+        )
+        main()
+        result = json.loads(capsys.readouterr().out)
+        assert result == {"events": [], "count": 0}
+
+    def test_main_dispatches_artifact_put(self, palace_path, capsys, monkeypatch):
+        monkeypatch.setattr(
+            sys,
+            "argv",
+            [
+                "mempalace",
+                "--palace",
+                palace_path,
+                "artifact",
+                "put",
+                "--kind",
+                "note",
+                "--created-by",
+                "mac-fable",
+                "--content",
+                "hello",
+                "--json",
+            ],
+        )
+        main()
+        artifact = json.loads(capsys.readouterr().out)
+        assert artifact["kind"] == "note"
+        assert artifact["size_bytes"] == 5
+
+
+class TestVerbatimNewlines:
+    """CRLF content must survive the CLI byte-for-byte.
+
+    Every read/write path here used Python text mode, whose universal-newline
+    translation rewrites \\r\\n to \\n on read and (on Windows) \\n back to
+    \\r\\n on write. For a store whose whole contract is verbatim bytes
+    addressed by sha256, that is silent corruption: a patch produced by a
+    Windows agent arrives on another machine as different bytes with a
+    different digest.
+
+    It also disarmed the CRLF warning in put_artifact — the \\r it looks for
+    was already stripped before the content got there, so the one check meant
+    to catch unappliable diffs could never fire on the platform that produces
+    them.
+    """
+
+    CRLF_PATCH = "diff --git a/x b/x\r\n--- a/x\r\n+++ b/x\r\n@@ -1 +1 @@\r\n-old\r\n+new\r\n"
+
+    def _stdin(self, monkeypatch, text):
+        """Stand in for a real console stdin: a text layer with universal
+        newlines (what Windows gives you) over a .buffer holding the true
+        bytes. Reading the text layer translates; reading .buffer does not.
+        """
+        import io
+
+        raw = io.BytesIO(text.encode("utf-8"))
+        stream = io.TextIOWrapper(raw, encoding="utf-8", newline=None)
+        monkeypatch.setattr(sys, "stdin", stream)
+
+    def test_put_from_file_preserves_crlf(self, palace_path, tmp_dir, capsys):
+        src = f"{tmp_dir}/in.patch"
+        with open(src, "wb") as fh:
+            fh.write(self.CRLF_PATCH.encode("utf-8"))
+
+        cmd_artifact(_put_args(palace_path, None, file=src))
+        artifact = json.loads(capsys.readouterr().out)
+
+        assert artifact["size_bytes"] == len(self.CRLF_PATCH.encode("utf-8"))
+        assert artifact["sha256"] == _sha256(self.CRLF_PATCH)
+
+    def test_put_from_stdin_preserves_crlf(self, palace_path, monkeypatch, capsys):
+        self._stdin(monkeypatch, self.CRLF_PATCH)
+        cmd_artifact(_put_args(palace_path, None))
+        artifact = json.loads(capsys.readouterr().out)
+        assert artifact["sha256"] == _sha256(self.CRLF_PATCH)
+
+    def test_put_from_stdin_dash_preserves_crlf(self, palace_path, monkeypatch, capsys):
+        self._stdin(monkeypatch, self.CRLF_PATCH)
+        cmd_artifact(_put_args(palace_path, None, file="-"))
+        artifact = json.loads(capsys.readouterr().out)
+        assert artifact["sha256"] == _sha256(self.CRLF_PATCH)
+
+    def test_crlf_warning_actually_fires(self, palace_path, tmp_dir, capsys):
+        """The CRLF warning is the reason this bug mattered — prove it fires."""
+        src = f"{tmp_dir}/in.patch"
+        with open(src, "wb") as fh:
+            fh.write(self.CRLF_PATCH.encode("utf-8"))
+
+        cmd_artifact(_put_args(palace_path, None, file=src, json=False))
+        captured = capsys.readouterr()
+        assert "carriage returns" in captured.err
+
+    def test_get_out_is_byte_identical(self, palace_path, tmp_dir, capsys):
+        src = f"{tmp_dir}/in.patch"
+        with open(src, "wb") as fh:
+            fh.write(self.CRLF_PATCH.encode("utf-8"))
+        cmd_artifact(_put_args(palace_path, None, file=src))
+        artifact = json.loads(capsys.readouterr().out)
+
+        out_path = f"{tmp_dir}/out.patch"
+        cmd_artifact(
+            SimpleNamespace(
+                palace=palace_path,
+                artifact_action="get",
+                artifact_id=artifact["id"],
+                out=out_path,
+                json=True,
+            )
+        )
+        with open(out_path, "rb") as fh:
+            written = fh.read()
+        assert written == self.CRLF_PATCH.encode("utf-8")
+
+    def test_append_body_file_preserves_crlf(self, palace_path, tmp_dir, capsys):
+        src = f"{tmp_dir}/body.txt"
+        with open(src, "wb") as fh:
+            fh.write("line one\r\nline two\r\n".encode("utf-8"))
+
+        cmd_logstream(_append_args(palace_path, body=None, body_file=src))
+        event = json.loads(capsys.readouterr().out)
+        assert event["body"] == "line one\r\nline two\r\n"
+
+
+def _sha256(text):
+    import hashlib
+
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()

tests/test_cli_source_adapters.py

@@ -0,0 +1,554 @@
+"""CLI coverage for explicit RFC 002 source-adapter dispatch (#2062)."""
+
+import argparse
+import contextlib
+import multiprocessing
+import os
+import sys
+import time
+
+import pytest
+
+from mempalace import cli
+from mempalace.sources import (
+    AdapterSchema,
+    BaseSourceAdapter,
+    DrawerRecord,
+    SourceItemMetadata,
+    register,
+    reset_adapters,
+    unregister,
+)
+
+
+class _FixtureAdapter(BaseSourceAdapter):
+    name = "fixture"
+    adapter_version = "0.1.0"
+    instances = []
+
+    def __init__(self):
+        self.source = None
+        self.palace = None
+        self.__class__.instances.append(self)
+
+    def ingest(self, *, source, palace):
+        self.source = source
+        self.palace = palace
+        yield DrawerRecord(content="fixture content", source_file="fixture://record")
+
+    def describe_schema(self):
+        return AdapterSchema(version="1.0", fields={})
+
+
+class _FakeCollection:
+    def __init__(self):
+        self.upserts = []
+
+    def upsert(self, **kwargs):
+        self.upserts.append(kwargs)
+
+
+class _FakeKnowledgeGraph:
+    instances = []
+
+    def __init__(self, db_path):
+        self.db_path = db_path
+        self.closed = False
+        self.__class__.instances.append(self)
+
+    def close(self):
+        self.closed = True
+
+
+class _FakeConfig:
+    def __init__(self, palace_path=None):
+        self.palace_path = palace_path or "/fake/palace"
+
+
+class _DirectMutationAdapter(BaseSourceAdapter):
+    name = "direct-mutation"
+    adapter_version = "0.1.0"
+    instances = []
+
+    def __init__(self):
+        self.palace = None
+        self.__class__.instances.append(self)
+
+    def ingest(self, *, source, palace):
+        self.palace = palace
+        palace.drawer_collection.upsert(
+            documents=["direct content"], ids=["direct"], metadatas=[{}]
+        )
+        palace.knowledge_graph.add_triple("direct", "writes", "kg")
+        yield DrawerRecord(content="fixture content", source_file="fixture://record")
+
+    def describe_schema(self):
+        return AdapterSchema(version="1.0", fields={})
+
+
+class _ReadAwareAdapter(BaseSourceAdapter):
+    name = "read-aware"
+    observed_count = None
+
+    def ingest(self, *, source, palace):
+        self.__class__.observed_count = palace.drawer_collection.count()
+        yield DrawerRecord(content="fixture content", source_file="fixture://record")
+
+    def describe_schema(self):
+        return AdapterSchema(version="1.0", fields={})
+
+
+class _IncrementalAdapter(BaseSourceAdapter):
+    name = "incremental"
+    capabilities = frozenset({"supports_incremental"})
+
+    def ingest(self, *, source, palace):
+        raise AssertionError("incremental adapters must be rejected before ingest")
+
+    def describe_schema(self):
+        return AdapterSchema(version="1.0", fields={})
+
+
+class _MetadataAdapter(BaseSourceAdapter):
+    name = "metadata"
+
+    def ingest(self, *, source, palace):
+        yield DrawerRecord(content="before metadata", source_file="fixture://record")
+        yield SourceItemMetadata(source_file="fixture://record", version="v1")
+
+    def describe_schema(self):
+        return AdapterSchema(version="1.0", fields={})
+
+
+class _InvalidResultAdapter(BaseSourceAdapter):
+    name = "invalid-result"
+
+    def ingest(self, *, source, palace):
+        yield object()
+
+    def describe_schema(self):
+        return AdapterSchema(version="1.0", fields={})
+
+
+def _hold_palace_lock(palace_path, ready_flag, release_flag):
+    """Hold a writer lease in a separate process for contention coverage."""
+    from mempalace.palace import mine_palace_lock
+
+    with mine_palace_lock(palace_path):
+        open(ready_flag, "w").close()
+        for _ in range(500):
+            if os.path.exists(release_flag):
+                return
+            time.sleep(0.01)
+
+
+@pytest.fixture(autouse=True)
+def _isolated_fixture_adapter():
+    _FixtureAdapter.instances.clear()
+    _DirectMutationAdapter.instances.clear()
+    _ReadAwareAdapter.observed_count = None
+    _FakeKnowledgeGraph.instances.clear()
+    reset_adapters()
+    try:
+        yield
+    finally:
+        unregister("fixture")
+        reset_adapters()
+
+
+def _mine_args(*, source=None, mode=None, dry_run=False, palace=None):
+    return argparse.Namespace(
+        dir="/source",
+        palace=palace,
+        source=source,
+        mode=mode,
+        wing=None,
+        agent="mempalace",
+        limit=0,
+        dry_run=dry_run,
+        no_gitignore=False,
+        include_ignored=[],
+        extract="exchange",
+        daemon=False,
+        background=False,
+        max_chunks_per_file=None,
+        redetect_origin=False,
+    )
+
+
+def test_cmd_mine_source_dispatches_registered_adapter_through_palace_context(monkeypatch):
+    from mempalace import knowledge_graph, palace
+
+    collection = _FakeCollection()
+    register("fixture", _FixtureAdapter)
+    monkeypatch.setattr(cli, "MempalaceConfig", _FakeConfig)
+    monkeypatch.setattr(palace, "get_collection", lambda palace_path: collection)
+    monkeypatch.setattr(knowledge_graph, "KnowledgeGraph", _FakeKnowledgeGraph)
+
+    cli.cmd_mine(_mine_args(source="fixture"))
+
+    adapter = _FixtureAdapter.instances[0]
+    assert adapter.source.local_path == "/source"
+    assert adapter.palace.palace_path == "/fake/palace"
+    assert adapter.palace.adapter_name == "fixture"
+    assert adapter.palace.adapter_version == "0.1.0"
+    assert adapter.palace.drawer_collection is collection
+    assert adapter.palace.knowledge_graph is _FakeKnowledgeGraph.instances[0]
+    assert _FakeKnowledgeGraph.instances[0].closed is True
+    assert collection.upserts[0]["documents"] == ["fixture content"]
+    assert collection.upserts[0]["metadatas"][0]["adapter_name"] == "fixture"
+
+
+def test_cmd_mine_source_rejects_unknown_adapter(capsys):
+    with pytest.raises(SystemExit) as excinfo:
+        cli.cmd_mine(_mine_args(source="not-installed"))
+
+    assert excinfo.value.code == 2
+    assert "unknown source adapter 'not-installed'" in capsys.readouterr().err
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="cross-process lock semantics differ on Windows"
+)
+def test_cmd_mine_source_reports_real_contention_without_traceback(tmp_path, capsys):
+    """The CLI converts a real competing writer lease into a clean exit."""
+    register("fixture", _FixtureAdapter)
+    palace_path = str(tmp_path / "palace")
+    ready = str(tmp_path / "ready")
+    release = str(tmp_path / "release")
+    ctx = multiprocessing.get_context("spawn")
+    holder = ctx.Process(target=_hold_palace_lock, args=(palace_path, ready, release))
+    holder.start()
+    try:
+        for _ in range(500):
+            if os.path.exists(ready):
+                break
+            time.sleep(0.01)
+        assert os.path.exists(ready), "lock holder did not become ready"
+
+        with pytest.raises(SystemExit) as excinfo:
+            cli.cmd_mine(_mine_args(source="fixture", palace=palace_path))
+
+        assert excinfo.value.code == 1
+        error = capsys.readouterr().err
+        assert error.startswith(f"mempalace: palace {palace_path} is held by PID ")
+        assert "Traceback" not in error
+    finally:
+        open(release, "w").close()
+        holder.join(timeout=10)
+        if holder.is_alive():
+            holder.terminate()
+        assert holder.exitcode == 0
+
+
+def test_mine_source_dry_run_prevents_direct_collection_and_kg_mutations(monkeypatch):
+    from mempalace import knowledge_graph, palace
+
+    register("direct-mutation", _DirectMutationAdapter)
+    monkeypatch.setattr(cli, "MempalaceConfig", _FakeConfig)
+    # Dry runs must never reach the backend at all (see
+    # test_mine_source_dry_run_never_opens_existing_collection for the
+    # dedicated assertion); failing here catches a regression that
+    # reintroduces a real `get_collection` call in the dry-run path.
+    monkeypatch.setattr(palace, "get_collection", lambda *_a, **_k: pytest.fail("must not open"))
+    monkeypatch.setattr(knowledge_graph, "KnowledgeGraph", _FakeKnowledgeGraph)
+
+    drawers_written = cli.mine_source_adapter(
+        source_name="direct-mutation",
+        source_path="/source",
+        palace_path="/fake/palace",
+        dry_run=True,
+    )
+
+    adapter = _DirectMutationAdapter.instances[0]
+    assert drawers_written == 1
+    assert _FakeKnowledgeGraph.instances == []
+    assert [operation[0] for operation in adapter.palace.drawer_collection.operations] == [
+        "upsert",
+        "upsert",
+    ]
+    assert [operation[0] for operation in adapter.palace.knowledge_graph.operations] == [
+        "add_triple"
+    ]
+
+
+def test_mine_source_dry_run_never_opens_existing_collection(monkeypatch):
+    from mempalace import knowledge_graph, palace
+
+    register("read-aware", _ReadAwareAdapter)
+    monkeypatch.setattr(cli, "MempalaceConfig", _FakeConfig)
+    monkeypatch.setattr(
+        palace, "get_collection", lambda *_args, **_kwargs: pytest.fail("must not open")
+    )
+    monkeypatch.setattr(knowledge_graph, "KnowledgeGraph", _FakeKnowledgeGraph)
+
+    assert (
+        cli.mine_source_adapter(
+            source_name="read-aware",
+            source_path="/source",
+            palace_path="/fake/palace",
+            dry_run=True,
+        )
+        == 1
+    )
+
+    assert _ReadAwareAdapter.observed_count == 0
+
+
+def test_dry_run_collection_proxy_returns_backend_result_types():
+    from mempalace.backends import GetResult, QueryResult
+
+    collection = cli._DryRunCollectionProxy()
+
+    get_result = collection.get()
+    query_result = collection.query(query_texts=["one", "two"], include=["embeddings"])
+
+    assert isinstance(get_result, GetResult)
+    assert isinstance(query_result, QueryResult)
+    assert query_result.ids == [[], []]
+    assert query_result.embeddings == [[], []]
+
+
+def test_mine_source_dry_run_existing_uninitialized_palace_creates_no_chroma_artifacts(
+    tmp_path, monkeypatch
+):
+    """Dry runs must not initialize Chroma in an existing empty palace dir."""
+    register("fixture", _FixtureAdapter)
+    palace = tmp_path / "existing-palace"
+    palace.mkdir()
+    monkeypatch.setenv("MEMPALACE_BACKEND", "chroma")
+
+    assert (
+        cli.mine_source_adapter(
+            source_name="fixture",
+            source_path="/source",
+            palace_path=str(palace),
+            dry_run=True,
+        )
+        == 1
+    )
+
+    assert list(palace.iterdir()) == []
+    assert not (palace / "chroma.sqlite3").exists()
+
+
+def test_mine_source_dry_run_fresh_palace_creates_no_backend_artifacts(tmp_path, monkeypatch):
+    """Dry runs must not materialize a nonexistent Chroma palace."""
+    register("fixture", _FixtureAdapter)
+    palace = tmp_path / "fresh-palace"
+    monkeypatch.setenv("MEMPALACE_BACKEND", "chroma")
+
+    assert (
+        cli.mine_source_adapter(
+            source_name="fixture",
+            source_path="/source",
+            palace_path=str(palace),
+            dry_run=True,
+        )
+        == 1
+    )
+
+    assert not palace.exists()
+
+
+def test_mine_source_dry_run_preserves_initialized_sqlite_exact_artifacts(tmp_path, monkeypatch):
+    """Dry runs must not open or alter an existing sqlite_exact backend."""
+    from mempalace.backends.base import PalaceRef
+    from mempalace.backends.sqlite_exact import SQLiteExactBackend
+
+    register("fixture", _FixtureAdapter)
+    palace = tmp_path / "sqlite-palace"
+    backend = SQLiteExactBackend()
+    backend.get_collection(
+        palace=PalaceRef(id=str(palace), local_path=str(palace)),
+        collection_name="mempalace_drawers",
+        create=True,
+    )
+    backend.close()
+    monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact")
+
+    before = {path.name: path.read_bytes() for path in palace.iterdir()}
+    assert "sqlite_exact.sqlite3" in before
+
+    assert (
+        cli.mine_source_adapter(
+            source_name="fixture",
+            source_path="/source",
+            palace_path=str(palace),
+            dry_run=True,
+        )
+        == 1
+    )
+
+    after = {path.name: path.read_bytes() for path in palace.iterdir()}
+    assert after == before
+
+
+def test_mine_source_rejects_incremental_adapter_before_ingest():
+    register("incremental", _IncrementalAdapter)
+
+    with pytest.raises(cli.UnsupportedSourceAdapterProtocolError, match="incremental ingestion"):
+        cli.mine_source_adapter(
+            source_name="incremental",
+            source_path="/source",
+            palace_path="/fake/palace",
+        )
+
+
+def test_mine_source_accepts_non_incremental_metadata(monkeypatch, recwarn):
+    from mempalace import knowledge_graph, palace
+
+    register("metadata", _MetadataAdapter)
+    monkeypatch.setattr(cli, "MempalaceConfig", _FakeConfig)
+    collection = _FakeCollection()
+    monkeypatch.setattr(palace, "get_collection", lambda palace_path: collection)
+    monkeypatch.setattr(knowledge_graph, "KnowledgeGraph", _FakeKnowledgeGraph)
+
+    drawers_written = cli.mine_source_adapter(
+        source_name="metadata",
+        source_path="/source",
+        palace_path="/fake/palace",
+    )
+
+    assert drawers_written == 1
+    assert collection.upserts[0]["documents"] == ["before metadata"]
+    assert "non-incremental item metadata" in str(recwarn.pop(RuntimeWarning).message)
+
+
+def test_mine_source_rejects_unsupported_adapter_results():
+    """Unexpected adapter yields must fail rather than being silently dropped."""
+    register("invalid-result", _InvalidResultAdapter)
+
+    with pytest.raises(TypeError, match="unsupported result type object"):
+        cli.mine_source_adapter(
+            source_name="invalid-result",
+            source_path="/source",
+            palace_path="/fake/palace",
+            dry_run=True,
+        )
+
+
+def test_mine_source_holds_writer_lease_before_opening_handles(monkeypatch):
+    from mempalace import knowledge_graph, palace
+
+    collection = _FakeCollection()
+    active = False
+
+    @contextlib.contextmanager
+    def lock(_palace_path):
+        nonlocal active
+        active = True
+        try:
+            yield
+        finally:
+            active = False
+
+    def get_collection(_palace_path):
+        assert active
+        return collection
+
+    register("fixture", _FixtureAdapter)
+    monkeypatch.setattr(cli, "MempalaceConfig", _FakeConfig)
+    monkeypatch.setattr(palace, "mine_palace_lock", lock)
+    monkeypatch.setattr(palace, "get_collection", get_collection)
+    monkeypatch.setattr(knowledge_graph, "KnowledgeGraph", _FakeKnowledgeGraph)
+
+    assert (
+        cli.mine_source_adapter(
+            source_name="fixture", source_path="/source", palace_path="/fake/palace"
+        )
+        == 1
+    )
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="cross-process lock semantics differ on Windows"
+)
+def test_mine_source_refuses_held_writer_lease_before_opening_handles(tmp_path, monkeypatch):
+    """A competing writer prevents adapter ingest and all handle creation."""
+    from mempalace import knowledge_graph, palace
+
+    home = tmp_path / "home"
+    home.mkdir()
+    monkeypatch.setenv("HOME", str(home))
+    palace_path = str(tmp_path / "palace")
+    ready = str(tmp_path / "ready")
+    release = str(tmp_path / "release")
+    opened_collection = False
+    opened_kg = False
+
+    def get_collection(_palace_path):
+        nonlocal opened_collection
+        opened_collection = True
+        return _FakeCollection()
+
+    class TrackingKnowledgeGraph(_FakeKnowledgeGraph):
+        def __init__(self, db_path):
+            nonlocal opened_kg
+            opened_kg = True
+            super().__init__(db_path)
+
+    register("fixture", _FixtureAdapter)
+    monkeypatch.setattr(cli, "MempalaceConfig", _FakeConfig)
+    monkeypatch.setattr(palace, "get_collection", get_collection)
+    monkeypatch.setattr(knowledge_graph, "KnowledgeGraph", TrackingKnowledgeGraph)
+
+    ctx = multiprocessing.get_context("spawn")
+    holder = ctx.Process(target=_hold_palace_lock, args=(palace_path, ready, release))
+    holder.start()
+    try:
+        for _ in range(500):
+            if os.path.exists(ready):
+                break
+            time.sleep(0.01)
+        assert os.path.exists(ready), "lock holder did not become ready"
+
+        with pytest.raises(palace.MineAlreadyRunning):
+            cli.mine_source_adapter(
+                source_name="fixture", source_path="/source", palace_path=palace_path
+            )
+
+        assert len(_FixtureAdapter.instances) == 1
+        assert _FixtureAdapter.instances[0].palace is None
+        assert not opened_collection
+        assert not opened_kg
+    finally:
+        open(release, "w").close()
+        holder.join(timeout=10)
+        if holder.is_alive():
+            holder.terminate()
+        assert holder.exitcode == 0
+
+
+def test_cmd_mine_without_mode_preserves_projects_legacy_path(monkeypatch):
+    from unittest.mock import patch
+
+    monkeypatch.setattr(cli, "MempalaceConfig", _FakeConfig)
+    with patch("mempalace.miner.mine") as mine:
+        cli.cmd_mine(_mine_args())
+
+    mine.assert_called_once_with(
+        project_dir="/source",
+        palace_path="/fake/palace",
+        wing_override=None,
+        agent="mempalace",
+        limit=0,
+        dry_run=False,
+        respect_gitignore=True,
+        include_ignored=[],
+        max_chunks_per_file=None,
+    )
+
+
+def test_mine_parser_rejects_explicit_source_and_mode(monkeypatch, capsys):
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        ["mempalace", "mine", "/source", "--source", "fixture", "--mode", "projects"],
+    )
+
+    with pytest.raises(SystemExit) as excinfo:
+        cli.main()
+
+    assert excinfo.value.code == 2
+    assert "not allowed with argument" in capsys.readouterr().err

tests/test_closet_llm.py

@@ -8,9 +8,14 @@ These tests don't hit the network. They mock urllib to verify:
 """
 
 import json
+import os
+import subprocess
+import sys
 import tempfile
 from unittest.mock import patch
 
+import pytest
+
 from mempalace.closet_llm import (
     LLMConfig,
     _call_llm,
@@ -237,6 +242,73 @@ class TestRegenerateClosets:
             assert result["error"] == "missing-config"
             assert any("ENDPOINT" in m for m in result["missing"])
 
+    def test_regen_refuses_before_open_or_llm_when_palace_has_writer(self, tmp_path, monkeypatch):
+        """The non-dry-run operation must contend for process-lifetime palace
+        ownership before it opens sqlite_exact or performs expensive/external
+        work."""
+        from mempalace import closet_llm as closet_llm_mod
+        from mempalace.palace import MineAlreadyRunning
+
+        palace = str(tmp_path / "palace")
+        os.makedirs(palace)
+        monkeypatch.setenv("HOME", str(tmp_path / "home"))
+        monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact")
+        holder_code = """
+import sys
+from mempalace.palace import mine_palace_lock
+with mine_palace_lock(sys.argv[1]):
+    print("ready", flush=True)
+    sys.stdin.read()
+"""
+        holder = subprocess.Popen(
+            [sys.executable, "-c", holder_code, palace],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            env=os.environ.copy(),
+        )
+        try:
+            assert holder.stdout is not None
+            assert holder.stdout.readline().strip() == "ready"
+            cfg = LLMConfig(endpoint="http://local/v1", model="m")
+            with (
+                patch.object(
+                    closet_llm_mod,
+                    "get_collection",
+                    side_effect=AssertionError("collection opened before ownership"),
+                ),
+                patch.object(
+                    closet_llm_mod,
+                    "get_closets_collection",
+                    side_effect=AssertionError("closets opened before ownership"),
+                ),
+                patch.object(
+                    closet_llm_mod,
+                    "_call_llm",
+                    side_effect=AssertionError("LLM called before ownership"),
+                ),
+                patch.object(
+                    closet_llm_mod,
+                    "purge_file_closets",
+                    side_effect=AssertionError("delete reached before ownership"),
+                ),
+                patch.object(
+                    closet_llm_mod,
+                    "upsert_closet_lines",
+                    side_effect=AssertionError("upsert reached before ownership"),
+                ),
+            ):
+                with pytest.raises(MineAlreadyRunning):
+                    regenerate_closets(palace, cfg=cfg)
+        finally:
+            if holder.stdin is not None:
+                holder.stdin.close()
+            holder.wait(timeout=10)
+            if holder.returncode != 0:
+                stderr = holder.stderr.read() if holder.stderr is not None else ""
+                pytest.fail(f"lock holder failed with {holder.returncode}: {stderr}")
+
     def test_regen_purges_regex_closets_and_stamps_normalize_version(self, tmp_path):
         """Regression: before the hardening, regex closets for the same
         source survived alongside fresh LLM closets (the old path used a

tests/test_closets.py

@@ -1535,9 +1535,10 @@ class TestDrawerGrepExpansion:
         ``source_file + chunk_index`` pulls chunks from both groups as if
         they were sequential neighbors, corrupting the enriched text.
         Scoping by ``parent_drawer_id`` when present keeps each logical
-        group isolated. (``tool_diary_write`` chunks tag a different key
-        (``parent_entry_id``) and are written without ``source_file``, so
-        they never reach this enrichment path.)
+        group isolated. (``tool_diary_write`` chunks are written without
+        ``source_file``, so they never reach this enrichment path at all --
+        it returns early on the missing key -- regardless of which
+        parent-id key they carry.)
         """
         col = get_collection(palace_path)
         source = "shared.log"
@@ -1711,7 +1712,12 @@ class TestDrawerGrepExpansion:
         )
 
         result = search_memories("JWT authentication", palace_path)
-        assert result["results"]
+        # Surface the full envelope when search fails (Windows CI has flaked
+        # with KeyError on a bare ``result["results"]`` after a mid-query
+        # error dict that lacked the key).
+        assert "results" in result, f"search envelope missing results: {result!r}"
+        assert result["results"], f"hybrid search returned no hits: {result!r}"
+        assert "error" not in result, f"hybrid search failed: {result!r}"
         boosted = [h for h in result["results"] if h["matched_via"] == "drawer+closet"]
         assert boosted, "hybrid search should mark the closet-agreeing source"
         top = boosted[0]

tests/test_codex_plugin_manifest.py

@@ -0,0 +1,47 @@
+"""Contract tests for the Codex plugin marketplace metadata."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+MARKETPLACE_PATH = REPO_ROOT / ".agents" / "plugins" / "marketplace.json"
+MANIFEST_PATH = REPO_ROOT / ".codex-plugin" / "plugin.json"
+MCP_PATH = REPO_ROOT / ".mcp.json"
+
+
+def _read_json(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def test_marketplace_entry_uses_supported_codex_schema():
+    marketplace = _read_json(MARKETPLACE_PATH)
+    plugin = marketplace["plugins"][0]
+
+    assert plugin["name"] == "mempalace"
+    assert plugin["source"] == {"source": "local", "path": "./"}
+    assert plugin["policy"] == {
+        "installation": "AVAILABLE",
+        "authentication": "ON_INSTALL",
+    }
+
+
+def test_plugin_manifest_references_supported_components():
+    manifest = _read_json(MANIFEST_PATH)
+
+    assert manifest["mcpServers"] == "./.mcp.json"
+    assert "hooks" not in manifest
+
+
+def test_mcp_config_registers_mempalace_server():
+    config = _read_json(MCP_PATH)
+
+    assert config == {
+        "mcpServers": {
+            "mempalace": {
+                "command": "mempalace-mcp",
+            }
+        }
+    }

tests/test_config.py

@@ -531,7 +531,7 @@ def test_iso_temporal_normalizes_plus_zero_offset_to_z():
 # Backs the validated chunk_* properties added in #1024. Every property
 # resolves through ``_validated_chunk_config`` which (a) coerces to int
 # (or falls back to the documented default), (b) enforces the invariants
-# ``chunk_text()`` needs (chunk_size >= 1, chunk_overlap < chunk_size,
+# ``chunk_text()`` needs (chunk_size >= 1, chunk_overlap <= chunk_size // 2,
 # min_chunk_size <= chunk_size). A bad config.json must NEVER hang
 # ingest — repair, don't raise.
 
@@ -593,23 +593,55 @@ def test_chunk_config_zero_chunk_size_falls_back(tmp_path):
 
 
 def test_chunk_config_overlap_at_or_above_size_repaired(tmp_path):
-    """``chunk_overlap >= chunk_size`` is the hang condition; repair to
-    the documented default when the default fits, otherwise to
-    ``chunk_size - 1``."""
+    """``chunk_overlap`` above ``chunk_size // 2`` is the hang condition
+    (#2056); repair to the documented default when it stays at or below
+    half, otherwise clamp to ``chunk_size // 2``. Here the default fits."""
     cfg = _write_config(tmp_path, chunk_size=900, chunk_overlap=900)
     assert cfg.chunk_size == 900
-    # 100 (default) fits inside 900 → use the default.
+    # 100 (default) is at most 900 // 2, so use the default.
     assert cfg.chunk_overlap == 100
-    assert cfg.chunk_overlap < cfg.chunk_size
+    assert cfg.chunk_overlap <= cfg.chunk_size // 2
 
 
 def test_chunk_config_overlap_repair_when_default_doesnt_fit(tmp_path):
-    """Tiny chunk_size where the default overlap (100) wouldn't fit:
-    repair to ``chunk_size - 1`` instead."""
+    """Tiny chunk_size where the default overlap (100) exceeds half the
+    chunk size: clamp to ``chunk_size // 2``, the largest safe overlap."""
     cfg = _write_config(tmp_path, chunk_size=50, chunk_overlap=100)
     assert cfg.chunk_size == 50
-    assert cfg.chunk_overlap == 49  # max(0, chunk_size - 1)
-    assert cfg.chunk_overlap < cfg.chunk_size
+    assert cfg.chunk_overlap == 25  # min(DEFAULT_CHUNK_OVERLAP, chunk_size // 2)
+    assert cfg.chunk_overlap <= cfg.chunk_size // 2
+
+
+def test_chunk_config_overlap_above_half_repaired(tmp_path):
+    """#2056: an overlap between ``chunk_size // 2`` and ``chunk_size`` used
+    to pass validation and could hang the miner on short-line content. It is
+    now repaired down to a safe value."""
+    cfg = _write_config(tmp_path, chunk_size=100, chunk_overlap=80)
+    assert cfg.chunk_size == 100
+    assert cfg.chunk_overlap == 50  # min(100, 100 // 2)
+    assert cfg.chunk_overlap <= cfg.chunk_size // 2
+
+
+def test_chunk_config_overlap_at_half_preserved(tmp_path):
+    """Exactly 50% overlap (``== chunk_size // 2``) is safe and must be kept
+    unchanged."""
+    cfg = _write_config(tmp_path, chunk_size=800, chunk_overlap=400)
+    assert cfg.chunk_size == 800
+    assert cfg.chunk_overlap == 400
+    assert cfg.chunk_overlap <= cfg.chunk_size // 2
+
+
+def test_chunk_config_overlap_repair_odd_size_and_default_equals_half(tmp_path):
+    """Floor-boundary repairs (#2056): odd chunk_size floors ``// 2``, and the
+    case where DEFAULT_CHUNK_OVERLAP (100) equals chunk_size // 2 exactly."""
+    # Odd size: 101 // 2 == 50, so an over-half overlap clamps to min(100, 50).
+    cfg = _write_config(tmp_path, chunk_size=101, chunk_overlap=100)
+    assert cfg.chunk_overlap == 50
+    assert cfg.chunk_overlap <= cfg.chunk_size // 2
+    # DEFAULT_CHUNK_OVERLAP (100) == 200 // 2: repair clamps to exactly 100.
+    cfg2 = _write_config(tmp_path, chunk_size=200, chunk_overlap=180)
+    assert cfg2.chunk_overlap == 100
+    assert cfg2.chunk_overlap <= cfg2.chunk_size // 2
 
 
 def test_chunk_config_min_chunk_size_above_size_repaired(tmp_path):
@@ -710,13 +742,40 @@ def test_chunk_text_rejects_non_positive_chunk_size():
         chunk_text("some content", "src.txt", chunk_size=-1)
 
 
-def test_chunk_text_rejects_overlap_at_or_above_size():
+def test_chunk_text_rejects_overlap_above_half_size():
+    """#2056: chunk_overlap > chunk_size // 2 can loop forever on short-line
+    content, so chunk_text now rejects it fast (not only overlap >= size)."""
     from mempalace.miner import chunk_text
 
+    # overlap >= chunk_size (the original #1024 guard) stays rejected.
     with pytest.raises(ValueError, match="chunk_overlap"):
         chunk_text("some content", "src.txt", chunk_size=100, chunk_overlap=100)
     with pytest.raises(ValueError, match="chunk_overlap"):
         chunk_text("some content", "src.txt", chunk_size=100, chunk_overlap=200)
+    # NEW: overlap strictly above half is now rejected too.
+    with pytest.raises(ValueError, match="chunk_overlap"):
+        chunk_text("some content", "src.txt", chunk_size=100, chunk_overlap=51)
+    with pytest.raises(ValueError, match="chunk_overlap"):
+        chunk_text("some content", "src.txt", chunk_size=50, chunk_overlap=49)
+
+
+def test_chunk_text_overlap_boundary_at_half_size():
+    """The exact safety boundary is ``chunk_size // 2``: overlap == half is
+    accepted and terminates; overlap == half + 1 is rejected because it can
+    loop forever on content whose lines are about half the chunk size (#2056).
+    """
+    from mempalace.miner import chunk_text
+
+    worst = ("x" * 10 + "\n") * 40  # 11-char lines = 20 // 2 + 1
+    # overlap == chunk_size // 2 -> safe, returns a list (does not hang).
+    assert isinstance(chunk_text(worst, "src.txt", 20, 10), list)
+    # overlap == chunk_size // 2 + 1 -> rejected fast (would otherwise hang).
+    with pytest.raises(ValueError, match="chunk_overlap"):
+        chunk_text(worst, "src.txt", 20, 11)
+    # Odd chunk_size floors: 101 // 2 == 50, so 50 is accepted, 51 rejected.
+    assert isinstance(chunk_text("word " * 100, "src.txt", 101, 50), list)
+    with pytest.raises(ValueError, match="chunk_overlap"):
+        chunk_text("word " * 100, "src.txt", 101, 51)
 
 
 def test_chunk_text_rejects_negative_overlap():
@@ -903,3 +962,87 @@ def test_explicit_palace_path_overrides_env_and_file_config(monkeypatch, tmp_pat
     assert cfg.palace_path == expected
     assert cfg.hallway_file == os.path.join(os.path.dirname(expected), "hallways.json")
     assert cfg.tunnel_file == os.path.join(os.path.dirname(expected), "tunnels.json")
+
+
+# ── cfg.lang resolution ────────────────────────────────────────────────
+
+
+def test_lang_defaults_to_english():
+    cfg = MempalaceConfig(config_dir=tempfile.mkdtemp())
+    assert cfg.lang == "en"
+
+
+def test_lang_reads_config_file():
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "config.json"), "w") as f:
+        json.dump({"lang": "ja"}, f)
+    cfg = MempalaceConfig(config_dir=tmpdir)
+    assert cfg.lang == "ja"
+
+
+def test_lang_env_var_overrides_file():
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "config.json"), "w") as f:
+        json.dump({"lang": "ja"}, f)
+    os.environ["MEMPALACE_LANG"] = "ru"
+    try:
+        cfg = MempalaceConfig(config_dir=tmpdir)
+        assert cfg.lang == "ru"
+    finally:
+        del os.environ["MEMPALACE_LANG"]
+
+
+def test_lang_falls_back_to_entity_languages_first_entry():
+    """Without explicit lang, use entity_languages[0] so existing configs keep working."""
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "config.json"), "w") as f:
+        json.dump({"entity_languages": ["ko", "en"]}, f)
+    cfg = MempalaceConfig(config_dir=tmpdir)
+    assert cfg.lang == "ko"
+
+
+def test_lang_strips_whitespace():
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "config.json"), "w") as f:
+        json.dump({"lang": "  fr  "}, f)
+    cfg = MempalaceConfig(config_dir=tmpdir)
+    assert cfg.lang == "fr"
+
+
+# ── cfg.lang_explicit (opt-in signal) ──────────────────────────────────
+
+
+def test_lang_explicit_returns_none_without_user_config():
+    """Default palace has no explicit lang. Opt-in features must see None."""
+    cfg = MempalaceConfig(config_dir=tempfile.mkdtemp())
+    assert cfg.lang_explicit is None
+
+
+def test_lang_explicit_reads_config_file():
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "config.json"), "w") as f:
+        json.dump({"lang": "ja"}, f)
+    cfg = MempalaceConfig(config_dir=tmpdir)
+    assert cfg.lang_explicit == "ja"
+
+
+def test_lang_explicit_env_overrides_file():
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "config.json"), "w") as f:
+        json.dump({"lang": "ja"}, f)
+    os.environ["MEMPALACE_LANG"] = "ru"
+    try:
+        cfg = MempalaceConfig(config_dir=tmpdir)
+        assert cfg.lang_explicit == "ru"
+    finally:
+        del os.environ["MEMPALACE_LANG"]
+
+
+def test_lang_explicit_ignores_entity_languages_fallback():
+    """entity_languages drives cfg.lang for display, but not opt-in lang_explicit."""
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "config.json"), "w") as f:
+        json.dump({"entity_languages": ["ko", "en"]}, f)
+    cfg = MempalaceConfig(config_dir=tmpdir)
+    assert cfg.lang_explicit is None
+    assert cfg.lang == "ko"  # display-side fallback still works

tests/test_convo_miner.py

@@ -1,3 +1,4 @@
+import json
 import os
 import tempfile
 import shutil
@@ -572,6 +573,143 @@ def test_mine_convos_grown_file_purges_stale_drawers_not_additive(capsys):
         shutil.rmtree(tmpdir, ignore_errors=True)
 
 
+def test_mine_convos_skips_same_content_under_new_filename(capsys):
+    """Re-exporting the same conversation from Claude/ChatGPT under a new
+    filename (fresh export bundle, regenerated slug, etc.) must not create
+    a duplicate set of drawers -- only the exact-new content should file."""
+    tmpdir = tempfile.mkdtemp()
+    try:
+        transcript = (
+            "> What is the plan?\nStart with the schema, then the API.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n"
+        )
+        (Path(tmpdir) / "export_2026-01-01.txt").write_text(transcript)
+        palace_path = os.path.join(tmpdir, "palace")
+        mine_convos(tmpdir, palace_path, wing="test")
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        count_after_first = col.count()
+        assert count_after_first >= 2
+
+        # Simulate a later export: the same conversation lands under a new
+        # filename, alongside one genuinely new conversation.
+        (Path(tmpdir) / "export_2026-02-01.txt").write_text(transcript)
+        (Path(tmpdir) / "export_2026-02-01_new.txt").write_text(
+            "> What's next?\nUNIQUE_SECOND_EXPORT_MARKER covers the new work.\n"
+        )
+        mine_convos(tmpdir, palace_path, wing="test")
+        out = capsys.readouterr().out
+        assert "duplicate of export_2026-01-01.txt" in out
+
+        col = client.get_collection("mempalace_drawers")
+        docs = col.get(include=["documents"])["documents"]
+        dup_hits = sum(1 for d in docs if "Migration ordering is the main one" in d)
+        assert dup_hits == 1, f"duplicate transcript re-filed: {dup_hits} copies"
+        assert any("UNIQUE_SECOND_EXPORT_MARKER" in d for d in docs)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def _privacy_export_bundle(conversations):
+    """Build a Claude.ai privacy-export-shaped JSON payload: an array of
+    conversation objects, each with its own chat_messages list."""
+    return [
+        {
+            "chat_messages": [
+                {"sender": "human", "text": turn}
+                if i % 2 == 0
+                else {"sender": "assistant", "text": turn}
+                for i, turn in enumerate(turns)
+            ]
+        }
+        for turns in conversations
+    ]
+
+
+def test_mine_convos_skips_same_conversation_within_re_exported_bundle(capsys):
+    """A Claude.ai privacy export bundles every conversation into one JSON
+    file. Re-exporting that bundle under a new filename with one additional
+    conversation must not re-file the conversations that didn't change --
+    hashing the whole bundle would change the file-level hash the moment
+    any conversation is added, hiding the ones that are still duplicates.
+    """
+    tmpdir = tempfile.mkdtemp()
+    try:
+        convo_a = ["What is the plan?", "CONVO_A_MARKER: start with the schema."]
+        convo_b = ["Any risks?", "CONVO_B_MARKER: migration ordering is the main one."]
+        convo_c = ["What's next?", "CONVO_C_MARKER: covers the new work."]
+
+        bundle1 = _privacy_export_bundle([convo_a, convo_b])
+        (Path(tmpdir) / "export_2026-01-01.json").write_text(json.dumps(bundle1))
+
+        palace_path = os.path.join(tmpdir, "palace")
+        mine_convos(tmpdir, palace_path, wing="test")
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        docs_after_first = col.get(include=["documents"])["documents"]
+        assert any("CONVO_A_MARKER" in d for d in docs_after_first)
+        assert any("CONVO_B_MARKER" in d for d in docs_after_first)
+
+        # Re-export: same two conversations plus one genuinely new one, all
+        # under a fresh filename (as a real re-export from Claude would do).
+        bundle2 = _privacy_export_bundle([convo_a, convo_b, convo_c])
+        (Path(tmpdir) / "export_2026-02-01.json").write_text(json.dumps(bundle2))
+
+        mine_convos(tmpdir, palace_path, wing="test")
+
+        col = client.get_collection("mempalace_drawers")
+        docs = col.get(include=["documents"])["documents"]
+        a_hits = sum(1 for d in docs if "CONVO_A_MARKER" in d)
+        b_hits = sum(1 for d in docs if "CONVO_B_MARKER" in d)
+        c_hits = sum(1 for d in docs if "CONVO_C_MARKER" in d)
+        assert a_hits == 1, f"conversation A re-filed from the updated bundle: {a_hits} copies"
+        assert b_hits == 1, f"conversation B re-filed from the updated bundle: {b_hits} copies"
+        assert c_hits >= 1, "new conversation C was not filed at all"
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def test_content_dedup_is_scoped_per_wing():
+    """Mining the same transcript content into a second wing must file real
+    drawers there, not just the registry sentinel -- the content-hash map
+    is a dedup signal within a wing, not a cross-wing "already have this
+    content anywhere" gate.
+    """
+    tmpdir = tempfile.mkdtemp()
+    try:
+        transcript = (
+            "> What is the plan?\nStart with the schema, then the API.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n"
+        )
+        dir_a = Path(tmpdir) / "wing_a_src"
+        dir_b = Path(tmpdir) / "wing_b_src"
+        dir_a.mkdir()
+        dir_b.mkdir()
+        (dir_a / "session.txt").write_text(transcript)
+        (dir_b / "session.txt").write_text(transcript)
+
+        palace_path = os.path.join(tmpdir, "palace")
+        mine_convos(str(dir_a), palace_path, wing="wing_a")
+        mine_convos(str(dir_b), palace_path, wing="wing_b")
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        wing_b_docs = col.get(where={"wing": "wing_b"}, include=["documents", "metadatas"])
+        real_drawers = [
+            d
+            for d, m in zip(wing_b_docs["documents"], wing_b_docs["metadatas"])
+            if m.get("room") != "_registry"
+        ]
+        assert real_drawers, (
+            "wing_b holds only the registry sentinel -- content dedup leaked across wings"
+        )
+        assert any("Migration ordering is the main one" in d for d in real_drawers)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
 def test_prefetch_mined_set_returns_stored_mtime():
     """prefetch_mined_set's dict carries each source_file's stored mtime,
     not just membership."""
@@ -696,3 +834,230 @@ def test_register_file_sentinel_includes_source_mtime():
         assert abs(mined[str(tiny_file)] - os.path.getmtime(tiny_file)) < 0.001
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+# ---------------------------------------------------------------------------
+# file_conversation_exchange — canonical single-exchange write path
+# ---------------------------------------------------------------------------
+
+
+class _RecordingCollection:
+    """Captures upsert kwargs without a real ChromaDB behind it."""
+
+    def __init__(self):
+        self.upserts = []
+
+    def upsert(self, *, ids, documents, metadatas):
+        self.upserts.append({"ids": ids, "documents": documents, "metadatas": metadatas})
+
+
+def _exchange_kwargs(**overrides):
+    kwargs = {
+        "wing": "wing_dev",
+        "room": "conversations",
+        "text": "User: hi\n\nAssistant: hello",
+        "source_file": "hermes-session:s1",
+        "agent": "hermes",
+    }
+    kwargs.update(overrides)
+    return kwargs
+
+
+def test_file_conversation_exchange_extra_metadata_cannot_clobber_canonical():
+    """The docstring promises extras are append-only — colliding keys lose.
+
+    PR #1915 review: ``metadata.update(extra_metadata)`` let a caller
+    silently overwrite ``wing`` / ``filed_at`` / etc.
+    """
+    from mempalace.convo_miner import file_conversation_exchange
+
+    col = _RecordingCollection()
+    file_conversation_exchange(
+        col,
+        **_exchange_kwargs(),
+        extra_metadata={"wing": "wing_evil", "filed_at": "1970-01-01", "source": "hermes"},
+    )
+    meta = col.upserts[0]["metadatas"][0]
+    assert meta["wing"] == "wing_dev"
+    assert meta["filed_at"] != "1970-01-01"
+    # Non-colliding extras still land.
+    assert meta["source"] == "hermes"
+
+
+def test_file_conversation_exchange_invalid_wing_falls_back_to_wing_general():
+    """A bad configured wing must not drop the turn — verbatim first.
+
+    Same validation the MCP write tools apply (sanitize_name), but with a
+    wing_general fallback instead of an error: live filing losing turns
+    over a config typo would violate the 100%-recall promise.
+    """
+    from mempalace.convo_miner import file_conversation_exchange
+
+    col = _RecordingCollection()
+    file_conversation_exchange(col, **_exchange_kwargs(wing="../escape"))
+    meta = col.upserts[0]["metadatas"][0]
+    assert meta["wing"] == "wing_general"
+    assert col.upserts[0]["documents"] == ["User: hi\n\nAssistant: hello"]
+
+
+def test_file_conversation_exchange_invalid_room_falls_back_to_conversations():
+    from mempalace.convo_miner import file_conversation_exchange
+
+    col = _RecordingCollection()
+    file_conversation_exchange(col, **_exchange_kwargs(room="a/b"))
+    meta = col.upserts[0]["metadatas"][0]
+    assert meta["room"] == "conversations"
+
+
+def _write_dry_run_transcript(path: Path) -> None:
+    path.write_text(
+        "> What is the plan?\n"
+        "Start with the schema, then the API.\n\n"
+        "> Are there any risks?\n"
+        "Migration ordering is the main one.\n\n"
+        "> What comes next?\n"
+        "Run focused tests before the full suite.\n",
+        encoding="utf-8",
+    )
+
+
+def test_mine_convos_dry_run_skips_unchanged_mined_file(
+    tmp_path,
+    capsys,
+    monkeypatch,
+):
+    monkeypatch.setenv("HOME", str(tmp_path))
+
+    convo_dir = tmp_path / "convos"
+    convo_dir.mkdir()
+    transcript = convo_dir / "session.txt"
+    _write_dry_run_transcript(transcript)
+    palace_path = str(tmp_path / "palace")
+
+    mine_convos(
+        str(convo_dir),
+        palace_path,
+        wing="original",
+    )
+    capsys.readouterr()
+
+    mine_convos(
+        str(convo_dir),
+        palace_path,
+        wing="target",
+        dry_run=True,
+    )
+    output = capsys.readouterr().out
+
+    assert "[DRY RUN] session.txt" not in output
+    assert "Files processed: 0" in output
+    assert "Files skipped (already filed): 1" in output
+    assert "Drawers filed: 0" in output
+
+
+def test_mine_convos_dry_run_keeps_modified_file_as_work(
+    tmp_path,
+    capsys,
+    monkeypatch,
+):
+    monkeypatch.setenv("HOME", str(tmp_path))
+
+    convo_dir = tmp_path / "convos"
+    convo_dir.mkdir()
+    transcript = convo_dir / "session.txt"
+    _write_dry_run_transcript(transcript)
+    palace_path = str(tmp_path / "palace")
+
+    mine_convos(
+        str(convo_dir),
+        palace_path,
+        wing="original",
+    )
+    capsys.readouterr()
+
+    transcript.write_text(
+        transcript.read_text(encoding="utf-8")
+        + "\n> Did the plan change?\n"
+        + "Yes, add a migration rollback test.\n",
+        encoding="utf-8",
+    )
+
+    future = time.time() + 60
+    os.utime(transcript, (future, future))
+
+    mine_convos(
+        str(convo_dir),
+        palace_path,
+        wing="target",
+        dry_run=True,
+    )
+    output = capsys.readouterr().out
+
+    assert "[DRY RUN] session.txt" in output
+    assert "Files processed: 1" in output
+    assert "Files skipped (already filed): 0" in output
+
+
+def test_mine_convos_dry_run_missing_palace_does_not_create_it(
+    tmp_path,
+    capsys,
+    monkeypatch,
+):
+    monkeypatch.setenv("HOME", str(tmp_path))
+
+    convo_dir = tmp_path / "convos"
+    convo_dir.mkdir()
+    transcript = convo_dir / "session.txt"
+    _write_dry_run_transcript(transcript)
+    palace_path = tmp_path / "palace"
+
+    mine_convos(
+        str(convo_dir),
+        str(palace_path),
+        wing="target",
+        dry_run=True,
+    )
+    output = capsys.readouterr().out
+
+    assert "[DRY RUN] session.txt" in output
+    assert "Files processed: 1" in output
+    assert "Files skipped (already filed): 0" in output
+    assert not palace_path.exists()
+
+
+def test_mine_convos_dry_run_single_file_does_not_scan_siblings(
+    tmp_path,
+    capsys,
+):
+    selected = tmp_path / "selected.txt"
+    sibling = tmp_path / "sibling.txt"
+
+    selected.write_text(
+        "> Which transcript should be mined?\n"
+        "SELECTED_ONLY_MARKER belongs to the active transcript.\n\n"
+        "> Should sibling files be included?\n"
+        "No. Only the selected transcript should be scanned.\n",
+        encoding="utf-8",
+    )
+    sibling.write_text(
+        "> Should this sibling be mined?\n"
+        "SIBLING_SHOULD_NOT_BE_MINED by the single-file invocation.\n\n"
+        "> Is that important?\n"
+        "Yes. It keeps hook-triggered mining narrowly scoped.\n",
+        encoding="utf-8",
+    )
+
+    palace_path = tmp_path / "palace"
+
+    mine_convos(
+        str(selected),
+        str(palace_path),
+        wing="sessions",
+        dry_run=True,
+    )
+    output = capsys.readouterr().out
+
+    assert "Files:   1" in output
+    assert "[DRY RUN] selected.txt" in output
+    assert "sibling.txt" not in output
+    assert not palace_path.exists()

tests/test_convo_miner_unit.py

@@ -10,6 +10,7 @@ from mempalace.convo_miner import (
     _emit_bounded,
     _extract_authored_at,
     _file_chunks_locked,
+    _source_file_delete_ids,
     chunk_exchanges,
     detect_convo_room,
     scan_convos,
@@ -509,6 +510,101 @@ class TestScanConvos:
         assert "SKIP: unreadable.txt" in err
         assert "stat error" in err
 
+    def test_scan_skips_subagent_dirs_by_default(self, tmp_path):
+        # Mimic Claude Code layout: ~/.claude/projects/<slug>/<session>/subagents/agent-*.jsonl
+        session_dir = tmp_path / "session-abc"
+        session_dir.mkdir()
+        (session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+        subagents_dir = session_dir / "subagents"
+        subagents_dir.mkdir()
+        (subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+        (subagents_dir / "agent-def.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = [f.name for f in files]
+
+        assert "main.jsonl" in names
+        assert "agent-abc.jsonl" not in names
+        assert "agent-def.jsonl" not in names
+
+    def test_scan_includes_subagent_dirs_when_opted_in(self, tmp_path):
+        session_dir = tmp_path / "session-abc"
+        session_dir.mkdir()
+        (session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+        subagents_dir = session_dir / "subagents"
+        subagents_dir.mkdir()
+        (subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+
+        files = scan_convos(str(tmp_path), include_subagents=True)
+        names = [f.name for f in files]
+
+        assert "main.jsonl" in names
+        assert "agent-abc.jsonl" in names
+
+    def test_scan_skips_subagent_dirs_at_any_depth(self, tmp_path):
+        # The "subagents" name match is by directory name, not by depth: verify
+        # both shallow (top-level) and nested subagents/ get skipped.
+        (tmp_path / "subagents").mkdir()
+        (tmp_path / "subagents" / "agent-top.jsonl").write_text("{}", encoding="utf-8")
+        nested = tmp_path / "session" / "subagents"
+        nested.mkdir(parents=True)
+        (nested / "agent-deep.jsonl").write_text("{}", encoding="utf-8")
+        (tmp_path / "session" / "main.jsonl").write_text("{}", encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = [f.name for f in files]
+
+        assert "main.jsonl" in names
+        assert "agent-top.jsonl" not in names
+        assert "agent-deep.jsonl" not in names
+
+    def test_scan_does_not_skip_suffix_named_dirs(self, tmp_path):
+        # Exact name match only: 'mysubagents' or 'subagentsbackup' must still
+        # be mined. Guards against future regression to substring/regex match.
+        for dir_name in ("mysubagents", "subagentsbackup", "subagent"):
+            d = tmp_path / dir_name
+            d.mkdir()
+            (d / f"{dir_name}.jsonl").write_text("{}", encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = {f.name for f in files}
+
+        assert "mysubagents.jsonl" in names
+        assert "subagentsbackup.jsonl" in names
+        assert "subagent.jsonl" in names
+
+    def test_scan_skips_subagents_case_insensitive(self, tmp_path):
+        # On Windows + macOS APFS the filesystem is case-preserving; if Claude
+        # Code or a plugin ever emits 'Subagents' (capitalized), the filter
+        # must still match. Only one variant per tmp_path because case-
+        # insensitive filesystems collapse 'Subagents' and 'SUBAGENTS'.
+        d = tmp_path / "Subagents"
+        d.mkdir()
+        (d / "agent.jsonl").write_text("{}", encoding="utf-8")
+        (tmp_path / "main.jsonl").write_text("{}", encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = {f.name for f in files}
+
+        assert "main.jsonl" in names
+        assert "agent.jsonl" not in names
+
+    def test_scan_mines_an_explicitly_named_file_inside_subagents(self, tmp_path):
+        # The skip is directory pruning, so it cannot reach a caller who names
+        # one file: that path feeds a single synthetic entry with no directories
+        # to prune. The split is deliberate -- --include-subagents governs what a
+        # directory walk sweeps up, while naming a path is an explicit request
+        # and stays honored. Pinned because the two behaviours were written
+        # independently and nothing else exercises them together.
+        subagents_dir = tmp_path / "session-abc" / "subagents"
+        subagents_dir.mkdir(parents=True)
+        target = subagents_dir / "agent-abc.jsonl"
+        target.write_text('{"type":"user"}\n', encoding="utf-8")
+
+        files = scan_convos(str(target))
+
+        assert [f.name for f in files] == ["agent-abc.jsonl"]
+
 
 class TestFileChunksLocked:
     def test_uses_bounded_upsert_batches(self, monkeypatch):
@@ -583,6 +679,75 @@ class TestFileChunksLocked:
         assert "rag/foo.py" in entities
         assert "do_thing_now" in entities
 
+    def test_aborts_when_stale_drawer_purge_fails(self, monkeypatch):
+        """#105: a failed purge must abort the mine attempt, not silently
+        proceed to upsert on top of it — the same swallow already fixed
+        for miner.py's process_file at #23, own instance here."""
+        import mempalace.convo_miner as convo_miner
+
+        class FailingPurgeCol:
+            def __init__(self):
+                self.upsert_called = False
+
+            def get(self, *args, **kwargs):
+                raise RuntimeError("simulated transient backend error")
+
+            def delete(self, *args, **kwargs):
+                pass
+
+            def upsert(self, documents, ids, metadatas):
+                self.upsert_called = True
+
+        chunks = [{"content": f"chunk {i} " * 20, "chunk_index": i} for i in range(3)]
+        col = FailingPurgeCol()
+        monkeypatch.setattr(
+            convo_miner, "file_already_mined", lambda collection, source_file, **kwargs: False
+        )
+        monkeypatch.setattr(convo_miner, "mine_lock", lambda source_file: contextlib.nullcontext())
+        monkeypatch.setattr(convo_miner, "_detect_hall_cached", lambda content: "conversations")
+
+        drawers, room_counts, skipped = _file_chunks_locked(
+            col, "chat.txt", chunks, "wing", "general", "agent", "exchange"
+        )
+
+        assert col.upsert_called is False, (
+            "_file_chunks_locked inserted new chunks even though the "
+            "stale-drawer purge raised — old and new rows can now coexist "
+            "as duplicates/orphans"
+        )
+        assert drawers == 0
+        assert skipped is True
+
+
+class TestSourceFileDeleteIds:
+    """#104: the sweeper writes drawers with no extract_mode at all
+    (ingest_mode="sweep"). convo_miner's default exchange-mode purge
+    must not scoop those up — they were never meant to carry
+    extract_mode, unlike a genuine legacy pre-schema convo_miner row."""
+
+    def test_excludes_sweeper_rows_from_exchange_mode_purge(self):
+        class FakeCol:
+            def get(self, where=None, limit=None, offset=0, include=None):
+                if offset > 0:
+                    return {"ids": [], "metadatas": []}
+                return {
+                    "ids": ["sweep_1", "exchange_1", "legacy_1"],
+                    "metadatas": [
+                        {"ingest_mode": "sweep", "session_id": "s1", "role": "user"},
+                        {"ingest_mode": "convos", "extract_mode": "exchange"},
+                        {"source_file": "chat.txt"},  # pre-ingest_mode legacy row
+                    ],
+                }
+
+        delete_ids = _source_file_delete_ids(FakeCol(), "chat.txt", "exchange")
+
+        assert "sweep_1" not in delete_ids, (
+            "sweeper's drawer was scooped into convo_miner's default "
+            "exchange-mode purge and would be deleted on the next re-mine"
+        )
+        assert "exchange_1" in delete_ids
+        assert "legacy_1" in delete_ids
+
 
 class TestExtractAuthoredAt:
     """authored_at = max per-line ``timestamp`` in a transcript (real authored date,
@@ -643,3 +808,21 @@ class TestExtractAuthoredAt:
         f = tmp_path / "session.jsonl"
         f.write_text('{"timestamp": 1}\n{"timestamp": false}\n')
         assert _extract_authored_at(f) is None
+
+
+def test_scan_convos_accepts_one_file_without_scanning_siblings(
+    tmp_path,
+):
+    selected = tmp_path / "selected.jsonl"
+    sibling = tmp_path / "sibling.jsonl"
+
+    selected.write_text(
+        '{"type": "user"}\n',
+        encoding="utf-8",
+    )
+    sibling.write_text(
+        '{"type": "user"}\n',
+        encoding="utf-8",
+    )
+
+    assert scan_convos(str(selected)) == [selected.resolve()]

tests/test_daemon.py

@@ -1,5 +1,7 @@
 import os
+import sqlite3
 import subprocess
+import sys
 import threading
 import time
 
@@ -10,6 +12,16 @@ from _chroma_palace_helper import make_minimal_chroma_sqlite
 from mempalace import daemon
 from mempalace import service
 
+_LOCK_CONTENDER = """
+from mempalace.palace import MineAlreadyRunning, mine_palace_lock
+import sys
+try:
+    with mine_palace_lock(sys.argv[1]):
+        raise SystemExit(0)
+except MineAlreadyRunning:
+    raise SystemExit(23)
+"""
+
 # POSIX file-mode bits (0600/0700) are not representable on Windows: os.chmod
 # can only toggle the read-only attribute, so a "private" file still reports
 # 0o666. The daemon relies on the user-profile directory ACLs for privacy
@@ -132,6 +144,31 @@ def test_daemon_http_lifecycle_executes_job(tmp_path, monkeypatch):
     _stop_server(client, thread, holders)
 
 
+def test_daemon_holds_local_backend_writer_lease_for_lifetime(tmp_path, monkeypatch):
+    monkeypatch.setenv("HOME", str(tmp_path / "home"))
+    client, thread, palace, holders = _start_server(
+        tmp_path, monkeypatch, lambda kind, payload: {"success": True, "exit_code": 0}
+    )
+    try:
+        result = subprocess.run(
+            [sys.executable, "-c", _LOCK_CONTENDER, str(palace)],
+            check=False,
+            env=os.environ.copy(),
+            timeout=10,
+        )
+        assert result.returncode == 23
+    finally:
+        _stop_server(client, thread, holders)
+
+    released = subprocess.run(
+        [sys.executable, "-c", _LOCK_CONTENDER, str(palace)],
+        check=False,
+        env=os.environ.copy(),
+        timeout=10,
+    )
+    assert released.returncode == 0
+
+
 def test_submit_job_uses_client_and_waits(monkeypatch, tmp_path):
     palace = tmp_path / "palace"
     palace.mkdir()
@@ -144,7 +181,7 @@ def test_submit_job_uses_client_and_waits(monkeypatch, tmp_path):
             self.submitted = (kind, payload, dedupe_key, priority)
             return {"id": "job-1", "state": "queued"}
 
-        def wait(self, job_id, timeout=daemon.DEFAULT_WAIT_TIMEOUT):
+        def wait(self, job_id, timeout=daemon.DEFAULT_WAIT_TIMEOUT, stop_on_lock_deferral=False):
             assert job_id == "job-1"
             return {
                 "id": "job-1",
@@ -340,8 +377,30 @@ def test_shutdown_cancels_active_job(tmp_path, monkeypatch):
     # And recover_running must not re-queue a cancelled job.
     assert store.recover_running() == 0
 
+    # The server thread is gone, but its timed-out worker is still executing.
+    # Writer ownership must stay with that worker until it truly exits.
+    contender = subprocess.run(
+        [sys.executable, "-c", _LOCK_CONTENDER, str(palace)],
+        check=False,
+        env=os.environ.copy(),
+        timeout=10,
+    )
+    assert contender.returncode == 23
+
     # Release the blocked worker so it (and the daemon thread) can exit.
     block.set()
+    deadline = time.monotonic() + 5
+    while time.monotonic() < deadline:
+        released = subprocess.run(
+            [sys.executable, "-c", _LOCK_CONTENDER, str(palace)],
+            check=False,
+            env=os.environ.copy(),
+            timeout=10,
+        )
+        if released.returncode == 0:
+            break
+        time.sleep(0.02)
+    assert released.returncode == 0
 
 
 def test_recover_running_dead_letters_exhausted_jobs(tmp_path, monkeypatch):
@@ -367,6 +426,488 @@ def test_recover_running_dead_letters_exhausted_jobs(tmp_path, monkeypatch):
     assert final.attempts == daemon.MAX_ATTEMPTS
 
 
+# --- #2014: a lease refusal is a deferral, not a failure ---
+
+
+def test_lease_refusal_defers_job_and_runs_it_on_the_next_claim(tmp_path, monkeypatch):
+    """A LockHeldByOtherProcess refusal means the palace write never landed, so
+    re-running it cannot duplicate content. It must be deferred and picked up
+    again, not dead-lettered. Regression for #2014, where a lock conflict ended
+    terminal 'failed' and only ever ran again if some hook happened to re-submit
+    equivalent work."""
+    calls = {"n": 0}
+
+    def fake_execute(kind, payload):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            return {
+                "success": False,
+                "error": "palace /p is held by PID 999 (mempalace-mcp)",
+                "error_class": daemon.LOCK_REFUSAL_ERROR_CLASS,
+                "exit_code": 1,
+            }
+        return {"success": True, "exit_code": 0}
+
+    monkeypatch.setattr(daemon, "LOCK_DEFER_BACKOFF_SECONDS", 0.05)
+    client, thread, palace, holders = _start_server(tmp_path, monkeypatch, fake_execute)
+    try:
+        job = client.submit("diary_write", {"entry": "verbatim"})
+        finished = client.wait(job["id"], timeout=10)
+        assert finished["state"] == "succeeded"
+        assert calls["n"] == 2  # refused once, then executed for real
+        # The refusal spent no attempt: only the successful claim counted.
+        assert finished["attempts"] == 1
+    finally:
+        _stop_server(client, thread, holders)
+
+
+def test_repeated_lease_refusal_never_exhausts_attempts(tmp_path, monkeypatch):
+    """#2014's actual promise: a palace locked across many claims must never
+    dead-letter the work. One refusal cannot show that -- MAX_ATTEMPTS is 3, so
+    the job has to survive more refusals than that and still run."""
+    calls = {"n": 0}
+    refusals = daemon.MAX_ATTEMPTS + 2
+
+    def fake_execute(kind, payload):
+        calls["n"] += 1
+        if calls["n"] <= refusals:
+            return {
+                "success": False,
+                "error": "palace /p is held by PID 999",
+                "error_class": daemon.LOCK_REFUSAL_ERROR_CLASS,
+                "exit_code": 1,
+            }
+        return {"success": True, "exit_code": 0}
+
+    monkeypatch.setattr(daemon, "LOCK_DEFER_BACKOFF_SECONDS", 0.02)
+    client, thread, palace, holders = _start_server(tmp_path, monkeypatch, fake_execute)
+    try:
+        job = client.submit("mine", {"source": "src"})
+        finished = client.wait(job["id"], timeout=20)
+        assert finished["state"] == "succeeded"
+        assert calls["n"] == refusals + 1
+        assert finished["attempts"] == 1  # every refusal refunded its claim
+    finally:
+        _stop_server(client, thread, holders)
+
+
+def test_wait_returns_a_lock_deferred_job_instead_of_waiting_out_the_holder(tmp_path, monkeypatch):
+    """An interactive caller must not block until a deferred job goes terminal --
+    it never will while the holder lives, so the default hour-long wait would
+    strand the terminal behind a peer session. Background waiters keep waiting."""
+
+    def fake_execute(kind, payload):
+        return {
+            "success": False,
+            "error": "palace /p is held by PID 999 (mempalace-mcp)",
+            "error_class": daemon.LOCK_REFUSAL_ERROR_CLASS,
+            "exit_code": 1,
+        }
+
+    monkeypatch.setattr(daemon, "LOCK_DEFER_BACKOFF_SECONDS", 5.0)
+    client, thread, palace, holders = _start_server(tmp_path, monkeypatch, fake_execute)
+    try:
+        job = client.submit("mine", {"source": "src"})
+        parked = client.wait(job["id"], timeout=10, stop_on_lock_deferral=True)
+        assert parked["state"] == "queued"  # returned early, not terminal
+        assert daemon.job_deferred_by_lock(parked)
+        assert "PID 999" in parked["error"]["message"]
+
+        # Without the flag the same job is not terminal, so the wait times out
+        # rather than returning -- the hang a naive deferral would introduce.
+        with pytest.raises(daemon.DaemonError):
+            client.wait(job["id"], timeout=0.5)
+    finally:
+        _stop_server(client, thread, holders)
+
+
+def test_job_deferred_by_lock_ignores_ordinary_queued_and_failed_jobs(tmp_path, monkeypatch):
+    """The predicate must not mistake a job merely awaiting its turn -- or a
+    genuinely failed one -- for a lock deferral, or the CLI would bail out of a
+    healthy queue."""
+    assert not daemon.job_deferred_by_lock({"state": "queued", "error": None})
+    assert not daemon.job_deferred_by_lock({"state": "queued", "error": {}})
+    assert not daemon.job_deferred_by_lock(
+        {"state": "failed", "error": {"error_class": daemon.LOCK_REFUSAL_ERROR_CLASS}}
+    )
+    assert not daemon.job_deferred_by_lock(
+        {"state": "queued", "error": {"error_class": "ValueError"}}
+    )
+    assert daemon.job_deferred_by_lock(
+        {"state": "queued", "error": {"error_class": daemon.LOCK_REFUSAL_ERROR_CLASS}}
+    )
+
+
+def test_claim_clears_a_stale_deferral_reason(tmp_path, monkeypatch):
+    """The reason defer parks on a queued job describes the claim that ended.
+    Leaving it would outlive its cause: the predicate would report a lock
+    refusal for a job that later crashed, and recover_running's dead-letter
+    (which COALESCEs error_json) would blame the lock instead of
+    MaxAttemptsExceeded."""
+    monkeypatch.setenv(daemon.STATE_ROOT_ENV, str(tmp_path / "state"))
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    store = daemon.QueueStore(daemon.queue_path(str(palace)))
+    job = store.enqueue("diary_write", {"entry": "x"})
+
+    store.claim_next()
+    store.defer(job.id, error={"error_class": daemon.LOCK_REFUSAL_ERROR_CLASS, "message": "held"})
+    assert daemon.job_deferred_by_lock(daemon.job_to_dict(store.get(job.id)))
+
+    claimed = store.claim_next()
+    assert claimed.error is None  # the refusal belonged to the previous claim
+
+    # Now crash out at MAX_ATTEMPTS: the dead-letter must name the real cause.
+    with store._lock, store._connect() as conn:
+        conn.execute(
+            "UPDATE jobs SET state='running', attempts=? WHERE id=?",
+            (daemon.MAX_ATTEMPTS, job.id),
+        )
+    store.recover_running()
+    final = store.get(job.id)
+    assert final.state == "failed"
+    assert final.error["error_class"] == "MaxAttemptsExceeded"
+
+
+def test_defer_records_why_the_job_went_back_to_the_queue(tmp_path, monkeypatch):
+    """A job parked behind the palace lock must be distinguishable from one
+    merely awaiting its turn.
+
+    The reason is what job_deferred_by_lock keys on, and what the foreground CLI
+    and the hook log quote back at the operator; without it a parked job and a
+    job simply waiting its turn are the same row. (`daemon jobs` prints only
+    id/state/kind/created_at, so it does not show this -- surfacing it there is
+    a separate change.)"""
+    monkeypatch.setenv(daemon.STATE_ROOT_ENV, str(tmp_path / "state"))
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    store = daemon.QueueStore(daemon.queue_path(str(palace)))
+    job = store.enqueue("mine", {"source": "src"})
+    store.claim_next()
+
+    deferred = store.defer(
+        job.id,
+        error={"error_class": daemon.LOCK_REFUSAL_ERROR_CLASS, "message": "held by PID 999"},
+    )
+    assert deferred.state == "queued"
+    assert deferred.error["error_class"] == daemon.LOCK_REFUSAL_ERROR_CLASS
+    assert "PID 999" in deferred.error["message"]
+
+
+def test_invalid_lock_backoff_env_falls_back_to_the_default(monkeypatch):
+    """Anything that is not a positive, finite number yields the default.
+
+    A typo must not crash the daemon at import; 0 or a negative value would make
+    the cooldown expire instantly and spin the worker against a lock it cannot
+    take; and inf/nan are the sharp edges of float() -- inf parses fine and is
+    > 0, so a bare positivity check would park the job until the daemon
+    restarts, which is exactly the dropped work #2014 is about."""
+    for bad in ("not-a-float", "-5", "0", "", "inf", "-inf", "nan"):
+        monkeypatch.setenv("MEMPALACE_DAEMON_LOCK_BACKOFF_SECONDS", bad)
+        assert daemon._lock_defer_backoff_seconds() == daemon._DEFAULT_LOCK_BACKOFF_SECONDS, bad
+
+    monkeypatch.setenv("MEMPALACE_DAEMON_LOCK_BACKOFF_SECONDS", "0.5")
+    assert daemon._lock_defer_backoff_seconds() == 0.5
+
+    monkeypatch.delenv("MEMPALACE_DAEMON_LOCK_BACKOFF_SECONDS")
+    assert daemon._lock_defer_backoff_seconds() == daemon._DEFAULT_LOCK_BACKOFF_SECONDS
+
+
+def test_defer_returns_job_to_queued_without_spending_an_attempt(tmp_path, monkeypatch):
+    """defer must undo claim_next's increment, so a palace held across many
+    claims cannot walk a job to MAX_ATTEMPTS on work that never landed."""
+    monkeypatch.setenv(daemon.STATE_ROOT_ENV, str(tmp_path / "state"))
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    store = daemon.QueueStore(daemon.queue_path(str(palace)))
+    job = store.enqueue("mine", {"source": "src"})
+
+    claimed = store.claim_next()
+    assert claimed.attempts == 1
+    assert claimed.state == "running"
+
+    deferred = store.defer(job.id)
+    assert deferred.state == "queued"
+    assert deferred.attempts == 0
+    assert deferred.started_at is None
+    # Still claimable -- the work is held, not lost.
+    assert store.claim_next().id == job.id
+
+
+def test_defer_scoped_to_a_stale_claim_is_a_no_op(tmp_path, monkeypatch):
+    """defer is scoped to the claim that was refused. If the row was meanwhile
+    re-queued and re-claimed (recover_running on another daemon start), a late
+    defer keyed to the old claim must not flip the newer claim back to 'queued'
+    -- that would refund its attempt and re-run work whose outcome the newer
+    claimant already owns."""
+    monkeypatch.setenv(daemon.STATE_ROOT_ENV, str(tmp_path / "state"))
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    store = daemon.QueueStore(daemon.queue_path(str(palace)))
+    job = store.enqueue("mine", {"source": "s"})
+
+    # started_at is the claim token this test compares, and adjacent claims can
+    # land inside one clock tick on hosts with a coarse wall clock (Windows
+    # datetime.now() ticks at ~15.6ms before 3.13). Make every stamp unique so
+    # the assertions exercise the guard, not the clock.
+    real_now = daemon._now
+    stamps = iter(range(10_000))
+    monkeypatch.setattr(daemon, "_now", lambda: f"{real_now()}#{next(stamps):04d}")
+
+    stale = store.claim_next()  # the claim that got the refusal
+    deferred = store.defer(job.id, claimed_started_at=stale.started_at)
+    assert deferred.state == "queued"  # scoping to one's own claim still defers
+
+    fresh = store.claim_next()  # the row is claimed again behind the first claimant
+    assert fresh.state == "running"
+
+    late = store.defer(job.id, claimed_started_at=stale.started_at)
+    assert late.state == "running", "a defer scoped to a stale claim must not fire"
+    assert late.attempts == fresh.attempts
+
+
+def test_defer_preserves_attempts_spent_on_earlier_real_executions(tmp_path, monkeypatch):
+    """The refund undoes one claim, not the job's whole history. An execution
+    that crashed left an unknown outcome and must keep costing its attempt --
+    otherwise an alternating crash/refusal pattern would defeat MAX_ATTEMPTS and
+    re-file verbatim content forever."""
+    monkeypatch.setenv(daemon.STATE_ROOT_ENV, str(tmp_path / "state"))
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    store = daemon.QueueStore(daemon.queue_path(str(palace)))
+    job = store.enqueue("diary_write", {"entry": "x"})
+
+    store.claim_next()  # attempt 1: crashes mid-execution (outcome unknown)
+    assert store.recover_running() == 1  # re-queued, attempt NOT refunded
+    assert store.get(job.id).attempts == 1
+
+    store.claim_next()  # attempt 2: refused the lock this time
+    deferred = store.defer(job.id)
+    assert deferred.attempts == 1  # only the refused claim was refunded, not the crash
+
+
+def test_shutdown_while_a_job_is_cooling_leaves_it_queued(tmp_path, monkeypatch):
+    """A job already deferred must survive shutdown as 'queued', not be swept
+    into a terminal state and dropped.
+
+    The worker does not hold a deferred job: it records the cooldown and moves
+    on, so `finally` clears active_job_id and the drain (which joins the worker
+    before reading it) finds nothing in flight to cancel. Pin that, because a
+    shutdown that cancelled a deferred job would be #2014 again."""
+    monkeypatch.setattr(daemon, "LOCK_DEFER_BACKOFF_SECONDS", 5.0)
+
+    def fake_execute(kind, payload):
+        return {
+            "success": False,
+            "error": "palace /p is held by PID 999",
+            "error_class": daemon.LOCK_REFUSAL_ERROR_CLASS,
+            "exit_code": 1,
+        }
+
+    client, thread, palace, holders = _start_server(tmp_path, monkeypatch, fake_execute)
+    try:
+        job = client.submit("diary_write", {"entry": "verbatim"})
+        # Wait for the REFUSAL, not merely for 'queued': a job is queued the moment
+        # it is submitted, so waiting on the state alone would pass this test before
+        # any deferral had happened. The marker only appears once defer ran.
+        deadline = time.monotonic() + 10
+        while time.monotonic() < deadline:
+            if daemon.job_deferred_by_lock(client.get_job(job["id"])):
+                break
+            time.sleep(0.02)
+        assert daemon.job_deferred_by_lock(client.get_job(job["id"])), "never reached a deferral"
+    finally:
+        _stop_server(client, thread, holders)
+
+    store = daemon.QueueStore(daemon.queue_path(str(palace)))
+    final = store.get(job["id"])
+    assert final.state == "queued", "a deferred job must not be cancelled by shutdown"
+    assert store.claim_next().id == job["id"]  # still runnable on the next start
+
+
+def test_defer_does_not_resurrect_a_cancelled_job(tmp_path, monkeypatch):
+    """defer mirrors finish(only_if_running=True): once shutdown cancelled an
+    in-flight job, a late deferral must not queue it for re-execution."""
+    monkeypatch.setenv(daemon.STATE_ROOT_ENV, str(tmp_path / "state"))
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    store = daemon.QueueStore(daemon.queue_path(str(palace)))
+    job = store.enqueue("diary_write", {"entry": "x"})
+    store.claim_next()
+    store.finish(job.id, state="cancelled", result={})
+
+    after = store.defer(job.id)
+    assert after.state == "cancelled"
+    assert store.claim_next() is None
+
+
+def test_claim_next_skips_excluded_jobs_without_binding_a_parameter_each(tmp_path, monkeypatch):
+    """The cooling set must not be spliced into the SQL as one host parameter per
+    job: SQLITE_MAX_VARIABLE_NUMBER defaults to 999 below SQLite 3.32 and this
+    project still supports Python 3.9. Over the cap the query raises into the
+    worker's catch-all, which retries against a set that only ages out on the
+    cooldown, so the queue stalls in cooldown-long fits. It also has to keep
+    picking the right row: the oldest queued job that is not excluded."""
+    monkeypatch.setenv(daemon.STATE_ROOT_ENV, str(tmp_path / "state"))
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    store = daemon.QueueStore(daemon.queue_path(str(palace)))
+
+    first = store.enqueue("mine", {"source": "first"})
+    second = store.enqueue("mine", {"source": "second"})
+
+    # Excluding the oldest hands back the next one, not None.
+    claimed = store.claim_next(exclude={first.id})
+    assert claimed is not None
+    assert claimed.id == second.id
+
+    # Excluding every queued job yields nothing rather than the wrong job.
+    store.defer(second.id)
+    assert store.claim_next(exclude={first.id, second.id}) is None
+
+    if not hasattr(sqlite3.Connection, "setlimit"):  # setlimit is 3.11+
+        pytest.skip("sqlite3.Connection.setlimit needed to pin the parameter cap")
+
+    # Force the pre-3.32 cap of 999 rather than trusting whatever SQLite this
+    # interpreter bundles: on a build with a large cap the old
+    # `id NOT IN (?, ?, ...)` shape passes and this test would guard nothing.
+    real_connect = sqlite3.connect
+
+    def capped_connect(*args, **kwargs):
+        conn = real_connect(*args, **kwargs)
+        conn.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 999)
+        return conn
+
+    monkeypatch.setattr(sqlite3, "connect", capped_connect)
+
+    huge = {f"cooling-{i:07d}" for i in range(5_000)}  # 5x the cap
+    claimed = store.claim_next(exclude=huge)
+    assert claimed is not None
+    assert claimed.id == first.id  # priority DESC, created_at ASC still honoured
+
+    # And priority still wins over age, with an exclusion in play.
+    store.defer(first.id)
+    urgent = store.enqueue("diary_write", {"entry": "x"}, priority=10)
+    assert store.claim_next(exclude=huge).id == urgent.id
+
+
+def test_submit_job_forwards_stop_on_lock_deferral_to_wait(monkeypatch):
+    """submit_job is the only door the CLI and the hooks use, so if it drops the
+    flag on the floor every caller silently goes back to waiting out a holder
+    that may never let go. Nothing else in the suite covers the forwarding."""
+    seen = {}
+
+    class _Client:
+        def submit(self, kind, payload, dedupe_key=None, priority=0):
+            return {"id": "job-1", "state": "queued"}
+
+        def wait(self, job_id, *, timeout=None, stop_on_lock_deferral=False):
+            seen["stop_on_lock_deferral"] = stop_on_lock_deferral
+            return {"id": job_id, "state": "queued"}
+
+    monkeypatch.setattr(daemon, "ensure_client", lambda *a, **kw: _Client())
+
+    daemon.submit_job("mine", {"source": "s"}, palace_path="/p", stop_on_lock_deferral=True)
+    assert seen["stop_on_lock_deferral"] is True
+
+    daemon.submit_job("mine", {"source": "s"}, palace_path="/p")
+    assert seen["stop_on_lock_deferral"] is False  # default stays the old waiting behaviour
+
+
+def test_a_deferred_job_does_not_stall_unrelated_queued_work(tmp_path, monkeypatch):
+    """The worker is the only one, and claim_next hands back the oldest queued
+    job -- which, after a refusal, is the deferred job itself. If the worker
+    waited out the cooldown in line, that one job would hold the queue hostage
+    for as long as the holder lived (the flock is held while the write runs, and
+    a wedged write holds it indefinitely, #2024), and nothing else would run.
+    The cooldown must skip the job, not block the loop."""
+    monkeypatch.setattr(daemon, "LOCK_DEFER_BACKOFF_SECONDS", 30.0)
+    ran = []
+
+    def fake_execute(kind, payload):
+        ran.append(kind)
+        if kind == "mine":  # older, and refused for the whole test
+            return {
+                "success": False,
+                "error": "palace /p is held by PID 999",
+                "error_class": daemon.LOCK_REFUSAL_ERROR_CLASS,
+                "exit_code": 1,
+            }
+        return {"success": True, "exit_code": 0}
+
+    client, thread, palace, holders = _start_server(tmp_path, monkeypatch, fake_execute)
+    try:
+        blocked = client.submit("mine", {"source": "src"})
+        unrelated = client.submit("diary_write", {"entry": "verbatim"})
+
+        # The unrelated job must run despite a 30s cooldown on the older one.
+        done = client.wait(unrelated["id"], timeout=10)
+        assert done["state"] == "succeeded"
+        assert "diary_write" in ran
+
+        # And the blocked job is still held, not lost.
+        parked = client.get_job(blocked["id"])
+        assert parked["state"] == "queued"
+        assert daemon.job_deferred_by_lock(parked)
+    finally:
+        _stop_server(client, thread, holders)
+
+
+def test_a_job_queued_behind_a_deferred_one_gets_its_own_refusal_marker(tmp_path, monkeypatch):
+    """A caller waiting on a job that is merely queued behind a deferred job has
+    nothing to key on: the job was never claimed, so it carries no marker, and
+    stop_on_lock_deferral cannot fire. It would wait out its full timeout and
+    then report a failure that never happened -- #2014 through the back door.
+    Because the cooldown skips the deferred job instead of blocking the worker,
+    the second job is claimed, refused, and marked on its own."""
+    monkeypatch.setattr(daemon, "LOCK_DEFER_BACKOFF_SECONDS", 30.0)
+
+    def fake_execute(kind, payload):
+        return {
+            "success": False,
+            "error": "palace /p is held by PID 999 (mempalace-mcp)",
+            "error_class": daemon.LOCK_REFUSAL_ERROR_CLASS,
+            "exit_code": 1,
+        }
+
+    client, thread, palace, holders = _start_server(tmp_path, monkeypatch, fake_execute)
+    try:
+        first = client.submit("mine", {"source": "a"})
+        deadline = time.monotonic() + 10
+        while time.monotonic() < deadline:
+            if daemon.job_deferred_by_lock(client.get_job(first["id"])):
+                break
+            time.sleep(0.02)
+        assert daemon.job_deferred_by_lock(client.get_job(first["id"]))
+
+        # Submitted while the first job is cooling: it must still be claimed.
+        second = client.submit("mine", {"source": "b"})
+        parked = client.wait(second["id"], timeout=10, stop_on_lock_deferral=True)
+        assert parked["state"] == "queued"
+        assert daemon.job_deferred_by_lock(parked), "the CLI/hook short-circuit cannot fire"
+        assert "PID 999" in parked["error"]["message"]
+    finally:
+        _stop_server(client, thread, holders)
+
+
+def test_failure_that_is_not_a_lease_refusal_stays_terminal(tmp_path, monkeypatch):
+    """Only a lease refusal defers. A genuine failure has an unknown outcome --
+    the daemon may have written before dying -- which is exactly what MAX_ATTEMPTS
+    guards, so it must stay terminal and never be blindly re-run."""
+
+    def fake_execute(kind, payload):
+        return {"success": False, "error": "boom", "error_class": "ValueError", "exit_code": 1}
+
+    client, thread, palace, holders = _start_server(tmp_path, monkeypatch, fake_execute)
+    try:
+        job = client.submit("diary_write", {"entry": "x"})
+        finished = client.wait(job["id"], timeout=10)
+        assert finished["state"] == "failed"
+        assert finished["error"]["message"] == "boom"
+    finally:
+        _stop_server(client, thread, holders)
+
+
 def test_claim_next_does_not_reclaim_running_job(tmp_path, monkeypatch):
     """The conditional UPDATE (WHERE state='queued') means a job already
     flipped to 'running' cannot be claimed again — the cross-process
@@ -641,6 +1182,42 @@ def test_run_mine_invalid_mode_returns_structured_error(tmp_path):
     assert out["exit_code"] == 2
 
 
+def test_run_mine_source_adapter_dispatches_through_adapter_runner(tmp_path, monkeypatch):
+    """Daemon mine jobs preserve the CLI's explicit source-adapter dispatch."""
+    from mempalace import cli, service
+
+    received = {}
+
+    def fake_mine_source_adapter(**kwargs):
+        received.update(kwargs)
+        return 3
+
+    monkeypatch.setattr(cli, "mine_source_adapter", fake_mine_source_adapter)
+    palace = tmp_path / "palace"
+    out = service.run_mine(
+        {
+            "palace_path": str(palace),
+            "source": "/source",
+            "source_adapter": "fixture",
+            "dry_run": True,
+        }
+    )
+
+    assert received == {
+        "source_name": "fixture",
+        "source_path": "/source",
+        "palace_path": str(palace.resolve()),
+        "dry_run": True,
+    }
+    assert out == {
+        "success": True,
+        "kind": "mine",
+        "mode": "source",
+        "dry_run": True,
+        "exit_code": 0,
+    }
+
+
 def test_run_mcp_tool_rejects_non_dict_arguments():
     from mempalace import service
 
@@ -755,6 +1332,53 @@ def test_run_sync_structured_errors_on_sync_failures(tmp_path, monkeypatch):
     assert "sync failed" in r["error"]
 
 
+def test_run_mine_lease_refusal_returns_the_lock_error_class(tmp_path, monkeypatch):
+    """run_mine is the sole source of the refusal marker for kind 'mine' -- the
+    primary #2014 vector, submitted by the hooks and the CLI. The worker's
+    defer-vs-dead-letter gate keys on this error_class, so if a refactor drops
+    it (say, a broad handler reordered above MineAlreadyRunning), the daemon
+    silently goes back to dead-lettering refused mines while every worker test
+    stays green: they all inject the marker by hand."""
+    import mempalace.miner as miner_module
+    from mempalace import service
+    from mempalace.palace import MineAlreadyRunning
+
+    palace = tmp_path / "palace"
+    palace.mkdir()
+
+    def _locked(**kw):
+        raise MineAlreadyRunning("palace is held by PID 999")
+
+    monkeypatch.setattr(miner_module, "mine", _locked)
+    r = service.run_mine(
+        {"palace_path": str(palace), "source": str(tmp_path / "src"), "mode": "projects"}
+    )
+    assert r["success"] is False
+    assert r["error_class"] == daemon.LOCK_REFUSAL_ERROR_CLASS
+    assert r["exit_code"] == 1
+
+
+def test_safe_defer_swallows_store_errors_and_leaves_the_job_running(tmp_path, monkeypatch):
+    """_safe_defer mirrors _safe_finish: a queue-DB hiccup during defer must not
+    kill the worker. The job stays 'running' for recover_running to re-queue on
+    the next start. Dropping the swallow would let the error reach the worker's
+    outer except, which dead-letters the lock-refused job -- the #2014 outcome
+    this branch exists to remove."""
+    monkeypatch.setenv(daemon.STATE_ROOT_ENV, str(tmp_path / "state"))
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    runtime = daemon.DaemonRuntime(str(palace))
+    job = runtime.store.enqueue("mine", {"source": "s"})
+    runtime.store.claim_next()
+
+    def _boom(*args, **kwargs):
+        raise sqlite3.OperationalError("database is locked")
+
+    monkeypatch.setattr(runtime.store, "defer", _boom)
+    runtime._safe_defer(job.id, error={"error_class": daemon.LOCK_REFUSAL_ERROR_CLASS})
+    assert runtime.store.get(job.id).state == "running"
+
+
 # --- post-merge review follow-ups (Copilot review on #1826) ---
 
 

tests/test_date_window.py

@@ -0,0 +1,100 @@
+"""Unit tests for mempalace.date_window — shared since/before parsing (#463).
+
+The same helpers back the ``list_drawers`` filter (#1128) via aliases in
+``mcp_server``; behavioral coverage for that surface lives in
+``test_mcp_server.py``. These tests pin the module contract directly.
+"""
+
+from datetime import datetime
+
+import pytest
+
+from mempalace.date_window import filed_at_in_window, parse_date_bound, parse_window
+
+
+class TestParseDateBound:
+    def test_none_and_blank_mean_no_filter(self):
+        assert parse_date_bound(None) is None
+        assert parse_date_bound("") is None
+        assert parse_date_bound("   ") is None
+
+    def test_date_only(self):
+        assert parse_date_bound("2026-04-01") == datetime(2026, 4, 1)
+
+    def test_naive_datetime(self):
+        assert parse_date_bound("2026-04-01T09:30:00") == datetime(2026, 4, 1, 9, 30)
+
+    def test_fractional_seconds(self):
+        assert parse_date_bound("2026-04-01T09:30:00.250000") == datetime(
+            2026, 4, 1, 9, 30, 0, 250000
+        )
+
+    def test_zulu_datetime_parses_on_39_floor(self):
+        assert parse_date_bound("2026-04-01T09:30:00Z") == datetime(2026, 4, 1, 9, 30)
+
+    def test_zulu_date_only(self):
+        # Regression: appending "+00:00" instead of stripping Z broke this
+        # exact shape on Python 3.9/3.10 (caught by review on #1891).
+        assert parse_date_bound("2026-04-01Z") == datetime(2026, 4, 1)
+
+    def test_offset_dropped_wall_clock(self):
+        assert parse_date_bound("2026-04-01T09:30:00+05:00") == datetime(2026, 4, 1, 9, 30)
+
+    def test_non_string_rejected(self):
+        with pytest.raises(ValueError, match="since"):
+            parse_date_bound(20260401, "since")
+
+    def test_garbage_rejected_names_field(self):
+        with pytest.raises(ValueError, match="before"):
+            parse_date_bound("next tuesday", "before")
+
+
+class TestParseWindow:
+    def test_both_none(self):
+        assert parse_window() == (None, None)
+
+    def test_valid_window(self):
+        since_dt, before_dt = parse_window("2026-04-01", "2026-04-10")
+        assert since_dt == datetime(2026, 4, 1)
+        assert before_dt == datetime(2026, 4, 10)
+
+    def test_inverted_window_rejected(self):
+        with pytest.raises(ValueError, match="must be earlier than"):
+            parse_window("2026-04-10", "2026-04-01")
+
+    def test_equal_bounds_rejected(self):
+        with pytest.raises(ValueError, match="must be earlier than"):
+            parse_window("2026-04-01", "2026-04-01")
+
+    def test_invalid_since_names_field(self):
+        with pytest.raises(ValueError, match="since"):
+            parse_window("nope", None)
+
+
+class TestFiledAtInWindow:
+    def test_no_bounds_accepts_anything(self):
+        assert filed_at_in_window("2026-01-01T00:00:00", None, None)
+
+    def test_since_inclusive(self):
+        since = datetime(2026, 1, 2)
+        assert filed_at_in_window("2026-01-02T00:00:00", since, None)
+        assert not filed_at_in_window("2026-01-01T23:59:59", since, None)
+
+    def test_before_exclusive(self):
+        before = datetime(2026, 1, 4)
+        assert filed_at_in_window("2026-01-03T23:59:59", None, before)
+        assert not filed_at_in_window("2026-01-04T00:00:00", None, before)
+
+    def test_missing_filed_at_excluded_when_bound_active(self):
+        assert not filed_at_in_window(None, datetime(2026, 1, 1), None)
+        assert not filed_at_in_window("", datetime(2026, 1, 1), None)
+
+    def test_unparseable_filed_at_excluded(self):
+        assert not filed_at_in_window("not-a-date", datetime(2026, 1, 1), None)
+
+    def test_aware_filed_at_compared_wall_clock(self):
+        # diary_ingest stamps aware UTC ("+00:00"); the offset is dropped and
+        # the wall-clock fields are compared.
+        since = datetime(2026, 1, 5)
+        assert filed_at_in_window("2026-01-05T00:00:00+00:00", since, None)
+        assert not filed_at_in_window("2026-01-04T23:59:59+00:00", since, None)

tests/test_dedup.py

@@ -9,6 +9,22 @@ from mempalace import dedup
 # ── get_source_groups ─────────────────────────────────────────────────
 
 
+def test_get_source_groups_aborts_on_hnsw_divergence():
+    """count() on a diverged HNSW segment can hard-crash the process
+    (#1222); get_source_groups must not call it when hnsw_capacity_status
+    reports divergence for the caller-supplied palace_path (#92). Omitting
+    palace_path (as every existing test above does) skips the check
+    entirely -- this only fires when a real caller passes one."""
+    col = MagicMock()
+    with patch(
+        "mempalace.backends.chroma.hnsw_capacity_status",
+        return_value={"diverged": True, "message": "test divergence"},
+    ):
+        groups = dedup.get_source_groups(col, palace_path="/fake/palace")
+    assert groups == {}
+    col.count.assert_not_called()
+
+
 def test_get_source_groups_basic():
     col = MagicMock()
     col.count.return_value = 5
@@ -243,6 +259,26 @@ def test_dedup_palace_dry_run(mock_get_collection, mock_groups, mock_dedup_group
     mock_dedup_group.assert_called_once()
 
 
+@patch("mempalace.dedup.get_source_groups")
+@patch("mempalace.dedup.get_collection")
+def test_dedup_palace_aborts_on_hnsw_divergence(mock_get_collection, mock_groups, tmp_path):
+    """dedup_palace's OWN count() print (a few lines before it calls
+    get_source_groups) is a separate call site from #92 -- count() on a
+    diverged segment can hard-crash the process, so this print must never
+    be reached either when hnsw_capacity_status reports divergence."""
+    mock_col = MagicMock()
+    mock_col.count.side_effect = AssertionError("count() must not be called when diverged")
+    _install_mock_collection(mock_get_collection, mock_col)
+
+    with patch(
+        "mempalace.backends.chroma.hnsw_capacity_status",
+        return_value={"diverged": True, "message": "test divergence"},
+    ):
+        dedup.dedup_palace(palace_path=str(tmp_path), dry_run=True)
+
+    mock_groups.assert_not_called()
+
+
 @patch("mempalace.dedup.dedup_source_group")
 @patch("mempalace.dedup.get_source_groups")
 @patch("mempalace.dedup.get_collection")
@@ -253,7 +289,9 @@ def test_dedup_palace_with_wing(mock_get_collection, mock_groups, mock_dedup_gro
 
     mock_groups.return_value = {}
     dedup.dedup_palace(palace_path=str(tmp_path), wing="test_wing", dry_run=True)
-    mock_groups.assert_called_once_with(mock_col, 5, None, wing="test_wing")
+    mock_groups.assert_called_once_with(
+        mock_col, 5, None, wing="test_wing", palace_path=str(tmp_path)
+    )
 
 
 @patch("mempalace.dedup.dedup_source_group")

tests/test_embedder_identity.py

@@ -10,6 +10,7 @@ only the configured model name (cheap), and persistence is exercised with
 
 import os
 import warnings
+from unittest.mock import MagicMock, patch
 
 import pytest
 
@@ -28,6 +29,31 @@ from mempalace.backends.base import (
 # ---------------------------------------------------------------------------
 
 
+def test_enforce_embedder_identity_skips_count_on_hnsw_divergence():
+    """count() on a diverged HNSW segment can hard-crash the process
+    (#1222) -- a try/except cannot save it, since a native crash takes the
+    whole process down regardless. _enforce_embedder_identity runs on
+    EVERY get_collection() call (the universal chokepoint every tool
+    passes through) and must never reach count() when hnsw_capacity_status
+    reports divergence (#89)."""
+    from mempalace.palace import _enforce_embedder_identity
+
+    collection = MagicMock()
+    collection.effective_embedder_identity.side_effect = Exception("not supported")
+    collection.get_stored_embedder_identity.return_value = None
+
+    with (
+        patch("mempalace.embedding.current_model_name", return_value="minilm"),
+        patch(
+            "mempalace.backends.chroma.hnsw_capacity_status",
+            return_value={"diverged": True, "message": "test divergence"},
+        ),
+    ):
+        _enforce_embedder_identity(collection, "/fake/palace", "mempalace_drawers", create=True)
+
+    collection.count.assert_not_called()
+
+
 def test_unknown_when_nothing_stored():
     assert check_embedder_identity(None, EmbedderIdentity("minilm", 384)) == "unknown"
 
@@ -221,6 +247,24 @@ def test_enforcement_brand_new_records_current_model(tmp_path, monkeypatch, clea
     assert got is not None and got.model_name == "minilm"
 
 
+def test_clear_validated_embedder_identity_scoped_to_palace(
+    tmp_path, monkeypatch, clear_identity_cache
+):
+    monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact")
+    monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm")
+    from mempalace import palace as P
+
+    keep = (str(tmp_path / "other"), "mempalace_drawers", "minilm")
+    drop = (str(tmp_path), "mempalace_drawers", "minilm")
+    P._VALIDATED_IDENTITY.add(keep)
+    P._VALIDATED_IDENTITY.add(drop)
+
+    P.clear_validated_embedder_identity(str(tmp_path))
+
+    assert keep in P._VALIDATED_IDENTITY
+    assert drop not in P._VALIDATED_IDENTITY
+
+
 def test_enforcement_legacy_with_data_warns(tmp_path, monkeypatch, clear_identity_cache):
     monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact")
     monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm")

tests/test_embedding.py

@@ -192,3 +192,129 @@ def test_describe_device_uses_resolved_effective_device(monkeypatch):
     )
 
     assert embedding.describe_device("auto") == "cuda"
+
+
+# ---------------------------------------------------------------------------
+# embedding -> backend handoff
+#
+# These live in this module on purpose: conftest's autouse
+# ``_stable_embedding_function_for_tests`` replaces
+# ``embedding_wrapper._embed_texts`` outright for every other test module, so a
+# defect in the real function is invisible there. ``test_embedding`` is in
+# ``_REAL_EMBEDDING_TEST_MODULES`` and runs unstubbed.
+# ---------------------------------------------------------------------------
+
+
+class _NumpyEmbeddingFunction:
+    """Mimics the real EF contract: a list of float32 ``np.ndarray`` rows.
+
+    Both shipped embedders (ChromaDB's ONNX MiniLM and EmbeddingGemma) return
+    numpy arrays, not Python lists — that difference is the whole point here.
+    """
+
+    def __init__(self, dim: int = 8):
+        self.dim = dim
+
+    def __call__(self, input):
+        import numpy as np
+
+        return [np.full(self.dim, 0.1, dtype=np.float32) for _ in list(input or [])]
+
+
+def test_embed_texts_returns_plain_python_floats(monkeypatch):
+    """``list(ndarray)`` yields ``np.float32`` scalars, which ChromaDB rejects.
+
+    Regression for the default (chroma) backend failing every write with
+    "Expected embeddings to be a list of floats or ints, a list of lists, a
+    numpy array, or a list of numpy arrays" once chroma began declaring
+    ``requires_explicit_embeddings`` and routing through EmbeddingCollection.
+    """
+    from mempalace.backends import embedding_wrapper as ew
+
+    monkeypatch.setattr(
+        embedding, "get_embedding_function", lambda *_, **__: _NumpyEmbeddingFunction()
+    )
+
+    vectors = ew._embed_texts(["hello", "world"])
+
+    assert len(vectors) == 2
+    for row in vectors:
+        assert isinstance(row, list)
+        assert all(type(x) is float for x in row), f"got {type(row[0])}, not builtin float"
+
+
+def test_embedding_collection_upsert_accepts_numpy_backed_vectors(tmp_path, monkeypatch):
+    """End-to-end: a real Chroma collection must accept what the wrapper emits.
+
+    Asserting on float types alone would not catch a future ChromaDB tightening
+    its accepted shapes, so drive an actual upsert + read-back.
+    """
+    from mempalace.backends.chroma import ChromaBackend
+    from mempalace.backends.base import PalaceRef
+    from mempalace.backends.embedding_wrapper import EmbeddingCollection
+
+    monkeypatch.setattr(
+        embedding, "get_embedding_function", lambda *_, **__: _NumpyEmbeddingFunction()
+    )
+
+    backend = ChromaBackend()
+    palace = tmp_path / "palace"
+    ref = PalaceRef(id=str(palace), local_path=str(palace))
+    try:
+        inner = backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True)
+        col = EmbeddingCollection(inner)
+
+        col.upsert(documents=["verbatim drawer text"], ids=["drawer-1"], metadatas=[{"wing": "w"}])
+
+        assert col.get(ids=["drawer-1"]).documents == ["verbatim drawer text"]
+    finally:
+        backend.close()
+
+
+def test_embed_texts_handles_plain_sequence_embedders(monkeypatch):
+    """The ``float(x)`` fallback must convert plain sequences, not just ndarrays.
+
+    ``_embed_texts`` branches on ``hasattr(v, "tolist")``. The numpy side is
+    covered above, but the fallback exists for embedders that hand back plain
+    sequences (custom/BYO EFs, and rows that arrive as tuples), and nothing
+    exercised it — so a regression there would surface only in the field, on a
+    non-default embedder, as the same ChromaDB ``ValueError``.
+
+    Yields ``Decimal`` rather than ``float`` so the assertion proves a real
+    conversion happened rather than passing values through unchanged.
+    """
+    from decimal import Decimal
+
+    from mempalace.backends import embedding_wrapper as ew
+
+    class _PlainSequenceEmbeddingFunction:
+        def __call__(self, input):
+            return [(Decimal("0.5"), Decimal("0.25")) for _ in list(input or [])]
+
+    monkeypatch.setattr(
+        embedding, "get_embedding_function", lambda *_, **__: _PlainSequenceEmbeddingFunction()
+    )
+
+    vectors = ew._embed_texts(["a", "b"])
+
+    assert vectors == [[0.5, 0.25], [0.5, 0.25]]
+    for row in vectors:
+        assert isinstance(row, list)
+        assert all(type(x) is float for x in row), f"got {type(row[0])}, not builtin float"
+
+
+def test_embed_texts_short_circuits_on_empty_input(monkeypatch):
+    """Empty input must return ``[]`` without constructing an embedding function.
+
+    Callers pass empty batches (a drawer set fully filtered by dedup), and
+    loading the EF is the expensive part — on the ONNX default it spins up a
+    native session. Guards the early return so it cannot be refactored away.
+    """
+    from mempalace.backends import embedding_wrapper as ew
+
+    def _explode(*_, **__):
+        raise AssertionError("get_embedding_function must not be called for an empty batch")
+
+    monkeypatch.setattr(embedding, "get_embedding_function", _explode)
+
+    assert ew._embed_texts([]) == []

tests/test_embedding_api.py

@@ -0,0 +1,352 @@
+"""Tests for the OpenAI-compatible embedding API backend (issue #1559).
+
+Covers ``OpenAICompatEmbeddingFunction``, the ``embedding_model ==
+"openai-compat"`` selection branch in ``get_embedding_function``, and the
+``MempalaceConfig`` properties that are the single source of truth for the
+endpoint settings. No server required — ``urllib.request.urlopen`` is mocked.
+"""
+
+import json
+
+import pytest
+
+import mempalace.embedding as embedding
+from mempalace.config import MempalaceConfig
+
+
+@pytest.fixture(autouse=True)
+def isolate_embedding_cache(monkeypatch):
+    monkeypatch.setattr(embedding, "_EF_CACHE", {})
+
+
+# ── Fake HTTP layer ───────────────────────────────────────────────────────
+
+
+class _FakeResp:
+    def __init__(self, body: bytes):
+        self._body = body
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+    def read(self):
+        return self._body
+
+
+def _fake_urlopen(*, dim=4, one_hot=False, shuffle=False, captured=None):
+    """Return a urlopen stand-in that echoes one embedding per input text.
+
+    ``one_hot`` makes each vector a unit vector at its own index (so order is
+    observable); ``shuffle`` reverses the returned rows to prove the EF
+    re-sorts by ``index``; ``captured`` collects the Request objects.
+    """
+
+    def fake(req, timeout=None):
+        if captured is not None:
+            captured.append(req)
+        body = json.loads(req.data.decode())
+        n = len(body["input"])
+        rows = []
+        for i in range(n):
+            if one_hot:
+                vec = [0.0] * max(dim, n)
+                vec[i] = 1.0
+            else:
+                vec = [float(i + 1)] * dim
+            rows.append({"index": i, "embedding": vec})
+        if shuffle:
+            rows = list(reversed(rows))
+        return _FakeResp(json.dumps({"data": rows, "model": body["model"]}).encode())
+
+    return fake
+
+
+# ── OpenAICompatEmbeddingFunction ─────────────────────────────────────────
+
+
+def test_resolve_url_variants():
+    ef = embedding.OpenAICompatEmbeddingFunction
+    assert ef("http://h:8420", "m")._url == "http://h:8420/v1/embeddings"
+    assert ef("http://h:8420/", "m")._url == "http://h:8420/v1/embeddings"
+    assert ef("http://h:8420/v1", "m")._url == "http://h:8420/v1/embeddings"
+    assert ef("http://h:8420/v1/embeddings", "m")._url == "http://h:8420/v1/embeddings"
+
+
+def test_name_encodes_model():
+    ef = embedding.OpenAICompatEmbeddingFunction
+    assert ef("http://h", "small").name() == "openai_compat_emb_small"
+    # HF-style ids with slashes are flattened to a safe identifier
+    assert ef("http://h", "Qwen/Qwen3-Embedding-0.6B").name() == (
+        "openai_compat_emb_Qwen_Qwen3-Embedding-0.6B"
+    )
+
+
+def test_embeds_and_l2_normalizes(monkeypatch):
+    monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen(dim=4))
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h:8420", "small")
+    out = ef(["a", "b"])
+    assert len(out) == 2
+    assert len(out[0]) == 4
+    for vec in out:
+        assert abs(sum(x * x for x in vec) ** 0.5 - 1.0) < 1e-6
+
+
+def test_sorts_response_by_index(monkeypatch):
+    monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen(one_hot=True, shuffle=True))
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    out = ef(["x", "y", "z"])
+    # Server returned rows reversed; the EF must realign by index so out[i]
+    # is the one-hot vector for position i.
+    for i, vec in enumerate(out):
+        assert max(range(len(vec)), key=lambda j: vec[j]) == i
+
+
+def test_sends_bearer_header_when_key_set(monkeypatch):
+    captured = []
+    monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen(captured=captured))
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m", api_key="sk-secret")
+    ef(["a"])
+    assert captured[0].get_header("Authorization") == "Bearer sk-secret"
+
+
+def test_no_auth_header_without_key(monkeypatch):
+    captured = []
+    monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen(captured=captured))
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    ef(["a"])
+    assert captured[0].get_header("Authorization") is None
+
+
+def test_batches_large_input(monkeypatch):
+    captured = []
+    monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen(captured=captured))
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    out = ef([f"t{i}" for i in range(130)])  # > _EF_API_BATCH (64)
+    assert len(out) == 130
+    assert len(captured) == 3  # 64 + 64 + 2
+
+
+def test_embed_query_delegates_to_call(monkeypatch):
+    monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen(dim=4))
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    assert ef.embed_query(["q"]) == ef(["q"])
+
+
+def test_raises_on_count_mismatch(monkeypatch):
+    def short(req, timeout=None):
+        return _FakeResp(json.dumps({"data": [{"index": 0, "embedding": [1.0]}]}).encode())
+
+    monkeypatch.setattr("urllib.request.urlopen", short)
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    with pytest.raises(RuntimeError, match="embeddings for"):
+        ef(["a", "b"])
+
+
+def test_raises_on_transport_error(monkeypatch):
+    from urllib.error import URLError
+
+    def boom(req, timeout=None):
+        raise URLError("connection refused")
+
+    monkeypatch.setattr("urllib.request.urlopen", boom)
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    with pytest.raises(RuntimeError, match="failed"):
+        ef(["a"])
+
+
+# ── get_embedding_function selection branch ───────────────────────────────
+
+
+class _FakeCfg:
+    def __init__(self, url=None, model=None, key=None, embedding_model="openai-compat"):
+        self.embedding_api_url = url
+        self.embedding_api_model = model
+        self.embedding_api_key = key
+        self.embedding_model = embedding_model
+
+
+def test_get_embedding_function_selects_openai_compat(monkeypatch):
+    monkeypatch.setattr(
+        "mempalace.config.MempalaceConfig", lambda *a, **k: _FakeCfg("http://h:8420", "small")
+    )
+    monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen(dim=4))
+    ef = embedding.get_embedding_function(device="cpu", model="openai-compat")
+    assert isinstance(ef, embedding.OpenAICompatEmbeddingFunction)
+    assert ef.name() == "openai_compat_emb_small"
+    assert len(ef(["hi"])[0]) == 4
+
+
+def test_openai_compat_requires_url(monkeypatch):
+    monkeypatch.setattr("mempalace.config.MempalaceConfig", lambda *a, **k: _FakeCfg(None, "small"))
+    with pytest.raises(ValueError, match="requires an endpoint"):
+        embedding.get_embedding_function(device="cpu", model="openai-compat")
+
+
+def test_openai_compat_requires_model(monkeypatch):
+    monkeypatch.setattr(
+        "mempalace.config.MempalaceConfig", lambda *a, **k: _FakeCfg("http://h", None)
+    )
+    with pytest.raises(ValueError, match="requires a model"):
+        embedding.get_embedding_function(device="cpu", model="openai-compat")
+
+
+# ── MempalaceConfig endpoint settings (single source of truth) ────────────
+
+
+def test_config_api_url_from_file(tmp_path, monkeypatch):
+    monkeypatch.delenv("MEMPALACE_EMBEDDING_API_URL", raising=False)
+    (tmp_path / "config.json").write_text(json.dumps({"embedding_api_url": "http://host:8420"}))
+    assert MempalaceConfig(config_dir=str(tmp_path)).embedding_api_url == "http://host:8420"
+
+
+def test_config_api_env_overrides_file(tmp_path, monkeypatch):
+    (tmp_path / "config.json").write_text(json.dumps({"embedding_api_url": "http://from-config"}))
+    monkeypatch.setenv("MEMPALACE_EMBEDDING_API_URL", "  http://from-env  ")
+    assert MempalaceConfig(config_dir=str(tmp_path)).embedding_api_url == "http://from-env"
+
+
+def test_config_api_unset_is_none(tmp_path, monkeypatch):
+    for var in ("MEMPALACE_EMBEDDING_API_URL", "MEMPALACE_EMBEDDING_API_MODEL"):
+        monkeypatch.delenv(var, raising=False)
+    cfg = MempalaceConfig(config_dir=str(tmp_path))
+    assert cfg.embedding_api_url is None
+    assert cfg.embedding_api_model is None
+
+
+def test_config_api_blank_value_is_none(tmp_path, monkeypatch):
+    monkeypatch.delenv("MEMPALACE_EMBEDDING_API_MODEL", raising=False)
+    (tmp_path / "config.json").write_text(json.dumps({"embedding_api_model": "   "}))
+    assert MempalaceConfig(config_dir=str(tmp_path)).embedding_api_model is None
+
+
+def test_config_api_model_and_key_preserve_case(tmp_path, monkeypatch):
+    for var in ("MEMPALACE_EMBEDDING_API_MODEL", "MEMPALACE_EMBEDDING_API_KEY"):
+        monkeypatch.delenv(var, raising=False)
+    (tmp_path / "config.json").write_text(
+        json.dumps({"embedding_api_model": "Qwen3-Embedding", "embedding_api_key": "AbC-XyZ"})
+    )
+    cfg = MempalaceConfig(config_dir=str(tmp_path))
+    assert cfg.embedding_api_model == "Qwen3-Embedding"
+    assert cfg.embedding_api_key == "AbC-XyZ"
+
+
+def test_config_api_blank_env_falls_through_to_file(tmp_path, monkeypatch):
+    (tmp_path / "config.json").write_text(json.dumps({"embedding_api_url": "http://from-config"}))
+    monkeypatch.setenv("MEMPALACE_EMBEDDING_API_URL", "   ")  # blank must not mask the file value
+    assert MempalaceConfig(config_dir=str(tmp_path)).embedding_api_url == "http://from-config"
+
+
+# ── request shape + malformed-response hardening (review findings) ────────
+
+
+def test_request_targets_v1_embeddings_with_expected_body(monkeypatch):
+    captured = []
+    monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen(captured=captured))
+    embedding.OpenAICompatEmbeddingFunction("http://h:8420", "small")(["a", "b"])
+    req = captured[0]
+    assert req.full_url == "http://h:8420/v1/embeddings"
+    assert req.get_header("Content-type") == "application/json"
+    # Custom User-Agent so Cloudflare-fronted endpoints don't 403 us (#1570).
+    assert req.get_header("User-agent", "").startswith("mempalace/")
+    assert json.loads(req.data) == {
+        "model": "small",
+        "input": ["a", "b"],
+        "encoding_format": "float",
+    }
+
+
+def test_embedding_api_error_is_runtimeerror():
+    assert issubclass(embedding.EmbeddingAPIError, RuntimeError)
+
+
+def test_raises_on_missing_embedding_key(monkeypatch):
+    def bad(req, timeout=None):
+        return _FakeResp(json.dumps({"data": [{"index": 0}]}).encode())  # no "embedding"
+
+    monkeypatch.setattr("urllib.request.urlopen", bad)
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    with pytest.raises(embedding.EmbeddingAPIError, match="malformed embeddings"):
+        ef(["a"])
+
+
+def test_raises_on_non_contiguous_indices(monkeypatch):
+    # Count matches (2 rows for 2 inputs) but the indices are absolute, not
+    # 0..n-1 — sort+positional-zip would silently misalign vectors with texts.
+    def bad(req, timeout=None):
+        rows = [{"index": 64, "embedding": [1.0]}, {"index": 65, "embedding": [2.0]}]
+        return _FakeResp(json.dumps({"data": rows}).encode())
+
+    monkeypatch.setattr("urllib.request.urlopen", bad)
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    with pytest.raises(embedding.EmbeddingAPIError, match="non-contiguous"):
+        ef(["a", "b"])
+
+
+def test_raises_on_http_protocol_exception(monkeypatch):
+    # BadStatusLine / IncompleteRead — common with local/overloaded servers.
+    from http.client import HTTPException
+
+    def boom(req, timeout=None):
+        raise HTTPException("incomplete read")
+
+    monkeypatch.setattr("urllib.request.urlopen", boom)
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    with pytest.raises(embedding.EmbeddingAPIError, match="failed"):
+        ef(["a"])
+
+
+def test_raises_on_value_error_from_urlopen(monkeypatch):
+    # urlopen raises ValueError on an invalid/missing URL scheme.
+    def boom(req, timeout=None):
+        raise ValueError("unknown url type")
+
+    monkeypatch.setattr("urllib.request.urlopen", boom)
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    with pytest.raises(embedding.EmbeddingAPIError, match="failed"):
+        ef(["a"])
+
+
+def test_raises_on_non_object_response(monkeypatch):
+    def bad(req, timeout=None):
+        return _FakeResp(json.dumps([1, 2, 3]).encode())  # JSON list, not an object
+
+    monkeypatch.setattr("urllib.request.urlopen", bad)
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    with pytest.raises(embedding.EmbeddingAPIError, match="non-object response"):
+        ef(["a"])
+
+
+def test_raises_and_surfaces_server_error_body(monkeypatch):
+    # HTTP 200 with an OpenAI-style error envelope (no "data") — surface it.
+    def err(req, timeout=None):
+        return _FakeResp(json.dumps({"error": {"message": "model not found"}}).encode())
+
+    monkeypatch.setattr("urllib.request.urlopen", err)
+    ef = embedding.OpenAICompatEmbeddingFunction("http://h", "m")
+    with pytest.raises(embedding.EmbeddingAPIError, match="model not found"):
+        ef(["a"])
+
+
+def test_get_embedding_function_caches_instance(monkeypatch):
+    monkeypatch.setattr(
+        "mempalace.config.MempalaceConfig", lambda *a, **k: _FakeCfg("http://h:8420", "small", "k")
+    )
+    a = embedding.get_embedding_function(device="cpu", model="openai-compat")
+    b = embedding.get_embedding_function(device="cpu", model="openai-compat")
+    assert a is b
+
+
+def test_describe_device_reports_openai_compat_endpoint(monkeypatch):
+    monkeypatch.setattr(
+        "mempalace.config.MempalaceConfig",
+        lambda *a, **k: _FakeCfg("http://10.0.0.1:8420", "small"),
+    )
+    assert embedding.describe_device() == "openai-compat (http://10.0.0.1:8420)"
+
+
+def test_describe_device_openai_compat_without_url(monkeypatch):
+    monkeypatch.setattr("mempalace.config.MempalaceConfig", lambda *a, **k: _FakeCfg(None, "small"))
+    assert embedding.describe_device() == "openai-compat"

tests/test_embeddinggemma.py

@@ -70,6 +70,12 @@ class _FakeTokenizer:
         self._truncation_enabled = True
         self._truncation_max = max_length
 
+    def get_vocab_size(self, with_added_tokens=True):
+        return 262145 if with_added_tokens else 262144
+
+    def token_to_id(self, token):
+        return 3 if token == "<unk>" else None
+
     def encode_batch(self, texts):
         class _Enc:
             def __init__(self, n):
@@ -184,7 +190,9 @@ def test_call_chunks_large_batches(patched_lazy_load, monkeypatch):
     monkeypatch.setattr(_FakeTokenizer, "encode_batch", recording_encode_batch)
     ef = embedding.EmbeddinggemmaONNX()
     n = embedding._EMBEDDINGGEMMA_BATCH_SIZE * 2 + 6
-    docs = [f"doc {i}" for i in range(n)]
+    # Descending sizes, so size-sorted order is the reverse of arrival order
+    # and the assertion below cannot pass on both.
+    docs = [f"{'x' * (n - i)} doc {i}" for i in range(n)]
     out = ef(docs)
 
     assert batch_sizes == [
@@ -192,9 +200,11 @@ def test_call_chunks_large_batches(patched_lazy_load, monkeypatch):
         embedding._EMBEDDINGGEMMA_BATCH_SIZE,
         6,
     ], f"expected bounded sub-batches, got {batch_sizes}"
-    # Sub-batches must cover the input in order; combined with the per-chunk
-    # extend in __call__ this pins output row order to input order.
-    assert captured_texts == [embedding._EMBEDDINGGEMMA_PREFIX + d for d in docs]
+    # Sub-batches cover the input in size-sorted order; the scatter in
+    # __call__ puts every row back at its own input index afterwards.
+    ordered = sorted(docs, key=lambda d: len(d.encode("utf-8")))
+    assert ordered != docs, "fixture must not already be in size order"
+    assert captured_texts == [embedding._EMBEDDINGGEMMA_PREFIX + d for d in ordered]
     arr = np.asarray(out)
     assert arr.shape == (n, 384), f"chunked outputs must concatenate to (n, 384), got {arr.shape}"
     assert np.allclose(np.linalg.norm(arr, axis=1), 1.0, atol=1e-5)
@@ -245,6 +255,200 @@ def test_custom_batch_size_is_honored(patched_lazy_load, monkeypatch):
     assert len(out) == 24
 
 
+# Bound before any test can monkeypatch the method, so the width helper
+# below measures with the real fake and never re-enters a recorder.
+_UNPATCHED_ENCODE_BATCH = _FakeTokenizer.encode_batch
+
+
+def _record_batches(monkeypatch, sink):
+    """Capture the texts handed to each encode_batch call, in order."""
+
+    def recording_encode_batch(self, texts):
+        sink.append(list(texts))
+        return _UNPATCHED_ENCODE_BATCH(self, texts)
+
+    monkeypatch.setattr(_FakeTokenizer, "encode_batch", recording_encode_batch)
+
+
+def _fake_padded_width(batches):
+    """Total padded token slots the fake tokenizer produces for `batches`.
+
+    Measured by encoding, not by re-deriving the fake's padding rule, so the
+    two cannot drift apart and quietly turn the assertion into a tautology.
+    """
+    tokenizer = _FakeTokenizer()
+    return sum(sum(len(e.ids) for e in _UNPATCHED_ENCODE_BATCH(tokenizer, b)) for b in batches)
+
+
+def test_call_groups_documents_by_size(patched_lazy_load, monkeypatch):
+    """Similar-size documents must share a sub-batch.
+
+    encode_batch pads every row to the longest sequence in the sub-batch and
+    attention cost per layer is batch x heads x length^2, so interleaving one
+    long document with short ones makes the short ones pay the long length
+    (#2104). Grouping by size is what keeps that bill proportional to the
+    text actually being embedded.
+    """
+    batches = []
+    _record_batches(monkeypatch, batches)
+    # One long document per sub-batch's worth of short ones: the pathological
+    # arrival order a verbatim transcript sweep produces.
+    long_doc = " ".join(["word"] * 200)
+    docs = [long_doc if i % _B == 0 else f"short {i}" for i in range(4 * _B)]
+
+    ef = embedding.EmbeddinggemmaONNX()
+    out = ef(docs)
+    assert len(out) == len(docs)
+
+    for texts in batches:
+        sizes = [len(t.encode("utf-8")) for t in texts]
+        assert sizes == sorted(sizes), f"sub-batch is not size-grouped: {sizes}"
+
+    prefixed = [embedding._EMBEDDINGGEMMA_PREFIX + d for d in docs]
+    arrival_order = [prefixed[s : s + _B] for s in range(0, len(prefixed), _B)]
+    assert _fake_padded_width(batches) < _fake_padded_width(arrival_order), (
+        "size grouping must lower the total padded width"
+    )
+
+
+def test_call_groups_by_utf8_size_not_character_count(patched_lazy_load, monkeypatch):
+    """The key is UTF-8 bytes, because this model is multilingual.
+
+    A CJK document is ~3 bytes per character and roughly a token per
+    character, so ordering by character count would file it next to Latin
+    documents several times cheaper to embed.
+    """
+    batches = []
+    _record_batches(monkeypatch, batches)
+    # Same character count, very different byte count (and token count).
+    docs = ["a" * 90] * _B + ["中" * 90] * _B
+    ef = embedding.EmbeddinggemmaONNX()
+    ef(list(reversed(docs)))
+
+    assert len(batches) == 2, f"expected two sub-batches, got {len(batches)}"
+    sizes = [[len(t.encode("utf-8")) for t in b] for b in batches]
+    assert max(sizes[0]) < min(sizes[1]), (
+        f"CJK documents must not share a sub-batch with Latin ones: {sizes}"
+    )
+
+
+def test_call_size_grouping_is_stable(patched_lazy_load, monkeypatch):
+    """Equal-size documents keep their arrival order.
+
+    An unstable sort would make the sub-batch split depend on nothing the
+    caller can see, so two identical inputs could take different code paths.
+    """
+    batches = []
+    _record_batches(monkeypatch, batches)
+    docs = [f"doc{i:03d}" for i in range(_B + 8)]  # identical size, distinct text
+    ef = embedding.EmbeddinggemmaONNX()
+    ef(docs)
+    captured = [t for b in batches for t in b]
+    assert captured == [embedding._EMBEDDINGGEMMA_PREFIX + d for d in docs]
+
+
+def test_call_keeps_arrival_order_within_a_single_sub_batch(patched_lazy_load, monkeypatch):
+    """An input that fits one sub-batch is not reordered.
+
+    Every row pads to the same width either way, so the sort would buy
+    nothing and only add keys to compute on the search hot path.
+    """
+    batches = []
+    _record_batches(monkeypatch, batches)
+    docs = [f"{'x' * (_B - i)} doc {i}" for i in range(_B)]  # descending size
+    ef = embedding.EmbeddinggemmaONNX()
+    ef(docs)
+    assert batches == [[embedding._EMBEDDINGGEMMA_PREFIX + d for d in docs]]
+
+
+class _MarkerTokenizer(_FakeTokenizer):
+    """Tokenizer whose first token id carries that document's own length.
+
+    ``_FakeTokenizer`` emits all-zero ids padded to one width, so a fake
+    session cannot tell its rows apart, which is exactly what an
+    order-restoration test has to observe.
+    """
+
+    def encode_batch(self, texts):
+        widths = [len(t) for t in texts]
+        padded = max(widths)
+
+        class _Enc:
+            def __init__(self, marker):
+                self.ids = [marker] + [0] * (padded - 1)
+                self.attention_mask = [1] * marker + [0] * (padded - marker)
+
+        return [_Enc(w) for w in widths]
+
+
+class _MarkerSession:
+    """Emit a vector whose first two dims encode the row's marker id.
+
+    Both dims scale by the same L2 norm, so ``row[0] / row[1]`` survives
+    normalization and identifies which document produced the row.
+    """
+
+    _WIDTH = 2 * embedding._EMBEDDINGGEMMA_DIM
+
+    def run(self, _output_names, feed):
+        ids = feed["input_ids"]
+        batch, length = ids.shape
+        sent = np.zeros((batch, self._WIDTH), dtype=np.float64)
+        sent[:, 0] = ids[:, 0]
+        sent[:, 1] = 1.0
+        return [np.zeros((batch, length, self._WIDTH), dtype=np.float64), sent]
+
+
+def _marker_ef(patched_lazy_load, session=None):
+    """An EF wired to the marker fakes, with the real lazy load short-circuited."""
+    # patched_lazy_load is taken so a future tightening of _lazy_load's
+    # early-return cannot turn these tests into a 300 MB model download.
+    ef = embedding.EmbeddinggemmaONNX()
+    ef._tokenizer = _MarkerTokenizer()
+    ef._session = session if session is not None else _MarkerSession()
+    ef._output_idx = 1
+    ef._np = np
+    return ef
+
+
+def test_call_returns_rows_at_their_input_index(patched_lazy_load):
+    """Row i of the result must be the embedding of document i.
+
+    Sub-batching by size reorders the work; ChromaDB zips the returned
+    vectors against the ids positionally, so grouping without the matching
+    scatter would file every drawer under another drawer's vector. That
+    half-applied state is what this pins: arrival order trivially satisfies
+    it, so it is the grouping tests that cover the other direction.
+    """
+    ef = _marker_ef(patched_lazy_load)
+    # Strictly descending sizes, so grouping reverses arrival order and an
+    # unscattered result would be visibly wrong.
+    docs = ["x" * n for n in range(200, 200 - 3 * _B, -1)]
+    out = ef(docs)
+
+    assert len(out) == len(docs)
+    assert all(row is not None for row in out), "every index must be filled"
+    markers = [round(row[0] / row[1]) for row in out]
+    assert markers == [len(embedding._EMBEDDINGGEMMA_PREFIX + d) for d in docs]
+
+
+def test_call_rejects_a_short_row_count_from_the_session(patched_lazy_load):
+    """A session returning fewer rows than documents must fail loudly.
+
+    Scattering by index would otherwise leave a None in the result and the
+    caller would only trip over it much later, converting to an array.
+    """
+
+    class _ShortSession(_MarkerSession):
+        def run(self, output_names, feed):
+            last_hidden, sent = super().run(output_names, feed)
+            return [last_hidden, sent[:-1]]
+
+    ef = _marker_ef(patched_lazy_load, session=_ShortSession())
+    with pytest.raises(RuntimeError, match="rows for a"):
+        ef(["x" * n for n in range(200, 200 - 2 * _B, -1)])
+
+
 def test_batch_size_below_one_is_rejected():
     """A zero or negative batch size would loop forever or embed nothing."""
     with pytest.raises(ValueError, match="batch_size"):
@@ -397,3 +601,59 @@ def test_config_embedding_model_default_is_minilm(monkeypatch):
 
     monkeypatch.delenv("MEMPALACE_EMBEDDING_MODEL", raising=False)
     assert MempalaceConfig().embedding_model == "minilm"
+
+
+def test_out_of_range_added_token_is_remapped_to_unknown(
+    patched_lazy_load,
+    monkeypatch,
+    caplog,
+):
+    class EncodingWithAddedToken:
+        ids = [2, 262144, 1]
+        attention_mask = [1, 1, 1]
+
+    def encode_with_added_token(_self, texts):
+        return [EncodingWithAddedToken() for _ in texts]
+
+    monkeypatch.setattr(
+        _FakeTokenizer,
+        "encode_batch",
+        encode_with_added_token,
+    )
+
+    captured = {}
+    fake_session_class = _make_fake_session()
+
+    class BoundsCheckingSession(fake_session_class):
+        def run(self, output_names, feed):
+            captured["input_ids"] = feed["input_ids"].copy()
+
+            assert np.all(feed["input_ids"] >= 0)
+            assert np.all(feed["input_ids"] < 262144)
+
+            return super().run(
+                output_names,
+                feed,
+            )
+
+    import onnxruntime
+
+    monkeypatch.setattr(
+        onnxruntime,
+        "InferenceSession",
+        lambda *_args, **_kwargs: BoundsCheckingSession(),
+    )
+
+    caplog.set_level(
+        "WARNING",
+        logger=embedding.__name__,
+    )
+
+    embedding_function = embedding.EmbeddinggemmaONNX()
+
+    result = embedding_function(["literal <image_soft_token> in source"])
+
+    assert captured["input_ids"].tolist() == [[2, 3, 1]]
+    assert np.asarray(result).shape == (1, 384)
+    assert "remapping to <unk>" in caplog.text
+    assert patched_lazy_load["hf_hub_download"] == 3