Merge pull request #2020 from MemPalace/develop

Release v3.6.0 — promote develop to main

.claude-plugin/README.md

@@ -1,6 +1,6 @@
 # MemPalace Claude Code Plugin
 
-A Claude Code plugin that gives your AI a persistent memory system. Mine projects and conversations into a searchable palace backed by ChromaDB, with 35 MCP tools, auto-save hooks, and 5 guided skills.
+A Claude Code plugin that gives your AI a persistent memory system. Mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and 5 guided skills.
 
 ## Prerequisites
 
@@ -51,7 +51,7 @@ Set the `MEMPAL_DIR` environment variable to a directory path to automatically r
 
 ## MCP Server
 
-The plugin automatically configures a local MCP server with 34 tools for storing, searching, and managing memories. No manual MCP setup is required -- `/mempalace:init` handles everything.
+The plugin automatically configures a local MCP server with 36 tools for storing, searching, and managing memories. No manual MCP setup is required -- `/mempalace:init` handles everything.
 
 ## Full Documentation
 

.claude-plugin/marketplace.json

@@ -8,8 +8,8 @@
     {
       "name": "mempalace",
       "source": "./.claude-plugin",
-      "description": "AI memory system — mine projects and conversations into a searchable palace. 35 MCP tools, auto-save hooks, guided setup.",
-      "version": "3.5.0",
+      "description": "AI memory system — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, guided setup.",
+      "version": "3.6.0",
       "author": {
         "name": "milla-jovovich"
       }

.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "mempalace",
-  "version": "3.5.0",
-  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 35 MCP tools, auto-save hooks, and guided setup.",
+  "version": "3.6.0",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.",
   "author": {
     "name": "milla-jovovich"
   },

.codex-plugin/README.md

@@ -1,6 +1,6 @@
 # MemPalace - Codex CLI Plugin
 
-Give your AI a persistent memory -- mine projects and conversations into a searchable palace backed by ChromaDB, with 35 MCP tools, auto-save hooks, and guided skills.
+Give your AI a persistent memory -- mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and guided skills.
 
 ## Prerequisites
 

.codex-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "mempalace",
-  "version": "3.5.0",
-  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 35 MCP tools, auto-save hooks, and guided setup.",
+  "version": "3.6.0",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.",
   "author": {
     "name": "milla-jovovich"
   },
@@ -27,7 +27,7 @@
   "interface": {
     "displayName": "MemPalace",
     "shortDescription": "AI memory system for Codex",
-    "longDescription": "Give your AI a persistent memory — mine projects and conversations into a searchable palace backed by ChromaDB, with 35 MCP tools, auto-save hooks, and guided skills.",
+    "longDescription": "Give your AI a persistent memory — mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and guided skills.",
     "developerName": "milla-jovovich",
     "category": "Coding",
     "capabilities": [

.cursor-plugin/README.md

@@ -1,6 +1,6 @@
 # MemPalace Cursor Plugin
 
-A Cursor IDE plugin that gives your agent a persistent memory system. Auto-registers the `mempalace-mcp` server (35 MCP tools), ships 5 slash commands, two model-invocable skills (setup/mining/search and a recall protocol), and an optional recall rule.
+A Cursor IDE plugin that gives your agent a persistent memory system. Auto-registers the `mempalace-mcp` server (36 MCP tools), ships 5 slash commands, two model-invocable skills (setup/mining/search and a recall protocol), and an optional recall rule.
 
 > Hooks (auto-save + session-start memory recall) are shipped separately under `hooks/cursor/` so the plugin is safe to install in any Cursor workspace without touching the agent loop. See [Hooks](#hooks-optional) below.
 
@@ -87,7 +87,7 @@ This plugin ships `mcp.json` at the plugin root, so Cursor auto-loads the `mempa
 }
 ```
 
-All 34 MemPalace MCP tools (`mempalace_search`, `mempalace_add_drawer`, `mempalace_diary_write`, `mempalace_check_duplicate`, `mempalace_diary_read`, …) become available to the agent immediately. No manual `~/.cursor/mcp.json` edit required.
+All 36 MemPalace MCP tools (`mempalace_search`, `mempalace_add_drawer`, `mempalace_diary_write`, `mempalace_check_duplicate`, `mempalace_diary_read`, …) become available to the agent immediately. No manual `~/.cursor/mcp.json` edit required.
 
 If the server doesn't appear, confirm `mempalace-mcp` is on the user `$PATH`:
 

.cursor-plugin/marketplace.json

@@ -8,7 +8,7 @@
     {
       "name": "mempalace",
       "source": ".",
-      "description": "AI memory system — mine projects and conversations into a searchable palace. 35 MCP tools, slash commands, and a guided skill for Cursor.",
+      "description": "AI memory system — mine projects and conversations into a searchable palace. 36 MCP tools, slash commands, and a guided skill for Cursor.",
       "author": {
         "name": "milla-jovovich"
       }

.cursor-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "mempalace",
-  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 35 MCP tools, slash commands, and a guided skill for Cursor.",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, slash commands, and a guided skill for Cursor.",
   "author": {
     "name": "milla-jovovich"
   },

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
       matrix:
         python-version: ["3.9", "3.11", "3.13"]
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
@@ -24,7 +24,7 @@ jobs:
   test-windows:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v6
         with:
           python-version: "3.13"
@@ -41,7 +41,7 @@ jobs:
   test-macos:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v6
         with:
           python-version: "3.13"
@@ -51,7 +51,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v6
         with:
           python-version: "3.11"

.github/workflows/deploy-docs.yml

@@ -23,7 +23,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           fetch-depth: 0
 

.github/workflows/docker-publish.yml

@@ -22,14 +22,14 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       # Needed for the emulated linux/arm64 build on real pushes.
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       # Only authenticate + push for in-repo events. Fork PRs lack the
       # packages:write token, so they build (to validate the Dockerfile) but
@@ -79,10 +79,10 @@ jobs:
   build-gpu:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Build GPU image (validation only — not published)
         uses: docker/build-push-action@v7

.github/workflows/publish.yml

@@ -73,7 +73,7 @@ jobs:
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
           echo "Resolved tag: $tag"
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           # Fully-qualified refs/tags/ so an unqualified name can't resolve to a
           # same-named *branch* instead of the tag (checkout prefers branches).

.github/workflows/version-guard.yml

@@ -16,7 +16,7 @@ jobs:
   check-versions:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Extract versions from all sources
         id: versions

CHANGELOG.md

@@ -10,6 +10,68 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ---
 
+## [3.6.0] — 2026-07-14
+
+### Features
+
+- **Turnkey secure remote / team server.** `mempalace serve` exposes the full MCP surface over HTTP with secure defaults: non-loopback binds require a bearer token, generated tokens are stored with restrictive permissions and passed through the child environment rather than argv, native TLS 1.2+ is supported, and `--read-only` both hides and refuses mutating tools. Docker Compose, systemd, and environment templates are included for deployment. (#1877, #1897, #1900)
+
+- **Milvus storage backend.** The new opt-in `milvus` backend supports embedded Milvus Lite, self-hosted Milvus, and Zilliz Cloud through `pymilvus`, with COSINE vector search, native BM25 lexical search for new collections, namespace isolation, target-mismatch protection, and an optional `milvus` dependency extra. (#1899)
+
+- **Atomic knowledge-graph fact replacement.** `supersede()` / `mempalace_kg_supersede` closes an open fact and opens its successor at one shared instant, so model, employer, address, and other single-valued facts can change without a hand-written invalidate/add race. Temporal queries now use half-open intervals at timestamp precision, returning only the successor at the transition boundary while preserving whole-day semantics for date-only facts. (#1913)
+
+- **Conversation chronology is preserved.** Conversation drawers now retain transcript time as `authored_at` alongside ingest time, search results expose it, and exact hybrid-score ties prefer the more recently authored drawer. An idempotent, dry-run-first backfill script adds the metadata to existing palaces without re-embedding. `mempalace_list_drawers` also accepts inclusive `since` and exclusive `before` filing-time bounds. (#1890, #1891)
+
+- **Mined sessions populate the associative graph.** A precision-biased, no-LLM structural extractor records code symbols, URLs, paths, and qualified identifiers as drawer entities; conversation mining then derives hallways from them. The new `mempalace hallways` CLI command exposes the graph. (#1894, #1895)
+
+- **Per-project mining exclusions.** `exclude_patterns` in `mempalace.yaml` filters pre-scanned project files without changing the corrected `--limit` semantics. (#1213, #1953)
+
+- **LaTeX project coverage.** `.tex` and `.bib` files are now treated as readable prose sources. (#1901)
+
+### Performance
+
+- **MCP initialization no longer waits for palace-wide integrity work.** stdio answers the JSON-RPC `initialize` request immediately while startup preflight runs on a background thread; a tool arriving mid-probe joins the same verdict instead of launching duplicate work. The startup SQLite probe is also skipped above a configurable size limit (512 MiB by default), while repair preflights remain strict. (#1911, #1987)
+
+- **Qdrant metadata counts use server-side facets.** Taxonomy, overview, and status paths can count metadata remotely instead of fetching and tallying every record; the embedding wrapper now forwards the facet and bulk-metadata capabilities correctly. (#1868, #1898)
+
+- **pgvector metadata-only fetches omit document payloads**, reducing transfer and decoding work on enumeration paths. (#1892)
+
+### Bug Fixes
+
+- **Chroma recovery distinguishes derived-index damage from data loss.** Valid all-layer-0 HNSW segments with an empty `link_lists.bin` are no longer quarantined repeatedly; isolated FTS5 inverted-index corruption is rebuilt from intact content during repair and after mining; embedded NUL bytes are sanitized before reaching ChromaDB. (#1716, #1872, #1878, #1927, #1928)
+
+- **SQLite recovery is bounded, atomic, and honest.** Integrity checks wait up to 15 seconds for transient writer contention instead of reporting a healthy palace as corrupt. In-place recovery archives use atomic `os.rename` rather than `shutil.move`'s destructive copy/delete fallback. `repair --mode from-sqlite` now rebuilds FTS5, vacuums, and requires a clean final `PRAGMA quick_check`; cleanup failures return non-zero with recovery details instead of printing a false success banner. (#1945, #2015, #2017)
+
+- **CLI search checks HNSW divergence before opening ChromaDB.** A diverged palace routes directly to the existing SQLite BM25 fallback, avoiding embedder or collection initialization against damaged native index state while leaving healthy Chroma and non-Chroma backends on their normal vector path. (#2016)
+
+- **Writer leases recover instead of stranding servers read-only.** A server refused while a peer owns the palace now retries on each mutating call and promotes itself after the peer exits. Status remains read-only and cannot acquire the writer lease, the in-process re-entrant holder set is released even across asynchronous interruption, and `checkpoint` / `delete_by_source` are correctly classified as mutations for both read-only serving and peer-writer protection. (#1923, #1930, #1934, #1960, #1970, #1971)
+
+- **Status reports only checks that actually ran.** Non-Chroma backends now mark the Chroma SQLite integrity check as not applicable rather than claiming an unperformed success. (#1931, #1946)
+
+- **Conversation transcripts are treated as mutable.** Appended or rewritten sessions are purged and re-filed based on modification time, preventing later turns from being silently skipped. Claude Code `tool-results/` sidecars are excluded from conversation scans so raw machine dumps cannot flood the embedding space. (#1957, #2010)
+
+- **Explicit palace selection now scopes derived graph state.** Project, conversation, and format mining write hallways and tunnels beside the selected `--palace` instead of leaking them into the ambient default palace. (#2018)
+
+- **Remote and local server hardening.** Non-loopback HTTP now requires a token unless an explicit insecure override is supplied; optional-provider probes cannot reuse an ambient external key before the user selects that provider; hook transcript paths and file-copy/repair paths receive stricter local validation and no-follow handling. Importing the MCP server no longer clobbers the host application's root logger, HTTP writes can re-enter the process-wide palace lock safely, and routine client disconnects no longer emit server tracebacks. (#1859, #1860, #1864, #1885, #2003, #2004)
+
+- **Mining and configuration correctness.** Oversized files produce a visible stderr warning, `~` in configured palace paths expands consistently, wing slugs handle special characters, and Windows background daemon / synchronous hook mines use `CREATE_NO_WINDOW`. (#923, #1852, #1857, #1863, #1865)
+
+- **`mempalace init` handles non-ASCII `.gitignore` files on Windows.** The project-file ignore guard now reads and appends UTF-8 explicitly instead of relying on locale defaults such as GBK. (#1648)
+
+- **L1 wake-up surfaces the latest moments.** Drawers with equal importance are now ordered by `filed_at` recency rather than insertion order, so startup context prefers recent memories instead of the oldest ones. (#1630)
+
+- **Backend detection requires a SQLite magic header** before classifying a target as Chroma or `sqlite_exact`, preventing unrelated files from being mistaken for a palace. (#1893, #1896)
+
+### Documentation
+
+- Added a storage-backend configuration reference, a remote/team-server deployment guide, `authored_at` migration guidance, and refreshed the OpenClaw integration for the full 36-tool MCP surface. (#1719, #1877, #1890, #1904, #1905)
+
+### Internal
+
+- Expanded WAL crash-safety, idempotence, and redaction-path coverage (#1869); updated GitHub Actions and Ruff; repaired `uv.lock` drift so it again matches the Ruff 0.15.20 pin and includes Python 3.9 dependency markers introduced by the Milvus resolution.
+
+---
+
 ## [3.5.0] — 2026-06-22
 
 ### Features
@@ -568,7 +630,9 @@ Initial public release.
 
 ---
 
-[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.4.1...HEAD
+[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.6.0...HEAD
+[3.6.0]: https://github.com/MemPalace/mempalace/compare/v3.5.0...v3.6.0
+[3.5.0]: https://github.com/MemPalace/mempalace/compare/v3.4.1...v3.5.0
 [3.4.1]: https://github.com/MemPalace/mempalace/compare/v3.4.0...v3.4.1
 [3.4.0]: https://github.com/MemPalace/mempalace/compare/v3.3.6...v3.4.0
 [3.3.6]: https://github.com/MemPalace/mempalace/compare/v3.3.5...v3.3.6

README.md

@@ -108,39 +108,23 @@ CUDA-accelerated embeddings, build the GPU variant with
 
 ## Storage backends
 
-ChromaDB is the default. For the pluggable-backend preview, MemPalace also
-ships `sqlite_exact` for local exact-vector correctness checks, and two opt-in
-external service backends — `qdrant` (REST) and `pgvector` (Postgres). The two
-external backends exercise the storage contract on different substrates (a
-REST/dict store and a SQL/JSONB store), so it is not accidentally shaped around
-one vendor.
+ChromaDB is the default and needs no configuration. MemPalace also ships a
+pluggable backend contract, exercised across deliberately different substrates
+so the contract is never accidentally shaped around one vendor. Every
+non-default backend is opt-in.
 
-```bash
-# local no-service backend
-mempalace mine ~/projects/myapp --backend sqlite_exact
+| Backend | Mode | Install | Namespaces | Lexical | Configure with |
+| ------- | ---- | ------- | :--------: | :-----: | -------------- |
+| `chroma` _(default)_ | Local (embedded) | bundled | – | ✓ | – |
+| `sqlite_exact` | Local (exact) | bundled | – | ✓ | – |
+| `milvus` | Local (Lite) · Server opt-in | `mempalace[milvus]` | ✓ | ✓ | `MEMPALACE_MILVUS_URI` |
+| `qdrant` | Server (REST) | bundled | ✓ | ✓ | `MEMPALACE_QDRANT_URL` |
+| `pgvector` | Server (Postgres) | `mempalace[pgvector]` | ✓ | ✓ | `MEMPALACE_PGVECTOR_DSN` |
 
-# Qdrant backend, defaulting to http://localhost:6333
-MEMPALACE_QDRANT_URL=http://localhost:6333 \
-  mempalace mine ~/projects/myapp --backend qdrant
-
-# Postgres + pgvector backend, defaulting to postgresql://localhost:5432/mempalace
-#   needs the optional driver: pip install mempalace[pgvector]
-#   and the `vector` extension available on the server
-MEMPALACE_PGVECTOR_DSN=postgresql://localhost:5432/mempalace \
-  mempalace mine ~/projects/myapp --backend pgvector
-```
-
-Qdrant can also be configured with `MEMPALACE_QDRANT_API_KEY`,
-`MEMPALACE_QDRANT_NAMESPACE`, and `MEMPALACE_QDRANT_TIMEOUT`; pgvector with
-`MEMPALACE_PGVECTOR_NAMESPACE`. Both external backends isolate tenants by
-namespace (advertised via the `supports_namespace_isolation` capability) and
-write a local marker (`qdrant_backend.json` / `pgvector_backend.json`) to guard
-against silently opening a palace against the wrong server.
-
-When `MEMPALACE_QDRANT_URL` or `MEMPALACE_PGVECTOR_DSN` points anywhere other
-than your own local or trusted self-hosted service, MemPalace will send and
-store verbatim drawer text and metadata there. That is an explicit opt-in
-backend choice, never the default.
+Select with `--backend <name>`, `MEMPALACE_BACKEND=<name>`, or
+`"backend": "<name>"` in `config.json`. See
+[Storage backends](/guide/configuration#storage-backends) for connection
+variables, namespace behavior, and deployment notes.
 
 ## Quickstart
 
@@ -225,7 +209,7 @@ Usage and tool reference:
 
 ## MCP server
 
-35 MCP tools cover palace reads/writes, knowledge-graph operations,
+36 MCP tools cover palace reads/writes, knowledge-graph operations,
 cross-wing navigation, drawer management, and agent diaries. Installation
 and the full tool list:
 [mempalaceofficial.com/reference/mcp-tools](https://mempalaceofficial.com/reference/mcp-tools.html).
@@ -285,7 +269,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
 MIT — see [LICENSE](LICENSE).
 
 <!-- Link Definitions -->
-[version-shield]: https://img.shields.io/badge/version-3.5.0-4dc9f6?style=flat-square&labelColor=0a0e14
+[version-shield]: https://img.shields.io/badge/version-3.6.0-4dc9f6?style=flat-square&labelColor=0a0e14
 [release-link]: https://github.com/MemPalace/mempalace/releases
 [python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8
 [python-link]: https://www.python.org/

deploy/docker-compose.server.yml

@@ -0,0 +1,71 @@
+# MemPalace remote team server — MCP over HTTP, backed by a central Qdrant.
+#
+# One command stands up a shared memory service a whole team's AI clients
+# connect to. Embeddings are still produced locally inside the mempalace
+# container; only your own Qdrant ever receives the vectors and text.
+#
+#   1. cp deploy/server.env.example deploy/.env  &&  edit deploy/.env
+#      (at minimum set MEMPALACE_MCP_HTTP_TOKEN to a long random secret)
+#   2. docker compose -f deploy/docker-compose.server.yml --env-file deploy/.env up -d
+#   3. connect a client (see the Remote / Team Server guide):
+#      claude mcp add --transport http mempalace http://YOUR_HOST:8765/mcp \
+#        --header "Authorization: Bearer $MEMPALACE_MCP_HTTP_TOKEN"
+#
+# SECURITY: this exposes plaintext HTTP on :8765. For anything beyond a trusted
+# private network, put a TLS-terminating reverse proxy in front (nginx/Caddy/
+# Traefik) and only expose the proxy. The bearer token is mandatory for the
+# network-exposed (0.0.0.0) bind.
+
+services:
+  qdrant:
+    image: qdrant/qdrant:latest
+    restart: unless-stopped
+    volumes:
+      - qdrant-storage:/qdrant/storage
+    # Not published to the host: only the mempalace service reaches it over the
+    # internal compose network. Uncomment to inspect Qdrant directly.
+    # ports:
+    #   - "6333:6333"
+
+  mempalace:
+    image: ghcr.io/mempalace/mempalace:latest
+    restart: unless-stopped
+    depends_on:
+      - qdrant
+    command:
+      - serve
+      - --host
+      - "0.0.0.0"
+      - --port
+      - "8765"
+      - --backend
+      - qdrant
+      # Uncomment to expose recall without write access to most clients:
+      # - --read-only
+    environment:
+      # Required for the network-exposed bind. Set in deploy/.env.
+      MEMPALACE_MCP_HTTP_TOKEN: ${MEMPALACE_MCP_HTTP_TOKEN:?set MEMPALACE_MCP_HTTP_TOKEN in deploy/.env}
+      MEMPALACE_QDRANT_URL: http://qdrant:6333
+      MEMPALACE_QDRANT_API_KEY: ${MEMPALACE_QDRANT_API_KEY:-}
+      # Set to cuda/dml/coreml on an accelerated host (see Dockerfile.gpu).
+      MEMPALACE_EMBEDDING_DEVICE: ${MEMPALACE_EMBEDDING_DEVICE:-auto}
+    ports:
+      - "8765:8765"
+    volumes:
+      - mempalace-data:/data
+    healthcheck:
+      # The image has no curl; use Python (always present). /healthz needs no auth.
+      # If you enable TLS on the server itself, switch this to https + ssl context.
+      test:
+        - CMD
+        - python
+        - -c
+        - "import urllib.request,sys; sys.exit(0) if urllib.request.urlopen('http://127.0.0.1:8765/healthz').read().strip()==b'ok' else sys.exit(1)"
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 40s
+
+volumes:
+  qdrant-storage:
+  mempalace-data:

deploy/mempalace-server.service

@@ -0,0 +1,48 @@
+# MemPalace remote MCP server — systemd unit template.
+#
+# Install:
+#   sudo useradd --system --home /var/lib/mempalace --shell /usr/sbin/nologin mempalace
+#   sudo install -d -o mempalace -g mempalace -m 750 /var/lib/mempalace /etc/mempalace
+#   sudo cp deploy/server.env.example /etc/mempalace/server.env   # then edit + chmod 600
+#   sudo install -m 600 -o mempalace -g mempalace /etc/mempalace/server.env /etc/mempalace/server.env
+#   # install mempalace into a venv on PATH, or adjust ExecStart to its absolute path
+#   sudo cp deploy/mempalace-server.service /etc/systemd/system/
+#   sudo systemctl daemon-reload && sudo systemctl enable --now mempalace-server
+#
+# This binds 0.0.0.0:8765 and requires MEMPALACE_MCP_HTTP_TOKEN (set in the
+# EnvironmentFile). Front it with a TLS-terminating reverse proxy, or set
+# MEMPALACE_MCP_TLS_CERT / _KEY in the EnvironmentFile for native TLS.
+
+[Unit]
+Description=MemPalace remote MCP server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=exec
+User=mempalace
+Group=mempalace
+EnvironmentFile=/etc/mempalace/server.env
+ExecStart=mempalace serve --host 0.0.0.0 --port 8765
+Restart=on-failure
+RestartSec=2
+
+# --- Hardening ---------------------------------------------------------------
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+RestrictNamespaces=true
+LockPersonality=true
+MemoryDenyWriteExecute=false
+# The palace and any local state live here; everything else is read-only.
+ReadWritePaths=/var/lib/mempalace
+StateDirectory=mempalace
+
+[Install]
+WantedBy=multi-user.target

deploy/server.env.example

@@ -0,0 +1,28 @@
+# MemPalace remote server environment.
+# Copy to deploy/.env (compose) or /etc/mempalace/server.env (systemd) and edit.
+# Keep this file readable only by the service account: chmod 600.
+
+# --- Required for a network-exposed (0.0.0.0) bind ------------------------------
+# Clients send: Authorization: Bearer <this value>. Generate a strong secret:
+#   openssl rand -hex 32
+MEMPALACE_MCP_HTTP_TOKEN=
+
+# --- Storage backend ------------------------------------------------------------
+# The team server should use a networked backend so the palace is shared.
+MEMPALACE_BACKEND=qdrant
+MEMPALACE_QDRANT_URL=http://qdrant:6333
+# MEMPALACE_QDRANT_API_KEY=
+
+# --- Embedding ------------------------------------------------------------------
+# auto | cpu | cuda | dml | coreml. Use cuda on a GPU host (needs the GPU image).
+MEMPALACE_EMBEDDING_DEVICE=auto
+
+# --- Optional: native TLS (otherwise terminate TLS at a reverse proxy) ----------
+# Point these at a PEM cert/key the service account can read. When set, serve
+# speaks HTTPS directly and clients connect to https://...
+# MEMPALACE_MCP_TLS_CERT=/etc/mempalace/tls/cert.pem
+# MEMPALACE_MCP_TLS_KEY=/etc/mempalace/tls/key.pem
+
+# --- Palace location (systemd / bare-metal) -------------------------------------
+# In Docker the palace lives on the mempalace-data volume by default.
+# MEMPALACE_PALACE_PATH=/var/lib/mempalace/palace

docs/authored-at.md

@@ -0,0 +1,45 @@
+# Authored date (`authored_at`)
+
+Conversation transcripts carry a per-line ISO-8601 `timestamp` (both Claude Code and
+Codex JSONL). The miner records the most recent one per file as the drawer's
+**`authored_at`** — when the content was actually written.
+
+This is distinct from the ingest date:
+
+| Field | Meaning |
+|-------|---------|
+| `filed_at` / result `created_at` | When the drawer was **mined** (written to the palace). A bulk re-mine collapses these to a single instant. |
+| `authored_at` | When the underlying content was **written**, recovered from the transcript timestamps. Survives re-mining. |
+
+`authored_at` is surfaced in search results (and shown in the CLI `search` output), and is
+used as a deterministic tie-break in hybrid ranking: candidates with identical scores order
+with the more recently authored drawer first. Drawers without per-line timestamps (e.g.
+markdown) fall back to `filed_at`.
+
+## Backfilling existing memory
+
+New mines populate `authored_at` automatically. Drawers mined before this feature only have
+`filed_at`. Re-mining does **not** fix them — the scanner skips files already mined at the
+current `NORMALIZE_VERSION`. Two options:
+
+1. **In-place backfill (recommended — no re-embedding).** `scripts/backfill_authored_at.py`
+   reads each convos drawer's source transcript and updates only the `authored_at` metadata.
+   Idempotent and safe to re-run; embeddings are untouched.
+
+   ```bash
+   python scripts/backfill_authored_at.py \
+       --palace ~/.mempalace/palace \
+       --sessions ~/.claude --sessions ~/.codex          # dry run
+   python scripts/backfill_authored_at.py \
+       --palace ~/.mempalace/palace \
+       --sessions ~/.claude --sessions ~/.codex --apply  # write
+   ```
+
+   For the Docker MCP image, mount the volume and session dirs read-only — see the header of
+   `scripts/backfill_authored_at.py` for the exact `docker run` invocation.
+
+   > Back up first: `tar czf palace-backup.tgz -C <palace-dir> .` (or snapshot the
+   > `mempalace-data` volume).
+
+2. **Drop and recreate.** Delete the affected drawers and re-mine the transcripts; the fresh
+   mine stamps `authored_at`. Simpler, but re-embeds everything.

docs/write-routing-policy.md

@@ -0,0 +1,119 @@
+# Write routing policy
+
+This document defines the shared policy used by the staged Tier 3 daemon
+rollout tracked in #1963.
+
+This foundation PR does not change existing hook or CLI routing. It provides
+one tested policy model that later hook and CLI PRs can consume without
+inventing different fallback rules.
+
+## Policies
+
+`direct`
+
+    Always execute through the existing direct local path.
+
+`prefer`
+
+    Use an available daemon. A caller that is allowed to start the daemon may
+    do so. Otherwise, fall back to the direct path.
+
+`require`
+
+    Use an available daemon. A caller that is allowed to start the daemon may
+    do so. If neither is possible, block the operation. Never fall back to a
+    direct ChromaDB writer.
+
+## Concrete routing outcomes
+
+The shared decision function returns one of:
+
+- `direct`
+- `daemon`
+- `blocked`
+
+It also reports whether the caller should auto-start the daemon and why the
+route was selected.
+
+Hooks generally pass `daemon_can_start=False` because hook execution has a
+tight latency budget.
+
+Interactive CLI commands can pass `daemon_can_start=True`.
+
+## Configuration
+
+Global environment policy:
+
+    MEMPALACE_WRITE_ROUTING=direct|prefer|require
+
+Hook-specific environment policy:
+
+    MEMPALACE_HOOK_WRITE_ROUTING=direct|prefer|require
+
+CLI-specific environment policy:
+
+    MEMPALACE_CLI_WRITE_ROUTING=direct|prefer|require
+
+Configuration-file shape:
+
+    {
+      "write_routing": {
+        "default": "direct",
+        "hooks": "prefer",
+        "cli": "require"
+      }
+    }
+
+## Precedence
+
+For hooks:
+
+1. `MEMPALACE_HOOK_WRITE_ROUTING`
+2. `MEMPALACE_WRITE_ROUTING`
+3. legacy `MEMPALACE_HOOKS_DAEMON`
+4. `write_routing.hooks`
+5. `write_routing.default`
+6. legacy `hooks.daemon`
+7. `direct`
+
+For CLI writes:
+
+1. `MEMPALACE_CLI_WRITE_ROUTING`
+2. `MEMPALACE_WRITE_ROUTING`
+3. `write_routing.cli`
+4. `write_routing.default`
+5. `direct`
+
+## Backward compatibility
+
+The existing `MEMPALACE_HOOKS_DAEMON` environment variable and
+`hooks.daemon` config value remain supported.
+
+Legacy true values map to `prefer`.
+
+Legacy false values map to `direct`.
+
+The existing `MempalaceConfig.hook_use_daemon` property is intentionally
+unchanged in this PR. Hook and CLI behavior remains unchanged until their
+policy-aware rollout PRs land.
+
+## Invalid policy values
+
+New policy settings accept only:
+
+- `direct`
+- `prefer`
+- `require`
+
+Invalid values fail with a source-specific error rather than silently falling
+back. This is important because silently turning a misspelled `require` into a
+direct write would violate the safety purpose of the policy.
+
+## Follow-up PRs
+
+PR 2 will apply the policy to hook-triggered writes.
+
+PR 3 will apply the policy to routine CLI writes.
+
+Maintenance operations such as repair, migration, and index rebuild are not
+ordinary routed writes. They require a separate exclusive-maintenance policy.

hooks/mempal_precompact_hook.sh

@@ -182,14 +182,14 @@ echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$
 #   1. TRANSCRIPT_PATH (from Claude Code) → parent dir, --mode convos
 #   2. MEMPAL_DIR → --mode projects
 if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
-    mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
+    "$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
         >> "$STATE_DIR/hook.log" 2>&1
 elif [ -n "$TRANSCRIPT_PATH" ]; then
     echo "[$(date '+%H:%M:%S')] Skipping missing or invalid transcript path after normalization: $TRANSCRIPT_PATH" \
         >> "$STATE_DIR/hook.log"
 fi
 if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
-    mempalace mine "$MEMPAL_DIR" --mode projects \
+    "$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \
         >> "$STATE_DIR/hook.log" 2>&1
 fi
 

hooks/mempal_save_hook.sh

@@ -265,14 +265,14 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then
     # MEMPAL_DIR is *additive*, not an override: a user with MEMPAL_DIR
     # pointed at their project still gets the active conversation mined.
     if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
-        mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
+        "$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
             >> "$STATE_DIR/hook.log" 2>&1 &
     elif [ -n "$TRANSCRIPT_PATH" ]; then
         echo "[$(date '+%H:%M:%S')] Skipping invalid transcript path: $TRANSCRIPT_PATH" \
             >> "$STATE_DIR/hook.log"
     fi
     if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
-        mempalace mine "$MEMPAL_DIR" --mode projects \
+        "$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \
             >> "$STATE_DIR/hook.log" 2>&1 &
     fi
 

integrations/openclaw/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: mempalace
 description: "MemPalace — Local AI memory with 96.6% recall. Semantic search, temporal knowledge graph, palace architecture (wings/rooms/drawers). Free, no cloud, no API keys."
-version: 3.3.0
+version: 3.6.0
 homepage: https://github.com/MemPalace/mempalace
 user-invocable: true
 metadata:
@@ -46,6 +46,10 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
 
 ## Available Tools
 
+Full MCP surface: 36 tools. Destructive or host-level tools are documented so
+you know they exist, but use them only when the user explicitly asks or when a
+tool-specific workflow below says to.
+
 ### Search & Browse
 - `mempalace_search` — Semantic search across all memories. Always start here.
   - `query` (required): natural language search — keep it short, keywords or a question. Do NOT include system prompts or conversation context.
@@ -58,6 +62,14 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
 - `mempalace_status` — Palace overview: total drawers, wings, rooms, AAAK spec
 - `mempalace_list_wings` — All wings with drawer counts
 - `mempalace_list_rooms` — Rooms within a wing (optional wing filter)
+- `mempalace_list_drawers` — Paginated drawer listing
+  - `wing`, `room`: optional filters
+  - `since`: only drawers filed on/after this ISO date/time
+  - `before`: only drawers filed before this ISO date/time
+  - `limit`: max results (default 20)
+  - `offset`: pagination offset (default 0)
+- `mempalace_get_drawer` — Fetch a single drawer by ID. Returns full verbatim content and metadata.
+  - `drawer_id` (required)
 - `mempalace_get_taxonomy` — Full wing/room/count tree
 - `mempalace_get_aaak_spec` — Get AAAK compression dialect specification
 
@@ -81,17 +93,59 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
 - `mempalace_traverse` — Walk from a room, find connected ideas across wings
   - `start_room` (required): room to start from
   - `max_hops`: connection depth (default 2)
-- `mempalace_find_tunnels` — Find rooms that bridge two wings
-  - `wing_a`, `wing_b` (required)
+- `mempalace_find_tunnels` — Find rooms that bridge two wings via *implicit* overlap (rooms whose drawers naturally share content across wings — discovered, not declared)
+  - `wing_a`, `wing_b`: optional filters; omit both to scan all wing pairs
+- `mempalace_create_tunnel` — Create an *explicit* cross-wing tunnel: a user/agent-declared link between two locations. Use when you notice content in one project relates to another (e.g. API design in `project_api` connects to schema in `project_database`).
+  - `source_wing`, `source_room`, `target_wing`, `target_room` (required)
+  - `label`: short description of the relationship
+  - `source_drawer_id`, `target_drawer_id`: anchor to specific drawers
+- `mempalace_list_tunnels` — List all explicit tunnels, optionally filtered by wing
+  - `wing`: optional filter
+- `mempalace_delete_tunnel` — Remove an explicit tunnel by ID
+  - `tunnel_id` (required)
+- `mempalace_list_hallways` — List within-wing entity hallways (entity-to-entity co-occurrence links built at mine time)
+  - `wing`: optional filter
+- `mempalace_delete_hallway` — Remove a hallway record by ID
+  - `hallway_id` (required)
+- `mempalace_follow_tunnels` — From a room, follow explicit tunnels to connected drawers in other wings
+  - `wing`, `room` (required)
 - `mempalace_graph_stats` — Graph connectivity overview
 
 ### Write
 - `mempalace_add_drawer` — Store verbatim content into a wing/room
   - `wing`, `room`, `content` (required)
   - `source_file`: optional source reference
+  - `added_by`: optional filing agent label
   - Checks for duplicates automatically
+- `mempalace_checkpoint` — Save a whole session in one call: dedup each item, file non-duplicates, then write one diary entry
+  - `items` (required): array of `{wing, room, content}`; content must be verbatim
+  - `diary`: optional `{agent_name, entry, topic?, wing?}`; entry should use AAAK format
+  - `dedup_threshold`: similarity threshold (default 0.9)
+  - `added_by`: optional filing agent label (defaults to the diary `agent_name`, else `checkpoint`)
+- `mempalace_update_drawer` — Update an existing drawer's content and/or move it to a different wing/room
+  - `drawer_id` (required)
+  - `content`, `wing`, `room`: at least one must be provided (no-op otherwise)
 - `mempalace_delete_drawer` — Remove a drawer by ID
   - `drawer_id` (required)
+
+### Ingest & Cleanup
+- `mempalace_mine` — Mine a directory into the palace. Host-level ingest; call only when the user asks to import files.
+  - `source` (required): directory to mine
+  - `mode`: `projects` (default), `convos`, or `extract`
+  - `wing`: target wing (default: source directory name)
+  - `agent`: recorded on every drawer (default `mempalace`)
+  - `limit`: max files to process (0 = all)
+  - `dry_run`: preview without writing
+  - `extract`: convos extraction strategy (`exchange` default, or `general`)
+- `mempalace_sync` — Prune drawers whose source files are gitignored, deleted, or moved. Use dry-run first.
+  - `project_dir`: optional project root scope
+  - `wing`: optional wing scope
+  - `apply`: actually delete; default is dry-run preview
+- `mempalace_delete_by_source` — Bulk-delete drawers with one exact `source_file`. Destructive; use dry-run first.
+  - `source_file` (required): exact metadata value to remove
+  - `dry_run`: preview match count and sample (default true)
+
+### Diary & Session
 - `mempalace_diary_write` — Write a session diary entry
   - `agent_name` (required): your name/identifier
   - `entry` (required): what happened, what you learned, what matters
@@ -99,6 +153,15 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
 - `mempalace_diary_read` — Read recent diary entries
   - `agent_name` (required)
   - `last_n`: number of entries (default 10)
+- `mempalace_memories_filed_away` — Acknowledge the latest silent auto-save checkpoint.
+  - Returns: how many messages were tucked into drawers since the last ack
+  - When to call: at the START of a session, to confirm prior-conversation persistence
+
+### System
+- `mempalace_hook_settings` — Get or set auto-save hook behavior. Host-level setting; do not change silently.
+  - `silent_save`: true saves directly without MCP-level clutter
+  - `desktop_toast`: true shows a desktop notification when saves complete
+- `mempalace_reconnect` — Force reconnect to the palace database after external writes or stale index state
 
 ## Setup
 

mempalace/backends/__init__.py

@@ -36,6 +36,7 @@ from .base import (
     UnsupportedMaintenanceKindError,
 )
 from .chroma import ChromaBackend, ChromaCollection
+from .milvus import MilvusBackend, MilvusCollection
 from .pgvector import PgVectorBackend, PgVectorCollection
 from .qdrant import QdrantBackend, QdrantCollection
 from .sqlite_exact import SQLiteExactBackend, SQLiteExactCollection
@@ -67,6 +68,8 @@ __all__ = [
     "LexicalHit",
     "LexicalResult",
     "MaintenanceResult",
+    "MilvusBackend",
+    "MilvusCollection",
     "PalaceNotFoundError",
     "PalaceRef",
     "PgVectorBackend",

mempalace/backends/base.py

@@ -501,6 +501,15 @@ class BaseCollection(ABC):
             offset += len(batch_meta)
         return all_meta
 
+    def facet_counts(
+        self,
+        field: str,
+        where: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        """Return counts for each distinct value of a metadata field."""
+        raise UnsupportedCapabilityError("backend does not support facet_counts")
+
     def maintenance_state(self) -> dict:
         """Return a structured snapshot of this collection's maintenance state.
 

mempalace/backends/chroma.py

@@ -8,6 +8,7 @@ import math
 import os
 import pickle
 import re
+import shlex
 import sqlite3
 import time
 from collections import defaultdict
@@ -81,13 +82,52 @@ def _hnsw_link_to_data_ratio(seg_dir: str) -> Optional[float]:
     return link_size / data_size
 
 
-def _hnsw_link_lists_is_usable_for_payload(seg_dir: str) -> bool:
-    """Return False when a non-trivial HNSW payload lacks usable link lists.
+def _hnsw_metadata_marker_intact(seg_dir: str) -> bool:
+    """Return True when ``index_metadata.pickle`` bears a complete envelope.
 
-    A missing or empty link_lists.bin is acceptable only for a fresh/empty
-    segment. Once data_level0.bin has real payload, a zero-byte link_lists.bin
-    is not a harmless async-flush shape: ChromaDB can later hand the broken
-    graph to hnswlib and crash in native code.
+    ChromaDB writes ``index_metadata.pickle`` last during a persist, so an
+    intact pickle envelope — protocol marker ``0x80`` at the head, ``STOP``
+    byte ``0x2e`` at the tail — proves the flush finished. Used as a
+    persist-completion marker: when present, the segment's on-disk shape
+    (including an all-layer-0 index with an empty ``link_lists.bin``) is the
+    one chromadb intentionally serialized, not a half-written one.
+
+    Deliberately byte-sniffs only; never deserializes. Deserialization can
+    execute arbitrary code, and the byte-sniff is enough to tell a complete
+    write from truncation or zero-fill. Assumes pickle protocol >= 2.
+    """
+    meta_path = os.path.join(seg_dir, "index_metadata.pickle")
+    try:
+        if not os.path.isfile(meta_path) or os.path.getsize(meta_path) < 16:
+            return False
+        with open(meta_path, "rb") as f:
+            head = f.read(2)
+            f.seek(-1, 2)  # last byte
+            tail = f.read(1)
+    except OSError:
+        return False
+    return len(head) == 2 and head[0] == 0x80 and tail == b"\x2e"
+
+
+def _hnsw_link_lists_is_usable_for_payload(seg_dir: str) -> bool:
+    """Return False when a non-trivial HNSW payload looks like a partial flush.
+
+    A zero-byte ``link_lists.bin`` is *not* corruption on its own. hnswlib
+    stores the entire layer-0 graph inside ``data_level0.bin`` and only writes
+    ``link_lists.bin`` for elements promoted to level > 0. A small or
+    low-fanout index where every element stays on layer 0 therefore
+    serializes an empty ``link_lists.bin`` and loads/searches fine (#1716).
+    Treating that shape as corruption produced a self-perpetuating quarantine
+    loop: repair rebuilt the byte-identical all-layer-0 segment, which the next
+    cold start quarantined again, accumulating drift dirs without bound.
+
+    An empty ``link_lists.bin`` only signals trouble when the persist was
+    interrupted before it could finish. ChromaDB writes
+    ``index_metadata.pickle`` last, so an intact metadata envelope proves the
+    flush completed and the empty ``link_lists.bin`` is the legitimate
+    all-layer-0 shape. Only when there is real payload, an empty
+    ``link_lists.bin``, *and* no completion marker do we treat the segment as a
+    partial flush.
     """
     data_path = os.path.join(seg_dir, "data_level0.bin")
     link_path = os.path.join(seg_dir, "link_lists.bin")
@@ -100,10 +140,16 @@ def _hnsw_link_lists_is_usable_for_payload(seg_dir: str) -> bool:
         if data_size <= _HNSW_MISSING_METADATA_DATA_FLOOR:
             return True
 
-        return os.path.isfile(link_path) and os.path.getsize(link_path) > 0
+        if os.path.isfile(link_path) and os.path.getsize(link_path) > 0:
+            return True
     except OSError:
         return False
 
+    # Real payload with an empty/absent link_lists.bin: legitimate only when
+    # the persist completed (all-layer-0 index), proven by an intact metadata
+    # marker. Otherwise it is a half-written segment chromadb could segfault on.
+    return _hnsw_metadata_marker_intact(seg_dir)
+
 
 def _hnsw_payload_appears_sane(seg_dir: str) -> bool:
     """Return False when HNSW payload files are structurally implausible."""
@@ -336,17 +382,7 @@ def _segment_appears_healthy(seg_dir: str) -> bool:
     if not _hnsw_payload_appears_sane(seg_dir):
         return False
 
-    try:
-        size = os.path.getsize(meta_path)
-        if size < 16:
-            return False
-        with open(meta_path, "rb") as f:
-            head = f.read(2)
-            f.seek(-1, 2)  # last byte
-            tail = f.read(1)
-    except OSError:
-        return False
-    return len(head) == 2 and head[0] == 0x80 and tail == b"\x2e"
+    return _hnsw_metadata_marker_intact(seg_dir)
 
 
 def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 300.0) -> list[str]:
@@ -1339,8 +1375,8 @@ class ChromaCollection(BaseCollection):
 
     @staticmethod
     def _sanitize_documents_for_chromadb(documents):
-        """Strip lone UTF-16 surrogates from every document before it reaches
-        the chromadb client.
+        """Strip lone UTF-16 surrogates and embedded NUL characters from every
+        document before it reaches the chromadb client.
 
         A single lone surrogate (U+D800–U+DFFF) raises ``UnicodeEncodeError``
         inside chromadb's encode path and aborts the *entire* add/upsert batch
@@ -1353,18 +1389,29 @@ class ChromaCollection(BaseCollection):
         directly. Sanitising here makes the chokepoint catch-all complete: the
         sibling :meth:`_sanitize_metadatas_for_chromadb` already guarantees this
         for metadata one method over; documents get the same guarantee.
+
+        A document containing an embedded NUL (U+0000) — routine in mined
+        Bash tool output — is well-formed UTF-8, unlike a lone surrogate, but
+        can corrupt the FTS5 inverted index for the whole collection rather
+        than just failing to store that one row (a ChromaDB-side bug; tracked
+        upstream). Stripping it here is the same defense-in-depth already
+        applied to surrogates: sanitize input we don't control before it
+        reaches a datastore we don't control.
         """
         if documents is None:
             return None
-        from ..config import strip_lone_surrogates
+        from ..config import strip_lone_surrogates, strip_nul_bytes
+
+        def _sanitize(d):
+            return strip_nul_bytes(strip_lone_surrogates(d))
 
         # chromadb accepts OneOrMany[Document]: a bare str is a single document,
         # not an iterable of characters. Handle it explicitly so we don't split
         # it into per-character documents — that would be exactly the kind of
         # silent corruption this method exists to prevent.
         if isinstance(documents, str):
-            return strip_lone_surrogates(documents)
-        return [strip_lone_surrogates(d) if isinstance(d, str) else d for d in documents]
+            return _sanitize(documents)
+        return [_sanitize(d) if isinstance(d, str) else d for d in documents]
 
     def add(self, *, documents, ids, metadatas=None, embeddings=None):
         kwargs: dict[str, Any] = {
@@ -1924,6 +1971,7 @@ class ChromaBackend(BaseBackend):
             current_model = MempalaceConfig().embedding_model
         except Exception:
             current_model = "unknown"
+        rebuild_cmd = f"mempalace --palace {shlex.quote(palace_path)} repair rebuild-index"
         return (
             f"Embedding model mismatch reading palace at {palace_path!r}.\n"
             f"  Underlying ChromaDB error: {msg}\n"
@@ -1931,8 +1979,8 @@ class ChromaBackend(BaseBackend):
             f"  The palace was built with a different embedding model. Either:\n"
             f"    (a) revert the model: unset MEMPALACE_EMBEDDING_MODEL (or set "
             f"the previous value), or\n"
-            f"    (b) re-embed in place: `mempalace repair rebuild-index "
-            f"--palace {palace_path}` (writes new vectors with the current model)."
+            f"    (b) re-embed in place: `{rebuild_cmd}` "
+            f"(writes new vectors with the current model)."
         )
 
     # ------------------------------------------------------------------
@@ -2191,7 +2239,26 @@ class ChromaBackend(BaseBackend):
 
     @classmethod
     def detect(cls, path: str) -> bool:
-        return os.path.isfile(os.path.join(path, "chroma.sqlite3"))
+        """Return True when ``path`` looks like a chroma palace.
+
+        Verifies the SQLite magic header rather than file presence alone.
+        Bare ``sqlite3.connect()`` against a missing path leaves a 0-byte
+        file behind (the SQLite header is written on the first statement,
+        not on connection), so file-presence alone treats those artifacts
+        as real chroma palaces and breaks multi-backend resolution. The
+        16-byte ``SQLite format 3\\x00`` magic prefix is written as soon
+        as chromadb's ``PersistentClient`` does any work, so this check
+        accepts every real chroma palace while rejecting empty / garbage
+        files. See #1893.
+        """
+        db_path = os.path.join(path, "chroma.sqlite3")
+        if not os.path.isfile(db_path):
+            return False
+        try:
+            with open(db_path, "rb") as f:
+                return f.read(16) == b"SQLite format 3\x00"
+        except OSError:
+            return False
 
     # ------------------------------------------------------------------
     # Legacy (pre-RFC 001) surface — retained while callers migrate.

mempalace/backends/embedding_wrapper.py

@@ -155,6 +155,30 @@ class EmbeddingCollection(BaseCollection):
     def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None):
         return self._inner.lexical_search(query=query, n_results=n_results, where=where)
 
+    def facet_counts(
+        self, field: str, where: Optional[dict] = None, limit: int = 1000
+    ) -> dict[str, int]:
+        # ``BaseCollection.facet_counts`` is a concrete method that raises
+        # ``UnsupportedCapabilityError`` as its default. MRO resolves it on
+        # this subclass before ``__getattr__`` ever fires, so without an
+        # explicit forwarder every facet call against a wrapped backend
+        # (qdrant, pgvector, sqlite_exact) raises and silently degrades to
+        # client-side counting in mcp_server's try/except.
+        return self._inner.facet_counts(field, where=where, limit=limit)
+
+    def get_all_metadata(self, where: Optional[dict] = None) -> list[dict]:
+        # ``BaseCollection.get_all_metadata`` ships a concrete default that
+        # pages through ``self.get(include=["metadatas"])``. Without this
+        # forwarder, MRO resolves the call here on the subclass and runs the
+        # base default — which routes back through ``self.get()`` (the
+        # wrapper's get, then ``__getattr__`` to the inner's get). Result:
+        # the inner's overridden ``get_all_metadata`` (e.g. pgvector's
+        # ``with_document=False`` fast path from #1892) is never reached,
+        # and every metadata-only fetch transfers the full document column
+        # over the wire. Same MRO-shadow pattern as ``facet_counts`` /
+        # ``lexical_search`` above.
+        return self._inner.get_all_metadata(where=where)
+
     def update(self, *, ids, documents=None, metadatas=None, embeddings=None):
         ids = _as_list(ids)
         if documents is not None:

mempalace/backends/pgvector.py

@@ -674,13 +674,19 @@ class _PgVectorClient:
         *,
         where: Optional[dict] = None,
         with_embedding: bool = False,
+        with_document: bool = True,
         limit: Optional[int] = None,
         offset: Optional[int] = None,
     ) -> list[dict]:
         qi = _quote_identifier(table)
         params: list = []
         where_sql = _where_to_sql(where, params) if where else "TRUE"
-        cols = "id, document, metadata"
+        # Project NULL into the document slot when the caller only needs
+        # metadata (e.g. mempalace_status's wing/room tally). Keeps the
+        # positional _row parser unchanged — document remains record[1] —
+        # while avoiding O(n × document_size) bytes over the wire on remote
+        # pgvector deployments. Follow-up to #1840.
+        cols = "id, document, metadata" if with_document else "id, NULL::text, metadata"
         if with_embedding:
             cols += ", embedding"
         sql = f"SELECT {cols} FROM {qi} WHERE {where_sql}"
@@ -855,7 +861,15 @@ class PgVectorCollection(BaseCollection):
                 )
             self._known_dimension = existing_dim or dimension
 
-    def _scroll(self, *, where=None, with_embedding=False, limit=None, offset=None) -> list[dict]:
+    def _scroll(
+        self,
+        *,
+        where=None,
+        with_embedding=False,
+        with_document=True,
+        limit=None,
+        offset=None,
+    ) -> list[dict]:
         self._ensure_open()
         if not self._table_exists():
             if self._marker_exists():
@@ -865,10 +879,39 @@ class PgVectorCollection(BaseCollection):
             self._table,
             where=where,
             with_embedding=with_embedding,
+            with_document=with_document,
             limit=limit,
             offset=offset,
         )
 
+    def get_all_metadata(self, where=None) -> list[dict]:
+        """Single-pass metadata-only fetch — projects out the document column.
+
+        The base implementation pages through ``get(include=["metadatas"])``,
+        which routes here via ``_scroll`` and (pre-this-override) always sent
+        the ``document`` text over the wire even when nothing consumed it.
+        For pgvector deployments where the client is remote (TLS over WAN),
+        that meant ``mempalace_status`` transferred O(n × document_size)
+        bytes per call, dominating wall time. With ``with_document=False``
+        the SELECT replaces document with NULL, dropping the per-row payload
+        to id + metadata for every caller of this method.
+
+        Filtered fetches still need the ``_matches_where`` post-filter for
+        non-pushdown semantics (array/object values where ``metadata @> ...``
+        is broader than the exact match the caller asked for — same
+        correctness contract as #1840's filtered ``get`` path). Since that
+        post-filter only reads ``metadata``, we keep the single-scroll +
+        ``with_document=False`` fast path and just apply the filter locally
+        on the metadata dicts before returning. This extends the wire-byte
+        win to filtered callers as well.
+        """
+        _validate_where(where)
+        pushdown = None if _requires_local_filter(where) else where
+        rows = self._scroll(where=pushdown, with_document=False)
+        if where is None:
+            return [row["metadata"] for row in rows]
+        return [row["metadata"] for row in rows if _matches_where(row["metadata"], where)]
+
     def _rows(
         self,
         *,

mempalace/backends/qdrant.py

@@ -40,6 +40,7 @@ from .base import (
     PalaceNotFoundError,
     PalaceRef,
     QueryResult,
+    UnsupportedCapabilityError,
     UnsupportedFilterError,
     _IncludeSpec,
 )
@@ -543,6 +544,38 @@ class _QdrantRESTClient:
         result = response.get("result") or {}
         return int(result.get("count") or 0)
 
+    def facet_counts(
+        self,
+        collection: str,
+        *,
+        field: str,
+        qdrant_filter: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        body: dict[str, Any] = {
+            "key": field,
+            "exact": True,
+            "limit": limit,
+        }
+
+        if qdrant_filter:
+            body["filter"] = qdrant_filter
+
+        response = self.request(
+            "POST",
+            f"/collections/{urlparse.quote(collection, safe='')}/facet",
+            body=body,
+        )
+
+        result = response.get("result") or {}
+        hits = result.get("hits") or []
+
+        return {
+            str(hit["value"]): int(hit.get("count") or 0)
+            for hit in hits
+            if hit.get("value") is not None
+        }
+
     def delete_collection(self, collection: str) -> None:
         self.request("DELETE", f"/collections/{urlparse.quote(collection, safe='')}")
 
@@ -1035,6 +1068,34 @@ class QdrantCollection(BaseCollection):
         rows = self._rows(where=where)
         return [row["metadata"] for row in rows]
 
+    def facet_counts(
+        self,
+        field: str,
+        where: Optional[dict] = None,
+        limit: int = 1000,
+    ) -> dict[str, int]:
+        self._ensure_open()
+        # Validate the filter before the existence short-circuit so an
+        # unsupported local-only filter raises regardless of whether the
+        # collection has been materialized yet — matching the order used by
+        # get()/lexical_search() above (#1835 review).
+        _validate_where(where)
+        if _requires_local_filter(where):
+            raise UnsupportedCapabilityError("facet_counts does not support local-only filters")
+        if not self._remote_exists():
+            if self._marker_exists():
+                raise CollectionNotInitializedError(self._collection_name)
+            return {}
+
+        q_filter = _qdrant_filter(where)
+
+        return self._client.facet_counts(
+            self._remote_collection,
+            field=f"{_PAYLOAD_METADATA}.{field}",
+            qdrant_filter=q_filter,
+            limit=limit,
+        )
+
     def delete(self, *, ids=None, where=None):
         _validate_where(where)
         if not self._remote_exists():
@@ -1125,6 +1186,7 @@ class QdrantBackend(BaseBackend):
             "supports_embeddings_out",
             "supports_metadata_filters",
             "supports_lexical_search",
+            "supports_metadata_facets",
             "supports_namespace_isolation",
             "server_mode",
         }

mempalace/backends/registry.py

@@ -207,6 +207,7 @@ def resolve_backend_for_palace(
 def _register_builtins() -> None:
     """Register chroma as the in-tree default."""
     from .chroma import ChromaBackend
+    from .milvus import MilvusBackend
     from .pgvector import PgVectorBackend
     from .qdrant import QdrantBackend
     from .sqlite_exact import SQLiteExactBackend
@@ -214,6 +215,8 @@ def _register_builtins() -> None:
     # Use setdefault semantics so a caller that pre-registered for tests wins.
     if "chroma" not in _registry:
         _registry["chroma"] = ChromaBackend
+    if "milvus" not in _registry:
+        _registry["milvus"] = MilvusBackend
     if "qdrant" not in _registry:
         _registry["qdrant"] = QdrantBackend
     if "sqlite_exact" not in _registry:

mempalace/backends/sqlite_exact.py

@@ -1045,7 +1045,23 @@ class SQLiteExactBackend(BaseBackend):
 
     @classmethod
     def detect(cls, path: str) -> bool:
-        return os.path.isfile(os.path.join(path, _DB_FILENAME))
+        """Return True when ``path`` looks like a sqlite_exact palace.
+
+        Verifies the SQLite magic header rather than file presence alone, for
+        the same reason as :py:meth:`mempalace.backends.chroma.ChromaBackend.detect`:
+        bare ``sqlite3.connect()`` against a missing path leaves a 0-byte file
+        behind because the SQLite header is written on the first statement,
+        not on connection. The 16-byte ``SQLite format 3\\x00`` magic prefix
+        accepts every real palace while rejecting empty / garbage files. See #1893.
+        """
+        db_path = os.path.join(path, _DB_FILENAME)
+        if not os.path.isfile(db_path):
+            return False
+        try:
+            with open(db_path, "rb") as f:
+                return f.read(16) == b"SQLite format 3\x00"
+        except OSError:
+            return False
 
     def create_collection(self, palace_path: str, collection_name: str) -> SQLiteExactCollection:
         return self.get_collection(palace_path, collection_name, create=True)

mempalace/cli.py

@@ -259,14 +259,16 @@ def _ensure_mempalace_files_gitignored(project_dir) -> bool:
     if not (project_path / ".git").exists():
         return False
     gitignore = project_path / ".gitignore"
-    existing = gitignore.read_text() if gitignore.exists() else ""
+    # Force UTF-8: Windows defaults to GBK and chokes on non-ASCII .gitignore
+    # comments, killing auto-init even though the file is valid UTF-8.
+    existing = gitignore.read_text(encoding="utf-8", errors="replace") if gitignore.exists() else ""
     existing_lines = {line.strip() for line in existing.splitlines()}
     missing = [p for p in _MEMPALACE_PROJECT_FILES if p not in existing_lines]
     if not missing:
         return False
     prefix = "" if not existing or existing.endswith("\n") else "\n"
     block = prefix + "\n# MemPalace per-project files (issue #185)\n" + "\n".join(missing) + "\n"
-    with open(gitignore, "a") as f:
+    with open(gitignore, "a", encoding="utf-8") as f:
         f.write(block)
     print(f"  Added {', '.join(missing)} to {gitignore.name}")
     return True
@@ -313,7 +315,16 @@ def cmd_init(args):
                 endpoint=getattr(args, "llm_endpoint", None),
                 api_key=getattr(args, "llm_api_key", None),
             )
-            ok, msg = candidate.check_available()
+            if (
+                provider_name == "openai-compat"
+                and getattr(candidate, "api_key_source", None) == "env"
+                and candidate.is_external_service
+            ):
+                ok = False
+                msg = "external openai-compat init requires explicit --llm-api-key"
+                print(f"  LLM skipped: {msg}")
+            else:
+                ok, msg = candidate.check_available()
             if ok:
                 llm_provider = candidate
                 print(f"  LLM enabled: {provider_name}/{provider_model}")
@@ -992,6 +1003,29 @@ def cmd_migrate_wings(args):
     )
 
 
+def cmd_hallways(args):
+    """List within-wing entity hallways (the auto-built associative graph)."""
+    from .hallways import list_hallways
+
+    palace_path = (
+        os.path.expanduser(args.palace)
+        if getattr(args, "palace", None)
+        else MempalaceConfig().palace_path
+    )
+    rows = list_hallways(
+        getattr(args, "wing", None),
+        config=MempalaceConfig(palace_path=palace_path),
+    )
+    if not rows:
+        print("No hallways yet — they are built from drawer entities when you mine.")
+        return
+    rows.sort(key=lambda h: h.get("co_occurrence_count", 0), reverse=True)
+    print(f"  {len(rows)} hallway(s):")
+    for h in rows[: max(0, args.limit)]:
+        label = h.get("label") or f"{h.get('entity_a', '?')} <-> {h.get('entity_b', '?')}"
+        print(f"    {label}")
+
+
 def cmd_status(args):
     from .miner import status
 
@@ -1083,11 +1117,16 @@ def cmd_repair(args):
         _rebuild_collection_via_temp,
         check_extraction_safety,
         index_read_recovery_guidance,
+        maybe_autoheal_fts5_index,
         maybe_repair_poisoned_max_seq_id_before_rebuild,
         print_sqlite_integrity_abort,
         sqlite_integrity_errors,
     )
 
+    if getattr(args, "repair_action", None) == "rebuild-index":
+        args.mode = "from-sqlite"
+        args.archive_existing = True
+
     if getattr(args, "mode", "legacy") == "max-seq-id":
         from .repair import repair_max_seq_id
 
@@ -1103,7 +1142,7 @@ def cmd_repair(args):
 
     if getattr(args, "mode", "legacy") == "from-sqlite":
         from .migrate import confirm_destructive_action
-        from .repair import RebuildPartialError, rebuild_from_sqlite
+        from .repair import RebuildCleanupError, RebuildPartialError, rebuild_from_sqlite
 
         source_path = getattr(args, "source", None)
         source_path = (
@@ -1141,6 +1180,13 @@ def cmd_repair(args):
                 f"Failed in collection: {exc.failed_collection}"
             )
             sys.exit(1)
+        except RebuildCleanupError:
+            # All rows may have landed, but rebuild_from_sqlite deliberately
+            # withholds success until FTS5 rebuild, VACUUM, and quick_check are
+            # clean. Its exception already includes the retained destination
+            # and archive/source recovery paths.
+            print("\n  Rebuild cleanup failed — see recovery details above.")
+            sys.exit(1)
         # An empty counts dict is rebuild_from_sqlite's documented signal
         # for a validation refusal (missing source, existing dest,
         # in-place without --archive-existing). The library already
@@ -1169,6 +1215,8 @@ def cmd_repair(args):
     # here so we can surface the clear recovery instructions and exit
     # cleanly before chromadb's compactor touches the disk.
     sqlite_errors = sqlite_integrity_errors(palace_path)
+    if sqlite_errors:
+        sqlite_errors = maybe_autoheal_fts5_index(palace_path, sqlite_errors)
     if sqlite_errors:
         print_sqlite_integrity_abort(palace_path, sqlite_errors)
         sys.exit(1)
@@ -1323,6 +1371,154 @@ def cmd_mcp(args):
         print(f"  {base_server_cmd} --palace /path/to/palace")
 
 
+_SERVER_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", "[::1]"}
+_SERVER_BIND_ALL_HOSTS = {"0.0.0.0", "::", "[::]"}
+
+
+def _server_is_loopback(host: str) -> bool:
+    return (host or "").strip().lower() in _SERVER_LOOPBACK_HOSTS
+
+
+def _server_token_path(palace_path: str) -> Path:
+    """Per-palace location for the auto-generated server bearer token.
+
+    Distinct from the daemon's token dir; keyed by the canonical palace path so
+    one server per palace reuses a stable token across restarts.
+    """
+    import hashlib
+
+    canonical = os.path.abspath(os.path.realpath(os.path.expanduser(palace_path)))
+    key = hashlib.sha256(os.path.normcase(canonical).encode("utf-8")).hexdigest()[:24]
+    return Path.home() / ".mempalace" / "server" / key / "token"
+
+
+def _load_or_create_server_token(palace_path: str) -> tuple[str, bool]:
+    """Return (token, created). Reuse an existing 0600 token or mint a new one."""
+    import secrets
+
+    token_path = _server_token_path(palace_path)
+    if token_path.exists():
+        existing = token_path.read_text(encoding="utf-8").strip()
+        if existing:
+            return existing, False
+    token = secrets.token_urlsafe(32)
+    token_path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        os.chmod(str(token_path.parent), 0o700)
+    except OSError:
+        pass
+    # O_CREAT with 0600 so the token is never briefly world-readable on disk.
+    fd = os.open(str(token_path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        fh.write(token + "\n")
+    return token, True
+
+
+def cmd_serve(args):
+    """Run a secure remote HTTP MCP server for a team to share one palace (#1877).
+
+    A turnkey wrapper over ``mempalace-mcp --transport http``: it resolves a
+    bearer token (auto-generating a strong one for non-loopback binds), prints a
+    ready-to-paste client config, then execs the real server in the foreground so
+    Docker/systemd own the process lifecycle. The token is passed via the
+    environment, never argv, so it can't leak through ``ps``.
+    """
+    host = args.host
+    port = int(args.port)
+    loopback = _server_is_loopback(host)
+    palace_path = (
+        os.path.abspath(os.path.expanduser(args.palace))
+        if args.palace
+        else MempalaceConfig().palace_path
+    )
+    backend = _backend_arg(args)
+
+    tls_cert = os.path.expanduser(args.tls_cert) if args.tls_cert else None
+    tls_key = os.path.expanduser(args.tls_key) if args.tls_key else None
+    if bool(tls_cert) != bool(tls_key):
+        print("mempalace: --tls-cert and --tls-key must be given together", file=sys.stderr)
+        sys.exit(2)
+    for label, path in (("--tls-cert", tls_cert), ("--tls-key", tls_key)):
+        if path and not os.path.isfile(path):
+            print(f"mempalace: {label} file not found: {path}", file=sys.stderr)
+            sys.exit(2)
+    scheme = "https" if tls_cert else "http"
+
+    # Token resolution. Explicit flag > existing env > (non-loopback) auto-generated.
+    token = (args.token or os.environ.get("MEMPALACE_MCP_HTTP_TOKEN", "")).strip()
+    token_created = False
+    if not token and not loopback and not args.allow_insecure:
+        token, token_created = _load_or_create_server_token(palace_path)
+
+    # Build the child environment. Token rides in the env (never argv) so it
+    # stays out of the process table.
+    env = dict(os.environ)
+    env["MEMPALACE_PALACE_PATH"] = palace_path
+    if backend:
+        env["MEMPALACE_BACKEND"] = str(backend).strip().lower()
+    if token:
+        env["MEMPALACE_MCP_HTTP_TOKEN"] = token
+    if args.allow_insecure:
+        env["MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN"] = "1"
+
+    child = [
+        sys.executable,
+        "-m",
+        "mempalace.mcp_server",
+        "--transport",
+        "http",
+        "--host",
+        host,
+        "--port",
+        str(port),
+    ]
+    if backend:
+        child += ["--backend", str(backend).strip().lower()]
+    child += ["--palace", palace_path]
+    if tls_cert:
+        child += ["--tls-cert", tls_cert, "--tls-key", tls_key]
+    if args.read_only:
+        child.append("--read-only")
+
+    # Client-facing address: 0.0.0.0/:: means "all interfaces" — clients dial a
+    # real reachable host, so show a placeholder rather than the bind wildcard.
+    client_host = "YOUR_SERVER_HOST" if host.strip().lower() in _SERVER_BIND_ALL_HOSTS else host
+    url = f"{scheme}://{client_host}:{port}/mcp"
+
+    print("Starting MemPalace remote MCP server")
+    print(f"  palace   : {palace_path}")
+    print(f"  backend  : {(backend or 'default').strip().lower() if backend else 'default'}")
+    print(f"  bind     : {host}:{port}  ({'loopback' if loopback else 'network-exposed'})")
+    print(f"  tls      : {'on' if tls_cert else 'off (plaintext — terminate TLS at a proxy)'}")
+    print(f"  read-only: {'yes' if args.read_only else 'no'}")
+    if token_created:
+        print("\n  A new bearer token was generated and stored 0600 at:")
+        print(f"    {_server_token_path(palace_path)}")
+        print("  Store it securely — clients need it to connect:")
+        print(f"    {token}")
+    print("\nConnect a client:")
+    if token:
+        print(
+            f"  claude mcp add --transport http mempalace {url} "
+            f'--header "Authorization: Bearer {token if token_created else "$MEMPALACE_MCP_HTTP_TOKEN"}"'
+        )
+    else:
+        print(f"  claude mcp add --transport http mempalace {url}")
+    print(f"  curl {scheme}://{client_host}:{port}/healthz   # liveness (no auth)\n")
+    sys.stdout.flush()
+
+    # Foreground: hand the process to the real server so signals (SIGTERM from
+    # Docker/systemd) reach it directly. exec on POSIX; subprocess on Windows
+    # (no exec semantics) propagating the exit code.
+    if os.name == "posix":
+        os.execve(sys.executable, child, env)
+    else:
+        import subprocess
+
+        completed = subprocess.run(child, env=env)
+        sys.exit(completed.returncode)
+
+
 def cmd_compress(args):
     """Compress drawers in a wing using AAAK Dialect."""
     from .dialect import Dialect
@@ -1823,6 +2019,15 @@ def main():
     p_repair.add_argument(
         "--yes", action="store_true", help="Skip confirmation for destructive changes"
     )
+    p_repair.add_argument(
+        "repair_action",
+        nargs="?",
+        choices=["rebuild-index"],
+        help=(
+            "Re-embed the palace from SQLite using the current embedding model "
+            "(alias for --mode from-sqlite --archive-existing)."
+        ),
+    )
     p_repair.add_argument(
         "--confirm-truncation-ok",
         action="store_true",
@@ -1925,6 +2130,38 @@ def main():
         help="Storage backend to include in the MCP startup command",
     )
 
+    # serve — turnkey remote HTTP MCP server (#1877)
+    p_serve = sub.add_parser(
+        "serve",
+        help="Run a secure remote HTTP MCP server for a team to share one palace",
+    )
+    p_serve.add_argument(
+        "--host", default="127.0.0.1", help="Bind address (use 0.0.0.0 for remote clients)"
+    )
+    p_serve.add_argument("--port", type=int, default=8765, help="Bind port (default: 8765)")
+    p_serve.add_argument(
+        "--backend", default=None, help="Storage backend (default: config/env/detected)"
+    )
+    p_serve.add_argument("--palace", default=None, help="Palace path (overrides config/env)")
+    p_serve.add_argument(
+        "--token",
+        default=None,
+        help="Bearer token clients must present. Default: reuse/auto-generate one for "
+        "non-loopback binds (stored 0600 under ~/.mempalace/server/).",
+    )
+    p_serve.add_argument("--tls-cert", default=None, help="PEM certificate to enable TLS")
+    p_serve.add_argument("--tls-key", default=None, help="PEM private key matching --tls-cert")
+    p_serve.add_argument(
+        "--read-only",
+        action="store_true",
+        help="Expose recall only: mutating tools are hidden and refused",
+    )
+    p_serve.add_argument(
+        "--allow-insecure",
+        action="store_true",
+        help="Permit a non-loopback bind with no token (only behind a trusted proxy)",
+    )
+
     # status
     # migrate
     p_migrate = sub.add_parser(
@@ -1952,6 +2189,9 @@ def main():
     )
     p_migrate_wings.add_argument("--yes", action="store_true", help="Skip the confirmation prompt")
 
+    p_hallways = sub.add_parser("hallways", help="List entity hallways (associative graph)")
+    p_hallways.add_argument("--wing", default=None, help="Filter to one wing")
+    p_hallways.add_argument("--limit", type=int, default=50, help="Max hallways to show")
     p_status = sub.add_parser("status", help="Show what's been filed")
     p_status.add_argument(
         "--backend",
@@ -2030,12 +2270,14 @@ def main():
         "sweep": cmd_sweep,
         "sync": cmd_sync,
         "mcp": cmd_mcp,
+        "serve": cmd_serve,
         "compress": cmd_compress,
         "wake-up": cmd_wakeup,
         "repair": cmd_repair,
         "repair-status": cmd_repair_status,
         "migrate": cmd_migrate,
         "migrate-wings": cmd_migrate_wings,
+        "hallways": cmd_hallways,
         "status": cmd_status,
     }
     dispatch[args.command](args)

mempalace/config.py

@@ -11,6 +11,14 @@ from datetime import date, datetime
 from functools import lru_cache
 from pathlib import Path
 
+from .write_routing import (
+    ResolvedWriteRoutingPolicy,
+    RoutingPolicyCandidate,
+    WriteRoutingError,
+    WriteRoutingPolicy,
+    resolve_write_routing_policy,
+)
+
 # ── Input validation ──────────────────────────────────────────────────────────
 # Shared sanitizers for wing/room/entity names. Prevents path traversal,
 # excessively long strings, and special characters that could cause issues
@@ -31,6 +39,22 @@ def strip_lone_surrogates(text: str) -> str:
     return _LONE_SURROGATE_RE.sub("�", text)
 
 
+# Tool output mined from real transcripts routinely embeds a NUL character
+# (U+0000) — e.g. captured Bash output where a reader raced a background
+# writer, or genuine binary/NUL-delimited command output. A document
+# containing one is otherwise valid, well-formed text (unlike a lone
+# surrogate, which is invalid UTF-8), but handing it to ChromaDB's
+# SQLite/FTS5 layer can corrupt the FTS5 inverted index for the *whole*
+# collection (``PRAGMA quick_check`` reports "malformed inverted index for
+# FTS5 table"), not just fail to store that one document. Stripping it
+# before it reaches the chromadb client is the same defense-in-depth this
+# module already applies to lone surrogates (#1235) — sanitize input we
+# don't control before it reaches a datastore we don't control.
+def strip_nul_bytes(text: str) -> str:
+    """Replace embedded NUL characters with U+FFFD before ChromaDB storage."""
+    return text.replace("\x00", "�")
+
+
 def normalize_wing_name(name: str) -> str:
     """Lower-case + collapse separators (`-`, ` `) to `_` for wing slugs.
 
@@ -197,6 +221,13 @@ def sanitize_content(value: str, max_length: int = 100_000) -> str:
 DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
 DEFAULT_COLLECTION_NAME = "mempalace_drawers"
 DEFAULT_BACKEND = "chroma"
+DEFAULT_MILVUS_CONSISTENCY_LEVEL = "Strong"
+_MILVUS_CONSISTENCY_LEVELS = {
+    "strong": "Strong",
+    "session": "Session",
+    "bounded": "Bounded",
+    "eventually": "Eventually",
+}
 
 # How many timestamped palace backups to retain before the oldest are
 # pruned. Applies to the accumulating backups written by ``mempalace
@@ -205,6 +236,15 @@ DEFAULT_BACKEND = "chroma"
 DEFAULT_MAX_BACKUPS = 10
 
 
+def normalize_milvus_consistency_level(value) -> str:
+    raw = str(value).strip() if value else DEFAULT_MILVUS_CONSISTENCY_LEVEL
+    normalized = _MILVUS_CONSISTENCY_LEVELS.get(raw.lower())
+    if normalized:
+        return normalized
+    allowed = ", ".join(_MILVUS_CONSISTENCY_LEVELS.values())
+    raise ValueError(f"milvus_consistency_level must be one of: {allowed}")
+
+
 def sqlite_read_uri(db_path: str) -> str:
     """Return a read-only ``file:`` URI for ``sqlite3.connect(..., uri=True)``.
 
@@ -310,18 +350,26 @@ class MempalaceConfig:
     Load order: env vars > config file > defaults.
     """
 
-    def __init__(self, config_dir=None):
+    def __init__(self, config_dir=None, palace_path=None):
         """Initialize config.
 
         Args:
             config_dir: Override config directory (useful for testing).
                         Defaults to ~/.mempalace.
+            palace_path: Explicit palace data directory. This is primarily
+                         used by CLI operations that received ``--palace``;
+                         it takes precedence over environment and file config.
         """
         self._config_dir = (
             Path(config_dir) if config_dir else Path(os.path.expanduser("~/.mempalace"))
         )
         self._config_file = self._config_dir / "config.json"
         self._people_map_file = self._config_dir / "people_map.json"
+        self._palace_path_override = (
+            os.path.abspath(os.path.expanduser(str(palace_path)))
+            if palace_path is not None
+            else None
+        )
         self._file_config = {}
 
         if self._config_file.exists():
@@ -334,13 +382,15 @@ class MempalaceConfig:
     @property
     def palace_path(self):
         """Path to the memory palace data directory."""
+        if self._palace_path_override is not None:
+            return self._palace_path_override
         env_val = os.environ.get("MEMPALACE_PALACE_PATH") or os.environ.get("MEMPAL_PALACE_PATH")
         if env_val:
             # Normalize: expand ~ and collapse .. to match the CLI --palace
             # code path (mcp_server.py:62) and prevent surprise redirection
             # when the env var contains unresolved components.
             return os.path.abspath(os.path.expanduser(env_val))
-        return self._file_config.get("palace_path", DEFAULT_PALACE_PATH)
+        return os.path.expanduser(self._file_config.get("palace_path", DEFAULT_PALACE_PATH))
 
     @property
     def tunnel_file(self):
@@ -421,6 +471,56 @@ class MempalaceConfig:
             timeout = 10.0
         return timeout if timeout > 0 else 10.0
 
+    @property
+    def milvus_uri(self):
+        """Milvus endpoint for the opt-in ``milvus`` backend.
+
+        Defaults to ``None`` so selecting Milvus uses per-palace Milvus Lite at
+        ``<palace>/milvus.db``. Set this only to deliberately use a shared
+        Milvus server, Zilliz Cloud, or a custom local Lite file.
+        """
+        env_val = os.environ.get("MEMPALACE_MILVUS_URI")
+        if env_val:
+            return env_val.strip()
+        value = self._file_config.get("milvus_uri")
+        return str(value).strip() if value else None
+
+    @property
+    def milvus_token(self):
+        """Token for the opt-in ``milvus`` backend, if configured."""
+        env_val = os.environ.get("MEMPALACE_MILVUS_TOKEN")
+        if env_val:
+            return env_val
+        value = self._file_config.get("milvus_token")
+        return str(value) if value else None
+
+    @property
+    def milvus_db_name(self):
+        """Optional Milvus database name for the opt-in ``milvus`` backend."""
+        env_val = os.environ.get("MEMPALACE_MILVUS_DB_NAME")
+        if env_val:
+            return env_val.strip()
+        value = self._file_config.get("milvus_db_name")
+        return str(value).strip() if value else None
+
+    @property
+    def milvus_namespace(self):
+        """Optional Milvus collection namespace/prefix."""
+        env_val = os.environ.get("MEMPALACE_MILVUS_NAMESPACE")
+        if env_val:
+            return env_val.strip()
+        value = self._file_config.get("milvus_namespace")
+        return str(value).strip() if value else None
+
+    @property
+    def milvus_consistency_level(self):
+        """Milvus read consistency level for the opt-in ``milvus`` backend."""
+        env_val = os.environ.get("MEMPALACE_MILVUS_CONSISTENCY_LEVEL")
+        if env_val:
+            return normalize_milvus_consistency_level(env_val)
+        value = self._file_config.get("milvus_consistency_level")
+        return normalize_milvus_consistency_level(value)
+
     @property
     def pgvector_dsn(self):
         """Postgres DSN for the opt-in ``pgvector`` backend.
@@ -795,6 +895,102 @@ class MempalaceConfig:
         """Whether the stop hook shows a desktop notification via notify-send."""
         return self._file_config.get("hooks", {}).get("desktop_toast", False)
 
+    def resolve_write_routing(self, scope: str) -> ResolvedWriteRoutingPolicy:
+        """Resolve the configured write policy for ``hooks`` or ``cli``.
+
+        Precedence is:
+
+        1. scope-specific environment variable;
+        2. global environment variable;
+        3. legacy hook environment variable;
+        4. scope-specific config value;
+        5. global config value;
+        6. legacy hook config value;
+        7. ``direct``.
+
+        This foundation does not change current hook or CLI behavior. The
+        policy-aware consumers are introduced by follow-up PRs.
+        """
+
+        normalized_scope = str(scope).strip().lower()
+        env_names = {
+            "hooks": "MEMPALACE_HOOK_WRITE_ROUTING",
+            "cli": "MEMPALACE_CLI_WRITE_ROUTING",
+        }
+
+        if normalized_scope not in env_names:
+            raise WriteRoutingError("write routing scope must be 'hooks' or 'cli'")
+
+        routing_config = self._file_config.get("write_routing", {})
+        if routing_config is None:
+            routing_config = {}
+
+        if not isinstance(routing_config, dict):
+            raise WriteRoutingError("config write_routing must be an object")
+
+        candidates = [
+            RoutingPolicyCandidate(
+                env_names[normalized_scope],
+                os.environ.get(env_names[normalized_scope]),
+            ),
+            RoutingPolicyCandidate(
+                "MEMPALACE_WRITE_ROUTING",
+                os.environ.get("MEMPALACE_WRITE_ROUTING"),
+            ),
+        ]
+
+        if normalized_scope == "hooks":
+            candidates.append(
+                RoutingPolicyCandidate(
+                    "MEMPALACE_HOOKS_DAEMON (legacy)",
+                    os.environ.get("MEMPALACE_HOOKS_DAEMON"),
+                    legacy_boolean=True,
+                )
+            )
+
+        candidates.extend(
+            [
+                RoutingPolicyCandidate(
+                    f"config write_routing.{normalized_scope}",
+                    routing_config.get(normalized_scope),
+                ),
+                RoutingPolicyCandidate(
+                    "config write_routing.default",
+                    routing_config.get("default"),
+                ),
+            ]
+        )
+
+        if normalized_scope == "hooks":
+            hooks_config = self._file_config.get("hooks", {})
+            if hooks_config is None:
+                hooks_config = {}
+
+            if not isinstance(hooks_config, dict):
+                raise WriteRoutingError("config hooks must be an object")
+
+            candidates.append(
+                RoutingPolicyCandidate(
+                    "config hooks.daemon (legacy)",
+                    hooks_config.get("daemon"),
+                    legacy_boolean=True,
+                )
+            )
+
+        return resolve_write_routing_policy(candidates)
+
+    @property
+    def hook_write_routing(self) -> WriteRoutingPolicy:
+        """Resolved future routing policy for hook-triggered writes."""
+
+        return self.resolve_write_routing("hooks").policy
+
+    @property
+    def cli_write_routing(self) -> WriteRoutingPolicy:
+        """Resolved future routing policy for routine CLI writes."""
+
+        return self.resolve_write_routing("cli").policy
+
     @property
     def hook_use_daemon(self):
         """Whether hooks should submit save/mine work to the opt-in daemon."""

mempalace/convo_miner.py

@@ -10,7 +10,9 @@ Same palace as project mining. Different ingest strategy.
 
 import os
 import sys
+import json
 import logging
+import stat
 from pathlib import Path
 from datetime import datetime
 from collections import defaultdict
@@ -19,6 +21,7 @@ from typing import Optional
 from .collision_scan import assert_no_collisions
 from .ids import ID_RECIPE, make_convo_drawer_id, make_convo_sentinel_id
 from .normalize import normalize
+from .entities import entities_metadata
 from .palace import (
     NORMALIZE_VERSION,
     SKIP_DIRS,
@@ -62,6 +65,16 @@ CONVO_EXTENSIONS = {
     ".jsonl",
 }
 
+# Directories inside conversation sources that never hold conversations.
+# ``tool-results``: Claude Code pages large tool outputs to
+# ``<session>/tool-results/*.txt`` inside ``~/.claude/projects/<slug>/``.
+# They are raw machine dumps referenced from the transcript JSONL — mining
+# them stores megabytes of command output as "memories" (field measurement:
+# 12.8k drawers from tool-results files on one palace; a single file
+# produced 3.6k). Extends the generic SKIP_DIRS set for the convo scanner
+# only — project mining semantics are unchanged.
+CONVO_SKIP_DIRS = SKIP_DIRS | {"tool-results"}
+
 MIN_CHUNK_SIZE = 30
 CHUNK_SIZE = 800  # chars per drawer — align with miner.py
 _LINE_GROUP_SIZE = 25  # lines per fallback group when no paragraph breaks
@@ -77,30 +90,67 @@ MAX_FILE_SIZE = 500 * 1024 * 1024  # 500 MB — skip files larger than this.
 # use also scales with source size.
 
 
+def _path_within_root(path: Path, root: Path) -> bool:
+    try:
+        path.expanduser().resolve().relative_to(root.expanduser().resolve())
+        return True
+    except (OSError, ValueError):
+        return False
+
+
+def _is_regular_source_file(filepath: Path, root: Path) -> bool:
+    if not _path_within_root(filepath, root):
+        return False
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    fd = -1
+    try:
+        fd = os.open(filepath, flags)
+        st = os.fstat(fd)
+        return stat.S_ISREG(st.st_mode) and st.st_size <= MAX_FILE_SIZE
+    except OSError:
+        return False
+    finally:
+        if fd != -1:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+
+
 def _register_file(collection, source_file: str, wing: str, agent: str, extract_mode: str):
     """Write a sentinel so file_already_mined() returns True for 0-chunk files.
 
     Without this, files that normalize to nothing or produce zero chunks are
     re-read and re-processed on every mine run because nothing was written to
     ChromaDB on the first pass.
+
+    Stamps source_mtime like every real drawer does, so a file that later
+    grows past the min-chunk-size floor (e.g. a short session that gets
+    extended) is correctly detected as changed on the next mine instead of
+    being skipped forever by this sentinel.
     """
+    try:
+        source_mtime = os.path.getmtime(source_file)
+    except OSError:
+        source_mtime = None
     sentinel_id = make_convo_sentinel_id(source_file, extract_mode)
+    meta = {
+        "wing": wing,
+        "room": "_registry",
+        "source_file": source_file,
+        "added_by": agent,
+        "filed_at": datetime.now().isoformat(),
+        "ingest_mode": "registry",
+        "extract_mode": extract_mode,
+        "normalize_version": NORMALIZE_VERSION,
+        "id_recipe": ID_RECIPE,
+    }
+    if source_mtime is not None:
+        meta["source_mtime"] = source_mtime
     collection.upsert(
         documents=[f"[registry] {source_file}"],
         ids=[sentinel_id],
-        metadatas=[
-            {
-                "wing": wing,
-                "room": "_registry",
-                "source_file": source_file,
-                "added_by": agent,
-                "filed_at": datetime.now().isoformat(),
-                "ingest_mode": "registry",
-                "extract_mode": extract_mode,
-                "normalize_version": NORMALIZE_VERSION,
-                "id_recipe": ID_RECIPE,
-            }
-        ],
+        metadatas=[meta],
     )
 
 
@@ -355,7 +405,7 @@ def scan_convos(convo_dir: str) -> list:
     convo_path = Path(convo_dir).expanduser().resolve()
     files = []
     for root, dirs, filenames in os.walk(convo_path):
-        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+        dirs[:] = [d for d in dirs if d not in CONVO_SKIP_DIRS]
         for filename in filenames:
             if filename.endswith(".meta.json"):
                 continue
@@ -369,10 +419,30 @@ def scan_convos(convo_dir: str) -> list:
                     except OSError:
                         pass
                     continue
+                # Skip files exceeding size limit, or those whose stat() raises
+                # (permission denied, racing delete, broken symlink that
+                # survived the earlier is_symlink check). Both branches log
+                # to stderr to match the SKIP: (symlink) line above; silent
+                # drops at this gate were the original #923 complaint.
                 try:
-                    if filepath.stat().st_size > MAX_FILE_SIZE:
+                    file_size = filepath.stat().st_size
+                    if file_size > MAX_FILE_SIZE:
+                        print(
+                            f"  SKIP: {filepath.name} ({file_size / (1024 * 1024):.1f} MB)"
+                            f" exceeds {MAX_FILE_SIZE // (1024 * 1024)} MB limit",
+                            file=sys.stderr,
+                        )
                         continue
-                except OSError:
+                except OSError as exc:
+                    # Prefer ``exc.strerror`` so the path isn't duplicated in
+                    # the output (see the matching comment in
+                    # ``miner.scan_project``).
+                    print(
+                        f"  SKIP: {filepath.name} (stat error: {exc.strerror or exc})",
+                        file=sys.stderr,
+                    )
+                    continue
+                if not _is_regular_source_file(filepath, convo_path):
                     continue
                 files.append(filepath)
     return files
@@ -383,12 +453,51 @@ def scan_convos(convo_dir: str) -> list:
 # =============================================================================
 
 
-def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extract_mode):
+def _extract_authored_at(filepath):
+    """Most-recent message timestamp in a transcript, used as the drawer's authored date.
+
+    Both Claude Code and Codex JSONL transcripts carry a top-level ISO-8601
+    ``timestamp`` on each line. We take the max so ``authored_at`` reflects when the
+    content was actually written, independent of when it was mined (``filed_at``).
+    This restores chronology: a session from days ago keeps its real date even when
+    re-mined today, instead of every drawer collapsing to ingest time. Returns None
+    for formats without per-line timestamps (e.g. plain ``.md``).
+    """
+    path = Path(filepath)
+    if path.suffix != ".jsonl":
+        return None
+    latest = None
+    try:
+        with path.open(encoding="utf-8", errors="ignore") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    ts = json.loads(line).get("timestamp")
+                except (ValueError, TypeError, AttributeError):
+                    continue
+                # ISO-8601 timestamps are strings; guard against a non-string
+                # ``timestamp`` so a malformed line can't raise TypeError on compare.
+                if isinstance(ts, str) and (latest is None or ts > latest):
+                    latest = ts
+    except OSError:
+        return None
+    return latest
+
+
+def _file_chunks_locked(
+    collection, source_file, chunks, wing, room, agent, extract_mode, authored_at=None
+):
     """Lock the source file, purge stale drawers, and upsert fresh chunks.
 
     Combines the per-file serialization that prevents concurrent agents from
-    duplicating work (via mine_lock) with the normalize-version rebuild
-    contract (purge-before-insert so pre-v2 drawers don't survive).
+    duplicating work (via mine_lock) with the rebuild contract
+    (purge-before-insert so stale drawers never survive) that fires on
+    either a normalize-version bump OR a changed/grown source file (mtime
+    differs from what's stored) -- transcripts are not assumed immutable,
+    since a Claude Code session keeps appending to its own file while
+    active and /compact or /clear can rewrite one in place.
 
     Returns (drawers_added, room_counts_delta, skipped).
     """
@@ -396,14 +505,15 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
     drawers_added = 0
     with mine_lock(source_file):
         # Re-check after lock — another agent may have just finished this file
-        # at the current schema. A stale-version hit here returns False, so we
+        # at the current schema/mtime. A stale hit here returns False, so we
         # still fall through to the purge+rebuild path below.
-        if file_already_mined(collection, source_file, extract_mode=extract_mode):
+        if file_already_mined(collection, source_file, check_mtime=True, extract_mode=extract_mode):
             return 0, room_counts_delta, True
 
-        # Purge stale drawers first. When the normalize schema bumps,
-        # file_already_mined() returned False for pre-v2 drawers — clean
-        # them out so the source doesn't end up with mixed old/new drawers.
+        # Purge stale drawers first. Fires both on a normalize-schema bump
+        # (file_already_mined() returned False for pre-v2 drawers) and on a
+        # changed/grown transcript (mtime differs) — clean them out so the
+        # source doesn't end up with mixed old/new drawers.
         try:
             delete_ids = _source_file_delete_ids(collection, source_file, extract_mode)
             if delete_ids:
@@ -416,6 +526,10 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
         # one filed_at per source file so all transcript drawers share an
         # ingest timestamp.
         filed_at = datetime.now().isoformat()
+        try:
+            source_mtime = os.path.getmtime(source_file)
+        except OSError:
+            source_mtime = None
         for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE):
             batch_docs: list = []
             batch_ids: list = []
@@ -429,21 +543,24 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
                 )
                 batch_docs.append(chunk["content"])
                 batch_ids.append(drawer_id)
-                batch_metas.append(
-                    {
-                        "wing": wing,
-                        "room": chunk_room,
-                        "hall": _detect_hall_cached(chunk["content"]),
-                        "source_file": source_file,
-                        "chunk_index": chunk["chunk_index"],
-                        "added_by": agent,
-                        "filed_at": filed_at,
-                        "ingest_mode": "convos",
-                        "extract_mode": extract_mode,
-                        "normalize_version": NORMALIZE_VERSION,
-                        "id_recipe": ID_RECIPE,
-                    }
-                )
+                meta = {
+                    "wing": wing,
+                    "room": chunk_room,
+                    "hall": _detect_hall_cached(chunk["content"]),
+                    "source_file": source_file,
+                    "chunk_index": chunk["chunk_index"],
+                    "added_by": agent,
+                    "filed_at": filed_at,
+                    "entities": entities_metadata(chunk["content"]),
+                    "authored_at": authored_at if authored_at is not None else filed_at,
+                    "ingest_mode": "convos",
+                    "extract_mode": extract_mode,
+                    "normalize_version": NORMALIZE_VERSION,
+                    "id_recipe": ID_RECIPE,
+                }
+                if source_mtime is not None:
+                    meta["source_mtime"] = source_mtime
+                batch_metas.append(meta)
             assert_no_collisions(list(zip(batch_ids, batch_metas)), collection)
             try:
                 collection.upsert(
@@ -487,6 +604,28 @@ def _is_ai_tool_path(path: Path) -> bool:
     return False
 
 
+def _is_unchanged_since_last_mine(source_file: str, mined_mtimes: dict) -> bool:
+    """True iff source_file was mined at the current schema AND its on-disk
+    mtime still matches what was stored -- the mtime-aware replacement for
+    "we've seen this source_file before" (transcripts are not immutable).
+
+    False (re-mine) whenever the file isn't in mined_mtimes at all, its
+    stored mtime is None (never recorded -- pre-mtime-tracking drawer, or
+    getmtime failed when it was written), or getmtime fails right now
+    (treat as changed rather than silently trusting stale data).
+    """
+    if source_file not in mined_mtimes:
+        return False
+    stored_mtime = mined_mtimes[source_file]
+    if stored_mtime is None:
+        return False
+    try:
+        current_mtime = os.path.getmtime(source_file)
+    except OSError:
+        return False
+    return abs(stored_mtime - current_mtime) < 0.001
+
+
 def _resolve_wing(convo_path: Path, wing: Optional[str]) -> str:
     """Determine the destination wing for ``mine_convos``.
 
@@ -566,6 +705,22 @@ def mine_convos(
         )
 
 
+def _compute_hallways_for_wing_safe(wing, collection, drawers_filed, config=None):
+    """Auto-populate the associative graph from the entities just mined.
+
+    Best-effort: hallway computation must never fail an otherwise-good mine, and is
+    skipped when nothing new was filed.
+    """
+    if drawers_filed <= 0:
+        return
+    try:
+        from .hallways import compute_hallways_for_wing
+
+        compute_hallways_for_wing(wing, col=collection, config=config)
+    except Exception as exc:
+        print(f"  (hallways skipped: {exc})")
+
+
 def _mine_convos_impl(
     convo_dir: str,
     palace_path: str,
@@ -577,7 +732,7 @@ def _mine_convos_impl(
 ):
     from .config import MempalaceConfig
 
-    palace_config = MempalaceConfig()
+    palace_config = MempalaceConfig(palace_path=palace_path)
     cfg_chunk_size = palace_config.chunk_size
     # Only override convo_miner's MIN_CHUNK_SIZE when the user has set
     # min_chunk_size explicitly. min_chunk_size_explicit returns the
@@ -609,13 +764,14 @@ def _mine_convos_impl(
 
     collection = get_collection(palace_path) if not dry_run else None
 
-    # Bulk pre-fetch already-mined set in one paginated pass instead of
-    # `len(files)` separate WHERE-source_file queries. On a 150k-drawer
-    # palace each per-file query costs ~2s, so a 2000-file sweep used to
-    # spend >1h just deciding to skip. prefetch_mined_set() does the same
-    # decisions in a single scan; loop body becomes an O(1) set check.
-    mined_set: set[str] = (
-        prefetch_mined_set(collection, extract_mode=extract_mode) if not dry_run else set()
+    # Bulk pre-fetch already-mined source_file -> stored mtime in one
+    # paginated pass instead of `len(files)` separate WHERE-source_file
+    # queries. On a 150k-drawer palace each per-file query costs ~2s, so a
+    # 2000-file sweep used to spend >1h just deciding to skip.
+    # prefetch_mined_set() does the same decisions in a single scan; loop
+    # body becomes an O(1) dict lookup + a cheap local mtime comparison.
+    mined_mtimes: dict = (
+        prefetch_mined_set(collection, extract_mode=extract_mode) if not dry_run else {}
     )
 
     total_drawers = 0
@@ -628,8 +784,19 @@ def _mine_convos_impl(
         files_processed = i
         source_file = str(filepath)
 
-        # Skip if already filed at current NORMALIZE_VERSION
-        if not dry_run and source_file in mined_set:
+        # Skip only if already filed at the current NORMALIZE_VERSION AND
+        # unchanged on disk since. Transcripts are NOT assumed immutable:
+        # a Claude Code session keeps appending to the same file while
+        # active, and /compact or /clear can rewrite one in place -- so
+        # "we've seen this source_file before" alone is not sufficient.
+        # Falling through re-mines: _file_chunks_locked purges this
+        # source_file's stale drawers before inserting fresh ones, so this
+        # never leaves duplicates behind.
+        if not dry_run and _is_unchanged_since_last_mine(source_file, mined_mtimes):
+            files_skipped += 1
+            continue
+
+        if not _is_regular_source_file(filepath, Path(convo_dir).expanduser().resolve()):
             files_skipped += 1
             continue
 
@@ -697,7 +864,14 @@ def _mine_convos_impl(
         # Lock + purge stale + file fresh chunks. Lock serializes concurrent
         # agents; purge removes pre-v2 drawers so the schema bump applies.
         drawers_added, room_delta, skipped = _file_chunks_locked(
-            collection, source_file, chunks, wing, room, agent, extract_mode
+            collection,
+            source_file,
+            chunks,
+            wing,
+            room,
+            agent,
+            extract_mode,
+            authored_at=_extract_authored_at(filepath),
         )
         if skipped:
             files_skipped += 1
@@ -712,6 +886,10 @@ def _mine_convos_impl(
             break
 
     if not dry_run:
+        # Compute hallways before the FTS5 validation: the latter opens a direct sqlite
+        # connection to the Chroma DB, which can invalidate the live collection handle on
+        # some Chroma builds and make the hallway fetch fail.
+        _compute_hallways_for_wing_safe(wing, collection, total_drawers, config=palace_config)
         _validate_palace_fts5_after_mine(palace_path)
 
     print(f"\n{'=' * 55}")

mempalace/daemon.py

@@ -963,6 +963,21 @@ def get_client_if_running(palace_path: str, *, health_timeout: float = 5.0) -> D
 
 
 def _detached_kwargs(log_path: Path) -> dict[str, Any]:
+    """Kwargs that spawn the daemon with a hidden console, detached from the CLI.
+
+    Shares the Windows flag logic with ``hooks_cli._detached_popen_kwargs``
+    (which takes no args and opens no log file). On Windows we use
+    ``CREATE_NO_WINDOW`` rather than ``DETACHED_PROCESS``: the latter gives the
+    child no console, so a console-subsystem grandchild later allocates a fresh
+    *visible* window (#1783); ``CREATE_NO_WINDOW`` gives a hidden console that
+    descendants inherit instead. Surviving the launching terminal is carried by
+    ``CREATE_BREAKAWAY_FROM_JOB`` (escapes the parent Job Object's kill-on-close)
+    plus the daemon never being attached to that console -- not by the console
+    flag -- while ``CREATE_NEW_PROCESS_GROUP`` isolates Ctrl-C/Ctrl-Break routing.
+    ``CREATE_NO_WINDOW`` is ignored when OR'd with ``DETACHED_PROCESS``, so this
+    replaces that flag rather than adding it. ``stdin=DEVNULL`` and stdout/stderr
+    redirected to the log avoid the #1268 parent hang.
+    """
     log_path.parent.mkdir(parents=True, exist_ok=True)
     log_fh = open(log_path, "a", encoding="utf-8")
     # The daemon log may capture verbatim content in tracebacks — owner-only.
@@ -975,7 +990,7 @@ def _detached_kwargs(log_path: Path) -> dict[str, Any]:
     }
     if os.name == "nt":
         flags = 0
-        for name in ("DETACHED_PROCESS", "CREATE_NEW_PROCESS_GROUP", "CREATE_BREAKAWAY_FROM_JOB"):
+        for name in ("CREATE_NO_WINDOW", "CREATE_NEW_PROCESS_GROUP", "CREATE_BREAKAWAY_FROM_JOB"):
             flags |= getattr(subprocess, name, 0)
         if flags:
             kwargs["creationflags"] = flags

mempalace/entities.py

@@ -0,0 +1,71 @@
+"""No-LLM structural entity extraction for the associative graph.
+
+Pulls deterministic, *structural* tokens from text — author-quoted code spans, URLs,
+file paths, qualified identifiers, and CamelCase symbols — to populate the ``entities``
+drawer-metadata field that hallways/tunnels consume. Structural-only by design: no
+wordlists, no NLP models, no domain vocabulary, so it stays language-neutral and
+predictable, and biases to precision (only tokens that are unambiguously "a thing being
+referred to") over recall.
+
+The output format matches what ``hallways._parse_entities`` expects: a ``;``-joined string.
+"""
+
+import re
+
+# Author-quoted code spans are the highest-signal structural marker: `foo`, `obj.method()`.
+_BACKTICK = re.compile(r"`([^`\n]{2,64})`")
+# URLs.
+_URL = re.compile(r"https?://[^\s)>\]}\"']+")
+# Paths with a separator and a short extension: rag/foo.py, a/b/c.tsx.
+_PATH = re.compile(r"\b[\w.-]+/[\w./-]*\.[A-Za-z][A-Za-z0-9]{0,4}\b")
+# Qualified dotted identifiers; each segment starts with a letter and is >=2 chars, so
+# "1.2.3", "e.g", and "i.e" are excluded: module.func, pkg.Class.method.
+_QUALIFIED = re.compile(r"\b[A-Za-z][A-Za-z0-9_]+(?:\.[A-Za-z][A-Za-z0-9_]+)+\b")
+# CamelCase with >=2 humps — strongly code-specific: ChromaBackend, MemoryStack.
+_CAMEL = re.compile(r"\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b")
+# snake_case (must contain an underscore, so it can't match plain English): do_thing,
+# _extract_authored_at. The optional leading/trailing `_?` matches dunder-style names
+# whose underscore would otherwise fall outside the `\b` boundary (`_` is a word char).
+_SNAKE = re.compile(r"\b_?[a-z][a-z0-9]*(?:_[a-z0-9]+)+_?\b")
+
+_MAX_ENTITIES = 24
+_MIN_LEN = 2
+_MAX_LEN = 64
+
+
+def _clean(token):
+    # `;` is the entities-metadata separator, so it must never survive inside an entity
+    # (e.g. a URL query string or a backtick span) or it would split the field.
+    return token.replace(";", " ").strip().strip("`.,:()[]{}<>\"'").strip()
+
+
+def extract_structural_entities(text, max_entities=_MAX_ENTITIES):
+    """Return up to ``max_entities`` structural entities from ``text``.
+
+    Deterministic and order-stable: entities are ranked by occurrence count (ties broken
+    by first appearance), deduplicated case-insensitively, preserving the first-seen
+    surface form.
+    """
+    if not text:
+        return []
+    counts = {}
+    order = {}
+    seq = 0
+    for pattern in (_BACKTICK, _URL, _PATH, _QUALIFIED, _CAMEL, _SNAKE):
+        for match in pattern.finditer(text):
+            token = _clean(match.group(1) if pattern is _BACKTICK else match.group(0))
+            if not (_MIN_LEN <= len(token) <= _MAX_LEN):
+                continue
+            key = token.lower()
+            if key not in counts:
+                counts[key] = 0
+                order[key] = (seq, token)
+                seq += 1
+            counts[key] += 1
+    ranked = sorted(order, key=lambda k: (-counts[k], order[k][0]))
+    return [order[k][1] for k in ranked[:max_entities]]
+
+
+def entities_metadata(text, max_entities=_MAX_ENTITIES):
+    """``;``-joined entity string for drawer metadata, or ``""`` when none are found."""
+    return ";".join(extract_structural_entities(text, max_entities=max_entities))

mempalace/entity_detector.py

@@ -215,6 +215,8 @@ PROSE_EXTENSIONS = {
     ".md",
     ".rst",
     ".csv",
+    ".tex",
+    ".bib",
 }
 
 READABLE_EXTENSIONS = {

mempalace/format_miner.py

@@ -750,7 +750,7 @@ def mine_formats(
     # min_chunk_size) are now threaded through chunk_text below, so users
     # who customized their config see the effect in format-mode mining.
     # Per PR #1555 review (Gemini #3).
-    palace_config = MempalaceConfig()
+    palace_config = MempalaceConfig(palace_path=palace_path)
 
     format_path = Path(format_dir).expanduser().resolve()
     if not wing:
@@ -945,7 +945,7 @@ def mine_formats(
         # skipped quietly.
         if not dry_run:
             try:
-                tunnels_added = _compute_topic_tunnels_for_wing(wing)
+                tunnels_added = _compute_topic_tunnels_for_wing(wing, config=palace_config)
                 if tunnels_added:
                     print(f"\n  Topic tunnels: +{tunnels_added} cross-wing link(s)")
             except Exception as exc:

mempalace/hallways.py

@@ -199,6 +199,7 @@ def compute_hallways_for_wing(
     wing: str,
     col=None,
     min_count: int = 2,
+    config=None,
 ) -> list[dict]:
     """Compute entity-pair hallways for one wing.
 
@@ -229,6 +230,10 @@ def compute_hallways_for_wing(
             hallway between two entities. Default 2 — single co-occurrences
             are noise (entities mentioned together once in one drawer);
             two or more is a real signal. Clamped to ``>=1``.
+        config: Optional ``MempalaceConfig`` selecting the palace-scoped
+            hallway sidecar. Callers using an explicit palace path must pass
+            the matching config so derived graph state cannot leak into the
+            default palace.
 
     Returns:
         List of hallway dicts created for this wing. Records for other
@@ -310,7 +315,7 @@ def compute_hallways_for_wing(
     #    across recomputes. Without this preservation, every mine wipes
     #    the connection weights accumulated through use — defeating the
     #    living-connection layer entirely.
-    existing = _load_hallways()
+    existing = _load_hallways(config)
     existing_dynamics_lookup: dict = {}
     for h in existing:
         if h.get("wing") != wing:
@@ -360,7 +365,7 @@ def compute_hallways_for_wing(
 
     # 4. Persist — preserve other-wing records, replace this wing's records.
     preserved_other_wings = [h for h in existing if h.get("wing") != wing]
-    _save_hallways(preserved_other_wings + created)
+    _save_hallways(preserved_other_wings + created, config)
 
     return created
 
@@ -370,19 +375,19 @@ def compute_hallways_for_wing(
 # ─────────────────────────────────────────────────────────────────────────────
 
 
-def list_hallways(wing: Optional[str] = None) -> list[dict]:
+def list_hallways(wing: Optional[str] = None, config=None) -> list[dict]:
     """List hallway records. Filter by ``wing`` if specified."""
-    all_hallways = _load_hallways()
+    all_hallways = _load_hallways(config)
     if wing is None:
         return list(all_hallways)
     return [h for h in all_hallways if h.get("wing") == wing]
 
 
-def delete_hallway(hallway_id: str) -> bool:
+def delete_hallway(hallway_id: str, config=None) -> bool:
     """Remove one hallway record by id. Returns True if a record was removed."""
-    hallways = _load_hallways()
+    hallways = _load_hallways(config)
     filtered = [h for h in hallways if h.get("id") != hallway_id]
     if len(filtered) == len(hallways):
         return False
-    _save_hallways(filtered)
+    _save_hallways(filtered, config)
     return True

mempalace/hooks_cli.py

@@ -660,6 +660,11 @@ def _mine_sync():
                     stdout=log_f,
                     stderr=log_f,
                     timeout=60,
+                    # Windows: hide the conhost window this sync mine would
+                    # otherwise flash on every fire. Mirrors the async paths
+                    # (_spawn_mine / _desktop_toast) via _detached_popen_kwargs().
+                    # 0 is a no-op off-Windows.
+                    creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
                 )
         except (OSError, subprocess.TimeoutExpired):
             pass
@@ -851,8 +856,13 @@ def _save_diary_direct(
 
 def _ingest_transcript(transcript_path: str):
     """Mine a Claude Code session transcript into the palace as a conversation."""
-    path = Path(transcript_path).expanduser()
-    if not path.is_file() or path.stat().st_size < 100:
+    path = _validate_transcript_path(transcript_path)
+    if path is None:
+        return
+    try:
+        if not path.is_file() or path.stat().st_size < 100:
+            return
+    except OSError:
         return
 
     try:
@@ -951,6 +961,26 @@ _ENCODED_PARENT_PREFIXES = (
 )
 
 
+def _safe_wing_slug(name: str) -> str:
+    """Normalize a project directory name into a wing slug ``sanitize_name`` accepts.
+
+    Builds on the historical space/hyphen handling: map characters outside
+    ``sanitize_name``'s set to ``_`` while keeping ``.`` and ``'`` so existing wings
+    for names like ``my.app`` are preserved (renaming a wing would orphan diary
+    entries already filed under it), collapse ``..`` which the validator rejects as
+    path traversal, trim edge separators it won't accept, and cap the length so
+    ``wing_<slug>`` stays within ``sanitize_name``'s 128-character limit. Without
+    this a folder containing e.g. a leading ``+`` produced ``wing_+project``, which
+    ``sanitize_name`` rejects — silently breaking diary auto-save. Falls back to
+    ``sessions`` when a name reduces to nothing (e.g. ``+``).
+    """
+    slug = name.lower().replace(" ", "_").replace("-", "_")
+    slug = re.sub(r"[^\w.']+", "_", slug)
+    slug = re.sub(r"\.{2,}", ".", slug)
+    slug = slug[:120].strip("_.'")
+    return slug or "sessions"
+
+
 def _wing_from_jsonl_cwd(transcript_path: str) -> Optional[str]:
     """Read ``cwd`` from the first JSONL line that records it.
 
@@ -984,8 +1014,7 @@ def _wing_from_jsonl_cwd(transcript_path: str) -> Optional[str]:
                     continue
                 project = cwd_norm.rsplit("/", 1)[-1]
                 if project:
-                    slug = project.lower().replace(" ", "_").replace("-", "_")
-                    return f"wing_{slug}"
+                    return f"wing_{_safe_wing_slug(project)}"
     except OSError:
         pass
     return None
@@ -1042,15 +1071,12 @@ def _wing_from_transcript_path(transcript_path: str) -> str:
             if encoded.startswith(prefix):
                 encoded = encoded[len(prefix) :]
                 break
-        project = encoded.lower().replace(" ", "_").replace("-", "_")
-        if project:
-            return f"wing_{project}"
+        return f"wing_{_safe_wing_slug(encoded)}"
 
     # 3. Legacy — explicit -Projects-<name> segment
     match = re.search(r"-Projects-([^/]+?)(?:/|$)", normalized)
     if match:
-        project = match.group(1).lower().replace(" ", "_").replace("-", "_")
-        return f"wing_{project}"
+        return f"wing_{_safe_wing_slug(match.group(1))}"
 
     # 4. Default
     return "wing_sessions"

mempalace/knowledge_graph.py

@@ -39,7 +39,7 @@ import json
 import os
 import sqlite3
 import threading
-from datetime import date, datetime
+from datetime import date, datetime, timezone
 from pathlib import Path
 from typing import Optional
 from .config import sanitize_iso_temporal
@@ -113,6 +113,14 @@ def _temporal_filter_sql(as_of: str) -> tuple[str, list[str]]:
 
     This keeps legacy date-only facts working when callers query with
     canonical UTC datetimes such as '2026-05-06T15:00:00Z'.
+
+    The upper bound is *strict* (``valid_to > as_of``): a fact whose
+    ``valid_to`` equals the query instant has already ended at that instant,
+    so the interval is treated as half-open ``[valid_from, valid_to)``. This
+    is what lets a fact and its successor share a boundary instant without an
+    as-of query returning both. Date-only ``valid_to`` still expands to the
+    end of that day (``T23:59:59Z``), so a standalone date-only fact stays
+    valid through its whole final day exactly as before.
     """
 
     as_of_key = _temporal_start_key(as_of)
@@ -121,7 +129,7 @@ def _temporal_filter_sql(as_of: str) -> tuple[str, list[str]]:
 
     return (
         f" AND (t.valid_from IS NULL OR {valid_from_expr} <= ?) "
-        f"AND (t.valid_to IS NULL OR {valid_to_expr} >= ?)",
+        f"AND (t.valid_to IS NULL OR {valid_to_expr} > ?)",
         [as_of_key, as_of_key],
     )
 
@@ -359,6 +367,125 @@ class KnowledgeGraph:
                     (ended, sub_id, pred, obj_id),
                 )
 
+    def supersede(
+        self,
+        subject: str,
+        predicate: str,
+        old_obj: str,
+        new_obj: str,
+        at: str = None,
+        confidence: float = 1.0,
+        source_closet: str = None,
+        source_file: str = None,
+        source_drawer_id: str = None,
+        adapter_name: str = None,
+    ):
+        """Atomically replace one fact with another at a single shared boundary.
+
+        Closes the currently-open ``(subject, predicate, old_obj)`` triple with
+        ``valid_to = at`` and opens ``(subject, predicate, new_obj)`` with
+        ``valid_from = at`` in one transaction, at a single shared instant.
+        Paired with the half-open upper bound in ``_temporal_filter_sql``, an
+        as-of query at that instant returns only the successor.
+
+        This is the primitive for a value change. Hand-rolling a handover as
+        ``invalidate(ended=D)`` + ``add_triple(valid_from=D)`` with date-only
+        ``D`` leaves two facts sharing the whole day ``D`` (``valid_to`` expands
+        to ``T23:59:59Z`` while ``valid_from`` expands to ``T00:00:00Z``), so an
+        as-of query on ``D`` returns both. ``supersede`` avoids this by writing
+        one identical precise instant to both sides.
+
+        ``at`` defaults to the current UTC instant. A date-only ``at`` is
+        normalized to ``<date>T00:00:00Z`` so both sides carry the same precise
+        value rather than the asymmetric whole-day expansion.
+
+        Returns the new triple's id. If no open ``old_obj`` triple exists the
+        successor is still opened, so ``supersede`` degrades to ``add_triple``.
+        """
+        if at is None:
+            boundary = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        elif _is_date_only_temporal(at):
+            boundary = f"{at}T00:00:00Z"
+        else:
+            boundary = at
+        boundary = sanitize_iso_temporal(boundary, "at")
+
+        sub_id = self._entity_id(subject)
+        old_id = self._entity_id(old_obj)
+        new_id = self._entity_id(new_obj)
+        pred = predicate.lower().replace(" ", "_")
+
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                # Only create entities we actually open a fact for. old_obj is
+                # matched by id in the UPDATE below whether or not its row
+                # exists, so inserting it would just orphan an entity when no
+                # open old fact is present (the degrade-to-add path).
+                for name, eid in ((subject, sub_id), (new_obj, new_id)):
+                    conn.execute(
+                        "INSERT OR IGNORE INTO entities (id, name) VALUES (?, ?)",
+                        (eid, name),
+                    )
+
+                # Reject a boundary that precedes the old fact's start — an
+                # inverted interval would be invisible to every KG query.
+                rows = conn.execute(
+                    "SELECT valid_from FROM triples "
+                    "WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
+                    (sub_id, pred, old_id),
+                ).fetchall()
+                for row in rows:
+                    valid_from = row["valid_from"]
+                    if valid_from is not None and _temporal_end_key(boundary) < _temporal_start_key(
+                        valid_from
+                    ):
+                        raise ValueError(
+                            f"at={boundary!r} is before valid_from={valid_from!r}; "
+                            "an inverted interval would be invisible to every KG query"
+                        )
+
+                # Close the open old fact at the shared boundary.
+                conn.execute(
+                    "UPDATE triples SET valid_to=? "
+                    "WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
+                    (boundary, sub_id, pred, old_id),
+                )
+
+                # Open the successor at the same instant (idempotent if already open).
+                existing = conn.execute(
+                    "SELECT id FROM triples "
+                    "WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
+                    (sub_id, pred, new_id),
+                ).fetchone()
+                if existing:
+                    return existing["id"]
+
+                triple_id = make_triple_id(
+                    sub_id, pred, new_id, boundary, datetime.now().isoformat()
+                )
+                conn.execute(
+                    """INSERT INTO triples (
+                        id, subject, predicate, object, valid_from, valid_to,
+                        confidence, source_closet, source_file,
+                        source_drawer_id, adapter_name
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                    (
+                        triple_id,
+                        sub_id,
+                        pred,
+                        new_id,
+                        boundary,
+                        None,
+                        confidence,
+                        source_closet,
+                        source_file,
+                        source_drawer_id,
+                        adapter_name,
+                    ),
+                )
+                return triple_id
+
     # ── Query operations ──────────────────────────────────────────────────
 
     def query_entity(self, name: str, as_of: str = None, direction: str = "outgoing"):

mempalace/layers.py

@@ -126,12 +126,21 @@ class Layer1:
         if not docs:
             return "## L1 — No memories yet."
 
-        # Score each drawer: prefer high importance, recent filing
+        # Score each drawer: prefer high importance, then most-recent filing.
+        # NOTE: the ingest pipeline (miner, convo_miner, diary, add_drawer)
+        # records provenance metadata — wing/room/source/chunk/filed_at — but
+        # never an evaluative importance/weight field. So `importance` is
+        # absent on virtually every drawer and ties at the default, which used
+        # to collapse the sort to insertion order (oldest first). `filed_at`
+        # is present on every drawer, so it is the *effective* ordering signal:
+        # newest first. This keeps importance as the primary key for the day a
+        # scoring pass populates it, while making the "recent filing" half of
+        # the promise true today with data we already have.
         scored = []
         for doc, meta in zip(docs, metas):
             meta = meta or {}
             doc = doc or ""
-            importance = 3
+            importance = 3.0
             # Try multiple metadata keys that might carry weight info
             for key in ("importance", "emotional_weight", "weight"):
                 val = meta.get(key)
@@ -141,11 +150,15 @@ class Layer1:
                     except (ValueError, TypeError):
                         pass
                     break
-            scored.append((importance, meta, doc))
+            # filed_at is an ISO-8601 string; ISO strings sort lexicographically
+            # in chronological order. Coerce to str so a missing/odd value sorts
+            # oldest rather than raising during the comparison.
+            recency = str(meta.get("filed_at") or "")
+            scored.append((importance, recency, meta, doc))
 
-        # Sort by importance descending, take top N
-        scored.sort(key=lambda x: x[0], reverse=True)
-        top = scored[: self.MAX_DRAWERS]
+        # Sort by importance desc, then recency (filed_at) desc; take top N.
+        scored.sort(key=lambda x: (x[0], x[1]), reverse=True)
+        top = [(imp, meta, doc) for imp, _recency, meta, doc in scored[: self.MAX_DRAWERS]]
 
         # Group by room for readability
         by_room = defaultdict(list)
@@ -306,6 +319,9 @@ class Layer3:
             lines.append(f"      {snippet}")
             if source:
                 lines.append(f"      src: {source}")
+            authored = (meta.get("authored_at") or "")[:10]
+            if authored:
+                lines.append(f"      authored: {authored}")
 
         return "\n".join(lines)
 

mempalace/llm_client.py

@@ -323,7 +323,7 @@ class OpenAICompatProvider(LLMProvider):
         base = base.removesuffix("/chat/completions").removesuffix("/v1")
         try:
             req = Request(f"{base}/v1/models")
-            if self.api_key:
+            if self.api_key and (self.api_key_source != "env" or not self.is_external_service):
                 req.add_header("Authorization", f"Bearer {self.api_key}")
             with urlopen(req, timeout=5):
                 pass

mempalace/migrate.py

@@ -295,7 +295,7 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     backup_path = f"{palace_path}.pre-migrate.{timestamp}"
     print(f"\n  Backing up to {backup_path}...")
-    shutil.copytree(palace_path, backup_path)
+    shutil.copytree(palace_path, backup_path, symlinks=True)
 
     # Enforce backup retention so repeated migrations cannot fill the disk
     # with full-palace copies. The backup we just created is the newest, so

mempalace/miner.py

@@ -14,6 +14,7 @@ import shlex
 import hashlib
 import fnmatch
 import logging
+import stat
 from pathlib import Path
 from datetime import datetime
 from collections import defaultdict
@@ -47,6 +48,38 @@ from .ids import ID_RECIPE, make_drawer_id_from_chunk
 
 logger = logging.getLogger("mempalace_mcp")
 
+
+def _path_within_root(path: Path, root: Path) -> bool:
+    try:
+        path.expanduser().resolve().relative_to(root.expanduser().resolve())
+        return True
+    except (OSError, ValueError):
+        return False
+
+
+def _read_text_no_follow(filepath: Path, root: Path) -> Optional[str]:
+    if not _path_within_root(filepath, root):
+        return None
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    fd = -1
+    try:
+        fd = os.open(filepath, flags)
+        st = os.fstat(fd)
+        if not stat.S_ISREG(st.st_mode) or st.st_size > MAX_FILE_SIZE:
+            return None
+        with os.fdopen(fd, "r", encoding="utf-8", errors="replace") as f:
+            fd = -1
+            return f.read()
+    except OSError:
+        return None
+    finally:
+        if fd != -1:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+
+
 PHP_EXTENSIONS = {
     # Compound Blade templates such as ``view.blade.php`` are covered by the
     # final ``.php`` suffix.
@@ -100,6 +133,8 @@ READABLE_EXTENSIONS = {
     ".csv",
     ".sql",
     ".toml",
+    ".tex",
+    ".bib",
     # C# / .NET
     ".cs",
     ".csproj",
@@ -209,20 +244,16 @@ class GitignoreMatcher:
         self.base_dir = base_dir
         self.rules = rules
 
-    @classmethod
-    def from_dir(cls, dir_path: Path):
-        gitignore_path = dir_path / ".gitignore"
-        if not gitignore_path.is_file():
-            return None
-
-        try:
-            lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines()
-        except Exception:
-            return None
+    @staticmethod
+    def _parse_rules(lines):
+        """Parse an iterable of raw pattern strings into a list of rule dicts.
 
+        Shared by :meth:`from_dir` and :meth:`from_patterns` so both paths
+        stay in sync with the gitignore parsing semantics.
+        """
         rules = []
         for raw_line in lines:
-            line = raw_line.strip()
+            line = str(raw_line).strip()
             if not line:
                 continue
 
@@ -254,12 +285,46 @@ class GitignoreMatcher:
                     "negated": negated,
                 }
             )
+        return rules
 
-        if not rules:
+    @classmethod
+    def from_dir(cls, dir_path: Path):
+        gitignore_path = dir_path / ".gitignore"
+        if not gitignore_path.is_file():
             return None
 
+        try:
+            lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines()
+        except Exception:
+            return None
+
+        rules = cls._parse_rules(lines)
+        if not rules:
+            return None
         return cls(dir_path, rules)
 
+    @classmethod
+    def from_patterns(cls, base_dir: Path, patterns):
+        """Create a matcher from an explicit list of gitignore-inspired pattern strings.
+
+        Supports a gitignore-inspired subset of pattern syntax for
+        ``exclude_patterns`` in ``mempalace.yaml``.  Patterns are parsed by
+        the same rule parser used for ``.gitignore`` files, so familiar
+        constructs (trailing ``/`` for dir-only, leading ``/`` for anchoring,
+        ``!`` negation, ``**`` globs) work as expected.  Full gitignore
+        semantics are not guaranteed for every edge case.
+
+        ``patterns`` must be a list of strings.  A single string is coerced
+        to a one-element list for convenience.  Non-string entries are
+        converted via ``str()``.
+        """
+        if isinstance(patterns, str):
+            patterns = [patterns]
+        rules = cls._parse_rules(patterns)
+        if not rules:
+            return None
+        return cls(base_dir, rules)
+
     def matches(self, path: Path, is_dir: bool = None):
         try:
             relative = path.relative_to(self.base_dir).as_posix().strip("/")
@@ -387,6 +452,29 @@ def is_force_included(path: Path, project_path: Path, include_paths: set) -> boo
     return False
 
 
+def _apply_exclude_patterns_to_prescanned_files(
+    files: list, project_path: Path, exclude_patterns: list, include_ignored: list
+) -> list:
+    """Filter a caller-provided (already-scanned) file list by exclude_patterns.
+
+    scan_project() applies exclude_patterns itself, but a caller that passes
+    its own pre-scanned ``files`` list bypasses that -- so callers reusing a
+    scan (e.g. the init double-scan) still need per-project exclusions
+    applied here. force_include paths are preserved, matching scan_project's
+    own precedence.
+    """
+    include_paths = normalize_include_paths(include_ignored)
+    exclude_matcher = GitignoreMatcher.from_patterns(project_path, exclude_patterns)
+    if exclude_matcher is None:
+        return files
+    return [
+        file_path
+        for file_path in files
+        if is_force_included(file_path, project_path, include_paths)
+        or exclude_matcher.matches(file_path, is_dir=False) is not True
+    ]
+
+
 # =============================================================================
 # CONFIG
 # =============================================================================
@@ -1341,9 +1429,8 @@ def process_file(
     if not dry_run and file_already_mined(collection, source_file, check_mtime=True):
         return 0, "general", None
 
-    try:
-        content = filepath.read_text(encoding="utf-8", errors="replace")
-    except OSError:
+    content = _read_text_no_follow(filepath, project_path)
+    if content is None:
         return 0, "general", None
 
     content = content.strip()
@@ -1497,6 +1584,7 @@ def scan_project(
     project_dir: str,
     respect_gitignore: bool = True,
     include_ignored: list = None,
+    exclude_patterns: list = None,
 ) -> list:
     """Return list of all readable file paths under ``project_dir``.
 
@@ -1509,6 +1597,7 @@ def scan_project(
     active_matchers = []
     matcher_cache = {}
     include_paths = normalize_include_paths(include_ignored)
+    exclude_matcher = GitignoreMatcher.from_patterns(project_path, exclude_patterns or [])
 
     for root, dirs, filenames in os.walk(project_path):
         root_path = Path(root)
@@ -1536,6 +1625,13 @@ def scan_project(
                 if is_force_included(root_path / d, project_path, include_paths)
                 or not is_gitignored(root_path / d, active_matchers, is_dir=True)
             ]
+        if exclude_matcher:
+            dirs[:] = [
+                d
+                for d in dirs
+                if is_force_included(root_path / d, project_path, include_paths)
+                or exclude_matcher.matches(root_path / d, is_dir=True) is not True
+            ]
 
         for filename in filenames:
             filepath = root_path / filename
@@ -1549,6 +1645,15 @@ def scan_project(
             if respect_gitignore and active_matchers and not force_include:
                 if is_gitignored(filepath, active_matchers, is_dir=False):
                     continue
+            # Skip files matching project-level exclude_patterns (mempalace.yaml).
+            # force_include paths bypass this check so callers can selectively
+            # re-include files that would otherwise be excluded.
+            if (
+                not force_include
+                and exclude_matcher
+                and exclude_matcher.matches(filepath, is_dir=False)
+            ):
+                continue
             # Skip symlinks — prevents following links to /dev/urandom, etc.
             if filepath.is_symlink():
                 rel = filepath.relative_to(project_path).as_posix()
@@ -1557,11 +1662,30 @@ def scan_project(
                 except OSError:
                     pass
                 continue
-            # Skip files exceeding size limit
+            # Skip files exceeding size limit, or those whose stat() raises
+            # (permission denied, racing delete, broken symlink that survived
+            # the earlier is_symlink check). Both branches log to stderr to
+            # match the SKIP: (symlink) line above; silent drops at this
+            # gate were the original #923 complaint.
             try:
-                if filepath.stat().st_size > MAX_FILE_SIZE:
+                file_size = filepath.stat().st_size
+                if file_size > MAX_FILE_SIZE:
+                    print(
+                        f"  SKIP: {filepath.name} ({file_size / (1024 * 1024):.1f} MB)"
+                        f" exceeds {MAX_FILE_SIZE // (1024 * 1024)} MB limit",
+                        file=sys.stderr,
+                    )
                     continue
-            except OSError:
+            except OSError as exc:
+                # Prefer ``exc.strerror`` so the path isn't duplicated in the
+                # output (PermissionError stringifies to
+                # ``[Errno 13] Permission denied: '<path>'`` and the path is
+                # already in the SKIP prefix). Falls back to the default repr
+                # when strerror is unset.
+                print(
+                    f"  SKIP: {filepath.name} (stat error: {exc.strerror or exc})",
+                    file=sys.stderr,
+                )
                 continue
             files.append(filepath)
     return files
@@ -1653,13 +1777,20 @@ def _mine_impl(
 
     wing = wing_override or config["wing"]
     rooms = config.get("rooms", [{"name": "general", "description": "All project files"}])
+    exclude_patterns = config.get("exclude_patterns", [])
 
     if files is None:
         files = scan_project(
             project_dir,
             respect_gitignore=respect_gitignore,
             include_ignored=include_ignored,
+            exclude_patterns=exclude_patterns,
         )
+    elif exclude_patterns:
+        files = _apply_exclude_patterns_to_prescanned_files(
+            files, project_path, exclude_patterns, include_ignored
+        )
+
     from .embedding import describe_device
 
     print(f"\n{'=' * 55}")
@@ -1742,12 +1873,15 @@ def _mine_impl(
                     break
 
         if not dry_run:
+            from .config import MempalaceConfig
+
+            graph_config = MempalaceConfig(palace_path=palace_path)
             # Cross-wing topic tunnels: after every file in this wing has been
             # processed, link this wing to any other wing that shares a
             # confirmed TOPIC label. Out of scope for v1: manifest-dependency
             # overlap, per-topic allow/deny lists, search-result surfacing.
             try:
-                tunnels_added = _compute_topic_tunnels_for_wing(wing)
+                tunnels_added = _compute_topic_tunnels_for_wing(wing, config=graph_config)
                 if tunnels_added:
                     print(f"\n  Topic tunnels: +{tunnels_added} cross-wing link(s)")
             except Exception as e:
@@ -1763,7 +1897,9 @@ def _mine_impl(
             # must never fail a mine; it's a derived analytic, not load-bearing
             # for the drawer write that already committed above.
             try:
-                hallways_created = compute_hallways_for_wing(wing, col=collection)
+                hallways_created = compute_hallways_for_wing(
+                    wing, col=collection, config=graph_config
+                )
                 if hallways_created:
                     print(f"\n  Hallways: +{len(hallways_created)} within-wing entity link(s)")
             except Exception as e:
@@ -1779,7 +1915,7 @@ def _mine_impl(
             # ``kind="entity"`` / ``kind="topic"``. Same fault-tolerance
             # pattern: never fail a mine over a derived analytic.
             try:
-                entity_tunnels_added = _compute_entity_tunnels_for_wing(wing)
+                entity_tunnels_added = _compute_entity_tunnels_for_wing(wing, config=graph_config)
                 if entity_tunnels_added:
                     print(f"\n  Entity tunnels: +{entity_tunnels_added} cross-wing entity link(s)")
             except Exception as e:
@@ -1903,7 +2039,7 @@ def _cleanup_mine_pid_file() -> None:
         pass
 
 
-def _compute_topic_tunnels_for_wing(wing: str) -> int:
+def _compute_topic_tunnels_for_wing(wing: str, config=None) -> int:
     """Drop tunnels between ``wing`` and every other wing that shares
     confirmed topics, honoring the ``topic_tunnel_min_count`` config knob.
 
@@ -1916,13 +2052,13 @@ def _compute_topic_tunnels_for_wing(wing: str) -> int:
     topics_map = get_topics_by_wing()
     if not topics_map or wing not in topics_map:
         return 0
-    cfg = MempalaceConfig()
+    cfg = config or MempalaceConfig()
     min_count = cfg.topic_tunnel_min_count
-    created = topic_tunnels_for_wing(wing, topics_map, min_count=min_count)
+    created = topic_tunnels_for_wing(wing, topics_map, min_count=min_count, config=cfg)
     return len(created)
 
 
-def _compute_entity_tunnels_for_wing(wing: str) -> int:
+def _compute_entity_tunnels_for_wing(wing: str, config=None) -> int:
     """Drop tunnels between ``wing`` and every other wing that shares an
     entity via the within-wing hallway primitive.
 
@@ -1940,10 +2076,10 @@ def _compute_entity_tunnels_for_wing(wing: str) -> int:
     from .hallways import list_hallways
     from .palace_graph import entity_tunnels_for_wing
 
-    hallways = list_hallways()
+    hallways = list_hallways(config=config)
     if not hallways:
         return 0
-    created = entity_tunnels_for_wing(wing, hallways)
+    created = entity_tunnels_for_wing(wing, hallways, config=config)
     return len(created)
 
 

mempalace/normalize.py

@@ -21,6 +21,7 @@ No API key. No internet. Everything local.
 import json
 import os
 import re
+import stat
 from pathlib import Path
 from typing import Optional
 
@@ -118,17 +119,28 @@ def normalize(filepath: str) -> str:
     Load a file and normalize to transcript format if it's a chat export.
     Plain text files pass through unchanged.
     """
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    if os.path.islink(filepath):
+        raise IOError(f"Could not read {filepath}: symlinked files are skipped")
+    fd = -1
     try:
-        file_size = os.path.getsize(filepath)
-    except OSError as e:
-        raise IOError(f"Could not read {filepath}: {e}") from e
-    if file_size > 500 * 1024 * 1024:  # 500 MB safety limit
-        raise IOError(f"File too large ({file_size // (1024 * 1024)} MB): {filepath}")
-    try:
-        with open(filepath, "r", encoding="utf-8-sig", errors="replace") as f:
+        fd = os.open(filepath, flags)
+        file_stat = os.fstat(fd)
+        if not stat.S_ISREG(file_stat.st_mode):
+            raise IOError(f"Could not read {filepath}: not a regular file")
+        if file_stat.st_size > 500 * 1024 * 1024:  # 500 MB safety limit
+            raise IOError(f"File too large ({file_stat.st_size // (1024 * 1024)} MB): {filepath}")
+        with os.fdopen(fd, "r", encoding="utf-8-sig", errors="replace") as f:
+            fd = -1
             content = f.read()
     except OSError as e:
         raise IOError(f"Could not read {filepath}: {e}") from e
+    finally:
+        if fd != -1:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
 
     if not content.strip():
         return content

mempalace/palace.py

@@ -928,12 +928,19 @@ def _validate_palace_fts5_after_mine(palace_path: str) -> None:
     Reuses the same primitive that `cmd_repair` already runs as preflight so the
     operator sees the same recovery banner regardless of which command surfaces
     the bug.
+
+    An isolated FTS5 inverted-index corruption (the common case after a
+    killed-mid-write mine, #1596) is auto-healed in place first via the same
+    `maybe_autoheal_fts5_index` rebuild `cmd_repair` already runs as its own
+    preflight step — so a normal `mine` self-heals the recoverable case
+    instead of forcing the operator to run `mempalace repair` by hand for a
+    derived index that regenerates from intact content.
     """
     if resolve_backend_name(palace_path) != "chroma":
         return
 
     # Defer-import: keeps the repair module graph out of mine's hot import path.
-    from .repair import _close_chroma_handles, sqlite_integrity_errors
+    from .repair import _close_chroma_handles, maybe_autoheal_fts5_index, sqlite_integrity_errors
 
     # Pass the live singleton so the writer's cached PersistentClient actually
     # gets closed and WAL flushes before the read-only sqlite3 re-open.
@@ -943,49 +950,89 @@ def _validate_palace_fts5_after_mine(palace_path: str) -> None:
     _close_chroma_handles(palace_path, backend=_DEFAULT_BACKEND)
 
     errors = sqlite_integrity_errors(palace_path)
+    if errors:
+        # progress=logger.info, not the default print: this runs inside the
+        # MCP server process too (mcp_server.tool_mine -> miner.mine), where
+        # stdout is the JSON-RPC transport -- a stray print() here would
+        # corrupt the protocol stream and crash the connection.
+        errors = maybe_autoheal_fts5_index(palace_path, errors, progress=logger.info)
     if errors:
         raise MineValidationError(palace_path, errors)
 
 
-# Per-thread record of palaces this thread already holds the lock for. Used by
-# `mine_palace_lock` to short-circuit re-entrant acquisition from the same
-# thread (e.g. miner.mine() acquires the outer lock then calls
+# Process-wide record of palaces this PROCESS already holds the lock for. Used
+# by `mine_palace_lock` to short-circuit re-entrant acquisition from the same
+# process (e.g. miner.mine() acquires the outer lock then calls
 # ChromaCollection.upsert which now also tries to acquire). Without this guard
 # the inner call would block on its own outer flock (Linux fcntl locks are per
-# open file description, so a same-thread second open of the lock file is a
-# distinct lock and self-deadlocks).
+# open file description, so a second open of the lock file from the same process
+# is a distinct lock and self-conflicts / EWOULDBLOCKs).
 #
-# The holder set is tagged with ``pid`` so that a forked child does NOT
-# inherit re-entrant credit from its parent: the OS-level flock IS NOT
-# inherited as a "we hold it" semantically — the child must reacquire — but
-# Python's ``threading.local`` IS inherited across fork. The pid check
-# clears stale state so a forked child correctly hits the fcntl path.
-_palace_lock_holders = threading.local()
+# This MUST be process-wide, not thread-local: the MCP HTTP transport
+# (ThreadingHTTPServer) acquires the long-lived writer-lease on one thread
+# (`mcp_server._acquire_mcp_writer_lock`) but dispatches each write request on a
+# different worker thread. A thread-local guard makes those handlers fail to see
+# the process-held lease, re-acquire the flock, and self-conflict
+# ("palace ... is held by PID <self>"). flock is per-process and HTTP writes are
+# serialized by `_HTTP_REQUEST_LOCK`, so the process is the correct re-entrancy
+# boundary.
+#
+# The holder set is tagged with ``pid`` so that a forked child does NOT inherit
+# re-entrant credit from its parent: the OS-level flock IS NOT inherited as a
+# "we hold it" semantically — the child must reacquire. The pid check clears
+# stale state so a forked child correctly hits the fcntl path. Access is guarded
+# by ``_palace_lock_guard`` because the set is now shared across threads.
+#
+# Fork safety: ``_palace_lock_guard`` is a real ``threading.Lock``, so a child
+# forked while another thread held it would inherit it locked (the holder thread
+# does not exist in the child) and deadlock on the next acquire. An at-fork
+# handler (registered below) replaces the guard with a fresh unlocked lock and
+# clears state in the child, which must reacquire the flock anyway.
+_palace_lock_guard = threading.Lock()
+_palace_lock_pid = None
+_palace_lock_keys = set()
 
 
-def _holder_state():
-    """Return the per-thread (pid, keys) record, refreshing after fork."""
-    keys = getattr(_palace_lock_holders, "keys", None)
-    pid = getattr(_palace_lock_holders, "pid", None)
+def _reset_palace_lock_state_after_fork() -> None:
+    """Reset lock state in a forked child to avoid an inherited-locked deadlock."""
+    global _palace_lock_guard, _palace_lock_pid, _palace_lock_keys
+    _palace_lock_guard = threading.Lock()
+    _palace_lock_keys = set()
+    _palace_lock_pid = os.getpid()
+
+
+# Availability: Unix (no-op elsewhere — Windows has no fork()).
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=_reset_palace_lock_state_after_fork)
+
+
+def _holder_keys_locked():
+    """Return the process-wide held-key set, refreshing after fork.
+
+    Caller MUST hold ``_palace_lock_guard``.
+    """
+    global _palace_lock_pid, _palace_lock_keys
     current_pid = os.getpid()
-    if keys is None or pid != current_pid:
-        keys = set()
-        _palace_lock_holders.keys = keys
-        _palace_lock_holders.pid = current_pid
-    return keys
+    if _palace_lock_pid != current_pid:
+        _palace_lock_keys = set()
+        _palace_lock_pid = current_pid
+    return _palace_lock_keys
 
 
-def _held_by_this_thread(lock_key: str) -> bool:
-    """Return True if this thread already holds ``mine_palace_lock`` for ``lock_key``."""
-    return lock_key in _holder_state()
+def _held_by_this_process(lock_key: str) -> bool:
+    """Return True if this process already holds ``mine_palace_lock`` for ``lock_key``."""
+    with _palace_lock_guard:
+        return lock_key in _holder_keys_locked()
 
 
 def _mark_held(lock_key: str) -> None:
-    _holder_state().add(lock_key)
+    with _palace_lock_guard:
+        _holder_keys_locked().add(lock_key)
 
 
 def _mark_released(lock_key: str) -> None:
-    _holder_state().discard(lock_key)
+    with _palace_lock_guard:
+        _holder_keys_locked().discard(lock_key)
 
 
 def _format_lock_holder(content: str) -> str:
@@ -1064,11 +1111,13 @@ def mine_palace_lock(palace_path: str):
     raise MineAlreadyRunning so the caller can exit cleanly instead of
     piling up as a waiting worker.
 
-    Re-entrant: if the current thread already holds the lock for the same
+    Re-entrant: if the current process already holds the lock for the same
     palace, the context manager passes through without re-acquiring. This
     lets ChromaCollection write methods (which acquire the lock themselves
     to protect MCP/direct callers) compose with miner.mine() (which holds
-    the outer lock for the entire mine pipeline) without self-deadlock.
+    the outer lock for the entire mine pipeline) without self-deadlock, and
+    lets the threaded MCP HTTP transport write from a worker thread while the
+    long-lived writer-lease is held on another thread of the same process.
     """
     lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks")
     os.makedirs(lock_dir, exist_ok=True)
@@ -1077,8 +1126,8 @@ def mine_palace_lock(palace_path: str):
     palace_key = hashlib.sha256(lock_key_source.encode()).hexdigest()[:16]
     lock_path = os.path.join(lock_dir, f"mine_palace_{palace_key}.lock")
 
-    if _held_by_this_thread(palace_key):
-        # Same thread already holds the lock for this palace — pass through.
+    if _held_by_this_process(palace_key):
+        # This process already holds the lock for this palace — pass through.
         yield
         return
 
@@ -1129,8 +1178,14 @@ def mine_palace_lock(palace_path: str):
                 ) from exc
         # Record our own identity for any later contender's diagnostic message.
         _write_lock_holder(lf)
-        _mark_held(palace_key)
+        # Mark the hold from inside the try so it always pairs with
+        # _mark_released. If it sat before the try, an async exception
+        # (a SIGINT/KeyboardInterrupt) landing in the gap would orphan
+        # palace_key in the holder set while the outer finally frees the
+        # flock, so the in-memory hold would outlive the OS lock and a later
+        # re-entrant acquire would pass through and write without the flock.
         try:
+            _mark_held(palace_key)
             yield
         finally:
             _mark_released(palace_key)
@@ -1179,9 +1234,15 @@ def file_already_mined(
         schema (triggers silent rebuild after a normalization upgrade)
       - `check_mtime=True` and the file's mtime differs from the stored one
 
-    When check_mtime=True (used by project miner), also re-mines on content
-    change. When check_mtime=False (used by convo miner), transcripts are
-    assumed immutable, so only the version gate triggers a rebuild.
+    When check_mtime=True (used by the project miner, and by the convo
+    miner's in-lock recheck), also re-mines on content change. Conversation
+    transcripts are NOT assumed immutable: a Claude Code session keeps
+    appending to its own file while active, and /compact or /clear can
+    rewrite one in place. The convo miner's bulk skip-check uses
+    prefetch_mined_set()'s stored mtimes instead of calling this function
+    per file (same mtime-aware decision, without the O(n) per-file query
+    cost); this function's check_mtime=True path remains its per-file,
+    lock-held race-condition recheck.
 
     When extract_mode is set (used by convo miner), idempotency is scoped to
     that extraction mode so exchange-mode and general-mode drawers can coexist
@@ -1238,42 +1299,25 @@ def file_already_mined(
         return False
 
 
-def bulk_check_mined(collection) -> dict[str, float]:
-    """Pre-fetch source_file/source_mtime pairs for all documents in the collection.
+def prefetch_mined_set(
+    collection, extract_mode: Optional[str] = None
+) -> dict[str, Optional[float]]:
+    """Pre-fetch source_file -> stored source_mtime for files already mined
+    at the current NORMALIZE_VERSION, in one bulk pass instead of one
+    ChromaDB query per file.
 
-    Returns a dict mapping source_file -> source_mtime (as float) for every
-    document that has both fields.  Callers can check membership and compare
-    mtimes locally instead of issuing one ChromaDB query per file.
-
-    Fetches the full collection in paginated batches (like palace_graph.py)
-    since a WHERE-IN filter on thousands of paths is not supported by ChromaDB.
-    """
-    mined: dict[str, float] = {}
-    try:
-        total = collection.count()
-        offset = 0
-        while offset < total:
-            batch = collection.get(limit=1000, offset=offset, include=["metadatas"])
-            for meta in batch["metadatas"]:
-                src = meta.get("source_file")
-                mtime = meta.get("source_mtime")
-                if src and mtime is not None:
-                    mined[src] = float(mtime)
-            if not batch["ids"]:
-                break
-            offset += len(batch["ids"])
-    except Exception:
-        logger.warning("bulk_check_mined: partial fetch, %d files loaded", len(mined))
-    return mined
-
-
-def prefetch_mined_set(collection, extract_mode: Optional[str] = None) -> set[str]:
-    """Pre-fetch the set of source_files already mined at the current NORMALIZE_VERSION.
-
-    Mirrors file_already_mined()'s version-gate semantics (check_mtime=False
-    branch) but in one bulk pass instead of one ChromaDB query per file.
-    Returns a set of source_file paths whose stored drawers are at or above
-    NORMALIZE_VERSION; callers do `if path in result_set: skip`.
+    Return type is a dict rather than a bare set so callers get mtime
+    awareness "for free": conversation transcripts are not immutable once
+    mined (a Claude Code session keeps appending to the same file while
+    active, and /compact or /clear can rewrite one in place), so "we've
+    seen this source_file before" is not suffient to skip it -- the caller
+    must also confirm its current on-disk mtime still matches what was
+    stored. `if src in mined_set` still means the same thing as the old
+    set-based return (dict `in` checks keys); a caller that wants staleness
+    detection reads `mined_set[src]` and compares against
+    os.path.getmtime(src) itself. `None` means either no mtime was ever
+    stored (drawers written before this field existed) or getmtime failed
+    when the drawer was written -- both should be treated as stale.
 
     When extract_mode is set, mirrors file_already_mined(..., extract_mode=...)
     so conversation mines skip per extraction mode rather than per source file.
@@ -1283,7 +1327,7 @@ def prefetch_mined_set(collection, extract_mode: Optional[str] = None) -> set[st
     palace, making a 2000-file sweep take >1h of pure skip-checking. This
     helper drops that to a single paginated scan plus O(1) lookups.
     """
-    mined: set[str] = set()
+    mined: dict[str, Optional[float]] = {}
     try:
         total = collection.count()
         offset = 0
@@ -1299,7 +1343,8 @@ def prefetch_mined_set(collection, extract_mode: Optional[str] = None) -> set[st
                 # Same default as file_already_mined: missing version == 1
                 version = meta.get("normalize_version", 1)
                 if version >= NORMALIZE_VERSION:
-                    mined.add(src)
+                    stored_mtime = meta.get("source_mtime")
+                    mined[src] = float(stored_mtime) if stored_mtime is not None else None
             if not batch["ids"]:
                 break
             offset += len(batch["ids"])

mempalace/palace_graph.py

@@ -494,6 +494,7 @@ def create_tunnel(
     source_drawer_id: str = None,
     target_drawer_id: str = None,
     kind: str = "explicit",
+    config=None,
 ):
     """Create an explicit (symmetric) tunnel between two locations in the palace.
 
@@ -520,6 +521,9 @@ def create_tunnel(
             topical link where rooms are synthetic ``topic:<name>``
             identifiers). Preserved on the stored dict so readers can
             distinguish real-room traversals from topic connections.
+        config: Optional ``MempalaceConfig`` selecting the palace and its
+            tunnel sidecar. Explicit-path callers must pass the matching
+            config instead of falling back to the ambient default palace.
 
     Returns:
         The stored tunnel dict.
@@ -545,7 +549,7 @@ def create_tunnel(
     # mempalace.yaml from disk; before this change the helpers each
     # instantiated their own, triggering several redundant disk reads per
     # create_tunnel call (flagged by gemini-code-assist on #1469).
-    config = MempalaceConfig()
+    config = config or MempalaceConfig()
 
     # Validate room existence for explicit tunnels only. Use the verbatim wing
     # slugs here so #1504's hyphen-preserving write path remains intact.
@@ -742,6 +746,7 @@ def compute_topic_tunnels(
     topics_by_wing: dict,
     min_count: int = 1,
     label_prefix: str = "shared topic",
+    config=None,
 ) -> list[dict]:
     """Create tunnels for every pair of wings that share >= ``min_count`` topics.
 
@@ -817,6 +822,7 @@ def compute_topic_tunnels(
                     target_room=room,
                     label=f"{label_prefix}: {topic_name}",
                     kind="topic",
+                    config=config,
                 )
                 created.append(tunnel)
     return created
@@ -827,6 +833,7 @@ def topic_tunnels_for_wing(
     topics_by_wing: dict,
     min_count: int = 1,
     label_prefix: str = "shared topic",
+    config=None,
 ) -> list[dict]:
     """Compute topic tunnels involving a single wing.
 
@@ -871,6 +878,7 @@ def topic_tunnels_for_wing(
                 slice_map,
                 min_count=min_count,
                 label_prefix=label_prefix,
+                config=config,
             )
         )
     return created
@@ -880,6 +888,7 @@ def entity_tunnels_for_wing(
     wing: str,
     hallways: list,
     label_prefix: str = "shared entity",
+    config=None,
 ) -> list:
     """Compute entity tunnels involving a single wing.
 
@@ -946,6 +955,7 @@ def entity_tunnels_for_wing(
                 target_room=room,
                 label=f"{label_prefix}: {entity}",
                 kind="entity",
+                config=config,
             )
             created.append(tunnel)
     return created

mempalace/repair.py

@@ -33,6 +33,7 @@ import argparse
 import os
 import shutil
 import sqlite3
+import stat
 import time
 from collections import defaultdict
 from contextlib import closing
@@ -57,6 +58,78 @@ REPAIR_TEMP_COLLECTION = f"{COLLECTION_NAME}__repair_tmp"
 CLOSETS_COLLECTION_NAME = "mempalace_closets"
 
 
+def _no_follow_flag() -> int:
+    return getattr(os, "O_NOFOLLOW", 0)
+
+
+def _open_regular_file_no_follow(path: str) -> int:
+    if os.path.islink(path):
+        raise RuntimeError(f"Refusing symlinked file: {path}")
+    fd = os.open(path, os.O_RDONLY | _no_follow_flag())
+    try:
+        st = os.fstat(fd)
+        if not stat.S_ISREG(st.st_mode):
+            raise RuntimeError(f"Refusing non-regular file: {path}")
+        return fd
+    except Exception:
+        os.close(fd)
+        raise
+
+
+def _write_text_replace_no_follow(path: str, text: str) -> None:
+    directory = os.path.dirname(path) or "."
+    basename = os.path.basename(path)
+    tmp_path = os.path.join(
+        directory,
+        f".{basename}.{os.getpid()}.{int(time.time() * 1_000_000)}.tmp",
+    )
+    fd = os.open(
+        tmp_path,
+        os.O_WRONLY | os.O_CREAT | os.O_EXCL | _no_follow_flag(),
+        0o600,
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            f.write(text)
+        os.replace(tmp_path, path)
+    except Exception:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise
+
+
+def _copy_file_no_follow(src: str, dst: str, *, replace: bool = False) -> None:
+    src_fd = _open_regular_file_no_follow(src)
+    try:
+        src_f = os.fdopen(src_fd, "rb")
+    except Exception:
+        os.close(src_fd)
+        raise
+    # ``src_f`` now owns ``src_fd`` and closes it on exit.
+    with src_f:
+        flags = os.O_WRONLY | os.O_CREAT | _no_follow_flag()
+        flags |= os.O_TRUNC if replace else os.O_EXCL
+        dst_fd = os.open(dst, flags, 0o600)
+        try:
+            dst_f = os.fdopen(dst_fd, "wb")
+        except Exception:
+            os.close(dst_fd)
+            raise
+        with dst_f:
+            shutil.copyfileobj(src_f, dst_f)
+    try:
+        shutil.copystat(src, dst, follow_symlinks=False)
+    except OSError:
+        pass
+
+
+def _unique_backup_path(path: str, label: str) -> str:
+    stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    return f"{path}.{label}.{stamp}.{os.getpid()}"
+
+
 def _drawers_collection_name() -> str:
     """Resolve the drawers collection name from user config, falling back
     to the module default ``COLLECTION_NAME`` if config is unreadable.
@@ -316,9 +389,10 @@ def scan_palace(palace_path=None, only_wing=None, collection_name: Optional[str]
     print(f"  BAD:  {len(bad_set):,}  ({len(bad_set) / max(len(all_ids), 1) * 100:.1f}%)")
 
     bad_file = os.path.join(palace_path, "corrupt_ids.txt")
-    with open(bad_file, "w") as f:
-        for bid in sorted(bad_set):
-            f.write(bid + "\n")
+    _write_text_replace_no_follow(
+        bad_file,
+        "".join(f"{bid}\n" for bid in sorted(bad_set)),
+    )
     print(f"\n  Bad IDs written to: {bad_file}")
     return good_set, bad_set
 
@@ -499,6 +573,9 @@ def sqlite_drawer_count(palace_path: str, collection_name: Optional[str] = None)
         return None
 
 
+_SQLITE_INTEGRITY_BUSY_TIMEOUT_SECONDS = 15.0
+
+
 def sqlite_integrity_errors(palace_path: str) -> list[str]:
     """Return SQLite quick_check errors for chroma.sqlite3.
 
@@ -517,7 +594,16 @@ def sqlite_integrity_errors(palace_path: str) -> list[str]:
         return []
 
     try:
-        with sqlite3.connect(sqlite_read_uri(sqlite_path), uri=True) as conn:
+        # A writer holding SQLite's lock is contention, not corruption. The
+        # sqlite3 module defaults to five seconds, which is shorter than
+        # routine batch mines and curator writes on rollback-journal palaces.
+        # Give those writers a bounded grace period before surfacing BUSY to
+        # callers; genuine corruption still comes from PRAGMA quick_check.
+        with sqlite3.connect(
+            sqlite_read_uri(sqlite_path),
+            uri=True,
+            timeout=_SQLITE_INTEGRITY_BUSY_TIMEOUT_SECONDS,
+        ) as conn:
             rows = conn.execute("PRAGMA quick_check").fetchall()
     except sqlite3.Error as e:
         return [f"PRAGMA quick_check failed: {e}"]
@@ -564,6 +650,93 @@ def print_sqlite_integrity_abort(palace_path: str, errors: list[str]) -> None:
     print("    6. Re-run `mempalace repair --yes`.")
 
 
+# quick_check's wording for a corrupt FTS5 inverted index differs by SQLite
+# version — both forms describe the same recoverable condition:
+#   "malformed inverted index for FTS5 table main.embedding_fulltext_search"
+#     (older SQLite)
+#   "fts5: corruption found reading blob N from table \"embedding_fulltext_search\""
+#     (SQLite >= ~3.5x, confirmed on 3.53.2 / Python 3.13.7 — the exact
+#     message this repo's own test fixture produces on that build)
+# Either failure is recoverable in place: the index is derived from the
+# intact ``embedding_fulltext_search_content`` shadow table, so rebuilding it
+# restores full-text search without touching any drawer rows. Concurrent
+# killed-mid-write mines are the usual cause (#1596). A regex matching only
+# the older phrasing would silently decline to auto-heal on newer SQLite —
+# the exact failure this repo's own test suite caught (test_repair.py's two
+# auto-heal tests failed on this machine until this pattern was widened).
+_FTS5_MALFORMED_RE = re.compile(
+    r"malformed inverted index for fts5 table|fts5:\s*corruption found",
+    re.IGNORECASE,
+)
+
+
+def _errors_are_isolated_fts5(errors: list[str]) -> bool:
+    """True when every quick_check error is a malformed FTS5 inverted index.
+
+    Only an isolated FTS5 failure is safe to auto-heal: the inverted index is
+    derived data that ``rebuild`` regenerates from the content shadow table. If
+    quick_check also reports page/row corruption, the data itself may be damaged
+    and rebuilding the index over it would mask real loss — that still aborts.
+    """
+    return bool(errors) and all(_FTS5_MALFORMED_RE.search(e) for e in errors)
+
+
+def maybe_autoheal_fts5_index(palace_path: str, errors: list[str], *, progress=print) -> list[str]:
+    """Rebuild a malformed FTS5 inverted index in place; return remaining errors.
+
+    The repair preflight aborts when ``PRAGMA quick_check`` reports SQLite-layer
+    corruption. After concurrent killed-mid-write mines (#1596) the common
+    failure is an isolated ``malformed inverted index for FTS5 table``, which is
+    fully recoverable: the index rebuilds from the intact
+    ``embedding_fulltext_search_content`` table without touching drawer rows.
+
+    When the errors are isolated to FTS5, rebuild the index under the palace
+    write lock (so a live mine cannot race the rebuild) and re-run quick_check.
+    Returns the remaining quick_check errors — empty when the heal succeeded.
+    Broader corruption, a lock held by another writer, or a rebuild failure
+    leaves ``errors`` unchanged so the caller still aborts with the banner.
+    """
+    if not _errors_are_isolated_fts5(errors):
+        return errors
+
+    sqlite_path = os.path.join(palace_path, "chroma.sqlite3")
+    if not os.path.exists(sqlite_path):
+        return errors
+
+    # Lazy import: palace.py is heavier and importing it at module load would
+    # widen repair.py's import graph for callers that never hit this path.
+    from .palace import MineAlreadyRunning, mine_palace_lock
+
+    progress(
+        "\n  Isolated FTS5 inverted-index corruption detected; attempting an\n"
+        "  in-place rebuild from the intact content table before aborting."
+    )
+    try:
+        with mine_palace_lock(palace_path):
+            with closing(sqlite3.connect(sqlite_path, isolation_level=None)) as conn:
+                conn.execute(
+                    "INSERT INTO embedding_fulltext_search"
+                    "(embedding_fulltext_search) VALUES('rebuild')"
+                )
+                conn.commit()
+    except MineAlreadyRunning as exc:
+        progress(
+            f"  Skipped FTS5 rebuild: palace is being written by another process ({exc}). "
+            "Stop it and re-run."
+        )
+        return errors
+    except Exception as exc:
+        progress(f"  FTS5 rebuild failed (leaving palace untouched): {exc}")
+        return errors
+
+    remaining = sqlite_integrity_errors(palace_path)
+    if remaining:
+        progress("  FTS5 rebuild did not clear quick_check; aborting for safety.")
+    else:
+        progress("  FTS5 index rebuilt from intact content; quick_check is clean.")
+    return remaining
+
+
 def index_read_recovery_guidance() -> str:
     """Recovery guidance for a failed drawer-index read in the legacy paths.
 
@@ -716,7 +889,12 @@ class _DefaultProgress:
         return f" (elapsed {_format_eta(elapsed)}, rate {rate:.1f}/s, ETA {_format_eta(eta)})"
 
 
-def _vacuum_and_rebuild_fts5(palace_path: str, progress=print) -> None:
+def _vacuum_and_rebuild_fts5(
+    palace_path: str,
+    progress=print,
+    *,
+    strict: bool = False,
+) -> None:
     """VACUUM the palace SQLite file and rebuild the FTS5 index if present.
 
     Repeated ``repair --yes`` runs delete and recreate the drawers collection,
@@ -725,12 +903,16 @@ def _vacuum_and_rebuild_fts5(palace_path: str, progress=print) -> None:
     internally inconsistent after multiple collection deletes; the rebuild
     command fixes it atomically without touching any row data.
 
-    Failures are non-fatal: a warning is printed and the caller continues.
-    The repair itself succeeded at this point — VACUUM/FTS5 are best-effort
-    cleanup, not correctness requirements.
+    Existing repair paths use the default best-effort behavior: failures log a
+    warning and return because their primary rebuild has already succeeded.
+    SQLite recovery passes ``strict=True`` because its bulk upserts can leave
+    this derived index malformed; that path must not report success until the
+    rebuild, VACUUM, and a final quick_check all complete.
     """
     sqlite_path = os.path.join(palace_path, "chroma.sqlite3")
     if not os.path.exists(sqlite_path):
+        if strict:
+            raise FileNotFoundError(f"recovered palace has no SQLite database: {sqlite_path}")
         return
     try:
         with closing(sqlite3.connect(sqlite_path, isolation_level=None)) as conn:
@@ -746,7 +928,21 @@ def _vacuum_and_rebuild_fts5(palace_path: str, progress=print) -> None:
                 progress("  FTS5 index rebuilt.")
             conn.execute("VACUUM")
             progress("  SQLite VACUUM complete.")
+            if strict:
+                rows = conn.execute("PRAGMA quick_check").fetchall()
+                errors = [str(row[0]) for row in rows if row and str(row[0]).lower() != "ok"]
+                if errors:
+                    raise sqlite3.DatabaseError(
+                        "post-recovery quick_check failed: " + "; ".join(errors[:3])
+                    )
+                progress("  SQLite quick_check clean.")
     except Exception as exc:
+        if strict:
+            # Preserve the concrete SQLite/filesystem exception for callers
+            # that need to classify the failure. The strict recovery caller
+            # owns the user-facing error message, so logging here would also
+            # print the same failure twice.
+            raise
         progress(f"  Warning: post-repair cleanup failed (non-fatal): {exc}")
 
 
@@ -809,6 +1005,8 @@ def rebuild_index(
     # corruption here lets us surface the clear recovery instructions and
     # exit cleanly before chromadb's compactor touches the disk.
     sqlite_errors = sqlite_integrity_errors(palace_path)
+    if sqlite_errors:
+        sqlite_errors = maybe_autoheal_fts5_index(palace_path, sqlite_errors, progress=progress)
     if sqlite_errors:
         print_sqlite_integrity_abort(palace_path, sqlite_errors)
         return
@@ -858,10 +1056,10 @@ def rebuild_index(
 
     # Back up ONLY the SQLite database, not the bloated HNSW files
     sqlite_path = os.path.join(palace_path, "chroma.sqlite3")
-    backup_path = sqlite_path + ".backup"
+    backup_path = _unique_backup_path(sqlite_path, "backup")
     if os.path.exists(sqlite_path):
         progress(f"  Backing up chroma.sqlite3 ({os.path.getsize(sqlite_path) / 1e6:.0f} MB)...")
-        shutil.copy2(sqlite_path, backup_path)
+        _copy_file_no_follow(sqlite_path, backup_path)
         progress(f"  Backup: {backup_path}")
 
     # Rebuild with correct HNSW settings
@@ -885,7 +1083,7 @@ def rebuild_index(
             try:
                 _close_chroma_handles(palace_path, backend=backend)
                 _delete_collection_if_exists(backend, palace_path, collection_name)
-                shutil.copy2(backup_path, sqlite_path)
+                _copy_file_no_follow(backup_path, sqlite_path, replace=True)
                 progress("  Backup restored. Palace is back to pre-repair state.")
             except Exception as restore_error:
                 progress(f"  Backup restore failed: {restore_error}")
@@ -930,6 +1128,29 @@ class RebuildPartialError(Exception):
         self.archive_path = archive_path
 
 
+class RebuildCleanupError(Exception):
+    """Raised when all recoverable rows landed but final cleanup failed.
+
+    The destination is intentionally retained for inspection, and an in-place
+    rebuild's original archive remains untouched. Callers must not treat this
+    as success because the derived FTS5 index has not been verified clean.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        counts: dict[str, int],
+        dest_palace: str,
+        archive_path: Optional[str],
+    ):
+        super().__init__(message)
+        self.message = message
+        self.counts = counts
+        self.dest_palace = dest_palace
+        self.archive_path = archive_path
+
+
 def _rebuild_one_collection(
     *,
     backend: ChromaBackend,
@@ -1116,7 +1337,10 @@ def _preserve_knowledge_graph_sqlite(source_palace: str, dest_palace: str) -> li
             continue
 
         os.makedirs(dest_palace, exist_ok=True)
-        shutil.copy2(src, dst)
+        try:
+            _copy_file_no_follow(src, dst, replace=True)
+        except RuntimeError:
+            continue
         copied.append(filename)
 
     if copied:
@@ -1176,7 +1400,9 @@ def rebuild_from_sqlite(
     chromadb upsert fails partway through; the dest palace is left in
     place so the user can inspect what landed, and the in-place archive
     (when applicable) is reported in the error so the user can re-run
-    against it.
+    against it. Raises :class:`RebuildCleanupError` if all rows land but the
+    required FTS5 rebuild, VACUUM, or final quick_check fails; this prevents a
+    structurally unverified recovery from being reported as complete.
 
     .. warning::
 
@@ -1250,7 +1476,30 @@ def rebuild_from_sqlite(
         ts = datetime.now().strftime("%Y%m%d-%H%M%S")
         archive_path = f"{dest_palace}.pre-rebuild-{ts}"
         print(f"  Archiving {dest_palace} → {archive_path}")
-        shutil.move(dest_palace, archive_path)
+        # os.rename, NOT shutil.move. When any file inside the palace is
+        # held open by another process (MCP server, a running mine, another
+        # harness), renaming the directory fails atomically UP FRONT on
+        # Windows and the palace is left untouched. shutil.move's fallback
+        # for a failed rename is copytree + rmtree — and that rmtree
+        # deletes the live palace file-by-file until it hits the first
+        # locked file, leaving the palace partially gutted next to a
+        # partial archive copy. Observed live (Windows 11, 2026-07-05/06):
+        # two interrupted in-place repairs; the palace survived only
+        # because the locked chroma.sqlite3/*.bin themselves could not be
+        # unlinked. Same lesson as the source-validation comment above:
+        # fail cleanly before touching anything.
+        try:
+            os.rename(dest_palace, archive_path)
+        except OSError as exc:
+            print(
+                f"\n  Cannot archive the existing palace: a file inside it "
+                f"is held open by another process (MCP server, running "
+                f"mine, another harness?).\n"
+                f"  Close those processes and retry. The palace was left "
+                f"untouched.\n"
+                f"  ({exc})"
+            )
+            return {}
         source_palace = archive_path
         src_db = os.path.join(source_palace, "chroma.sqlite3")
 
@@ -1302,14 +1551,39 @@ def rebuild_from_sqlite(
             else:
                 print(f"    done: {upserted} rows in {cname}")
 
-        print(f"\n  Rebuild complete. {sum(counts.values())} total rows.")
-        if archive_path is not None:
-            print(f"  Original palace archived at: {archive_path}")
-        print(f"{'=' * 55}\n")
-        return counts
     finally:
         backend.close()
 
+    # Bulk Chroma upserts can leave the derived FTS5 index internally
+    # inconsistent even when all source rows landed.  Rebuild it only after
+    # the backend releases its SQLite handle; otherwise VACUUM cannot obtain
+    # the exclusive lock it needs on Windows.
+    try:
+        _vacuum_and_rebuild_fts5(dest_palace, strict=True)
+    except Exception as exc:
+        message_parts = [
+            f"Post-recovery cleanup failed after {sum(counts.values())} rows were rebuilt: {exc}",
+            f"Recovered palace retained at: {dest_palace}",
+        ]
+        if archive_path is not None:
+            message_parts.append(f"Original palace remains archived at: {archive_path}")
+        else:
+            message_parts.append(f"Source palace is unchanged at: {source_palace}")
+        message = "\n  ".join(message_parts)
+        print(f"\n  ERROR: {message}")
+        raise RebuildCleanupError(
+            message,
+            counts=dict(counts),
+            dest_palace=dest_palace,
+            archive_path=archive_path,
+        ) from exc
+
+    print(f"\n  Rebuild complete. {sum(counts.values())} total rows.")
+    if archive_path is not None:
+        print(f"  Original palace archived at: {archive_path}")
+    print(f"{'=' * 55}\n")
+    return counts
+
 
 def status(palace_path=None, collection_name: Optional[str] = None) -> dict:
     """Read-only health check: compare sqlite vs HNSW element counts.

mempalace/searcher.py

@@ -221,7 +221,18 @@ def _hybrid_rank(
         r["bm25_score"] = round(raw, 3)
         scored.append((vector_weight * vec_sim + bm25_weight * norm, r))
 
-    scored.sort(key=lambda pair: pair[0], reverse=True)
+    # Break exact score ties toward the more recently authored drawer so equal-score
+    # candidates rank chronologically instead of in arbitrary backend order. ISO-8601
+    # ``authored_at`` strings sort chronologically; missing dates sort oldest.
+    # authored_at lives at the top level on the search_memories path and nested under
+    # "metadata" on the candidate-union path; check both so the tie-break works for each.
+    scored.sort(
+        key=lambda pair: (
+            pair[0],
+            pair[1].get("authored_at") or pair[1].get("metadata", {}).get("authored_at") or "",
+        ),
+        reverse=True,
+    )
     results[:] = [r for _, r in scored]
     return results
 
@@ -396,11 +407,109 @@ def _warn_if_legacy_metric(col) -> None:
     )
 
 
+def _hnsw_capacity_diverged(palace_path: str) -> bool:
+    """Return True if HNSW divergence is severe enough to crash ChromaDB.
+
+    Thin, exception-safe wrapper around
+    :func:`mempalace.backends.chroma.hnsw_capacity_status`. Used by the
+    CLI search path to short-circuit to the BM25-only fallback before
+    opening a Chroma client. Client construction and collection identity
+    checks can themselves touch the damaged index, so guarding only
+    ``col.query()`` is too late (#1222 covers the MCP path via the module-level
+    ``_vector_disabled`` flag; this covers the CLI path).
+
+    A probe that raises falls through to ``False`` so the caller proceeds
+    to the normal vector path — the underlying query then either succeeds
+    (probe was a false negative) or raises its own diagnostic error. The
+    probe itself must never be the thing that crashes search.
+    """
+    try:
+        from .backends.chroma import hnsw_capacity_status
+        from .config import get_configured_collection_name
+
+        info = hnsw_capacity_status(palace_path, get_configured_collection_name())
+        return bool(info.get("diverged"))
+    except Exception:
+        logger.debug("HNSW capacity probe raised; proceeding to vector path", exc_info=True)
+        return False
+
+
+def _print_search_results_bm25_only(
+    query: str, palace_path: str, wing: str, room: str, n_results: int
+) -> None:
+    """CLI fallback printer for when HNSW divergence fences off vector search.
+
+    Mirrors the vector-path output shape so users get lexical matches in
+    the format they expect, plus a clear notice pointing at
+    ``mempalace repair``. Replaces the silent SIGBUS users otherwise hit
+    when the CLI called ``col.query()`` against a diverged segment.
+    """
+    result = _bm25_only_via_sqlite(
+        query=query,
+        palace_path=palace_path,
+        wing=wing,
+        room=room,
+        n_results=n_results,
+    )
+    hits = result.get("results", [])
+
+    print(
+        "\n  NOTICE: vector search disabled — HNSW index has diverged from SQLite.\n"
+        "          Showing BM25-only results. Run `mempalace repair` to restore "
+        "vector search.\n"
+    )
+    print(f"{'=' * 60}")
+    print(f'  Results for: "{query}"')
+    if wing:
+        print(f"  Wing: {wing}")
+    if room:
+        print(f"  Room: {room}")
+    print(f"{'=' * 60}\n")
+
+    if not hits:
+        print(f'  No results found for: "{query}"')
+        return
+
+    for i, hit in enumerate(hits, 1):
+        bm25 = hit.get("bm25_score", 0.0)
+        wing_name = hit.get("wing", "?")
+        room_name = hit.get("room", "?")
+        source = Path(hit.get("source_file", "?")).name
+
+        print(f"  [{i}] {wing_name} / {room_name}")
+        print(f"      Source: {source}")
+        print(f"      Match:  bm25={bm25}  (vector disabled)")
+        print()
+        for line in (hit.get("text", "") or "").strip().split("\n"):
+            print(f"      {line}")
+        print()
+        print(f"  {'─' * 56}")
+
+    print()
+
+
 def search(query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5):
     """
     Search the palace. Returns verbatim drawer content.
     Optionally filter by wing (project) or room (aspect).
     """
+    # Probe a Chroma palace before get_collection(). Opening the client can
+    # load native index state, and embedder-identity enforcement may call
+    # collection.count(); both happen before the old query-only guard and can
+    # hit the same native crash. Non-Chroma backends never use Chroma's HNSW
+    # files or sqlite-specific fallback and proceed normally.
+    try:
+        backend_name = resolve_backend_name(palace_path)
+    except (BackendMismatchError, KeyError):
+        # Preserve _open_collection_or_explain's state-specific diagnostics
+        # for mixed artifacts and unknown backend selections. This probe is
+        # only an early Chroma safety fence; it must not become a second,
+        # less-helpful backend validation path.
+        backend_name = None
+
+    if backend_name == "chroma" and _hnsw_capacity_diverged(palace_path):
+        return _print_search_results_bm25_only(query, palace_path, wing, room, n_results)
+
     col = _open_collection_or_explain(palace_path, opener=get_collection)
     if col is None:
         if not os.path.isdir(palace_path):
@@ -681,6 +790,7 @@ def _bm25_only_via_sqlite(
                 "source_file": Path(full_source).name if full_source else "?",
                 "source_path": full_source,
                 "created_at": meta.get("filed_at", "unknown"),
+                "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")),
                 # No vector distance available in BM25-only mode.
                 "similarity": None,
                 "distance": None,
@@ -783,6 +893,7 @@ def _merge_bm25_union_candidates(
                 "source_file": Path(full_source).name if full_source else "?",
                 "source_path": full_source,
                 "created_at": meta.get("filed_at", "unknown"),
+                "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")),
                 "similarity": None,
                 "distance": None,
                 "effective_distance": None,
@@ -1198,6 +1309,7 @@ def search_memories(
             "source_file": Path(source).name if source else "?",
             "source_path": source,
             "created_at": meta.get("filed_at", "unknown"),
+            "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")),
             "similarity": round(_distance_to_similarity(effective_dist, metric), 3),
             "distance": round(dist, 4),
             "effective_distance": round(effective_dist, 4),

mempalace/service.py

@@ -55,6 +55,7 @@ WRITE_TOOLS = frozenset(
     {
         "mempalace_add_drawer",
         "mempalace_checkpoint",
+        "mempalace_delete_by_source",
         "mempalace_delete_drawer",
         "mempalace_update_drawer",
         "mempalace_diary_write",

mempalace/version.py

@@ -1,3 +1,3 @@
 """Single source of truth for the MemPalace package version."""
 
-__version__ = "3.5.0"
+__version__ = "3.6.0"

mempalace/write_routing.py

@@ -0,0 +1,210 @@
+"""Shared daemon-routing policy for MemPalace write callers.
+
+The policy is deliberately transport-agnostic. Hook and CLI consumers decide
+whether a daemon is already available and whether they are allowed to start
+one; this module turns those facts plus a policy into one explicit route.
+
+This module changes no caller defaults by itself. Hook and CLI adoption are
+separate follow-up PRs.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Iterable
+
+
+class WriteRoutingError(ValueError):
+    """Raised when a write-routing policy is invalid or cannot be applied."""
+
+
+class WriteRoutingPolicy(str, Enum):
+    """User-selected policy for routine write operations."""
+
+    DIRECT = "direct"
+    PREFER = "prefer"
+    REQUIRE = "require"
+
+
+class WriteRoutingTarget(str, Enum):
+    """Concrete route selected for one write operation."""
+
+    DIRECT = "direct"
+    DAEMON = "daemon"
+    BLOCKED = "blocked"
+
+
+@dataclass(frozen=True)
+class RoutingPolicyCandidate:
+    """One precedence-ordered source for a routing policy."""
+
+    source: str
+    value: Any
+    legacy_boolean: bool = False
+
+
+@dataclass(frozen=True)
+class ResolvedWriteRoutingPolicy:
+    """A normalized policy plus the source that selected it."""
+
+    policy: WriteRoutingPolicy
+    source: str
+
+
+@dataclass(frozen=True)
+class WriteRoutingDecision:
+    """Concrete routing decision for a single operation."""
+
+    policy: WriteRoutingPolicy
+    target: WriteRoutingTarget
+    auto_start_daemon: bool
+    reason: str
+
+    @property
+    def use_daemon(self) -> bool:
+        """Whether this operation should use the daemon."""
+        return self.target is WriteRoutingTarget.DAEMON
+
+    @property
+    def blocked(self) -> bool:
+        """Whether the operation must stop instead of writing directly."""
+        return self.target is WriteRoutingTarget.BLOCKED
+
+
+_LEGACY_TRUE = {"1", "true", "yes", "on", "daemon"}
+_LEGACY_FALSE = {"0", "false", "no", "off"}
+
+
+def parse_write_routing_policy(
+    value: Any,
+    *,
+    legacy_boolean: bool = False,
+) -> WriteRoutingPolicy:
+    """Normalize one routing-policy value.
+
+    New policy settings accept only ``direct``, ``prefer``, and ``require``.
+
+    Legacy boolean settings additionally map truthy values to ``prefer`` and
+    falsy values to ``direct`` so existing ``hooks.daemon`` configurations
+    retain their historical behavior.
+    """
+
+    if isinstance(value, WriteRoutingPolicy):
+        return value
+
+    if legacy_boolean and isinstance(value, bool):
+        return WriteRoutingPolicy.PREFER if value else WriteRoutingPolicy.DIRECT
+
+    if legacy_boolean and isinstance(value, int) and not isinstance(value, bool):
+        if value == 1:
+            return WriteRoutingPolicy.PREFER
+        if value == 0:
+            return WriteRoutingPolicy.DIRECT
+
+    if not isinstance(value, str):
+        raise WriteRoutingError("write routing policy must be one of: direct, prefer, require")
+
+    normalized = value.strip().lower()
+
+    try:
+        return WriteRoutingPolicy(normalized)
+    except ValueError:
+        pass
+
+    if legacy_boolean:
+        if normalized in _LEGACY_TRUE:
+            return WriteRoutingPolicy.PREFER
+        if normalized in _LEGACY_FALSE:
+            return WriteRoutingPolicy.DIRECT
+
+    raise WriteRoutingError(
+        f"invalid write routing policy {value!r}; expected direct, prefer, or require"
+    )
+
+
+def resolve_write_routing_policy(
+    candidates: Iterable[RoutingPolicyCandidate],
+    *,
+    default: WriteRoutingPolicy = WriteRoutingPolicy.DIRECT,
+) -> ResolvedWriteRoutingPolicy:
+    """Return the first configured policy from ordered policy sources."""
+
+    for candidate in candidates:
+        if candidate.value is None:
+            continue
+
+        try:
+            policy = parse_write_routing_policy(
+                candidate.value,
+                legacy_boolean=candidate.legacy_boolean,
+            )
+        except WriteRoutingError as exc:
+            raise WriteRoutingError(f"{candidate.source}: {exc}") from exc
+
+        return ResolvedWriteRoutingPolicy(
+            policy=policy,
+            source=candidate.source,
+        )
+
+    return ResolvedWriteRoutingPolicy(
+        policy=default,
+        source="default",
+    )
+
+
+def choose_write_route(
+    policy: WriteRoutingPolicy,
+    *,
+    daemon_available: bool,
+    daemon_can_start: bool,
+) -> WriteRoutingDecision:
+    """Choose direct, daemon, or blocked for one routine write.
+
+    ``daemon_can_start`` is normally false for latency-sensitive hooks and
+    true for interactive CLI commands.
+
+    The key safety guarantee is that ``require`` never degrades to a direct
+    write when the daemon is unavailable.
+    """
+
+    policy = parse_write_routing_policy(policy)
+
+    if policy is WriteRoutingPolicy.DIRECT:
+        return WriteRoutingDecision(
+            policy=policy,
+            target=WriteRoutingTarget.DIRECT,
+            auto_start_daemon=False,
+            reason="policy-direct",
+        )
+
+    if daemon_available:
+        return WriteRoutingDecision(
+            policy=policy,
+            target=WriteRoutingTarget.DAEMON,
+            auto_start_daemon=False,
+            reason="daemon-available",
+        )
+
+    if daemon_can_start:
+        return WriteRoutingDecision(
+            policy=policy,
+            target=WriteRoutingTarget.DAEMON,
+            auto_start_daemon=True,
+            reason="daemon-auto-start",
+        )
+
+    if policy is WriteRoutingPolicy.PREFER:
+        return WriteRoutingDecision(
+            policy=policy,
+            target=WriteRoutingTarget.DIRECT,
+            auto_start_daemon=False,
+            reason="daemon-unavailable-fallback",
+        )
+
+    return WriteRoutingDecision(
+        policy=policy,
+        target=WriteRoutingTarget.BLOCKED,
+        auto_start_daemon=False,
+        reason="daemon-required-unavailable",
+    )

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mempalace"
-version = "3.5.0"
+version = "3.6.0"
 description = "Give your AI a memory — mine projects and conversations into a searchable palace. No API key required."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -60,6 +60,7 @@ mempalace-mcp = "mempalace.mcp_server:main"
 
 [project.entry-points."mempalace.backends"]
 chroma = "mempalace.backends.chroma:ChromaBackend"
+milvus = "mempalace.backends.milvus:MilvusBackend"
 pgvector = "mempalace.backends.pgvector:PgVectorBackend"
 qdrant = "mempalace.backends.qdrant:QdrantBackend"
 sqlite_exact = "mempalace.backends.sqlite_exact:SQLiteExactBackend"
@@ -78,7 +79,7 @@ dev = [
     # (wired only into the test-windows job in ci.yml, scoped to the specific
     # error). Local/Linux/macOS runs never rerun, so real failures stay loud.
     "pytest-rerunfailures>=12.0",
-    "ruff==0.15.18",
+    "ruff==0.15.20",
     "psutil>=5.9",
     # Property-based testing — generates hundreds of random inputs per
     # test to find counterexamples the hand-written positive tests miss.
@@ -99,6 +100,13 @@ dev = [
     "mypy>=1.0",
 ]
 spellcheck = ["autocorrect>=2.0"]
+# Opt-in Milvus backend. Defaults to per-palace Milvus Lite
+# (`<palace>/milvus.db`); set MEMPALACE_MILVUS_URI and
+# MEMPALACE_MILVUS_TOKEN to point at Milvus server or Zilliz Cloud.
+milvus = [
+    "pymilvus>=3.0.0",
+    "milvus-lite>=3.0; python_version >= '3.10' and sys_platform != 'win32'",
+]
 # Opt-in Postgres + pgvector backend. Only the psycopg driver is needed on the
 # client; the server must have the `vector` extension available. Selected via
 # MEMPALACE_BACKEND=pgvector / --backend pgvector; never required for the
@@ -139,7 +147,7 @@ dev = [
     # (wired only into the test-windows job in ci.yml, scoped to the specific
     # error). Local/Linux/macOS runs never rerun, so real failures stay loud.
     "pytest-rerunfailures>=12.0",
-    "ruff==0.15.18",
+    "ruff==0.15.20",
     "psutil>=5.9",
     "hypothesis>=6.0",
     "pre-commit>=3.0",

scripts/backfill_authored_at.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Backfill ``authored_at`` onto existing conversation drawers.
+
+New mines stamp ``authored_at`` automatically (see ``convo_miner._extract_authored_at``),
+but drawers mined before that change only have ``filed_at`` (ingest time). Re-mining does
+NOT fix them: the scanner skips files already mined at the current ``NORMALIZE_VERSION``.
+
+This migration updates the affected drawers IN PLACE — metadata only, embeddings are left
+untouched, so there is no re-embedding cost. It is idempotent (drawers already correct are
+skipped) and safe to re-run. It only touches ``ingest_mode == "convos"`` drawers; markdown
+drawers have no per-line timestamps and keep their ``filed_at`` fallback.
+
+Drawers whose source transcript is no longer on disk are left as-is (they keep falling back
+to ``filed_at``), so point ``--sessions`` at the directories that still hold your ``.jsonl``
+transcripts (e.g. ``~/.claude`` and ``~/.codex``).
+
+Usage (dry-run prints what would change; pass --apply to write):
+
+    python scripts/backfill_authored_at.py \
+        --palace ~/.mempalace/palace \
+        --sessions ~/.claude --sessions ~/.codex [--apply]
+
+In Docker (the MCP image), mount the volume and your session dirs read-only:
+
+    docker run --rm \
+      -v mempalace-data:/data \
+      -v ~/.claude:/sessions/claude:ro -v ~/.codex:/sessions/codex:ro \
+      -v "$PWD/scripts/backfill_authored_at.py:/tmp/backfill.py:ro" \
+      --entrypoint /app/.venv/bin/python mempalace:local \
+      /tmp/backfill.py --palace /data/.mempalace/palace \
+        --sessions /sessions/claude --sessions /sessions/codex --apply
+"""
+
+import argparse
+import glob
+import os
+
+import chromadb
+
+from mempalace.convo_miner import _extract_authored_at
+
+COLLECTION = "mempalace_drawers"
+PAGE = 2000
+BATCH = 1000
+
+
+def _index_sessions(session_dirs):
+    """Map ``basename.jsonl -> realpath`` for every transcript under the given dirs."""
+    index = {}
+    for root in session_dirs:
+        for f in glob.glob(os.path.join(os.path.expanduser(root), "**", "*.jsonl"), recursive=True):
+            index.setdefault(os.path.basename(f), f)
+    return index
+
+
+def backfill_authored_at(collection, session_dirs, apply=False):
+    """Stamp ``authored_at`` on convos drawers from their source transcript timestamps.
+
+    Returns a stats dict: ``scanned``, ``updated``, ``resolved_files``, ``unresolved_files``.
+    """
+    index = _index_sessions(session_dirs)
+    cache = {}
+    unresolved = set()
+    pending_ids, pending_metas = [], []
+    scanned = updated = 0
+
+    def flush():
+        nonlocal pending_ids, pending_metas, updated
+        if pending_ids and apply:
+            collection.update(ids=pending_ids, metadatas=pending_metas)
+        updated += len(pending_ids)
+        pending_ids, pending_metas = [], []
+
+    offset = 0
+    while True:
+        res = collection.get(
+            where={"ingest_mode": "convos"}, include=["metadatas"], limit=PAGE, offset=offset
+        )
+        ids = res["ids"]
+        if not ids:
+            break
+        for drawer_id, meta in zip(ids, res["metadatas"]):
+            scanned += 1
+            basename = os.path.basename(meta.get("source_file") or "")
+            if basename in cache:
+                authored = cache[basename]
+            else:
+                path = index.get(basename)
+                authored = _extract_authored_at(path) if path else None
+                cache[basename] = authored
+                if path is None and basename:
+                    unresolved.add(basename)
+            if authored and meta.get("authored_at") != authored:
+                new_meta = dict(meta)
+                new_meta["authored_at"] = authored
+                pending_ids.append(drawer_id)
+                pending_metas.append(new_meta)
+                if len(pending_ids) >= BATCH:
+                    flush()
+        offset += len(ids)
+    flush()
+    return {
+        "scanned": scanned,
+        "updated": updated,
+        "resolved_files": sum(1 for v in cache.values() if v),
+        "unresolved_files": len(unresolved),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--palace", required=True, help="Path to the ChromaDB palace dir")
+    parser.add_argument(
+        "--sessions",
+        action="append",
+        default=[],
+        required=True,
+        help="Directory holding .jsonl transcripts (repeatable)",
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Write changes (default is a dry run that only reports counts)",
+    )
+    args = parser.parse_args()
+
+    client = chromadb.PersistentClient(path=os.path.expanduser(args.palace))
+    collection = client.get_collection(COLLECTION)
+    stats = backfill_authored_at(collection, args.sessions, apply=args.apply)
+    mode = "APPLIED" if args.apply else "DRY-RUN (use --apply to write)"
+    print(
+        f"{mode}: scanned={stats['scanned']} updated={stats['updated']} "
+        f"resolved_files={stats['resolved_files']} unresolved_files={stats['unresolved_files']}"
+    )
+
+
+if __name__ == "__main__":
+    main()

skills/mempalace/SKILL.md

@@ -42,6 +42,6 @@ search-before-answer so the agent reads the palace instead of guessing.
 
 ## Cursor-specific notes
 
-- The `mempalace-mcp` server is auto-registered by this plugin. Once installed, all 34 MemPalace MCP tools (`mempalace_search`, `mempalace_add_drawer`, `mempalace_diary_write`, `mempalace_check_duplicate`, `mempalace_diary_read`, etc.) are available to the agent without any further configuration.
+- The `mempalace-mcp` server is auto-registered by this plugin. Once installed, all 36 MemPalace MCP tools (`mempalace_search`, `mempalace_add_drawer`, `mempalace_diary_write`, `mempalace_check_duplicate`, `mempalace_diary_read`, etc.) are available to the agent without any further configuration.
 - For automatic background saving every N agent turns plus session-start memory recall, also install the Cursor hooks separately by running `hooks/cursor/install.sh --scope user` from a cloned MemPalace repo. See [`website/guide/cursor-hooks.md`](../../website/guide/cursor-hooks.md) for the full walkthrough.
 - The recommended `agent_name` when calling `mempalace_diary_write` from a Cursor session is `cursor-ide` (matches the precedent of `claude-code` and `codex`).

tests/_chroma_palace_helper.py

@@ -0,0 +1,57 @@
+"""Shared helpers: create minimal valid palace-marker SQLite files for tests.
+
+Many tests want to stand up "a chroma / sqlite_exact palace" cheaply —
+historically they did this with ``(path / "<marker>.sqlite3").touch()`` or
+``write_bytes(b"")``, relying on the backend ``detect()`` methods' old
+``os.path.isfile()`` semantics. Post-#1893, both ``ChromaBackend.detect()``
+and ``SQLiteExactBackend.detect()`` require a valid SQLite magic header, so
+the empty stand-in no longer registers. These helpers create the minimum
+required to make detection fire without standing up a full palace.
+
+This module is intentionally not a ``test_*`` file: it ships utilities, not
+tests.
+"""
+
+import sqlite3
+from pathlib import Path
+from typing import Union
+
+
+def _write_minimal_sqlite_file(db_path: Path) -> None:
+    """Write a valid SQLite magic header at ``db_path``.
+
+    Writing any statement is sufficient to land the 16-byte
+    ``SQLite format 3\\x00`` magic prefix that the backend ``detect()``
+    methods check.
+    """
+
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute("CREATE TABLE _detect_smoke(x)")
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def make_minimal_chroma_sqlite(palace_path: Union[Path, str]) -> Path:
+    """Create ``<palace_path>/chroma.sqlite3`` with a valid SQLite header.
+
+    Returns the path to the file. Backs
+    :py:meth:`mempalace.backends.chroma.ChromaBackend.detect`.
+    """
+
+    db_path = Path(palace_path) / "chroma.sqlite3"
+    _write_minimal_sqlite_file(db_path)
+    return db_path
+
+
+def make_minimal_sqlite_exact_sqlite(palace_path: Union[Path, str]) -> Path:
+    """Create ``<palace_path>/sqlite_exact.sqlite3`` with a valid SQLite header.
+
+    Returns the path to the file. Backs
+    :py:meth:`mempalace.backends.sqlite_exact.SQLiteExactBackend.detect`.
+    """
+
+    db_path = Path(palace_path) / "sqlite_exact.sqlite3"
+    _write_minimal_sqlite_file(db_path)
+    return db_path

tests/conftest.py

@@ -77,6 +77,17 @@ def _reset_mcp_cache():
                 if hasattr(mcp_server, "_kg_by_path"):
                     mcp_server._kg_by_path.clear()
 
+                # Close (not just dereference) the cached chromadb client so its
+                # rust-side file handles are released; on Windows a bare deref
+                # leaves them locked and leaks across the session (#1128).
+                cached_client = getattr(mcp_server, "_client_cache", None)
+                if cached_client is not None:
+                    close = getattr(cached_client, "close", None)
+                    if callable(close):
+                        try:
+                            close()
+                        except Exception:
+                            pass
                 mcp_server._client_cache = None
                 mcp_server._collection_cache = None
                 if hasattr(mcp_server, "_collection_cache_backend"):
@@ -97,6 +108,29 @@ def _reset_mcp_cache():
         except (ImportError, AttributeError):
             pass
 
+        # Release chromadb clients opened through the backend layer. Many tests
+        # reach the store via palace.get_collection() (sweep, repair, CLI, ...),
+        # which caches one PersistentClient per palace_path on the long-lived
+        # backend singleton and never closes it. chromadb frees the rust-side
+        # SQLite/HNSW file handles only on client.close(); on POSIX the open
+        # handles are harmless, but on Windows they stay locked and accumulate
+        # across the session until a later test's HNSW segment write fails
+        # (#1128 Windows CI). close_palace() closes the client and drops the
+        # handle without marking the backend closed, so it stays reusable.
+        try:
+            from mempalace import palace as _palace
+
+            backend = getattr(_palace, "_DEFAULT_BACKEND", None)
+            clients = getattr(backend, "_clients", None)
+            if clients:
+                for path in list(clients):
+                    try:
+                        backend.close_palace(path)
+                    except Exception:
+                        pass
+        except (ImportError, AttributeError):
+            pass
+
     _clear_cache()
     yield
     _clear_cache()
@@ -154,7 +188,11 @@ def collection(palace_path):
     col = client.get_or_create_collection("mempalace_drawers", metadata={"hnsw:space": "cosine"})
     yield col
     client.delete_collection("mempalace_drawers")
-    del client
+    # close() (not a bare dereference) releases chromadb's rust-side SQLite/HNSW
+    # file handles. On Windows a mere `del` leaves them locked, so the temp
+    # palace cannot be removed and handles leak across the whole test session
+    # until a later test's HNSW write fails (#1128 Windows CI).
+    client.close()
 
 
 @pytest.fixture

tests/test_backends.py

@@ -178,12 +178,44 @@ def test_resolve_backend_priority_order(tmp_path):
     assert resolve_backend_for_palace() == "chroma"
 
 
-def test_chroma_detect_matches_palace_with_chroma_sqlite(tmp_path):
-    (tmp_path / "chroma.sqlite3").write_bytes(b"")
+def test_chroma_detect_matches_palace_with_sqlite_header(tmp_path):
+    """A real SQLite database at ``<path>/chroma.sqlite3`` registers as chroma.
+
+    Uses ``sqlite3.connect`` + a write so the SQLite magic header is actually
+    on disk — the only thing detection looks at.
+    """
+    db_path = tmp_path / "chroma.sqlite3"
+    conn = sqlite3.connect(db_path)
+    conn.execute("CREATE TABLE detect_smoke(x)")
+    conn.commit()
+    conn.close()
     assert ChromaBackend.detect(str(tmp_path)) is True
     assert ChromaBackend.detect(str(tmp_path.parent)) is False
 
 
+def test_chroma_detect_rejects_empty_chroma_sqlite(tmp_path):
+    """A 0-byte ``chroma.sqlite3`` is not a chroma palace (closes #1893).
+
+    Bare ``sqlite3.connect()`` against a missing path leaves a 0-byte file
+    behind because the SQLite header is written on the first statement, not
+    on connect. Detection must reject that artifact so it cannot trip
+    ``BackendMismatchError`` against a real non-chroma backend marker in the
+    same directory.
+    """
+    (tmp_path / "chroma.sqlite3").write_bytes(b"")
+    assert ChromaBackend.detect(str(tmp_path)) is False
+
+
+def test_chroma_detect_rejects_non_sqlite_file(tmp_path):
+    """A non-SQLite file at the ``chroma.sqlite3`` path is not chroma.
+
+    Defends against partial writes / garbage content / anything that lands at
+    the canonical path but isn't actually a SQLite database.
+    """
+    (tmp_path / "chroma.sqlite3").write_bytes(b"not a sqlite file" * 4)
+    assert ChromaBackend.detect(str(tmp_path)) is False
+
+
 def test_chroma_lexical_search_uses_sqlite_fts_not_full_collection_scan(tmp_path):
     db_path = tmp_path / "chroma.sqlite3"
     conn = sqlite3.connect(db_path)
@@ -1810,7 +1842,7 @@ def test_explain_ef_mismatch_recognizes_chromadb_conflict():
     assert msg is not None
     assert "/tmp/palace.db" in msg
     assert "MEMPALACE_EMBEDDING_MODEL" in msg
-    assert "rebuild-index" in msg
+    assert "mempalace --palace /tmp/palace.db repair rebuild-index" in msg
 
 
 def test_explain_ef_mismatch_returns_none_for_unrelated_errors():

tests/test_backfill_authored_at.py

@@ -0,0 +1,86 @@
+"""Integration tests for the authored_at backfill migration (scripts/)."""
+
+import importlib.util
+import uuid
+from pathlib import Path
+
+import chromadb
+
+# The migration ships as a script, not a package module; load it directly.
+_SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "backfill_authored_at.py"
+_spec = importlib.util.spec_from_file_location("backfill_authored_at", _SCRIPT)
+backfill_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(backfill_mod)
+
+
+def _collection():
+    # Unique name per call: EphemeralClient shares one in-memory instance across the
+    # process, so a fixed collection name would leak drawers between tests.
+    client = chromadb.EphemeralClient()
+    return client.create_collection(f"drawers_{uuid.uuid4().hex}")
+
+
+def _add(col, drawer_id, source_file, authored_at=None):
+    meta = {"ingest_mode": "convos", "source_file": source_file, "filed_at": "2026-06-27T00:00:00"}
+    if authored_at is not None:
+        meta["authored_at"] = authored_at
+    col.add(ids=[drawer_id], documents=["hello"], metadatas=[meta], embeddings=[[0.1, 0.2, 0.3]])
+
+
+def _transcript(dir_path, name, *timestamps):
+    dir_path.mkdir(parents=True, exist_ok=True)
+    f = dir_path / name
+    f.write_text("".join(f'{{"timestamp": "{ts}"}}\n' for ts in timestamps))
+    return f
+
+
+def test_backfill_sets_latest_timestamp(tmp_path):
+    sessions = tmp_path / "claude"
+    _transcript(sessions, "abc.jsonl", "2026-06-10T08:00:00.000Z", "2026-06-12T09:00:00.000Z")
+    col = _collection()
+    # Stored source_file uses an old mount prefix; resolution is by basename.
+    _add(col, "d1", "/old/mount/abc.jsonl")
+
+    stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True)
+
+    assert stats["scanned"] == 1
+    assert stats["updated"] == 1
+    got = col.get(ids=["d1"], include=["metadatas"])["metadatas"][0]
+    assert got["authored_at"] == "2026-06-12T09:00:00.000Z"
+
+
+def test_dry_run_writes_nothing(tmp_path):
+    sessions = tmp_path / "claude"
+    _transcript(sessions, "abc.jsonl", "2026-06-12T09:00:00.000Z")
+    col = _collection()
+    _add(col, "d1", "/old/mount/abc.jsonl")
+
+    stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=False)
+
+    assert stats["updated"] == 1  # would update
+    assert "authored_at" not in col.get(ids=["d1"], include=["metadatas"])["metadatas"][0]
+
+
+def test_idempotent_second_run_updates_nothing(tmp_path):
+    sessions = tmp_path / "claude"
+    _transcript(sessions, "abc.jsonl", "2026-06-12T09:00:00.000Z")
+    col = _collection()
+    _add(col, "d1", "/old/mount/abc.jsonl")
+
+    backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True)
+    stats2 = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True)
+
+    assert stats2["updated"] == 0
+
+
+def test_unresolved_transcript_is_left_alone(tmp_path):
+    sessions = tmp_path / "claude"
+    sessions.mkdir()
+    col = _collection()
+    _add(col, "d1", "/old/mount/missing.jsonl")  # no file on disk
+
+    stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True)
+
+    assert stats["updated"] == 0
+    assert stats["unresolved_files"] == 1
+    assert "authored_at" not in col.get(ids=["d1"], include=["metadatas"])["metadatas"][0]

tests/test_clean_nul_bytes.py

@@ -0,0 +1,163 @@
+"""Tests for embedded-NUL sanitisation in the bulk-ingest write path.
+
+Mirrors ``test_clean_lone_surrogates.py`` (issue #1235) for a sibling class of
+bad input: a document containing an embedded NUL character (U+0000).
+
+Unlike a lone surrogate, a NUL-containing string is well-formed UTF-8 — it
+doesn't raise on ``.encode()``. But handing it to ChromaDB's SQLite/FTS5
+layer can corrupt the FTS5 inverted index for the *whole* collection
+(``PRAGMA quick_check`` reports "malformed inverted index for FTS5 table"),
+not just fail to store that one document. This is routine in mined Bash
+tool output — e.g. a reader racing a background writer, or genuine
+NUL-delimited/binary command output captured verbatim.
+
+``sanitize_content`` (the MCP write-tool path) already rejects NUL bytes
+outright (``ValueError``) — that behaviour is unchanged and still tested
+here for completeness. The bulk ingest paths (miner, convo_miner, sweeper,
+diary_ingest) build documents without routing through that helper and reach
+``ChromaCollection`` directly, exactly as #1235 found for surrogates. This
+file covers the chokepoint fix that closes that gap: replace, not reject, so
+a NUL-containing chunk doesn't abort or corrupt an otherwise-good mine.
+"""
+
+from mempalace.config import strip_nul_bytes
+
+
+# ── Unit tests ─────────────────────────────────────────────────────────────
+
+
+class TestStripNulBytes:
+    def test_passthrough_normal(self):
+        assert strip_nul_bytes("hello world") == "hello world"
+        assert strip_nul_bytes("你好世界") == "你好世界"
+
+    def test_empty_string(self):
+        assert strip_nul_bytes("") == ""
+
+    def test_replaces_single_nul(self):
+        assert strip_nul_bytes("hello\x00world") == "hello�world"
+
+    def test_replaces_multiple_nuls(self):
+        assert strip_nul_bytes("\x00\x00\x00") == "���"
+        assert strip_nul_bytes("a\x00b\x00c") == "a�b�c"
+
+    def test_replaces_large_nul_run(self):
+        """The reporter-shaped case: a large contiguous NUL run inside
+        otherwise-ordinary captured tool output."""
+        dirty = "before-nuls " + ("\x00" * 10291) + " after-nuls"
+        clean = strip_nul_bytes(dirty)
+        assert "\x00" not in clean
+        assert clean.startswith("before-nuls ")
+        assert clean.endswith(" after-nuls")
+        assert clean.count("�") == 10291
+
+    def test_preserves_real_content_around_nuls(self):
+        assert (
+            strip_nul_bytes("ordinary log line one\x00\x00\x00ordinary log line two")
+            == "ordinary log line one���ordinary log line two"
+        )
+
+
+# ── Existing sanitizer behavior (unchanged) ─────────────────────────────────
+
+
+class TestSanitizeContentStillRejectsNuls:
+    """sanitize_content (the MCP write-tool path) already guards against NUL
+    bytes by rejecting them outright — that's a reasonable choice for an
+    interactive tool call and is untouched by this fix. Verifies it still
+    holds now that a sibling path (the backend chokepoint) takes the
+    replace-not-reject approach instead, for the bulk-ingest paths where
+    rejecting one bad chunk would otherwise abort or corrupt a whole mine.
+    """
+
+    def test_sanitize_content_rejects_nul(self):
+        from mempalace.config import sanitize_content
+
+        import pytest
+
+        with pytest.raises(ValueError, match="null bytes"):
+            sanitize_content("hello\x00world")
+
+
+# ── Backend chokepoint (bulk ingest paths) ──────────────────────────────────
+
+
+class _CapturingCollection:
+    """Minimal chromadb.Collection stand-in that records the kwargs it receives,
+    so we can assert what actually reaches the chromadb client."""
+
+    def __init__(self):
+        self.calls = []
+
+    def add(self, **kwargs):
+        self.calls.append(("add", kwargs))
+
+    def upsert(self, **kwargs):
+        self.calls.append(("upsert", kwargs))
+
+    def update(self, **kwargs):
+        self.calls.append(("update", kwargs))
+
+
+class TestBackendChokepointStripsNulBytes:
+    """The bulk ingest paths (miner, convo_miner, sweeper, diary_ingest) build
+    documents without routing through ``sanitize_content`` and reach
+    ``ChromaCollection`` directly — the same gap #1235 found for surrogates.
+    A NUL byte in *document* text can corrupt the FTS5 inverted index for the
+    whole collection. The backend chokepoint must strip it, mirroring the
+    existing surrogate handling in the same method."""
+
+    @staticmethod
+    def _collection():
+        from mempalace.backends.chroma import ChromaCollection
+
+        fake = _CapturingCollection()
+        return fake, ChromaCollection(fake)
+
+    def test_add_strips_nul_in_document(self):
+        fake, col = self._collection()
+        col.add(documents=["clean\x00doc"], ids=["1"])
+        _, kwargs = fake.calls[0]
+        assert kwargs["documents"] == ["clean�doc"]
+
+    def test_upsert_strips_nul_in_document(self):
+        fake, col = self._collection()
+        col.upsert(documents=["a\x00b"], ids=["1"], metadatas=[{"wing": "w"}])
+        _, kwargs = fake.calls[0]
+        assert kwargs["documents"] == ["a�b"]
+
+    def test_update_strips_nul_in_document(self):
+        fake, col = self._collection()
+        col.update(ids=["1"], documents=["x\x00y"])
+        _, kwargs = fake.calls[0]
+        assert kwargs["documents"] == ["x�y"]
+
+    def test_one_nul_message_does_not_corrupt_the_batch(self):
+        """The reporter's shape: a single NUL-heavy row alongside ordinary
+        ones in the same batch. All rows survive, sanitised."""
+        fake, col = self._collection()
+        col.upsert(
+            documents=["ok one", "poison\x00row", "ok three"],
+            ids=["1", "2", "3"],
+        )
+        _, kwargs = fake.calls[0]
+        assert kwargs["documents"] == ["ok one", "poison�row", "ok three"]
+        assert len(kwargs["ids"]) == 3
+
+    def test_both_surrogates_and_nuls_in_same_document(self):
+        """Both sanitizers apply to the same document — order doesn't
+        leave either kind of bad character behind."""
+        fake, col = self._collection()
+        col.upsert(documents=["a\udc95b\x00c"], ids=["1"])
+        _, kwargs = fake.calls[0]
+        assert kwargs["documents"] == ["a�b�c"]
+
+    def test_single_string_document_is_not_split_into_chars(self):
+        """chromadb accepts a bare str as one document (OneOrMany[Document]).
+        The sanitiser must keep it whole and clean, not split it into
+        per-character documents."""
+        fake, col = self._collection()
+        col.upsert(documents="one\x00document", ids=["1"])
+        _, kwargs = fake.calls[0]
+        assert kwargs["documents"] == "one�document"
+        assert isinstance(kwargs["documents"], str)

tests/test_cli.py

@@ -975,6 +975,16 @@ def test_main_repair_dispatches():
         mock_cmd.assert_called_once()
 
 
+def test_main_repair_rebuild_index_dispatches():
+    with (
+        patch("sys.argv", ["mempalace", "repair", "rebuild-index"]),
+        patch("mempalace.cli.cmd_repair") as mock_cmd,
+    ):
+        main()
+        args = mock_cmd.call_args.args[0]
+        assert args.repair_action == "rebuild-index"
+
+
 def test_main_compress_dispatches():
     with (
         patch("sys.argv", ["mempalace", "compress"]),
@@ -1767,3 +1777,57 @@ def test_cmd_repair_from_sqlite_success_does_not_exit(mock_config_cls, tmp_path)
     with patch("mempalace.repair.rebuild_from_sqlite", return_value=fake_counts):
         # Should return cleanly; no SystemExit raised.
         cmd_repair(args)
+
+
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_from_sqlite_cleanup_failure_exits_nonzero(mock_config_cls, tmp_path, capsys):
+    from mempalace.repair import RebuildCleanupError
+
+    palace_dir = tmp_path / "palace"
+    source_dir = tmp_path / "source"
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+    args = argparse.Namespace(
+        palace=str(palace_dir),
+        mode="from-sqlite",
+        source=str(source_dir),
+        archive_existing=False,
+        yes=True,
+    )
+    failure = RebuildCleanupError(
+        "cleanup failed",
+        counts={"mempalace_drawers": 1},
+        dest_palace=str(palace_dir),
+        archive_path=None,
+    )
+    with patch("mempalace.repair.rebuild_from_sqlite", side_effect=failure):
+        with pytest.raises(SystemExit) as excinfo:
+            cmd_repair(args)
+
+    assert excinfo.value.code == 1
+    assert "Rebuild cleanup failed" in capsys.readouterr().out
+
+
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_repair_rebuild_index_alias_uses_sqlite_archive(mock_config_cls, tmp_path):
+    """``repair rebuild-index`` must bypass Chroma reads and rebuild from SQLite."""
+    palace_dir = tmp_path / "palace"
+    palace_dir.mkdir()
+    mock_config_cls.return_value.palace_path = str(palace_dir)
+
+    args = argparse.Namespace(
+        palace=str(palace_dir),
+        repair_action="rebuild-index",
+        mode="legacy",
+        source=None,
+        archive_existing=False,
+        yes=True,
+    )
+    fake_counts = {"mempalace_drawers": 1, "mempalace_closets": 0}
+    with patch("mempalace.repair.rebuild_from_sqlite", return_value=fake_counts) as rebuild:
+        cmd_repair(args)
+
+    rebuild.assert_called_once_with(
+        source_palace=str(palace_dir),
+        dest_palace=str(palace_dir),
+        archive_existing_dest=True,
+    )

tests/test_cli_hallways.py

@@ -0,0 +1,74 @@
+"""Tests for the `hallways` CLI command."""
+
+from argparse import Namespace
+
+import mempalace.hallways as hallways_mod
+from mempalace.cli import cmd_hallways
+
+
+def test_lists_sorted_by_count(monkeypatch, capsys):
+    rows = [
+        {
+            "entity_a": "C",
+            "entity_b": "D",
+            "co_occurrence_count": 1,
+            "wing": "w",
+            "label": "C <-> D (x1)",
+        },
+        {
+            "entity_a": "A",
+            "entity_b": "B",
+            "co_occurrence_count": 3,
+            "wing": "w",
+            "label": "A <-> B (x3)",
+        },
+    ]
+    monkeypatch.setattr(hallways_mod, "list_hallways", lambda wing=None, config=None: list(rows))
+    cmd_hallways(Namespace(wing=None, limit=50))
+    out = capsys.readouterr().out
+    assert "2 hallway(s)" in out
+    assert "A <-> B (x3)" in out
+    # Highest co-occurrence first.
+    assert out.index("A <-> B") < out.index("C <-> D")
+
+
+def test_respects_limit(monkeypatch, capsys):
+    rows = [
+        {"entity_a": f"E{i}", "entity_b": "X", "co_occurrence_count": i, "label": f"E{i} <-> X"}
+        for i in range(5)
+    ]
+    monkeypatch.setattr(hallways_mod, "list_hallways", lambda wing=None, config=None: list(rows))
+    cmd_hallways(Namespace(wing=None, limit=2))
+    assert capsys.readouterr().out.count("<->") == 2
+
+
+def test_negative_limit_shows_nothing_not_tail(monkeypatch, capsys):
+    rows = [
+        {"entity_a": f"E{i}", "entity_b": "X", "co_occurrence_count": i, "label": f"E{i} <-> X"}
+        for i in range(5)
+    ]
+    monkeypatch.setattr(hallways_mod, "list_hallways", lambda wing=None, config=None: list(rows))
+    cmd_hallways(Namespace(wing=None, limit=-2))
+    # A negative limit must not slice from the end (which would print all-but-2).
+    assert capsys.readouterr().out.count("<->") == 0
+
+
+def test_empty_message(monkeypatch, capsys):
+    monkeypatch.setattr(hallways_mod, "list_hallways", lambda wing=None, config=None: [])
+    cmd_hallways(Namespace(wing="x", limit=50))
+    assert "No hallways yet" in capsys.readouterr().out
+
+
+def test_explicit_palace_scopes_hallway_listing(monkeypatch, tmp_path):
+    calls = []
+
+    def fake_list(wing=None, config=None):
+        calls.append((wing, config.palace_path))
+        return []
+
+    selected = tmp_path / "selected" / "palace"
+    monkeypatch.setattr(hallways_mod, "list_hallways", fake_list)
+
+    cmd_hallways(Namespace(wing="wing_aya", limit=50, palace=str(selected)))
+
+    assert calls == [("wing_aya", str(selected))]

tests/test_config.py

@@ -70,6 +70,41 @@ def test_qdrant_config_from_env_and_file(tmp_path, monkeypatch):
     assert cfg.qdrant_timeout == 3.5
 
 
+def test_milvus_config_from_env_and_file(tmp_path, monkeypatch):
+    with open(tmp_path / "config.json", "w") as f:
+        json.dump(
+            {
+                "milvus_uri": "https://config.example",
+                "milvus_token": "config-token",
+                "milvus_db_name": "config-db",
+                "milvus_namespace": "config-ns",
+                "milvus_consistency_level": "bounded",
+            },
+            f,
+        )
+    monkeypatch.setenv("MEMPALACE_MILVUS_URI", "https://env.example")
+    monkeypatch.setenv("MEMPALACE_MILVUS_TOKEN", "env-token")
+    monkeypatch.setenv("MEMPALACE_MILVUS_DB_NAME", "env-db")
+    monkeypatch.setenv("MEMPALACE_MILVUS_NAMESPACE", "env-ns")
+    monkeypatch.setenv("MEMPALACE_MILVUS_CONSISTENCY_LEVEL", "eventually")
+
+    cfg = MempalaceConfig(config_dir=str(tmp_path))
+
+    assert cfg.milvus_uri == "https://env.example"
+    assert cfg.milvus_token == "env-token"
+    assert cfg.milvus_db_name == "env-db"
+    assert cfg.milvus_namespace == "env-ns"
+    assert cfg.milvus_consistency_level == "Eventually"
+
+
+def test_milvus_config_rejects_invalid_consistency_level(tmp_path, monkeypatch):
+    monkeypatch.setenv("MEMPALACE_MILVUS_CONSISTENCY_LEVEL", "linearizable")
+    cfg = MempalaceConfig(config_dir=str(tmp_path))
+
+    with pytest.raises(ValueError, match="milvus_consistency_level"):
+        cfg.milvus_consistency_level
+
+
 def test_set_backend_persists_choice(tmp_path):
     cfg = MempalaceConfig(config_dir=str(tmp_path))
     cfg.set_backend("sqlite_exact")
@@ -853,3 +888,18 @@ def test_max_backups_bad_env_falls_back_to_config(monkeypatch, tmp_path):
     monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "garbage")
     cfg = MempalaceConfig(config_dir=str(tmp_path))
     assert cfg.max_backups == 4
+
+
+def test_explicit_palace_path_overrides_env_and_file_config(monkeypatch, tmp_path):
+    configured = tmp_path / "configured" / "palace"
+    explicit = tmp_path / "explicit" / "../explicit" / "palace"
+    with open(tmp_path / "config.json", "w") as f:
+        json.dump({"palace_path": str(configured)}, f)
+    monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_path / "environment" / "palace"))
+
+    cfg = MempalaceConfig(config_dir=str(tmp_path), palace_path=str(explicit))
+
+    expected = os.path.abspath(os.path.expanduser(str(explicit)))
+    assert cfg.palace_path == expected
+    assert cfg.hallway_file == os.path.join(os.path.dirname(expected), "hallways.json")
+    assert cfg.tunnel_file == os.path.join(os.path.dirname(expected), "tunnels.json")

tests/test_config_palace_path.py

@@ -0,0 +1,33 @@
+"""Tests for palace_path tilde expansion in MempalaceConfig."""
+
+import os
+from mempalace.config import MempalaceConfig
+
+
+def test_palace_path_expands_tilde_from_config_file():
+    """palace_path must expand ~ even when read from config.json, not env."""
+    cfg = MempalaceConfig()
+    cfg._file_config["palace_path"] = "~/.mempalace/palace"
+    result = cfg.palace_path
+    assert not result.startswith("~"), (
+        f"palace_path returned unexpanded tilde: {result!r}. "
+        "This causes mempalace mine to create a literal '~' directory "
+        "relative to CWD instead of writing to the home directory."
+    )
+    assert result == os.path.expanduser("~/.mempalace/palace")
+
+
+def test_palace_path_expands_tilde_nested():
+    """Nested tilde paths (e.g. ~/custom/palace) are also expanded."""
+    cfg = MempalaceConfig()
+    cfg._file_config["palace_path"] = "~/custom/mempalace"
+    result = cfg.palace_path
+    assert not result.startswith("~")
+    assert result == os.path.expanduser("~/custom/mempalace")
+
+
+def test_palace_path_absolute_unchanged():
+    """Absolute paths pass through without modification."""
+    cfg = MempalaceConfig()
+    cfg._file_config["palace_path"] = "/tmp/test_palace"
+    assert cfg.palace_path == "/tmp/test_palace"

tests/test_convo_miner.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 import shutil
+import time
 from pathlib import Path
 
 import chromadb
@@ -8,10 +9,11 @@ import pytest
 
 from mempalace.convo_miner import (
     _is_ai_tool_path,
+    _register_file,
     _resolve_wing,
     mine_convos,
 )
-from mempalace.palace import MineAlreadyRunning, file_already_mined
+from mempalace.palace import MineAlreadyRunning, file_already_mined, prefetch_mined_set
 
 
 def test_convo_mining():
@@ -454,3 +456,243 @@ def test_mine_convos_limit_skips_already_mined(capsys):
                 break
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+# ── mtime-aware re-mining ────────────────────────────────────────────
+#
+# Conversation transcripts are NOT immutable: a Claude Code session keeps
+# appending to its own file while active, and /compact or /clear can
+# rewrite one in place. These tests cover the fix -- convo mining used to
+# treat "we've seen this source_file before" as sufficient to skip it
+# forever (transcripts were assumed immutable), silently missing content
+# appended after the first mine.
+
+
+def test_mine_convos_reprocesses_when_file_grows(capsys):
+    """A session file that grows after being mined must be picked up on
+    the next mine, not skipped forever."""
+    tmpdir = tempfile.mkdtemp()
+    try:
+        convo_path = Path(tmpdir) / "session.txt"
+        convo_path.write_text(
+            "> What is the plan?\nStart with the schema, then the API.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n"
+        )
+        palace_path = os.path.join(tmpdir, "palace")
+
+        mine_convos(tmpdir, palace_path, wing="test")
+        capsys.readouterr()
+
+        # Simulate the session being extended: real content added, mtime
+        # bumped forward (avoids same-second mtime resolution flakiness).
+        convo_path.write_text(
+            "> What is the plan?\nStart with the schema, then the API.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n\n"
+            "> UNIQUE_GROWN_SESSION_MARKER, did we resolve it?\n"
+            "Yes, resolved by locking the migration order explicitly.\n"
+        )
+        future = time.time() + 60
+        os.utime(convo_path, (future, future))
+
+        mine_convos(tmpdir, palace_path, wing="test")
+        out = capsys.readouterr().out
+        assert "Files skipped (already filed): 1" not in out
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        docs = col.get(include=["documents"])["documents"]
+        assert any("UNIQUE_GROWN_SESSION_MARKER" in d for d in docs), (
+            "grown session content was not picked up on re-mine"
+        )
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def test_mine_convos_unchanged_file_still_skipped(capsys):
+    """A file whose content and mtime are unchanged must still be skipped
+    -- the mtime check must not defeat the existing skip-on-unchanged
+    optimization."""
+    tmpdir = tempfile.mkdtemp()
+    try:
+        convo_path = Path(tmpdir) / "session.txt"
+        convo_path.write_text(
+            "> What is the plan?\nStart with the schema, then the API.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n"
+        )
+        palace_path = os.path.join(tmpdir, "palace")
+
+        mine_convos(tmpdir, palace_path, wing="test")
+        capsys.readouterr()
+
+        mine_convos(tmpdir, palace_path, wing="test")
+        out = capsys.readouterr().out
+        assert "Files skipped (already filed): 1" in out
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def test_mine_convos_grown_file_purges_stale_drawers_not_additive(capsys):
+    """Re-mining a grown file must not leave duplicate/stale drawers behind
+    -- purge-then-insert, not additive accumulation. Checks content
+    directly (a drawer count comparison is fragile: ChromaDB collections
+    can carry non-drawer bookkeeping rows unrelated to this behavior)."""
+    tmpdir = tempfile.mkdtemp()
+    try:
+        convo_path = Path(tmpdir) / "session.txt"
+        convo_path.write_text(
+            "> What is the plan?\nUNIQUE_ORIGINAL_EXCHANGE_MARKER here.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n"
+        )
+        palace_path = os.path.join(tmpdir, "palace")
+
+        mine_convos(tmpdir, palace_path, wing="test")
+        capsys.readouterr()
+
+        convo_path.write_text(
+            "> What is the plan?\nUNIQUE_ORIGINAL_EXCHANGE_MARKER here.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n\n"
+            "> One more exchange?\nUNIQUE_NEW_EXCHANGE_MARKER here.\n"
+        )
+        future = time.time() + 60
+        os.utime(convo_path, (future, future))
+        mine_convos(tmpdir, palace_path, wing="test")
+        capsys.readouterr()
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        docs = col.get(include=["documents"])["documents"]
+
+        original_hits = sum(1 for d in docs if "UNIQUE_ORIGINAL_EXCHANGE_MARKER" in d)
+        new_hits = sum(1 for d in docs if "UNIQUE_NEW_EXCHANGE_MARKER" in d)
+        assert original_hits == 1, (
+            f"original exchange duplicated across re-mine: {original_hits} copies"
+        )
+        assert new_hits == 1, f"new exchange should appear exactly once, got {new_hits}"
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def test_prefetch_mined_set_returns_stored_mtime():
+    """prefetch_mined_set's dict carries each source_file's stored mtime,
+    not just membership."""
+    tmpdir = tempfile.mkdtemp()
+    try:
+        convo_path = Path(tmpdir) / "session.txt"
+        convo_path.write_text(
+            "> What is the plan?\nStart with the schema, then the API.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n"
+        )
+        palace_path = os.path.join(tmpdir, "palace")
+        mine_convos(tmpdir, palace_path, wing="test")
+
+        resolved_file = str(convo_path.resolve())
+        actual_mtime = os.path.getmtime(resolved_file)
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        mined = prefetch_mined_set(col, extract_mode="exchange")
+
+        assert resolved_file in mined
+        assert mined[resolved_file] is not None
+        assert abs(mined[resolved_file] - actual_mtime) < 0.001
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def test_prefetch_mined_set_none_for_drawer_without_stored_mtime():
+    """A drawer written before source_mtime existed (or with getmtime
+    failure at write time) must surface as None, not be silently absent --
+    None must be treated as stale by callers, not as 'unknown, assume ok'."""
+    tmpdir = tempfile.mkdtemp()
+    try:
+        palace_path = os.path.join(tmpdir, "palace")
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_or_create_collection("mempalace_drawers")
+        col.upsert(
+            ids=["drawer_legacy_1"],
+            documents=["legacy content with no source_mtime field"],
+            metadatas=[
+                {
+                    "wing": "test",
+                    "room": "general",
+                    "source_file": "/fake/legacy/file.txt",
+                    "chunk_index": 0,
+                    "extract_mode": "exchange",
+                    "normalize_version": 999,  # force >= current version
+                }
+            ],
+        )
+        mined = prefetch_mined_set(col, extract_mode="exchange")
+        assert "/fake/legacy/file.txt" in mined
+        assert mined["/fake/legacy/file.txt"] is None
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def test_mine_convos_reprocesses_legacy_drawer_without_stored_mtime(capsys):
+    """A file mined before source_mtime was tracked (simulated: drawer
+    written directly, no source_mtime field) must be re-mined on the next
+    run, not skipped forever -- this is the one-time backfill behavior."""
+    tmpdir = tempfile.mkdtemp()
+    try:
+        convo_path = Path(tmpdir) / "session.txt"
+        convo_path.write_text(
+            "> What is the plan?\nUNIQUE_LEGACY_BACKFILL_MARKER here.\n\n"
+            "> Any risks?\nMigration ordering is the main one.\n"
+        )
+        resolved_file = str(convo_path.resolve())
+        palace_path = os.path.join(tmpdir, "palace")
+
+        # Simulate a pre-existing drawer from before source_mtime existed.
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_or_create_collection("mempalace_drawers")
+        from mempalace.palace import NORMALIZE_VERSION
+
+        col.upsert(
+            ids=["drawer_legacy_session_1"],
+            documents=["stale legacy content, no mtime field"],
+            metadatas=[
+                {
+                    "wing": "test",
+                    "room": "general",
+                    "source_file": resolved_file,
+                    "chunk_index": 0,
+                    "extract_mode": "exchange",
+                    "normalize_version": NORMALIZE_VERSION,
+                }
+            ],
+        )
+        del col, client
+
+        mine_convos(tmpdir, palace_path, wing="test")
+        out = capsys.readouterr().out
+        assert "Files skipped (already filed): 1" not in out
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        docs = col.get(include=["documents"])["documents"]
+        assert any("UNIQUE_LEGACY_BACKFILL_MARKER" in d for d in docs)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def test_register_file_sentinel_includes_source_mtime():
+    """The 0-chunk sentinel must stamp source_mtime too, so a file that
+    later grows past the min-chunk-size floor is detected as changed
+    instead of being skipped forever by the sentinel."""
+    tmpdir = tempfile.mkdtemp()
+    try:
+        tiny_file = Path(tmpdir) / "tiny.txt"
+        tiny_file.write_text("hi")
+        palace_path = os.path.join(tmpdir, "palace")
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_or_create_collection("mempalace_drawers")
+
+        _register_file(col, str(tiny_file), "test", "mempalace", "exchange")
+
+        mined = prefetch_mined_set(col, extract_mode="exchange")
+        assert str(tiny_file) in mined
+        assert mined[str(tiny_file)] is not None
+        assert abs(mined[str(tiny_file)] - os.path.getmtime(tiny_file)) < 0.001
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)

tests/test_convo_miner_unit.py

@@ -8,6 +8,7 @@ import pytest
 from mempalace.convo_miner import (
     CHUNK_SIZE,
     _emit_bounded,
+    _extract_authored_at,
     _file_chunks_locked,
     chunk_exchanges,
     detect_convo_room,
@@ -362,6 +363,30 @@ class TestScanConvos:
         files = scan_convos(str(tmp_path))
         assert files == []
 
+    def test_scan_skips_tool_results_dirs(self, tmp_path):
+        # Claude Code pages large tool outputs to <session>/tool-results/*.txt
+        # inside ~/.claude/projects/<slug>/. These are raw machine dumps
+        # referenced from the transcript JSONL, not conversations — mining
+        # them floods the palace (12.8k drawers measured in the field, one
+        # single file produced 3.6k). The scanner must not descend into them.
+        session_dir = tmp_path / "1234-5678-session"
+        tool_results = session_dir / "tool-results"
+        tool_results.mkdir(parents=True)
+        (tool_results / "bipc8jdx0.txt").write_text("raw tool dump " * 100, encoding="utf-8")
+        (tmp_path / "session.jsonl").write_text('{"type": "user"}', encoding="utf-8")
+        files = scan_convos(str(tmp_path))
+        names = [f.name for f in files]
+        assert "session.jsonl" in names
+        assert "bipc8jdx0.txt" not in names
+
+    def test_scan_keeps_regular_nested_dirs(self, tmp_path):
+        # The tool-results skip must not turn into a blanket nested-dir skip.
+        nested = tmp_path / "archive"
+        nested.mkdir()
+        (nested / "old-chat.md").write_text("> q\na\n> q2\na2\n> q3\na3", encoding="utf-8")
+        files = scan_convos(str(tmp_path))
+        assert [f.name for f in files] == ["old-chat.md"]
+
     @pytest.mark.skipif(
         sys.platform == "win32",
         reason="symlink creation requires elevated privileges on Windows",
@@ -431,6 +456,59 @@ class TestScanConvos:
         assert "deep/subdir/nested.jsonl" in err
         assert "(symlink)" in err
 
+    def test_scan_skips_oversized_files(self, tmp_path, capsys, monkeypatch):
+        import re
+
+        import mempalace.convo_miner as convo_mod
+
+        monkeypatch.setattr(convo_mod, "MAX_FILE_SIZE", 100)
+
+        (tmp_path / "small.txt").write_text("hello " * 5, encoding="utf-8")
+        (tmp_path / "big.txt").write_text("hello " * 100, encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = [f.name for f in files]
+        assert "small.txt" in names
+        assert "big.txt" not in names
+
+        err = capsys.readouterr().err
+        # SKIP message goes to stderr, matching the existing
+        # `SKIP: <rel> (symlink)` line in the same function.
+        assert "SKIP: big.txt" in err
+        # Validate the full template so a drop of the MB suffix or a
+        # regression to bare-substring output trips the test.
+        assert re.search(r"SKIP: big\.txt \(\d+\.\d+ MB\) exceeds \d+ MB limit", err), err
+
+    def test_scan_skips_unreadable_files(self, tmp_path, capsys, monkeypatch):
+        from pathlib import Path
+
+        # .txt is in CONVO_EXTENSIONS so it reaches the size-check gate.
+        (tmp_path / "readable.txt").write_text("hi", encoding="utf-8")
+        unreadable = tmp_path / "unreadable.txt"
+        unreadable.write_text("hi", encoding="utf-8")
+
+        real_stat = Path.stat
+
+        def selective_stat(self, *args, **kwargs):
+            # On Py 3.10+, Path.is_symlink() routes through lstat ->
+            # stat(follow_symlinks=False). Only raise for the follow-symlinks
+            # call that the actual size-check makes, otherwise the test
+            # never reaches the size-check arm we want to exercise.
+            if self.name == "unreadable.txt" and kwargs.get("follow_symlinks", True):
+                raise PermissionError(13, "Permission denied", str(self))
+            return real_stat(self, *args, **kwargs)
+
+        monkeypatch.setattr(Path, "stat", selective_stat)
+
+        files = scan_convos(str(tmp_path))
+        names = [f.name for f in files]
+        assert "readable.txt" in names
+        assert "unreadable.txt" not in names
+
+        err = capsys.readouterr().err
+        assert "SKIP: unreadable.txt" in err
+        assert "stat error" in err
+
 
 class TestFileChunksLocked:
     def test_uses_bounded_upsert_batches(self, monkeypatch):
@@ -468,3 +546,100 @@ class TestFileChunksLocked:
         assert dict(room_counts) == {}
         assert skipped is False
         assert col.batch_sizes == [2, 2, 1]
+
+    def test_populates_entities_metadata(self, monkeypatch):
+        import mempalace.convo_miner as convo_miner
+
+        class FakeCol:
+            def __init__(self):
+                self.metas = []
+
+            def delete(self, *args, **kwargs):
+                pass
+
+            def get(self, ids=None, include=None, **kwargs):
+                return {"ids": [], "metadatas": []}
+
+            def upsert(self, documents, ids, metadatas):
+                self.metas.extend(metadatas)
+
+        chunks = [
+            {
+                "content": "We changed `MemoryStack` in rag/foo.py via do_thing_now().",
+                "chunk_index": 0,
+            }
+        ]
+        col = FakeCol()
+        monkeypatch.setattr(
+            convo_miner, "file_already_mined", lambda collection, source_file, **kwargs: False
+        )
+        monkeypatch.setattr(convo_miner, "mine_lock", lambda source_file: contextlib.nullcontext())
+        monkeypatch.setattr(convo_miner, "_detect_hall_cached", lambda content: "conversations")
+
+        _file_chunks_locked(col, "chat.txt", chunks, "wing", "general", "agent", "exchange")
+
+        entities = col.metas[0]["entities"].split(";")
+        assert "MemoryStack" in entities
+        assert "rag/foo.py" in entities
+        assert "do_thing_now" in entities
+
+
+class TestExtractAuthoredAt:
+    """authored_at = max per-line ``timestamp`` in a transcript (real authored date,
+    independent of mine time). Both Claude Code and Codex JSONL carry a top-level
+    ISO-8601 ``timestamp`` per line."""
+
+    def test_returns_latest_timestamp(self, tmp_path):
+        f = tmp_path / "session.jsonl"
+        f.write_text(
+            '{"type": "user", "timestamp": "2026-06-21T10:00:00.000Z"}\n'
+            '{"type": "assistant", "timestamp": "2026-06-23T14:30:00.000Z"}\n'
+            '{"type": "user", "timestamp": "2026-06-22T09:00:00.000Z"}\n'
+        )
+        assert _extract_authored_at(f) == "2026-06-23T14:30:00.000Z"
+
+    def test_ignores_lines_without_timestamp(self, tmp_path):
+        f = tmp_path / "session.jsonl"
+        f.write_text(
+            '{"type": "summary", "summary": "x"}\n'
+            '{"type": "assistant", "timestamp": "2026-06-23T14:30:00.000Z"}\n'
+        )
+        assert _extract_authored_at(f) == "2026-06-23T14:30:00.000Z"
+
+    def test_tolerates_blank_and_malformed_lines(self, tmp_path):
+        f = tmp_path / "session.jsonl"
+        f.write_text(
+            "\n"
+            "not json\n"
+            "[1, 2, 3]\n"  # valid JSON, but no .get()
+            '{"timestamp": "2026-06-25T00:00:00.000Z"}\n'
+        )
+        assert _extract_authored_at(f) == "2026-06-25T00:00:00.000Z"
+
+    def test_none_for_non_jsonl(self, tmp_path):
+        f = tmp_path / "notes.md"
+        f.write_text("# heading\n")
+        assert _extract_authored_at(f) is None
+
+    def test_none_when_no_timestamps(self, tmp_path):
+        f = tmp_path / "session.jsonl"
+        f.write_text('{"type": "user", "content": "hi"}\n')
+        assert _extract_authored_at(f) is None
+
+    def test_none_for_missing_file(self, tmp_path):
+        assert _extract_authored_at(tmp_path / "absent.jsonl") is None
+
+    def test_non_string_timestamp_does_not_crash(self, tmp_path):
+        # A non-string timestamp must be skipped, not raise TypeError on compare.
+        f = tmp_path / "session.jsonl"
+        f.write_text(
+            '{"type": "user", "timestamp": 1234567890}\n'
+            '{"type": "assistant", "timestamp": {"nested": true}}\n'
+            '{"type": "user", "timestamp": "2026-06-24T00:00:00.000Z"}\n'
+        )
+        assert _extract_authored_at(f) == "2026-06-24T00:00:00.000Z"
+
+    def test_only_non_string_timestamps_returns_none(self, tmp_path):
+        f = tmp_path / "session.jsonl"
+        f.write_text('{"timestamp": 1}\n{"timestamp": false}\n')
+        assert _extract_authored_at(f) is None

tests/test_daemon.py

@@ -1,9 +1,12 @@
 import os
+import subprocess
 import threading
 import time
 
 import pytest
 
+from _chroma_palace_helper import make_minimal_chroma_sqlite
+
 from mempalace import daemon
 from mempalace import service
 
@@ -171,6 +174,7 @@ def test_service_tool_classification():
     assert service.classify_tool("mempalace_search") == "read"
     assert service.classify_tool("mempalace_add_drawer") == "write"
     assert service.classify_tool("mempalace_checkpoint") == "write"
+    assert service.classify_tool("mempalace_delete_by_source") == "write"
     assert service.classify_tool("mempalace_mine") == "maintenance"
     assert service.classify_tool("unknown") == "unknown"
 
@@ -727,7 +731,7 @@ def test_run_sync_structured_errors_on_sync_failures(tmp_path, monkeypatch):
 
     palace = tmp_path / "palace"
     palace.mkdir()
-    (palace / "chroma.sqlite3").touch()
+    make_minimal_chroma_sqlite(palace)
 
     def _raise(exc):
         def fn(**kw):
@@ -855,3 +859,50 @@ def test_get_client_if_running_uses_short_probe_timeout(monkeypatch):
     client = daemon.get_client_if_running("/p", health_timeout=daemon.HOOK_PROBE_TIMEOUT)
     assert client is not None
     assert captured["timeout"] == daemon.HOOK_PROBE_TIMEOUT
+
+
+# --- _detached_kwargs ---
+
+
+def test_detached_kwargs_posix(tmp_path, monkeypatch):
+    monkeypatch.setattr("mempalace.daemon.os.name", "posix")
+    kwargs = daemon._detached_kwargs(tmp_path / "daemon.log")
+    fh = kwargs["stdout"]
+    try:
+        assert kwargs.get("start_new_session") is True
+        assert kwargs.get("stdin") is subprocess.DEVNULL
+        assert kwargs.get("close_fds") is True
+        assert "creationflags" not in kwargs
+    finally:
+        fh.close()
+
+
+def test_detached_kwargs_windows(tmp_path, monkeypatch):
+    monkeypatch.setattr("mempalace.daemon.os.name", "nt")
+    monkeypatch.setattr("mempalace.daemon.subprocess.CREATE_NO_WINDOW", 0x08000000, raising=False)
+    monkeypatch.setattr("mempalace.daemon.subprocess.DETACHED_PROCESS", 0x00000008, raising=False)
+    monkeypatch.setattr(
+        "mempalace.daemon.subprocess.CREATE_NEW_PROCESS_GROUP", 0x00000200, raising=False
+    )
+    monkeypatch.setattr(
+        "mempalace.daemon.subprocess.CREATE_BREAKAWAY_FROM_JOB", 0x01000000, raising=False
+    )
+    kwargs = daemon._detached_kwargs(tmp_path / "daemon.log")
+    fh = kwargs["stdout"]
+    try:
+        flags = kwargs.get("creationflags", 0)
+        assert flags & 0x08000000, "CREATE_NO_WINDOW must be set"
+        assert not (flags & 0x00000008), (
+            "DETACHED_PROCESS must NOT be set (it suppresses CREATE_NO_WINDOW)"
+        )
+        assert flags & 0x00000200, "CREATE_NEW_PROCESS_GROUP preserved (Ctrl-Break group isolation)"
+        assert flags & 0x01000000, (
+            "CREATE_BREAKAWAY_FROM_JOB preserved (survive parent Job-Object close)"
+        )
+        assert kwargs.get("stdin") is subprocess.DEVNULL
+        assert kwargs.get("close_fds") is True
+        # creationflags + start_new_session are mutually exclusive on Windows
+        # (Popen raises ValueError); the nt branch must not set the latter.
+        assert "start_new_session" not in kwargs
+    finally:
+        fh.close()

tests/test_embedding_wrapper.py

@@ -124,3 +124,98 @@ def test_add_wraps_bare_string_ids_and_dict_metadatas(monkeypatch):
     # documents / embeddings / ids / metadatas all length-aligned at 1
     assert call["documents"] == ["solo"]
     assert len(call["embeddings"]) == 1
+
+
+def test_facet_counts_forwards_to_inner():
+    """``BaseCollection.facet_counts`` is a concrete method (raises) — Python
+    MRO resolves it on the wrapper subclass before ``__getattr__`` ever fires,
+    so without an explicit forwarder the wrapper would raise
+    ``UnsupportedCapabilityError`` and silently degrade every wrapped backend
+    (qdrant/pgvector/sqlite_exact) to client-side counting in mcp_server."""
+
+    class _Inner:
+        def __init__(self):
+            self.calls = []
+
+        def facet_counts(self, field, where=None, limit=1000):
+            self.calls.append((field, where, limit))
+            return {"alpha": 2, "beta": 1}
+
+    inner = _Inner()
+    out = ew.EmbeddingCollection(inner).facet_counts("wing", where={"wing": "alpha"}, limit=50)
+    assert out == {"alpha": 2, "beta": 1}
+    assert inner.calls == [("wing", {"wing": "alpha"}, 50)]
+
+
+def test_get_all_metadata_forwards_to_inner():
+    """Same MRO-shadow pattern as ``facet_counts``: ``BaseCollection.get_all_
+    metadata`` ships a concrete default that pages through ``self.get()``.
+    Without this forwarder the wrapper runs the base default, the inner
+    backend's overridden ``get_all_metadata`` (e.g. pgvector's
+    ``with_document=False`` fast path from #1892) is unreachable, and every
+    metadata-only fetch transfers the full document column over the wire."""
+
+    class _Inner:
+        def __init__(self):
+            self.calls = []
+
+        def get_all_metadata(self, where=None):
+            self.calls.append(where)
+            return [{"wing": "alpha"}, {"wing": "beta"}]
+
+        # Provide a ``get`` so a missing forwarder would silently succeed via
+        # the BaseCollection default rather than crash — this is exactly the
+        # shape that hid the bug in production. Returning a sentinel here
+        # makes a wrong delegation observable: the test would receive
+        # ``[{"WRONG"}]``, not the inner's real list.
+        def get(self, **_kw):
+            from mempalace.backends.base import GetResult
+
+            return GetResult(ids=["bad"], documents=[""], metadatas=[{"WRONG": True}])
+
+    inner = _Inner()
+    out = ew.EmbeddingCollection(inner).get_all_metadata(where={"wing": "alpha"})
+    assert out == [{"wing": "alpha"}, {"wing": "beta"}]
+    assert inner.calls == [{"wing": "alpha"}]
+
+
+def test_wrapper_forwards_all_concrete_basecollection_methods():
+    """Every concrete (non-abstract) public method on ``BaseCollection`` must
+    be explicitly defined on ``EmbeddingCollection``. Without an override,
+    Python MRO resolves the call to ``BaseCollection``'s default before
+    ``__getattr__`` ever fires, silently shadowing the wrapped backend's real
+    implementation.
+
+    The bug class this catches: any new ``BaseCollection`` method with a
+    concrete default body (raises, returns ``{}``, returns a no-op, pages
+    through ``self.get()``) becomes a silent regression for every wrapped
+    backend the moment a backend overrides it. ``facet_counts`` (#1868) and
+    ``get_all_metadata`` (#1796 / #1892) both hit this; ``lexical_search``
+    would have hit it earlier if the wrapper hadn't been updated by hand.
+
+    If this test fails, add an explicit forwarder on ``EmbeddingCollection``
+    that calls ``self._inner.<name>(...)``."""
+    import inspect
+
+    from mempalace.backends.base import BaseCollection
+    from mempalace.backends.embedding_wrapper import EmbeddingCollection
+
+    concrete: set[str] = set()
+    for name, member in inspect.getmembers(BaseCollection):
+        if name.startswith("_"):
+            continue
+        if isinstance(member, property):
+            if member.fget and not getattr(member.fget, "__isabstractmethod__", False):
+                concrete.add(name)
+            continue
+        if callable(member) and not getattr(member, "__isabstractmethod__", False):
+            concrete.add(name)
+
+    forwarded = set(vars(EmbeddingCollection).keys())
+    missing = sorted(concrete - forwarded)
+    assert not missing, (
+        "EmbeddingCollection must explicitly forward these concrete "
+        "BaseCollection methods (otherwise MRO resolves the call here and "
+        "shadows the inner backend's implementation): "
+        f"{missing}"
+    )

tests/test_entities.py

@@ -0,0 +1,69 @@
+"""Tests for no-LLM structural entity extraction."""
+
+from mempalace.entities import entities_metadata, extract_structural_entities
+
+
+def test_extracts_code_symbols_paths_urls():
+    text = (
+        "We patched `_extract_authored_at` in rag/convo_miner.py so MemoryStack and "
+        "ChromaBackend agree. See module.func and pkg.Class.method, plus do_thing_now. "
+        "Ref https://github.com/MemPalace/mempalace/pull/1890 for details."
+    )
+    ents = set(extract_structural_entities(text))
+    assert "_extract_authored_at" in ents
+    assert "rag/convo_miner.py" in ents
+    assert "MemoryStack" in ents
+    assert "ChromaBackend" in ents
+    assert "module.func" in ents
+    assert "pkg.Class.method" in ents
+    assert "do_thing_now" in ents
+    assert any(e.startswith("https://github.com/MemPalace") for e in ents)
+
+
+def test_excludes_prose_noise():
+    text = "This is a normal sentence, e.g. with i.e. abbreviations and version 1.2.3 here."
+    ents = extract_structural_entities(text)
+    # No plain prose words, no "e.g"/"i.e", no bare version numbers.
+    assert ents == []
+
+
+def test_ranked_by_frequency_then_order():
+    text = "alpha_one alpha_one alpha_one beta_two beta_two gamma_three"
+    ents = extract_structural_entities(text)
+    assert ents[:3] == ["alpha_one", "beta_two", "gamma_three"]
+
+
+def test_dedup_case_insensitive_keeps_first_form():
+    text = "`MemoryStack` and memorystack and MEMORYSTACK"
+    ents = extract_structural_entities(text)
+    assert ents.count("MemoryStack") == 1
+    assert ents == ["MemoryStack"]
+
+
+def test_respects_max_entities():
+    text = " ".join(f"sym_{i}_x" for i in range(50))
+    assert len(extract_structural_entities(text, max_entities=10)) == 10
+
+
+def test_extracts_leading_underscore_snake_in_plain_text():
+    # Not in backticks — must still be caught by the snake-case pattern.
+    ents = extract_structural_entities("we called _extract_authored_at and _do_thing here")
+    assert "_extract_authored_at" in ents
+    assert "_do_thing" in ents
+
+
+def test_semicolon_in_entity_does_not_corrupt_metadata():
+    # A backtick span containing ';' must not split the ;-joined metadata field.
+    md = entities_metadata("see `a(); b()` and TwoThing")
+    parts = md.split(";")
+    # Every part is a whole entity — no fragment is a bare separator artifact.
+    assert all(p.strip() for p in parts)
+    assert "TwoThing" in parts
+
+
+def test_metadata_is_semicolon_joined():
+    text = "`one_thing` and TwoThing"
+    md = entities_metadata(text)
+    assert md == "one_thing;TwoThing"
+    assert entities_metadata("") == ""
+    assert entities_metadata("just plain prose with nothing structural") == ""

tests/test_entity_detector.py

@@ -531,6 +531,24 @@ def test_scan_for_detection_skips_git_dir(tmp_path):
     assert not any(".git" in f for f in file_strs)
 
 
+def test_scan_for_detection_includes_latex_prose(tmp_path):
+    # .tex and .bib are prose-heavy (author names, abstracts, citations) and
+    # belong in the preferred PROSE_EXTENSIONS bucket alongside .md / .rst,
+    # not the code-file fallback. .bib in particular is almost entirely
+    # author names — high entity density per byte.
+    (tmp_path / "paper.tex").write_text(
+        "\\documentclass{article}\\author{Leslie Lamport}\\begin{document}Body.\\end{document}"
+    )
+    (tmp_path / "refs.bib").write_text(
+        "@article{l86, author={Leslie Lamport}, title={LaTeX}, year={1986}}"
+    )
+    (tmp_path / "code.py").write_text("import os")
+    files = scan_for_detection(str(tmp_path))
+    extensions = {os.path.splitext(str(f))[1] for f in files}
+    assert ".tex" in extensions
+    assert ".bib" in extensions
+
+
 # ── module-level constants ──────────────────────────────────────────────
 
 
@@ -543,6 +561,8 @@ def test_stopwords_contains_common_words():
 def test_prose_extensions():
     assert ".txt" in PROSE_EXTENSIONS
     assert ".md" in PROSE_EXTENSIONS
+    assert ".tex" in PROSE_EXTENSIONS
+    assert ".bib" in PROSE_EXTENSIONS
 
 
 # ── _print_entity_list ─────────────────────────────────────────────────

tests/test_format_miner.py

@@ -1158,7 +1158,9 @@ def test_mine_formats_calls_compute_topic_tunnels_after_loop(_mine_formats_mocks
         patch("mempalace.format_miner._compute_topic_tunnels_for_wing", return_value=0) as p_tun,
     ):
         mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_aya")
-    p_tun.assert_called_once_with("wing_aya")
+    p_tun.assert_called_once()
+    assert p_tun.call_args.args == ("wing_aya",)
+    assert p_tun.call_args.kwargs["config"].palace_path == str(tmp / "palace")
 
 
 def test_mine_formats_tunnel_failure_does_not_crash_mine(_mine_formats_mocks):
@@ -1518,13 +1520,16 @@ def test_mine_formats_threads_chunk_size_from_user_config(monkeypatch, tmp_path:
 
     # Inject custom config values via a fake MempalaceConfig.
     class _FakeMempalaceConfig:
+        def __init__(self, **kwargs):
+            self._palace_path = kwargs.get("palace_path", str(tmp_path / "palace"))
+
         chunk_size = 1234
         chunk_overlap = 56
         min_chunk_size = 78
 
         @property
         def palace_path(self):
-            return str(tmp_path / "palace")
+            return self._palace_path
 
     monkeypatch.setattr(format_miner, "MempalaceConfig", _FakeMempalaceConfig)
 

tests/test_hallways.py

@@ -92,6 +92,24 @@ class TestHallwayStorage:
 
 
 class TestComputeHallways:
+    def test_explicit_config_scopes_persistence_to_selected_palace(self, tmp_path):
+        from mempalace.config import MempalaceConfig
+
+        default_cfg = MempalaceConfig(palace_path=tmp_path / "default" / "palace")
+        selected_cfg = MempalaceConfig(palace_path=tmp_path / "selected" / "palace")
+        col = _fake_collection(
+            [
+                {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi"},
+                {"wing": "wing_aya", "room": "letters", "entities": "Aya;Lumi"},
+            ]
+        )
+
+        created = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, config=selected_cfg)
+
+        assert len(created) == 1
+        assert hallways_mod.list_hallways(config=selected_cfg) == created
+        assert hallways_mod.list_hallways(config=default_cfg) == []
+
     def test_returns_empty_for_unknown_wing(self, tmp_path, monkeypatch):
         """Wing with no drawers → no hallways, no crash."""
         _use_tmp_hallway_file(monkeypatch, tmp_path)
@@ -300,6 +318,19 @@ class TestHallwayQuery:
         hallways_mod._save_hallways([{"id": "h1", "wing": "wing_aya"}])
         assert hallways_mod.delete_hallway("nonexistent") is False
 
+    def test_delete_hallway_uses_selected_palace_config(self, tmp_path):
+        from mempalace.config import MempalaceConfig
+
+        default_cfg = MempalaceConfig(palace_path=tmp_path / "default" / "palace")
+        selected_cfg = MempalaceConfig(palace_path=tmp_path / "selected" / "palace")
+        record = {"id": "h1", "wing": "wing_aya"}
+        hallways_mod._save_hallways([record], default_cfg)
+        hallways_mod._save_hallways([record], selected_cfg)
+
+        assert hallways_mod.delete_hallway("h1", config=selected_cfg) is True
+        assert hallways_mod.list_hallways(config=selected_cfg) == []
+        assert hallways_mod.list_hallways(config=default_cfg) == [record]
+
 
 # ─────────────────────────────────────────────────────────────────────────────
 # L7 dynamics integration — hallway records carry strength/stability/etc

tests/test_hnsw_payload_health.py

@@ -113,8 +113,11 @@ def test_quarantine_leaves_reasonable_payload_in_place(tmp_path):
     assert seg_dir.exists()
 
 
-def test_segment_health_rejects_zero_byte_link_lists_with_payload(tmp_path):
-    """Regression #1457: real HNSW payload with empty link_lists.bin is corrupt."""
+def test_segment_health_accepts_zero_byte_link_lists_with_valid_pickle(tmp_path):
+    """Regression #1716: an all-layer-0 HNSW index serializes an empty
+    link_lists.bin (level-0 links live in data_level0.bin). With a complete
+    index_metadata.pickle the persist finished, so the segment is healthy —
+    not the partial-flush corruption a missing marker would imply."""
     seg_dir = tmp_path / "11111111-2222-3333-4444-555555555555"
 
     _write_segment(
@@ -124,11 +127,32 @@ def test_segment_health_rejects_zero_byte_link_lists_with_payload(tmp_path):
         write_metadata=True,
     )
 
+    assert _segment_appears_healthy(str(seg_dir))
+
+
+def test_segment_health_rejects_zero_byte_link_lists_with_truncated_pickle(tmp_path):
+    """An empty link_lists.bin with real payload but a truncated metadata
+    envelope is a partial flush, not an all-layer-0 index — still rejected.
+    The completion marker (intact pickle), not link_lists content, is what
+    distinguishes the two."""
+    seg_dir = tmp_path / "11111111-2222-3333-4444-555555555555"
+
+    _write_segment(
+        seg_dir,
+        data_size=2_000,
+        link_size=0,
+        write_metadata=False,
+    )
+    # Persist started (0x80 head) but never wrote the STOP terminator.
+    (seg_dir / "index_metadata.pickle").write_bytes(b"\x80" + b"x" * 20)
+
     assert not _segment_appears_healthy(str(seg_dir))
 
 
-def test_quarantine_catches_zero_byte_link_lists_when_stale(tmp_path):
-    """Regression #1457: stale segments with empty link_lists.bin are quarantined."""
+def test_quarantine_leaves_zero_byte_link_lists_with_valid_pickle(tmp_path):
+    """Regression #1716: a stale all-layer-0 segment with a complete pickle is
+    left in place. Quarantining it spawned the self-perpetuating loop — repair
+    rebuilds the byte-identical empty-link_lists shape and it re-quarantines."""
     palace = tmp_path / "palace"
     palace.mkdir()
 
@@ -150,9 +174,5 @@ def test_quarantine_catches_zero_byte_link_lists_when_stale(tmp_path):
 
     moved = quarantine_stale_hnsw(str(palace), stale_seconds=300)
 
-    assert len(moved) == 1
-    assert not seg_dir.exists()
-
-    moved_path = Path(moved[0])
-    assert moved_path.exists()
-    assert moved_path.name.startswith("11111111-2222-3333-4444-555555555555.drift-")
+    assert moved == []
+    assert seg_dir.exists()

tests/test_hooks_cli.py

@@ -8,8 +8,11 @@ from pathlib import Path
 from unittest.mock import MagicMock, patch
 
 import pytest
+from hypothesis import given
+from hypothesis import strategies as st
 
 import mempalace.hooks_cli as hooks_cli_mod
+from mempalace.config import sanitize_name
 from mempalace.hooks_cli import (
     SAVE_INTERVAL,
     _count_human_messages,
@@ -23,6 +26,7 @@ from mempalace.hooks_cli import (
     _mine_already_running,
     _mine_sync,
     _parse_harness_input,
+    _safe_wing_slug,
     _sanitize_session_id,
     _save_diary_direct,
     _validate_transcript_path,
@@ -722,6 +726,84 @@ def test_wing_from_transcript_path_cwd_handles_non_string_cwd(tmp_path):
     assert _wing_from_transcript_path(str(transcript)) == "wing_proper_name"
 
 
+# --- _safe_wing_slug: special-character project dirs (e.g. +project) ---
+
+
+def test_safe_wing_slug_strips_leading_plus():
+    """Regression: a ``+``-prefixed folder (e.g. ``+project``) leaked ``+`` into the
+    slug, producing ``wing_+project`` which ``sanitize_name`` rejects — silently
+    breaking diary auto-save for that project."""
+    assert _safe_wing_slug("+project") == "project"
+
+
+def test_safe_wing_slug_replaces_inner_special_chars():
+    assert _safe_wing_slug("foo+bar") == "foo_bar"
+
+
+def test_safe_wing_slug_preserves_space_and_hyphen_behavior():
+    # spaces and hyphens still collapse to underscores (no regression)
+    assert _safe_wing_slug("React Native") == "react_native"
+    assert _safe_wing_slug("claude-code") == "claude_code"
+
+
+def test_safe_wing_slug_preserves_existing_valid_names():
+    """Backward compatibility: a name that already produced a valid wing keeps the
+    same slug, so previously-filed diary entries aren't orphaned (AGENTS.md: never
+    destroy existing data). ``.`` and ``'`` are both accepted by sanitize_name and
+    must survive."""
+    assert _safe_wing_slug("myapp") == "myapp"
+    assert _safe_wing_slug("my.app") == "my.app"
+    assert _safe_wing_slug("v1.2.3") == "v1.2.3"
+    assert _safe_wing_slug("o'brien") == "o'brien"
+
+
+def test_safe_wing_slug_collapses_double_dots():
+    """``..`` must collapse — sanitize_name rejects it as path traversal."""
+    assert _safe_wing_slug("my..app") == "my.app"
+    assert _safe_wing_slug("..hidden..") == "hidden"
+
+
+def test_safe_wing_slug_caps_length_for_sanitize_name():
+    """sanitize_name rejects names over 128 chars; the slug stays short enough that
+    wing_<slug> never trips that limit, even for very long directory names."""
+    slug = _safe_wing_slug("a" * 500)
+    assert len(slug) <= 120
+    sanitize_name(f"wing_{slug}")  # must not raise
+
+
+def test_safe_wing_slug_falls_back_to_sessions_when_empty():
+    # names made entirely of disallowed characters reduce to nothing
+    assert _safe_wing_slug("+") == "sessions"
+    assert _safe_wing_slug("@#$") == "sessions"
+
+
+def test_wing_from_transcript_path_cwd_plus_prefixed_dir(tmp_path):
+    """Reporter's case: a ``+``-prefixed working directory must yield a sanitizable
+    wing (``wing_project``), not the rejected ``wing_+project``."""
+    project_dir = tmp_path / "encoded-dir"
+    project_dir.mkdir()
+    transcript = project_dir / "session.jsonl"
+    transcript.write_text(
+        '{"type":"user","cwd":"/Users/me/code/+project","content":"hi"}\n',
+        encoding="utf-8",
+    )
+    assert _wing_from_transcript_path(str(transcript)) == "wing_project"
+
+
+def test_wing_from_transcript_path_legacy_plus_prefixed_project():
+    """Legacy ``-Projects-<name>`` path with a ``+``-prefixed project folder."""
+    path = "/Users/me/foo/-Projects-+app/session.jsonl"
+    assert _wing_from_transcript_path(path) == "wing_app"
+
+
+@given(st.text(min_size=1, max_size=300))
+def test_safe_wing_slug_always_yields_sanitizable_wing(name):
+    """Property: for ANY non-empty input, ``wing_<slug>`` must pass sanitize_name —
+    the entire contract of the helper (a rejected wing silently breaks auto-save)."""
+    wing = f"wing_{_safe_wing_slug(name)}"
+    assert sanitize_name(wing) == wing
+
+
 # --- _log ---
 
 

tests/test_hybrid_search.py

@@ -12,7 +12,7 @@ from mempalace.palace import (
     get_collection,
     upsert_closet_lines,
 )
-from mempalace.searcher import search_memories
+from mempalace.searcher import _hybrid_rank, search_memories
 
 
 def _seed_drawers(palace_path):
@@ -173,3 +173,34 @@ class TestSourceFileFilter:
         ids = [h["source_file"] for h in result["results"]]
         assert "fixture_D1.md" not in ids
         assert set(ids) <= {"fixture_D4.md"}
+
+
+def test_hybrid_rank_breaks_score_ties_by_authored_at():
+    """Identical-content hits get identical vector + BM25 scores; the tie must break
+    toward the more recently authored drawer, not arbitrary backend order."""
+    older = {
+        "text": "alpha beta gamma",
+        "distance": 0.2,
+        "metadata": {"authored_at": "2026-06-21T10:00:00.000Z"},
+    }
+    newer = {
+        "text": "alpha beta gamma",
+        "distance": 0.2,
+        "metadata": {"authored_at": "2026-06-27T10:00:00.000Z"},
+    }
+    # Input order puts the older drawer first; the tiebreak should reorder it.
+    results = [older, newer]
+    _hybrid_rank(results, "alpha beta gamma")
+    assert results[0]["metadata"]["authored_at"] == "2026-06-27T10:00:00.000Z"
+    assert results[1]["metadata"]["authored_at"] == "2026-06-21T10:00:00.000Z"
+
+
+def test_hybrid_rank_tiebreak_handles_top_level_authored_at():
+    """The search_memories path puts authored_at at the top level (no `metadata`
+    nesting); the tie-break must read it there too."""
+    older = {"text": "alpha beta gamma", "distance": 0.2, "authored_at": "2026-06-21T10:00:00.000Z"}
+    newer = {"text": "alpha beta gamma", "distance": 0.2, "authored_at": "2026-06-27T10:00:00.000Z"}
+    results = [older, newer]
+    _hybrid_rank(results, "alpha beta gamma")
+    assert results[0]["authored_at"] == "2026-06-27T10:00:00.000Z"
+    assert results[1]["authored_at"] == "2026-06-21T10:00:00.000Z"

tests/test_init_gitignore_protection.py

@@ -7,6 +7,7 @@ the two filenames to `.gitignore` when `<dir>` is a git repository.
 """
 
 from pathlib import Path
+from unittest.mock import mock_open, patch
 
 from mempalace.cli import _ensure_mempalace_files_gitignored
 
@@ -60,3 +61,20 @@ def test_handles_gitignore_without_trailing_newline(tmp_path):
     assert "dist\n" in contents
     assert "mempalace.yaml" in contents
     assert "entities.json" in contents
+
+
+def test_gitignore_io_pins_utf8_and_defensive_decode(tmp_path):
+    """Regression for #1648: never fall back to the Windows locale codec."""
+    _git_init(tmp_path)
+    gitignore = tmp_path / ".gitignore"
+    gitignore.write_text("# café\n", encoding="utf-8")
+    append_handle = mock_open()
+
+    with (
+        patch.object(Path, "read_text", return_value="# café\n") as read_text,
+        patch("builtins.open", append_handle),
+    ):
+        assert _ensure_mempalace_files_gitignored(tmp_path) is True
+
+    read_text.assert_called_once_with(encoding="utf-8", errors="replace")
+    append_handle.assert_called_once_with(gitignore, "a", encoding="utf-8")

tests/test_knowledge_graph.py

@@ -322,3 +322,71 @@ class TestKnowledgeGraphConnectionCleanup:
         assert kg._connection is None
         with pytest.raises(sqlite3.ProgrammingError):
             conn.execute("SELECT 1")
+
+
+class TestSupersessionBoundary:
+    """Regression coverage for the as-of boundary double-count (issue #1913):
+    an as-of query at the instant one fact ends and its successor begins must
+    return only the successor for a single-valued predicate."""
+
+    def _models(self, kg, as_of):
+        return sorted(
+            f["object"]
+            for f in kg.query_entity("Bot", as_of=as_of, direction="outgoing")
+            if f["predicate"] == "uses_model"
+        )
+
+    def test_exact_datetime_boundary_returns_only_successor(self, kg):
+        # Two facts sharing a precise instant: half-open upper bound (strict >)
+        # means the fact ending at T no longer matches at T.
+        kg.add_triple(
+            "Bot",
+            "uses_model",
+            "A",
+            valid_from="2026-05-01T00:00:00Z",
+            valid_to="2026-06-02T12:00:00Z",
+        )
+        kg.add_triple("Bot", "uses_model", "B", valid_from="2026-06-02T12:00:00Z")
+
+        assert self._models(kg, "2026-06-02T11:59:59Z") == ["A"]
+        assert self._models(kg, "2026-06-02T12:00:00Z") == ["B"]
+        assert self._models(kg, "2026-06-02T12:00:01Z") == ["B"]
+
+    def test_supersede_date_only_resolves_to_successor(self, kg):
+        kg.add_triple("Bot", "uses_model", "claude-opus-4-7", valid_from="2026-05-01")
+        kg.supersede("Bot", "uses_model", "claude-opus-4-7", "claude-opus-4-8", at="2026-06-02")
+
+        assert self._models(kg, "2026-06-01") == ["claude-opus-4-7"]
+        assert self._models(kg, "2026-06-02") == ["claude-opus-4-8"]
+        assert self._models(kg, "2026-06-03") == ["claude-opus-4-8"]
+
+    def test_supersede_datetime_boundary_resolves_to_successor(self, kg):
+        kg.add_triple("Bot", "uses_model", "A", valid_from="2026-05-01T00:00:00Z")
+        kg.supersede("Bot", "uses_model", "A", "B", at="2026-06-02T12:00:00Z")
+
+        assert self._models(kg, "2026-06-02T11:59:59Z") == ["A"]
+        assert self._models(kg, "2026-06-02T12:00:00Z") == ["B"]
+
+    def test_supersede_default_now_closes_old_and_opens_new(self, kg):
+        kg.add_triple("Bot", "uses_model", "A", valid_from="2026-05-01")
+        kg.supersede("Bot", "uses_model", "A", "B")
+        # A far-future as-of sees only the successor; the old fact was closed.
+        assert self._models(kg, "2099-01-01") == ["B"]
+
+    def test_supersede_degrades_to_add_when_no_open_old(self, kg):
+        tid = kg.supersede("Bot", "uses_model", "missing", "B", at="2026-01-01")
+        assert tid.startswith("t_bot_uses_model_b_")
+        assert self._models(kg, "2099-01-01") == ["B"]
+
+    def test_supersede_rejects_boundary_before_valid_from(self, kg):
+        kg.add_triple("Bot", "uses_model", "A", valid_from="2026-06-01")
+        with pytest.raises(ValueError, match="before valid_from"):
+            kg.supersede("Bot", "uses_model", "A", "B", at="2026-05-01")
+
+    def test_standalone_date_only_end_stays_valid_all_day(self, kg):
+        # Half-open change must NOT shrink a standalone date-only fact: it stays
+        # valid through the end of its final day (whole-day expansion retained).
+        kg.add_triple("Bot", "uses_model", "A", valid_from="2026-05-01", valid_to="2026-06-02")
+        assert self._models(kg, "2026-06-02") == ["A"]
+        assert self._models(kg, "2026-06-02T23:00:00Z") == ["A"]
+        assert self._models(kg, "2026-06-03") == []

tests/test_layers.py

@@ -201,6 +201,27 @@ def test_layer1_importance_from_various_keys():
     assert "ESSENTIAL STORY" in result
 
 
+def test_layer1_breaks_importance_ties_by_filed_at_recency():
+    """Equal-importance drawers surface newest-first instead of insertion order."""
+    docs = ["oldest memory", "newest memory", "middle memory"]
+    metas = [
+        {"room": "moments", "importance": 3, "filed_at": "2026-01-01T00:00:00Z"},
+        {"room": "moments", "importance": 3, "filed_at": "2026-03-01T00:00:00Z"},
+        {"room": "moments", "importance": 3, "filed_at": "2026-02-01T00:00:00Z"},
+    ]
+    mock_col = _mock_chromadb_for_layer(docs, metas)
+
+    with (
+        patch("mempalace.layers.MempalaceConfig") as mock_cfg,
+        patch("mempalace.layers._get_collection", return_value=mock_col),
+    ):
+        mock_cfg.return_value.palace_path = "/fake"
+        result = Layer1(palace_path="/fake").generate()
+
+    assert result.index("newest memory") < result.index("middle memory")
+    assert result.index("middle memory") < result.index("oldest memory")
+
+
 def test_layer1_batch_exception_breaks():
     """If col.get raises on a batch, loop breaks gracefully."""
     mock_col = MagicMock()

tests/test_mcp_http_transport.py

@@ -19,6 +19,9 @@ Design constraints
 
 import http.client
 import json
+import logging
+import socketserver
+import ssl
 import threading
 
 import pytest
@@ -197,6 +200,184 @@ def test_bearer_token_enforced_when_configured(monkeypatch):
         thread.join(timeout=5)
 
 
+def test_read_only_hides_and_refuses_mutating_tools(http_server, monkeypatch):
+    """Read-only mode (#1877): mutating tools are hidden from tools/list AND
+    refused at dispatch with -32003, while read tools still work."""
+    monkeypatch.setattr(mcp, "_READ_ONLY", True)
+    port, _ = http_server
+
+    status, body = _post(port, "/mcp", {"jsonrpc": "2.0", "id": 1, "method": "tools/list"})
+    assert status == 200
+    names = {t["name"] for t in json.loads(body)["result"]["tools"]}
+    assert "mempalace_search" in names  # read tool stays
+    assert "mempalace_add_drawer" not in names  # mutating tool hidden
+    assert names.isdisjoint(mcp._MUTATING_TOOLS)
+
+    status, body = _post(
+        port,
+        "/mcp",
+        {
+            "jsonrpc": "2.0",
+            "id": 2,
+            "method": "tools/call",
+            "params": {"name": "mempalace_add_drawer", "arguments": {"content": "x"}},
+        },
+    )
+    assert status == 200
+    assert json.loads(body)["error"]["code"] == -32003
+
+
+def test_read_only_off_exposes_mutating_tools(http_server):
+    """Sanity: without read-only, mutating tools are present (guards the test above)."""
+    port, _ = http_server
+    status, body = _post(port, "/mcp", {"jsonrpc": "2.0", "id": 1, "method": "tools/list"})
+    names = {t["name"] for t in json.loads(body)["result"]["tools"]}
+    assert "mempalace_add_drawer" in names
+
+
+@pytest.mark.parametrize(
+    "disconnect_exc",
+    [
+        ConnectionResetError(104, "connection reset by peer"),
+        BrokenPipeError(32, "broken pipe"),
+        ssl.SSLEOFError("unexpected eof while reading"),
+    ],
+    ids=["connreset", "brokenpipe", "ssleof"],
+)
+def test_handle_error_quiets_client_disconnect(caplog, monkeypatch, disconnect_exc):
+    """Regression for #2003: a client that hangs up mid-response makes the send
+    path raise ConnectionError (BrokenPipeError / ConnectionResetError), or
+    ssl.SSLEOFError on the TLS transport. The server must log that quietly at
+    DEBUG instead of routing it to the default handler's per-request traceback.
+    """
+    httpd = mcp._build_http_server("127.0.0.1", 0)
+    try:
+        delegated = []
+        monkeypatch.setattr(
+            socketserver.BaseServer,
+            "handle_error",
+            lambda self, request, addr: delegated.append(addr),
+        )
+        addr = ("127.0.0.1", 51234)
+
+        with caplog.at_level(logging.DEBUG, logger="mempalace_mcp"):
+            try:
+                raise disconnect_exc
+            except type(disconnect_exc):
+                httpd.handle_error(None, addr)
+
+        assert delegated == []  # noisy default handler NOT invoked
+        rec = next(r for r in caplog.records if "disconnect" in r.getMessage().lower())
+        assert rec.levelno == logging.DEBUG
+        assert rec.name == "mempalace_mcp"
+    finally:
+        httpd.server_close()
+
+
+def test_handle_error_delegates_real_errors(monkeypatch):
+    """A genuine error is NOT misclassified as a disconnect: it reaches the
+    default handler, so its traceback is still surfaced.
+    """
+    httpd = mcp._build_http_server("127.0.0.1", 0)
+    try:
+        delegated = []
+        monkeypatch.setattr(
+            socketserver.BaseServer,
+            "handle_error",
+            lambda self, request, addr: delegated.append(addr),
+        )
+        addr = ("127.0.0.1", 51234)
+        try:
+            raise ValueError("boom")
+        except ValueError:
+            httpd.handle_error(None, addr)
+        assert delegated == [addr]
+    finally:
+        httpd.server_close()
+
+
+def _make_self_signed_cert(tmp_path):
+    """Write a throwaway self-signed cert/key via openssl; skip if unavailable."""
+    import shutil
+    import subprocess
+
+    if shutil.which("openssl") is None:
+        pytest.skip("openssl not available to generate a test certificate")
+    cert = tmp_path / "cert.pem"
+    key = tmp_path / "key.pem"
+    subprocess.run(
+        [
+            "openssl",
+            "req",
+            "-x509",
+            "-newkey",
+            "rsa:2048",
+            "-keyout",
+            str(key),
+            "-out",
+            str(cert),
+            "-days",
+            "1",
+            "-nodes",
+            "-subj",
+            "/CN=localhost",
+        ],
+        check=True,
+        capture_output=True,
+    )
+    return cert, key
+
+
+def test_tls_serves_https(tmp_path, monkeypatch):
+    """With --tls-cert/--tls-key (via env), the server speaks TLS: a plain HTTP
+    client cannot read it, and an HTTPS client trusting the cert can."""
+    import ssl
+
+    cert, key = _make_self_signed_cert(tmp_path)
+    monkeypatch.setenv("MEMPALACE_MCP_TLS_CERT", str(cert))
+    monkeypatch.setenv("MEMPALACE_MCP_TLS_KEY", str(key))
+
+    httpd = mcp._build_http_server("127.0.0.1", 0)
+    assert getattr(httpd, "scheme", "http") == "https"
+    port = httpd.server_address[1]
+    thread = threading.Thread(
+        target=httpd.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True
+    )
+    thread.start()
+    try:
+        # Full verification on: trust the self-signed cert as the CA and dial
+        # "localhost" (the cert CN, resolves to 127.0.0.1) so hostname checking
+        # passes without being disabled.
+        ctx = ssl.create_default_context(cafile=str(cert))
+        conn = http.client.HTTPSConnection("localhost", port, context=ctx, timeout=5)
+        try:
+            conn.request("GET", "/healthz")
+            resp = conn.getresponse()
+            assert resp.status == 200
+            assert resp.read() == b"ok\n"
+        finally:
+            conn.close()
+
+        # A plaintext HTTP client must NOT be able to talk to the TLS socket.
+        with pytest.raises(Exception):
+            plain = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
+            plain.request("GET", "/healthz")
+            plain.getresponse()
+    finally:
+        httpd.shutdown()
+        httpd.server_close()
+        thread.join(timeout=5)
+
+
+def test_tls_requires_both_cert_and_key(tmp_path, monkeypatch):
+    """A cert without a key (or vice versa) is a startup error, not a silent skip."""
+    cert, _key = _make_self_signed_cert(tmp_path)
+    monkeypatch.setenv("MEMPALACE_MCP_TLS_CERT", str(cert))
+    monkeypatch.delenv("MEMPALACE_MCP_TLS_KEY", raising=False)
+    with pytest.raises(ValueError, match="both"):
+        mcp._build_http_server("127.0.0.1", 0)
+
+
 def test_loopback_and_origin_helpers():
     assert mcp._http_is_loopback("127.0.0.1")
     assert mcp._http_is_loopback("localhost")

tests/test_milvus_backend.py

@@ -0,0 +1,370 @@
+import json
+import os
+import sys
+import uuid
+
+import pytest
+
+from _backend_conformance import assert_partition_isolation
+
+from mempalace.backends import (
+    BackendError,
+    BackendMismatchError,
+    DimensionMismatchError,
+    PalaceRef,
+    UnsupportedFilterError,
+    available_backends,
+)
+from mempalace.backends.milvus import (
+    DOCUMENT_MAX_LENGTH,
+    MilvusBackend,
+    translate_where,
+    translate_where_document,
+)
+
+
+def _require_milvus_lite():
+    pytest.importorskip("pymilvus")
+    if sys.platform == "win32":
+        pytest.skip("milvus-lite is not distributed on Windows")
+    pytest.importorskip("milvus_lite")
+
+
+def _new_lite_collection(tmp_path, name="drawers", namespace=None):
+    _require_milvus_lite()
+    backend = MilvusBackend()
+    palace_path = tmp_path / (namespace or "palace")
+    palace = PalaceRef(
+        id=str(palace_path),
+        local_path=str(palace_path),
+        namespace=namespace,
+    )
+    collection = backend.get_collection(palace=palace, collection_name=name, create=True)
+    return backend, palace, collection
+
+
+def test_registry_exposes_milvus():
+    assert "milvus" in available_backends()
+
+
+def test_translate_where_supports_portable_filter_subset():
+    assert translate_where(None) == ""
+    assert translate_where({"wing": "project"}) == 'wing == "project"'
+    assert translate_where({"rank": {"$gte": 2}}) == "rank >= 2"
+    assert translate_where({"wing": {"$in": ["a", "b"]}}) == 'wing in ["a", "b"]'
+    assert (
+        translate_where({"$and": [{"wing": "p"}, {"room": "r"}]}) == '(wing == "p" and room == "r")'
+    )
+    assert (
+        translate_where({"$or": [{"wing": "p"}, {"wing": "q"}]}) == '(wing == "p" or wing == "q")'
+    )
+    assert translate_where({"wing": "p", "room": "r"}) == 'wing == "p" and room == "r"'
+    assert translate_where_document({"$contains": "needle"}) == 'document like "%needle%"'
+
+
+def test_translate_where_rejects_unsafe_fields_and_operators():
+    with pytest.raises(UnsupportedFilterError):
+        translate_where({"bad field": "x"})
+    with pytest.raises(UnsupportedFilterError):
+        translate_where({"rank": {"$regex": "x"}})
+    with pytest.raises(UnsupportedFilterError):
+        translate_where({"$nor": [{"wing": "x"}]})
+
+
+def test_milvus_lite_add_query_filter_lexical_and_marker(tmp_path):
+    backend, palace, col = _new_lite_collection(tmp_path)
+    try:
+        col.add(
+            ids=["a", "b", "c"],
+            documents=[
+                "alpha backend note",
+                "rareterm milvus backend note",
+                "frontend design note",
+            ],
+            metadatas=[
+                {"wing": "project", "room": "backend", "rank": 1},
+                {"wing": "project", "room": "backend", "rank": 3},
+                {"wing": "project", "room": "frontend", "rank": 2},
+            ],
+            embeddings=[[1, 0], [0.9, 0.1], [0, 1]],
+        )
+
+        assert MilvusBackend.detect(palace.local_path)
+        assert os.path.isfile(os.path.join(palace.local_path, "milvus_backend.json"))
+        assert os.path.exists(os.path.join(palace.local_path, "milvus.db"))
+        assert col._has_native_lexical()
+        assert col.count() == 3
+
+        result = col.query(
+            query_embeddings=[[1, 0]],
+            n_results=3,
+            where={"rank": {"$gte": 2}},
+            include=["documents", "metadatas", "distances", "embeddings"],
+        )
+        assert result.ids == [["b", "c"]]
+        assert result.documents[0][0] == "rareterm milvus backend note"
+        assert result.metadatas[0][0]["rank"] == 3
+        assert result.embeddings[0][0] == pytest.approx([0.9, 0.1])
+        assert result.distances[0] == sorted(result.distances[0])
+
+        hits = col.lexical_search(query="rareterm backend", n_results=2).hits
+        assert [hit.id for hit in hits] == ["b", "a"]
+    finally:
+        backend.close()
+
+
+def test_milvus_lite_upsert_update_delete_get_order_and_multi_collection(tmp_path):
+    backend, palace, drawers = _new_lite_collection(tmp_path, "drawers")
+    closets = backend.get_collection(palace=palace, collection_name="closets", create=True)
+    try:
+        drawers.upsert(
+            ids=["one", "two"],
+            documents=["first document", "second document"],
+            metadatas=[{"wing": "a"}, {"wing": "b"}],
+            embeddings=[[1, 0], [0, 1]],
+        )
+        closets.add(
+            ids=["one"],
+            documents=["closet document"],
+            metadatas=[{"wing": "closet"}],
+            embeddings=[[0.5, 0.5]],
+        )
+
+        got = drawers.get(ids=["two", "one", "two"], include=["documents", "metadatas"])
+        assert got.ids == ["two", "one", "two"]
+        assert got.documents == ["second document", "first document", "second document"]
+
+        drawers.update(ids=["one", "missing"], metadatas=[{"room": "updated"}, {"room": "skip"}])
+        assert drawers.get(ids=["one"]).metadatas == [{"wing": "a", "room": "updated"}]
+        assert drawers.get(ids=["missing"]).ids == []
+
+        drawers.delete(where={"wing": "b"})
+        assert drawers.get().ids == ["one"]
+        assert closets.get().ids == ["one"]
+    finally:
+        backend.close()
+
+
+def test_milvus_lite_db_name_and_consistency_options(tmp_path):
+    _require_milvus_lite()
+    backend = MilvusBackend()
+    palace = PalaceRef(id=str(tmp_path / "palace"), local_path=str(tmp_path / "palace"))
+    col = backend.get_collection(
+        palace=palace,
+        collection_name="drawers",
+        create=True,
+        options={"db_name": "mempalace_test", "consistency_level": "eventually"},
+    )
+    try:
+        assert col._config.db_name == "mempalace_test"
+        assert col._config.consistency_level == "Eventually"
+
+        col.upsert(ids=["a"], documents=["native bm25 document"], embeddings=[[1, 0]])
+
+        with open(os.path.join(palace.local_path, "milvus_backend.json"), encoding="utf-8") as f:
+            marker = json.load(f)
+        assert marker["milvus"]["db_name"] == "mempalace_test"
+        assert col.lexical_search(query="native", n_results=1).hits[0].id == "a"
+    finally:
+        backend.close()
+
+
+def test_milvus_rejects_invalid_consistency_level(tmp_path):
+    _require_milvus_lite()
+    backend = MilvusBackend()
+    palace = PalaceRef(id=str(tmp_path / "palace"), local_path=str(tmp_path / "palace"))
+    try:
+        with pytest.raises(BackendError, match="consistency_level"):
+            backend.get_collection(
+                palace=palace,
+                collection_name="drawers",
+                create=True,
+                options={"consistency_level": "linearizable"},
+            )
+    finally:
+        backend.close()
+
+
+def test_milvus_lite_dimension_and_validation_errors(tmp_path):
+    backend, _palace, col = _new_lite_collection(tmp_path)
+    try:
+        col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
+
+        with pytest.raises(DimensionMismatchError):
+            col.upsert(ids=["b"], documents=["two"], metadatas=[{}], embeddings=[[1, 0, 0]])
+        with pytest.raises(ValueError, match="unique"):
+            col.add(ids=["dup", "dup"], documents=["a", "b"], embeddings=[[1, 0], [0, 1]])
+        with pytest.raises(ValueError, match="reserved"):
+            col.add(ids=["reserved"], documents=["x"], metadatas=[{"id": "x"}], embeddings=[[1, 0]])
+        with pytest.raises(ValueError, match="delete requires"):
+            col.delete()
+        with pytest.raises(ValueError, match="document byte length"):
+            col.add(
+                ids=["long"], documents=["你" * (DOCUMENT_MAX_LENGTH // 3 + 1)], embeddings=[[1, 0]]
+            )
+    finally:
+        backend.close()
+
+
+def test_milvus_writes_marker_when_upserting_existing_remote_collection(tmp_path):
+    _require_milvus_lite()
+    shared_uri = str(tmp_path / "shared.db")
+
+    backend_a = MilvusBackend()
+    palace_a = PalaceRef(id="palace-a", local_path=str(tmp_path / "palace-a"))
+    col_a = backend_a.get_collection(
+        palace=palace_a,
+        collection_name="drawers",
+        create=True,
+        options={"uri": shared_uri},
+    )
+    try:
+        col_a.upsert(ids=["a"], documents=["first"], embeddings=[[1, 0]])
+    finally:
+        backend_a.close()
+
+    backend_b = MilvusBackend()
+    palace_b = PalaceRef(id="palace-b", local_path=str(tmp_path / "palace-b"))
+    marker_b = os.path.join(palace_b.local_path, "milvus_backend.json")
+    col_b = backend_b.get_collection(
+        palace=palace_b,
+        collection_name="drawers",
+        create=True,
+        options={"uri": shared_uri},
+    )
+    try:
+        assert not os.path.exists(marker_b)
+        col_b.upsert(ids=["b"], documents=["second"], embeddings=[[0, 1]])
+        assert os.path.isfile(marker_b)
+    finally:
+        backend_b.close()
+
+
+def test_milvus_marker_rejects_remote_target_change(tmp_path, monkeypatch):
+    backend, palace, col = _new_lite_collection(tmp_path)
+    try:
+        col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
+        backend.close()
+
+        monkeypatch.setenv("MEMPALACE_MILVUS_URI", str(tmp_path / "other.db"))
+        with pytest.raises(BackendMismatchError, match="target"):
+            MilvusBackend().get_collection(palace=palace, collection_name="drawers", create=False)
+    finally:
+        backend.close()
+
+
+def test_palace_wrapper_embeds_for_milvus(tmp_path, monkeypatch):
+    _require_milvus_lite()
+    import mempalace.backends.embedding_wrapper as embedding_wrapper
+    from mempalace import palace
+    from mempalace.backends import reset_backends
+
+    monkeypatch.setattr(
+        embedding_wrapper,
+        "_embed_texts",
+        lambda texts: [[1.0, 0.0] for _ in texts],
+    )
+    monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "milvus")
+    monkeypatch.setenv("MEMPALACE_BACKEND", "milvus")
+    reset_backends()
+    try:
+        col = palace.get_collection(str(tmp_path / "wrapped"), "drawers", create=True)
+        col.add(documents=["wrapped milvus document"], ids=["wrapped"], metadatas=[{"wing": "w"}])
+        result = col.query(query_texts=["wrapped"], n_results=1)
+        assert result.ids == [["wrapped"]]
+    finally:
+        reset_backends()
+
+
+def test_milvus_cross_palace_isolation_conformance(tmp_path):
+    _require_milvus_lite()
+    backend = MilvusBackend()
+    cols = []
+    try:
+        for label in ("alpha", "beta"):
+            path = tmp_path / label
+            ref = PalaceRef(id=str(path), local_path=str(path))
+            cols.append(backend.get_collection(palace=ref, collection_name="drawers", create=True))
+        assert_partition_isolation(backend, cols[0], cols[1], embedding=[1.0, 0.0, 0.0, 0.0])
+    finally:
+        backend.close()
+
+
+def test_milvus_namespace_isolation_conformance(tmp_path):
+    _require_milvus_lite()
+    assert "supports_namespace_isolation" in MilvusBackend.capabilities
+    backend = MilvusBackend()
+    try:
+        ref_a = PalaceRef(
+            id=str(tmp_path / "tenant-a"),
+            local_path=str(tmp_path / "tenant-a"),
+            namespace="tenant-a",
+        )
+        ref_b = PalaceRef(
+            id=str(tmp_path / "tenant-b"),
+            local_path=str(tmp_path / "tenant-b"),
+            namespace="tenant-b",
+        )
+        col_a = backend.get_collection(palace=ref_a, collection_name="drawers", create=True)
+        col_b = backend.get_collection(palace=ref_b, collection_name="drawers", create=True)
+        assert col_a._remote_collection != col_b._remote_collection
+        assert_partition_isolation(backend, col_a, col_b, embedding=[1.0, 0.0, 0.0, 0.0])
+    finally:
+        backend.close()
+
+
+def test_milvus_zilliz_cloud_roundtrip_when_enabled(tmp_path):
+    pytest.importorskip("pymilvus")
+    uri = os.environ.get("MEMPALACE_MILVUS_URI")
+    token = os.environ.get("MEMPALACE_MILVUS_TOKEN")
+    if not uri or not token:
+        pytest.skip(
+            "set MEMPALACE_MILVUS_URI and MEMPALACE_MILVUS_TOKEN "
+            "to run live Milvus/Zilliz Cloud test"
+        )
+
+    backend = MilvusBackend()
+    namespace = f"live_{uuid.uuid4().hex}"
+    palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path), namespace=namespace)
+    col = backend.get_collection(
+        palace=palace,
+        collection_name="drawers",
+        create=True,
+        options={"uri": uri, "token": token, "namespace": namespace},
+    )
+    try:
+        col.upsert(
+            ids=["cloud-a", "cloud-b"],
+            documents=["rareterm live milvus backend", "other live document"],
+            metadatas=[{"wing": "live", "rank": 2}, {"wing": "other", "rank": 1}],
+            embeddings=[[1.0, 0.0], [0.0, 1.0]],
+        )
+        assert col.count() == 2
+
+        result = col.query(
+            query_embeddings=[[1.0, 0.0]],
+            n_results=2,
+            where={"wing": "live"},
+        )
+        assert result.ids == [["cloud-a"]]
+
+        ranked = col.query(
+            query_embeddings=[[1.0, 0.0]],
+            n_results=2,
+            include=["distances"],
+        )
+        assert ranked.ids == [["cloud-a", "cloud-b"]]
+        assert ranked.distances[0] == pytest.approx([0.0, 1.0])
+
+        hits = col.lexical_search(query="rareterm", n_results=1).hits
+        assert hits and hits[0].id == "cloud-a"
+
+        col.delete(ids=["cloud-a"])
+        assert col.get(ids=["cloud-a"]).ids == []
+    finally:
+        try:
+            if col._client.has_collection(col._remote_collection):
+                col._client.drop_collection(col._remote_collection)
+        except Exception:
+            pass
+        backend.close()

tests/test_miner.py

@@ -95,8 +95,8 @@ def test_mine_computes_hallways_for_wing_post_mine(monkeypatch):
 
     hallway_calls = []
 
-    def fake_compute(wing, col=None, min_count=2):
-        hallway_calls.append({"wing": wing, "col": col, "min_count": min_count})
+    def fake_compute(wing, col=None, min_count=2, config=None):
+        hallway_calls.append({"wing": wing, "col": col, "min_count": min_count, "config": config})
         return []  # no hallways materialized — that's not what we're testing
 
     # Patch at the call site (mempalace.miner.compute_hallways_for_wing) so
@@ -134,6 +134,7 @@ def test_mine_computes_hallways_for_wing_post_mine(monkeypatch):
         assert call["col"] is not None, (
             "must pass the live collection so hallways can query drawers"
         )
+        assert call["config"].palace_path == str(palace_path)
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)
 
@@ -147,7 +148,7 @@ def test_mine_hallway_failure_does_not_crash_mine(monkeypatch):
     """
     from mempalace import miner as miner_mod
 
-    def angry_compute(wing, col=None, min_count=2):
+    def angry_compute(wing, col=None, min_count=2, config=None):
         raise RuntimeError("simulated hallway-compute explosion")
 
     monkeypatch.setattr(miner_mod, "compute_hallways_for_wing", angry_compute)
@@ -194,8 +195,8 @@ def test_mine_computes_entity_tunnels_for_wing_post_mine(monkeypatch):
 
     entity_tunnel_calls = []
 
-    def fake_compute(wing):
-        entity_tunnel_calls.append({"wing": wing})
+    def fake_compute(wing, config=None):
+        entity_tunnel_calls.append({"wing": wing, "config": config})
         return 0  # no tunnels — that's not what we're testing here
 
     # Patch at the call site (mempalace.miner._compute_entity_tunnels_for_wing)
@@ -227,6 +228,7 @@ def test_mine_computes_entity_tunnels_for_wing_post_mine(monkeypatch):
             f"got {len(entity_tunnel_calls)}"
         )
         assert entity_tunnel_calls[0]["wing"] == "test_project"
+        assert entity_tunnel_calls[0]["config"].palace_path == str(palace_path)
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)
 
@@ -241,7 +243,7 @@ def test_mine_entity_tunnel_failure_does_not_crash_mine(monkeypatch):
     """
     from mempalace import miner as miner_mod
 
-    def angry_compute(wing):
+    def angry_compute(wing, config=None):
         raise RuntimeError("simulated entity-tunnel-compute explosion")
 
     monkeypatch.setattr(miner_mod, "_compute_entity_tunnels_for_wing", angry_compute)
@@ -349,6 +351,20 @@ def test_scan_project_includes_kotlin_files():
         ]
 
 
+def test_scan_project_includes_latex_files():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        project_root = Path(tmpdir).resolve()
+        write_file(
+            project_root / "main.tex",
+            "\\documentclass{article}\n\\begin{document}\nHello, world.\n\\end{document}\n" * 20,
+        )
+        write_file(
+            project_root / "refs.bib",
+            "@article{lamport1986, author={Leslie Lamport}, title={LaTeX}, year={1986}}\n" * 20,
+        )
+        assert scanned_files(project_root) == ["main.tex", "refs.bib"]
+
+
 def test_scan_project_respects_gitignore():
     tmpdir = tempfile.mkdtemp()
     try:
@@ -575,6 +591,115 @@ def test_scan_project_logs_nested_symlink_with_relative_path(tmp_path, capsys):
     assert "(symlink)" in err
 
 
+def test_scan_project_exclude_patterns_skips_matching_files():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        project_root = Path(tmpdir).resolve()
+
+        write_file(project_root / "src" / "app.py", "print('app')\n" * 20)
+        write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20)
+        write_file(project_root / "README.md", "# README\n" * 20)
+
+        # *.md excludes README.md and docs/guide.md; docs/* would also cover docs/
+        # subtree.  Patterns follow .gitignore syntax via GitignoreMatcher.
+        assert scanned_files(
+            project_root,
+            exclude_patterns=["*.md", "docs/*"],
+        ) == ["src/app.py"]
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def test_scan_project_exclude_patterns_prunes_entire_directory():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        project_root = Path(tmpdir).resolve()
+
+        write_file(project_root / "src" / "app.py", "print('app')\n" * 20)
+        write_file(project_root / "docs" / "sub" / "deep.md", "# Deep\n" * 20)
+
+        # A trailing-slash pattern (dir-only) prunes the whole directory so
+        # os.walk never descends into it.
+        assert scanned_files(project_root, exclude_patterns=["docs/"]) == ["src/app.py"]
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def test_scan_project_exclude_patterns_include_ignored_bypasses_exclusion():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        project_root = Path(tmpdir).resolve()
+
+        write_file(project_root / "src" / "app.py", "print('app')\n" * 20)
+        write_file(project_root / ".gitignore", "docs/\n")
+        write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20)
+
+        # docs/ is both gitignored and matched by exclude_patterns, but
+        # include_ignored should override both filters and bring it back.
+        assert scanned_files(
+            project_root,
+            exclude_patterns=["docs/*"],
+            include_ignored=["docs"],
+        ) == ["docs/guide.md", "src/app.py"]
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def test_mine_exclude_patterns_applied_to_prescan_files(monkeypatch):
+    """exclude_patterns from config must be applied when mine() is given a pre-scanned list.
+
+    The init command calls scan_project() first to show a file-count estimate,
+    then passes the result to mine(..., files=...) to avoid walking the tree
+    twice.  The _mine_impl elif branch must apply exclude_patterns to that list
+    so per-project exclusions still take effect.
+    """
+    import mempalace.miner as miner_mod
+
+    tmpdir = tempfile.mkdtemp()
+    try:
+        project_root = Path(tmpdir).resolve()
+
+        write_file(project_root / "src" / "app.py", "print('app')\n" * 20)
+        write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20)
+        write_file(project_root / "README.md", "# README\n" * 20)
+
+        with open(project_root / "mempalace.yaml", "w") as f:
+            yaml.dump(
+                {
+                    "wing": "test_project",
+                    "rooms": [{"name": "general", "description": "General"}],
+                    "exclude_patterns": ["*.md", "docs/"],
+                },
+                f,
+            )
+
+        # Simulate the init double-scan: scan without exclude_patterns first
+        # (representing the caller that doesn't know the config), then pass
+        # the full list to mine() so the elif branch applies the config exclusions.
+        prescan = scan_project(str(project_root), exclude_patterns=None)
+        assert len(prescan) >= 2, "prescan should include .md files before filtering"
+
+        # Capture which files process_file is called with during dry_run
+        processed = []
+        _real_process_file = miner_mod.process_file
+
+        def _capture_process_file(filepath, **kwargs):
+            processed.append(filepath)
+            return _real_process_file(filepath, **kwargs)
+
+        monkeypatch.setattr(miner_mod, "process_file", _capture_process_file)
+
+        palace_path = project_root / "palace"
+        mine(str(project_root), str(palace_path), dry_run=True, files=prescan)
+
+        processed_rel = [p.relative_to(project_root).as_posix() for p in processed]
+        assert "src/app.py" in processed_rel
+        assert "docs/guide.md" not in processed_rel
+        assert "README.md" not in processed_rel
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
 def test_entity_metadata_finds_cyrillic_names(monkeypatch):
     """Entity extraction must find non-Latin names when entity_languages includes the locale."""
     import mempalace.palace as palace_mod
@@ -626,6 +751,62 @@ def test_entity_metadata_matches_known_names_case_insensitively(monkeypatch):
     assert "Lumi" in matched_mixed
 
 
+def test_scan_project_skips_oversized_files(tmp_path, capsys, monkeypatch):
+    import re
+
+    import mempalace.miner as miner_mod
+
+    monkeypatch.setattr(miner_mod, "MAX_FILE_SIZE", 100)
+
+    write_file(tmp_path / "small.py", "x = 1\n" * 10)
+    write_file(tmp_path / "big.py", "x = 1\n" * 100)
+
+    files = scan_project(str(tmp_path))
+    names = [f.name for f in files]
+    assert "small.py" in names
+    assert "big.py" not in names
+
+    err = capsys.readouterr().err
+    # SKIP message goes to stderr, matching the existing
+    # `SKIP: <rel> (symlink)` line in the same function.
+    assert "SKIP: big.py" in err
+    # Validate the full template shape so a regression to bare-substring
+    # output (or a drop of the MB suffix) trips the test instead of
+    # silently passing.
+    assert re.search(r"SKIP: big\.py \(\d+\.\d+ MB\) exceeds \d+ MB limit", err), err
+
+
+def test_scan_project_skips_unreadable_files(tmp_path, capsys, monkeypatch):
+    from pathlib import Path
+
+    # Two real files; the unreadable one will have stat() raise.
+    write_file(tmp_path / "readable.py", "x = 1\n")
+    unreadable = tmp_path / "unreadable.py"
+    write_file(unreadable, "x = 1\n")
+
+    real_stat = Path.stat
+
+    def selective_stat(self, *args, **kwargs):
+        # On Py 3.10+, Path.is_symlink() routes through lstat ->
+        # stat(follow_symlinks=False). Only raise for the follow-symlinks
+        # call that the actual size-check makes, otherwise the test
+        # never reaches the size-check arm we want to exercise.
+        if self.name == "unreadable.py" and kwargs.get("follow_symlinks", True):
+            raise PermissionError(13, "Permission denied", str(self))
+        return real_stat(self, *args, **kwargs)
+
+    monkeypatch.setattr(Path, "stat", selective_stat)
+
+    files = scan_project(str(tmp_path))
+    names = [f.name for f in files]
+    assert "readable.py" in names
+    assert "unreadable.py" not in names
+
+    err = capsys.readouterr().err
+    assert "SKIP: unreadable.py" in err
+    assert "stat error" in err
+
+
 def test_file_already_mined_check_mtime():
     tmpdir = tempfile.mkdtemp()
     try:

tests/test_miner_fts5_validation.py

@@ -150,17 +150,25 @@ def test_helper_raises_on_page_mangled_sqlite(tmp_path):
     assert "malformed" in combined or "quick_check failed" in combined
 
 
-def test_helper_raises_on_fts5_segment_corruption(tmp_path):
-    """The reporter-shaped failure: FTS5 inverted index malformed, main pages OK."""
+def test_helper_auto_heals_fts5_segment_corruption(tmp_path):
+    """The reporter-shaped failure: FTS5 inverted index malformed, main pages
+    OK. This is exactly the isolated-FTS5 condition
+    ``maybe_autoheal_fts5_index`` (#1926/#1928) rebuilds in place, so the
+    validator must return silently (healed) rather than raise.
+    """
+    import sqlite3
+    from contextlib import closing
+
     palace = tmp_path / "palace"
     _build_palace_with_drawer(palace)
     _corrupt_fts5_segment(palace / "chroma.sqlite3")
 
-    with pytest.raises(MineValidationError) as exc_info:
-        _validate_palace_fts5_after_mine(str(palace))
+    # Must not raise: isolated FTS5 corruption is auto-healed before the
+    # validator would otherwise raise MineValidationError.
+    assert _validate_palace_fts5_after_mine(str(palace)) is None
 
-    err = exc_info.value
-    assert "fts5" in " ".join(err.errors).lower()
+    with closing(sqlite3.connect(str(palace / "chroma.sqlite3"))) as conn:
+        assert conn.execute("PRAGMA quick_check").fetchall() == [("ok",)]
 
 
 # ── 3. cmd_mine surfaces MineValidationError as exit-1 + banner ─────
@@ -246,11 +254,25 @@ def test_validate_skipped_on_dry_run(tmp_path, monkeypatch):
 
 
 def test_full_chain_raises_through_mine_impl(tmp_path, monkeypatch):
-    """Run a real mine, corrupt FTS5 mid-process, re-mine. The validator
-    inside _mine_impl must raise MineValidationError as the explicit source
-    (spy verifies). The new `except MineValidationError: raise` clause in
-    miner._mine_impl bypasses the partial-progress "Mine aborted" banner so
-    cmd_mine prints the single, authoritative recovery message.
+    """Run a real mine, then force the validator to raise on the re-mine.
+    The validator inside _mine_impl must raise MineValidationError as the
+    explicit source (spy verifies). The new `except MineValidationError:
+    raise` clause in miner._mine_impl bypasses the partial-progress "Mine
+    aborted" banner so cmd_mine prints the single, authoritative recovery
+    message.
+
+    The validator is monkeypatched to raise directly rather than corrupting
+    the real sqlite file: page-level (non-isolated) corruption severe
+    enough to make quick_check report non-FTS5 errors is also severe enough
+    that ChromaDB's own Rust bindings panic just opening the file for the
+    re-mine's ``get_collection()`` call -- before this test's own validator
+    ever runs. That's a real, useful thing to know (native panic rather
+    than a catchable Python exception, on a sufficiently corrupted file),
+    but it's a different failure mode than what this test is about: proving
+    _mine_impl's exception-passthrough logic runs cleanly when the
+    validator raises. Isolated FTS5-only corruption is covered separately
+    by test_full_chain_auto_heals_isolated_fts5_corruption below, and would
+    never reach this raise path at all now that it's auto-healed.
     """
     from mempalace import miner as miner_mod
     from mempalace import palace as palace_mod
@@ -269,17 +291,20 @@ def test_full_chain_raises_through_mine_impl(tmp_path, monkeypatch):
         dry_run=False,
     )
 
-    _corrupt_fts5_segment(palace / "chroma.sqlite3")
+    # Second mine needs new content to process, else file_already_mined's
+    # mtime check short-circuits before the validator ever runs.
+    (src / "big.md").write_text("lorem ipsum dolor " * 200)
 
     # Spy on the validator so we can prove IT was the raise-source, not
     # some other exception masquerading as MineValidationError.
     called = []
-    real_validator = palace_mod._validate_palace_fts5_after_mine
+    real_errors = ["malformed inverted index for FTS5 table main.embedding_fulltext_search"]
 
     def _validator_spy(path):
         called.append(path)
-        return real_validator(path)
+        raise MineValidationError(path, real_errors)
 
+    monkeypatch.setattr(palace_mod, "_validate_palace_fts5_after_mine", _validator_spy)
     monkeypatch.setattr(miner_mod, "_validate_palace_fts5_after_mine", _validator_spy)
 
     with pytest.raises(MineValidationError) as exc_info:
@@ -293,15 +318,15 @@ def test_full_chain_raises_through_mine_impl(tmp_path, monkeypatch):
         )
 
     assert called == [str(palace)], f"validator must be the raise-source; spy recorded: {called}"
-    assert "fts5" in " ".join(exc_info.value.errors).lower()
+    assert list(exc_info.value.errors) == real_errors
     assert exc_info.value.palace_path == str(palace)
 
 
-def test_mine_impl_does_not_print_partial_summary_on_validation_error(tmp_path, capsys):
-    """When _validate_palace_fts5_after_mine raises, miner._mine_impl must
-    NOT print the "Mine aborted by exception" partial-progress banner that
-    `except Exception` adds. That banner is reserved for true mid-loop
-    failures and would double-up with cmd_mine's recovery banner.
+def test_full_chain_auto_heals_isolated_fts5_corruption(tmp_path):
+    """Run a real mine, corrupt only the FTS5 index (main pages intact),
+    re-mine. The full _mine_impl chain must auto-heal and succeed silently
+    -- not raise -- since this is exactly the isolated condition
+    maybe_autoheal_fts5_index rebuilds in place (#1926/#1928).
     """
     from mempalace import miner as miner_mod
 
@@ -318,8 +343,165 @@ def test_mine_impl_does_not_print_partial_summary_on_validation_error(tmp_path,
         limit=0,
         dry_run=False,
     )
+
     _corrupt_fts5_segment(palace / "chroma.sqlite3")
 
+    # Editing big.md so the re-mine has new content to process (otherwise
+    # file_already_mined's mtime check would short-circuit before the
+    # validator ever runs).
+    (src / "big.md").write_text("lorem ipsum dolor " * 200)
+
+    # Must not raise: the corruption is healed before mine_impl returns.
+    miner_mod.mine(
+        project_dir=str(src),
+        palace_path=str(palace),
+        wing_override=None,
+        agent="mempalace",
+        limit=0,
+        dry_run=False,
+    )
+
+
+def test_validator_suppresses_raise_when_autoheal_clears(tmp_path, monkeypatch):
+    """Build-independent companion to test_full_chain_auto_heals_isolated_fts5_corruption.
+
+    _corrupt_fts5_segment pytest.skips on SQLite builds that refuse direct
+    FTS5 shadow-table writes, so on those builds the auto-heal wiring is
+    never actually exercised. Stubbing sqlite_integrity_errors and
+    maybe_autoheal_fts5_index directly (rather than fabricating real
+    corruption) proves the wiring in _validate_palace_fts5_after_mine itself,
+    deterministically, on every build.
+    """
+    from mempalace import repair as repair_mod
+
+    palace = tmp_path / "palace"
+    _build_palace_with_drawer(palace)
+
+    fts_error = ["malformed inverted index for FTS5 table main.embedding_fulltext_search"]
+    heal_calls = []
+
+    monkeypatch.setattr(repair_mod, "sqlite_integrity_errors", lambda _p: list(fts_error))
+
+    def _fake_heal(palace_path, errors, **_kwargs):
+        heal_calls.append((palace_path, tuple(errors)))
+        return []  # healed: no remaining errors
+
+    monkeypatch.setattr(repair_mod, "maybe_autoheal_fts5_index", _fake_heal)
+
+    _validate_palace_fts5_after_mine(str(palace))  # must not raise
+
+    assert heal_calls == [(str(palace), tuple(fts_error))]
+
+
+def test_validator_passes_logger_progress_not_print_to_autoheal(tmp_path, monkeypatch, capsys):
+    """The actual review-comment fix (gemini-code-assist, PR #1928): this
+    validator runs inside the MCP server process too (mcp_server.tool_mine ->
+    miner.mine), where stdout is the JSON-RPC transport. maybe_autoheal_fts5_index
+    defaults to progress=print, so _validate_palace_fts5_after_mine must
+    override it with a logging call instead -- otherwise a healed mine would
+    print raw progress text onto the transport stream mid-response.
+
+    The two tests above only prove the suppress/raise wiring and would pass
+    even if this line still defaulted to print (they mock
+    maybe_autoheal_fts5_index entirely, discarding whatever kwargs it's
+    called with). This test instead captures the actual kwargs passed and
+    invokes the captured progress callable, asserting nothing lands on stdout.
+
+    Deliberately not pinned to logger.info specifically: which severity level
+    is used is a verbosity choice, not a correctness requirement -- the bug
+    was "raw print() to stdout", not "wrong log level". Asserting equality to
+    one specific level method would make the test fail on a reasonable future
+    change (e.g. to logger.debug) that doesn't reintroduce the actual bug.
+    """
+    from mempalace import repair as repair_mod
+    from mempalace.palace import logger as palace_logger
+
+    palace = tmp_path / "palace"
+    _build_palace_with_drawer(palace)
+
+    fts_error = ["malformed inverted index for FTS5 table main.embedding_fulltext_search"]
+    captured_kwargs = {}
+
+    monkeypatch.setattr(repair_mod, "sqlite_integrity_errors", lambda _p: list(fts_error))
+
+    def _fake_heal(palace_path, errors, **kwargs):
+        captured_kwargs.update(kwargs)
+        kwargs.get("progress", print)("probe progress message")
+        return []
+
+    monkeypatch.setattr(repair_mod, "maybe_autoheal_fts5_index", _fake_heal)
+
+    _validate_palace_fts5_after_mine(str(palace))
+
+    progress = captured_kwargs.get("progress")
+    assert progress is not None
+    assert progress is not print
+    # Any severity is fine; what matters is that it's a bound method of this
+    # module's own logger (routed through logging), not the raw print builtin.
+    assert getattr(progress, "__self__", None) is palace_logger
+    out, _ = capsys.readouterr()
+    assert out == ""
+
+
+def test_validator_still_raises_when_autoheal_cannot_clear(tmp_path, monkeypatch):
+    """Auto-heal is a best-effort first attempt, not a reason to swallow real
+    corruption: when maybe_autoheal_fts5_index can't clear the condition
+    (broader corruption, lock contention, rebuild failure), the validator
+    must still raise. Same build-independence rationale as the test above.
+    """
+    from mempalace import repair as repair_mod
+
+    palace = tmp_path / "palace"
+    _build_palace_with_drawer(palace)
+
+    errors = ["database disk image is malformed"]
+    monkeypatch.setattr(repair_mod, "sqlite_integrity_errors", lambda _p: list(errors))
+    monkeypatch.setattr(
+        repair_mod, "maybe_autoheal_fts5_index", lambda palace_path, errs, **_kwargs: errs
+    )
+
+    with pytest.raises(MineValidationError):
+        _validate_palace_fts5_after_mine(str(palace))
+
+
+def test_mine_impl_does_not_print_partial_summary_on_validation_error(
+    tmp_path, capsys, monkeypatch
+):
+    """When _validate_palace_fts5_after_mine raises, miner._mine_impl must
+    NOT print the "Mine aborted by exception" partial-progress banner that
+    `except Exception` adds. That banner is reserved for true mid-loop
+    failures and would double-up with cmd_mine's recovery banner.
+
+    The validator is monkeypatched to raise directly rather than corrupting
+    the real sqlite file -- see test_full_chain_raises_through_mine_impl's
+    docstring for why real non-isolated corruption isn't usable here
+    (ChromaDB's own Rust bindings panic opening a sufficiently corrupted
+    file, before this test's validator would ever run).
+    """
+    from mempalace import miner as miner_mod
+    from mempalace import palace as palace_mod
+
+    palace = tmp_path / "palace"
+    src = tmp_path / "src"
+    src.mkdir()
+    (src / "big.md").write_text("lorem ipsum " * 200)
+
+    miner_mod.mine(
+        project_dir=str(src),
+        palace_path=str(palace),
+        wing_override=None,
+        agent="mempalace",
+        limit=0,
+        dry_run=False,
+    )
+    (src / "big.md").write_text("lorem ipsum dolor " * 200)
+
+    def _raise(path):
+        raise MineValidationError(path, ["malformed inverted index for FTS5 table"])
+
+    monkeypatch.setattr(palace_mod, "_validate_palace_fts5_after_mine", _raise)
+    monkeypatch.setattr(miner_mod, "_validate_palace_fts5_after_mine", _raise)
+
     with pytest.raises(MineValidationError):
         miner_mod.mine(
             project_dir=str(src),
@@ -525,28 +707,32 @@ def test_mine_formats_keyboard_interrupt_skips_validator(tmp_path, monkeypatch):
 
 
 def test_mine_formats_full_chain_raises_when_fts5_corrupt(tmp_path, monkeypatch):
-    """End-to-end: a palace with corrupted FTS5 + real mine_formats call
-    must propagate MineValidationError from `_validate_palace_fts5_after_mine`.
-    Mirrors `test_full_chain_raises_through_mine_impl` for the extract path.
+    """End-to-end: mine_formats must propagate MineValidationError from
+    `_validate_palace_fts5_after_mine`. Mirrors
+    `test_full_chain_raises_through_mine_impl` for the extract path.
     Empty source dir is sufficient: the for-loop iterates zero times, the
-    `else` branch runs, validation fires on the pre-corrupted sqlite.
+    `else` branch runs, validation fires on the (mock-corrupted) sqlite.
+
+    The validator is monkeypatched to raise directly rather than corrupting
+    the real sqlite file -- see test_full_chain_raises_through_mine_impl's
+    docstring for why real non-isolated corruption isn't usable here
+    (ChromaDB's own Rust bindings panic opening a sufficiently corrupted
+    file, before this test's validator would ever run).
     """
     from mempalace import format_miner as format_mod
-    from mempalace import palace as palace_mod
 
     palace = tmp_path / "palace"
     _build_palace_with_drawer(palace)
-    _corrupt_fts5_segment(palace / "chroma.sqlite3")
 
     src = tmp_path / "docs"
     src.mkdir()
 
     called = []
-    real_validator = palace_mod._validate_palace_fts5_after_mine
+    real_errors = ["malformed inverted index for FTS5 table main.embedding_fulltext_search"]
 
     def _validator_spy(path):
         called.append(path)
-        return real_validator(path)
+        raise MineValidationError(path, real_errors)
 
     monkeypatch.setattr(format_mod, "_validate_palace_fts5_after_mine", _validator_spy)
 
@@ -561,5 +747,31 @@ def test_mine_formats_full_chain_raises_when_fts5_corrupt(tmp_path, monkeypatch)
         )
 
     assert called == [str(palace)], f"validator must be the raise-source; spy recorded: {called}"
-    assert "fts5" in " ".join(exc_info.value.errors).lower()
+    assert list(exc_info.value.errors) == real_errors
     assert exc_info.value.palace_path == str(palace)
+
+
+def test_mine_formats_full_chain_auto_heals_isolated_fts5_corruption(tmp_path):
+    """End-to-end for the extract path: isolated FTS5-only corruption must
+    be auto-healed by `_validate_palace_fts5_after_mine`, not raise.
+    Mirrors `test_full_chain_auto_heals_isolated_fts5_corruption` for the
+    project-miner path (#1926/#1928).
+    """
+    from mempalace import format_miner as format_mod
+
+    palace = tmp_path / "palace"
+    _build_palace_with_drawer(palace)
+    _corrupt_fts5_segment(palace / "chroma.sqlite3")
+
+    src = tmp_path / "docs"
+    src.mkdir()
+
+    # Must not raise: the corruption is healed before mine_formats returns.
+    format_mod.mine_formats(
+        format_dir=str(src),
+        palace_path=str(palace),
+        wing="testwing",
+        agent="mempalace",
+        limit=0,
+        dry_run=False,
+    )

tests/test_normalize.py

@@ -1,4 +1,5 @@
 import json
+import stat
 from unittest.mock import patch
 
 from mempalace.normalize import (
@@ -1680,11 +1681,23 @@ def test_claude_code_jsonl_thinking_blocks_ignored():
     assert "A" in result
 
 
-def test_normalize_rejects_large_file():
-    """Files over 500 MB should raise IOError before reading."""
-    with patch("mempalace.normalize.os.path.getsize", return_value=600 * 1024 * 1024):
+def test_normalize_rejects_large_file(tmp_path):
+    """Files over the 500 MB safety limit raise IOError instead of being read.
+
+    Size is checked via ``os.fstat`` on the already-open descriptor (no TOCTOU
+    gap), so we stub fstat to report a huge regular file rather than mocking the
+    pre-open ``os.path.getsize``.
+    """
+    big = tmp_path / "huge_file.txt"
+    big.write_text("not actually huge")
+
+    class _HugeStat:
+        st_mode = stat.S_IFREG | 0o644
+        st_size = 600 * 1024 * 1024
+
+    with patch("mempalace.normalize.os.fstat", return_value=_HugeStat()):
         try:
-            normalize("/fake/huge_file.txt")
+            normalize(str(big))
             assert False, "Should have raised IOError"
         except IOError as e:
             assert "too large" in str(e).lower()

tests/test_palace.py

@@ -2,6 +2,8 @@
 
 import chromadb
 
+from _chroma_palace_helper import make_minimal_chroma_sqlite
+
 from mempalace.backends import CollectionNotInitializedError, PalaceNotFoundError
 from mempalace.palace import _open_collection_or_explain, get_collection
 
@@ -94,7 +96,7 @@ def test_open_collection_or_explain_state_e_unexpected_error(tmp_path, monkeypat
     emit, lines = _capture()
     palace = tmp_path / "palace"
     palace.mkdir()
-    (palace / "chroma.sqlite3").touch()  # pass the isfile guard
+    make_minimal_chroma_sqlite(palace)  # pass the isfile guard
 
     def boom(*args, **kwargs):
         raise RuntimeError("disk on fire")
@@ -125,7 +127,7 @@ def test_open_collection_or_explain_propagates_palace_not_found_from_backend(tmp
     emit, lines = _capture()
     palace = tmp_path / "palace"
     palace.mkdir()
-    (palace / "chroma.sqlite3").touch()
+    make_minimal_chroma_sqlite(palace)
 
     def raise_pnf(*args, **kwargs):
         raise PalaceNotFoundError(str(palace))
@@ -151,7 +153,7 @@ def test_open_collection_or_explain_reraises_backend_closed_error(tmp_path, monk
 
     palace = tmp_path / "palace"
     palace.mkdir()
-    (palace / "chroma.sqlite3").touch()
+    make_minimal_chroma_sqlite(palace)
 
     def raise_closed(*args, **kwargs):
         raise BackendClosedError("ChromaBackend has been closed")
@@ -171,7 +173,7 @@ def test_open_collection_or_explain_distinguishes_collection_subclass(tmp_path,
     emit, lines = _capture()
     palace = tmp_path / "palace"
     palace.mkdir()
-    (palace / "chroma.sqlite3").touch()
+    make_minimal_chroma_sqlite(palace)
 
     def raise_cnie(*args, **kwargs):
         raise CollectionNotInitializedError(str(palace))

tests/test_palace_graph_tunnels.py

@@ -201,6 +201,21 @@ class TestTopicTunnels:
     dedup and persistence with explicit tunnels.
     """
 
+    def test_explicit_config_scopes_persistence_to_selected_palace(self, tmp_path):
+        from mempalace.config import MempalaceConfig
+
+        default_cfg = MempalaceConfig(palace_path=tmp_path / "default" / "palace")
+        selected_cfg = MempalaceConfig(palace_path=tmp_path / "selected" / "palace")
+        topics_by_wing = {"wing_alpha": ["OpenAPI"], "wing_beta": ["OpenAPI"]}
+
+        created = palace_graph.compute_topic_tunnels(
+            topics_by_wing, min_count=1, config=selected_cfg
+        )
+
+        assert len(created) == 1
+        assert palace_graph._load_tunnels(selected_cfg) == created
+        assert palace_graph._load_tunnels(default_cfg) == []
+
     def test_compute_topic_tunnels_creates_link_for_shared_topic(self, tmp_path, monkeypatch):
         _use_tmp_tunnel_file(monkeypatch, tmp_path)
         topics_by_wing = {

tests/test_palace_locks.py

@@ -11,11 +11,13 @@ from __future__ import annotations
 
 import multiprocessing
 import os
+import threading
 import time
 import sys
 
 import pytest
 
+import mempalace.palace as palace_mod
 from mempalace.palace import (
     _write_lock_holder,
     MineAlreadyRunning,
@@ -68,6 +70,45 @@ def _hold_lock(palace_path: str, ready_flag: str, release_flag: str) -> int:
 # ---------------------------------------------------------------------------
 
 
+def test_mine_palace_lock_reentrant_across_threads_same_process(tmp_path):
+    """Process-wide re-entrancy: a second acquisition from a *different thread*
+    of the same process passes through instead of self-conflicting.
+
+    Regression for the MCP HTTP transport (ThreadingHTTPServer): the writer
+    lease is acquired on one thread (mcp_server._acquire_mcp_writer_lock) but
+    write requests are dispatched on other worker threads. With the old
+    thread-local re-entrancy those handlers re-acquired the process-held flock
+    and raised MineAlreadyRunning ("palace ... is held by PID <self>").
+    Re-entrancy is now process-wide, so same-process cross-thread acquisition is
+    a pass-through.
+    """
+    palace = str(tmp_path / "palace")
+    os.makedirs(palace, exist_ok=True)
+
+    outer = mine_palace_lock(palace)
+    outer.__enter__()  # main thread holds the lease, like the MCP writer-lease
+    try:
+        result: dict = {}
+
+        def worker():
+            try:
+                with mine_palace_lock(palace):
+                    result["acquired"] = True
+            except MineAlreadyRunning as exc:  # pragma: no cover - failure path
+                result["error"] = str(exc)
+
+        t = threading.Thread(target=worker)
+        t.start()
+        t.join(timeout=5)
+
+        assert not t.is_alive(), "worker thread hung acquiring the palace lock"
+        assert result.get("acquired") is True, (
+            f"cross-thread same-process acquisition should pass through, got: {result}"
+        )
+    finally:
+        outer.__exit__(None, None, None)
+
+
 def test_single_acquire_succeeds(tmp_path, monkeypatch):
     monkeypatch.setenv("HOME", str(tmp_path))
     with mine_palace_lock(str(tmp_path / "palace")):
@@ -354,3 +395,47 @@ def test_mine_global_lock_is_alias_for_back_compat(tmp_path, monkeypatch):
     assert mine_global_lock is mine_palace_lock
     with mine_global_lock(str(tmp_path / "palace")):
         pass  # the alias accepts the same palace_path argument
+
+
+def test_holder_set_not_orphaned_by_interrupt_after_mark_held(tmp_path, monkeypatch):
+    """An async interrupt right after the hold is recorded must not leave the
+    palace key stranded in the process-wide holder set.
+
+    ``_mark_held`` sits inside the ``try`` whose ``finally`` runs
+    ``_mark_released``, so the two are paired on every exit. If ``_mark_held``
+    ran before the ``try``, a ``KeyboardInterrupt``/signal landing in the gap
+    would strand the key: the outer ``finally`` still frees the flock, so
+    ``_held_by_this_process`` would report a hold the OS lock no longer backs,
+    and the next re-entrant acquire in this process would pass through and
+    write without the flock (two concurrent writers into one palace).
+    """
+    monkeypatch.setenv("HOME", str(tmp_path))
+    palace = str(tmp_path / "palace")
+
+    before = set(palace_mod._palace_lock_keys)
+
+    # Model the interrupt: run the real _mark_held (records the hold), then
+    # raise, as a signal arriving at that instant would.
+    real_mark_held = palace_mod._mark_held
+
+    def _mark_then_interrupt(lock_key):
+        real_mark_held(lock_key)
+        raise KeyboardInterrupt
+
+    palace_mod._mark_held = _mark_then_interrupt
+    try:
+        with pytest.raises(KeyboardInterrupt):
+            with mine_palace_lock(palace):
+                pass
+    finally:
+        # Restore by hand (not monkeypatch.setattr, which unpatches only at
+        # teardown) so the reuse check below calls the real _mark_held.
+        palace_mod._mark_held = real_mark_held
+
+    assert set(palace_mod._palace_lock_keys) == before, (
+        "palace key was stranded in the holder set after an interrupt: "
+        "the in-memory hold outlived the flock"
+    )
+    # The flock was freed and no stale hold remains, so the lock is reusable.
+    with mine_palace_lock(palace):
+        pass

tests/test_pgvector_backend.py

@@ -93,8 +93,19 @@ class _FakePgVectorClient:
             out.append(item)
         return out
 
-    def scroll_rows(self, table, *, where=None, with_embedding=False, limit=None, offset=None):
-        self.scroll_calls.append({"where": where, "limit": limit, "offset": offset})
+    def scroll_rows(
+        self,
+        table,
+        *,
+        where=None,
+        with_embedding=False,
+        with_document=True,
+        limit=None,
+        offset=None,
+    ):
+        self.scroll_calls.append(
+            {"where": where, "limit": limit, "offset": offset, "with_document": with_document}
+        )
         rows = self._filtered(table, where)
         if limit is not None or offset:
             # Mirror the real backend: ORDER BY id, then LIMIT/OFFSET.
@@ -108,7 +119,9 @@ class _FakePgVectorClient:
             out.append(
                 {
                     "id": row["id"],
-                    "document": row["document"],
+                    # Match the real backend: NULL document becomes empty string
+                    # via the SELECT NULL::text projection when with_document=False.
+                    "document": row["document"] if with_document else "",
                     "metadata": row.get("metadata") or {},
                     "embedding": row.get("embedding") if with_embedding else None,
                     "distance": None,
@@ -390,7 +403,7 @@ def test_pgvector_get_unfiltered_page_pushes_limit_offset(tmp_path, fake_pgvecto
 
     # An unfiltered page is pushed to SQL as LIMIT/OFFSET instead of fetching
     # the whole table and slicing in Python (the O(rows x pages) path).
-    assert client.scroll_calls == [{"where": None, "limit": 2, "offset": 1}]
+    assert client.scroll_calls == [{"where": None, "limit": 2, "offset": 1, "with_document": True}]
     # ORDER BY id, then OFFSET 1 LIMIT 2 -> b, c.
     assert page.ids == ["b", "c"]
 
@@ -410,7 +423,9 @@ def test_pgvector_get_filtered_page_stays_on_full_scan(tmp_path, fake_pgvector):
 
     # A filtered get keeps the full-scan path (no LIMIT/OFFSET pushed) so the
     # exact _matches_where re-filter runs before pagination.
-    assert client.scroll_calls == [{"where": {"wing": "x"}, "limit": None, "offset": None}]
+    assert client.scroll_calls == [
+        {"where": {"wing": "x"}, "limit": None, "offset": None, "with_document": True}
+    ]
     assert page.ids == ["c"]
 
 
@@ -427,13 +442,17 @@ def test_pgvector_get_offset_only_and_limit_only_push(tmp_path, fake_pgvector):
     # offset-only (limit=None) is pushed.
     client.scroll_calls.clear()
     page = col.get(offset=2, include=["metadatas"])
-    assert client.scroll_calls == [{"where": None, "limit": None, "offset": 2}]
+    assert client.scroll_calls == [
+        {"where": None, "limit": None, "offset": 2, "with_document": True}
+    ]
     assert page.ids == ["c", "d"]
 
     # limit-only (offset=None) is pushed.
     client.scroll_calls.clear()
     page = col.get(limit=2, include=["metadatas"])
-    assert client.scroll_calls == [{"where": None, "limit": 2, "offset": None}]
+    assert client.scroll_calls == [
+        {"where": None, "limit": 2, "offset": None, "with_document": True}
+    ]
     assert page.ids == ["a", "b"]
 
 
@@ -451,7 +470,9 @@ def test_pgvector_get_negative_bounds_use_python_slice(tmp_path, fake_pgvector):
     # A negative offset must not reach SQL (OFFSET -1 would error); it falls
     # through to the unchanged full-scan + Python-slice path.
     page = col.get(offset=-1, include=["metadatas"])
-    assert client.scroll_calls == [{"where": None, "limit": None, "offset": None}]
+    assert client.scroll_calls == [
+        {"where": None, "limit": None, "offset": None, "with_document": True}
+    ]
     assert page.ids == ["c"]
 
 
@@ -473,6 +494,73 @@ def test_pgvector_get_pages_tile_without_overlap(tmp_path, fake_pgvector):
     assert p1 + p2 + p3 == ["a", "b", "c", "d", "e"]
 
 
+def test_pgvector_get_all_metadata_skips_document_column(tmp_path, fake_pgvector):
+    """The metadata-only fast path must NOT pull document text over the wire.
+
+    Default base ``get_all_metadata`` pages through ``get(include=["metadatas"])``,
+    which used to route here via scroll_rows with documents always selected — the
+    "separate follow-up" #1840 flagged. This override calls scroll_rows with
+    with_document=False so the SELECT projects NULL into the document slot,
+    dropping per-row payload for remote (TLS over WAN) clients where status
+    otherwise dominates wall time.
+    """
+    _backend, col = _collection(tmp_path)
+    col.add(
+        ids=["a", "b", "c"],
+        documents=["doc_a", "doc_b", "doc_c"],
+        metadatas=[
+            {"wing": "p", "room": "backend"},
+            {"wing": "p", "room": "frontend"},
+            {"wing": "q", "room": "backend"},
+        ],
+        embeddings=[[1, 0], [0, 1], [0.5, 0.5]],
+    )
+    client = fake_pgvector.instances[0]
+    client.scroll_calls.clear()
+
+    metas = col.get_all_metadata()
+
+    # Exactly one scroll, with_document=False (no document text on the wire).
+    assert client.scroll_calls == [
+        {"where": None, "limit": None, "offset": None, "with_document": False}
+    ]
+    # Returns just the metadata dicts (full set, any order — sort by wing+room for stability).
+    metas_sorted = sorted(metas, key=lambda m: (m["wing"], m["room"]))
+    assert metas_sorted == [
+        {"wing": "p", "room": "backend"},
+        {"wing": "p", "room": "frontend"},
+        {"wing": "q", "room": "backend"},
+    ]
+
+
+def test_pgvector_get_all_metadata_filtered_uses_fast_path(tmp_path, fake_pgvector):
+    """Filtered get_all_metadata uses the single-pass metadata-only fast path.
+
+    ``_matches_where`` only reads ``metadata``, so we keep ``with_document=False``
+    and apply the post-filter locally on the metadata dicts. SQL pushdown still
+    happens when the filter is pushdownable; the local ``_matches_where`` re-runs
+    for array/object semantics #1840's filtered path required.
+    """
+    _backend, col = _collection(tmp_path)
+    col.add(
+        ids=["a", "b", "c"],
+        documents=["doc_a", "doc_b", "doc_c"],
+        metadatas=[{"wing": "x"}, {"wing": "y"}, {"wing": "x"}],
+        embeddings=[[1, 0], [0, 1], [0.5, 0.5]],
+    )
+    client = fake_pgvector.instances[0]
+    client.scroll_calls.clear()
+
+    metas = col.get_all_metadata(where={"wing": "x"})
+
+    # Exactly one scroll with with_document=False — pushdown forwards the
+    # equality filter to SQL; no document text on the wire.
+    assert client.scroll_calls == [
+        {"where": {"wing": "x"}, "limit": None, "offset": None, "with_document": False}
+    ]
+    assert sorted(metas, key=lambda m: m["wing"]) == [{"wing": "x"}, {"wing": "x"}]
+
+
 def test_pgvector_delete_by_where_pushdown_and_local(tmp_path, fake_pgvector):
     _backend, col = _collection(tmp_path)
     col.add(

tests/test_qdrant_backend.py

@@ -5,6 +5,7 @@ import numpy as np
 import pytest
 
 from _backend_conformance import assert_partition_isolation
+from _chroma_palace_helper import make_minimal_chroma_sqlite
 
 from mempalace.backends import (
     BackendError,
@@ -12,6 +13,7 @@ from mempalace.backends import (
     CollectionNotInitializedError,
     DimensionMismatchError,
     PalaceRef,
+    UnsupportedCapabilityError,
     available_backends,
 )
 from mempalace.backends.qdrant import QdrantBackend
@@ -82,6 +84,7 @@ class _FakeQdrantClient:
         self.query_calls = []
         self.scroll_calls = []
         self.created_indexes = []
+        self.facet_calls = []
         _FakeQdrantClient.instances.append(self)
 
     def request(self, *_args, **_kwargs):
@@ -176,6 +179,33 @@ class _FakeQdrantClient:
     def delete_collection(self, collection):
         self.collections.pop(collection, None)
 
+    def facet_counts(
+        self,
+        collection,
+        *,
+        field,
+        qdrant_filter=None,
+        limit=1000,
+    ):
+        self.facet_calls.append((field, qdrant_filter))
+
+        counts = {}
+
+        points = list(self.collections.get(collection, {"points": {}})["points"].values())
+
+        points = [point for point in points if _fake_match_filter(point, qdrant_filter)]
+
+        for point in points:
+            metadata = point["payload"].get("metadata", {})
+            actual_field = field.split(".", 1)[-1] if field.startswith("metadata.") else field
+            value = metadata.get(actual_field)
+
+            if value is None:
+                continue
+            counts[value] = counts.get(value, 0) + 1
+
+        return counts
+
 
 @pytest.fixture
 def fake_qdrant(monkeypatch):
@@ -362,7 +392,7 @@ def test_qdrant_marker_participates_in_backend_mismatch(tmp_path, monkeypatch, f
     backend, col = _collection(tmp_path)
     col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
     backend.close()
-    (tmp_path / "chroma.sqlite3").write_bytes(b"")
+    make_minimal_chroma_sqlite(tmp_path)
     monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "chroma")
 
     with pytest.raises(BackendMismatchError):
@@ -527,3 +557,112 @@ def test_qdrant_live_rest_roundtrip_when_enabled(tmp_path):
         except Exception:
             pass
         backend.close()
+
+
+def test_qdrant_facet_counts(tmp_path, fake_qdrant):
+    _, collection = _collection(tmp_path)
+    collection.upsert(
+        ids=["1", "2", "3", "4"],
+        documents=["a", "b", "c", "d"],
+        metadatas=[
+            {"wing": "alpha"},
+            {"wing": "alpha"},
+            {"wing": "beta"},
+            {"wing": "gamma"},
+        ],
+        embeddings=[
+            [1, 0],
+            [1, 0],
+            [1, 0],
+            [1, 0],
+        ],
+    )
+    assert collection.facet_counts("wing") == {
+        "alpha": 2,
+        "beta": 1,
+        "gamma": 1,
+    }
+
+
+def test_qdrant_facet_counts_where(tmp_path, fake_qdrant):
+    _, collection = _collection(tmp_path)
+    collection.upsert(
+        ids=["1", "2", "3"],
+        documents=["a", "b", "c"],
+        metadatas=[
+            {"wing": "engineering", "room": "backend"},
+            {"wing": "engineering", "room": "frontend"},
+            {"wing": "design", "room": "ux"},
+        ],
+        embeddings=[
+            [1, 0],
+            [1, 0],
+            [1, 0],
+        ],
+    )
+    assert collection.facet_counts(
+        "room",
+        where={"wing": "engineering"},
+    ) == {
+        "backend": 1,
+        "frontend": 1,
+    }
+
+
+def test_qdrant_facet_counts_rejects_local_filters(tmp_path, fake_qdrant):
+    _, collection = _collection(tmp_path)
+    with pytest.raises(UnsupportedCapabilityError):
+        collection.facet_counts(
+            "room",
+            where={
+                "$or": [
+                    {"wing": "a"},
+                    {"wing": "b"},
+                ]
+            },
+        )
+
+
+def test_qdrant_facet_counts_passes_filter(tmp_path, fake_qdrant):
+    _, collection = _collection(tmp_path)
+    collection.upsert(
+        ids=["1"],
+        documents=["doc"],
+        metadatas=[{"wing": "engineering", "room": "backend"}],
+        embeddings=[[1, 0]],
+    )
+    collection.facet_counts(
+        "room",
+        where={"wing": "engineering"},
+    )
+    client = fake_qdrant.instances[0]
+    assert len(client.facet_calls) == 1
+    field, qfilter = client.facet_calls[0]
+    assert field == "metadata.room"
+    assert qfilter == {
+        "must": [
+            {
+                "key": "metadata.wing",
+                "match": {"value": "engineering"},
+            }
+        ]
+    }
+
+
+def test_qdrant_facet_counts_ignores_missing_metadata(tmp_path, fake_qdrant):
+    _, collection = _collection(tmp_path)
+    collection.upsert(
+        ids=["1", "2"],
+        documents=["a", "b"],
+        metadatas=[
+            {"wing": "alpha"},
+            {},
+        ],
+        embeddings=[
+            [1, 0],
+            [1, 0],
+        ],
+    )
+    assert collection.facet_counts("wing") == {
+        "alpha": 1,
+    }

tests/test_repair.py

@@ -7,6 +7,8 @@ from unittest.mock import MagicMock, call, patch
 
 import pytest
 
+from _chroma_palace_helper import make_minimal_chroma_sqlite
+
 from mempalace import repair
 
 
@@ -338,9 +340,9 @@ def test_index_read_recovery_guidance_recommends_from_sqlite():
     assert "may need to be re-mined" not in msg
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
-def test_rebuild_index_success(mock_backend_cls, mock_shutil, tmp_path):
+def test_rebuild_index_success(mock_backend_cls, mock_copy, tmp_path):
     # Create a valid sqlite file so the repair preflight can run quick_check.
     sqlite_path = tmp_path / "chroma.sqlite3"
     with sqlite3.connect(sqlite_path) as conn:
@@ -365,8 +367,8 @@ def test_rebuild_index_success(mock_backend_cls, mock_shutil, tmp_path):
     repair.rebuild_index(palace_path=str(tmp_path))
 
     # Verify: backed up sqlite only, not copytree.
-    mock_shutil.copy2.assert_called_once()
-    assert "chroma.sqlite3" in str(mock_shutil.copy2.call_args)
+    mock_copy.assert_called_once()
+    assert "chroma.sqlite3" in str(mock_copy.call_args)
 
     # Verify: deleted and recreated (cosine is the backend default)
     assert mock_backend.create_collection.call_args_list == [
@@ -385,19 +387,19 @@ def test_rebuild_index_success(mock_backend_cls, mock_shutil, tmp_path):
     mock_new_col.add.assert_not_called()
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
 def test_rebuild_index_ignores_missing_temp_collection_at_start(
-    mock_backend_cls, mock_shutil, tmp_path
+    mock_backend_cls, mock_copy, tmp_path
 ):
     sqlite_path = tmp_path / "chroma.sqlite3"
     sqlite3.connect(str(sqlite_path)).close()
 
-    def _fake_copy2(src, dst):
+    def _fake_copy2(src, dst, **_):
         with open(dst, "w") as handle:
             handle.write("backup")
 
-    mock_shutil.copy2.side_effect = _fake_copy2
+    mock_copy.side_effect = _fake_copy2
 
     mock_col = MagicMock()
     mock_col.count.return_value = 2
@@ -421,7 +423,7 @@ def test_rebuild_index_ignores_missing_temp_collection_at_start(
 
     repair.rebuild_index(palace_path=str(tmp_path))
 
-    assert mock_shutil.copy2.call_count == 1
+    assert mock_copy.call_count == 1
     assert mock_backend.delete_collection.call_args_list == [
         call(str(tmp_path), "mempalace_drawers__repair_tmp"),
         call(str(tmp_path), "mempalace_drawers"),
@@ -583,7 +585,7 @@ def test_status_returns_empty_when_db_present_no_drawers(tmp_path, capsys):
     'uninitialized' (#1498). Mocks sqlite_drawer_count to assert the
     return-shape contract; see the real-disk sibling below for the
     no-chromadb-client invariant."""
-    (tmp_path / "chroma.sqlite3").touch()
+    make_minimal_chroma_sqlite(tmp_path)
     with patch("mempalace.repair.sqlite_drawer_count", return_value=0):
         result = repair.status(palace_path=str(tmp_path))
 
@@ -622,7 +624,7 @@ def test_status_falls_through_to_capacity_when_sqlite_count_unreadable(tmp_path)
     """When sqlite_drawer_count returns None (schema drift / locked file),
     repair.status must fall through to hnsw_capacity_status instead of
     short-circuiting on 'empty' (#1498)."""
-    (tmp_path / "chroma.sqlite3").touch()
+    make_minimal_chroma_sqlite(tmp_path)
     with (
         patch("mempalace.repair.sqlite_drawer_count", return_value=None),
         patch("mempalace.repair.hnsw_capacity_status") as capacity_status,
@@ -658,7 +660,7 @@ def test_status_default_uses_configured_drawer_collection(tmp_path):
     # Provide the on-disk preconditions the stratified state helper (#1498)
     # checks before reaching the capacity probe: chroma.sqlite3 file exists
     # and sqlite_drawer_count returns a positive number (palace not empty).
-    (tmp_path / "chroma.sqlite3").touch()
+    make_minimal_chroma_sqlite(tmp_path)
     with (
         patch("mempalace.repair._drawers_collection_name", return_value="custom_drawers"),
         patch("mempalace.repair.sqlite_drawer_count", return_value=1),
@@ -688,9 +690,9 @@ def test_status_default_uses_configured_drawer_collection(tmp_path):
     assert capacity_status.call_args_list[1].args == (str(tmp_path), "mempalace_closets")
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
-def test_rebuild_index_aborts_on_truncation_signal(mock_backend_cls, mock_shutil, tmp_path):
+def test_rebuild_index_aborts_on_truncation_signal(mock_backend_cls, mock_copy, tmp_path):
     """rebuild_index honors the safety guard: SQLite says 67k, get() returns
     10k → no delete_collection, no upsert, no backup."""
     mock_backend = MagicMock()
@@ -714,7 +716,7 @@ def test_rebuild_index_aborts_on_truncation_signal(mock_backend_cls, mock_shutil
     # Guard fired: nothing destructive happened
     mock_backend.delete_collection.assert_not_called()
     mock_backend.create_collection.assert_not_called()
-    mock_shutil.copy2.assert_not_called()
+    mock_copy.assert_not_called()
 
 
 @patch("mempalace.repair.shutil")
@@ -749,10 +751,10 @@ def test_rebuild_index_proceeds_with_override(mock_backend_cls, mock_shutil, tmp
     mock_new_col.upsert.assert_called()
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
 def test_rebuild_index_stage_failure_leaves_live_collection_untouched(
-    mock_backend_cls, mock_shutil, tmp_path
+    mock_backend_cls, mock_copy, tmp_path
 ):
     sqlite_path = tmp_path / "chroma.sqlite3"
     sqlite3.connect(str(sqlite_path)).close()
@@ -773,24 +775,24 @@ def test_rebuild_index_stage_failure_leaves_live_collection_untouched(
         repair.rebuild_index(palace_path=str(tmp_path))
 
     assert excinfo.value.live_replaced is False
-    assert mock_shutil.copy2.call_count == 1
+    assert mock_copy.call_count == 1
     assert mock_backend.delete_collection.call_args_list == [
         call(str(tmp_path), "mempalace_drawers__repair_tmp"),
         call(str(tmp_path), "mempalace_drawers__repair_tmp"),
     ]
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
-def test_rebuild_index_live_failure_restores_backup(mock_backend_cls, mock_shutil, tmp_path):
+def test_rebuild_index_live_failure_restores_backup(mock_backend_cls, mock_copy, tmp_path):
     sqlite_path = tmp_path / "chroma.sqlite3"
     sqlite3.connect(str(sqlite_path)).close()
 
-    def _fake_copy2(src, dst):
+    def _fake_copy2(src, dst, **_):
         with open(dst, "w") as handle:
             handle.write("backup")
 
-    mock_shutil.copy2.side_effect = _fake_copy2
+    mock_copy.side_effect = _fake_copy2
 
     mock_col = MagicMock()
     mock_col.count.return_value = 2
@@ -813,7 +815,7 @@ def test_rebuild_index_live_failure_restores_backup(mock_backend_cls, mock_shuti
         repair.rebuild_index(palace_path=str(tmp_path))
 
     assert excinfo.value.live_replaced is True
-    assert mock_shutil.copy2.call_count == 2
+    assert mock_copy.call_count == 2
     assert active_backend.delete_collection.call_args_list == [
         call(str(tmp_path), "mempalace_drawers__repair_tmp"),
         call(str(tmp_path), "mempalace_drawers"),
@@ -824,19 +826,19 @@ def test_rebuild_index_live_failure_restores_backup(mock_backend_cls, mock_shuti
     helper_backend.close_palace.assert_not_called()
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
 def test_rebuild_index_live_delete_missing_still_restores_backup(
-    mock_backend_cls, mock_shutil, tmp_path
+    mock_backend_cls, mock_copy, tmp_path
 ):
     sqlite_path = tmp_path / "chroma.sqlite3"
     sqlite3.connect(str(sqlite_path)).close()
 
-    def _fake_copy2(src, dst):
+    def _fake_copy2(src, dst, **_):
         with open(dst, "w") as handle:
             handle.write("backup")
 
-    mock_shutil.copy2.side_effect = _fake_copy2
+    mock_copy.side_effect = _fake_copy2
 
     mock_col = MagicMock()
     mock_col.count.return_value = 2
@@ -860,7 +862,7 @@ def test_rebuild_index_live_delete_missing_still_restores_backup(
         repair.rebuild_index(palace_path=str(tmp_path))
 
     assert excinfo.value.live_replaced is True
-    assert mock_shutil.copy2.call_count == 2
+    assert mock_copy.call_count == 2
     assert mock_backend.delete_collection.call_args_list == [
         call(str(tmp_path), "mempalace_drawers__repair_tmp"),
         call(str(tmp_path), "mempalace_drawers"),
@@ -869,21 +871,22 @@ def test_rebuild_index_live_delete_missing_still_restores_backup(
     ]
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
 def test_rebuild_index_restore_failure_preserves_original_error(
-    mock_backend_cls, mock_shutil, tmp_path, capsys
+    mock_backend_cls, mock_copy, tmp_path, capsys
 ):
     sqlite_path = tmp_path / "chroma.sqlite3"
     sqlite3.connect(str(sqlite_path)).close()
 
-    def _copy2_side_effect(src, dst):
-        if str(src).endswith(".backup"):
+    def _copy_side_effect(src, dst, **_):
+        # The restore copy reads from the timestamped backup file.
+        if ".backup." in str(src):
             raise PermissionError("locked sqlite")
         with open(dst, "w") as handle:
             handle.write("backup")
 
-    mock_shutil.copy2.side_effect = _copy2_side_effect
+    mock_copy.side_effect = _copy_side_effect
 
     mock_col = MagicMock()
     mock_col.count.return_value = 2
@@ -944,19 +947,19 @@ def test_rebuild_collection_via_temp_keeps_original_error_when_cleanup_fails(
     ]
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
 def test_rebuild_index_ignores_temp_cleanup_failure_after_success(
-    mock_backend_cls, mock_shutil, tmp_path
+    mock_backend_cls, mock_copy, tmp_path
 ):
     sqlite_path = tmp_path / "chroma.sqlite3"
     sqlite3.connect(str(sqlite_path)).close()
 
-    def _fake_copy2(src, dst):
+    def _fake_copy2(src, dst, **_):
         with open(dst, "w") as handle:
             handle.write("backup")
 
-    mock_shutil.copy2.side_effect = _fake_copy2
+    mock_copy.side_effect = _fake_copy2
 
     mock_col = MagicMock()
     mock_col.count.return_value = 2
@@ -979,7 +982,7 @@ def test_rebuild_index_ignores_temp_cleanup_failure_after_success(
 
     repair.rebuild_index(palace_path=str(tmp_path))
 
-    assert mock_shutil.copy2.call_count == 1
+    assert mock_copy.call_count == 1
     assert mock_backend.delete_collection.call_args_list == [
         call(str(tmp_path), "mempalace_drawers__repair_tmp"),
         call(str(tmp_path), "mempalace_drawers"),
@@ -1355,6 +1358,57 @@ def test_sqlite_integrity_errors_returns_empty_for_healthy_db(tmp_path):
     assert repair.sqlite_integrity_errors(str(palace)) == []
 
 
+def test_sqlite_integrity_errors_uses_bounded_contention_timeout(tmp_path, monkeypatch):
+    """Integrity checks wait out routine writers without a real-time sleep.
+
+    Assert the sqlite connection contract directly so this regression test is
+    deterministic and does not add the seven-second delay from the original
+    proposal to every test run.
+    """
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    db_path = palace / "chroma.sqlite3"
+    db_path.touch()
+
+    calls = []
+
+    class _Result:
+        @staticmethod
+        def fetchall():
+            return [("ok",)]
+
+    class _Connection:
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc, tb):
+            return False
+
+        def execute(self, statement):
+            calls.append(("execute", statement))
+            return _Result()
+
+    def _connect(database, **kwargs):
+        calls.append(("connect", database, kwargs))
+        return _Connection()
+
+    monkeypatch.setattr(repair.sqlite3, "connect", _connect)
+
+    assert repair.sqlite_integrity_errors(str(palace)) == []
+    assert calls == [
+        (
+            "connect",
+            repair.sqlite_read_uri(str(db_path)),
+            {
+                "uri": True,
+                "timeout": repair._SQLITE_INTEGRITY_BUSY_TIMEOUT_SECONDS,
+            },
+        ),
+        ("execute", "PRAGMA quick_check"),
+    ]
+    assert repair._SQLITE_INTEGRITY_BUSY_TIMEOUT_SECONDS == 15.0
+
+
 def test_sqlite_integrity_errors_reports_unreadable_sqlite_file(tmp_path):
     palace = tmp_path / "palace"
     palace.mkdir()
@@ -1367,11 +1421,11 @@ def test_sqlite_integrity_errors_reports_unreadable_sqlite_file(tmp_path):
     assert "quick_check failed" in errors[0]
 
 
-@patch("mempalace.repair.shutil")
+@patch("mempalace.repair._copy_file_no_follow")
 @patch("mempalace.repair.ChromaBackend")
 def test_rebuild_index_aborts_on_sqlite_integrity_errors_before_delete_collection(
     mock_backend_cls,
-    mock_shutil,
+    mock_copy,
     tmp_path,
     capsys,
 ):
@@ -1410,7 +1464,7 @@ def test_rebuild_index_aborts_on_sqlite_integrity_errors_before_delete_collectio
 
     mock_backend.delete_collection.assert_not_called()
     mock_backend.create_collection.assert_not_called()
-    mock_shutil.copy2.assert_not_called()
+    mock_copy.assert_not_called()
 
 
 def test_rebuild_index_runs_sqlite_preflight_before_chromadb_open(tmp_path, capsys):
@@ -1749,6 +1803,60 @@ def test_rebuild_from_sqlite_roundtrips_via_real_chromadb(tmp_path):
     assert closet_row["metadatas"][0] == {"wing": "alpha"}
 
 
+def test_rebuild_from_sqlite_rebuilds_fts5_after_chroma_closes(tmp_path, monkeypatch):
+    """The SQLite recovery path must finish by rebuilding Chroma's FTS5 index.
+
+    Large bulk upserts can leave the derived full-text index malformed even
+    when every source drawer survived.  The repair is not complete until the
+    Chroma client releases its SQLite handle and FTS5 is rebuilt.
+    """
+    source = tmp_path / "source"
+    dest = tmp_path / "dest"
+    _seed_palace(source, "mempalace_drawers", [("d1", "doc", {"wing": "w"})])
+
+    calls = []
+    real_rebuild = repair._vacuum_and_rebuild_fts5
+
+    def _spy(path, progress=print, *, strict=False):
+        calls.append((path, strict))
+        return real_rebuild(path, progress=progress, strict=strict)
+
+    monkeypatch.setattr(repair, "_vacuum_and_rebuild_fts5", _spy)
+
+    counts = repair.rebuild_from_sqlite(str(source), str(dest))
+
+    assert counts["mempalace_drawers"] == 1
+    assert calls == [(str(dest), True)]
+
+
+def test_rebuild_from_sqlite_cleanup_failure_is_not_reported_as_success(
+    tmp_path, monkeypatch, capsys
+):
+    source = tmp_path / "source"
+    dest = tmp_path / "dest"
+    _seed_palace(source, "mempalace_drawers", [("d1", "verbatim", {"wing": "w"})])
+
+    def _fail_cleanup(path, progress=print, *, strict=False):
+        assert path == str(dest)
+        assert strict is True
+        raise RuntimeError("simulated FTS5 rebuild failure")
+
+    monkeypatch.setattr(repair, "_vacuum_and_rebuild_fts5", _fail_cleanup)
+
+    with pytest.raises(repair.RebuildCleanupError) as excinfo:
+        repair.rebuild_from_sqlite(str(source), str(dest))
+
+    exc = excinfo.value
+    assert exc.counts["mempalace_drawers"] == 1
+    assert exc.dest_palace == str(dest)
+    assert exc.archive_path is None
+    assert dest.exists()
+    assert (source / "chroma.sqlite3").exists()
+    output = capsys.readouterr().out
+    assert "Rebuild complete" not in output
+    assert "Post-recovery cleanup failed" in output
+
+
 def test_rebuild_from_sqlite_refuses_existing_dest(tmp_path):
     """Refuse to write into a directory that already exists when source
     and dest differ. Without this, an unattended re-run would silently
@@ -1814,6 +1922,45 @@ def test_rebuild_from_sqlite_in_place_refuses_without_archive_flag(tmp_path):
     assert archives == []
 
 
+def test_rebuild_from_sqlite_in_place_archive_failure_leaves_palace_untouched(
+    tmp_path, monkeypatch
+):
+    """A file inside the palace held open by another process (MCP server,
+    a running mine, another harness) must abort the archive step cleanly,
+    leaving the live palace fully intact.
+
+    Regression test for a real-world incident (2026-07-05/06, Windows 11):
+    the archive step used ``shutil.move``, whose fallback for a failed
+    ``os.rename`` is copytree + rmtree. That rmtree deletes the live
+    palace file-by-file until it hits the first locked file, so an
+    in-progress mine or a live MCP server holding one file open left the
+    palace partially gutted next to a partial archive copy -- twice, on
+    two separate nights. os.rename fails atomically up front with nothing
+    touched; this test locks that behaviour in so a future change back to
+    shutil.move (or an equivalent copy+delete fallback) fails loudly.
+    """
+    palace = tmp_path / "palace"
+    rows = [("d1", "doc one", {"wing": "w"}), ("d2", "doc two", {"wing": "w"})]
+    _seed_palace(palace, "mempalace_drawers", rows)
+    sqlite_before = (palace / "chroma.sqlite3").stat().st_size
+    entries_before = sorted(p.name for p in palace.iterdir())
+
+    def _raise(*_args, **_kwargs):
+        raise PermissionError("[WinError 32] simulated: file held open by another process")
+
+    monkeypatch.setattr(repair.os, "rename", _raise)
+
+    counts = repair.rebuild_from_sqlite(str(palace), str(palace), archive_existing_dest=True)
+
+    assert counts == {}
+    # Palace directory contents and the sqlite file itself are byte-for-byte
+    # untouched -- no partial delete, no partial archive left behind.
+    assert sorted(p.name for p in palace.iterdir()) == entries_before
+    assert (palace / "chroma.sqlite3").stat().st_size == sqlite_before
+    archives = [p for p in tmp_path.iterdir() if "pre-rebuild" in p.name]
+    assert archives == []
+
+
 def test_rebuild_from_sqlite_source_missing_chroma_db(tmp_path):
     """Source dir exists but has no chroma.sqlite3 → returns empty,
     leaves dest untouched."""
@@ -1975,6 +2122,147 @@ def test_vacuum_and_rebuild_fts5_missing_sqlite(tmp_path):
     repair._vacuum_and_rebuild_fts5(str(tmp_path))  # no file — must not raise
 
 
+def test_vacuum_and_rebuild_fts5_strict_requires_sqlite(tmp_path):
+    with pytest.raises(FileNotFoundError, match="has no SQLite database"):
+        repair._vacuum_and_rebuild_fts5(str(tmp_path), strict=True)
+
+
+def test_vacuum_and_rebuild_fts5_strict_preserves_exception_type(tmp_path, monkeypatch):
+    sqlite_path = tmp_path / "chroma.sqlite3"
+    sqlite_path.touch()
+    messages = []
+
+    def _raise_database_error(*args, **kwargs):
+        raise sqlite3.DatabaseError("simulated cleanup failure")
+
+    monkeypatch.setattr(repair.sqlite3, "connect", _raise_database_error)
+
+    with pytest.raises(sqlite3.DatabaseError, match="simulated cleanup failure"):
+        repair._vacuum_and_rebuild_fts5(str(tmp_path), progress=messages.append, strict=True)
+
+    assert messages == []
+
+
+# ── FTS5 inverted-index auto-heal (#1596) ─────────────────────────────
+
+
+def _make_fts5_palace(tmp_path, *, corrupt: bool) -> str:
+    """Build a palace whose embedding_fulltext_search index is optionally
+    corrupted to the malformed-inverted-index quick_check state #1596 hits."""
+    sqlite_path = tmp_path / "chroma.sqlite3"
+    with closing(sqlite3.connect(str(sqlite_path))) as conn:
+        conn.execute(
+            "CREATE VIRTUAL TABLE embedding_fulltext_search"
+            " USING fts5(string_value, tokenize='unicode61')"
+        )
+        for i in range(200):
+            conn.execute(
+                "INSERT INTO embedding_fulltext_search(string_value) VALUES(?)",
+                (f"alpha beta gamma row{i} delta epsilon",),
+            )
+        conn.commit()
+        if corrupt:
+            # Zero the last index segment leaf: quick_check then reports
+            # "malformed inverted index" while the content table stays intact.
+            conn.execute(
+                "UPDATE embedding_fulltext_search_data SET block=zeroblob(length(block)) "
+                "WHERE id=(SELECT max(id) FROM embedding_fulltext_search_data)"
+            )
+            conn.commit()
+    return str(tmp_path)
+
+
+def test_errors_are_isolated_fts5_classification():
+    fts = "malformed inverted index for FTS5 table main.embedding_fulltext_search"
+    # SQLite >= ~3.5x (confirmed on 3.53.2 / Python 3.13.7) reports isolated
+    # FTS5 corruption with this wording instead of the older phrasing above.
+    # A regex matching only the old phrasing silently declines to auto-heal
+    # on any machine running a newer SQLite -- caught by this repo's own
+    # test_maybe_autoheal_fts5_index_heals_isolated_corruption failing on
+    # this exact build before _FTS5_MALFORMED_RE was widened to cover both.
+    fts_new = (
+        'fts5: corruption found reading blob 137438953474 from table "embedding_fulltext_search"'
+    )
+    page = "Page 4 of B-tree 12345: database disk image is malformed"
+    assert repair._errors_are_isolated_fts5([fts])
+    assert repair._errors_are_isolated_fts5([fts, fts])
+    assert repair._errors_are_isolated_fts5([fts_new])
+    assert repair._errors_are_isolated_fts5([fts, fts_new])
+    assert not repair._errors_are_isolated_fts5([])
+    assert not repair._errors_are_isolated_fts5([page])
+    # Any non-FTS5 error in the set means the data itself may be damaged.
+    assert not repair._errors_are_isolated_fts5([fts, page])
+    assert not repair._errors_are_isolated_fts5([fts_new, page])
+
+
+def test_maybe_autoheal_fts5_index_heals_isolated_corruption(tmp_path):
+    palace = _make_fts5_palace(tmp_path, corrupt=True)
+    errors = repair.sqlite_integrity_errors(palace)
+    assert errors and repair._errors_are_isolated_fts5(errors)
+
+    remaining = repair.maybe_autoheal_fts5_index(palace, errors, progress=lambda *_: None)
+
+    assert remaining == []
+    # quick_check is clean and full-text search works again.
+    assert repair.sqlite_integrity_errors(palace) == []
+    with closing(sqlite3.connect(str(tmp_path / "chroma.sqlite3"))) as conn:
+        hits = conn.execute(
+            "SELECT count(*) FROM embedding_fulltext_search "
+            "WHERE embedding_fulltext_search MATCH 'gamma'"
+        ).fetchone()[0]
+    assert hits == 200
+
+
+def test_maybe_autoheal_fts5_index_leaves_non_fts5_errors_untouched(tmp_path):
+    palace = _make_fts5_palace(tmp_path, corrupt=False)
+    page_errors = ["Page 4 of B-tree 12345: database disk image is malformed"]
+
+    # Not isolated FTS5: returned unchanged and the rebuild is never attempted.
+    with patch("mempalace.palace.mine_palace_lock") as lock:
+        remaining = repair.maybe_autoheal_fts5_index(palace, page_errors, progress=lambda *_: None)
+    assert remaining == page_errors
+    lock.assert_not_called()
+
+
+def test_maybe_autoheal_fts5_index_skips_when_palace_is_being_mined(tmp_path):
+    from mempalace.palace import MineAlreadyRunning
+
+    palace = _make_fts5_palace(tmp_path, corrupt=True)
+    errors = repair.sqlite_integrity_errors(palace)
+
+    def _raise(_path):
+        raise MineAlreadyRunning("held by pid 999")
+
+    # A live mine holds the lock: do not race the rebuild — surface and abort.
+    with patch("mempalace.palace.mine_palace_lock", side_effect=_raise):
+        remaining = repair.maybe_autoheal_fts5_index(palace, errors, progress=lambda *_: None)
+
+    assert remaining == errors
+    # The FTS index is still corrupt because we refused to rebuild under contention.
+    assert repair.sqlite_integrity_errors(palace) == errors
+
+
+def test_rebuild_index_preflight_autoheals_isolated_fts5_then_proceeds(tmp_path, monkeypatch):
+    """The preflight no longer hard-aborts on isolated FTS5 corruption (#1596):
+    it rebuilds the index, then continues into the rebuild path."""
+    palace = _make_fts5_palace(tmp_path, corrupt=True)
+
+    called = {}
+
+    def _fake_max_seq(_palace_path, **_kwargs):
+        # Reached only if the preflight did NOT abort — record and stop early
+        # so the test doesn't need a real chromadb collection.
+        called["reached"] = True
+        return {"stopped": True}
+
+    monkeypatch.setattr(repair, "maybe_repair_poisoned_max_seq_id_before_rebuild", _fake_max_seq)
+
+    repair.rebuild_index(palace_path=palace, progress=lambda *_: None)
+
+    assert called.get("reached") is True
+    assert repair.sqlite_integrity_errors(palace) == []
+
+
 @patch("mempalace.repair.shutil")
 @patch("mempalace.repair.ChromaBackend")
 def test_rebuild_index_calls_vacuum(mock_backend_cls, mock_shutil, tmp_path):

tests/test_searcher.py

@@ -9,7 +9,16 @@ from unittest.mock import MagicMock, patch
 
 import pytest
 
-from mempalace.searcher import SearchError, build_where_filter, search, search_memories
+from _chroma_palace_helper import make_minimal_chroma_sqlite
+
+from mempalace.backends import BackendMismatchError
+from mempalace.searcher import (
+    SearchError,
+    build_where_filter,
+    get_collection,
+    search,
+    search_memories,
+)
 
 
 # ── build_where_filter (unit) ──────────────────────────────────────────
@@ -349,7 +358,7 @@ def fake_palace_path(tmp_path):
     backend instead of raising on State A / State B."""
     p = tmp_path / "palace"
     p.mkdir()
-    (p / "chroma.sqlite3").touch()
+    make_minimal_chroma_sqlite(p)
     return str(p)
 
 
@@ -517,3 +526,119 @@ class TestSearchCLI:
         captured = capsys.readouterr()
         assert "[1]" in captured.out
         assert "[2]" in captured.out
+
+    def test_search_routes_to_bm25_when_hnsw_diverged(self, fake_palace_path, capsys):
+        """Regression: `mempalace search` on a diverged HNSW segment must not
+        segfault ChromaDB's Rust bindings.
+
+        The MCP path gates this via ``_vector_disabled`` (#1222); the CLI
+        path was missing the gate, so any query into a diverged palace
+        exited 139 (SIGBUS) at ``chromadb/api/rust.py:_query`` with zero
+        diagnostic output. This test verifies the CLI now probes
+        ``hnsw_capacity_status`` and routes to the BM25-only sqlite
+        fallback instead of calling ``col.query()`` against a segment
+        that would crash it.
+        """
+        bm25_result = {
+            "query": "anything",
+            "filters": {},
+            "total_before_filter": 1,
+            "results": [
+                {
+                    "text": "diary entry that matches the query",
+                    "wing": "wing_test",
+                    "room": "diary",
+                    "source_file": "test.jsonl",
+                    "bm25_score": 1.5,
+                    "distance": None,
+                }
+            ],
+            "fallback": "bm25_only_via_sqlite",
+            "fallback_reason": "vector_search_disabled",
+        }
+        with (
+            patch("mempalace.searcher.resolve_backend_name", return_value="chroma"),
+            patch(
+                "mempalace.backends.chroma.hnsw_capacity_status",
+                return_value={"diverged": True, "message": "test divergence"},
+            ),
+            patch("mempalace.searcher._bm25_only_via_sqlite", return_value=bm25_result),
+            patch("mempalace.searcher.get_collection") as mock_get_collection,
+        ):
+            search("anything", fake_palace_path)
+        captured = capsys.readouterr()
+        # Routed to BM25 before opening Chroma at all. Client construction and
+        # identity enforcement can touch the same damaged native index, so a
+        # query-only guard is insufficient.
+        mock_get_collection.assert_not_called()
+        # User got actionable output, not a silent crash.
+        assert "mempalace repair" in captured.out
+        assert "diary entry that matches" in captured.out
+
+    def test_search_proceeds_to_vector_when_hnsw_healthy(self, fake_palace_path, capsys):
+        """Paired guard: when HNSW is healthy, the divergence probe must NOT
+        short-circuit to BM25 — vector search proceeds normally.
+
+        Prevents a regression where the gate accidentally always fires.
+        """
+        mock_col = MagicMock()
+        mock_col.metadata = {"hnsw:space": "cosine"}
+        mock_col.query.return_value = {
+            "documents": [["a matching doc"]],
+            "metadatas": [[{"source_file": "a.md", "wing": "w", "room": "r"}]],
+            "distances": [[0.1]],
+        }
+        with (
+            patch("mempalace.searcher.resolve_backend_name", return_value="chroma"),
+            patch(
+                "mempalace.backends.chroma.hnsw_capacity_status",
+                return_value={"diverged": False, "status": "ok"},
+            ),
+            patch("mempalace.searcher._bm25_only_via_sqlite") as mock_bm25,
+            patch("mempalace.searcher.get_collection", return_value=mock_col),
+        ):
+            search("anything", fake_palace_path)
+        captured = capsys.readouterr()
+        # Vector path ran.
+        mock_col.query.assert_called_once()
+        # BM25 fallback was NOT invoked.
+        mock_bm25.assert_not_called()
+        assert "a matching doc" in captured.out
+
+    def test_search_does_not_run_chroma_probe_for_other_backends(self, fake_palace_path, capsys):
+        """The HNSW guard is Chroma-specific and must not fence other backends."""
+        mock_col = MagicMock()
+        mock_col.query.return_value = {
+            "documents": [["backend-native result"]],
+            "metadatas": [[{"source_file": "native.md", "wing": "w", "room": "r"}]],
+            "distances": [[0.1]],
+        }
+        with (
+            patch("mempalace.searcher.resolve_backend_name", return_value="sqlite_exact"),
+            patch("mempalace.backends.chroma.hnsw_capacity_status") as mock_probe,
+            patch("mempalace.searcher.get_collection", return_value=mock_col),
+        ):
+            search("anything", fake_palace_path)
+
+        mock_probe.assert_not_called()
+        mock_col.query.assert_called_once()
+        assert "backend-native result" in capsys.readouterr().out
+
+    @pytest.mark.parametrize(
+        "resolution_error",
+        [BackendMismatchError("mixed backend artifacts"), KeyError("unknown_backend")],
+    )
+    def test_search_delegates_backend_resolution_errors_to_open_diagnostic(
+        self, fake_palace_path, resolution_error
+    ):
+        """The early HNSW fence must not replace normal CLI diagnostics."""
+        with (
+            patch("mempalace.searcher.resolve_backend_name", side_effect=resolution_error),
+            patch("mempalace.searcher._hnsw_capacity_diverged") as mock_probe,
+            patch("mempalace.searcher._open_collection_or_explain", return_value=None) as mock_open,
+        ):
+            with pytest.raises(SearchError):
+                search("anything", fake_palace_path)
+
+        mock_probe.assert_not_called()
+        mock_open.assert_called_once_with(fake_palace_path, opener=get_collection)

tests/test_serve.py

@@ -0,0 +1,160 @@
+"""Tests for the turnkey `mempalace serve` command (#1877).
+
+These exercise the wrapper's security-relevant behavior — token autogeneration
+and 0600 persistence, the secure-by-default non-loopback gate, and that the
+bearer token is passed via the environment (never argv, so it can't leak via
+``ps``) — without binding a real socket. ``cmd_serve`` ends by exec'ing the real
+server; we intercept ``os.execve`` to capture the child invocation instead.
+"""
+
+import argparse
+import os
+import stat
+
+import pytest
+
+from mempalace import cli
+
+
+class _ExecCalled(Exception):
+    """Raised by the patched os.execve to stop cmd_serve at the exec boundary."""
+
+
+@pytest.fixture
+def isolated_home(tmp_path, monkeypatch):
+    """Point ~ at a temp dir so server token state never touches the real home."""
+    monkeypatch.setenv("HOME", str(tmp_path))
+    monkeypatch.setenv("USERPROFILE", str(tmp_path))  # Windows
+    # Don't inherit a token from the ambient environment.
+    monkeypatch.delenv("MEMPALACE_MCP_HTTP_TOKEN", raising=False)
+    return tmp_path
+
+
+@pytest.fixture
+def capture_exec(monkeypatch):
+    """Capture the child argv/env cmd_serve would launch, instead of running it.
+
+    cmd_serve takes the os.execve branch on POSIX and the subprocess.run branch
+    on Windows. Patch both (rather than forcing os.name, which breaks
+    Path.home() on Windows) so the test is platform-agnostic.
+    """
+    import subprocess
+
+    captured = {}
+
+    def _capture(argv, env):
+        captured["argv"] = argv
+        captured["env"] = env
+        raise _ExecCalled()
+
+    monkeypatch.setattr(cli.os, "execve", lambda path, argv, env: _capture(argv, env))
+    monkeypatch.setattr(subprocess, "run", lambda argv, env=None, **kw: _capture(argv, env))
+    return captured
+
+
+def _serve_args(tmp_path, **over):
+    base = dict(
+        host="127.0.0.1",
+        port=8765,
+        backend=None,
+        global_backend=None,
+        palace=str(tmp_path / "palace"),
+        token=None,
+        tls_cert=None,
+        tls_key=None,
+        read_only=False,
+        allow_insecure=False,
+    )
+    base.update(over)
+    return argparse.Namespace(**base)
+
+
+def test_token_helper_creates_0600_and_reuses(isolated_home):
+    palace = str(isolated_home / "palace")
+    token1, created1 = cli._load_or_create_server_token(palace)
+    assert created1 is True
+    assert token1
+
+    path = cli._server_token_path(palace)
+    assert path.exists()
+    if os.name == "posix":
+        # POSIX permission bits aren't meaningful on Windows (files report 0o666).
+        mode = stat.S_IMODE(path.stat().st_mode)
+        assert mode == 0o600, oct(mode)
+        dir_mode = stat.S_IMODE(path.parent.stat().st_mode)
+        assert dir_mode == 0o700, oct(dir_mode)
+
+    token2, created2 = cli._load_or_create_server_token(palace)
+    assert created2 is False
+    assert token2 == token1  # stable across restarts
+
+
+def test_loopback_serve_needs_no_token(isolated_home, capture_exec):
+    with pytest.raises(_ExecCalled):
+        cli.cmd_serve(_serve_args(isolated_home, host="127.0.0.1"))
+    env = capture_exec["env"]
+    assert "MEMPALACE_MCP_HTTP_TOKEN" not in env
+    assert "MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN" not in env
+    # No token persisted for a loopback bind.
+    assert not cli._server_token_path(str(isolated_home / "palace")).exists()
+
+
+def test_non_loopback_autogenerates_token_in_env_not_argv(isolated_home, capture_exec):
+    with pytest.raises(_ExecCalled):
+        cli.cmd_serve(_serve_args(isolated_home, host="0.0.0.0"))
+    env = capture_exec["env"]
+    argv = capture_exec["argv"]
+    token = env.get("MEMPALACE_MCP_HTTP_TOKEN")
+    assert token, "a token must be generated for a network-exposed bind"
+    # Security: the token rides in the env, never on the command line.
+    assert all(token not in part for part in argv)
+    assert "--token" not in argv
+    # And it was persisted for reuse on the next start (0600 on POSIX).
+    path = cli._server_token_path(str(isolated_home / "palace"))
+    assert path.exists()
+    if os.name == "posix":
+        assert stat.S_IMODE(path.stat().st_mode) == 0o600
+
+
+def test_allow_insecure_skips_token_and_sets_escape_hatch(isolated_home, capture_exec):
+    with pytest.raises(_ExecCalled):
+        cli.cmd_serve(_serve_args(isolated_home, host="0.0.0.0", allow_insecure=True))
+    env = capture_exec["env"]
+    assert env.get("MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN") == "1"
+    assert "MEMPALACE_MCP_HTTP_TOKEN" not in env
+    assert not cli._server_token_path(str(isolated_home / "palace")).exists()
+
+
+def test_read_only_flag_forwarded_to_child(isolated_home, capture_exec):
+    with pytest.raises(_ExecCalled):
+        cli.cmd_serve(_serve_args(isolated_home, read_only=True))
+    assert "--read-only" in capture_exec["argv"]
+
+
+def test_explicit_token_is_used_and_not_in_argv(isolated_home, capture_exec):
+    with pytest.raises(_ExecCalled):
+        cli.cmd_serve(_serve_args(isolated_home, host="0.0.0.0", token="my-secret-token"))
+    env = capture_exec["env"]
+    argv = capture_exec["argv"]
+    assert env["MEMPALACE_MCP_HTTP_TOKEN"] == "my-secret-token"
+    assert all("my-secret-token" not in part for part in argv)
+    # An explicitly-provided token is not persisted to the server token file.
+    assert not cli._server_token_path(str(isolated_home / "palace")).exists()
+
+
+def test_tls_paths_forwarded_and_validated(isolated_home, capture_exec, tmp_path):
+    cert = tmp_path / "cert.pem"
+    key = tmp_path / "key.pem"
+    cert.write_text("x")
+    key.write_text("x")
+    with pytest.raises(_ExecCalled):
+        cli.cmd_serve(_serve_args(isolated_home, tls_cert=str(cert), tls_key=str(key)))
+    argv = capture_exec["argv"]
+    assert "--tls-cert" in argv and "--tls-key" in argv
+
+
+def test_tls_requires_both_cert_and_key(isolated_home, capture_exec, tmp_path):
+    cert = tmp_path / "cert.pem"
+    cert.write_text("x")
+    with pytest.raises(SystemExit):
+        cli.cmd_serve(_serve_args(isolated_home, tls_cert=str(cert), tls_key=None))

tests/test_sqlite_exact_backend.py

@@ -4,6 +4,8 @@ import threading
 
 import pytest
 
+from _chroma_palace_helper import make_minimal_chroma_sqlite, make_minimal_sqlite_exact_sqlite
+
 import mempalace.backends.sqlite_exact as sqlite_exact_module
 from mempalace.backends import (
     BackendMismatchError,
@@ -410,7 +412,7 @@ def test_palace_wrapper_embeds_for_sqlite_exact(tmp_path, monkeypatch):
 def test_backend_mismatch_protection(tmp_path, monkeypatch):
     from mempalace.palace import get_collection
 
-    (tmp_path / "chroma.sqlite3").write_bytes(b"")
+    make_minimal_chroma_sqlite(tmp_path)
     monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact")
 
     with pytest.raises(BackendMismatchError):
@@ -420,14 +422,46 @@ def test_backend_mismatch_protection(tmp_path, monkeypatch):
 def test_mixed_backend_artifacts_are_rejected_even_when_chroma_selected(tmp_path, monkeypatch):
     from mempalace.palace import resolve_backend_name
 
-    (tmp_path / "chroma.sqlite3").write_bytes(b"")
-    (tmp_path / "sqlite_exact.sqlite3").write_bytes(b"")
+    make_minimal_chroma_sqlite(tmp_path)
+    make_minimal_sqlite_exact_sqlite(tmp_path)
     monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "chroma")
 
     with pytest.raises(BackendMismatchError):
         resolve_backend_name(str(tmp_path))
 
 
+def test_sqlite_exact_detect_matches_palace_with_sqlite_header(tmp_path):
+    """A real SQLite database at ``<path>/sqlite_exact.sqlite3`` registers
+    as sqlite_exact. Mirrors the chroma analog at
+    ``test_chroma_detect_matches_palace_with_sqlite_header``.
+    """
+    make_minimal_sqlite_exact_sqlite(tmp_path)
+    assert SQLiteExactBackend.detect(str(tmp_path)) is True
+    assert SQLiteExactBackend.detect(str(tmp_path.parent)) is False
+
+
+def test_sqlite_exact_detect_rejects_empty_sqlite_exact_sqlite(tmp_path):
+    """A 0-byte ``sqlite_exact.sqlite3`` is not a sqlite_exact palace (#1893).
+
+    Same root cause as the chroma side: bare ``sqlite3.connect()`` against
+    a missing path leaves a 0-byte file behind because the SQLite header is
+    written on the first statement, not on connect. Detection must reject
+    that artifact so it cannot trip ``BackendMismatchError`` against a real
+    non-sqlite_exact backend marker in the same directory.
+    """
+    (tmp_path / "sqlite_exact.sqlite3").write_bytes(b"")
+    assert SQLiteExactBackend.detect(str(tmp_path)) is False
+
+
+def test_sqlite_exact_detect_rejects_non_sqlite_file(tmp_path):
+    """A non-SQLite file at the ``sqlite_exact.sqlite3`` path is not
+    sqlite_exact. Defends against partial writes / garbage content / anything
+    that lands at the canonical path but isn't actually a SQLite database.
+    """
+    (tmp_path / "sqlite_exact.sqlite3").write_bytes(b"not a sqlite file" * 4)
+    assert SQLiteExactBackend.detect(str(tmp_path)) is False
+
+
 def test_sqlite_exact_exact_ranking_uses_cosine(tmp_path):
     _backend, col = _collection(tmp_path)
     halfway = [0.5, math.sqrt(0.75)]

tests/test_sync.py

@@ -11,6 +11,8 @@ from pathlib import Path
 import chromadb
 import pytest
 
+from _chroma_palace_helper import make_minimal_chroma_sqlite
+
 
 def _seed_drawers(palace_path, repo_path, deleted_path, elsewhere_path):
     """Populate the drawers collection with 6 entries covering all buckets."""
@@ -1446,7 +1448,7 @@ class TestServiceRunSyncReport:
         os.makedirs(palace)
         # Satisfy run_sync's detect_backend_for_path guard without spinning up
         # the real Chroma/embedder stack (which would disturb sys.stdout).
-        Path(palace, "chroma.sqlite3").touch()
+        make_minimal_chroma_sqlite(palace)
         monkeypatch.setattr(
             sync_module,
             "sync_palace",
@@ -1472,7 +1474,7 @@ class TestServiceRunSyncReport:
 
         palace = os.path.join(tmp_dir, "palace")
         os.makedirs(palace)
-        Path(palace, "chroma.sqlite3").touch()
+        make_minimal_chroma_sqlite(palace)
         monkeypatch.setattr(
             sync_module,
             "sync_palace",

tests/test_wal.py

@@ -40,3 +40,131 @@ def test_wal_log_redacts_and_writes(tmp_path, monkeypatch):
     assert entry["operation"] == "op"
     assert entry["params"]["entry"].startswith("[REDACTED")
     assert entry["params"]["safe"] == "ok"
+
+
+def test_wal_ensure_is_idempotent_and_cached(tmp_path, monkeypatch):
+    """_ensure_wal hardens the dir once, then short-circuits on the cached path.
+
+    Covers the cache-hit early return (wal.py:58): once _WAL_INITIALIZED_DIR
+    matches the WAL dir, a second call must not touch the filesystem again. This
+    is what stops a persistent chmod failure on a restricted FS from being
+    retried on every single write.
+    """
+    from pathlib import Path
+
+    from mempalace import wal
+
+    wal_dir = tmp_path / "wal"
+    wal_dir.mkdir()
+    monkeypatch.setattr(wal, "_WAL_FILE", wal_dir / "write_log.jsonl")
+    monkeypatch.setattr(wal, "_WAL_INITIALIZED_DIR", None)
+
+    wal._ensure_wal()
+    assert wal._WAL_INITIALIZED_DIR == wal_dir
+
+    # After caching, a second call must return before reaching any chmod/mkdir.
+    def _boom(self, *args, **kwargs):
+        raise AssertionError("filesystem touched again after dir was cached")
+
+    monkeypatch.setattr(Path, "chmod", _boom)
+    monkeypatch.setattr(Path, "mkdir", _boom)
+    wal._ensure_wal()  # must hit the cached early-return, not raise
+
+
+def test_wal_log_never_raises_when_write_fails(tmp_path, monkeypatch, caplog):
+    """A WAL write failure is logged and swallowed, never crashing the caller.
+
+    Covers wal.py:96-97 — the module docstring and _wal_log both promise that
+    any WAL failure is non-fatal, so a tool call is never broken by audit-log
+    I/O (e.g. a full disk or a read-only filesystem).
+    """
+    import logging
+
+    from mempalace import wal
+
+    monkeypatch.setattr(wal, "_WAL_FILE", tmp_path / "wal" / "write_log.jsonl")
+    monkeypatch.setattr(wal, "_WAL_INITIALIZED_DIR", None)
+
+    def _boom(*args, **kwargs):
+        raise OSError("no space left on device")
+
+    monkeypatch.setattr(wal.os, "open", _boom)
+
+    with caplog.at_level(logging.ERROR, logger="mempalace.wal"):
+        wal._wal_log("add_drawer", {"safe": "ok"})  # must not raise
+
+    assert any("WAL write failed" in r.getMessage() for r in caplog.records)
+
+
+def test_wal_ensure_swallows_chmod_failure_on_existing_dir(tmp_path, monkeypatch):
+    """A denied chmod on an existing dir is swallowed and the dir still caches.
+
+    Covers wal.py:67-68 — the WAL dir already exists (so no FileNotFoundError),
+    but chmod is denied (restricted FS). _ensure_wal must not raise and must
+    cache the dir so the failing chmod is not retried on every write.
+    """
+    from pathlib import Path
+
+    from mempalace import wal
+
+    wal_dir = tmp_path / "wal"
+    wal_dir.mkdir()
+    monkeypatch.setattr(wal, "_WAL_FILE", wal_dir / "write_log.jsonl")
+    monkeypatch.setattr(wal, "_WAL_INITIALIZED_DIR", None)
+
+    def _denied(self, *args, **kwargs):
+        raise OSError("operation not permitted")
+
+    monkeypatch.setattr(Path, "chmod", _denied)
+
+    wal._ensure_wal()  # must not raise
+    assert wal._WAL_INITIALIZED_DIR == wal_dir
+
+
+def test_wal_ensure_swallows_mkdir_failure(tmp_path, monkeypatch):
+    """A failed fallback mkdir is swallowed and the dir still caches.
+
+    Covers wal.py:65-66 — chmod raises FileNotFoundError (dir absent), the
+    fallback mkdir then fails too (read-only parent). _ensure_wal must not raise.
+    """
+    from pathlib import Path
+
+    from mempalace import wal
+
+    wal_dir = tmp_path / "missing" / "wal"
+    monkeypatch.setattr(wal, "_WAL_FILE", wal_dir / "write_log.jsonl")
+    monkeypatch.setattr(wal, "_WAL_INITIALIZED_DIR", None)
+
+    def _not_found(self, *args, **kwargs):
+        raise FileNotFoundError
+
+    def _mkdir_denied(self, *args, **kwargs):
+        raise OSError("read-only file system")
+
+    monkeypatch.setattr(Path, "chmod", _not_found)
+    monkeypatch.setattr(Path, "mkdir", _mkdir_denied)
+
+    wal._ensure_wal()  # must not raise
+    assert wal._WAL_INITIALIZED_DIR == wal_dir
+
+
+def test_wal_log_redacts_non_string_values(tmp_path, monkeypatch):
+    """Non-string values under a redact key use the plain [REDACTED] marker.
+
+    Covers the else-branch of the redaction ternary (wal.py:80): only str values
+    get the "[REDACTED N chars]" form; any other type is fully redacted without
+    calling len() on it.
+    """
+    import json
+
+    from mempalace import wal
+
+    wal_file = tmp_path / "wal" / "write_log.jsonl"
+    monkeypatch.setattr(wal, "_WAL_FILE", wal_file)
+    monkeypatch.setattr(wal, "_WAL_INITIALIZED_DIR", None)
+
+    wal._wal_log("kg_add", {"document": [1, 2, 3], "safe": "ok"})
+
+    entry = json.loads(wal_file.read_text().strip())
+    assert entry["params"]["document"] == "[REDACTED]"
+    assert entry["params"]["safe"] == "ok"

tests/test_write_routing.py

@@ -0,0 +1,423 @@
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from mempalace.config import MempalaceConfig
+from mempalace.write_routing import (
+    RoutingPolicyCandidate,
+    WriteRoutingError,
+    WriteRoutingPolicy,
+    WriteRoutingTarget,
+    choose_write_route,
+    parse_write_routing_policy,
+    resolve_write_routing_policy,
+)
+
+
+_ROUTING_ENV_KEYS = (
+    "MEMPALACE_WRITE_ROUTING",
+    "MEMPALACE_HOOK_WRITE_ROUTING",
+    "MEMPALACE_CLI_WRITE_ROUTING",
+    "MEMPALACE_HOOKS_DAEMON",
+)
+
+
+@pytest.fixture(autouse=True)
+def _clear_routing_env(monkeypatch):
+    for key in _ROUTING_ENV_KEYS:
+        monkeypatch.delenv(key, raising=False)
+
+
+def _write_config(tmp_path, payload):
+    (tmp_path / "config.json").write_text(
+        json.dumps(payload),
+        encoding="utf-8",
+    )
+
+
+@pytest.mark.parametrize(
+    "value",
+    ["direct", " DIRECT ", WriteRoutingPolicy.DIRECT],
+)
+def test_parse_direct(value):
+    assert parse_write_routing_policy(value) is WriteRoutingPolicy.DIRECT
+
+
+@pytest.mark.parametrize(
+    "value",
+    ["prefer", " PREFER ", WriteRoutingPolicy.PREFER],
+)
+def test_parse_prefer(value):
+    assert parse_write_routing_policy(value) is WriteRoutingPolicy.PREFER
+
+
+@pytest.mark.parametrize(
+    "value",
+    ["require", " REQUIRE ", WriteRoutingPolicy.REQUIRE],
+)
+def test_parse_require(value):
+    assert parse_write_routing_policy(value) is WriteRoutingPolicy.REQUIRE
+
+
+@pytest.mark.parametrize(
+    "value",
+    [True, 1, "true", "yes", "on", "daemon"],
+)
+def test_legacy_truthy_maps_to_prefer(value):
+    assert (
+        parse_write_routing_policy(
+            value,
+            legacy_boolean=True,
+        )
+        is WriteRoutingPolicy.PREFER
+    )
+
+
+@pytest.mark.parametrize(
+    "value",
+    [False, 0, "false", "no", "off"],
+)
+def test_legacy_falsy_maps_to_direct(value):
+    assert (
+        parse_write_routing_policy(
+            value,
+            legacy_boolean=True,
+        )
+        is WriteRoutingPolicy.DIRECT
+    )
+
+
+@pytest.mark.parametrize(
+    "value",
+    [True, False, 1, 0, "maybe", ""],
+)
+def test_new_policy_rejects_legacy_or_invalid_values(value):
+    with pytest.raises(WriteRoutingError):
+        parse_write_routing_policy(value)
+
+
+def test_resolver_returns_first_configured_candidate():
+    resolved = resolve_write_routing_policy(
+        [
+            RoutingPolicyCandidate("first", None),
+            RoutingPolicyCandidate("second", "require"),
+            RoutingPolicyCandidate("third", "direct"),
+        ]
+    )
+
+    assert resolved.policy is WriteRoutingPolicy.REQUIRE
+    assert resolved.source == "second"
+
+
+def test_resolver_names_invalid_source():
+    with pytest.raises(
+        WriteRoutingError,
+        match="MEMPALACE_WRITE_ROUTING",
+    ):
+        resolve_write_routing_policy(
+            [
+                RoutingPolicyCandidate(
+                    "MEMPALACE_WRITE_ROUTING",
+                    "typo",
+                )
+            ]
+        )
+
+
+@pytest.mark.parametrize(
+    (
+        "policy",
+        "available",
+        "can_start",
+        "target",
+        "auto_start",
+    ),
+    [
+        (
+            WriteRoutingPolicy.DIRECT,
+            False,
+            False,
+            WriteRoutingTarget.DIRECT,
+            False,
+        ),
+        (
+            WriteRoutingPolicy.DIRECT,
+            True,
+            True,
+            WriteRoutingTarget.DIRECT,
+            False,
+        ),
+        (
+            WriteRoutingPolicy.PREFER,
+            True,
+            False,
+            WriteRoutingTarget.DAEMON,
+            False,
+        ),
+        (
+            WriteRoutingPolicy.PREFER,
+            False,
+            True,
+            WriteRoutingTarget.DAEMON,
+            True,
+        ),
+        (
+            WriteRoutingPolicy.PREFER,
+            False,
+            False,
+            WriteRoutingTarget.DIRECT,
+            False,
+        ),
+        (
+            WriteRoutingPolicy.REQUIRE,
+            True,
+            False,
+            WriteRoutingTarget.DAEMON,
+            False,
+        ),
+        (
+            WriteRoutingPolicy.REQUIRE,
+            False,
+            True,
+            WriteRoutingTarget.DAEMON,
+            True,
+        ),
+        (
+            WriteRoutingPolicy.REQUIRE,
+            False,
+            False,
+            WriteRoutingTarget.BLOCKED,
+            False,
+        ),
+    ],
+)
+def test_decision_matrix(
+    policy,
+    available,
+    can_start,
+    target,
+    auto_start,
+):
+    decision = choose_write_route(
+        policy,
+        daemon_available=available,
+        daemon_can_start=can_start,
+    )
+
+    assert decision.target is target
+    assert decision.auto_start_daemon is auto_start
+    assert decision.use_daemon is (target is WriteRoutingTarget.DAEMON)
+    assert decision.blocked is (target is WriteRoutingTarget.BLOCKED)
+
+
+def test_hook_policy_defaults_to_direct(tmp_path):
+    cfg = MempalaceConfig(config_dir=tmp_path)
+
+    resolved = cfg.resolve_write_routing("hooks")
+
+    assert resolved.policy is WriteRoutingPolicy.DIRECT
+    assert resolved.source == "default"
+
+
+def test_cli_policy_defaults_to_direct(tmp_path):
+    cfg = MempalaceConfig(config_dir=tmp_path)
+
+    assert cfg.cli_write_routing is WriteRoutingPolicy.DIRECT
+
+
+def test_scoped_env_beats_global_and_config(
+    tmp_path,
+    monkeypatch,
+):
+    _write_config(
+        tmp_path,
+        {
+            "write_routing": {
+                "default": "direct",
+                "hooks": "prefer",
+            }
+        },
+    )
+
+    monkeypatch.setenv(
+        "MEMPALACE_WRITE_ROUTING",
+        "prefer",
+    )
+    monkeypatch.setenv(
+        "MEMPALACE_HOOK_WRITE_ROUTING",
+        "require",
+    )
+
+    resolved = MempalaceConfig(config_dir=tmp_path).resolve_write_routing("hooks")
+
+    assert resolved.policy is WriteRoutingPolicy.REQUIRE
+    assert resolved.source == "MEMPALACE_HOOK_WRITE_ROUTING"
+
+
+def test_global_env_beats_legacy_hook_env_and_config(
+    tmp_path,
+    monkeypatch,
+):
+    _write_config(
+        tmp_path,
+        {
+            "write_routing": {
+                "hooks": "direct",
+            },
+            "hooks": {
+                "daemon": False,
+            },
+        },
+    )
+
+    monkeypatch.setenv(
+        "MEMPALACE_WRITE_ROUTING",
+        "require",
+    )
+    monkeypatch.setenv(
+        "MEMPALACE_HOOKS_DAEMON",
+        "true",
+    )
+
+    cfg = MempalaceConfig(config_dir=tmp_path)
+
+    assert cfg.hook_write_routing is WriteRoutingPolicy.REQUIRE
+
+
+def test_legacy_hook_env_beats_config(
+    tmp_path,
+    monkeypatch,
+):
+    _write_config(
+        tmp_path,
+        {
+            "write_routing": {
+                "hooks": "direct",
+            },
+            "hooks": {
+                "daemon": False,
+            },
+        },
+    )
+
+    monkeypatch.setenv(
+        "MEMPALACE_HOOKS_DAEMON",
+        "true",
+    )
+
+    resolved = MempalaceConfig(config_dir=tmp_path).resolve_write_routing("hooks")
+
+    assert resolved.policy is WriteRoutingPolicy.PREFER
+    assert resolved.source == "MEMPALACE_HOOKS_DAEMON (legacy)"
+
+
+def test_scoped_config_beats_global_config(tmp_path):
+    _write_config(
+        tmp_path,
+        {
+            "write_routing": {
+                "default": "prefer",
+                "cli": "require",
+            }
+        },
+    )
+
+    resolved = MempalaceConfig(config_dir=tmp_path).resolve_write_routing("cli")
+
+    assert resolved.policy is WriteRoutingPolicy.REQUIRE
+    assert resolved.source == "config write_routing.cli"
+
+
+def test_global_config_applies_to_both_scopes(tmp_path):
+    _write_config(
+        tmp_path,
+        {
+            "write_routing": {
+                "default": "prefer",
+            }
+        },
+    )
+
+    cfg = MempalaceConfig(config_dir=tmp_path)
+
+    assert cfg.hook_write_routing is WriteRoutingPolicy.PREFER
+    assert cfg.cli_write_routing is WriteRoutingPolicy.PREFER
+
+
+def test_legacy_hook_config_maps_true_to_prefer(tmp_path):
+    _write_config(
+        tmp_path,
+        {
+            "hooks": {
+                "daemon": True,
+            }
+        },
+    )
+
+    resolved = MempalaceConfig(config_dir=tmp_path).resolve_write_routing("hooks")
+
+    assert resolved.policy is WriteRoutingPolicy.PREFER
+    assert resolved.source == "config hooks.daemon (legacy)"
+
+
+def test_existing_hook_use_daemon_behavior_is_unchanged(
+    tmp_path,
+    monkeypatch,
+):
+    _write_config(
+        tmp_path,
+        {
+            "hooks": {
+                "daemon": False,
+            }
+        },
+    )
+
+    monkeypatch.setenv(
+        "MEMPALACE_HOOKS_DAEMON",
+        "yes",
+    )
+
+    assert MempalaceConfig(config_dir=tmp_path).hook_use_daemon is True
+
+
+def test_invalid_scoped_policy_fails_loudly(tmp_path):
+    _write_config(
+        tmp_path,
+        {
+            "write_routing": {
+                "hooks": "typo",
+            }
+        },
+    )
+
+    with pytest.raises(
+        WriteRoutingError,
+        match="config write_routing.hooks",
+    ):
+        MempalaceConfig(config_dir=tmp_path).resolve_write_routing("hooks")
+
+
+def test_invalid_routing_object_fails_loudly(tmp_path):
+    _write_config(
+        tmp_path,
+        {
+            "write_routing": "require",
+        },
+    )
+
+    with pytest.raises(
+        WriteRoutingError,
+        match="must be an object",
+    ):
+        MempalaceConfig(config_dir=tmp_path).resolve_write_routing("cli")
+
+
+def test_unknown_scope_is_rejected(tmp_path):
+    with pytest.raises(
+        WriteRoutingError,
+        match="scope",
+    ):
+        MempalaceConfig(config_dir=tmp_path).resolve_write_routing("mcp")

uv.lock

@@ -234,6 +234,41 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4a/57/3b7d4dd193ade4641c865bc2b93aeeb71162e81fc348b8dad020215601ed/build-1.4.2-py3-none-any.whl", hash = "sha256:7a4d8651ea877cb2a89458b1b198f2e69f536c95e89129dbf5d448045d60db88", size = 24643, upload-time = "2026-03-25T14:20:26.568Z" },
 ]
 
+[[package]]
+name = "cachetools"
+version = "6.2.6"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/39/91/d9ae9a66b01102a18cd16db0cf4cd54187ffe10f0865cc80071a4104fbb3/cachetools-6.2.6.tar.gz", hash = "sha256:16c33e1f276b9a9c0b49ab5782d901e3ad3de0dd6da9bf9bcd29ac5672f2f9e6", size = 32363, upload-time = "2026-01-27T20:32:59.956Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/90/45/f458fa2c388e79dd9d8b9b0c99f1d31b568f27388f2fdba7bb66bbc0c6ed/cachetools-6.2.6-py3-none-any.whl", hash = "sha256:8c9717235b3c651603fff0076db52d6acbfd1b338b8ed50256092f7ce9c85bda", size = 11668, upload-time = "2026-01-27T20:32:58.527Z" },
+]
+
+[[package]]
+name = "cachetools"
+version = "7.1.4"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.15' and sys_platform == 'win32'",
+    "python_full_version >= '3.15' and sys_platform != 'win32'",
+    "python_full_version == '3.14.*' and sys_platform == 'win32'",
+    "python_full_version == '3.14.*' and sys_platform != 'win32'",
+    "python_full_version == '3.13.*' and sys_platform == 'win32'",
+    "python_full_version == '3.13.*' and sys_platform != 'win32'",
+    "python_full_version == '3.12.*' and sys_platform == 'win32'",
+    "python_full_version == '3.12.*' and sys_platform != 'win32'",
+    "python_full_version == '3.11.*' and sys_platform == 'win32'",
+    "python_full_version == '3.11.*' and sys_platform != 'win32'",
+    "python_full_version == '3.10.*' and sys_platform == 'win32'",
+    "python_full_version == '3.10.*' and sys_platform != 'win32'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f4/8b/0d3945a13955303b81272f759a0331e54c5c793da455e6f5706b89d2639c/cachetools-7.1.4.tar.gz", hash = "sha256:437f55a4e0c1b01a4f3077cc470e6991d47430970e36fbcb77e2be0df4fc1cd6", size = 40085, upload-time = "2026-05-21T22:40:43.376Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8c/7b/1fc1c09cc0756cf25861a3be10565915953876da48bb228fb9a672b20a42/cachetools-7.1.4-py3-none-any.whl", hash = "sha256:323dc4127934744db5b54eb4924482d7edafbf9554e820d1531c2e08c0e4ef54", size = 16761, upload-time = "2026-05-21T22:40:41.845Z" },
+]
+
 [[package]]
 name = "certifi"
 version = "2026.2.25"
@@ -976,6 +1011,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" },
 ]
 
+[[package]]
+name = "faiss-cpu"
+version = "1.14.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*' and sys_platform != 'win32'" },
+    { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform != 'win32'" },
+    { name = "packaging", marker = "python_full_version >= '3.10' and sys_platform != 'win32'" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/34/6b04ef5bae3eada6b5a9457d7875cce041c040d53c890815cbd1e9821c65/faiss_cpu-1.14.3-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f9d0e84d909194f63f027bbd3c1e35e905e48c9345c2db6e6f24da09a6bc5906", size = 6925734, upload-time = "2026-06-13T02:19:05.232Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/8a/b451af4b3c6dd18749ecfb58ccb68503b77e49c9aa4a89d950e9d521e058/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d734cfa9ac90b6a5dfed3a27cb706d05f22824703dafc3969b4e2071877a31c", size = 9661210, upload-time = "2026-06-13T02:19:06.99Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/ed/57335bc18c9e18677587bec9bf070c675b29c8e683e13f4def0440731ca0/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8780b526c06e57aad90a8c4655dfba2ff1b3195bd600ff4499752fc45159c9fc", size = 18506292, upload-time = "2026-06-13T02:19:09.621Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/d3/c6ca8c44a63b909e78aa8a69e14501c79c89613e62261d58455ea603d710/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b28ba083e8c02f2c9be03783402537fd3f00d27e68799c44bf88931500ee12ec", size = 11238800, upload-time = "2026-06-13T02:19:12.821Z" },
+    { url = "https://files.pythonhosted.org/packages/93/5f/b405692913a301251749cb175cb3f564ed257fdaa80a22c9a36444d0095d/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cdcb90850cb4b7c27d270839b37bcc0dc8d1fb6a62d1e13053e51ac95061ca25", size = 19237637, upload-time = "2026-06-13T02:19:15.531Z" },
+]
+
 [[package]]
 name = "filelock"
 version = "3.19.1"
@@ -1951,7 +2004,7 @@ wheels = [
 
 [[package]]
 name = "mempalace"
-version = "3.5.0"
+version = "3.6.0"
 source = { editable = "." }
 dependencies = [
     { name = "chromadb" },
@@ -2001,6 +2054,10 @@ gpu = [
     { name = "onnxruntime-gpu", version = "1.24.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "onnxruntime-gpu", version = "1.25.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
+milvus = [
+    { name = "milvus-lite", marker = "python_full_version >= '3.10' and sys_platform != 'win32'" },
+    { name = "pymilvus" },
+]
 pgvector = [
     { name = "psycopg", version = "3.2.13", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version < '3.10'" },
     { name = "psycopg", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version >= '3.10'" },
@@ -2033,6 +2090,7 @@ requires-dist = [
     { name = "huggingface-hub", specifier = ">=0.20" },
     { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.0" },
     { name = "markitdown", extras = ["docx", "pdf", "pptx", "xlsx"], marker = "python_full_version >= '3.10' and extra == 'extract'", specifier = ">=0.1.5" },
+    { name = "milvus-lite", marker = "python_full_version >= '3.10' and sys_platform != 'win32' and extra == 'milvus'", specifier = ">=3.0" },
     { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0" },
     { name = "numpy", specifier = ">=1.24" },
     { name = "onnxruntime", marker = "extra == 'coreml'", specifier = ">=1.16" },
@@ -2041,17 +2099,18 @@ requires-dist = [
     { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.0" },
     { name = "psutil", marker = "extra == 'dev'", specifier = ">=5.9" },
     { name = "psycopg", extras = ["binary"], marker = "extra == 'pgvector'", specifier = ">=3.1" },
+    { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=3.0.0" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0" },
     { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0" },
     { name = "pytest-rerunfailures", marker = "extra == 'dev'", specifier = ">=12.0" },
     { name = "python-dateutil", specifier = ">=2.8" },
     { name = "pyyaml", specifier = ">=6.0,<7" },
-    { name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.18" },
+    { name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.20" },
     { name = "striprtf", marker = "extra == 'extract'", specifier = ">=0.0.27" },
     { name = "tokenizers", specifier = ">=0.15" },
     { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" },
 ]
-provides-extras = ["dev", "spellcheck", "pgvector", "gpu", "dml", "coreml", "multilingual", "extract"]
+provides-extras = ["dev", "spellcheck", "milvus", "pgvector", "gpu", "dml", "coreml", "multilingual", "extract"]
 
 [package.metadata.requires-dev]
 dev = [
@@ -2062,7 +2121,24 @@ dev = [
     { name = "pytest", specifier = ">=7.0" },
     { name = "pytest-cov", specifier = ">=4.0" },
     { name = "pytest-rerunfailures", specifier = ">=12.0" },
-    { name = "ruff", specifier = "==0.15.18" },
+    { name = "ruff", specifier = "==0.15.20" },
+]
+
+[[package]]
+name = "milvus-lite"
+version = "3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "faiss-cpu", marker = "python_full_version >= '3.10' and sys_platform != 'win32'" },
+    { name = "grpcio", marker = "python_full_version >= '3.10' and sys_platform != 'win32'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*' and sys_platform != 'win32'" },
+    { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform != 'win32'" },
+    { name = "pyarrow", marker = "python_full_version >= '3.10' and sys_platform != 'win32'" },
+    { name = "tomli", marker = "python_full_version == '3.10.*' and sys_platform != 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a3/9a/d80d260e6fe1246818a8ef782c374ba9c6ca46ca3b987c14eabe914ef805/milvus_lite-3.0.tar.gz", hash = "sha256:2c35d0d046b1faae3402cde1fb73d65f51ee8c6aba65f54de1dda46f7bb18b9b", size = 589749, upload-time = "2026-05-13T07:14:05.827Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/59/42/dee08ae3bfc1731572f193d00b248c9370b0b9dff12becb0ffd8b2ee8d56/milvus_lite-3.0-py3-none-any.whl", hash = "sha256:d9a094eab84bdaa4253da3721482282c939da1cce6f4e1759f947e8d3e53406e", size = 230490, upload-time = "2026-05-13T07:14:00.816Z" },
 ]
 
 [[package]]
@@ -3306,12 +3382,14 @@ source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
     "python_full_version == '3.10.*' and sys_platform == 'win32'",
     "python_full_version == '3.10.*' and sys_platform != 'win32'",
+    "python_full_version < '3.10'",
 ]
 dependencies = [
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
-    { name = "python-dateutil", marker = "python_full_version == '3.10.*'" },
-    { name = "pytz", marker = "python_full_version == '3.10.*'" },
-    { name = "tzdata", marker = "python_full_version == '3.10.*'" },
+    { name = "python-dateutil", marker = "python_full_version < '3.11'" },
+    { name = "pytz", marker = "python_full_version < '3.11'" },
+    { name = "tzdata", marker = "python_full_version < '3.11'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" }
 wheels = [
@@ -3908,6 +3986,56 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" },
 ]
 
+[[package]]
+name = "pyarrow"
+version = "24.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a5/bf/a34fee1d624152124fa8355c42f34195ad5fe5233ce5bb87946432047d52/pyarrow-24.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:7c2b98645d576a0b9616892ead22b64a83a5f043c5e2ca15ebcefcb5b70c80cb", size = 35076681, upload-time = "2026-04-21T08:51:46.845Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/41/64180033d7027afce12dc96d0fe1f504c6fa112190582b458acea2399530/pyarrow-24.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:644a246325b8c69c595ad1dd4b463eba4b0cdb731370e4a86137d433208d6147", size = 36684260, upload-time = "2026-04-21T08:51:53.642Z" },
+    { url = "https://files.pythonhosted.org/packages/57/02/9b9320e673dd8a99411fac78690f3df92f6dd6f59754c750110bca66d64e/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3a577bd840ca83f646f0a625dbc571dba7044c43c2d1503afc378b570954345c", size = 45698566, upload-time = "2026-04-21T10:46:02.133Z" },
+    { url = "https://files.pythonhosted.org/packages/67/33/f75e91b9a64c3f33c787e263c93b871ad91b8a4a68c1d5cebddd9840e835/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:e3268e43984d0b1a185c89b4cfff282a7ead12fc93f56cfd7088bdbcbe727041", size = 48835562, upload-time = "2026-04-21T10:46:10.278Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/63/097510448e47e4091faa41c43ba92f97cecaab8f4535b56a3d149578f634/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2392d954fcb920f42d230284b677605e4e2fbb11f2821e823e642abd67fbb491", size = 49394997, upload-time = "2026-04-21T10:46:18.08Z" },
+    { url = "https://files.pythonhosted.org/packages/60/6b/c047d6222ab279024a062742d1807e2fbaf27bba88a98637299ff47b9236/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec9373df11544592b0ba7ec2af0e35059e5f0e7647c6183a854dedd193298f1", size = 51911424, upload-time = "2026-04-21T10:46:25.347Z" },
+    { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" },
+    { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" },
+    { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" },
+    { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" },
+    { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" },
+    { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" },
+    { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" },
+    { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" },
+    { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" },
+    { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" },
+    { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" },
+    { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" },
+    { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" },
+    { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" },
+    { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" },
+]
+
 [[package]]
 name = "pybase64"
 version = "1.4.3"
@@ -4315,6 +4443,29 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
 ]
 
+[[package]]
+name = "pymilvus"
+version = "3.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cachetools", version = "6.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "cachetools", version = "7.1.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "grpcio" },
+    { name = "orjson", version = "3.11.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "orjson", version = "3.11.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "protobuf" },
+    { name = "python-dotenv", version = "1.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "python-dotenv", version = "1.2.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "requests", version = "2.33.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "setuptools" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ac/c1/01647e61f3a82fd881382746b6dde3401d65b88cd4f75bd059901fb2392b/pymilvus-3.0.0-1-py3-none-any.whl", hash = "sha256:57c8e7c87fbbf579f122b4df893949bc78e50bca2988527864891bd544817b05", size = 344817, upload-time = "2026-05-07T14:57:45.235Z" },
+]
+
 [[package]]
 name = "pypdfium2"
 version = "5.8.0"
@@ -5068,27 +5219,36 @@ wheels = [
 
 [[package]]
 name = "ruff"
-version = "0.15.18"
+version = "0.15.20"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/74/98/1295ad5a5aa9bc85bdcdfa5d82fe7b49c61af5657df4f227637ff9de0da6/ruff-0.15.18.tar.gz", hash = "sha256:2698a964c70e8bf402dcb99c8810472d270d141e7aa8c4e13599fd52033a2f33", size = 4761437, upload-time = "2026-06-18T18:25:39.224Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/43/dc/35b341fc554ba02f217fc10da57d1a75168cfbcf75b0ef2202176d4c4f2d/ruff-0.15.20.tar.gz", hash = "sha256:1416eb04349192646b54de98f146c4f59afe37d0decfc02c3cbbf396f3a28566", size = 4755489, upload-time = "2026-06-25T17:20:37.578Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/b9/d0/686e984941269621e2be72612d5c1e461f8f7b38415a2a7d7a81c8ae6715/ruff-0.15.18-py3-none-linux_armv6l.whl", hash = "sha256:8b6850172348c8381b8b3084c5915a4393c2373b9b54cd5b5e1ea15812bc10df", size = 10887308, upload-time = "2026-06-18T18:25:03.062Z" },
-    { url = "https://files.pythonhosted.org/packages/ed/21/bc4123e3f5515ee99f8ce1eb93a14a0628fe4d1678663cd08f933ac16931/ruff-0.15.18-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3fccc153a85417dcd976883160cacce486997b0a0058dd18f54b8aaaac7d1ce2", size = 11281305, upload-time = "2026-06-18T18:25:30.026Z" },
-    { url = "https://files.pythonhosted.org/packages/51/93/4769464c25cf7ab2acb3c7dda9cad3d867eb41c59565b3e2a9d17249c90c/ruff-0.15.18-py3-none-macosx_11_0_arm64.whl", hash = "sha256:08d4c86a68f2c3ec2c9d56380a71fb4a4f65373055cbb8caabd645e9102f38d4", size = 10641215, upload-time = "2026-06-18T18:25:15.802Z" },
-    { url = "https://files.pythonhosted.org/packages/6c/42/56926d17120db2c208d76bf60a1a019644dd9e91dc27f0f95c9caddb1366/ruff-0.15.18-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37e5108745c2c0705da916d7d4de533ddf547051ef45f62888c31bae73f66318", size = 10957224, upload-time = "2026-06-18T18:25:36.955Z" },
-    { url = "https://files.pythonhosted.org/packages/22/4f/d43fab8d8189afde803103022d000a8ef9f230616d436d52a8b2b8d63b50/ruff-0.15.18-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56949a6ce8b3abde54c0bcb22cebfe57e8771cadc84b407ae8b8eaf67ebdcd43", size = 10699024, upload-time = "2026-06-18T18:25:05.707Z" },
-    { url = "https://files.pythonhosted.org/packages/63/42/1e3e4c68bd408b9768cf3e439acbe2c78245225faef253f7028a0cdb63e0/ruff-0.15.18-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01a754cd6a1b630d3f97e33eb452cf7a98040482318e870f8bc52a5a30e62657", size = 11491458, upload-time = "2026-06-18T18:25:20.275Z" },
-    { url = "https://files.pythonhosted.org/packages/20/77/47a3484bea8521e14a203d98c389c5c97846675e4f02734672da4a69b52a/ruff-0.15.18-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ba7a07e03a44dbf10bb086ee06705b173625014ec99f73a7e6836a5e5590a0c", size = 12383752, upload-time = "2026-06-18T18:25:22.535Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/ca/054159590787023d83b658a1a1819c4c8910114e7015069340b71c0961cb/ruff-0.15.18-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a2c40a41a4cadbcf5897b548ab29dfe248b20c540961c0247d98a3973c70403", size = 11577923, upload-time = "2026-06-18T18:25:10.702Z" },
-    { url = "https://files.pythonhosted.org/packages/6d/ff/d353d6b7bbd73cc0ec37f4463d7540e45e894338abdd9964eee0de332708/ruff-0.15.18-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f0480ce690cbb6c4db6e5d08f19fce98e10ba131a8b60c1bcdac42771e3ae2d", size = 11583925, upload-time = "2026-06-18T18:25:32.391Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/4a/891f89b9c296ed3e5f3ece1a5629badc989d9a8fdaa30431aaf4774bc1c2/ruff-0.15.18-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2330215f1f393fa8733f55edce04fcf94c36a2c460fcde31f78cc84e4951e9b1", size = 11582834, upload-time = "2026-06-18T18:25:27.309Z" },
-    { url = "https://files.pythonhosted.org/packages/32/a3/ed9e370154bf85de360b93c03026157f02d4943b2d01ff4945f4429f8e8a/ruff-0.15.18-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a6aa6a3d979e48ae617578183674bf264fbe7d0114a796a26bd678d67963c7ff", size = 10927328, upload-time = "2026-06-18T18:25:34.676Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/d1/5cf5909329fedb5d39d555ee818ba5cf4638e1a301b89785d34f2905bfcb/ruff-0.15.18-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a81beadbbff2c9c245561ae3f77b16709d87f35eec650d0501679239d3449b22", size = 10693187, upload-time = "2026-06-18T18:25:08.245Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/44/ff6c635cf2c4f4e7b618b6640da057376baa36014695487d88aed4794268/ruff-0.15.18-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2186d9e940ae332ab293623a75b5f4fe49565f449954d50a72a046683aa6b809", size = 11208721, upload-time = "2026-06-18T18:25:41.327Z" },
-    { url = "https://files.pythonhosted.org/packages/88/d9/5baa2a30861adfb7022cf33c1e35b2fc18085b08c16f83eff4c7b99a5f48/ruff-0.15.18-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5c2abf140438032bc77b2284a6c9944ecd8a19e5f1c7b52b1b8e4a0a80d19a7a", size = 11678599, upload-time = "2026-06-18T18:25:13.607Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/1a/0725a7cfdc32ff769efb96ee782bec882e16448c5d9e3be947ec4c04ce27/ruff-0.15.18-py3-none-win32.whl", hash = "sha256:02299e6e9fa5b297a3f6d5d10d7bcd655c925b028bb8b9d4588214549c6b9ec4", size = 10901903, upload-time = "2026-06-18T18:25:24.755Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/51/805d9f6fb7970505c3504794a5ec350f605361b807fef4dcf214ebd35e72/ruff-0.15.18-py3-none-win_amd64.whl", hash = "sha256:dac80dc8d26b2257dbefabed62f5d255c3937b4ccb122da1fc634794fa3578b3", size = 12041189, upload-time = "2026-06-18T18:25:17.915Z" },
-    { url = "https://files.pythonhosted.org/packages/29/4c/67bb45e41609eb4726f1bfeb59e083cf91d14c696d4bd14c234a980be93d/ruff-0.15.18-py3-none-win_arm64.whl", hash = "sha256:b2c9257fcbd4a3e5b977a1904e6facca016bafe2edc17df24db67cfaee03b4e4", size = 11329958, upload-time = "2026-06-18T18:25:43.686Z" },
+    { url = "https://files.pythonhosted.org/packages/94/d9/2d5014f0253ba541d2061d9fa7193f48e941c8b21bb88a7ff9bbe0bd0596/ruff-0.15.20-py3-none-linux_armv6l.whl", hash = "sha256:00e188c53e499c3c1637f73c91dcf2fb56d576cab76ce1be50a27c4e80e37078", size = 10839665, upload-time = "2026-06-25T17:19:44.702Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/d3/ac1798ba64f670698867fcfc591d50e7e421bef137db564858f619a30fcf/ruff-0.15.20-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9ebd1fd9b9c95fc0bd7b2761aebec1f030013d2e193a2901b224af68fe47251b", size = 11208649, upload-time = "2026-06-25T17:19:48.787Z" },
+    { url = "https://files.pythonhosted.org/packages/47/47/d3ac899991202095dfcf3d5176be4272642be3cf981a2f1a30f72a2afb95/ruff-0.15.20-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5b16cdd67ca108185cd36dce98c576350c03b1660a751de725fb049193a0632", size = 10622638, upload-time = "2026-06-25T17:19:51.354Z" },
+    { url = "https://files.pythonhosted.org/packages/33/13/4e043fe30aa94d4ff5213a9881fc296d12960f5971b234a5263fdc225312/ruff-0.15.20-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3413bb3c3d2ca6a8208f1f4809cd2dca3c6de6d0b491c0e70847672bde6e6efd", size = 10984227, upload-time = "2026-06-25T17:19:54.044Z" },
+    { url = "https://files.pythonhosted.org/packages/76/e6/92e7bf40388bc5800073b96564f56264f7e48bfd1a498f5ced6ae6d5a769/ruff-0.15.20-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd7ec42b3bb3da066488db093308a69c4ac5ee6d2af333a86ba6e2eb2e7dd44b", size = 10622882, upload-time = "2026-06-25T17:19:57.037Z" },
+    { url = "https://files.pythonhosted.org/packages/13/7a/43460be3f24495a3aa46d4b16873e2c4941b3b5f0b00cf88c03b7b94b339/ruff-0.15.20-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1a36ad0eb77fba9aabfb69ede54de6f376d04ac18ebea022847046d340a8267", size = 11474808, upload-time = "2026-06-25T17:20:00.357Z" },
+    { url = "https://files.pythonhosted.org/packages/27/a0/f37077884873221c6b33b4ab49eb18f9f88e54a16a25a5bca59bef46dd66/ruff-0.15.20-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6df3b1e4610432f0386dba04d853b5f08cbbc903410c6fcc02f620f05aff53c", size = 12293094, upload-time = "2026-06-25T17:20:03.446Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/74/165545b60256a9704c21ac0ec4a0d07933b320812f9584836c9f4aca4292/ruff-0.15.20-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e89f198a1ea6ef0d727c1cf16088bc91a6cb0ab947dedc966715691647186eae", size = 11526176, upload-time = "2026-06-25T17:20:06.301Z" },
+    { url = "https://files.pythonhosted.org/packages/86/b1/a976a136d40ade83ce743578399865f57001003a409acadc0ecbb3051082/ruff-0.15.20-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309809086c2acb67624950a3c8133e80f32d0d3e27106c0cd60ff26657c9f24b", size = 11520767, upload-time = "2026-06-25T17:20:09.191Z" },
+    { url = "https://files.pythonhosted.org/packages/19/0f/f032696cb01c9b54c0263fa393474d7758f1cdc021a01b04e3cbc2500999/ruff-0.15.20-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2d2374caa2f2c2f9e2b7da0a50802cfb8b79f55a9b5e49379f564544fbf56487", size = 11500132, upload-time = "2026-06-25T17:20:13.602Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/f4/51b1a14bc69e8c224b15dab9cce8e99b425e0455d462caa2b3c9be2b6a8e/ruff-0.15.20-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a1ed17b65293e0c2f22fc387bc13198a5de94bf4429589b0ff6946b0feaf21a3", size = 10943828, upload-time = "2026-06-25T17:20:16.635Z" },
+    { url = "https://files.pythonhosted.org/packages/71/4b/fe267640783cd02bf6c5cc290b1df1051be2ec294c678b5c15fe19e52343/ruff-0.15.20-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f701305e66b38ea6c91882490eb73459796808e4c6362a1b765255e0cdcd4053", size = 10645418, upload-time = "2026-06-25T17:20:19.4Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/c0/a65aa4ec2f5e87a1df32dc3ec1fede434fe3dfd5cbcf3b503cafc676ab54/ruff-0.15.20-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b9c0c367ad8e5d0d5b5b8537864c469a0a0e55417aadfbeca41fa61333be9f4", size = 11211770, upload-time = "2026-06-25T17:20:22.033Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/a4/0caa331d954ae2723d729d351c989cb4ca8b6077d5c6c2cb6de75e98c041/ruff-0.15.20-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:01cc00dd58f0df339d0e902219dd53990ea99996a0344e5d9cc8d45d5307e460", size = 11618698, upload-time = "2026-06-25T17:20:25.259Z" },
+    { url = "https://files.pythonhosted.org/packages/10/9b/5f14927848d2fd4aa891fd88d883788c5a7baba561c7874732364045708c/ruff-0.15.20-py3-none-win32.whl", hash = "sha256:ed65ef510e43a137207e0f01cfcf998aeddb1aeeda5c9d35023e910284d7cf21", size = 10857322, upload-time = "2026-06-25T17:20:28.612Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/f0/fe47c501f9dea92a26d788ff98bb5d92ed4cb4c88792c5c88af6b697dc8e/ruff-0.15.20-py3-none-win_amd64.whl", hash = "sha256:a525c81c70fb0380344dd1d8745d8cc1c890b7fc94a58d5a07bd8eb9557b8415", size = 11993274, upload-time = "2026-06-25T17:20:31.871Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca", size = 11343498, upload-time = "2026-06-25T17:20:35.03Z" },
+]
+
+[[package]]
+name = "setuptools"
+version = "82.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" },
 ]
 
 [[package]]