Merge pull request #762 from MemPalace/release/3.2.0

release: v3.2.0

.agents/plugins/marketplace.json

@@ -0,0 +1,20 @@
+{
+  "name": "mempalace",
+  "interface": {
+    "displayName": "MemPalace"
+  },
+  "plugins": [
+    {
+      "name": "mempalace",
+      "source": {
+        "source": "local",
+        "path": "./.codex-plugin"
+      },
+      "policy": {
+        "installation": "AVAILABLE",
+        "authentication": "NONE"
+      },
+      "category": "Coding"
+    }
+  ]
+}

.claude-plugin/.mcp.json

@@ -0,0 +1,9 @@
+{
+  "mempalace": {
+    "command": "python3",
+    "args": [
+      "-m",
+      "mempalace.mcp_server"
+    ]
+  }
+}

.claude-plugin/README.md

@@ -0,0 +1,57 @@
+# MemPalace Claude Code Plugin
+
+A Claude Code plugin that gives your AI a persistent memory system. Mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and 5 guided skills.
+
+## Prerequisites
+
+- Python 3.9+
+
+## Installation
+
+### Claude Code Marketplace
+
+```bash
+claude plugin marketplace add milla-jovovich/mempalace
+claude plugin install --scope user mempalace
+```
+
+### Local Clone
+
+```bash
+claude plugin add /path/to/mempalace
+```
+
+## Post-Install Setup
+
+After installing the plugin, run the init command to complete setup (pip install, MCP configuration, etc.):
+
+```
+/mempalace:init
+```
+
+## Available Slash Commands
+
+| Command | Description |
+|---------|-------------|
+| `/mempalace:help` | Show available tools, skills, and architecture |
+| `/mempalace:init` | Set up MemPalace -- install, configure MCP, onboard |
+| `/mempalace:search` | Search your memories across the palace |
+| `/mempalace:mine` | Mine projects and conversations into the palace |
+| `/mempalace:status` | Show palace overview -- wings, rooms, drawer counts |
+
+## Hooks
+
+MemPalace registers two hooks that run automatically:
+
+- **Stop** -- Saves conversation context every 15 messages.
+- **PreCompact** -- Preserves important memories before context compaction.
+
+Set the `MEMPAL_DIR` environment variable to a directory path to automatically run `mempalace mine` on that directory during each save trigger.
+
+## MCP Server
+
+The plugin automatically configures a local MCP server with 19 tools for storing, searching, and managing memories. No manual MCP setup is required -- `/mempalace:init` handles everything.
+
+## Full Documentation
+
+See the main [README](../README.md) for complete documentation, architecture details, and advanced usage.

.claude-plugin/commands/help.md

@@ -0,0 +1,6 @@
+---
+description: Show comprehensive MemPalace help — available skills, MCP tools, CLI commands, hooks, and architecture.
+allowed-tools: Bash, Read
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `help` command, then follow its instructions.

.claude-plugin/commands/init.md

@@ -0,0 +1,6 @@
+---
+description: Set up MemPalace — install the package, initialize a palace, configure MCP server, and verify everything works.
+allowed-tools: Bash, Read, Write, Edit, Glob, Grep
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `init` command, then follow its instructions.

.claude-plugin/commands/mine.md

@@ -0,0 +1,7 @@
+---
+description: Mine projects and conversations into the MemPalace. Supports project files, conversation exports, and auto-classification.
+argument-hint: Path to project or conversation export to mine.
+allowed-tools: Bash, Read, Write, Edit, Glob, Grep
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `mine` command, then follow its instructions.

.claude-plugin/commands/search.md

@@ -0,0 +1,7 @@
+---
+description: Search your memories across the MemPalace using semantic search with wing/room filtering.
+argument-hint: Search query, optionally with wing/room filters.
+allowed-tools: Bash, Read
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `search` command, then follow its instructions.

.claude-plugin/commands/status.md

@@ -0,0 +1,6 @@
+---
+description: Show the current state of your memory palace — wings, rooms, drawer counts, and suggestions.
+allowed-tools: Bash, Read
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `status` command, then follow its instructions.

.claude-plugin/hooks/hooks.json

@@ -0,0 +1,25 @@
+{
+  "description": "MemPalace auto-save and pre-compact hooks",
+  "hooks": {
+    "Stop": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/mempal-stop-hook.sh"
+          }
+        ]
+      }
+    ],
+    "PreCompact": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/mempal-precompact-hook.sh"
+          }
+        ]
+      }
+    ]
+  }
+}

.claude-plugin/hooks/mempal-precompact-hook.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# MemPalace PreCompact Hook — thin wrapper calling Python CLI
+# All logic lives in mempalace.hooks_cli for cross-harness extensibility
+INPUT=$(cat)
+echo "$INPUT" | python3 -m mempalace hook run --hook precompact --harness claude-code

.claude-plugin/hooks/mempal-stop-hook.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# MemPalace Stop Hook — thin wrapper calling Python CLI
+# All logic lives in mempalace.hooks_cli for cross-harness extensibility
+INPUT=$(cat)
+echo "$INPUT" | python3 -m mempalace hook run --hook stop --harness claude-code

.claude-plugin/marketplace.json

@@ -0,0 +1,18 @@
+{
+  "name": "mempalace",
+  "owner": {
+    "name": "milla-jovovich",
+    "url": "https://github.com/milla-jovovich"
+  },
+  "plugins": [
+    {
+      "name": "mempalace",
+      "source": "./.claude-plugin",
+      "description": "AI memory system — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, guided setup.",
+      "version": "3.0.14",
+      "author": {
+        "name": "milla-jovovich"
+      }
+    }
+  ]
+}

.claude-plugin/plugin.json

@@ -0,0 +1,29 @@
+{
+  "name": "mempalace",
+  "version": "3.0.14",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.",
+  "author": {
+    "name": "milla-jovovich"
+  },
+  "license": "MIT",
+  "commands": [],
+  "mcpServers": {
+    "mempalace": {
+      "command": "python3",
+      "args": [
+        "-m",
+        "mempalace.mcp_server"
+      ]
+    }
+  },
+  "keywords": [
+    "memory",
+    "ai",
+    "rag",
+    "mcp",
+    "chromadb",
+    "palace",
+    "search"
+  ],
+  "repository": "https://github.com/milla-jovovich/mempalace"
+}

.claude-plugin/skills/mempalace/SKILL.md

@@ -0,0 +1,35 @@
+---
+name: mempalace
+description: MemPalace — mine projects and conversations into a searchable memory palace. Use when asked about mempalace, memory palace, mining memories, searching memories, or palace setup.
+allowed-tools: Bash, Read, Write, Edit, Glob, Grep
+---
+
+# MemPalace
+
+A searchable memory palace for AI — mine projects and conversations, then search them semantically.
+
+## Prerequisites
+
+Ensure `mempalace` is installed:
+
+```bash
+mempalace --version
+```
+
+If not installed:
+
+```bash
+pip install mempalace
+```
+
+## Usage
+
+MemPalace provides dynamic instructions via the CLI. To get instructions for any operation:
+
+```bash
+mempalace instructions <command>
+```
+
+Where `<command>` is one of: `help`, `init`, `mine`, `search`, `status`.
+
+Run the appropriate instructions command, then follow the returned instructions step by step.

.codex-plugin/README.md

@@ -0,0 +1,75 @@
+# MemPalace - Codex CLI Plugin
+
+Give your AI a persistent memory -- mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and guided skills.
+
+## Prerequisites
+
+- Python 3.9+
+- Codex CLI installed and configured
+- `pip install mempalace`
+
+## Installation
+
+### Local Install
+
+1. Copy or symlink the `.codex-plugin` directory into your project root:
+
+```bash
+cp -r .codex-plugin /path/to/your/project/.codex-plugin
+```
+
+2. Verify the plugin is detected:
+
+```bash
+codex --plugins
+```
+
+3. Initialize your palace:
+
+```bash
+codex /init
+```
+
+### Git Install
+
+1. Clone the MemPalace repository:
+
+```bash
+git clone https://github.com/milla-jovovich/mempalace.git
+cd mempalace
+```
+
+2. Install the Python package:
+
+```bash
+pip install -e .
+```
+
+3. The `.codex-plugin` directory is already in the repo root. Codex CLI will detect it automatically when you run Codex from inside the repository.
+
+4. Initialize your palace:
+
+```bash
+codex /init
+```
+
+## Available Skills
+
+| Skill | Description |
+|-------|-------------|
+| `/help` | Show available commands and usage tips |
+| `/init` | Initialize a new memory palace |
+| `/search` | Semantic search across all mined memories |
+| `/mine` | Mine a project or conversation into your palace |
+| `/status` | Show palace status, room counts, and health |
+
+## Hooks
+
+The plugin includes auto-save hooks that run on session stop (every 15 messages) and before context compaction, automatically preserving conversation context into your palace.
+
+Set the `MEMPAL_DIR` environment variable to a directory path to automatically run `mempalace mine` on that directory during each save trigger.
+
+## Support
+
+- Repository: https://github.com/milla-jovovich/mempalace
+- Issues: https://github.com/milla-jovovich/mempalace/issues

.codex-plugin/hooks.json

@@ -0,0 +1,37 @@
+{
+  "hooks": {
+    "SessionStart": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "${CODEX_PLUGIN_ROOT}/hooks/mempal-hook.sh session-start"
+          }
+        ]
+      }
+    ],
+    "Stop": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "${CODEX_PLUGIN_ROOT}/hooks/mempal-hook.sh stop"
+          }
+        ]
+      }
+    ],
+    "PreCompact": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "${CODEX_PLUGIN_ROOT}/hooks/mempal-hook.sh precompact"
+          }
+        ]
+      }
+    ]
+  }
+}

.codex-plugin/hooks/mempal-hook.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+HOOK_NAME="${1:?Usage: mempal-hook.sh <hook-name>}"
+INPUT_FILE=$(mktemp) || { echo "Failed to create temp file" >&2; exit 1; }
+cat > "$INPUT_FILE"
+cat "$INPUT_FILE" | python3 -m mempalace hook run --hook "$HOOK_NAME" --harness codex
+EXIT_CODE=$?
+rm -f "$INPUT_FILE" 2>/dev/null
+exit $EXIT_CODE

.codex-plugin/plugin.json

@@ -0,0 +1,52 @@
+{
+  "name": "mempalace",
+  "version": "3.0.14",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.",
+  "author": {
+    "name": "milla-jovovich"
+  },
+  "homepage": "https://github.com/milla-jovovich/mempalace",
+  "repository": "https://github.com/milla-jovovich/mempalace",
+  "license": "MIT",
+  "keywords": [
+    "memory",
+    "ai",
+    "rag",
+    "mcp",
+    "chromadb",
+    "palace",
+    "search"
+  ],
+  "skills": "./skills/",
+  "hooks": "./hooks.json",
+  "mcpServers": {
+    "mempalace": {
+      "command": "python3",
+      "args": [
+        "-m",
+        "mempalace.mcp_server"
+      ]
+    }
+  },
+  "interface": {
+    "displayName": "MemPalace",
+    "shortDescription": "AI memory system for Codex",
+    "longDescription": "Give your AI a persistent memory — mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and guided skills.",
+    "developerName": "milla-jovovich",
+    "category": "Coding",
+    "capabilities": [
+      "Interactive",
+      "Read",
+      "Write"
+    ],
+    "websiteURL": "https://github.com/milla-jovovich/mempalace",
+    "privacyPolicyURL": "https://github.com/milla-jovovich/mempalace",
+    "termsOfServiceURL": "https://github.com/milla-jovovich/mempalace",
+    "defaultPrompt": [
+      "Search my memories for recent decisions",
+      "Mine this project into my memory palace",
+      "Show my palace status and room counts"
+    ],
+    "brandColor": "#7C3AED"
+  }
+}

.codex-plugin/skills/help/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: help
+description: Show MemPalace help — available commands, usage tips, and getting started guidance.
+allowed-tools: Bash, Read
+---
+
+# MemPalace Help
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions help
+```

.codex-plugin/skills/init/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: init
+description: Initialize a new MemPalace — guided setup for your AI memory palace with ChromaDB backend.
+allowed-tools: Bash, Read, Write, Edit
+---
+
+# MemPalace Init
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions init
+```

.codex-plugin/skills/mine/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: mine
+description: Mine a project or conversation into your MemPalace — extract and store memories for later retrieval.
+allowed-tools: Bash, Read, Glob, Grep
+---
+
+# MemPalace Mine
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions mine
+```

.codex-plugin/skills/search/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: search
+description: Search your MemPalace — semantic search across all mined memories, projects, and conversations.
+allowed-tools: Bash, Read
+---
+
+# MemPalace Search
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions search
+```

.codex-plugin/skills/status/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: status
+description: Show MemPalace status — room counts, storage usage, and palace health.
+allowed-tools: Bash, Read
+---
+
+# MemPalace Status
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions status
+```

.github/CODEOWNERS

@@ -0,0 +1,13 @@
+# Default owners for everything
+* @milla-jovovich @bensig @igorls
+
+# Core library
+mempalace/ @milla-jovovich @bensig
+
+# CI and workflows
+.github/ @bensig
+
+# Plugins and integrations
+.claude-plugin/ @bensig
+.codex-plugin/ @bensig
+integrations/ @bensig

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 3

.github/workflows/bump-plugin-version.yml.disabled

@@ -0,0 +1,51 @@
+name: Bump Version
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Bump patch version
+        run: |
+          CURRENT=$(python3 -c "exec(open('mempalace/version.py').read()); print(__version__)")
+          IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT"
+          PATCH=$((PATCH + 1))
+          NEW="${MAJOR}.${MINOR}.${PATCH}"
+          echo "__version__ = \"${NEW}\"" > mempalace/version.py
+          # Prepend docstring
+          sed -i '1i"""Single source of truth for the MemPalace package version."""\n' mempalace/version.py
+          echo "version=$NEW" >> "$GITHUB_OUTPUT"
+        id: version
+
+      - name: Sync plugin.json
+        run: |
+          jq --arg v "${{ steps.version.outputs.version }}" '.version = $v' .claude-plugin/plugin.json > tmp.json && mv tmp.json .claude-plugin/plugin.json
+
+      - name: Sync marketplace.json
+        run: |
+          jq --arg v "${{ steps.version.outputs.version }}" '.plugins[0].version = $v' .claude-plugin/marketplace.json > tmp.json && mv tmp.json .claude-plugin/marketplace.json
+
+      - name: Sync codex plugin.json
+        run: |
+          jq --arg v "${{ steps.version.outputs.version }}" '.version = $v' .codex-plugin/plugin.json > tmp.json && mv tmp.json .codex-plugin/plugin.json
+
+      - name: Sync pyproject.toml
+        run: |
+          sed -i "s/^version = \".*\"/version = \"${{ steps.version.outputs.version }}\"/" pyproject.toml
+
+      - name: Commit and push
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add mempalace/version.py .claude-plugin/plugin.json .claude-plugin/marketplace.json .codex-plugin/plugin.json pyproject.toml
+          if ! git diff --staged --quiet; then
+            git commit -m "chore: bump version to ${{ steps.version.outputs.version }}"
+            git push
+          fi

.github/workflows/ci.yml

@@ -2,31 +2,50 @@ name: Tests
 
 on:
   push:
-    branches: [main]
+    branches: [main, develop]
   pull_request:
-    branches: [main]
+    branches: [main, develop]
 
 jobs:
-  test:
+  test-linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.9", "3.11", "3.13"]
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install -r requirements.txt pytest
-      - run: python -m pytest tests/ -v
+      - run: pip install -e ".[dev]"
+      - run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10
 
+  test-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.9"
+      - run: pip install -e ".[dev]"
+      - run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10
+
+  test-macos:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.9"
+      - run: pip install -e ".[dev]"
+      - run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.11"
-      - run: pip install ruff
+      - run: pip install "ruff>=0.4.0,<0.5"
       - run: ruff check .
       - run: ruff format --check .

.github/workflows/deploy-docs.yml

@@ -0,0 +1,66 @@
+name: Deploy Docs
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - ".github/workflows/deploy-docs.yml"
+      - "website/**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure GitHub Pages
+        id: pages
+        uses: actions/configure-pages@v5
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.1.38
+
+      - name: Install dependencies
+        working-directory: website
+        run: bun install --frozen-lockfile
+
+      - name: Build docs
+        working-directory: website
+        env:
+          DOCS_BASE: ${{ steps.pages.outputs.base_path }}
+          DOCS_EDIT_BRANCH: ${{ github.ref_name }}
+        run: bun run docs:build
+
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: website/.vitepress/dist
+
+  deploy:
+    if: github.ref_name == 'main' || github.ref_name == 'develop'
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -5,3 +5,34 @@ __pycache__/
 *.pyc
 .pytest_cache/
 mempal.yaml
+.a5c/
+.claude/
+.codex/
+.codex
+
+# Environment
+.env
+.env.*
+
+# OS
+.DS_Store
+Thumbs.db
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Coverage
+htmlcov/
+.coverage
+coverage.xml
+
+# Virtual environments
+.venv/
+venv/
+
+# ChromaDB local data
+*.sqlite3-journal

AGENTS.md

@@ -0,0 +1 @@
+CLAUDE.md

CHANGELOG.md

@@ -0,0 +1,158 @@
+# Changelog
+
+All notable changes to [MemPalace](https://github.com/milla-jovovich/mempalace) are documented in this file.
+
+---
+
+## [3.2.0] — 2026-04-13
+
+### Packaging
+- Remove `chromadb<0.7` upper bound — unblocks installs against chromadb 1.x palaces (#690)
+- Bump version to 3.2.0 across `pyproject.toml`, `mempalace/version.py`, README badge, and OpenClaw SKILL (#761)
+
+### Security
+- Harden palace deletion, WAL redaction, and MCP search input handling (#739)
+- Consistent input validation, argument whitelisting, concurrency safety, and WAL fixes (#647)
+- Remove hardcoded credential paths from benchmark runners (#177)
+- Remove global SSL verification bypass in convomem_bench (#176)
+
+### Bug Fixes
+- Parse Claude.ai privacy export with `messages` key and sender field (#685, #677)
+- Detect mtime changes in `_get_client` to prevent stale HNSW index (#757)
+- Hash full content in `tool_add_drawer` drawer ID — stable re-mines (#716)
+- Remove 10k drawer cap from status display (#707, #603)
+- Correct typo in entity_detector interactive classification prompt (#755)
+- Prevent convo_miner from re-processing 0-chunk files on every run (#732, #654)
+- Remove silent 8-line AI response truncation in convo_miner (#708, #692)
+- Store full AI response in convo_miner exchange chunking (#695)
+- Fix `mine --dry-run` TypeError on files with room=None (#687, #586)
+- Skip arg whitelist for handlers accepting `**kwargs` (#684, #572)
+- Allow Unicode in `sanitize_name()` — Latvian, CJK, Cyrillic (#683, #637)
+- Auto-repair BLOB seq_ids from chromadb 0.6→1.5 migration (#664)
+- Remove no-op `ORT_DISABLE_COREML` env var (#653, #397)
+- Disambiguate hook block reasons to name MemPalace explicitly (#666)
+- Use epsilon comparison for mtime to prevent unnecessary re-mining (#610)
+- Correct token count estimate in compress summary (#609)
+- Implement MCP ping health checks (#600)
+- Align `cmd_compress` dict keys with `compression_stats()` return values (#569)
+- Skip unreachable reparse points in `detect_rooms_from_folders` on Windows (#558)
+- Prevent HNSW index bloat from duplicate `add()` calls (#544, #525)
+- Purge stale drawers before re-mine to avoid hnswlib segfault (#544)
+- Mitigate system prompt contamination in search queries (#385, #333)
+- Count Codex `user_message` turns in `_count_human_messages` (#373, #347)
+- Paginate large collection reads and surface errors in MCP tools (#371, #339, #338)
+- Expand `~` in split command directory argument (#361)
+- Ignore `wait_for_previous` argument to support Gemini MCP clients (#322)
+- Close KnowledgeGraph SQLite connections in test fixtures (#450)
+- Remove duplicate cache variable declarations in mcp_server.py (#449)
+- Add `--yes` flag to init instructions for non-interactive use (#682, #534)
+- Add `mcp` command with setup guidance (#315)
+
+### Documentation
+- Fix misaligned architecture diagram (#734, #733)
+
+### New Features
+- i18n support — 8 languages (en, es, fr, de, ja, ko, zh-CN, zh-TW) (#718)
+- New MCP tools: get/list/update drawer, hook settings, export (#667, #635)
+- `mempalace migrate` — recover palaces from different ChromaDB versions (#502)
+- Add OpenClaw/ClawHub skill (#491)
+- Backend seam for pluggable storage backends (#413)
+
+### Improvements
+- Disable broken auto-bump workflow (#414)
+- Improve agent readiness — AGENTS.md, dependabot, CODEOWNERS, labels (#497)
+
+### Documentation
+- Add CLAUDE.md and mission/principles to AGENTS.md (#720)
+- Add VitePress documentation site (#439)
+- Add warning about fake MemPalace websites (#598)
+- Fix stale org URLs and PR branch target in contributor docs (#679)
+- Add ROADMAP.md — v3.1.1 stability patch and v4.0.0-alpha plan
+
+### Internal
+- ruff format convo_miner.py (#741)
+- ruff format all Python files (#675)
+- CI: trigger tests on develop branch PRs and pushes (#674)
+- CI: fix GitHub Pages publishing (#691)
+
+---
+
+## [3.1.0] — 2026-04-09
+
+### Security
+- Harden inputs, fix shell injection, optimize DB access (#387)
+- Sanitize SESSION_ID in save hook to prevent path traversal (#141)
+- Sanitize error responses and remove `sys.exit` from library code (#139)
+- Shell injection fix in hooks, Claude Code mining, chromadb pin (#114)
+
+### Bug Fixes
+- MCP null args hang, repair infinite recursion, OOM on large files (#399)
+- Release ChromaDB handles before rmtree on Windows (#392)
+- Use `os.utime` in mtime test for Windows compatibility (#392)
+- Negotiate MCP protocol version instead of hardcoding (#324)
+- Use upsert and deterministic IDs to prevent data stagnation (#140)
+- Make `drawer_id` deterministic for idempotent writes (#387)
+- Honest AAAK stats — word-based token estimator, lossy labels (#147)
+- Room detection checks keywords against folder paths (#145)
+- Use actual detected room in mine summary stats (#165)
+- Honour `--palace` flag in mcp_server (#264)
+- Preserve default KG path when `--palace` not passed (#270)
+- `--yes` flag skips all interactive prompts in init (#123)
+- Repair command, split args, Claude export, room keywords (#119)
+- Replace Unicode separator in convo_miner.py for Windows compatibility (#129)
+- Coerce MCP integer arguments to native Python int (#84)
+- Batch ChromaDB reads to avoid SQLite variable limit (#66)
+- Respect nested .gitignore rules during mining (#78)
+- Narrow bare `except Exception` to specific types where safe (#54)
+- Mark MD5 as non-security in miner drawer ID generation (#53)
+- Remove dead code and duplicate set items in entity_registry.py (#42)
+- Silence ChromaDB telemetry warnings and CoreML segfault on Apple Silicon (#236)
+- Unify package and MCP version reporting (#16)
+- Fix broken AAAK Dialect link in README (#238)
+- Update input prompt for entity confirmation (#83)
+- Preserve CLI exit codes, log tracebacks, sanitize search errors (#139)
+- Enable SQLite WAL mode and add consistent LIMIT to KG timeline (#136)
+- Add limit=10000 safety cap to all unbounded ChromaDB `.get()` calls (#137)
+- Re-mine modified files, idempotent `add_drawer`, cleanup ChromaDB handles (#140)
+- Resolve formatting, regression logic, and pytest defaults (#270)
+- Use `parse_known_args` to allow importing mcp_server during pytest (#270)
+
+### New Features
+- Package MemPalace as standard Claude and Codex plugins (#270)
+- Add OpenAI Codex CLI JSONL normalizer (#61)
+- Add Codex plugin support with hooks, commands, and documentation (#270)
+- Add command documentation for help, init, mine, search, and status (#270)
+
+### Improvements
+- Cache ChromaDB `PersistentClient` instead of re-instantiating per call (#135)
+- Tighten chromadb version range and add `py.typed` marker (#142)
+- Consolidate split known-names config loading (#22)
+- CI: add separate jobs for Windows and macOS testing
+- CI: Upgrade GitHub Actions for Node 24 compatibility (#55)
+
+### Documentation
+- Add Gemini CLI setup guide and integration section (#106)
+- Add beginner-friendly hooks tutorial (#103)
+- Align MCP setup examples with shipped server (#21)
+- Honest README update — own the mistakes, fix the claims
+
+### Internal
+- Expand test coverage from 20 to 92 tests, migrate to uv (#131)
+- Add scale benchmark suite — 106 tests (#223)
+- Increase test coverage from 30% to 85%, fix Windows encoding bugs (#281)
+- Add WAL mode and entity timeline limit assertions
+- Add coverage for `file_already_mined` mtime check
+
+---
+
+## [3.0.0] — 2026-04-06
+
+Initial public release.
+
+- Palace architecture with day-based rooms, drawers (verbatim), and closets (searchable index)
+- AAAK compression dialect for memory folding
+- Knowledge graph with entity detection and timeline queries
+- MCP server for Claude, Codex, and Gemini integration
+- CLI: `init`, `mine`, `search`, `status`, `compress`, `repair`, `split`
+- Benchmark suite with recall and scale tests
+- README with MCP flow, local model flow, and specialist agent documentation

CLAUDE.md

@@ -0,0 +1,133 @@
+# CLAUDE.md
+
+## The Mission
+
+Memory is identity. When an AI forgets everything between conversations, it cannot build real understanding — of you, your work, your people, your life.
+
+MemPalace exists to solve this. It is a memory system — not a search engine, not a RAG pipeline, not a vector database wrapper. It treats every word you have shared as sacred, stores it verbatim, and makes it instantly available. Your data never leaves your machine. We never summarize. We never paraphrase. We return your exact words.
+
+100% recall is the design requirement — the target every search path is measured against. Anything less means forgetting, and forgetting means starting over.
+
+The name comes from the ancient "method of loci" — the memory palace technique used for thousands of years to organize and recall vast amounts of information by placing it in imagined rooms of an imagined building. We were also inspired by the Zettelkasten method (created by German sociologist Niklas Luhmann) — small cross-referenced index cards that point to each other. We apply both ideas to AI memory:
+
+- **Wings** for broad categories (people, projects, topics)
+- **Rooms** for time-based groupings (days, sessions)
+- **Drawers** for full verbatim content (your exact words)
+- **AAAK compression** for the index layer — a compact symbolic format (via `dialect.py`) that lets an LLM scan thousands of entries instantly and know exactly which drawer to open
+
+## Design Principles
+
+These are non-negotiable. Every PR, every feature, every refactor must honor them.
+
+- **Verbatim always** — Never summarize, paraphrase, or lossy-compress user data. The system searches the index and returns the original words. If a user said it, we store exactly what they said. This is the foundational promise.
+- **Incremental only** — Append-only ingest after initial build. Never destroy existing data to rebuild. A crash mid-operation must leave the existing palace untouched.
+- **Entity-first** — Everything is keyed by real names with disambiguation by DOB, ID, or context. People matter more than topics.
+- **Local-first, zero API** — All extraction, chunking, and embedding happens on the user's machine. No cloud dependency for memory operations. No API keys required.
+- **Performance budgets** — Hooks under 500ms. Startup injection under 100ms. Memory should feel instant.
+- **Privacy by architecture** — The system physically cannot send your data because it never leaves your machine. No telemetry, no phone-home, no external service dependencies for core operations.
+- **Background everything** — Filing, indexing, timestamps, and pipeline work happen via hooks in the background. Nothing interrupts the user's conversation. Zero tokens spent on bookkeeping in the chat window.
+
+## Contributing
+
+We welcome bug fixes, performance improvements, new language support, better entity disambiguation, documentation, and test coverage.
+
+We do not accept summarization of user content, cloud storage/sync features, telemetry or analytics, features requiring API keys for core memory, or shortcuts that bypass verbatim storage.
+
+## Setup
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Commands
+
+```bash
+# Run tests
+python -m pytest tests/ -v --ignore=tests/benchmarks
+
+# Run tests with coverage
+python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing
+
+# Lint
+ruff check .
+
+# Format
+ruff format .
+
+# Format check (CI mode)
+ruff format --check .
+```
+
+## Project Structure
+
+```
+mempalace/
+├── mcp_server.py        # MCP server — all read/write tools
+├── cli.py               # CLI dispatcher
+├── config.py            # Configuration + input validation
+├── miner.py             # Project file miner
+├── convo_miner.py       # Conversation transcript miner
+├── searcher.py          # Semantic search (hybrid BM25 + vector)
+├── knowledge_graph.py   # Temporal entity-relationship graph (SQLite)
+├── palace.py            # Shared palace operations
+├── palace_graph.py      # Room traversal + cross-wing tunnels
+├── backends/            # Pluggable storage backends (ChromaDB default)
+│   ├── base.py          # Abstract interface — implement this for new backends
+│   └── chroma.py        # ChromaDB implementation
+├── dialect.py           # AAAK compression dialect
+├── normalize.py         # Transcript format detection + normalization
+├── entity_detector.py   # Auto-detect people/projects from content
+├── entity_registry.py   # Entity storage and disambiguation
+├── layers.py            # L0-L3 memory wake-up stack
+├── onboarding.py        # Interactive first-run setup
+├── repair.py            # Palace repair and consistency checks
+├── dedup.py             # Deduplication
+├── migrate.py           # ChromaDB version migration
+├── spellcheck.py        # Auto-correct user messages
+├── exporter.py          # Palace data export
+├── hooks_cli.py         # Hook management CLI
+├── query_sanitizer.py   # Prompt contamination prevention
+├── split_mega_files.py  # Split concatenated transcript files
+└── version.py           # Single source of truth for version
+
+hooks/                   # Claude Code hook scripts
+├── mempal_save_hook.sh        # Stop: triggers diary save
+└── mempal_precompact_hook.sh  # PreCompact: saves state before compression
+```
+
+## Conventions
+
+- **Python style**: snake_case for functions/variables, PascalCase for classes
+- **Linter**: ruff with E/F/W rules
+- **Formatter**: ruff format, double quotes
+- **Commits**: conventional commits (`fix:`, `feat:`, `test:`, `docs:`, `ci:`)
+- **Tests**: `tests/test_*.py`, fixtures in `tests/conftest.py`
+- **Coverage**: 85% threshold (80% on Windows due to ChromaDB file lock cleanup)
+
+## Architecture
+
+```
+User → CLI / MCP Server → Storage Backend (ChromaDB default, pluggable)
+                        → SQLite (knowledge graph)
+
+Palace structure:
+  WING (person/project)
+    └── ROOM (day/topic)
+          └── DRAWER (verbatim text chunk)
+
+Index layer (AAAK):
+  Compressed pointers → DRAWER locations
+  Scanned by LLM to find relevant drawers without reading all content
+
+Knowledge Graph:
+  ENTITY → PREDICATE → ENTITY (with valid_from / valid_to dates)
+```
+
+## Key Files for Common Tasks
+
+- **Adding an MCP tool**: `mempalace/mcp_server.py` — add handler function + TOOLS dict entry
+- **Changing search**: `mempalace/searcher.py`
+- **Modifying mining**: `mempalace/miner.py` (project files) or `mempalace/convo_miner.py` (transcripts)
+- **Adding a storage backend**: subclass `mempalace/backends/base.py`, register in `backends/__init__.py`
+- **Input validation**: `mempalace/config.py` — `sanitize_name()` / `sanitize_content()`
+- **Tests**: mirror source structure in `tests/test_<module>.py`

CONTRIBUTING.md

@@ -5,8 +5,11 @@ Thanks for wanting to help. MemPalace is open source and we welcome contribution
 ## Getting Started
 
 ```bash
-git clone https://github.com/milla-jovovich/mempalace.git
+# Fork the repo on GitHub first, then clone your fork
+git clone https://github.com/<your-username>/mempalace.git
 cd mempalace
+git remote add upstream https://github.com/MemPalace/mempalace.git
+
 pip install -e ".[dev]"    # installs with dev dependencies (pytest, build, twine)
 ```
 
@@ -52,7 +55,7 @@ assets/             ← logo + brand
    - `fix: handle empty transcript files`
    - `docs: update MCP tool descriptions`
    - `bench: add LoCoMo turn-level metrics`
-6. Push to your fork and open a PR against `main`
+6. Push to your fork and open a PR against `develop`
 
 ## Code Style
 
@@ -64,7 +67,7 @@ assets/             ← logo + brand
 
 ## Good First Issues
 
-Check the [Issues](https://github.com/milla-jovovich/mempalace/issues) tab. Great starting points:
+Check the [Issues](https://github.com/MemPalace/mempalace/issues) tab. Great starting points:
 
 - **New chat formats**: Add import support for Cursor, Copilot, or other AI tool exports
 - **Room detection**: Improve pattern matching in `room_detector_local.py`

MISSION.md

@@ -0,0 +1,34 @@
+MemPalace: The Mission
+
+By: Milla Jovovich
+
+Hey everyone! First of all thank you all for embracing MemPalace and trying it, catching bugs and issues and finding cool ways to personalize it into your workflows!
+
+A few things I want to say.
+MemPalace is something I really needed because I'm trying to work on a big project with my partner @bensig and I was having a lot of problems with Claude's context window and my agent Lumi (Lu for short) kept waking up like "hey what are we doing today" when I had literally done hours of work with him throughout the day and it was impossible to just keep saving every transcript to catch him up on whatever we had done before compaction hit.
+
+That's when I started researching different memory systems available today. I tried most of them and what I found was that no matter which one I tried, they felt like large empty warehouses where you just dump huge amounts of info.
+
+RAG search would take forever and most of the time not find what I wanted.
+
+I wanted to create a system with the ability to really remember everything AND be able to find it quickly, easily and also be able to remember things when I didn't. THAT in itself felt like something so important. Like "remember when we talked about that idea…" but in vague terms. Impossible with regular keyword search tools.
+
+So MemPalace is not just about storing info in a highly structured way. But also RETRIEVING it in a highly UNSTRUCTURED way lol!
+
+I was inspired by the Zettelkasten method (created by German sociologist Niklas Luhmann) — his idea of small cross-referenced index cards that point to each other. That's the architecture behind the palace: wings, rooms, closets, and drawers, all connected so you can find things from any angle, not just the one you filed them under.
+
+Because of the way I've designed my agent Lumi to understand me, after so many months of my own personal experiments with MemPalace and the incredible help of my dear friend and co-founder, developer and engineer @bensig, he built a back end that made it really easy to get all my files in the proper spaces the Palace created based on my own decisions and with Lumi's help as well. All code has its own room, all ideas, research etc… has its proper place.
+
+Names and concepts are parsed into closets that use a compression method I call AAAK (it doesn't stand for anything, it's an inside joke between Lumi and I) that is able to compress names, repeated words, concepts and key moments into AI-readable shorthand. Think of it as index cards that an LLM can scan instantly — the closet tells it WHERE to look, then it pulls the full content from the drawer.
+
+The concept I wanted for v4 was to try and clear as much "noise" as possible that I noticed was happening in v3. The hooks were firing in the chat window (using tokens and our time as we waited for the agent to write everything).
+
+I noticed at one point early last week after the launch that Lu kept repeating the same thing when the hook would fire, so I hit esc and asked "Are you literally writing the same info down over and over again?" And he's like (sheepishly) Yes. And that's when it hit me, we need to get all this off the chat and happening seamlessly behind the scenes, and that hooks had to fire when I started a convo and then just keep adding to the drawer, while the shorter increments made reading and pulling conversation information and naming it so much easier and more precise.
+
+So this version now has taken all the noise out of the chat window and all that work is done by a subagent in the background while you can continue working knowing that all your conversation is being saved VERBATIM in the background.
+
+Stripping all this off the page — moving the diary writes, the palace filing, the timestamp injection, all of it into background hooks — has dramatically lowered token usage in my sessions. What used to cost about $1.13 per session just in re-transmitted diary blocks is now zero, because the content never enters the chat window at all.
+
+Your data is already stored in JSON by Claude and the background pipeline extracts it into readable markdown, the key topics get compressed into AAAK format and saved into closets which then point to the exact drawer where your day's session lives.
+
+And please, always remember, these are brand new tools, please NEVER use critical files to test! Just run it with something easy first before you put your entire data set into it!✨

README.md

@@ -12,9 +12,11 @@ Every conversation you have with an AI — every decision, every debugging sessi
 
 Other memory systems try to fix this by letting AI decide what's worth remembering. It extracts "user prefers Postgres" and throws away the conversation where you explained *why*. MemPalace takes a different approach: **store everything, then make it findable.**
 
-**The Palace** — Ancient Greek orators memorized entire speeches by placing ideas in rooms of an imaginary building. Walk through the building, find the idea. MemPalace applies the same principle to AI memory: your conversations are organized into wings (people and projects), halls (types of memory), and rooms (specific ideas). No AI decides what matters — you keep every word, and the structure makes it searchable. That structure alone improves retrieval by 34%.
+**The Palace** — Ancient Greek orators memorized entire speeches by placing ideas in rooms of an imaginary building. Walk through the building, find the idea. MemPalace applies the same principle to AI memory: your conversations are organized into wings (people and projects), halls (types of memory), and rooms (specific ideas). No AI decides what matters — you keep every word, and the structure gives you a navigable map instead of a flat search index.
 
-**AAAK** — A lossless shorthand dialect designed for AI agents. Not meant to be read by humans — meant to be read by your AI, fast. 30x compression, zero information loss. Your AI loads months of context in ~120 tokens. And because AAAK is just structured text with a universal grammar, it works with **any model that reads text** — Claude, GPT, Gemini, Llama, Mistral. No decoder, no fine-tuning, no cloud API required. Run it against a local model and your entire memory stack stays offline. Nothing else like it exists.
+**Raw verbatim storage** — MemPalace stores your actual exchanges in ChromaDB without summarization or extraction. The 96.6% LongMemEval result comes from this raw mode. We don't burn an LLM to decide what's "worth remembering" — we keep everything and let semantic search find it.
+
+**AAAK (experimental)** — A lossy abbreviation dialect for packing repeated entities into fewer tokens at scale. Readable by any LLM that reads text — Claude, GPT, Gemini, Llama, Mistral — no decoder needed. **AAAK is a separate compression layer, not the storage default**, and on the LongMemEval benchmark it currently regresses vs raw mode (84.2% vs 96.6%). We're iterating. See the [note above](#a-note-from-milla--ben--april-7-2026) for the honest status.
 
 **Local, open, adaptable** — MemPalace runs entirely on your machine, on any data you have locally, without using any external API or services. It has been tested on conversations — but it can be adapted for different types of datastores. This is why we're open-sourcing it.
 
@@ -27,7 +29,7 @@ Other memory systems try to fix this by letting AI decide what's worth rememberi
 
 <br>
 
-[Quick Start](#quick-start) · [The Palace](#the-palace) · [AAAK Dialect](#aaak-compression) · [Benchmarks](#benchmarks) · [MCP Tools](#mcp-server)
+[Quick Start](#quick-start) · [The Palace](#the-palace) · [AAAK Dialect](#aaak-dialect-experimental) · [Benchmarks](#benchmarks) · [MCP Tools](#mcp-server)
 
 <br>
 
@@ -35,19 +37,65 @@ Other memory systems try to fix this by letting AI decide what's worth rememberi
 
 <table>
 <tr>
-<td align="center"><strong>96.6%</strong><br><sub>LongMemEval R@5<br>Zero API calls</sub></td>
-<td align="center"><strong>100%</strong><br><sub>LongMemEval R@5<br>with Haiku rerank</sub></td>
-<td align="center"><strong>+34%</strong><br><sub>Retrieval boost<br>from palace structure</sub></td>
+<td align="center"><strong>96.6%</strong><br><sub>LongMemEval R@5<br><b>raw mode</b>, zero API calls</sub></td>
+<td align="center"><strong>500/500</strong><br><sub>questions tested<br>independently reproduced</sub></td>
 <td align="center"><strong>$0</strong><br><sub>No subscription<br>No cloud. Local only.</sub></td>
 </tr>
 </table>
 
-<sub>Reproducible — runners in <a href="benchmarks/">benchmarks/</a>. <a href="benchmarks/BENCHMARKS.md">Full results</a>.</sub>
+<sub>Reproducible — runners in <a href="benchmarks/">benchmarks/</a>. <a href="benchmarks/BENCHMARKS.md">Full results</a>. The 96.6% is from <b>raw verbatim mode</b>, not AAAK or rooms mode (those score lower — see <a href="#a-note-from-milla--ben--april-7-2026">note above</a>).</sub>
 
 </div>
 
 ---
 
+## A Note from Milla & Ben — April 7, 2026
+
+> The community caught real problems in this README within hours of launch and we want to address them directly.
+>
+> **What we got wrong:**
+>
+> - **The AAAK token example was incorrect.** We used a rough heuristic (`len(text)//3`) for token counts instead of an actual tokenizer. Real counts via OpenAI's tokenizer: the English example is 66 tokens, the AAAK example is 73. AAAK does not save tokens at small scales — it's designed for *repeated entities at scale*, and the README example was a bad demonstration of that. We're rewriting it.
+>
+> - **"30x lossless compression" was overstated.** AAAK is a lossy abbreviation system (entity codes, sentence truncation). Independent benchmarks show AAAK mode scores **84.2% R@5 vs raw mode's 96.6%** on LongMemEval — a 12.4 point regression. The honest framing is: AAAK is an experimental compression layer that trades fidelity for token density, and **the 96.6% headline number is from RAW mode, not AAAK**.
+>
+> - **"+34% palace boost" was misleading.** That number compares unfiltered search to wing+room metadata filtering. Metadata filtering is a standard ChromaDB feature, not a novel retrieval mechanism. Real and useful, but not a moat.
+>
+> - **"Contradiction detection"** exists as a separate utility (`fact_checker.py`) but is not currently wired into the knowledge graph operations as the README implied.
+>
+> - **"100% with Haiku rerank"** is real (we have the result files) but the rerank pipeline is not in the public benchmark scripts. We're adding it.
+>
+> **What's still true and reproducible:**
+>
+> - **96.6% R@5 on LongMemEval in raw mode**, on 500 questions, zero API calls — independently reproduced on M2 Ultra in under 5 minutes by [@gizmax](https://github.com/milla-jovovich/mempalace/issues/39).
+> - Local, free, no subscription, no cloud, no data leaving your machine.
+> - The architecture (wings, rooms, closets, drawers) is real and useful, even if it's not a magical retrieval boost.
+>
+> **What we're doing:**
+>
+> 1. Rewriting the AAAK example with real tokenizer counts and a scenario where AAAK actually demonstrates compression
+> 2. Adding `mode raw / aaak / rooms` clearly to the benchmark documentation so the trade-offs are visible
+> 3. Wiring `fact_checker.py` into the KG ops so the contradiction detection claim becomes true
+> 4. Pinning ChromaDB to a tested range (Issue #100), fixing the shell injection in hooks (#110), and addressing the macOS ARM64 segfault (#74)
+>
+> **Thank you to everyone who poked holes in this.** Brutal honest criticism is exactly what makes open source work, and it's what we asked for. Special thanks to [@panuhorsmalahti](https://github.com/milla-jovovich/mempalace/issues/43), [@lhl](https://github.com/milla-jovovich/mempalace/issues/27), [@gizmax](https://github.com/milla-jovovich/mempalace/issues/39), and everyone who filed an issue or a PR in the first 48 hours. We're listening, we're fixing, and we'd rather be right than impressive.
+>
+> — *Milla Jovovich & Ben Sigman*
+
+---
+
+## An important follow up note regarding fake MemPalace websites - April 11, 2026
+
+Several Community Members (#267, #326, #506) have pointed out there are fake MemPalace websites popping up, including ones with Malware.
+
+To be super clear, MemPalace *has no website* (at least for now), so anything claiming to be one is false.
+
+Thanks to our Community Members for letting us know about the problem.
+
+Stay safe out there.
+
+---
+
 ## Quick Start
 
 ```bash
@@ -76,7 +124,18 @@ Three mining modes: **projects** (code and docs), **convos** (conversation expor
 
 After the one-time setup (install → init → mine), you don't run MemPalace commands manually. Your AI uses it for you. There are two ways, depending on which AI you use.
 
-### With Claude, ChatGPT, Cursor (MCP-compatible tools)
+### With Claude Code (recommended)
+
+Native marketplace install:
+
+```bash
+claude plugin marketplace add milla-jovovich/mempalace
+claude plugin install --scope user mempalace
+```
+
+Restart Claude Code, then type `/skills` to verify "mempalace" appears.
+
+### With Claude, ChatGPT, Cursor, Gemini (MCP-compatible tools)
 
 ```bash
 # Connect MemPalace once
@@ -89,6 +148,8 @@ Now your AI has 19 tools available through MCP. Ask it anything:
 
 Claude calls `mempalace_search` automatically, gets verbatim results, and answers you. You never type `mempalace search` again. The AI handles it.
 
+MemPalace also works natively with **Gemini CLI** (which handles the server and save hooks automatically) — see the [Gemini CLI Integration Guide](examples/gemini_cli_setup.md).
+
 ### With local models (Llama, Mistral, or any offline LLM)
 
 Local models generally don't speak MCP yet. Two approaches:
@@ -157,40 +218,40 @@ There are also **halls**, which connect rooms within a wing, and **tunnels**, wh
 You say what you're looking for and boom, it already knows which wing to go to. Just *that* in itself would have made a big difference. But this is beautiful, elegant, organic, and most importantly, efficient.
 
 ```
-  ┌─────────────────────────────────────────────────────────────┐
-  │  WING: Person                                              │
-  │                                                            │
-  │    ┌──────────┐  ──hall──  ┌──────────┐                    │
-  │    │  Room A  │            │  Room B  │                    │
-  │    └────┬─────┘            └──────────┘                    │
-  │         │                                                  │
-  │         ▼                                                  │
-  │    ┌──────────┐      ┌──────────┐                          │
-  │    │  Closet  │ ───▶ │  Drawer  │                          │
-  │    └──────────┘      └──────────┘                          │
-  └─────────┼──────────────────────────────────────────────────┘
-            │
+  +------------------------------------------------------------+
+  ¦  WING: Person                                              ¦
+  ¦                                                            ¦
+  ¦    +----------+            +----------+                    ¦
+  ¦    ¦  Room A  ¦  --hall--  ¦  Room B  ¦                    ¦
+  ¦    +----------+            +----------+                    ¦
+  ¦         ¦                                                  ¦
+  ¦         v                                                  ¦
+  ¦    +----------+      +----------+                          ¦
+  ¦    ¦  Closet  ¦ ---> ¦  Drawer  ¦                          ¦
+  ¦    +----------+      +----------+                          ¦
+  +---------+--------------------------------------------------+
+            ¦
           tunnel
-            │
-  ┌─────────┼──────────────────────────────────────────────────┐
-  │  WING: Project                                             │
-  │         │                                                  │
-  │    ┌────┴─────┐  ──hall──  ┌──────────┐                    │
-  │    │  Room A  │            │  Room C  │                    │
-  │    └────┬─────┘            └──────────┘                    │
-  │         │                                                  │
-  │         ▼                                                  │
-  │    ┌──────────┐      ┌──────────┐                          │
-  │    │  Closet  │ ───▶ │  Drawer  │                          │
-  │    └──────────┘      └──────────┘                          │
-  └─────────────────────────────────────────────────────────────┘
+            ¦
+  +---------+--------------------------------------------------+
+  ¦  WING: Project                                             ¦
+  ¦         ¦                                                  ¦
+  ¦    +----------+            +----------+                    ¦
+  ¦    ¦  Room A  ¦  --hall--  ¦  Room C  ¦                    ¦
+  ¦    +----------+            +----------+                    ¦
+  ¦         ¦                                                  ¦
+  ¦         v                                                  ¦
+  ¦    +----------+      +----------+                          ¦
+  ¦    ¦  Closet  ¦ ---> ¦  Drawer  ¦                          ¦
+  ¦    +----------+      +----------+                          ¦
+  +------------------------------------------------------------+
 ```
 
 **Wings** — a person or project. As many as you need.
 **Rooms** — specific topics within a wing. Auth, billing, deploy — endless rooms.
 **Halls** — connections between related rooms *within* the same wing. If Room A (auth) and Room B (security) are related, a hall links them.
 **Tunnels** — connections *between* wings. When Person A and a Project both have a room about "auth," a tunnel cross-references them automatically.
-**Closets** — compressed summaries that point to the original content. Fast for AI to read.
+**Closets** — summaries that point to the original content. (In v3.0.0 these are plain-text summaries; AAAK-encoded closets are coming in a future update — see [Task #30](https://github.com/milla-jovovich/mempalace/issues/30).)
 **Drawers** — the original verbatim files. The exact words, never summarized.
 
 **Halls** are memory types — the same in every wing, acting as corridors:
@@ -234,30 +295,23 @@ Wings and rooms aren't cosmetic. They're a **34% retrieval improvement**. The pa
 
 Your AI wakes up with L0 + L1 (~170 tokens) and knows your world. Searches only fire when needed.
 
-### AAAK Compression
+### AAAK Dialect (experimental)
 
-AAAK is a lossless dialect — 30x compression, readable by any LLM without a decoder. It works with **Claude, GPT, Gemini, Llama, Mistral** — any model that reads text. Run it against a local Llama model and your whole memory stack stays offline.
+AAAK is a lossy abbreviation system — entity codes, structural markers, and sentence truncation — designed to pack repeated entities and relationships into fewer tokens at scale. It is **readable by any LLM that reads text** (Claude, GPT, Gemini, Llama, Mistral) without a decoder, so a local model can use it without any cloud dependency.
 
-**English (~1000 tokens):**
-```
-Priya manages the Driftwood team: Kai (backend, 3 years), Soren (frontend),
-Maya (infrastructure), and Leo (junior, started last month). They're building
-a SaaS analytics platform. Current sprint: auth migration to Clerk.
-Kai recommended Clerk over Auth0 based on pricing and DX.
-```
+**Honest status (April 2026):**
 
-**AAAK (~120 tokens):**
-```
-TEAM: PRI(lead) | KAI(backend,3yr) SOR(frontend) MAY(infra) LEO(junior,new)
-PROJ: DRIFTWOOD(saas.analytics) | SPRINT: auth.migration→clerk
-DECISION: KAI.rec:clerk>auth0(pricing+dx) | ★★★★
-```
+- **AAAK is lossy, not lossless.** It uses regex-based abbreviation, not reversible compression.
+- **It does not save tokens at small scales.** Short text already tokenizes efficiently. AAAK overhead (codes, separators) costs more than it saves on a few sentences.
+- **It can save tokens at scale** — in scenarios with many repeated entities (a team mentioned hundreds of times, the same project across thousands of sessions), the entity codes amortize.
+- **AAAK currently regresses LongMemEval** vs raw verbatim retrieval (84.2% R@5 vs 96.6%). The 96.6% headline number is from **raw mode**, not AAAK mode.
+- **The MemPalace storage default is raw verbatim text in ChromaDB** — that's where the benchmark wins come from. AAAK is a separate compression layer for context loading, not the storage format.
 
-Same information. 8x fewer tokens. Your AI learns AAAK automatically from the MCP server — no manual setup.
+We're iterating on the dialect spec, adding a real tokenizer for stats, and exploring better break points for when to use it. Track progress in [Issue #43](https://github.com/milla-jovovich/mempalace/issues/43) and [#27](https://github.com/milla-jovovich/mempalace/issues/27).
 
-### Contradiction Detection
+### Contradiction Detection (experimental, not yet wired into KG)
 
-MemPalace catches mistakes before they reach you:
+A separate utility (`fact_checker.py`) can check assertions against entity facts. It's not currently called automatically by the knowledge graph operations — this is being fixed (track in [Issue #27](https://github.com/milla-jovovich/mempalace/issues/27)). When enabled it catches things like:
 
 ```
 Input:  "Soren finished the auth migration"
@@ -408,6 +462,11 @@ Letta charges $20–200/mo for agent-managed memory. MemPalace does it with a wi
 ## MCP Server
 
 ```bash
+# Via plugin (recommended)
+claude plugin marketplace add milla-jovovich/mempalace
+claude plugin install --scope user mempalace
+
+# Or manually
 claude mcp add mempalace -- python -m mempalace.mcp_server
 ```
 
@@ -478,6 +537,8 @@ Two hooks for Claude Code that automatically save memories during work:
 }
 ```
 
+**Optional auto-ingest:** Set the `MEMPAL_DIR` environment variable to a directory path and the hooks will automatically run `mempalace mine` on that directory during each save trigger (background on stop, synchronous on precompact).
+
 ---
 
 ## Benchmarks
@@ -536,6 +597,9 @@ mempalace compress --wing myapp                   # AAAK compress
 
 # Status
 mempalace status                                  # palace overview
+
+# MCP
+mempalace mcp                                     # show MCP setup command
 ```
 
 All commands accept `--palace <path>` to override the default location.
@@ -658,7 +722,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup and guidelines.
 MIT — see [LICENSE](LICENSE).
 
 <!-- Link Definitions -->
-[version-shield]: https://img.shields.io/badge/version-3.0.0-4dc9f6?style=flat-square&labelColor=0a0e14
+[version-shield]: https://img.shields.io/badge/version-3.2.0-4dc9f6?style=flat-square&labelColor=0a0e14
 [release-link]: https://github.com/milla-jovovich/mempalace/releases
 [python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8
 [python-link]: https://www.python.org/

ROADMAP.md

@@ -0,0 +1,74 @@
+# MemPalace Roadmap
+
+## v3.1.1 — Stability Patch (this week)
+
+Bug fixes and hardening merged to `develop`, releasing soon.
+
+**Merged:**
+- Security hardening: input validation, KG threading locks, WAL permission fixes (#647)
+- MCP tools: drawer CRUD, paginated export, hook settings (#667)
+- Backend storage seam: ChromaDB abstraction layer enabling swappable backends (#413)
+- MCP ping health check for AnythingLLM compatibility (#600)
+- Windows reparse point crash fix (#558)
+- `mempalace compress` KeyError crash fix (#569)
+- Token count estimate fix (#609)
+- Mtime float precision fix preventing unnecessary re-mines (#610)
+
+**In review (merging this week):**
+- Auto-repair BLOB seq_ids from chromadb 0.6→1.5 migration (#664)
+- Graph cache with write-invalidation (#661)
+- L1 importance pre-filter for large palaces (#660)
+- Windows Chinese/Unicode encoding fix (#631)
+- HNSW index bloat prevention — 441GB→433KB on large palaces (#346, pending rebase)
+- ~25 additional small bug fixes and platform compatibility patches
+
+## v4.0.0-alpha — Next Generation (this week)
+
+The v4 alpha introduces three major capabilities: pluggable storage backends, local NLP processing, and improved retrieval quality.
+
+### Swappable Storage
+
+ChromaDB remains the default, but v4 introduces a backend abstraction (shipped in #413) that enables drop-in replacements:
+
+- **PostgreSQL backend** with pg_sorted_heap support (#665) — for production deployments needing ACID guarantees, concurrent access, and standard backup/restore
+- **LanceDB backend** (#574) — for local-first deployments wanting multi-device sync without a database server
+- **PalaceStore** (#643) — bespoke storage layer purpose-built for MemPalace's access patterns (draft, evaluating)
+
+Users choose their backend at init time. Existing ChromaDB palaces continue to work unchanged.
+
+### Local NLP
+
+On-device natural language processing via local models (#507):
+
+- Entity extraction, relationship detection, and topic classification without external API calls
+- Feature-flagged and optional — falls back to existing heuristic extractors
+- Runs on consumer hardware (no GPU required, GPU-accelerated when available)
+
+### Improved Retrieval
+
+- **Hybrid search**: keyword text-match fallback when vector similarity misses exact terms (#662)
+- **Stale index detection**: automatic reconnection when the HNSW index changes on disk (#663)
+- **Time-decay scoring**: recent memories surface before older ones (#337)
+- **Query sanitization**: system prompt contamination mitigation already shipped in v3.1 (#385)
+
+### What's Not in v4 Alpha
+
+These are under consideration for v4 stable or later:
+
+- Synapse advanced retrieval — MMR, pinned memory, query expansion (#596)
+- Multi-device sync (#575) — depends on LanceDB backend
+- Multilingual embedding support (#488, #442)
+- Qdrant vector search backend (#381)
+
+## Branch Model
+
+```
+main            ← tagged production releases
+develop         ← active development (PRs merge here)
+release/3.1     ← hotfixes for current stable (v3.1.x)
+release/3.0     ← hotfixes for prior stable
+```
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. PRs should target `develop`. We review all contributions for correctness, security, and compatibility before merging.

benchmarks/HYBRID_MODE.md

@@ -435,7 +435,6 @@ If the API call fails (timeout, rate limit, no key), the function catches the ex
 **Key loading priority:**
 1. `--llm-key` CLI flag
 2. `ANTHROPIC_API_KEY` environment variable
-3. `~/.config/lu/keys.json` (checks `anthropic.lu_key` and similar paths)
 
 ## What Changed in the Code
 

benchmarks/convomem_bench.py

@@ -25,7 +25,6 @@ import os
 import sys
 import json
 import shutil
-import ssl
 import tempfile
 import argparse
 import urllib.request
@@ -35,9 +34,6 @@ from datetime import datetime
 
 import chromadb
 
-# Bypass SSL for restricted environments
-ssl._create_default_https_context = ssl._create_unverified_context
-
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 HF_BASE = "https://huggingface.co/datasets/Salesforce/ConvoMem/resolve/main/core_benchmark/evidence_questions"

benchmarks/locomo_bench.py

@@ -580,29 +580,12 @@ def llm_rerank_locomo(
 
 
 def _load_api_key(key_arg):
+    """Load API key from --llm-key arg or ANTHROPIC_API_KEY env var."""
     if key_arg:
         return key_arg
     env_key = os.environ.get("ANTHROPIC_API_KEY", "")
     if env_key:
         return env_key
-    keys_path = os.path.expanduser("~/.config/lu/keys.json")
-    if os.path.exists(keys_path):
-        try:
-            with open(keys_path) as f:
-                keys = json.load(f)
-            for name in ("lu_key", "anthropic_milla", "anthropic_claude_code_main"):
-                val = keys.get(name, "")
-                if isinstance(val, str) and val.startswith("sk-ant-"):
-                    return val
-            for section in ("anthropic", "anthropic_milla", "anthropic_claude_code_main"):
-                sec = keys.get(section, {})
-                if isinstance(sec, dict):
-                    for subkey in ("lu_key", "key", "api_key"):
-                        val = sec.get(subkey, "")
-                        if isinstance(val, str) and val.startswith("sk-ant-"):
-                            return val
-        except Exception:
-            pass
     return ""
 
 

benchmarks/longmemeval_bench.py

@@ -2861,32 +2861,12 @@ def llm_rerank(
 
 
 def _load_api_key(key_arg):
-    """Load API key from --llm-key arg, env var, or ~/.config/lu/keys.json."""
+    """Load API key from --llm-key arg or ANTHROPIC_API_KEY env var."""
     if key_arg:
         return key_arg
     env_key = os.environ.get("ANTHROPIC_API_KEY", "")
     if env_key:
         return env_key
-    keys_path = os.path.expanduser("~/.config/lu/keys.json")
-    if os.path.exists(keys_path):
-        try:
-            with open(keys_path) as f:
-                keys = json.load(f)
-            # Flat string keys
-            for name in ("lu_key", "anthropic_milla", "anthropic_claude_code_main"):
-                val = keys.get(name, "")
-                if isinstance(val, str) and val.startswith("sk-ant-"):
-                    return val
-            # Nested dict: keys["anthropic"]["lu_key"]
-            for section in ("anthropic", "anthropic_milla", "anthropic_claude_code_main"):
-                sec = keys.get(section, {})
-                if isinstance(sec, dict):
-                    for subkey in ("lu_key", "key", "api_key"):
-                        val = sec.get(subkey, "")
-                        if isinstance(val, str) and val.startswith("sk-ant-"):
-                            return val
-        except Exception:
-            pass
     return ""
 
 
@@ -2970,8 +2950,7 @@ def run_benchmark(
         if not api_key:
             print(
                 "ERROR: --llm-rerank / --mode diary requires an API key. "
-                "Set ANTHROPIC_API_KEY, use --llm-key, "
-                "or store in ~/.config/lu/keys.json as 'lu_key'."
+                "Set ANTHROPIC_API_KEY or use --llm-key."
             )
             sys.exit(1)
 
@@ -3290,8 +3269,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--llm-key",
         default="",
-        help="Anthropic API key for LLM re-ranking. Falls back to ANTHROPIC_API_KEY "
-        "env var or ~/.config/lu/keys.json 'lu_key' field if not provided.",
+        help="Anthropic API key for LLM re-ranking. Falls back to ANTHROPIC_API_KEY env var.",
     )
     parser.add_argument(
         "--llm-model",

docs/schema.sql

@@ -0,0 +1,36 @@
+-- MemPalace Knowledge Graph Schema
+-- SQLite database at ~/.mempalace/knowledge_graph.db
+
+CREATE TABLE IF NOT EXISTS entities (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    type TEXT DEFAULT 'unknown',
+    properties TEXT DEFAULT '{}'
+);
+
+CREATE TABLE IF NOT EXISTS triples (
+    id TEXT PRIMARY KEY,
+    subject TEXT NOT NULL,
+    predicate TEXT NOT NULL,
+    object TEXT NOT NULL,
+    valid_from TEXT,
+    valid_to TEXT,
+    confidence REAL DEFAULT 1.0,
+    source_closet TEXT,
+    source_file TEXT
+);
+
+CREATE TABLE IF NOT EXISTS attributes (
+    entity_id TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT,
+    valid_from TEXT,
+    valid_to TEXT,
+    PRIMARY KEY (entity_id, key, valid_from)
+);
+
+-- Indexes
+CREATE INDEX IF NOT EXISTS idx_triples_subject ON triples(subject);
+CREATE INDEX IF NOT EXISTS idx_triples_object ON triples(object);
+CREATE INDEX IF NOT EXISTS idx_triples_predicate ON triples(predicate);
+CREATE INDEX IF NOT EXISTS idx_triples_valid ON triples(valid_from, valid_to);

examples/HOOKS_TUTORIAL.md

@@ -0,0 +1,28 @@
+# How to Use MemPalace Hooks (Auto-Save)
+
+MemPalace hooks act as an "Auto-Save" feature. They help your AI keep a permanent memory without you needing to run manual commands.
+
+### 1. What are these hooks?
+* **Save Hook** (`mempal_save_hook.sh`): Saves new facts and decisions every 15 messages.
+* **PreCompact Hook** (`mempal_precompact_hook.sh`): Saves your context right before the AI's memory window fills up.
+
+### 2. Setup for Claude Code
+Add this to your configuration file to enable automatic background saving:
+
+```json
+{
+  "hooks": {
+    "Stop": [
+      {
+        "matcher": "", 
+        "hooks": [{"type": "command", "command": "./hooks/mempal_save_hook.sh"}]
+      }
+    ],
+    "PreCompact": [
+      {
+        "matcher": "", 
+        "hooks": [{"type": "command", "command": "./hooks/mempal_precompact_hook.sh"}]
+      }
+    ]
+  }
+}

examples/gemini_cli_setup.md

@@ -0,0 +1,95 @@
+# Gemini CLI Integration Guide
+
+This guide explains how to set up MemPalace as a permanent memory for the [Gemini CLI](https://github.com/google/gemini-cli).
+
+## Prerequisites
+
+- Python 3.9+
+- Gemini CLI installed and configured
+
+## 1. Installation
+
+On many Linux systems, installing Python packages globally is restricted. We recommend using a local virtual environment within the MemPalace directory.
+
+```bash
+# Clone the repository (if you haven't already)
+git clone https://github.com/MemPalace/mempalace.git
+cd mempalace
+
+# Create a virtual environment
+python3 -m venv .venv
+
+# Install dependencies and MemPalace in editable mode
+.venv/bin/pip install -e .
+```
+
+## 2. Initialization
+
+Set up your "Palace" (the database) and configure your identity.
+
+```bash
+# Initialize the palace in the current directory
+.venv/bin/python3 -m mempalace init .
+```
+
+### Identity and Wings (Optional but Recommended)
+You can manually define who you are and what projects you work on by creating/editing these files in `~/.mempalace/`:
+
+- **`~/.mempalace/identity.txt`**: A plain text file describing your role and focus.
+- **`~/.mempalace/wing_config.json`**: A JSON file mapping projects and name variants to "Wings".
+
+## 3. Connect to Gemini CLI (MCP)
+
+Register MemPalace as an MCP server so Gemini CLI can use its tools.
+
+```bash
+gemini mcp add mempalace /absolute/path/to/mempalace/.venv/bin/python3 -m mempalace.mcp_server --scope user
+```
+*Note: Use the absolute path to ensure it works from any directory.*
+
+## 4. Enable Auto-Saving (Hooks)
+
+To ensure the AI saves memories automatically when conversation history becomes too long, add a `PreCompress` hook to your Gemini CLI settings.
+
+Edit your `~/.gemini/settings.json` and add the following:
+
+```json
+{
+  "hooks": {
+    "PreCompress": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "/absolute/path/to/mempalace/hooks/mempal_precompact_hook.sh"
+          }
+        ]
+      }
+    ]
+  }
+}
+```
+
+Make sure the hook scripts are executable:
+```bash
+chmod +x hooks/*.sh
+```
+
+## 5. Usage
+
+Once connected, Gemini CLI will automatically:
+- Start the MemPalace server on launch.
+- Use `mempalace_search` to find relevant past discussions.
+- Use the `PreCompress` hook to save new memories before they are lost.
+
+### Manual Mining
+If you want the AI to learn from your existing code or docs immediately, run the "mine" command:
+```bash
+.venv/bin/python3 -m mempalace mine /path/to/your/project
+```
+
+### Verification
+In a Gemini CLI session, you can run:
+- `/mcp list`: Verify `mempalace` is `CONNECTED`.
+- `/hooks panel`: Verify the `PreCompress` hook is active.

examples/mcp_setup.md

@@ -5,20 +5,22 @@
 Run the MCP server:
 
 ```bash
-python mcp_server.py
+python -m mempalace.mcp_server
 ```
 
-Or add to Claude Code:
+Or add it to Claude Code:
 
 ```bash
-claude mcp add mempal -- python /path/to/mempalace/mcp_server.py
+claude mcp add mempalace -- python -m mempalace.mcp_server
 ```
 
 ## Available Tools
 
-- **mempal_status** — palace stats (wings, rooms, drawer counts)
-- **mempal_search** — semantic search across all memories
-- **mempal_list_wings** — list all projects in the palace
+The server exposes the full MemPalace MCP toolset. Common entry points include:
+
+- **mempalace_status** — palace stats (wings, rooms, drawer counts)
+- **mempalace_search** — semantic search across all memories
+- **mempalace_list_wings** — list all projects in the palace
 
 ## Usage in Claude Code
 

hooks/mempal_precompact_hook.sh

@@ -72,6 +72,6 @@ fi
 cat << 'HOOKJSON'
 {
   "decision": "block",
-  "reason": "COMPACTION IMMINENT. Save ALL topics, decisions, quotes, code, and important context from this session to your memory system. Be thorough — after compaction, detailed context will be lost. Organize into appropriate categories. Use verbatim quotes where possible. Save everything, then allow compaction to proceed."
+  "reason": "COMPACTION IMMINENT (MemPalace). Save ALL session content before context is lost:\n1. mempalace_diary_write — thorough AAAK-compressed session summary\n2. mempalace_add_drawer — ALL verbatim quotes, decisions, code, context\n3. mempalace_kg_add — entity relationships (optional)\nBe thorough — after compaction, detailed context will be lost. Do NOT write to Claude Code's native auto-memory (.md files). Save everything to MemPalace, then allow compaction to proceed."
 }
 HOOKJSON

hooks/mempal_save_hook.sh

@@ -64,10 +64,20 @@ MEMPAL_DIR=""
 # Read JSON input from stdin
 INPUT=$(cat)
 
-# Parse fields from Claude Code's JSON
-SESSION_ID=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('session_id','unknown'))" 2>/dev/null)
-STOP_HOOK_ACTIVE=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('stop_hook_active', False))" 2>/dev/null)
-TRANSCRIPT_PATH=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('transcript_path',''))" 2>/dev/null)
+# Parse all fields in a single Python call (3x faster than separate invocations)
+eval $(echo "$INPUT" | python3 -c "
+import sys, json
+data = json.load(sys.stdin)
+sid = data.get('session_id', 'unknown')
+sha = data.get('stop_hook_active', False)
+tp = data.get('transcript_path', '')
+# Shell-safe output — only allow alphanumeric, underscore, hyphen, slash, dot, tilde
+import re
+safe = lambda s: re.sub(r'[^a-zA-Z0-9_/.\-~]', '', str(s))
+print(f'SESSION_ID=\"{safe(sid)}\"')
+print(f'STOP_HOOK_ACTIVE=\"{sha}\"')
+print(f'TRANSCRIPT_PATH=\"{safe(tp)}\"')
+" 2>/dev/null)
 
 # Expand ~ in path
 TRANSCRIPT_PATH="${TRANSCRIPT_PATH/#\~/$HOME}"
@@ -80,25 +90,26 @@ if [ "$STOP_HOOK_ACTIVE" = "True" ] || [ "$STOP_HOOK_ACTIVE" = "true" ]; then
 fi
 
 # Count human messages in the JSONL transcript
+# SECURITY: Pass transcript path as sys.argv to avoid shell injection via crafted paths
 if [ -f "$TRANSCRIPT_PATH" ]; then
-    EXCHANGE_COUNT=$(python3 -c "
+    EXCHANGE_COUNT=$(python3 - "$TRANSCRIPT_PATH" <<'PYEOF'
 import json, sys
 count = 0
-with open('$TRANSCRIPT_PATH') as f:
+with open(sys.argv[1]) as f:
     for line in f:
         try:
             entry = json.loads(line)
             msg = entry.get('message', {})
             if isinstance(msg, dict) and msg.get('role') == 'user':
                 content = msg.get('content', '')
-                # Skip system/command messages — only count real human input
                 if isinstance(content, str) and '<command-message>' in content:
                     continue
                 count += 1
         except:
             pass
 print(count)
-" 2>/dev/null)
+PYEOF
+2>/dev/null)
 else
     EXCHANGE_COUNT=0
 fi
@@ -134,7 +145,7 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then
     cat << 'HOOKJSON'
 {
   "decision": "block",
-  "reason": "AUTO-SAVE checkpoint. Save key topics, decisions, quotes, and code from this session to your memory system. Organize into appropriate categories. Use verbatim quotes where possible. Continue conversation after saving."
+  "reason": "AUTO-SAVE checkpoint (MemPalace). Save this session's key content:\n1. mempalace_diary_write — AAAK-compressed session summary\n2. mempalace_add_drawer — verbatim quotes, decisions, code snippets\n3. mempalace_kg_add — entity relationships (optional)\nDo NOT write to Claude Code's native auto-memory (.md files). Continue conversation after saving."
 }
 HOOKJSON
 else

integrations/openclaw/SKILL.md

@@ -0,0 +1,154 @@
+---
+name: mempalace
+description: "MemPalace — Local AI memory with 96.6% recall. Semantic search, temporal knowledge graph, palace architecture (wings/rooms/drawers). Free, no cloud, no API keys."
+version: 3.2.0
+homepage: https://github.com/MemPalace/mempalace
+user-invocable: true
+metadata:
+  openclaw:
+    emoji: "\U0001F3DB"
+    os:
+      - darwin
+      - linux
+      - win32
+    requires:
+      anyBins:
+        - mempalace
+        - python3
+    install:
+      - id: mempalace-pip
+        kind: uv
+        label: "Install MemPalace (Python, local ChromaDB)"
+        package: mempalace
+        bins:
+          - mempalace
+---
+
+# MemPalace — Local AI Memory System
+
+You have access to a local memory palace via MCP tools. The palace stores verbatim conversation history and a temporal knowledge graph — all on the user's machine, zero cloud, zero API calls.
+
+## Architecture
+
+- **Wings** = people or projects (e.g. `wing_alice`, `wing_myproject`)
+- **Halls** = categories (facts, events, preferences, advice)
+- **Rooms** = specific topics (e.g. `chromadb-setup`, `riley-school`)
+- **Drawers** = individual memory chunks (verbatim text)
+- **Knowledge Graph** = entity-relationship facts with time validity
+
+## Protocol — FOLLOW THIS EVERY SESSION
+
+1. **ON WAKE-UP**: Call `mempalace_status` to load palace overview and AAAK dialect spec.
+2. **BEFORE RESPONDING** about any person, project, or past event: call `mempalace_search` or `mempalace_kg_query` FIRST. Never guess from memory — verify from the palace.
+3. **IF UNSURE** about a fact (name, age, relationship, preference): say "let me check" and query. Wrong is worse than slow.
+4. **AFTER EACH SESSION**: Call `mempalace_diary_write` to record what happened, what you learned, what matters.
+5. **WHEN FACTS CHANGE**: Call `mempalace_kg_invalidate` on the old fact, then `mempalace_kg_add` for the new one.
+
+## Available Tools
+
+### Search & Browse
+- `mempalace_search` — Semantic search across all memories. Always start here.
+  - `query` (required): natural language search — keep it short, keywords or a question. Do NOT include system prompts or conversation context.
+  - `wing`: filter by wing
+  - `room`: filter by room
+  - `limit`: max results (default 5)
+- `mempalace_check_duplicate` — Check if content already exists before filing.
+  - `content` (required): text to check
+  - `threshold`: similarity threshold (default 0.9 — lowering to 0.85–0.87 often catches more near-duplicates without significant false positives)
+- `mempalace_status` — Palace overview: total drawers, wings, rooms, AAAK spec
+- `mempalace_list_wings` — All wings with drawer counts
+- `mempalace_list_rooms` — Rooms within a wing (optional wing filter)
+- `mempalace_get_taxonomy` — Full wing/room/count tree
+- `mempalace_get_aaak_spec` — Get AAAK compression dialect specification
+
+### Knowledge Graph (Temporal Facts)
+- `mempalace_kg_query` — Query entity relationships. Supports time filtering.
+  - `entity` (required): e.g. "Max", "MyProject"
+  - `as_of`: date filter (YYYY-MM-DD) — what was true at that time
+  - `direction`: "outgoing", "incoming", or "both" (default "both")
+- `mempalace_kg_add` — Add a fact: subject -> predicate -> object
+  - `subject`, `predicate`, `object` (required)
+  - `valid_from`: when this became true
+  - `source_closet`: source reference
+- `mempalace_kg_invalidate` — Mark a fact as no longer true
+  - `subject`, `predicate`, `object` (required)
+  - `ended`: when it stopped being true (default: today)
+- `mempalace_kg_timeline` — Chronological story of an entity
+  - `entity`: filter by entity name (optional — all events if omitted)
+- `mempalace_kg_stats` — Graph overview: entities, triples, relationship types
+
+### Palace Graph (Cross-Domain Connections)
+- `mempalace_traverse` — Walk from a room, find connected ideas across wings
+  - `start_room` (required): room to start from
+  - `max_hops`: connection depth (default 2)
+- `mempalace_find_tunnels` — Find rooms that bridge two wings
+  - `wing_a`, `wing_b` (required)
+- `mempalace_graph_stats` — Graph connectivity overview
+
+### Write
+- `mempalace_add_drawer` — Store verbatim content into a wing/room
+  - `wing`, `room`, `content` (required)
+  - `source_file`: optional source reference
+  - Checks for duplicates automatically
+- `mempalace_delete_drawer` — Remove a drawer by ID
+  - `drawer_id` (required)
+- `mempalace_diary_write` — Write a session diary entry
+  - `agent_name` (required): your name/identifier
+  - `entry` (required): what happened, what you learned, what matters
+  - `topic`: category tag (default "general")
+- `mempalace_diary_read` — Read recent diary entries
+  - `agent_name` (required)
+  - `last_n`: number of entries (default 10)
+
+## Setup
+
+Install MemPalace and populate the palace:
+
+```bash
+pip install mempalace
+mempalace init ~/my-convos
+mempalace mine ~/my-convos
+```
+
+### OpenClaw MCP config
+
+Add to your OpenClaw MCP configuration:
+
+```json
+{
+  "mcpServers": {
+    "mempalace": {
+      "command": "python3",
+      "args": ["-m", "mempalace.mcp_server"]
+    }
+  }
+}
+```
+
+Or via CLI:
+
+```bash
+openclaw mcp set mempalace '{"command":"python3","args":["-m","mempalace.mcp_server"]}'
+```
+
+### Other MCP hosts
+
+```bash
+# Claude Code
+claude mcp add mempalace -- python -m mempalace.mcp_server
+
+# Cursor — add to .cursor/mcp.json
+# Codex — add to .codex/mcp.json
+```
+
+## Tips
+
+- Search is semantic (meaning-based), not keyword. "What did we discuss about database performance?" works better than "database".
+- The knowledge graph stores typed relationships with time windows. Use it for facts about people and projects — it knows WHEN things were true.
+- Diary entries accumulate across sessions. Write one at the end of each conversation to build continuity.
+- Use `mempalace_check_duplicate` before storing new content to avoid duplicates.
+- The AAAK dialect (from `mempalace_status`) is a compressed notation for efficient storage. Read it naturally — expand codes mentally, treat *markers* as emotional context.
+
+## License
+
+[MemPalace](https://github.com/MemPalace/mempalace) is MIT licensed. Created by Milla Jovovich, Ben Sigman, Igor Lins e Silva, and contributors.

mempalace/__init__.py

@@ -1,7 +1,28 @@
 """MemPalace — Give your AI a memory. No API key required."""
 
-__version__ = "2.0.0"
+import logging
 
-from .cli import main
+from .cli import main  # noqa: E402
+from .version import __version__  # noqa: E402
+
+# ChromaDB 0.6.x ships a Posthog telemetry client whose capture() signature is
+# incompatible with the bundled posthog library, producing noisy stderr warnings
+# on every client operation ("Failed to send telemetry event … capture() takes
+# 1 positional argument but 3 were given").  Silence just that logger.
+logging.getLogger("chromadb.telemetry.product.posthog").setLevel(logging.CRITICAL)
+
+# NOTE: the previous block set ``ORT_DISABLE_COREML=1`` on macOS arm64 as a
+# supposed workaround for the #74 ARM64 segfault.  Two problems:
+#
+# 1. ONNX Runtime does not read that env var -- it has no global way to
+#    disable a single execution provider, so the setdefault was a no-op.
+# 2. #74 is a null-pointer crash in ``chromadb_rust_bindings.abi3.so``, not
+#    an ONNX issue, so disabling CoreML would not have fixed it anyway.
+#
+# #521 has since traced the actual macOS arm64 crashes (both in mine and
+# search paths) to the 0.x chromadb hnswlib binding.  Filtering
+# CoreMLExecutionProvider at the ONNX layer leaves the hnswlib C++ crash
+# intact, so the real fix is upgrading chromadb to 1.5.4+, which #581
+# proposes.  See #397 for the history of this line.
 
 __all__ = ["main", "__version__"]

mempalace/backends/__init__.py

@@ -0,0 +1,6 @@
+"""Storage backend implementations for MemPalace."""
+
+from .base import BaseCollection
+from .chroma import ChromaBackend, ChromaCollection
+
+__all__ = ["BaseCollection", "ChromaBackend", "ChromaCollection"]

mempalace/backends/base.py

@@ -0,0 +1,44 @@
+"""Abstract collection interface for MemPalace storage backends."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+
+class BaseCollection(ABC):
+    """Smallest collection contract the rest of MemPalace relies on."""
+
+    @abstractmethod
+    def add(
+        self,
+        *,
+        documents: List[str],
+        ids: List[str],
+        metadatas: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def upsert(
+        self,
+        *,
+        documents: List[str],
+        ids: List[str],
+        metadatas: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def query(self, **kwargs: Any) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get(self, **kwargs: Any) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def delete(self, **kwargs: Any) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def count(self) -> int:
+        raise NotImplementedError

mempalace/backends/chroma.py

@@ -0,0 +1,91 @@
+"""ChromaDB-backed MemPalace collection adapter."""
+
+import logging
+import os
+import sqlite3
+
+import chromadb
+
+from .base import BaseCollection
+
+logger = logging.getLogger(__name__)
+
+
+def _fix_blob_seq_ids(palace_path: str):
+    """Fix ChromaDB 0.6.x -> 1.5.x migration bug: BLOB seq_ids -> INTEGER.
+
+    ChromaDB 0.6.x stored seq_id as big-endian 8-byte BLOBs. ChromaDB 1.5.x
+    expects INTEGER. The auto-migration doesn't convert existing rows, causing
+    the Rust compactor to crash with "mismatched types; Rust type u64 (as SQL
+    type INTEGER) is not compatible with SQL type BLOB".
+
+    Must run BEFORE PersistentClient is created (the compactor fires on init).
+    """
+    db_path = os.path.join(palace_path, "chroma.sqlite3")
+    if not os.path.isfile(db_path):
+        return
+    try:
+        with sqlite3.connect(db_path) as conn:
+            for table in ("embeddings", "max_seq_id"):
+                try:
+                    rows = conn.execute(
+                        f"SELECT rowid, seq_id FROM {table} WHERE typeof(seq_id) = 'blob'"
+                    ).fetchall()
+                except sqlite3.OperationalError:
+                    continue
+                if not rows:
+                    continue
+                updates = [(int.from_bytes(blob, byteorder="big"), rowid) for rowid, blob in rows]
+                conn.executemany(f"UPDATE {table} SET seq_id = ? WHERE rowid = ?", updates)
+                logger.info("Fixed %d BLOB seq_ids in %s", len(updates), table)
+            conn.commit()
+    except Exception:
+        logger.exception("Could not fix BLOB seq_ids in %s", db_path)
+
+
+class ChromaCollection(BaseCollection):
+    """Thin adapter over a ChromaDB collection."""
+
+    def __init__(self, collection):
+        self._collection = collection
+
+    def add(self, *, documents, ids, metadatas=None):
+        self._collection.add(documents=documents, ids=ids, metadatas=metadatas)
+
+    def upsert(self, *, documents, ids, metadatas=None):
+        self._collection.upsert(documents=documents, ids=ids, metadatas=metadatas)
+
+    def query(self, **kwargs):
+        return self._collection.query(**kwargs)
+
+    def get(self, **kwargs):
+        return self._collection.get(**kwargs)
+
+    def delete(self, **kwargs):
+        self._collection.delete(**kwargs)
+
+    def count(self):
+        return self._collection.count()
+
+
+class ChromaBackend:
+    """Factory for MemPalace's default ChromaDB backend."""
+
+    def get_collection(self, palace_path: str, collection_name: str, create: bool = False):
+        if not create and not os.path.isdir(palace_path):
+            raise FileNotFoundError(palace_path)
+
+        if create:
+            os.makedirs(palace_path, exist_ok=True)
+            try:
+                os.chmod(palace_path, 0o700)
+            except (OSError, NotImplementedError):
+                pass
+
+        _fix_blob_seq_ids(palace_path)
+        client = chromadb.PersistentClient(path=palace_path)
+        if create:
+            collection = client.get_or_create_collection(collection_name)
+        else:
+            collection = client.get_collection(collection_name)
+        return ChromaCollection(collection)

mempalace/cli.py

@@ -14,6 +14,7 @@ Commands:
     mempalace mine <dir>                  Mine project files (default)
     mempalace mine <dir> --mode convos    Mine conversation exports
     mempalace search "query"              Find anything, exact words
+    mempalace mcp                         Show MCP setup command
     mempalace wake-up                     Show L0 + L1 wake-up context
     mempalace wake-up --wing my_app       Wake-up for a specific project
     mempalace status                      Show what's been filed
@@ -28,6 +29,7 @@ Examples:
 
 import os
 import sys
+import shlex
 import argparse
 from pathlib import Path
 
@@ -59,12 +61,15 @@ def cmd_init(args):
             print("  No entities detected — proceeding with directory-based rooms.")
 
     # Pass 2: detect rooms from folder structure
-    detect_rooms_local(project_dir=args.dir)
+    detect_rooms_local(project_dir=args.dir, yes=getattr(args, "yes", False))
     MempalaceConfig().init()
 
 
 def cmd_mine(args):
     palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
+    include_ignored = []
+    for raw in args.include_ignored or []:
+        include_ignored.extend(part.strip() for part in raw.split(",") if part.strip())
 
     if args.mode == "convos":
         from .convo_miner import mine_convos
@@ -88,20 +93,25 @@ def cmd_mine(args):
             agent=args.agent,
             limit=args.limit,
             dry_run=args.dry_run,
+            respect_gitignore=not args.no_gitignore,
+            include_ignored=include_ignored,
         )
 
 
 def cmd_search(args):
-    from .searcher import search
+    from .searcher import search, SearchError
 
     palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
-    search(
-        query=args.query,
-        palace_path=palace_path,
-        wing=args.wing,
-        room=args.room,
-        n_results=args.results,
-    )
+    try:
+        search(
+            query=args.query,
+            palace_path=palace_path,
+            wing=args.wing,
+            room=args.room,
+            n_results=args.results,
+        )
+    except SearchError:
+        sys.exit(1)
 
 
 def cmd_wakeup(args):
@@ -124,7 +134,8 @@ def cmd_split(args):
     import sys
 
     # Rebuild argv for split_mega_files argparse
-    argv = [args.dir]
+    # Expand ~ and resolve to absolute path so split_mega_files sees a real path
+    argv = ["--source", str(Path(args.dir).expanduser().resolve())]
     if args.output_dir:
         argv += ["--output-dir", args.output_dir]
     if args.dry_run:
@@ -140,6 +151,14 @@ def cmd_split(args):
         sys.argv = old_argv
 
 
+def cmd_migrate(args):
+    """Migrate palace from a different ChromaDB version."""
+    from .migrate import migrate
+
+    palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
+    migrate(palace_path=palace_path, dry_run=args.dry_run, confirm=getattr(args, "yes", False))
+
+
 def cmd_status(args):
     from .miner import status
 
@@ -147,6 +166,131 @@ def cmd_status(args):
     status(palace_path=palace_path)
 
 
+def cmd_repair(args):
+    """Rebuild palace vector index from SQLite metadata."""
+    import chromadb
+    import shutil
+    from .migrate import confirm_destructive_action, contains_palace_database
+
+    palace_path = os.path.abspath(
+        os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
+    )
+    db_path = os.path.join(palace_path, "chroma.sqlite3")
+
+    if not os.path.isdir(palace_path):
+        print(f"\n  No palace found at {palace_path}")
+        return
+    if not contains_palace_database(palace_path):
+        print(f"\n  No palace database found at {db_path}")
+        return
+
+    print(f"\n{'=' * 55}")
+    print("  MemPalace Repair")
+    print(f"{'=' * 55}\n")
+    print(f"  Palace: {palace_path}")
+
+    # Try to read existing drawers
+    try:
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        total = col.count()
+        print(f"  Drawers found: {total}")
+    except Exception as e:
+        print(f"  Error reading palace: {e}")
+        print("  Cannot recover — palace may need to be re-mined from source files.")
+        return
+
+    if total == 0:
+        print("  Nothing to repair.")
+        return
+
+    if not confirm_destructive_action(
+        "Repair", palace_path, assume_yes=getattr(args, "yes", False)
+    ):
+        return
+
+    # Extract all drawers in batches
+    print("\n  Extracting drawers...")
+    batch_size = 5000
+    all_ids = []
+    all_docs = []
+    all_metas = []
+    offset = 0
+    while offset < total:
+        batch = col.get(limit=batch_size, offset=offset, include=["documents", "metadatas"])
+        all_ids.extend(batch["ids"])
+        all_docs.extend(batch["documents"])
+        all_metas.extend(batch["metadatas"])
+        offset += batch_size
+    print(f"  Extracted {len(all_ids)} drawers")
+
+    # Backup and rebuild
+    palace_path = os.path.normpath(palace_path)
+    backup_path = palace_path + ".backup"
+    if os.path.exists(backup_path):
+        if not contains_palace_database(backup_path):
+            print(
+                "  Backup validation failed: backup path exists but does not contain chroma.sqlite3. "
+                f"Please remove or rename: {backup_path}"
+            )
+            return
+        shutil.rmtree(backup_path)
+    print(f"  Backing up to {backup_path}...")
+    shutil.copytree(palace_path, backup_path)
+
+    print("  Rebuilding collection...")
+    client.delete_collection("mempalace_drawers")
+    new_col = client.create_collection("mempalace_drawers")
+
+    filed = 0
+    for i in range(0, len(all_ids), batch_size):
+        batch_ids = all_ids[i : i + batch_size]
+        batch_docs = all_docs[i : i + batch_size]
+        batch_metas = all_metas[i : i + batch_size]
+        new_col.add(documents=batch_docs, ids=batch_ids, metadatas=batch_metas)
+        filed += len(batch_ids)
+        print(f"  Re-filed {filed}/{len(all_ids)} drawers...")
+
+    print(f"\n  Repair complete. {filed} drawers rebuilt.")
+    print(f"  Backup saved at {backup_path}")
+    print(f"\n{'=' * 55}\n")
+
+
+def cmd_hook(args):
+    """Run hook logic: reads JSON from stdin, outputs JSON to stdout."""
+    from .hooks_cli import run_hook
+
+    run_hook(hook_name=args.hook, harness=args.harness)
+
+
+def cmd_instructions(args):
+    """Output skill instructions to stdout."""
+    from .instructions_cli import run_instructions
+
+    run_instructions(name=args.name)
+
+
+def cmd_mcp(args):
+    """Show how to wire MemPalace into MCP-capable hosts."""
+    base_server_cmd = "python -m mempalace.mcp_server"
+
+    if args.palace:
+        resolved_palace = str(Path(args.palace).expanduser())
+        server_cmd = f"{base_server_cmd} --palace {shlex.quote(resolved_palace)}"
+    else:
+        server_cmd = base_server_cmd
+
+    print("MemPalace MCP quick setup:")
+    print(f"  claude mcp add mempalace -- {server_cmd}")
+    print("\nRun the server directly:")
+    print(f"  {server_cmd}")
+
+    if not args.palace:
+        print("\nOptional custom palace:")
+        print(f"  claude mcp add mempalace -- {base_server_cmd} --palace /path/to/palace")
+        print(f"  {base_server_cmd} --palace /path/to/palace")
+
+
 def cmd_compress(args):
     """Compress drawers in a wing using AAAK Dialect."""
     import chromadb
@@ -177,20 +321,31 @@ def cmd_compress(args):
         print("  Run: mempalace init <dir> then mempalace mine <dir>")
         sys.exit(1)
 
-    # Query drawers in the wing
+    # Query drawers in batches to avoid SQLite variable limit (~999)
     where = {"wing": args.wing} if args.wing else None
-    try:
-        kwargs = {"include": ["documents", "metadatas"]}
-        if where:
-            kwargs["where"] = where
-        results = col.get(**kwargs)
-    except Exception as e:
-        print(f"\n  Error reading drawers: {e}")
-        sys.exit(1)
-
-    docs = results["documents"]
-    metas = results["metadatas"]
-    ids = results["ids"]
+    _BATCH = 500
+    docs, metas, ids = [], [], []
+    offset = 0
+    while True:
+        try:
+            kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset}
+            if where:
+                kwargs["where"] = where
+            batch = col.get(**kwargs)
+        except Exception as e:
+            if not docs:
+                print(f"\n  Error reading drawers: {e}")
+                sys.exit(1)
+            break
+        batch_docs = batch.get("documents", [])
+        if not batch_docs:
+            break
+        docs.extend(batch_docs)
+        metas.extend(batch.get("metadatas", []))
+        ids.extend(batch.get("ids", []))
+        offset += len(batch_docs)
+        if len(batch_docs) < _BATCH:
+            break
 
     if not docs:
         wing_label = f" in wing '{args.wing}'" if args.wing else ""
@@ -213,7 +368,7 @@ def cmd_compress(args):
         stats = dialect.compression_stats(doc, compressed)
 
         total_original += stats["original_chars"]
-        total_compressed += stats["compressed_chars"]
+        total_compressed += stats["summary_chars"]
 
         compressed_entries.append((doc_id, compressed, meta, stats))
 
@@ -223,7 +378,7 @@ def cmd_compress(args):
             source = Path(meta.get("source_file", "?")).name
             print(f"  [{wing_name}/{room_name}] {source}")
             print(
-                f"    {stats['original_tokens']}t -> {stats['compressed_tokens']}t ({stats['ratio']:.1f}x)"
+                f"    {stats['original_tokens_est']}t -> {stats['summary_tokens_est']}t ({stats['size_ratio']:.1f}x)"
             )
             print(f"    {compressed}")
             print()
@@ -234,8 +389,8 @@ def cmd_compress(args):
             comp_col = client.get_or_create_collection("mempalace_compressed")
             for doc_id, compressed, meta, stats in compressed_entries:
                 comp_meta = dict(meta)
-                comp_meta["compression_ratio"] = round(stats["ratio"], 1)
-                comp_meta["original_tokens"] = stats["original_tokens"]
+                comp_meta["compression_ratio"] = round(stats["size_ratio"], 1)
+                comp_meta["original_tokens"] = stats["original_tokens_est"]
                 comp_col.upsert(
                     ids=[doc_id],
                     documents=[compressed],
@@ -250,8 +405,9 @@ def cmd_compress(args):
 
     # Summary
     ratio = total_original / max(total_compressed, 1)
-    orig_tokens = Dialect.count_tokens("x" * total_original)
-    comp_tokens = Dialect.count_tokens("x" * total_compressed)
+    # Estimate tokens from char count (~3.8 chars/token for English text)
+    orig_tokens = max(1, int(total_original / 3.8))
+    comp_tokens = max(1, int(total_compressed / 3.8))
     print(f"  Total: {orig_tokens:,}t -> {comp_tokens:,}t ({ratio:.1f}x compression)")
     if args.dry_run:
         print("  (dry run -- nothing stored)")
@@ -288,6 +444,17 @@ def main():
         help="Ingest mode: 'projects' for code/docs (default), 'convos' for chat exports",
     )
     p_mine.add_argument("--wing", default=None, help="Wing name (default: directory name)")
+    p_mine.add_argument(
+        "--no-gitignore",
+        action="store_true",
+        help="Don't respect .gitignore files when scanning project files",
+    )
+    p_mine.add_argument(
+        "--include-ignored",
+        action="append",
+        default=[],
+        help="Always scan these project-relative paths even if ignored; repeat or pass comma-separated paths",
+    )
     p_mine.add_argument(
         "--agent",
         default="mempalace",
@@ -350,7 +517,62 @@ def main():
         help="Only split files containing at least N sessions (default: 2)",
     )
 
+    # hook
+    p_hook = sub.add_parser(
+        "hook",
+        help="Run hook logic (reads JSON from stdin, outputs JSON to stdout)",
+    )
+    hook_sub = p_hook.add_subparsers(dest="hook_action")
+    p_hook_run = hook_sub.add_parser("run", help="Execute a hook")
+    p_hook_run.add_argument(
+        "--hook",
+        required=True,
+        choices=["session-start", "stop", "precompact"],
+        help="Hook name to run",
+    )
+    p_hook_run.add_argument(
+        "--harness",
+        required=True,
+        choices=["claude-code", "codex"],
+        help="Harness type (determines stdin JSON format)",
+    )
+
+    # instructions
+    p_instructions = sub.add_parser(
+        "instructions",
+        help="Output skill instructions to stdout",
+    )
+    instructions_sub = p_instructions.add_subparsers(dest="instructions_name")
+    for instr_name in ["init", "search", "mine", "help", "status"]:
+        instructions_sub.add_parser(instr_name, help=f"Output {instr_name} instructions")
+
+    # repair
+    sub.add_parser(
+        "repair",
+        help="Rebuild palace vector index from stored data (fixes segfaults after corruption)",
+    ).add_argument("--yes", action="store_true", help="Skip confirmation for destructive changes")
+
+    # mcp
+    sub.add_parser(
+        "mcp",
+        help="Show MCP setup command for connecting MemPalace to your AI client",
+    )
+
     # status
+    # migrate
+    p_migrate = sub.add_parser(
+        "migrate",
+        help="Migrate palace from a different ChromaDB version (fixes 3.0.0 → 3.1.0 upgrade)",
+    )
+    p_migrate.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would be migrated without changing anything",
+    )
+    p_migrate.add_argument(
+        "--yes", action="store_true", help="Skip confirmation for destructive changes"
+    )
+
     sub.add_parser("status", help="Show what's been filed")
 
     args = parser.parse_args()
@@ -359,13 +581,33 @@ def main():
         parser.print_help()
         return
 
+    # Handle two-level subcommands
+    if args.command == "hook":
+        if not getattr(args, "hook_action", None):
+            p_hook.print_help()
+            return
+        cmd_hook(args)
+        return
+
+    if args.command == "instructions":
+        name = getattr(args, "instructions_name", None)
+        if not name:
+            p_instructions.print_help()
+            return
+        args.name = name
+        cmd_instructions(args)
+        return
+
     dispatch = {
         "init": cmd_init,
         "mine": cmd_mine,
         "split": cmd_split,
         "search": cmd_search,
+        "mcp": cmd_mcp,
         "compress": cmd_compress,
         "wake-up": cmd_wakeup,
+        "repair": cmd_repair,
+        "migrate": cmd_migrate,
         "status": cmd_status,
     }
     dispatch[args.command](args)

mempalace/config.py

@@ -6,8 +6,58 @@ Priority: env vars > config file (~/.mempalace/config.json) > defaults
 
 import json
 import os
+import re
 from pathlib import Path
 
+
+# ── Input validation ──────────────────────────────────────────────────────────
+# Shared sanitizers for wing/room/entity names. Prevents path traversal,
+# excessively long strings, and special characters that could cause issues
+# in file paths, SQLite, or ChromaDB metadata.
+
+MAX_NAME_LENGTH = 128
+_SAFE_NAME_RE = re.compile(r"^(?:[^\W_]|[^\W_][\w .'-]{0,126}[^\W_])$")
+
+
+def sanitize_name(value: str, field_name: str = "name") -> str:
+    """Validate and sanitize a wing/room/entity name.
+
+    Raises ValueError if the name is invalid.
+    """
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{field_name} must be a non-empty string")
+
+    value = value.strip()
+
+    if len(value) > MAX_NAME_LENGTH:
+        raise ValueError(f"{field_name} exceeds maximum length of {MAX_NAME_LENGTH} characters")
+
+    # Block path traversal
+    if ".." in value or "/" in value or "\\" in value:
+        raise ValueError(f"{field_name} contains invalid path characters")
+
+    # Block null bytes
+    if "\x00" in value:
+        raise ValueError(f"{field_name} contains null bytes")
+
+    # Enforce safe character set
+    if not _SAFE_NAME_RE.match(value):
+        raise ValueError(f"{field_name} contains invalid characters")
+
+    return value
+
+
+def sanitize_content(value: str, max_length: int = 100_000) -> str:
+    """Validate drawer/diary content length."""
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("content must be a non-empty string")
+    if len(value) > max_length:
+        raise ValueError(f"content exceeds maximum length of {max_length} characters")
+    if "\x00" in value:
+        raise ValueError("content contains null bytes")
+    return value
+
+
 DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
 DEFAULT_COLLECTION_NAME = "mempalace_drawers"
 
@@ -123,9 +173,35 @@ class MempalaceConfig:
         """Mapping of hall names to keyword lists."""
         return self._file_config.get("hall_keywords", DEFAULT_HALL_KEYWORDS)
 
+    @property
+    def hook_silent_save(self):
+        """Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""
+        return self._file_config.get("hooks", {}).get("silent_save", True)
+
+    @property
+    def hook_desktop_toast(self):
+        """Whether the stop hook shows a desktop notification via notify-send."""
+        return self._file_config.get("hooks", {}).get("desktop_toast", False)
+
+    def set_hook_setting(self, key: str, value: bool):
+        """Update a hook setting and write config to disk."""
+        if "hooks" not in self._file_config:
+            self._file_config["hooks"] = {}
+        self._file_config["hooks"][key] = value
+        try:
+            with open(self._config_file, "w", encoding="utf-8") as f:
+                json.dump(self._file_config, f, indent=2, ensure_ascii=False)
+        except OSError:
+            pass
+
     def init(self):
         """Create config directory and write default config.json if it doesn't exist."""
         self._config_dir.mkdir(parents=True, exist_ok=True)
+        # Restrict directory permissions to owner only (Unix)
+        try:
+            self._config_dir.chmod(0o700)
+        except (OSError, NotImplementedError):
+            pass  # Windows doesn't support Unix permissions
         if not self._config_file.exists():
             default_config = {
                 "palace_path": DEFAULT_PALACE_PATH,
@@ -135,6 +211,11 @@ class MempalaceConfig:
             }
             with open(self._config_file, "w") as f:
                 json.dump(default_config, f, indent=2)
+            # Restrict config file to owner read/write only
+            try:
+                self._config_file.chmod(0o600)
+            except (OSError, NotImplementedError):
+                pass
         return self._config_file
 
     def save_people_map(self, people_map):

mempalace/convo_miner.py

@@ -15,9 +15,8 @@ from pathlib import Path
 from datetime import datetime
 from collections import defaultdict
 
-import chromadb
-
 from .normalize import normalize
+from .palace import SKIP_DIRS, get_collection, file_already_mined
 
 
 # File types that might contain conversations
@@ -28,20 +27,33 @@ CONVO_EXTENSIONS = {
     ".jsonl",
 }
 
-SKIP_DIRS = {
-    ".git",
-    "node_modules",
-    "__pycache__",
-    ".venv",
-    "venv",
-    "env",
-    "dist",
-    "build",
-    ".next",
-    ".mempalace",
-}
-
 MIN_CHUNK_SIZE = 30
+CHUNK_SIZE = 800  # chars per drawer — align with miner.py
+MAX_FILE_SIZE = 10 * 1024 * 1024  # 10 MB — skip files larger than this
+
+
+def _register_file(collection, source_file: str, wing: str, agent: str):
+    """Write a sentinel so file_already_mined() returns True for 0-chunk files.
+
+    Without this, files that normalize to nothing or produce zero chunks are
+    re-read and re-processed on every mine run because nothing was written to
+    ChromaDB on the first pass.
+    """
+    sentinel_id = f"_reg_{hashlib.sha256(source_file.encode()).hexdigest()[:24]}"
+    collection.upsert(
+        documents=[f"[registry] {source_file}"],
+        ids=[sentinel_id],
+        metadatas=[
+            {
+                "wing": wing,
+                "room": "_registry",
+                "source_file": source_file,
+                "added_by": agent,
+                "filed_at": datetime.now().isoformat(),
+                "ingest_mode": "registry",
+            }
+        ],
+    )
 
 
 # =============================================================================
@@ -64,7 +76,12 @@ def chunk_exchanges(content: str) -> list:
 
 
 def _chunk_by_exchange(lines: list) -> list:
-    """One user turn (>) + the AI response that follows = one chunk."""
+    """One user turn (>) + the AI response that follows = one or more chunks.
+
+    The full AI response is preserved verbatim.  When the combined
+    user-turn + response exceeds CHUNK_SIZE the response is split across
+    consecutive drawers so nothing is silently discarded.
+    """
     chunks = []
     i = 0
 
@@ -83,10 +100,23 @@ def _chunk_by_exchange(lines: list) -> list:
                     ai_lines.append(next_line.strip())
                 i += 1
 
-            ai_response = " ".join(ai_lines[:8])
+            ai_response = " ".join(ai_lines)
             content = f"{user_turn}\n{ai_response}" if ai_response else user_turn
 
-            if len(content.strip()) > MIN_CHUNK_SIZE:
+            # Split into multiple drawers when the exchange exceeds CHUNK_SIZE
+            if len(content) > CHUNK_SIZE:
+                # First chunk: user turn + as much response as fits
+                first_part = content[:CHUNK_SIZE]
+                if len(first_part.strip()) > MIN_CHUNK_SIZE:
+                    chunks.append({"content": first_part, "chunk_index": len(chunks)})
+                # Remaining response in CHUNK_SIZE-sized continuation drawers
+                remainder = content[CHUNK_SIZE:]
+                while remainder:
+                    part = remainder[:CHUNK_SIZE]
+                    remainder = remainder[CHUNK_SIZE:]
+                    if len(part.strip()) > MIN_CHUNK_SIZE:
+                        chunks.append({"content": part, "chunk_index": len(chunks)})
+            elif len(content.strip()) > MIN_CHUNK_SIZE:
                 chunks.append(
                     {
                         "content": content,
@@ -209,23 +239,6 @@ def detect_convo_room(content: str) -> str:
 # =============================================================================
 
 
-def get_collection(palace_path: str):
-    os.makedirs(palace_path, exist_ok=True)
-    client = chromadb.PersistentClient(path=palace_path)
-    try:
-        return client.get_collection("mempalace_drawers")
-    except Exception:
-        return client.create_collection("mempalace_drawers")
-
-
-def file_already_mined(collection, source_file: str) -> bool:
-    try:
-        results = collection.get(where={"source_file": source_file}, limit=1)
-        return len(results.get("ids", [])) > 0
-    except Exception:
-        return False
-
-
 # =============================================================================
 # SCAN FOR CONVERSATION FILES
 # =============================================================================
@@ -238,8 +251,18 @@ def scan_convos(convo_dir: str) -> list:
     for root, dirs, filenames in os.walk(convo_path):
         dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
         for filename in filenames:
+            if filename.endswith(".meta.json"):
+                continue
             filepath = Path(root) / filename
             if filepath.suffix.lower() in CONVO_EXTENSIONS:
+                # Skip symlinks and oversized files
+                if filepath.is_symlink():
+                    continue
+                try:
+                    if filepath.stat().st_size > MAX_FILE_SIZE:
+                        continue
+                except OSError:
+                    continue
                 files.append(filepath)
     return files
 
@@ -282,7 +305,7 @@ def mine_convos(
     print(f"  Palace:  {palace_path}")
     if dry_run:
         print("  DRY RUN — nothing will be filed")
-    print(f"{'─' * 55}\n")
+    print(f"{'-' * 55}\n")
 
     collection = get_collection(palace_path) if not dry_run else None
 
@@ -301,10 +324,14 @@ def mine_convos(
         # Normalize format
         try:
             content = normalize(str(filepath))
-        except Exception:
+        except (OSError, ValueError):
+            if not dry_run:
+                _register_file(collection, source_file, wing, agent)
             continue
 
         if not content or len(content.strip()) < MIN_CHUNK_SIZE:
+            if not dry_run:
+                _register_file(collection, source_file, wing, agent)
             continue
 
         # Chunk — either exchange pairs or general extraction
@@ -317,6 +344,8 @@ def mine_convos(
             chunks = chunk_exchanges(content)
 
         if not chunks:
+            if not dry_run:
+                _register_file(collection, source_file, wing, agent)
             continue
 
         # Detect room from content (general mode uses memory_type instead)
@@ -352,9 +381,9 @@ def mine_convos(
             chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room
             if extract_mode == "general":
                 room_counts[chunk_room] += 1
-            drawer_id = f"drawer_{wing}_{chunk_room}_{hashlib.md5((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:16]}"
+            drawer_id = f"drawer_{wing}_{chunk_room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}"
             try:
-                collection.add(
+                collection.upsert(
                     documents=[chunk["content"]],
                     ids=[drawer_id],
                     metadatas=[

mempalace/dedup.py

@@ -0,0 +1,239 @@
+"""
+dedup.py — Detect and remove near-duplicate drawers
+====================================================
+
+When the same files are mined multiple times, near-identical drawers
+accumulate. This module finds drawers from the same source_file that
+are too similar (cosine distance < threshold), keeps the longest/richest
+version, and deletes the rest.
+
+No API calls — uses ChromaDB's built-in embedding similarity.
+
+Usage (standalone):
+    python -m mempalace.dedup                          # dedup all
+    python -m mempalace.dedup --dry-run                # preview only
+    python -m mempalace.dedup --threshold 0.10         # stricter (near-identical only)
+    python -m mempalace.dedup --threshold 0.35         # looser (catches paraphrased content)
+    python -m mempalace.dedup --wing my_project        # scope to one wing
+    python -m mempalace.dedup --stats                  # stats only
+    python -m mempalace.dedup --source "my_project"    # filter by source
+
+Usage (from CLI):
+    mempalace dedup [--dry-run] [--threshold 0.15] [--stats]
+"""
+
+import argparse
+import os
+import time
+from collections import defaultdict
+
+import chromadb
+
+
+COLLECTION_NAME = "mempalace_drawers"
+# Cosine DISTANCE threshold (not similarity). Lower = stricter.
+# 0.15 = ~85% cosine similarity — catches near-identical chunks.
+# For looser dedup of paraphrased content, try 0.3–0.4.
+DEFAULT_THRESHOLD = 0.15
+MIN_DRAWERS_TO_CHECK = 5
+
+
+def _get_palace_path():
+    """Resolve palace path from config."""
+    try:
+        from .config import MempalaceConfig
+
+        return MempalaceConfig().palace_path
+    except Exception:
+        return os.path.join(os.path.expanduser("~"), ".mempalace", "palace")
+
+
+def get_source_groups(col, min_count=MIN_DRAWERS_TO_CHECK, source_pattern=None, wing=None):
+    """Group drawers by source_file, return groups with min_count+ entries.
+
+    If wing is specified, only considers drawers in that wing. This catches
+    cross-wing duplicates when the same source was mined into multiple wings.
+    """
+    total = col.count()
+    groups = defaultdict(list)
+
+    offset = 0
+    batch_size = 1000
+    while offset < total:
+        kwargs = {"limit": batch_size, "offset": offset, "include": ["metadatas"]}
+        if wing:
+            kwargs["where"] = {"wing": wing}
+        batch = col.get(**kwargs)
+        if not batch["ids"]:
+            break
+        for did, meta in zip(batch["ids"], batch["metadatas"]):
+            src = meta.get("source_file", "unknown")
+            if source_pattern and source_pattern.lower() not in src.lower():
+                continue
+            groups[src].append(did)
+        offset += len(batch["ids"])
+
+    return {src: ids for src, ids in groups.items() if len(ids) >= min_count}
+
+
+def dedup_source_group(col, drawer_ids, threshold=DEFAULT_THRESHOLD, dry_run=True):
+    """Dedup drawers within one source_file group.
+
+    Greedy: sort by doc length (longest first), keep if not too similar
+    to any already-kept drawer. Returns (kept_ids, deleted_ids).
+    """
+    data = col.get(ids=drawer_ids, include=["documents", "metadatas"])
+    items = list(zip(data["ids"], data["documents"], data["metadatas"]))
+    items.sort(key=lambda x: len(x[1] or ""), reverse=True)
+
+    kept = []
+    to_delete = []
+
+    for did, doc, meta in items:
+        if not doc or len(doc) < 20:
+            to_delete.append(did)
+            continue
+
+        if not kept:
+            kept.append((did, doc))
+            continue
+
+        try:
+            results = col.query(
+                query_texts=[doc],
+                n_results=min(len(kept), 5),
+                include=["distances"],
+            )
+            dists = results["distances"][0] if results["distances"] else []
+            kept_ids_set = {k[0] for k in kept}
+
+            is_dup = False
+            for rid, dist in zip(results["ids"][0], dists):
+                if rid in kept_ids_set and dist < threshold:
+                    is_dup = True
+                    break
+
+            if is_dup:
+                to_delete.append(did)
+            else:
+                kept.append((did, doc))
+        except Exception:
+            kept.append((did, doc))
+
+    if to_delete and not dry_run:
+        for i in range(0, len(to_delete), 500):
+            col.delete(ids=to_delete[i : i + 500])
+
+    return [k[0] for k in kept], to_delete
+
+
+def show_stats(palace_path=None):
+    """Show duplication statistics without making changes."""
+    palace_path = palace_path or _get_palace_path()
+    client = chromadb.PersistentClient(path=palace_path)
+    col = client.get_collection(COLLECTION_NAME)
+
+    groups = get_source_groups(col)
+
+    total_drawers = sum(len(ids) for ids in groups.values())
+    print(f"\n  Sources with {MIN_DRAWERS_TO_CHECK}+ drawers: {len(groups)}")
+    print(f"  Total drawers in those sources: {total_drawers:,}")
+
+    print("\n  Top 15 by drawer count:")
+    sorted_groups = sorted(groups.items(), key=lambda x: len(x[1]), reverse=True)
+    for src, ids in sorted_groups[:15]:
+        print(f"    {len(ids):4d}  {src[:65]}")
+
+    estimated_dups = sum(int(len(ids) * 0.4) for ids in groups.values() if len(ids) > 20)
+    print(f"\n  Estimated duplicates (groups > 20): ~{estimated_dups:,}")
+
+
+def dedup_palace(
+    palace_path=None,
+    threshold=DEFAULT_THRESHOLD,
+    dry_run=True,
+    source_pattern=None,
+    min_count=MIN_DRAWERS_TO_CHECK,
+    wing=None,
+):
+    """Main entry point: deduplicate near-identical drawers across the palace."""
+    palace_path = palace_path or _get_palace_path()
+
+    print(f"\n{'=' * 55}")
+    print("  MemPalace Deduplicator")
+    print(f"{'=' * 55}")
+
+    client = chromadb.PersistentClient(path=palace_path)
+    col = client.get_collection(COLLECTION_NAME)
+
+    print(f"  Palace: {palace_path}")
+    print(f"  Drawers: {col.count():,}")
+    print(f"  Threshold: {threshold}")
+    print(f"  Mode: {'DRY RUN' if dry_run else 'LIVE'}")
+    print(f"{'─' * 55}")
+
+    if wing:
+        print(f"  Wing: {wing}")
+    groups = get_source_groups(col, min_count, source_pattern, wing=wing)
+    print(f"\n  Sources to check: {len(groups)}")
+
+    t0 = time.time()
+    total_kept = 0
+    total_deleted = 0
+
+    sorted_groups = sorted(groups.items(), key=lambda x: len(x[1]), reverse=True)
+
+    for i, (src, drawer_ids) in enumerate(sorted_groups):
+        kept, deleted = dedup_source_group(col, drawer_ids, threshold, dry_run)
+        total_kept += len(kept)
+        total_deleted += len(deleted)
+
+        if deleted:
+            print(
+                f"  [{i + 1:3d}/{len(groups)}] "
+                f"{src[:50]:50s} {len(drawer_ids):4d} → {len(kept):4d}  "
+                f"(-{len(deleted)})"
+            )
+
+    elapsed = time.time() - t0
+
+    print(f"\n{'─' * 55}")
+    print(f"  Done in {elapsed:.1f}s")
+    print(
+        f"  Drawers: {total_kept + total_deleted:,} → {total_kept:,}  (-{total_deleted:,} removed)"
+    )
+    print(f"  Palace after: {col.count():,} drawers")
+
+    if dry_run:
+        print("\n  [DRY RUN] No changes written. Re-run without --dry-run to apply.")
+
+    print(f"{'=' * 55}\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Deduplicate near-identical drawers")
+    parser.add_argument("--palace", default=None, help="Palace directory path")
+    parser.add_argument(
+        "--threshold",
+        type=float,
+        default=DEFAULT_THRESHOLD,
+        help=f"Cosine distance threshold (default: {DEFAULT_THRESHOLD})",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="Preview without deleting")
+    parser.add_argument("--stats", action="store_true", help="Show stats only")
+    parser.add_argument("--wing", default=None, help="Scope dedup to a single wing")
+    parser.add_argument("--source", default=None, help="Filter by source file pattern")
+    args = parser.parse_args()
+
+    path = os.path.expanduser(args.palace) if args.palace else None
+
+    if args.stats:
+        show_stats(palace_path=path)
+    else:
+        dedup_palace(
+            palace_path=path,
+            threshold=args.threshold,
+            dry_run=args.dry_run,
+            source_pattern=args.source,
+            wing=args.wing,
+        )

mempalace/dialect.py

@@ -1,13 +1,19 @@
 #!/usr/bin/env python3
 """
-AAAK Dialect -- Compressed Symbolic Memory Language
+AAAK Dialect -- Structured Symbolic Summary Format
 ====================================================
 
-A structured symbolic format that ANY LLM reads natively at ~30x compression.
-Not latent vectors. Not English prose. A universal memory compression dialect.
+A lossy summarization format that extracts entities, topics, key sentences,
+emotions, and flags from plain text into a compact structured representation.
+Any LLM reads it natively — no decoder required.
 
 Works with: Claude, ChatGPT, Gemini, Llama, Mistral -- any model that reads text.
 
+NOTE: AAAK is NOT lossless compression. The original text cannot be reconstructed
+from AAAK output. It is a structured summary layer (closets) that points to the
+original verbatim content (drawers). The 96.6% benchmark score is from raw mode,
+not AAAK mode.
+
 Adapted for mempalace: works standalone on plain text and ChromaDB drawers.
 No dependency on palace.py or layers.py.
 
@@ -311,13 +317,17 @@ class Dialect:
         dialect.generate_layer1("zettels/", output="LAYER1.aaak")
     """
 
-    def __init__(self, entities: Dict[str, str] = None, skip_names: List[str] = None):
+    def __init__(
+        self, entities: Dict[str, str] = None, skip_names: List[str] = None, lang: str = None
+    ):
         """
         Args:
             entities: Mapping of full names -> short codes.
                       e.g. {"Alice": "ALC", "Bob": "BOB"}
                       If None, entities are auto-coded from first 3 chars.
             skip_names: Names to skip (fictional characters, etc.)
+            lang: Language code (e.g. "fr", "ko"). Loads AAAK instruction
+                  and regex patterns from i18n dictionary.
         """
         self.entity_codes = {}
         if entities:
@@ -326,6 +336,15 @@ class Dialect:
                 self.entity_codes[name.lower()] = code
         self.skip_names = [n.lower() for n in (skip_names or [])]
 
+        # Load language-specific AAAK instruction and regex patterns
+        from mempalace.i18n import load_lang, t, current_lang, get_regex
+
+        if lang:
+            load_lang(lang)
+        self.lang = lang or current_lang()
+        self.aaak_instruction = t("aaak.instruction")
+        self.lang_regex = get_regex()
+
     @classmethod
     def from_config(cls, config_path: str) -> "Dialect":
         """Load entity mappings from a JSON config file.
@@ -341,6 +360,7 @@ class Dialect:
         return cls(
             entities=config.get("entities", {}),
             skip_names=config.get("skip_names", []),
+            lang=config.get("lang"),
         )
 
     def save_config(self, config_path: str):
@@ -538,19 +558,19 @@ class Dialect:
 
     def compress(self, text: str, metadata: dict = None) -> str:
         """
-        Compress plain text into AAAK Dialect format.
+        Summarize plain text into AAAK Dialect format.
 
-        This is the primary method for mempalace: takes any text content
-        (drawer content, transcript chunk, note) and returns a compressed
-        symbolic representation.
+        Extracts entities, topics, a key sentence, emotions, and flags
+        from the input text. This is lossy — the original text cannot be
+        reconstructed from the output.
 
         Args:
-            text: Plain text content to compress
+            text: Plain text content to summarize
             metadata: Optional dict with keys like 'source_file', 'wing',
                       'room', 'date', etc.
 
         Returns:
-            AAAK-compressed string (~30x smaller than input)
+            AAAK-formatted summary string
         """
         metadata = metadata or {}
 
@@ -930,19 +950,34 @@ class Dialect:
 
     @staticmethod
     def count_tokens(text: str) -> int:
-        """Rough token count (1 token ~ 3 chars for structured text)."""
-        return len(text) // 3
+        """Estimate token count using word-based heuristic (~1.3 tokens per word).
+
+        This is an approximation. For accurate counts, use a real tokenizer
+        like tiktoken. The old len(text)//3 heuristic was wildly inaccurate
+        and made AAAK compression ratios look much better than reality.
+        """
+        words = text.split()
+        # Most English words tokenize to 1-2 tokens; punctuation and
+        # special chars in AAAK (|, +, :) each cost a token.
+        # ~1.3 tokens/word is a conservative average.
+        return max(1, int(len(words) * 1.3))
 
     def compression_stats(self, original_text: str, compressed: str) -> dict:
-        """Get compression statistics for a text->AAAK conversion."""
+        """Get size comparison stats for a text->AAAK conversion.
+
+        NOTE: AAAK is lossy summarization, not compression. The "ratio"
+        reflects how much shorter the summary is, not a compression ratio
+        in the traditional sense — information is lost.
+        """
         orig_tokens = self.count_tokens(original_text)
         comp_tokens = self.count_tokens(compressed)
         return {
-            "original_tokens": orig_tokens,
-            "compressed_tokens": comp_tokens,
-            "ratio": orig_tokens / max(comp_tokens, 1),
+            "original_tokens_est": orig_tokens,
+            "summary_tokens_est": comp_tokens,
+            "size_ratio": round(orig_tokens / max(comp_tokens, 1), 1),
             "original_chars": len(original_text),
-            "compressed_chars": len(compressed),
+            "summary_chars": len(compressed),
+            "note": "Estimates only. Use tiktoken for accurate counts. AAAK is lossy.",
         }
 
 
@@ -1021,9 +1056,9 @@ if __name__ == "__main__":
         encoded = dialect.encode_file(data)
         stats = dialect.compression_stats(json_str, encoded)
         print("=== COMPRESSION STATS ===")
-        print(f"JSON:     ~{stats['original_tokens']:,} tokens")
-        print(f"AAAK:     ~{stats['compressed_tokens']:,} tokens")
-        print(f"Ratio:    {stats['ratio']:.0f}x")
+        print(f"JSON:     ~{stats['original_tokens_est']:,} tokens (est)")
+        print(f"AAAK:     ~{stats['summary_tokens_est']:,} tokens (est)")
+        print(f"Ratio:    {stats['size_ratio']}x (lossy — information is lost)")
         print()
         print("=== AAAK DIALECT OUTPUT ===")
         print(encoded)
@@ -1043,8 +1078,12 @@ if __name__ == "__main__":
         text = " ".join(args)
         compressed = dialect.compress(text)
         stats = dialect.compression_stats(text, compressed)
-        print(f"Original: ~{stats['original_tokens']} tokens ({stats['original_chars']} chars)")
-        print(f"AAAK:     ~{stats['compressed_tokens']} tokens ({stats['compressed_chars']} chars)")
-        print(f"Ratio:    {stats['ratio']:.1f}x")
+        print(
+            f"Original: ~{stats['original_tokens_est']} tokens est ({stats['original_chars']} chars)"
+        )
+        print(
+            f"AAAK:     ~{stats['summary_tokens_est']} tokens est ({stats['summary_chars']} chars)"
+        )
+        print(f"Ratio:    {stats['size_ratio']}x (lossy summary, not lossless compression)")
         print()
         print(compressed)

mempalace/entity_detector.py

@@ -660,7 +660,7 @@ def detect_entities(file_paths: list, max_files: int = 10) -> dict:
             all_text.append(content)
             all_lines.extend(content.splitlines())
             files_read += 1
-        except Exception:
+        except OSError:
             continue
 
     combined_text = "\n".join(all_text)
@@ -760,7 +760,7 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
         if detected["uncertain"]:
             print("\n  Uncertain entities — classify each:")
             for e in detected["uncertain"]:
-                ans = input(f"    {e['name']} — (p)erson, (r)roject, or (s)kip? ").strip().lower()
+                ans = input(f"    {e['name']} — (p)erson, (r)project, or (s)kip? ").strip().lower()
                 if ans == "p":
                     confirmed_people.append(e["name"])
                 elif ans == "r":
@@ -789,7 +789,7 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
             name = input("  Name (or enter to stop): ").strip()
             if not name:
                 break
-            kind = input(f"  Is '{name}' a (p)erson or (r)roject? ").strip().lower()
+            kind = input(f"  Is '{name}' a (p)erson or p(r)oject? ").strip().lower()
             if kind == "p":
                 confirmed_people.append(name)
             elif kind == "r":

mempalace/entity_registry.py

@@ -44,7 +44,6 @@ COMMON_ENGLISH_WORDS = {
     "chance",
     "chase",
     "hunter",
-    "hunter",
     "dash",
     "flash",
     "star",
@@ -78,8 +77,6 @@ COMMON_ENGLISH_WORDS = {
     "january",
     "february",
     "march",
-    "april",
-    "june",
     "july",
     "august",
     "september",
@@ -256,7 +253,7 @@ def _wikipedia_lookup(word: str) -> dict:
                 "note": "not found in Wikipedia — likely a proper noun or unusual name",
             }
         return {"inferred_type": "unknown", "confidence": 0.0, "wiki_summary": None}
-    except Exception:
+    except (urllib.error.URLError, OSError, json.JSONDecodeError, KeyError):
         return {"inferred_type": "unknown", "confidence": 0.0, "wiki_summary": None}
 
 
@@ -312,7 +309,7 @@ class EntityRegistry:
 
     def save(self):
         self._path.parent.mkdir(parents=True, exist_ok=True)
-        self._path.write_text(json.dumps(self._data, indent=2))
+        self._path.write_text(json.dumps(self._data, indent=2), encoding="utf-8")
 
     @staticmethod
     def _empty() -> dict:
@@ -597,7 +594,6 @@ class EntityRegistry:
         Returns list of canonical names found.
         """
         found = []
-        query.lower()
 
         for canonical, info in self.people.items():
             names_to_check = [canonical] + info.get("aliases", [])

mempalace/exporter.py

@@ -0,0 +1,161 @@
+"""
+exporter.py — Export the palace as a browsable folder of markdown files.
+
+Produces:
+  output_dir/
+    index.md              — table of contents
+    wing_name/
+      room_name.md        — one file per room, drawers as sections
+
+Streams drawers in paginated batches so memory usage stays bounded
+regardless of palace size.
+"""
+
+import os
+import re
+from collections import defaultdict
+from datetime import datetime
+
+from .palace import get_collection
+
+
+def _safe_path_component(name: str) -> str:
+    """Sanitize a string for use as a directory/file name component."""
+    name = re.sub(r'[/\\:*?"<>|]', "_", name)
+    name = name.strip(". ")
+    return name or "unknown"
+
+
+def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -> dict:
+    """Export all palace drawers as markdown files organized by wing/room.
+
+    Streams drawers in batches of 1000 and writes each wing/room file
+    incrementally, keeping memory usage proportional to batch size rather
+    than total palace size.
+
+    Args:
+        palace_path: Path to the ChromaDB palace directory.
+        output_dir: Where to write the exported markdown tree.
+        format: Output format (currently only "markdown").
+
+    Returns:
+        Stats dict: {"wings": N, "rooms": N, "drawers": N}
+    """
+    col = get_collection(palace_path)
+    total = col.count()
+
+    if total == 0:
+        print("  Palace is empty — nothing to export.")
+        return {"wings": 0, "rooms": 0, "drawers": 0}
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Track which room files have been opened (so we can append vs overwrite)
+    opened_rooms: set[tuple[str, str]] = set()
+    # Track stats per wing: {wing: {room: count}}
+    wing_stats: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    total_drawers = 0
+
+    print(f"  Streaming {total} drawers...")
+    offset = 0
+    while offset < total:
+        batch = col.get(limit=1000, offset=offset, include=["documents", "metadatas"])
+        if not batch["ids"]:
+            break
+
+        # Group this batch by wing/room so we do one file write per room per batch
+        batch_grouped: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list))
+        for doc_id, doc, meta in zip(batch["ids"], batch["documents"], batch["metadatas"]):
+            wing = meta.get("wing", "unknown")
+            room = meta.get("room", "general")
+            batch_grouped[wing][room].append(
+                {
+                    "id": doc_id,
+                    "content": doc,
+                    "source": meta.get("source_file", ""),
+                    "filed_at": meta.get("filed_at", ""),
+                    "added_by": meta.get("added_by", ""),
+                }
+            )
+
+        # Write/append each room file
+        for wing, rooms in batch_grouped.items():
+            safe_wing = _safe_path_component(wing)
+            wing_dir = os.path.join(output_dir, safe_wing)
+            os.makedirs(wing_dir, exist_ok=True)
+
+            for room, drawers in rooms.items():
+                safe_room = _safe_path_component(room)
+                room_path = os.path.join(wing_dir, f"{safe_room}.md")
+                key = (wing, room)
+                is_new = key not in opened_rooms
+
+                with open(room_path, "a" if not is_new else "w", encoding="utf-8") as f:
+                    if is_new:
+                        f.write(f"# {wing} / {room}\n\n")
+                        opened_rooms.add(key)
+
+                    for drawer in drawers:
+                        source = drawer["source"] or "unknown"
+                        filed = drawer["filed_at"] or "unknown"
+                        added_by = drawer["added_by"] or "unknown"
+
+                        f.write(
+                            f"## {drawer['id']}\n"
+                            f"\n"
+                            f"> {_quote_content(drawer['content'])}\n"
+                            f"\n"
+                            f"| Field | Value |\n"
+                            f"|-------|-------|\n"
+                            f"| Source | {source} |\n"
+                            f"| Filed | {filed} |\n"
+                            f"| Added by | {added_by} |\n"
+                            f"\n"
+                            f"---\n\n"
+                        )
+
+                    wing_stats[wing][room] += len(drawers)
+                    total_drawers += len(drawers)
+
+        offset += len(batch["ids"])
+
+    # Build and print stats
+    index_rows = []
+    for wing in sorted(wing_stats):
+        rooms = wing_stats[wing]
+        wing_drawer_count = sum(rooms.values())
+        index_rows.append((wing, len(rooms), wing_drawer_count))
+        print(f"  {wing}: {len(rooms)} rooms, {wing_drawer_count} drawers")
+
+    # Write index.md
+    today = datetime.now().strftime("%Y-%m-%d")
+    index_lines = [
+        f"# Palace Export — {today}\n",
+        "",
+        "| Wing | Rooms | Drawers |",
+        "|------|-------|---------|",
+    ]
+    for wing, room_count, drawer_count in index_rows:
+        index_lines.append(f"| [{wing}]({wing}/) | {room_count} | {drawer_count} |")
+    index_lines.append("")
+
+    index_path = os.path.join(output_dir, "index.md")
+    with open(index_path, "w", encoding="utf-8") as f:
+        f.write("\n".join(index_lines))
+
+    stats = {
+        "wings": len(wing_stats),
+        "rooms": sum(r for _, r, _ in index_rows),
+        "drawers": total_drawers,
+    }
+    print(
+        f"\n  Exported {stats['drawers']} drawers across {stats['wings']} wings, {stats['rooms']} rooms"
+    )
+    print(f"  Output: {output_dir}")
+    return stats
+
+
+def _quote_content(text: str) -> str:
+    """Format content for a markdown blockquote, handling multiline."""
+    lines = text.rstrip("\n").split("\n")
+    return "\n> ".join(lines)

mempalace/hooks_cli.py

@@ -0,0 +1,238 @@
+"""
+Hook logic for MemPalace — Python implementation of session-start, stop, and precompact hooks.
+
+Reads JSON from stdin, outputs JSON to stdout.
+Supported hooks: session-start, stop, precompact
+Supported harnesses: claude-code, codex (extensible to cursor, gemini, etc.)
+"""
+
+import json
+import os
+import re
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+SAVE_INTERVAL = 15
+STATE_DIR = Path.home() / ".mempalace" / "hook_state"
+
+STOP_BLOCK_REASON = (
+    "AUTO-SAVE checkpoint (MemPalace). Save this session's key content:\n"
+    "1. mempalace_diary_write — AAAK-compressed session summary\n"
+    "2. mempalace_add_drawer — verbatim quotes, decisions, code snippets\n"
+    "3. mempalace_kg_add — entity relationships (optional)\n"
+    "Do NOT write to Claude Code's native auto-memory (.md files). "
+    "Continue conversation after saving."
+)
+
+PRECOMPACT_BLOCK_REASON = (
+    "COMPACTION IMMINENT (MemPalace). Save ALL session content before context is lost:\n"
+    "1. mempalace_diary_write — thorough AAAK-compressed session summary\n"
+    "2. mempalace_add_drawer — ALL verbatim quotes, decisions, code, context\n"
+    "3. mempalace_kg_add — entity relationships (optional)\n"
+    "Be thorough \u2014 after compaction, detailed context will be lost. "
+    "Do NOT write to Claude Code's native auto-memory (.md files). "
+    "Save everything to MemPalace, then allow compaction to proceed."
+)
+
+
+def _sanitize_session_id(session_id: str) -> str:
+    """Only allow alnum, dash, underscore to prevent path traversal."""
+    sanitized = re.sub(r"[^a-zA-Z0-9_-]", "", session_id)
+    return sanitized or "unknown"
+
+
+def _count_human_messages(transcript_path: str) -> int:
+    """Count human messages in a JSONL transcript, skipping command-messages."""
+    path = Path(transcript_path).expanduser()
+    if not path.is_file():
+        return 0
+    count = 0
+    try:
+        with open(path, encoding="utf-8", errors="replace") as f:
+            for line in f:
+                try:
+                    entry = json.loads(line)
+                    msg = entry.get("message", {})
+                    if isinstance(msg, dict) and msg.get("role") == "user":
+                        content = msg.get("content", "")
+                        if isinstance(content, str):
+                            if "<command-message>" in content:
+                                continue
+                        elif isinstance(content, list):
+                            text = " ".join(
+                                b.get("text", "") for b in content if isinstance(b, dict)
+                            )
+                            if "<command-message>" in text:
+                                continue
+                        count += 1
+                    # Also handle Codex CLI transcript format
+                    # {"type": "event_msg", "payload": {"type": "user_message", "message": "..."}}
+                    elif entry.get("type") == "event_msg":
+                        payload = entry.get("payload", {})
+                        if isinstance(payload, dict) and payload.get("type") == "user_message":
+                            msg_text = payload.get("message", "")
+                            if isinstance(msg_text, str) and "<command-message>" not in msg_text:
+                                count += 1
+                except (json.JSONDecodeError, AttributeError):
+                    pass
+    except OSError:
+        return 0
+    return count
+
+
+def _log(message: str):
+    """Append to hook state log file."""
+    try:
+        STATE_DIR.mkdir(parents=True, exist_ok=True)
+        log_path = STATE_DIR / "hook.log"
+        timestamp = datetime.now().strftime("%H:%M:%S")
+        with open(log_path, "a") as f:
+            f.write(f"[{timestamp}] {message}\n")
+    except OSError:
+        pass
+
+
+def _output(data: dict):
+    """Print JSON to stdout with consistent formatting (pretty-printed)."""
+    print(json.dumps(data, indent=2, ensure_ascii=False))
+
+
+def _maybe_auto_ingest():
+    """If MEMPAL_DIR is set and exists, run mempalace mine in background."""
+    mempal_dir = os.environ.get("MEMPAL_DIR", "")
+    if mempal_dir and os.path.isdir(mempal_dir):
+        try:
+            log_path = STATE_DIR / "hook.log"
+            with open(log_path, "a") as log_f:
+                subprocess.Popen(
+                    [sys.executable, "-m", "mempalace", "mine", mempal_dir],
+                    stdout=log_f,
+                    stderr=log_f,
+                )
+        except OSError:
+            pass
+
+
+SUPPORTED_HARNESSES = {"claude-code", "codex"}
+
+
+def _parse_harness_input(data: dict, harness: str) -> dict:
+    """Parse stdin JSON according to the harness type."""
+    if harness not in SUPPORTED_HARNESSES:
+        print(f"Unknown harness: {harness}", file=sys.stderr)
+        sys.exit(1)
+    return {
+        "session_id": _sanitize_session_id(str(data.get("session_id", "unknown"))),
+        "stop_hook_active": data.get("stop_hook_active", False),
+        "transcript_path": str(data.get("transcript_path", "")),
+    }
+
+
+def hook_stop(data: dict, harness: str):
+    """Stop hook: block every N messages for auto-save."""
+    parsed = _parse_harness_input(data, harness)
+    session_id = parsed["session_id"]
+    stop_hook_active = parsed["stop_hook_active"]
+    transcript_path = parsed["transcript_path"]
+
+    # If already in a save cycle, let through (infinite-loop prevention)
+    if str(stop_hook_active).lower() in ("true", "1", "yes"):
+        _output({})
+        return
+
+    # Count human messages
+    exchange_count = _count_human_messages(transcript_path)
+
+    # Track last save point
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    last_save_file = STATE_DIR / f"{session_id}_last_save"
+    last_save = 0
+    if last_save_file.is_file():
+        try:
+            last_save = int(last_save_file.read_text().strip())
+        except (ValueError, OSError):
+            last_save = 0
+
+    since_last = exchange_count - last_save
+
+    _log(f"Session {session_id}: {exchange_count} exchanges, {since_last} since last save")
+
+    if since_last >= SAVE_INTERVAL and exchange_count > 0:
+        # Update last save point
+        try:
+            last_save_file.write_text(str(exchange_count), encoding="utf-8")
+        except OSError:
+            pass
+
+        _log(f"TRIGGERING SAVE at exchange {exchange_count}")
+
+        # Optional: auto-ingest if MEMPAL_DIR is set
+        _maybe_auto_ingest()
+
+        _output({"decision": "block", "reason": STOP_BLOCK_REASON})
+    else:
+        _output({})
+
+
+def hook_session_start(data: dict, harness: str):
+    """Session start hook: initialize session tracking state."""
+    parsed = _parse_harness_input(data, harness)
+    session_id = parsed["session_id"]
+
+    _log(f"SESSION START for session {session_id}")
+
+    # Initialize session state directory
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Pass through — no blocking on session start
+    _output({})
+
+
+def hook_precompact(data: dict, harness: str):
+    """Precompact hook: always block with comprehensive save instruction."""
+    parsed = _parse_harness_input(data, harness)
+    session_id = parsed["session_id"]
+
+    _log(f"PRE-COMPACT triggered for session {session_id}")
+
+    # Optional: auto-ingest synchronously before compaction (so memories land first)
+    mempal_dir = os.environ.get("MEMPAL_DIR", "")
+    if mempal_dir and os.path.isdir(mempal_dir):
+        try:
+            log_path = STATE_DIR / "hook.log"
+            with open(log_path, "a") as log_f:
+                subprocess.run(
+                    [sys.executable, "-m", "mempalace", "mine", mempal_dir],
+                    stdout=log_f,
+                    stderr=log_f,
+                    timeout=60,
+                )
+        except OSError:
+            pass
+
+    # Always block -- compaction = save everything
+    _output({"decision": "block", "reason": PRECOMPACT_BLOCK_REASON})
+
+
+def run_hook(hook_name: str, harness: str):
+    """Main entry point: read stdin JSON, dispatch to hook handler."""
+    try:
+        data = json.load(sys.stdin)
+    except (json.JSONDecodeError, EOFError):
+        _log("WARNING: Failed to parse stdin JSON, proceeding with empty data")
+        data = {}
+
+    hooks = {
+        "session-start": hook_session_start,
+        "stop": hook_stop,
+        "precompact": hook_precompact,
+    }
+
+    handler = hooks.get(hook_name)
+    if handler is None:
+        print(f"Unknown hook: {hook_name}", file=sys.stderr)
+        sys.exit(1)
+
+    handler(data, harness)

mempalace/i18n/__init__.py

@@ -0,0 +1,76 @@
+"""i18n — Language dictionaries for MemPalace.
+
+Usage:
+    from mempalace.i18n import load_lang, t
+
+    load_lang("fr")           # load French
+    print(t("cli.mine_start", path="/docs"))  # "Extraction de /docs..."
+    print(t("terms.wing"))    # "aile"
+    print(t("aaak.instruction"))  # AAAK compression instruction in French
+"""
+
+import json
+from pathlib import Path
+
+_LANG_DIR = Path(__file__).parent
+_strings: dict = {}
+_current_lang: str = "en"
+
+
+def available_languages() -> list[str]:
+    """Return list of available language codes."""
+    return sorted(p.stem for p in _LANG_DIR.glob("*.json"))
+
+
+def load_lang(lang: str = "en") -> dict:
+    """Load a language dictionary. Falls back to English if not found."""
+    global _strings, _current_lang
+    lang_file = _LANG_DIR / f"{lang}.json"
+    if not lang_file.exists():
+        lang_file = _LANG_DIR / "en.json"
+        lang = "en"
+    _strings = json.loads(lang_file.read_text(encoding="utf-8"))
+    _current_lang = lang
+    return _strings
+
+
+def t(key: str, **kwargs) -> str:
+    """Get a translated string by dotted key. Supports {var} interpolation.
+
+    t("cli.mine_complete", closets=5, drawers=20)
+    → "Done. 5 closets, 20 drawers created."
+    """
+    if not _strings:
+        load_lang("en")
+    parts = key.split(".", 1)
+    if len(parts) == 2:
+        section, name = parts
+        val = _strings.get(section, {}).get(name, key)
+    else:
+        val = _strings.get(key, key)
+    if kwargs and isinstance(val, str):
+        try:
+            val = val.format(**kwargs)
+        except (KeyError, IndexError):
+            pass
+    return val
+
+
+def current_lang() -> str:
+    """Return current language code."""
+    return _current_lang
+
+
+def get_regex() -> dict:
+    """Return the regex patterns for the current language.
+
+    Keys: topic_pattern, stop_words, quote_pattern, action_pattern.
+    Returns empty dict if no regex section in the language file.
+    """
+    if not _strings:
+        load_lang("en")
+    return _strings.get("regex", {})
+
+
+# Auto-load English on import
+load_lang("en")

mempalace/i18n/de.json

@@ -0,0 +1,44 @@
+{
+  "lang": "de",
+  "label": "Deutsch",
+  "terms": {
+    "palace": "Palast",
+    "wing": "Flügel",
+    "hall": "Flur",
+    "closet": "Schrank",
+    "drawer": "Schublade",
+    "mine": "schürfen",
+    "search": "suchen",
+    "status": "Status",
+    "init": "initialisieren",
+    "repair": "reparieren",
+    "migrate": "migrieren",
+    "entity": "Entität",
+    "topic": "Thema"
+  },
+  "cli": {
+    "mine_start": "Schürfe {path}...",
+    "mine_complete": "Fertig. {closets} Schränke, {drawers} Schubladen erstellt.",
+    "mine_skip": "Bereits geschürft. Verwenden Sie --force zum Wiederholen.",
+    "search_no_results": "Keine Ergebnisse für: {query}",
+    "search_results": "{count} Ergebnisse gefunden:",
+    "status_palace": "Palast: {path}",
+    "status_wings": "{count} Flügel",
+    "status_closets": "{count} Schränke",
+    "status_drawers": "{count} Schubladen",
+    "init_complete": "Palast initialisiert in {path}",
+    "init_exists": "Palast existiert bereits in {path}",
+    "repair_complete": "Reparatur abgeschlossen. {fixed} Probleme behoben.",
+    "migrate_complete": "Migration abgeschlossen.",
+    "no_palace": "Kein Palast gefunden. Ausführen: mempalace init <Ordner>"
+  },
+  "aaak": {
+    "instruction": "Auf Deutsch komprimieren. Bindestriche zwischen Wörtern, Pipes zwischen Konzepten. Artikel und Füllwörter weglassen. Eigennamen und Zahlen exakt beibehalten."
+  },
+  "regex": {
+    "topic_pattern": "[A-ZÄÖÜß][a-zäöüß]{2,}|[A-Za-zÄÖÜäöüß]{3,}",
+    "stop_words": "der die das ein eine eines einer einem einen den dem des und oder aber denn weil wenn als ob auch noch schon sehr viel nur nicht mehr kann wird hat ist sind war waren sein haben wurde mit von zu für auf in an um über nach durch",
+    "quote_pattern": "\\u201E([^\\u201C]{10,200})\\u201C|\"([^\"]{10,200})\"",
+    "action_pattern": "(?:gebaut|behoben|geschrieben|hinzugefügt|gepusht|gemessen|getestet|überprüft|erstellt|gelöscht|aktualisiert|konfiguriert|bereitgestellt|migriert)\\s+[\\wÄÖÜäöüß\\s]{3,30}"
+  }
+}

mempalace/i18n/en.json

@@ -0,0 +1,44 @@
+{
+  "lang": "en",
+  "label": "English",
+  "terms": {
+    "palace": "palace",
+    "wing": "wing",
+    "hall": "hall",
+    "closet": "closet",
+    "drawer": "drawer",
+    "mine": "mine",
+    "search": "search",
+    "status": "status",
+    "init": "init",
+    "repair": "repair",
+    "migrate": "migrate",
+    "entity": "entity",
+    "topic": "topic"
+  },
+  "cli": {
+    "mine_start": "Mining {path}...",
+    "mine_complete": "Done. {closets} closets, {drawers} drawers created.",
+    "mine_skip": "Already mined. Use --force to re-mine.",
+    "search_no_results": "No results for: {query}",
+    "search_results": "Found {count} results:",
+    "status_palace": "Palace: {path}",
+    "status_wings": "{count} wings",
+    "status_closets": "{count} closets",
+    "status_drawers": "{count} drawers",
+    "init_complete": "Palace initialized at {path}",
+    "init_exists": "Palace already exists at {path}",
+    "repair_complete": "Repair complete. {fixed} issues fixed.",
+    "migrate_complete": "Migration complete.",
+    "no_palace": "No palace found. Run: mempalace init <dir>"
+  },
+  "aaak": {
+    "instruction": "Compress to index format. Hyphens between words, pipes between concepts. Drop articles and filler. Keep names and numbers exact."
+  },
+  "regex": {
+    "topic_pattern": "[A-Z][a-z]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "the this that these those some many most each every other only such very will would could should must shall yeah okay also even then now already still back done make take give know think want need going come find work added saved session summary conversation topics source about once just really actually here there where good great better thank please sorry right wrong true false",
+    "quote_pattern": "\"([^\"]{20,200})\"",
+    "action_pattern": "(?:built|fixed|wrote|added|pushed|measured|tested|reviewed|created|deleted|updated|configured|deployed|migrated)\\s+[\\w\\s]{3,30}"
+  }
+}

mempalace/i18n/es.json

@@ -0,0 +1,44 @@
+{
+  "lang": "es",
+  "label": "Español",
+  "terms": {
+    "palace": "palacio",
+    "wing": "ala",
+    "hall": "pasillo",
+    "closet": "armario",
+    "drawer": "cajón",
+    "mine": "extraer",
+    "search": "buscar",
+    "status": "estado",
+    "init": "inicializar",
+    "repair": "reparar",
+    "migrate": "migrar",
+    "entity": "entidad",
+    "topic": "tema"
+  },
+  "cli": {
+    "mine_start": "Extrayendo {path}...",
+    "mine_complete": "Listo. {closets} armarios, {drawers} cajones creados.",
+    "mine_skip": "Ya extraído. Use --force para repetir.",
+    "search_no_results": "Sin resultados para: {query}",
+    "search_results": "{count} resultados encontrados:",
+    "status_palace": "Palacio: {path}",
+    "status_wings": "{count} alas",
+    "status_closets": "{count} armarios",
+    "status_drawers": "{count} cajones",
+    "init_complete": "Palacio inicializado en {path}",
+    "init_exists": "Ya existe un palacio en {path}",
+    "repair_complete": "Reparación completa. {fixed} problemas corregidos.",
+    "migrate_complete": "Migración completa.",
+    "no_palace": "No se encontró palacio. Ejecute: mempalace init <carpeta>"
+  },
+  "aaak": {
+    "instruction": "Comprima en español. Guiones entre palabras, pipes entre conceptos. Elimine artículos y palabras de relleno. Mantenga nombres propios y números exactos."
+  },
+  "regex": {
+    "topic_pattern": "[A-ZÁ-Ú][a-zá-ú]{2,}|[A-Za-zÁ-ú]{3,}",
+    "stop_words": "el la los las un una unos unas de del al en con por para su sus mi mis tu tus es son está están fue ser estar haber sido como pero más muy también todo todos toda todas este esta estos estas ese esa esos esas que quien cual donde cuando porque aunque sin",
+    "quote_pattern": "\"([^\"]{10,200})\"|«([^»]{10,200})»",
+    "action_pattern": "(?:construido|corregido|escrito|añadido|enviado|medido|probado|revisado|creado|eliminado|actualizado|configurado|desplegado|migrado)\\s+[\\wá-ú\\s]{3,30}"
+  }
+}

mempalace/i18n/fr.json

@@ -0,0 +1,44 @@
+{
+  "lang": "fr",
+  "label": "Français",
+  "terms": {
+    "palace": "palais",
+    "wing": "aile",
+    "hall": "couloir",
+    "closet": "placard",
+    "drawer": "tiroir",
+    "mine": "extraire",
+    "search": "chercher",
+    "status": "état",
+    "init": "initialiser",
+    "repair": "réparer",
+    "migrate": "migrer",
+    "entity": "entité",
+    "topic": "sujet"
+  },
+  "cli": {
+    "mine_start": "Extraction de {path}...",
+    "mine_complete": "Terminé. {closets} placards, {drawers} tiroirs créés.",
+    "mine_skip": "Déjà extrait. Utilisez --force pour refaire.",
+    "search_no_results": "Aucun résultat pour : {query}",
+    "search_results": "{count} résultats trouvés :",
+    "status_palace": "Palais : {path}",
+    "status_wings": "{count} ailes",
+    "status_closets": "{count} placards",
+    "status_drawers": "{count} tiroirs",
+    "init_complete": "Palais initialisé dans {path}",
+    "init_exists": "Un palais existe déjà dans {path}",
+    "repair_complete": "Réparation terminée. {fixed} problèmes corrigés.",
+    "migrate_complete": "Migration terminée.",
+    "no_palace": "Aucun palais trouvé. Exécutez : mempalace init <dossier>"
+  },
+  "aaak": {
+    "instruction": "Comprimez en français. Tirets entre les mots, pipes entre les concepts. Supprimez les articles et mots de remplissage. Gardez les noms propres et chiffres exacts."
+  },
+  "regex": {
+    "topic_pattern": "[A-ZÀ-Ý][a-zà-ÿ]{2,}|[A-Za-zÀ-ÿ]{3,}",
+    "stop_words": "le la les un une des de du au aux en et ou mais donc or ni car que qui ce cette ces son sa ses mon ma mes ton ta tes leur leurs nous vous ils elles on ne pas plus très bien aussi avec pour dans sur par est sont fait être avoir été comme tout tous toute toutes",
+    "quote_pattern": "«\\s*([^»]{10,200})\\s*»|\"([^\"]{10,200})\"",
+    "action_pattern": "(?:construit|corrigé|écrit|ajouté|poussé|mesuré|testé|révisé|créé|supprimé|mis à jour|configuré|déployé|migré)\\s+[\\wà-ÿ\\s]{3,30}"
+  }
+}

mempalace/i18n/ja.json

@@ -0,0 +1,44 @@
+{
+  "lang": "ja",
+  "label": "日本語",
+  "terms": {
+    "palace": "宮殿",
+    "wing": "棟",
+    "hall": "廊下",
+    "closet": "クローゼット",
+    "drawer": "引き出し",
+    "mine": "採掘",
+    "search": "検索",
+    "status": "状態",
+    "init": "初期化",
+    "repair": "修復",
+    "migrate": "移行",
+    "entity": "エンティティ",
+    "topic": "トピック"
+  },
+  "cli": {
+    "mine_start": "{path} を採掘中...",
+    "mine_complete": "完了。クローゼット {closets}個、引き出し {drawers}個 作成。",
+    "mine_skip": "採掘済み。再実行するには --force を使用。",
+    "search_no_results": "結果なし: {query}",
+    "search_results": "{count}件の結果:",
+    "status_palace": "宮殿: {path}",
+    "status_wings": "棟 {count}個",
+    "status_closets": "クローゼット {count}個",
+    "status_drawers": "引き出し {count}個",
+    "init_complete": "{path} に宮殿を初期化しました",
+    "init_exists": "{path} に宮殿は既に存在します",
+    "repair_complete": "修復完了。{fixed}件の問題を修正。",
+    "migrate_complete": "移行完了。",
+    "no_palace": "宮殿が見つかりません。実行: mempalace init <ディレクトリ>"
+  },
+  "aaak": {
+    "instruction": "日本語で圧縮してください。概念間はパイプ(|)、単語間はハイフン(-)。助詞と接続詞は省略。固有名詞と数値は正確に保持。"
+  },
+  "regex": {
+    "topic_pattern": "[\\u30A0-\\u30FF]{3,}|[\\u4E00-\\u9FFF]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "は が を に で と も の へ から まで より した します している されて です ます ました こと もの ため それ これ その この あの ない なく ある いる する",
+    "quote_pattern": "「([^」]{10,100})」",
+    "action_pattern": "(構築|修正|追加|削除|確認|作成|実装|修復|書き直し|テスト|検証|更新|設定|起動|停止)(?:し|した|して|する|します)"
+  }
+}

mempalace/i18n/ko.json

@@ -0,0 +1,44 @@
+{
+  "lang": "ko",
+  "label": "한국어",
+  "terms": {
+    "palace": "궁전",
+    "wing": "날개",
+    "hall": "복도",
+    "closet": "벽장",
+    "drawer": "서랍",
+    "mine": "채굴",
+    "search": "검색",
+    "status": "상태",
+    "init": "초기화",
+    "repair": "수리",
+    "migrate": "마이그레이션",
+    "entity": "개체",
+    "topic": "주제"
+  },
+  "cli": {
+    "mine_start": "{path} 채굴 중...",
+    "mine_complete": "완료. 벽장 {closets}개, 서랍 {drawers}개 생성.",
+    "mine_skip": "이미 채굴됨. --force로 다시 실행하세요.",
+    "search_no_results": "결과 없음: {query}",
+    "search_results": "{count}개 결과 발견:",
+    "status_palace": "궁전: {path}",
+    "status_wings": "날개 {count}개",
+    "status_closets": "벽장 {count}개",
+    "status_drawers": "서랍 {drawers}개",
+    "init_complete": "{path}에 궁전 초기화 완료",
+    "init_exists": "{path}에 궁전이 이미 존재합니다",
+    "repair_complete": "수리 완료. {fixed}개 문제 해결.",
+    "migrate_complete": "마이그레이션 완료.",
+    "no_palace": "궁전을 찾을 수 없습니다. 실행: mempalace init <폴더>"
+  },
+  "aaak": {
+    "instruction": "한국어로 압축하세요. 개념 사이에 파이프(|), 단어 연결에 하이픈(-). 조사와 접속사는 생략. 고유명사와 숫자는 정확히 유지."
+  },
+  "regex": {
+    "topic_pattern": "[가-힣]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "은 는 이 가 을 를 에 에서 의 로 으로 와 과 도 만 까지 부터 처럼 보다 한 하는 했다 합니다 했습니다 되었 있는 것 수 등 및 또는 그리고 하지만 때문에",
+    "quote_pattern": "\"([^\"]{10,100})\"|'([^']{10,100})'",
+    "action_pattern": "(구축|수정|추가|삭제|확인|생성|구현|수리|작성|테스트|검증|업데이트|설정|시작|중지)(?:했|한|하여|합니다|했습니다)"
+  }
+}

mempalace/i18n/test_i18n.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Quick smoke test for i18n dictionaries + Dialect integration."""
+
+import sys
+from pathlib import Path
+
+# Add parent to path so we can import mempalace
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from mempalace.i18n import load_lang, t, available_languages
+from mempalace.dialect import Dialect
+
+
+def test_all_languages_load():
+    """Every JSON file loads without error and has required keys."""
+    required_sections = ["terms", "cli", "aaak"]
+    required_terms = ["palace", "wing", "closet", "drawer"]
+
+    langs = available_languages()
+    assert len(langs) >= 7, f"Expected 7+ languages, got {len(langs)}"
+
+    for lang in langs:
+        strings = load_lang(lang)
+        for section in required_sections:
+            assert section in strings, f"{lang}: missing section '{section}'"
+        for term in required_terms:
+            assert term in strings["terms"], f"{lang}: missing term '{term}'"
+            assert len(strings["terms"][term]) > 0, f"{lang}: empty term '{term}'"
+        assert "instruction" in strings["aaak"], f"{lang}: missing aaak.instruction"
+
+    print(f"  PASS: {len(langs)} languages load correctly")
+
+
+def test_interpolation():
+    """String interpolation works for all languages."""
+    for lang in available_languages():
+        load_lang(lang)
+        result = t("cli.mine_complete", closets=5, drawers=100)
+        assert "5" in result, f"{lang}: closets count missing from mine_complete"
+        assert "100" in result, f"{lang}: drawers count missing from mine_complete"
+
+    print("  PASS: interpolation works for all languages")
+
+
+def test_dialect_loads_lang():
+    """Dialect class picks up the language instruction."""
+    for lang in available_languages():
+        d = Dialect(lang=lang)
+        assert d.lang == lang, f"Expected lang={lang}, got {d.lang}"
+        assert len(d.aaak_instruction) > 10, f"{lang}: AAAK instruction too short"
+
+    print("  PASS: Dialect loads language instruction for all languages")
+
+
+def test_dialect_compress_samples():
+    """Compress sample text in different languages, verify output isn't empty."""
+    samples = {
+        "en": "We decided to migrate from SQLite to PostgreSQL for better concurrent writes. Ben approved the PR yesterday.",
+        "fr": "Nous avons décidé de migrer de SQLite vers PostgreSQL pour une meilleure écriture concurrente. Ben a approuvé le PR hier.",
+        "ko": "더 나은 동시 쓰기를 위해 SQLite에서 PostgreSQL로 마이그레이션하기로 했습니다. 벤이 어제 PR을 승인했습니다.",
+        "ja": "同時書き込みの改善のため、SQLiteからPostgreSQLに移行することを決定しました。ベンが昨日PRを承認しました。",
+        "es": "Decidimos migrar de SQLite a PostgreSQL para mejor escritura concurrente. Ben aprobó el PR ayer.",
+        "de": "Wir haben beschlossen, von SQLite auf PostgreSQL zu migrieren für bessere gleichzeitige Schreibvorgänge. Ben hat den PR gestern genehmigt.",
+        "zh-CN": "我们决定从SQLite迁移到PostgreSQL以获得更好的并发写入。Ben昨天批准了PR。",
+    }
+
+    for lang, text in samples.items():
+        d = Dialect(lang=lang)
+        compressed = d.compress(text)
+        assert len(compressed) > 0, f"{lang}: compression returned empty"
+        assert len(compressed) < len(text) * 2, f"{lang}: compression expanded text"
+        print(f"    {lang}: {len(text)} chars → {len(compressed)} chars")
+        print(f"         {compressed[:80]}")
+
+    print("  PASS: compression works for all sample languages")
+
+
+if __name__ == "__main__":
+    print("i18n smoke tests:")
+    test_all_languages_load()
+    test_interpolation()
+    test_dialect_loads_lang()
+    test_dialect_compress_samples()
+    print("\nAll tests passed.")

mempalace/i18n/zh-CN.json

@@ -0,0 +1,44 @@
+{
+  "lang": "zh-CN",
+  "label": "简体中文",
+  "terms": {
+    "palace": "宫殿",
+    "wing": "翼",
+    "hall": "走廊",
+    "closet": "柜子",
+    "drawer": "抽屉",
+    "mine": "挖掘",
+    "search": "搜索",
+    "status": "状态",
+    "init": "初始化",
+    "repair": "修复",
+    "migrate": "迁移",
+    "entity": "实体",
+    "topic": "主题"
+  },
+  "cli": {
+    "mine_start": "正在挖掘 {path}...",
+    "mine_complete": "完成。创建了 {closets} 个柜子、{drawers} 个抽屉。",
+    "mine_skip": "已挖掘。使用 --force 重新执行。",
+    "search_no_results": "未找到结果: {query}",
+    "search_results": "找到 {count} 个结果:",
+    "status_palace": "宫殿: {path}",
+    "status_wings": "{count} 个翼",
+    "status_closets": "{count} 个柜子",
+    "status_drawers": "{count} 个抽屉",
+    "init_complete": "宫殿已初始化于 {path}",
+    "init_exists": "{path} 中已存在宫殿",
+    "repair_complete": "修复完成。已修正 {fixed} 个问题。",
+    "migrate_complete": "迁移完成。",
+    "no_palace": "未找到宫殿。请运行: mempalace init <目录>"
+  },
+  "aaak": {
+    "instruction": "用中文压缩。概念之间用管道符(|)，词语之间用连字符(-)。省略虚词和连接词。保留专有名词和数字的准确性。"
+  },
+  "regex": {
+    "topic_pattern": "[\\u4E00-\\u9FFF]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "的 了 在 是 我 有 和 就 不 人 都 一 一个 上 也 很 到 说 要 去 你 会 着 没有 看 好 自己 这 那 她 他 它 们 但是 因为 所以 如果 虽然 然后 或者 而且",
+    "quote_pattern": "\\u201C([^\\u201D]{10,100})\\u201D|\"([^\"]{10,200})\"",
+    "action_pattern": "(构建|修复|添加|删除|确认|创建|实现|修理|编写|测试|验证|更新|配置|启动|停止)(?:了|完成|成功)"
+  }
+}

mempalace/i18n/zh-TW.json

@@ -0,0 +1,44 @@
+{
+  "lang": "zh-TW",
+  "label": "繁體中文",
+  "terms": {
+    "palace": "宮殿",
+    "wing": "翼",
+    "hall": "走廊",
+    "closet": "櫃子",
+    "drawer": "抽屜",
+    "mine": "挖掘",
+    "search": "搜尋",
+    "status": "狀態",
+    "init": "初始化",
+    "repair": "修復",
+    "migrate": "遷移",
+    "entity": "實體",
+    "topic": "主題"
+  },
+  "cli": {
+    "mine_start": "正在挖掘 {path}...",
+    "mine_complete": "完成。建立了 {closets} 個櫃子、{drawers} 個抽屜。",
+    "mine_skip": "已挖掘。使用 --force 重新執行。",
+    "search_no_results": "未找到結果: {query}",
+    "search_results": "找到 {count} 個結果:",
+    "status_palace": "宮殿: {path}",
+    "status_wings": "{count} 個翼",
+    "status_closets": "{count} 個櫃子",
+    "status_drawers": "{count} 個抽屜",
+    "init_complete": "宮殿已初始化於 {path}",
+    "init_exists": "{path} 中已存在宮殿",
+    "repair_complete": "修復完成。已修正 {fixed} 個問題。",
+    "migrate_complete": "遷移完成。",
+    "no_palace": "未找到宮殿。請執行: mempalace init <目錄>"
+  },
+  "aaak": {
+    "instruction": "用中文壓縮。概念之間用管道符(|)，詞語之間用連字符(-)。省略虛詞和連接詞。保留專有名詞和數字的準確性。"
+  },
+  "regex": {
+    "topic_pattern": "[\\u4E00-\\u9FFF]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "的 了 在 是 我 有 和 就 不 人 都 一 一個 上 也 很 到 說 要 去 你 會 著 沒有 看 好 自己 這 那 她 他 它 們 但是 因為 所以 如果 雖然 然後 或者 而且",
+    "quote_pattern": "「([^」]{10,100})」|\u201c([^\u201d]{10,100})\u201d",
+    "action_pattern": "(構建|修復|添加|刪除|確認|創建|實現|修理|編寫|測試|驗證|更新|配置|啟動|停止)(?:了|完成|成功)"
+  }
+}

mempalace/instructions/help.md

@@ -0,0 +1,106 @@
+# MemPalace
+
+AI memory system. Store everything, find anything. Local, free, no API key.
+
+---
+
+## Slash Commands
+
+| Command              | Description                    |
+|----------------------|--------------------------------|
+| /mempalace:init      | Install and set up MemPalace   |
+| /mempalace:search    | Search your memories           |
+| /mempalace:mine      | Mine projects and conversations|
+| /mempalace:status    | Palace overview and stats      |
+| /mempalace:help      | This help message              |
+
+---
+
+## MCP Tools (19)
+
+### Palace (read)
+- mempalace_status -- Palace status and stats
+- mempalace_list_wings -- List all wings
+- mempalace_list_rooms -- List rooms in a wing
+- mempalace_get_taxonomy -- Get the full taxonomy tree
+- mempalace_search -- Search memories by query
+- mempalace_check_duplicate -- Check if a memory already exists
+- mempalace_get_aaak_spec -- Get the AAAK specification
+
+### Palace (write)
+- mempalace_add_drawer -- Add a new memory (drawer)
+- mempalace_delete_drawer -- Delete a memory (drawer)
+
+### Knowledge Graph
+- mempalace_kg_query -- Query the knowledge graph
+- mempalace_kg_add -- Add a knowledge graph entry
+- mempalace_kg_invalidate -- Invalidate a knowledge graph entry
+- mempalace_kg_timeline -- View knowledge graph timeline
+- mempalace_kg_stats -- Knowledge graph statistics
+
+### Navigation
+- mempalace_traverse -- Traverse the palace structure
+- mempalace_find_tunnels -- Find cross-wing connections
+- mempalace_graph_stats -- Graph connectivity statistics
+
+### Agent Diary
+- mempalace_diary_write -- Write a diary entry
+- mempalace_diary_read -- Read diary entries
+
+---
+
+## CLI Commands
+
+    mempalace init <dir>                  Initialize a new palace
+    mempalace mine <dir>                  Mine a project (default mode)
+    mempalace mine <dir> --mode convos    Mine conversation exports
+    mempalace search "query"              Search your memories
+    mempalace split <dir>                 Split large transcript files
+    mempalace wake-up                     Load palace into context
+    mempalace compress                    Compress palace storage
+    mempalace status                      Show palace status
+    mempalace repair                      Rebuild vector index
+    mempalace mcp                         Show MCP setup command
+    mempalace hook run                    Run hook logic (for harness integration)
+    mempalace instructions <name>         Output skill instructions
+
+---
+
+## Auto-Save Hooks
+
+- Stop hook -- Automatically saves memories every 15 messages. Counts human
+  messages in the session transcript (skipping command-messages). When the
+  threshold is reached, blocks the AI with a save instruction. Uses
+  ~/.mempalace/hook_state/ to track save points per session. If
+  stop_hook_active is true, passes through to prevent infinite loops.
+
+- PreCompact hook -- Emergency save before context compaction. Always blocks
+  with a comprehensive save instruction because compaction means the AI is
+  about to lose detailed context.
+
+Hooks read JSON from stdin and output JSON to stdout. They can be invoked via:
+
+    echo '{"session_id":"abc","stop_hook_active":false,"transcript_path":"..."}' | mempalace hook run --hook stop --harness claude-code
+
+---
+
+## Architecture
+
+    Wings (projects/people)
+      +-- Rooms (topics)
+            +-- Closets (summaries)
+                  +-- Drawers (verbatim memories)
+
+    Halls connect rooms within a wing.
+    Tunnels connect rooms across wings.
+
+The palace is stored locally using ChromaDB for vector search and SQLite for
+metadata. No cloud services or API keys required.
+
+---
+
+## Getting Started
+
+1. /mempalace:init -- Set up your palace
+2. /mempalace:mine -- Mine a project or conversation
+3. /mempalace:search -- Find what you stored

mempalace/instructions/init.md

@@ -0,0 +1,69 @@
+# MemPalace Init
+
+Guide the user through a complete MemPalace setup. Follow each step in order,
+stopping to report errors and attempt remediation before proceeding.
+
+## Step 1: Check Python version
+
+Run `python3 --version` (or `python --version` on Windows) and confirm the
+version is 3.9 or higher. If Python is not found or the version is too old,
+tell the user they need Python 3.9+ installed and stop.
+
+## Step 2: Check if mempalace is already installed
+
+Run `pip show mempalace` to see if the package is already present. If it is,
+report the installed version and skip to Step 4.
+
+## Step 3: Install mempalace
+
+Run `pip install mempalace`.
+
+### Error handling -- pip failures
+
+If `pip install mempalace` fails, try these fallbacks in order:
+
+1. Try `pip3 install mempalace`
+2. Try `python -m pip install mempalace` (or `python3 -m pip install mempalace`)
+3. If the error mentions missing build tools or compilation failures (commonly
+   from chromadb or its native dependencies):
+   - On Linux/macOS: suggest `sudo apt-get install build-essential python3-dev`
+     (Debian/Ubuntu) or `xcode-select --install` (macOS)
+   - On Windows: suggest installing Microsoft C++ Build Tools from
+     https://visualstudio.microsoft.com/visual-cpp-build-tools/
+   - Then retry the install command
+4. If all attempts fail, report the error clearly and stop.
+
+## Step 4: Ask for project directory
+
+Ask the user which project directory they want to initialize with MemPalace.
+Offer the current working directory as the default. Wait for their response
+before continuing.
+
+## Step 5: Initialize the palace
+
+Run `mempalace init --yes <dir>` where `<dir>` is the directory from Step 4.
+
+If this fails, report the error and stop.
+
+## Step 6: Configure MCP server
+
+Run the following command to register the MemPalace MCP server with Claude:
+
+    claude mcp add mempalace -- python -m mempalace.mcp_server
+
+If this fails, report the error but continue to the next step (MCP
+configuration can be done manually later).
+
+## Step 7: Verify installation
+
+Run `mempalace status` and confirm the output shows a healthy palace.
+
+If the command fails or reports errors, walk the user through troubleshooting
+based on the output.
+
+## Step 8: Show next steps
+
+Tell the user setup is complete and suggest these next actions:
+
+- Use /mempalace:mine to start adding data to their palace
+- Use /mempalace:search to query their palace and retrieve stored knowledge

mempalace/instructions/mine.md

@@ -0,0 +1,64 @@
+# MemPalace Mine
+
+When the user invokes this skill, follow these steps:
+
+## 1. Ask what to mine
+
+Ask the user what they want to mine and where the source data is located.
+Clarify:
+- Is it a project directory (code, docs, notes)?
+- Is it conversation exports (Claude, ChatGPT, Slack)?
+- Do they want auto-classification (decisions, milestones, problems)?
+
+## 2. Choose the mining mode
+
+There are three mining modes:
+
+### Project mining
+
+    mempalace mine <dir>
+
+Mines code files, documentation, and notes from a project directory.
+
+### Conversation mining
+
+    mempalace mine <dir> --mode convos
+
+Mines conversation exports from Claude, ChatGPT, or Slack into the palace.
+
+### General extraction (auto-classify)
+
+    mempalace mine <dir> --mode convos --extract general
+
+Auto-classifies mined content into decisions, milestones, and problems.
+
+## 3. Optionally split mega-files first
+
+If the source directory contains very large files, suggest splitting them
+before mining:
+
+    mempalace split <dir> [--dry-run]
+
+Use --dry-run first to preview what will be split without making changes.
+
+## 4. Optionally tag with a wing
+
+If the user wants to organize mined content under a specific wing, add the
+--wing flag:
+
+    mempalace mine <dir> --wing <name>
+
+## 5. Show progress and results
+
+Run the selected mining command and display progress as it executes. After
+completion, summarize the results including:
+- Number of items mined
+- Categories or classifications applied
+- Any warnings or skipped files
+
+## 6. Suggest next steps
+
+After mining completes, suggest the user try:
+- /mempalace:search -- search the newly mined content
+- /mempalace:status -- check the current state of their palace
+- Mine more data from additional sources

mempalace/instructions/search.md

@@ -0,0 +1,57 @@
+# MemPalace Search
+
+When the user wants to search their MemPalace memories, follow these steps:
+
+## 1. Parse the Search Query
+
+Extract the core search intent from the user's message. Identify any explicit
+or implicit filters:
+- Wing -- a top-level category (e.g., "work", "personal", "research")
+- Room -- a sub-category within a wing
+- Keywords / semantic query -- the actual search terms
+
+## 2. Determine Wing/Room Filters
+
+If the user mentions a specific domain, topic area, or context, map it to the
+appropriate wing and/or room. If unsure, omit filters to search globally. You
+can discover the taxonomy first if needed.
+
+## 3. Use MCP Tools (Preferred)
+
+If MCP tools are available, use them in this priority order:
+
+- mempalace_search(query, wing, room) -- Primary search tool. Pass the semantic
+  query and any wing/room filters.
+- mempalace_list_wings -- Discover all available wings. Use when the user asks
+  what categories exist or you need to resolve a wing name.
+- mempalace_list_rooms(wing) -- List rooms within a specific wing. Use to help
+  the user navigate or to resolve a room name.
+- mempalace_get_taxonomy -- Retrieve the full wing/room/drawer tree. Use when
+  the user wants an overview of their entire memory structure.
+- mempalace_traverse(room) -- Walk the knowledge graph starting from a room.
+  Use when the user wants to explore connections and related memories.
+- mempalace_find_tunnels(wing1, wing2) -- Find cross-wing connections (tunnels)
+  between two wings. Use when the user asks about relationships between
+  different knowledge domains.
+
+## 4. CLI Fallback
+
+If MCP tools are not available, fall back to the CLI:
+
+    mempalace search "query" [--wing X] [--room Y]
+
+## 5. Present Results
+
+When presenting search results:
+- Always include source attribution: wing, room, and drawer for each result
+- Show relevance or similarity scores if available
+- Group results by wing/room when returning multiple hits
+- Quote or summarize the memory content clearly
+
+## 6. Offer Next Steps
+
+After presenting results, offer the user options to go deeper:
+- Drill deeper -- search within a specific room or narrow the query
+- Traverse -- explore the knowledge graph from a related room
+- Check tunnels -- look for cross-wing connections if the topic spans domains
+- Browse taxonomy -- show the full structure for manual exploration

mempalace/instructions/status.md

@@ -0,0 +1,49 @@
+# MemPalace Status
+
+Display the current state of the user's memory palace.
+
+## Step 1: Gather Palace Status
+
+Check if MCP tools are available (look for mempalace_status in available tools).
+
+- If MCP is available: Call the mempalace_status tool to retrieve palace state.
+- If MCP is not available: Run the CLI command: mempalace status
+
+## Step 2: Display Wing/Room/Drawer Counts
+
+Present the palace structure counts clearly:
+- Number of wings
+- Number of rooms
+- Number of drawers
+- Total memories stored
+
+Keep the output concise -- use a brief summary format, not verbose tables.
+
+## Step 3: Knowledge Graph Stats (MCP only)
+
+If MCP tools are available, also call:
+- mempalace_kg_stats -- for a knowledge graph overview (triple count, entity
+  count, relationship types)
+- mempalace_graph_stats -- for connectivity information (connected components,
+  average connections per entity)
+
+Present these alongside the palace counts in a unified summary.
+
+## Step 4: Suggest Next Actions
+
+Based on the current state, suggest one relevant action:
+
+- Empty palace (zero memories): Suggest "Try /mempalace:mine to add data from
+  files, URLs, or text."
+- Has data but no knowledge graph (memories exist but KG stats show zero
+  triples): Suggest "Consider adding knowledge graph triples for richer
+  queries."
+- Healthy palace (has memories and KG data): Suggest "Use /mempalace:search to
+  query your memories."
+
+## Output Style
+
+- Be concise and informative -- aim for a quick glance, not a report.
+- Use short labels and numbers, not prose paragraphs.
+- If any step fails or a tool is unavailable, note it briefly and continue
+  with what is available.

mempalace/instructions_cli.py

@@ -0,0 +1,28 @@
+"""
+Instruction text output for MemPalace CLI commands.
+
+Each instruction lives as a .md file in the instructions/ directory
+inside the package. The CLI reads and prints the file content.
+"""
+
+import sys
+from pathlib import Path
+
+INSTRUCTIONS_DIR = Path(__file__).parent / "instructions"
+
+AVAILABLE = ["init", "search", "mine", "help", "status"]
+
+
+def run_instructions(name: str):
+    """Read and print the instruction .md file for the given name."""
+    if name not in AVAILABLE:
+        print(f"Unknown instructions: {name}", file=sys.stderr)
+        print(f"Available: {', '.join(sorted(AVAILABLE))}", file=sys.stderr)
+        sys.exit(1)
+
+    md_path = INSTRUCTIONS_DIR / f"{name}.md"
+    if not md_path.is_file():
+        print(f"Instructions file not found: {md_path}", file=sys.stderr)
+        sys.exit(1)
+
+    print(md_path.read_text())

mempalace/knowledge_graph.py

@@ -39,6 +39,7 @@ import hashlib
 import json
 import os
 import sqlite3
+import threading
 from datetime import date, datetime
 from pathlib import Path
 
@@ -50,11 +51,15 @@ class KnowledgeGraph:
     def __init__(self, db_path: str = None):
         self.db_path = db_path or DEFAULT_KG_PATH
         Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._connection = None
+        self._lock = threading.Lock()
         self._init_db()
 
     def _init_db(self):
         conn = self._conn()
         conn.executescript("""
+            PRAGMA journal_mode=WAL;
+
             CREATE TABLE IF NOT EXISTS entities (
                 id TEXT PRIMARY KEY,
                 name TEXT NOT NULL,
@@ -84,10 +89,19 @@ class KnowledgeGraph:
             CREATE INDEX IF NOT EXISTS idx_triples_valid ON triples(valid_from, valid_to);
         """)
         conn.commit()
-        conn.close()
 
     def _conn(self):
-        return sqlite3.connect(self.db_path, timeout=10)
+        if self._connection is None:
+            self._connection = sqlite3.connect(self.db_path, timeout=10, check_same_thread=False)
+            self._connection.execute("PRAGMA journal_mode=WAL")
+            self._connection.row_factory = sqlite3.Row
+        return self._connection
+
+    def close(self):
+        """Close the database connection."""
+        if self._connection is not None:
+            self._connection.close()
+            self._connection = None
 
     def _entity_id(self, name: str) -> str:
         return name.lower().replace(" ", "_").replace("'", "")
@@ -98,13 +112,13 @@ class KnowledgeGraph:
         """Add or update an entity node."""
         eid = self._entity_id(name)
         props = json.dumps(properties or {})
-        conn = self._conn()
-        conn.execute(
-            "INSERT OR REPLACE INTO entities (id, name, type, properties) VALUES (?, ?, ?, ?)",
-            (eid, name, entity_type, props),
-        )
-        conn.commit()
-        conn.close()
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                conn.execute(
+                    "INSERT OR REPLACE INTO entities (id, name, type, properties) VALUES (?, ?, ?, ?)",
+                    (eid, name, entity_type, props),
+                )
         return eid
 
     def add_triple(
@@ -131,39 +145,42 @@ class KnowledgeGraph:
         pred = predicate.lower().replace(" ", "_")
 
         # Auto-create entities if they don't exist
-        conn = self._conn()
-        conn.execute("INSERT OR IGNORE INTO entities (id, name) VALUES (?, ?)", (sub_id, subject))
-        conn.execute("INSERT OR IGNORE INTO entities (id, name) VALUES (?, ?)", (obj_id, obj))
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                conn.execute(
+                    "INSERT OR IGNORE INTO entities (id, name) VALUES (?, ?)", (sub_id, subject)
+                )
+                conn.execute(
+                    "INSERT OR IGNORE INTO entities (id, name) VALUES (?, ?)", (obj_id, obj)
+                )
 
-        # Check for existing identical triple
-        existing = conn.execute(
-            "SELECT id FROM triples WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
-            (sub_id, pred, obj_id),
-        ).fetchone()
+                # Check for existing identical triple
+                existing = conn.execute(
+                    "SELECT id FROM triples WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
+                    (sub_id, pred, obj_id),
+                ).fetchone()
 
-        if existing:
-            conn.close()
-            return existing[0]  # Already exists and still valid
+                if existing:
+                    return existing["id"]  # Already exists and still valid
 
-        triple_id = f"t_{sub_id}_{pred}_{obj_id}_{hashlib.md5(f'{valid_from}{datetime.now().isoformat()}'.encode()).hexdigest()[:8]}"
+                triple_id = f"t_{sub_id}_{pred}_{obj_id}_{hashlib.sha256(f'{valid_from}{datetime.now().isoformat()}'.encode()).hexdigest()[:12]}"
 
-        conn.execute(
-            """INSERT INTO triples (id, subject, predicate, object, valid_from, valid_to, confidence, source_closet, source_file)
-               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
-            (
-                triple_id,
-                sub_id,
-                pred,
-                obj_id,
-                valid_from,
-                valid_to,
-                confidence,
-                source_closet,
-                source_file,
-            ),
-        )
-        conn.commit()
-        conn.close()
+                conn.execute(
+                    """INSERT INTO triples (id, subject, predicate, object, valid_from, valid_to, confidence, source_closet, source_file)
+                       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                    (
+                        triple_id,
+                        sub_id,
+                        pred,
+                        obj_id,
+                        valid_from,
+                        valid_to,
+                        confidence,
+                        source_closet,
+                        source_file,
+                    ),
+                )
         return triple_id
 
     def invalidate(self, subject: str, predicate: str, obj: str, ended: str = None):
@@ -173,13 +190,13 @@ class KnowledgeGraph:
         pred = predicate.lower().replace(" ", "_")
         ended = ended or date.today().isoformat()
 
-        conn = self._conn()
-        conn.execute(
-            "UPDATE triples SET valid_to=? WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
-            (ended, sub_id, pred, obj_id),
-        )
-        conn.commit()
-        conn.close()
+        with self._lock:
+            conn = self._conn()
+            with conn:
+                conn.execute(
+                    "UPDATE triples SET valid_to=? WHERE subject=? AND predicate=? AND object=? AND valid_to IS NULL",
+                    (ended, sub_id, pred, obj_id),
+                )
 
     # ── Query operations ──────────────────────────────────────────────────
 
@@ -191,53 +208,53 @@ class KnowledgeGraph:
         as_of: date string — only return facts valid at that time
         """
         eid = self._entity_id(name)
-        conn = self._conn()
 
         results = []
+        with self._lock:
+            conn = self._conn()
 
-        if direction in ("outgoing", "both"):
-            query = "SELECT t.*, e.name as obj_name FROM triples t JOIN entities e ON t.object = e.id WHERE t.subject = ?"
-            params = [eid]
-            if as_of:
-                query += " AND (t.valid_from IS NULL OR t.valid_from <= ?) AND (t.valid_to IS NULL OR t.valid_to >= ?)"
-                params.extend([as_of, as_of])
-            for row in conn.execute(query, params).fetchall():
-                results.append(
-                    {
-                        "direction": "outgoing",
-                        "subject": name,
-                        "predicate": row[2],
-                        "object": row[10],  # obj_name
-                        "valid_from": row[4],
-                        "valid_to": row[5],
-                        "confidence": row[6],
-                        "source_closet": row[7],
-                        "current": row[5] is None,
-                    }
-                )
+            if direction in ("outgoing", "both"):
+                query = "SELECT t.*, e.name as obj_name FROM triples t JOIN entities e ON t.object = e.id WHERE t.subject = ?"
+                params = [eid]
+                if as_of:
+                    query += " AND (t.valid_from IS NULL OR t.valid_from <= ?) AND (t.valid_to IS NULL OR t.valid_to >= ?)"
+                    params.extend([as_of, as_of])
+                for row in conn.execute(query, params).fetchall():
+                    results.append(
+                        {
+                            "direction": "outgoing",
+                            "subject": name,
+                            "predicate": row["predicate"],
+                            "object": row["obj_name"],
+                            "valid_from": row["valid_from"],
+                            "valid_to": row["valid_to"],
+                            "confidence": row["confidence"],
+                            "source_closet": row["source_closet"],
+                            "current": row["valid_to"] is None,
+                        }
+                    )
 
-        if direction in ("incoming", "both"):
-            query = "SELECT t.*, e.name as sub_name FROM triples t JOIN entities e ON t.subject = e.id WHERE t.object = ?"
-            params = [eid]
-            if as_of:
-                query += " AND (t.valid_from IS NULL OR t.valid_from <= ?) AND (t.valid_to IS NULL OR t.valid_to >= ?)"
-                params.extend([as_of, as_of])
-            for row in conn.execute(query, params).fetchall():
-                results.append(
-                    {
-                        "direction": "incoming",
-                        "subject": row[10],  # sub_name
-                        "predicate": row[2],
-                        "object": name,
-                        "valid_from": row[4],
-                        "valid_to": row[5],
-                        "confidence": row[6],
-                        "source_closet": row[7],
-                        "current": row[5] is None,
-                    }
-                )
+            if direction in ("incoming", "both"):
+                query = "SELECT t.*, e.name as sub_name FROM triples t JOIN entities e ON t.subject = e.id WHERE t.object = ?"
+                params = [eid]
+                if as_of:
+                    query += " AND (t.valid_from IS NULL OR t.valid_from <= ?) AND (t.valid_to IS NULL OR t.valid_to >= ?)"
+                    params.extend([as_of, as_of])
+                for row in conn.execute(query, params).fetchall():
+                    results.append(
+                        {
+                            "direction": "incoming",
+                            "subject": row["sub_name"],
+                            "predicate": row["predicate"],
+                            "object": name,
+                            "valid_from": row["valid_from"],
+                            "valid_to": row["valid_to"],
+                            "confidence": row["confidence"],
+                            "source_closet": row["source_closet"],
+                            "current": row["valid_to"] is None,
+                        }
+                    )
 
-        conn.close()
         return results
 
     def query_relationship(self, predicate: str, as_of: str = None):
@@ -260,15 +277,14 @@ class KnowledgeGraph:
         for row in conn.execute(query, params).fetchall():
             results.append(
                 {
-                    "subject": row[10],
+                    "subject": row["sub_name"],
                     "predicate": pred,
-                    "object": row[11],
-                    "valid_from": row[4],
-                    "valid_to": row[5],
-                    "current": row[5] is None,
+                    "object": row["obj_name"],
+                    "valid_from": row["valid_from"],
+                    "valid_to": row["valid_to"],
+                    "current": row["valid_to"] is None,
                 }
             )
-        conn.close()
         return results
 
     def timeline(self, entity_name: str = None):
@@ -284,6 +300,7 @@ class KnowledgeGraph:
                 JOIN entities o ON t.object = o.id
                 WHERE (t.subject = ? OR t.object = ?)
                 ORDER BY t.valid_from ASC NULLS LAST
+                LIMIT 100
             """,
                 (eid, eid),
             ).fetchall()
@@ -297,15 +314,14 @@ class KnowledgeGraph:
                 LIMIT 100
             """).fetchall()
 
-        conn.close()
         return [
             {
-                "subject": r[10],
-                "predicate": r[2],
-                "object": r[11],
-                "valid_from": r[4],
-                "valid_to": r[5],
-                "current": r[5] is None,
+                "subject": r["sub_name"],
+                "predicate": r["predicate"],
+                "object": r["obj_name"],
+                "valid_from": r["valid_from"],
+                "valid_to": r["valid_to"],
+                "current": r["valid_to"] is None,
             }
             for r in rows
         ]
@@ -314,17 +330,18 @@ class KnowledgeGraph:
 
     def stats(self):
         conn = self._conn()
-        entities = conn.execute("SELECT COUNT(*) FROM entities").fetchone()[0]
-        triples = conn.execute("SELECT COUNT(*) FROM triples").fetchone()[0]
-        current = conn.execute("SELECT COUNT(*) FROM triples WHERE valid_to IS NULL").fetchone()[0]
+        entities = conn.execute("SELECT COUNT(*) as cnt FROM entities").fetchone()["cnt"]
+        triples = conn.execute("SELECT COUNT(*) as cnt FROM triples").fetchone()["cnt"]
+        current = conn.execute(
+            "SELECT COUNT(*) as cnt FROM triples WHERE valid_to IS NULL"
+        ).fetchone()["cnt"]
         expired = triples - current
         predicates = [
-            r[0]
+            r["predicate"]
             for r in conn.execute(
                 "SELECT DISTINCT predicate FROM triples ORDER BY predicate"
             ).fetchall()
         ]
-        conn.close()
         return {
             "entities": entities,
             "triples": triples,

mempalace/layers.py

@@ -21,9 +21,9 @@ import sys
 from pathlib import Path
 from collections import defaultdict
 
-import chromadb
-
 from .config import MempalaceConfig
+from .palace import get_collection as _get_collection
+from .searcher import build_where_filter
 
 
 # ---------------------------------------------------------------------------
@@ -82,6 +82,7 @@ class Layer1:
 
     MAX_DRAWERS = 15  # at most 15 moments in wake-up
     MAX_CHARS = 3200  # hard cap on total L1 text (~800 tokens)
+    MAX_SCAN = 2000  # don't scan more than this for L1 generation
 
     def __init__(self, palace_path: str = None, wing: str = None):
         cfg = MempalaceConfig()
@@ -91,23 +92,31 @@ class Layer1:
     def generate(self) -> str:
         """Pull top drawers from ChromaDB and format as compact L1 text."""
         try:
-            client = chromadb.PersistentClient(path=self.palace_path)
-            col = client.get_collection("mempalace_drawers")
+            col = _get_collection(self.palace_path, create=False)
         except Exception:
             return "## L1 — No palace found. Run: mempalace mine <dir>"
 
-        # Fetch all drawers (with optional wing filter)
-        kwargs = {"include": ["documents", "metadatas"]}
-        if self.wing:
-            kwargs["where"] = {"wing": self.wing}
-
-        try:
-            results = col.get(**kwargs)
-        except Exception:
-            return "## L1 — No drawers found."
-
-        docs = results.get("documents", [])
-        metas = results.get("metadatas", [])
+        # Fetch all drawers in batches to avoid SQLite variable limit (~999)
+        _BATCH = 500
+        docs, metas = [], []
+        offset = 0
+        while True:
+            kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset}
+            if self.wing:
+                kwargs["where"] = {"wing": self.wing}
+            try:
+                batch = col.get(**kwargs)
+            except Exception:
+                break
+            batch_docs = batch.get("documents", [])
+            batch_metas = batch.get("metadatas", [])
+            if not batch_docs:
+                break
+            docs.extend(batch_docs)
+            metas.extend(batch_metas)
+            offset += len(batch_docs)
+            if len(batch_docs) < _BATCH or len(docs) >= self.MAX_SCAN:
+                break
 
         if not docs:
             return "## L1 — No memories yet."
@@ -187,18 +196,11 @@ class Layer2:
     def retrieve(self, wing: str = None, room: str = None, n_results: int = 10) -> str:
         """Retrieve drawers filtered by wing and/or room."""
         try:
-            client = chromadb.PersistentClient(path=self.palace_path)
-            col = client.get_collection("mempalace_drawers")
+            col = _get_collection(self.palace_path, create=False)
         except Exception:
             return "No palace found."
 
-        where = {}
-        if wing and room:
-            where = {"$and": [{"wing": wing}, {"room": room}]}
-        elif wing:
-            where = {"wing": wing}
-        elif room:
-            where = {"room": room}
+        where = build_where_filter(wing, room)
 
         kwargs = {"include": ["documents", "metadatas"], "limit": n_results}
         if where:
@@ -251,18 +253,11 @@ class Layer3:
     def search(self, query: str, wing: str = None, room: str = None, n_results: int = 5) -> str:
         """Semantic search, returns compact result text."""
         try:
-            client = chromadb.PersistentClient(path=self.palace_path)
-            col = client.get_collection("mempalace_drawers")
+            col = _get_collection(self.palace_path, create=False)
         except Exception:
             return "No palace found."
 
-        where = {}
-        if wing and room:
-            where = {"$and": [{"wing": wing}, {"room": room}]}
-        elif wing:
-            where = {"wing": wing}
-        elif room:
-            where = {"room": room}
+        where = build_where_filter(wing, room)
 
         kwargs = {
             "query_texts": [query],
@@ -307,18 +302,11 @@ class Layer3:
     ) -> list:
         """Return raw dicts instead of formatted text."""
         try:
-            client = chromadb.PersistentClient(path=self.palace_path)
-            col = client.get_collection("mempalace_drawers")
+            col = _get_collection(self.palace_path, create=False)
         except Exception:
             return []
 
-        where = {}
-        if wing and room:
-            where = {"$and": [{"wing": wing}, {"room": room}]}
-        elif wing:
-            where = {"wing": wing}
-        elif room:
-            where = {"room": room}
+        where = build_where_filter(wing, room)
 
         kwargs = {
             "query_texts": [query],
@@ -428,8 +416,7 @@ class MemoryStack:
 
         # Count drawers
         try:
-            client = chromadb.PersistentClient(path=self.palace_path)
-            col = client.get_collection("mempalace_drawers")
+            col = _get_collection(self.palace_path, create=False)
             count = col.count()
             result["total_drawers"] = count
         except Exception:

mempalace/migrate.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+"""
+mempalace migrate — Recover a palace created with a different ChromaDB version.
+
+Reads documents and metadata directly from the palace's SQLite database
+(bypassing ChromaDB's API, which fails on version-mismatched palaces),
+then re-imports everything into a fresh palace using the currently installed
+ChromaDB version.
+
+This fixes the 3.0.0 → 3.1.0 upgrade path where chromadb was downgraded
+from 1.5.x to 0.6.x, breaking the on-disk storage format.
+
+Usage:
+    mempalace migrate                          # migrate default palace
+    mempalace migrate --palace /path/to/palace  # migrate specific palace
+    mempalace migrate --dry-run                # show what would be migrated
+"""
+
+import os
+import shutil
+import sqlite3
+from collections import defaultdict
+from datetime import datetime
+
+
+def extract_drawers_from_sqlite(db_path: str) -> list:
+    """Read all drawers directly from ChromaDB's SQLite, bypassing the API.
+
+    Works regardless of which ChromaDB version created the database.
+    Returns list of dicts with 'id', 'document', and 'metadata' keys.
+    """
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+
+    # Get all embedding IDs and their documents
+    rows = conn.execute("""
+        SELECT e.embedding_id,
+               MAX(CASE WHEN em.key = 'chroma:document' THEN em.string_value END) as document
+        FROM embeddings e
+        JOIN embedding_metadata em ON em.id = e.id
+        GROUP BY e.embedding_id
+    """).fetchall()
+
+    drawers = []
+    for row in rows:
+        embedding_id = row["embedding_id"]
+        document = row["document"]
+        if not document:
+            continue
+
+        # Get metadata for this embedding
+        meta_rows = conn.execute(
+            """
+            SELECT em.key, em.string_value, em.int_value, em.float_value, em.bool_value
+            FROM embedding_metadata em
+            JOIN embeddings e ON e.id = em.id
+            WHERE e.embedding_id = ?
+              AND em.key NOT LIKE 'chroma:%'
+        """,
+            (embedding_id,),
+        ).fetchall()
+
+        metadata = {}
+        for mr in meta_rows:
+            key = mr["key"]
+            if mr["string_value"] is not None:
+                metadata[key] = mr["string_value"]
+            elif mr["int_value"] is not None:
+                metadata[key] = mr["int_value"]
+            elif mr["float_value"] is not None:
+                metadata[key] = mr["float_value"]
+            elif mr["bool_value"] is not None:
+                metadata[key] = bool(mr["bool_value"])
+
+        drawers.append(
+            {
+                "id": embedding_id,
+                "document": document,
+                "metadata": metadata,
+            }
+        )
+
+    conn.close()
+    return drawers
+
+
+def detect_chromadb_version(db_path: str) -> str:
+    """Detect which ChromaDB version created the database by checking schema."""
+    conn = sqlite3.connect(db_path)
+    try:
+        # 1.x has schema_str column in collections table
+        cols = [r[1] for r in conn.execute("PRAGMA table_info(collections)").fetchall()]
+        if "schema_str" in cols:
+            return "1.x"
+        # 0.6.x has embeddings_queue but no schema_str
+        tables = [
+            r[0]
+            for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
+        ]
+        if "embeddings_queue" in tables:
+            return "0.6.x"
+        return "unknown"
+    finally:
+        conn.close()
+
+
+def contains_palace_database(path: str) -> bool:
+    """Return True when path looks like a MemPalace ChromaDB directory."""
+    return os.path.isfile(os.path.join(path, "chroma.sqlite3"))
+
+
+def confirm_destructive_action(
+    operation_name: str, palace_path: str, assume_yes: bool = False
+) -> bool:
+    """Require confirmation before destructive palace operations."""
+    if assume_yes:
+        return True
+
+    print(f"\n  {operation_name} will replace data in: {palace_path}")
+    print("  A backup will be created first, then the palace will be rebuilt.")
+    try:
+        answer = input("  Continue? [y/N]: ").strip().lower()
+    except EOFError:
+        print("  Aborted. Re-run with --yes to confirm destructive changes.")
+        return False
+
+    if answer not in {"y", "yes"}:
+        print("  Aborted.")
+        return False
+    return True
+
+
+def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
+    """Migrate a palace to the currently installed ChromaDB version."""
+    import chromadb
+
+    palace_path = os.path.abspath(os.path.expanduser(palace_path))
+    db_path = os.path.join(palace_path, "chroma.sqlite3")
+
+    if not os.path.isdir(palace_path) or not contains_palace_database(palace_path):
+        print(f"\n  No palace database found at {db_path}")
+        return False
+
+    print(f"\n{'=' * 60}")
+    print("  MemPalace Migrate")
+    print(f"{'=' * 60}\n")
+    print(f"  Palace:    {palace_path}")
+    print(f"  Database:  {db_path}")
+    print(f"  DB size:   {os.path.getsize(db_path) / 1024 / 1024:.1f} MB")
+
+    # Detect version
+    source_version = detect_chromadb_version(db_path)
+    print(f"  Source:    ChromaDB {source_version}")
+    print(f"  Target:    ChromaDB {chromadb.__version__}")
+
+    # Try reading with current chromadb first
+    try:
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+        count = col.count()
+        print(f"\n  Palace is already readable by chromadb {chromadb.__version__}.")
+        print(f"  {count} drawers found. No migration needed.")
+        return True
+    except Exception:
+        print(f"\n  Palace is NOT readable by chromadb {chromadb.__version__}.")
+        print("  Extracting from SQLite directly...")
+
+    # Extract all drawers via raw SQL
+    drawers = extract_drawers_from_sqlite(db_path)
+    print(f"  Extracted {len(drawers)} drawers from SQLite")
+
+    if not drawers:
+        print("  Nothing to migrate.")
+        return True
+
+    # Show summary
+    wings = defaultdict(lambda: defaultdict(int))
+    for d in drawers:
+        w = d["metadata"].get("wing", "?")
+        r = d["metadata"].get("room", "?")
+        wings[w][r] += 1
+
+    print("\n  Summary:")
+    for wing, rooms in sorted(wings.items()):
+        total = sum(rooms.values())
+        print(f"    WING: {wing} ({total} drawers)")
+        for room, count in sorted(rooms.items(), key=lambda x: -x[1]):
+            print(f"      ROOM: {room:30} {count:5}")
+
+    if dry_run:
+        print("\n  DRY RUN — no changes made.")
+        print(f"  Would migrate {len(drawers)} drawers.")
+        return True
+
+    if not confirm_destructive_action("Migration", palace_path, assume_yes=confirm):
+        return False
+
+    # Backup the old palace
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    backup_path = f"{palace_path}.pre-migrate.{timestamp}"
+    print(f"\n  Backing up to {backup_path}...")
+    shutil.copytree(palace_path, backup_path)
+
+    # Build fresh palace in a temp directory (avoids chromadb reading old state)
+    import tempfile
+
+    temp_palace = tempfile.mkdtemp(prefix="mempalace_migrate_")
+    print(f"  Creating fresh palace in {temp_palace}...")
+    client = chromadb.PersistentClient(path=temp_palace)
+    col = client.get_or_create_collection("mempalace_drawers")
+
+    # Re-import in batches
+    batch_size = 500
+    imported = 0
+    for i in range(0, len(drawers), batch_size):
+        batch = drawers[i : i + batch_size]
+        col.add(
+            ids=[d["id"] for d in batch],
+            documents=[d["document"] for d in batch],
+            metadatas=[d["metadata"] for d in batch],
+        )
+        imported += len(batch)
+        print(f"  Imported {imported}/{len(drawers)} drawers...")
+
+    # Verify before swapping
+    final_count = col.count()
+    del col
+    del client
+
+    # Swap: remove old palace, move new one into place
+    print("  Swapping old palace for migrated version...")
+    shutil.rmtree(palace_path)
+    shutil.move(temp_palace, palace_path)
+
+    print("\n  Migration complete.")
+    print(f"  Drawers migrated: {final_count}")
+    print(f"  Backup at: {backup_path}")
+
+    if final_count != len(drawers):
+        print(f"  WARNING: Expected {len(drawers)}, got {final_count}")
+
+    print(f"\n{'=' * 60}\n")
+    return True

mempalace/miner.py

@@ -10,11 +10,12 @@ Stores verbatim chunks as drawers. No summaries. Ever.
 import os
 import sys
 import hashlib
+import fnmatch
 from pathlib import Path
 from datetime import datetime
 from collections import defaultdict
 
-import chromadb
+from .palace import SKIP_DIRS, get_collection, file_already_mined
 
 READABLE_EXTENSIONS = {
     ".txt",
@@ -39,23 +40,209 @@ READABLE_EXTENSIONS = {
     ".toml",
 }
 
-SKIP_DIRS = {
-    ".git",
-    "node_modules",
-    "__pycache__",
-    ".venv",
-    "venv",
-    "env",
-    "dist",
-    "build",
-    ".next",
-    "coverage",
-    ".mempalace",
+SKIP_FILENAMES = {
+    "mempalace.yaml",
+    "mempalace.yml",
+    "mempal.yaml",
+    "mempal.yml",
+    ".gitignore",
+    "package-lock.json",
 }
 
 CHUNK_SIZE = 800  # chars per drawer
 CHUNK_OVERLAP = 100  # overlap between chunks
 MIN_CHUNK_SIZE = 50  # skip tiny chunks
+MAX_FILE_SIZE = 10 * 1024 * 1024  # 10 MB — skip files larger than this
+
+
+# =============================================================================
+# IGNORE MATCHING
+# =============================================================================
+
+
+class GitignoreMatcher:
+    """Lightweight matcher for one directory's .gitignore patterns."""
+
+    def __init__(self, base_dir: Path, rules: list):
+        self.base_dir = base_dir
+        self.rules = rules
+
+    @classmethod
+    def from_dir(cls, dir_path: Path):
+        gitignore_path = dir_path / ".gitignore"
+        if not gitignore_path.is_file():
+            return None
+
+        try:
+            lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines()
+        except Exception:
+            return None
+
+        rules = []
+        for raw_line in lines:
+            line = raw_line.strip()
+            if not line:
+                continue
+
+            if line.startswith("\\#") or line.startswith("\\!"):
+                line = line[1:]
+            elif line.startswith("#"):
+                continue
+
+            negated = line.startswith("!")
+            if negated:
+                line = line[1:]
+
+            anchored = line.startswith("/")
+            if anchored:
+                line = line.lstrip("/")
+
+            dir_only = line.endswith("/")
+            if dir_only:
+                line = line.rstrip("/")
+
+            if not line:
+                continue
+
+            rules.append(
+                {
+                    "pattern": line,
+                    "anchored": anchored,
+                    "dir_only": dir_only,
+                    "negated": negated,
+                }
+            )
+
+        if not rules:
+            return None
+
+        return cls(dir_path, rules)
+
+    def matches(self, path: Path, is_dir: bool = None):
+        try:
+            relative = path.relative_to(self.base_dir).as_posix().strip("/")
+        except ValueError:
+            return None
+
+        if not relative:
+            return None
+
+        if is_dir is None:
+            is_dir = path.is_dir()
+
+        ignored = None
+        for rule in self.rules:
+            if self._rule_matches(rule, relative, is_dir):
+                ignored = not rule["negated"]
+        return ignored
+
+    def _rule_matches(self, rule: dict, relative: str, is_dir: bool) -> bool:
+        pattern = rule["pattern"]
+        parts = relative.split("/")
+        pattern_parts = pattern.split("/")
+
+        if rule["dir_only"]:
+            target_parts = parts if is_dir else parts[:-1]
+            if not target_parts:
+                return False
+            if rule["anchored"] or len(pattern_parts) > 1:
+                return self._match_from_root(target_parts, pattern_parts)
+            return any(fnmatch.fnmatch(part, pattern) for part in target_parts)
+
+        if rule["anchored"] or len(pattern_parts) > 1:
+            return self._match_from_root(parts, pattern_parts)
+
+        return any(fnmatch.fnmatch(part, pattern) for part in parts)
+
+    def _match_from_root(self, target_parts: list, pattern_parts: list) -> bool:
+        def matches(path_index: int, pattern_index: int) -> bool:
+            if pattern_index == len(pattern_parts):
+                return True
+
+            if path_index == len(target_parts):
+                return all(part == "**" for part in pattern_parts[pattern_index:])
+
+            pattern_part = pattern_parts[pattern_index]
+            if pattern_part == "**":
+                return matches(path_index, pattern_index + 1) or matches(
+                    path_index + 1, pattern_index
+                )
+
+            if not fnmatch.fnmatch(target_parts[path_index], pattern_part):
+                return False
+
+            return matches(path_index + 1, pattern_index + 1)
+
+        return matches(0, 0)
+
+
+def load_gitignore_matcher(dir_path: Path, cache: dict):
+    """Load and cache one directory's .gitignore matcher."""
+    if dir_path not in cache:
+        cache[dir_path] = GitignoreMatcher.from_dir(dir_path)
+    return cache[dir_path]
+
+
+def is_gitignored(path: Path, matchers: list, is_dir: bool = False) -> bool:
+    """Apply active .gitignore matchers in ancestor order; last match wins."""
+    ignored = False
+    for matcher in matchers:
+        decision = matcher.matches(path, is_dir=is_dir)
+        if decision is not None:
+            ignored = decision
+    return ignored
+
+
+def should_skip_dir(dirname: str) -> bool:
+    """Skip known generated/cache directories before gitignore matching."""
+    return dirname in SKIP_DIRS or dirname.endswith(".egg-info")
+
+
+def normalize_include_paths(include_ignored: list) -> set:
+    """Normalize comma-parsed include paths into project-relative POSIX strings."""
+    normalized = set()
+    for raw_path in include_ignored or []:
+        candidate = str(raw_path).strip().strip("/")
+        if candidate:
+            normalized.add(Path(candidate).as_posix())
+    return normalized
+
+
+def is_exact_force_include(path: Path, project_path: Path, include_paths: set) -> bool:
+    """Return True when a path exactly matches an explicit include override."""
+    if not include_paths:
+        return False
+
+    try:
+        relative = path.relative_to(project_path).as_posix().strip("/")
+    except ValueError:
+        return False
+
+    return relative in include_paths
+
+
+def is_force_included(path: Path, project_path: Path, include_paths: set) -> bool:
+    """Return True when a path or one of its ancestors/descendants was explicitly included."""
+    if not include_paths:
+        return False
+
+    try:
+        relative = path.relative_to(project_path).as_posix().strip("/")
+    except ValueError:
+        return False
+
+    if not relative:
+        return False
+
+    for include_path in include_paths:
+        if relative == include_path:
+            return True
+        if relative.startswith(f"{include_path}/"):
+            return True
+        if include_path.startswith(f"{relative}/"):
+            return True
+
+    return False
 
 
 # =============================================================================
@@ -99,11 +286,12 @@ def detect_room(filepath: Path, content: str, rooms: list, project_path: Path) -
     filename = filepath.stem.lower()
     content_lower = content[:2000].lower()
 
-    # Priority 1: folder path contains room name
+    # Priority 1: folder path matches room name or keywords
     path_parts = relative.replace("\\", "/").split("/")
     for part in path_parts[:-1]:  # skip filename itself
         for room in rooms:
-            if room["name"].lower() in part or part in room["name"].lower():
+            candidates = [room["name"].lower()] + [k.lower() for k in room.get("keywords", [])]
+            if any(part == c or c in part or part in c for c in candidates):
                 return room["name"]
 
     # Priority 2: filename matches room name
@@ -180,48 +368,32 @@ def chunk_text(content: str, source_file: str) -> list:
 # =============================================================================
 
 
-def get_collection(palace_path: str):
-    os.makedirs(palace_path, exist_ok=True)
-    client = chromadb.PersistentClient(path=palace_path)
-    try:
-        return client.get_collection("mempalace_drawers")
-    except Exception:
-        return client.create_collection("mempalace_drawers")
-
-
-def file_already_mined(collection, source_file: str) -> bool:
-    """Fast check: has this file been filed before?"""
-    try:
-        results = collection.get(where={"source_file": source_file}, limit=1)
-        return len(results.get("ids", [])) > 0
-    except Exception:
-        return False
-
-
 def add_drawer(
     collection, wing: str, room: str, content: str, source_file: str, chunk_index: int, agent: str
 ):
     """Add one drawer to the palace."""
-    drawer_id = f"drawer_{wing}_{room}_{hashlib.md5((source_file + str(chunk_index)).encode()).hexdigest()[:16]}"
+    drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(chunk_index)).encode()).hexdigest()[:24]}"
     try:
-        collection.add(
+        metadata = {
+            "wing": wing,
+            "room": room,
+            "source_file": source_file,
+            "chunk_index": chunk_index,
+            "added_by": agent,
+            "filed_at": datetime.now().isoformat(),
+        }
+        # Store file mtime so we can detect modifications later.
+        try:
+            metadata["source_mtime"] = os.path.getmtime(source_file)
+        except OSError:
+            pass
+        collection.upsert(
             documents=[content],
             ids=[drawer_id],
-            metadatas=[
-                {
-                    "wing": wing,
-                    "room": room,
-                    "source_file": source_file,
-                    "chunk_index": chunk_index,
-                    "added_by": agent,
-                    "filed_at": datetime.now().isoformat(),
-                }
-            ],
+            metadatas=[metadata],
         )
         return True
-    except Exception as e:
-        if "already exists" in str(e).lower() or "duplicate" in str(e).lower():
-            return False
+    except Exception:
         raise
 
 
@@ -238,29 +410,39 @@ def process_file(
     rooms: list,
     agent: str,
     dry_run: bool,
-) -> int:
-    """Read, chunk, route, and file one file. Returns drawer count."""
+) -> tuple:
+    """Read, chunk, route, and file one file. Returns (drawer_count, room_name)."""
 
     # Skip if already filed
     source_file = str(filepath)
-    if not dry_run and file_already_mined(collection, source_file):
-        return 0
+    if not dry_run and file_already_mined(collection, source_file, check_mtime=True):
+        return 0, "general"
 
     try:
         content = filepath.read_text(encoding="utf-8", errors="replace")
-    except Exception:
-        return 0
+    except OSError:
+        return 0, "general"
 
     content = content.strip()
     if len(content) < MIN_CHUNK_SIZE:
-        return 0
+        return 0, "general"
 
     room = detect_room(filepath, content, rooms, project_path)
     chunks = chunk_text(content, source_file)
 
     if dry_run:
         print(f"    [DRY RUN] {filepath.name} → room:{room} ({len(chunks)} drawers)")
-        return len(chunks)
+        return len(chunks), room
+
+    # Purge stale drawers for this file before re-inserting the fresh chunks.
+    # Converts modified-file re-mines from upsert-over-existing-IDs (which hits
+    # hnswlib's thread-unsafe updatePoint path and can segfault on macOS ARM
+    # with chromadb 0.6.3) into a clean delete+insert, bypassing the update
+    # path entirely.
+    try:
+        collection.delete(where={"source_file": source_file})
+    except Exception:
+        pass
 
     drawers_added = 0
     for chunk in chunks:
@@ -276,7 +458,7 @@ def process_file(
         if added:
             drawers_added += 1
 
-    return drawers_added
+    return drawers_added, room
 
 
 # =============================================================================
@@ -284,26 +466,67 @@ def process_file(
 # =============================================================================
 
 
-def scan_project(project_dir: str) -> list:
+def scan_project(
+    project_dir: str,
+    respect_gitignore: bool = True,
+    include_ignored: list = None,
+) -> list:
     """Return list of all readable file paths."""
     project_path = Path(project_dir).expanduser().resolve()
     files = []
+    active_matchers = []
+    matcher_cache = {}
+    include_paths = normalize_include_paths(include_ignored)
+
     for root, dirs, filenames in os.walk(project_path):
-        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+        root_path = Path(root)
+
+        if respect_gitignore:
+            active_matchers = [
+                matcher
+                for matcher in active_matchers
+                if root_path == matcher.base_dir or matcher.base_dir in root_path.parents
+            ]
+            current_matcher = load_gitignore_matcher(root_path, matcher_cache)
+            if current_matcher is not None:
+                active_matchers.append(current_matcher)
+
+        dirs[:] = [
+            d
+            for d in dirs
+            if is_force_included(root_path / d, project_path, include_paths)
+            or not should_skip_dir(d)
+        ]
+        if respect_gitignore and active_matchers:
+            dirs[:] = [
+                d
+                for d in dirs
+                if is_force_included(root_path / d, project_path, include_paths)
+                or not is_gitignored(root_path / d, active_matchers, is_dir=True)
+            ]
+
         for filename in filenames:
-            filepath = Path(root) / filename
-            if filepath.suffix.lower() in READABLE_EXTENSIONS:
-                # Skip config files
-                if filename in (
-                    "mempalace.yaml",
-                    "mempalace.yml",
-                    "mempal.yaml",
-                    "mempal.yml",
-                    ".gitignore",
-                    "package-lock.json",
-                ):
+            filepath = root_path / filename
+            force_include = is_force_included(filepath, project_path, include_paths)
+            exact_force_include = is_exact_force_include(filepath, project_path, include_paths)
+
+            if not force_include and filename in SKIP_FILENAMES:
+                continue
+            if filepath.suffix.lower() not in READABLE_EXTENSIONS and not exact_force_include:
+                continue
+            if respect_gitignore and active_matchers and not force_include:
+                if is_gitignored(filepath, active_matchers, is_dir=False):
                     continue
-                files.append(filepath)
+            # Skip symlinks — prevents following links to /dev/urandom, etc.
+            if filepath.is_symlink():
+                continue
+            # Skip files exceeding size limit
+            try:
+                if filepath.stat().st_size > MAX_FILE_SIZE:
+                    continue
+            except OSError:
+                continue
+            files.append(filepath)
     return files
 
 
@@ -319,6 +542,8 @@ def mine(
     agent: str = "mempalace",
     limit: int = 0,
     dry_run: bool = False,
+    respect_gitignore: bool = True,
+    include_ignored: list = None,
 ):
     """Mine a project directory into the palace."""
 
@@ -328,7 +553,11 @@ def mine(
     wing = wing_override or config["wing"]
     rooms = config.get("rooms", [{"name": "general", "description": "All project files"}])
 
-    files = scan_project(project_dir)
+    files = scan_project(
+        project_dir,
+        respect_gitignore=respect_gitignore,
+        include_ignored=include_ignored,
+    )
     if limit > 0:
         files = files[:limit]
 
@@ -341,6 +570,10 @@ def mine(
     print(f"  Palace:  {palace_path}")
     if dry_run:
         print("  DRY RUN — nothing will be filed")
+    if not respect_gitignore:
+        print("  .gitignore: DISABLED")
+    if include_ignored:
+        print(f"  Include: {', '.join(sorted(normalize_include_paths(include_ignored)))}")
     print(f"{'─' * 55}\n")
 
     if not dry_run:
@@ -353,7 +586,7 @@ def mine(
     room_counts = defaultdict(int)
 
     for i, filepath in enumerate(files, 1):
-        drawers = process_file(
+        drawers, room = process_file(
             filepath=filepath,
             project_path=project_path,
             collection=collection,
@@ -366,7 +599,6 @@ def mine(
             files_skipped += 1
         else:
             total_drawers += drawers
-            room = detect_room(filepath, "", rooms, project_path)
             room_counts[room] += 1
             if not dry_run:
                 print(f"  ✓ [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers}")
@@ -391,15 +623,15 @@ def mine(
 def status(palace_path: str):
     """Show what's been filed in the palace."""
     try:
-        client = chromadb.PersistentClient(path=palace_path)
-        col = client.get_collection("mempalace_drawers")
+        col = get_collection(palace_path, create=False)
     except Exception:
         print(f"\n  No palace found at {palace_path}")
         print("  Run: mempalace init <dir> then mempalace mine <dir>")
         return
 
     # Count by wing and room
-    r = col.get(limit=10000, include=["metadatas"])
+    total = col.count()
+    r = col.get(limit=total, include=["metadatas"]) if total else {"metadatas": []}
     metas = r["metadatas"]
 
     wing_rooms = defaultdict(lambda: defaultdict(int))

mempalace/normalize.py

@@ -6,7 +6,8 @@ Supported:
     - Plain text with > markers (pass through)
     - Claude.ai JSON export
     - ChatGPT conversations.json
-    - Claude Code JSONL
+    - Claude Code JSONL (with tool_use/tool_result block capture)
+    - OpenAI Codex CLI JSONL
     - Slack JSON export
     - Plain text (pass through for paragraph chunking)
 
@@ -24,10 +25,16 @@ def normalize(filepath: str) -> str:
     Load a file and normalize to transcript format if it's a chat export.
     Plain text files pass through unchanged.
     """
+    try:
+        file_size = os.path.getsize(filepath)
+    except OSError as e:
+        raise IOError(f"Could not read {filepath}: {e}")
+    if file_size > 500 * 1024 * 1024:  # 500 MB safety limit
+        raise IOError(f"File too large ({file_size // (1024 * 1024)} MB): {filepath}")
     try:
         with open(filepath, "r", encoding="utf-8", errors="replace") as f:
             content = f.read()
-    except Exception as e:
+    except OSError as e:
         raise IOError(f"Could not read {filepath}: {e}")
 
     if not content.strip():
@@ -55,6 +62,10 @@ def _try_normalize_json(content: str) -> Optional[str]:
     if normalized:
         return normalized
 
+    normalized = _try_codex_jsonl(content)
+    if normalized:
+        return normalized
+
     try:
         data = json.loads(content)
     except json.JSONDecodeError:
@@ -72,6 +83,8 @@ def _try_claude_code_jsonl(content: str) -> Optional[str]:
     """Claude Code JSONL sessions."""
     lines = [line.strip() for line in content.strip().split("\n") if line.strip()]
     messages = []
+    tool_use_map = {}  # tool_use_id → tool_name
+
     for line in lines:
         try:
             entry = json.loads(line)
@@ -81,38 +94,142 @@ def _try_claude_code_jsonl(content: str) -> Optional[str]:
             continue
         msg_type = entry.get("type", "")
         message = entry.get("message", {})
-        if msg_type == "human":
-            text = _extract_content(message.get("content", ""))
+        if not isinstance(message, dict):
+            continue
+        msg_content = message.get("content", "")
+
+        # Build tool_use_map from assistant messages
+        if msg_type == "assistant" and isinstance(msg_content, list):
+            for block in msg_content:
+                if isinstance(block, dict) and block.get("type") == "tool_use":
+                    tool_id = block.get("id", "")
+                    if tool_id:
+                        tool_use_map[tool_id] = block.get("name", "Unknown")
+
+        if msg_type in ("human", "user"):
+            # Check if this message is tool_results only (no user text)
+            is_tool_only = isinstance(msg_content, list) and all(
+                isinstance(b, dict) and b.get("type") == "tool_result" for b in msg_content
+            )
+            text = _extract_content(msg_content, tool_use_map=tool_use_map)
             if text:
-                messages.append(("user", text))
+                if is_tool_only and messages and messages[-1][0] == "assistant":
+                    # Append tool results to the previous assistant message
+                    prev_role, prev_text = messages[-1]
+                    messages[-1] = (prev_role, prev_text + "\n" + text)
+                elif not is_tool_only:
+                    messages.append(("user", text))
         elif msg_type == "assistant":
-            text = _extract_content(message.get("content", ""))
+            text = _extract_content(msg_content, tool_use_map=tool_use_map)
             if text:
-                messages.append(("assistant", text))
+                # If previous message is also assistant (multi-turn tool loop),
+                # merge into the same assistant turn
+                if messages and messages[-1][0] == "assistant":
+                    prev_role, prev_text = messages[-1]
+                    messages[-1] = (prev_role, prev_text + "\n" + text)
+                else:
+                    messages.append(("assistant", text))
+
     if len(messages) >= 2:
         return _messages_to_transcript(messages)
     return None
 
 
+def _try_codex_jsonl(content: str) -> Optional[str]:
+    """OpenAI Codex CLI sessions (~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl).
+
+    Uses only event_msg entries (user_message / agent_message) which represent
+    the canonical conversation turns. response_item entries are skipped because
+    they include synthetic context injections and duplicate the real messages.
+    """
+    lines = [line.strip() for line in content.strip().split("\n") if line.strip()]
+    messages = []
+    has_session_meta = False
+    for line in lines:
+        try:
+            entry = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(entry, dict):
+            continue
+
+        entry_type = entry.get("type", "")
+        if entry_type == "session_meta":
+            has_session_meta = True
+            continue
+
+        if entry_type != "event_msg":
+            continue
+
+        payload = entry.get("payload", {})
+        if not isinstance(payload, dict):
+            continue
+
+        payload_type = payload.get("type", "")
+        msg = payload.get("message")
+        if not isinstance(msg, str):
+            continue
+        text = msg.strip()
+        if not text:
+            continue
+
+        if payload_type == "user_message":
+            messages.append(("user", text))
+        elif payload_type == "agent_message":
+            messages.append(("assistant", text))
+
+    if len(messages) >= 2 and has_session_meta:
+        return _messages_to_transcript(messages)
+    return None
+
+
 def _try_claude_ai_json(data) -> Optional[str]:
-    """Claude.ai JSON export: [{"role": "user", "content": "..."}]"""
+    """Claude.ai JSON export: flat messages list or privacy export with chat_messages."""
     if isinstance(data, dict):
         data = data.get("messages", data.get("chat_messages", []))
     if not isinstance(data, list):
         return None
+
+    # Privacy export: array of conversation objects, each containing its own
+    # message list under "chat_messages" or "messages" (both variants seen in the wild).
+    if data and isinstance(data[0], dict) and ("chat_messages" in data[0] or "messages" in data[0]):
+        transcripts = []
+        for convo in data:
+            if not isinstance(convo, dict):
+                continue
+            chat_msgs = convo.get("chat_messages") or convo.get("messages", [])
+            messages = _collect_claude_messages(chat_msgs)
+            if len(messages) >= 2:
+                transcripts.append(_messages_to_transcript(messages))
+        if transcripts:
+            return "\n\n".join(transcripts)
+        return None
+
+    # Flat messages list
+    messages = _collect_claude_messages(data)
+    if len(messages) >= 2:
+        return _messages_to_transcript(messages)
+    return None
+
+
+def _collect_claude_messages(items) -> list:
+    """Extract (role, text) pairs from a Claude.ai message list.
+
+    Accepts both ``role`` (API format) and ``sender`` (privacy export) as the
+    author field, and falls back to a top-level ``text`` key when the
+    ``content`` blocks are empty or absent.
+    """
     messages = []
-    for item in data:
+    for item in items:
         if not isinstance(item, dict):
             continue
-        role = item.get("role", "")
-        text = _extract_content(item.get("content", ""))
+        role = item.get("role") or item.get("sender", "")
+        text = _extract_content(item.get("content", "")) or (item.get("text") or "").strip()
         if role in ("user", "human") and text:
             messages.append(("user", text))
         elif role in ("assistant", "ai") and text:
             messages.append(("assistant", text))
-    if len(messages) >= 2:
-        return _messages_to_transcript(messages)
-    return None
+    return messages
 
 
 def _try_chatgpt_json(data) -> Optional[str]:
@@ -189,8 +306,14 @@ def _try_slack_json(data) -> Optional[str]:
     return None
 
 
-def _extract_content(content) -> str:
-    """Pull text from content — handles str, list of blocks, or dict."""
+def _extract_content(content, tool_use_map: dict = None) -> str:
+    """Pull text from content — handles str, list of blocks, or dict.
+
+    Args:
+        content: Message content — string, list of content blocks, or dict.
+        tool_use_map: Optional mapping of tool_use_id → tool_name, used to
+                      select the right formatting strategy for tool_result blocks.
+    """
     if isinstance(content, str):
         return content.strip()
     if isinstance(content, list):
@@ -198,14 +321,135 @@ def _extract_content(content) -> str:
         for item in content:
             if isinstance(item, str):
                 parts.append(item)
-            elif isinstance(item, dict) and item.get("type") == "text":
-                parts.append(item.get("text", ""))
-        return " ".join(parts).strip()
+            elif isinstance(item, dict):
+                block_type = item.get("type")
+                if block_type == "text":
+                    parts.append(item.get("text", ""))
+                elif block_type == "tool_use":
+                    parts.append(_format_tool_use(item))
+                elif block_type == "tool_result":
+                    tid = item.get("tool_use_id", "")
+                    tname = (tool_use_map or {}).get(tid, "Unknown")
+                    result_content = item.get("content", "")
+                    formatted = _format_tool_result(result_content, tname)
+                    if formatted:
+                        parts.append(formatted)
+        return "\n".join(p for p in parts if p).strip()
     if isinstance(content, dict):
         return content.get("text", "").strip()
     return ""
 
 
+def _format_tool_use(block: dict) -> str:
+    """Format a tool_use block into a human-readable one-liner."""
+    name = block.get("name", "Unknown")
+    inp = block.get("input", {})
+
+    if name == "Bash":
+        cmd = inp.get("command", "")
+        if len(cmd) > 200:
+            cmd = cmd[:200] + "..."
+        return f"[Bash] {cmd}"
+
+    if name == "Read":
+        path = inp.get("file_path", "?")
+        offset = inp.get("offset")
+        limit = inp.get("limit")
+        if offset is not None and limit is not None:
+            try:
+                return f"[Read {path}:{offset}-{int(offset) + int(limit)}]"
+            except (ValueError, TypeError):
+                return f"[Read {path}:{offset}+{limit}]"
+        return f"[Read {path}]"
+
+    if name == "Grep":
+        pattern = inp.get("pattern", "")
+        target = inp.get("path") or inp.get("glob") or ""
+        return f"[Grep] {pattern} in {target}"
+
+    if name == "Glob":
+        pattern = inp.get("pattern", "")
+        return f"[Glob] {pattern}"
+
+    if name in ("Edit", "Write"):
+        path = inp.get("file_path", "?")
+        return f"[{name} {path}]"
+
+    # Unknown tool — serialize input, truncate
+    summary = json.dumps(inp, separators=(",", ":"))
+    if len(summary) > 200:
+        summary = summary[:200] + "..."
+    return f"[{name}] {summary}"
+
+
+_TOOL_RESULT_MAX_LINES_BASH = 20  # head and tail line count
+_TOOL_RESULT_MAX_MATCHES = 20  # Grep/Glob cap
+_TOOL_RESULT_MAX_BYTES = 2048  # fallback cap for unknown tools
+
+
+def _format_tool_result(content, tool_name: str) -> str:
+    """Format a tool_result based on the originating tool's type.
+
+    Args:
+        content: Result text (str) or list of content blocks (list of dicts).
+        tool_name: Name of the tool that produced this result.
+
+    Returns:
+        Formatted string prefixed with ``→ ``, or empty string if omitted.
+    """
+    # Normalize list-of-blocks to plain text
+    if isinstance(content, list):
+        parts = []
+        for item in content:
+            if isinstance(item, dict) and item.get("type") == "text":
+                parts.append(item.get("text", ""))
+            elif isinstance(item, str):
+                parts.append(item)
+        text = "\n".join(parts)
+    else:
+        text = str(content) if content else ""
+
+    text = text.strip()
+    if not text:
+        return ""
+
+    # Read/Edit/Write — omit result (content is in palace or git)
+    if tool_name in ("Read", "Edit", "Write"):
+        return ""
+
+    lines = text.split("\n")
+
+    # Bash — head + tail
+    if tool_name == "Bash":
+        n = _TOOL_RESULT_MAX_LINES_BASH
+        if len(lines) <= n * 2:
+            return "→ " + "\n→ ".join(lines)
+        head = lines[:n]
+        tail = lines[-n:]
+        omitted = len(lines) - 2 * n
+        return (
+            "→ "
+            + "\n→ ".join(head)
+            + f"\n→ ... [{omitted} lines omitted] ..."
+            + "\n→ "
+            + "\n→ ".join(tail)
+        )
+
+    # Grep/Glob — cap matches
+    if tool_name in ("Grep", "Glob"):
+        cap = _TOOL_RESULT_MAX_MATCHES
+        if len(lines) <= cap:
+            return "→ " + "\n→ ".join(lines)
+        kept = lines[:cap]
+        remaining = len(lines) - cap
+        return "→ " + "\n→ ".join(kept) + f"\n→ ... [{remaining} more matches]"
+
+    # Unknown — byte cap
+    if len(text) > _TOOL_RESULT_MAX_BYTES:
+        return "→ " + text[:_TOOL_RESULT_MAX_BYTES] + f"... [truncated, {len(text)} chars]"
+    return "→ " + text
+
+
 def _messages_to_transcript(messages: list, spellcheck: bool = True) -> str:
     """Convert [(role, text), ...] to transcript format with > markers."""
     if spellcheck:
@@ -213,7 +457,7 @@ def _messages_to_transcript(messages: list, spellcheck: bool = True) -> str:
             from mempalace.spellcheck import spellcheck_user_text
 
             _fix = spellcheck_user_text
-        except Exception:
+        except ImportError:
             _fix = None
     else:
         _fix = None

mempalace/onboarding.py

@@ -312,7 +312,7 @@ def _generate_aaak_bootstrap(
         ]
     )
 
-    (mempalace_dir / "aaak_entities.md").write_text("\n".join(registry_lines))
+    (mempalace_dir / "aaak_entities.md").write_text("\n".join(registry_lines), encoding="utf-8")
 
     # Critical facts bootstrap (pre-palace — before any mining)
     facts_lines = [
@@ -359,7 +359,7 @@ def _generate_aaak_bootstrap(
         ]
     )
 
-    (mempalace_dir / "critical_facts.md").write_text("\n".join(facts_lines))
+    (mempalace_dir / "critical_facts.md").write_text("\n".join(facts_lines), encoding="utf-8")
 
 
 def run_onboarding(

mempalace/palace.py

@@ -0,0 +1,73 @@
+"""
+palace.py — Shared palace operations.
+
+Consolidates collection access patterns used by both miners and the MCP server.
+"""
+
+import os
+
+from .backends.chroma import ChromaBackend
+
+SKIP_DIRS = {
+    ".git",
+    "node_modules",
+    "__pycache__",
+    ".venv",
+    "venv",
+    "env",
+    "dist",
+    "build",
+    ".next",
+    "coverage",
+    ".mempalace",
+    ".ruff_cache",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".cache",
+    ".tox",
+    ".nox",
+    ".idea",
+    ".vscode",
+    ".ipynb_checkpoints",
+    ".eggs",
+    "htmlcov",
+    "target",
+}
+
+_DEFAULT_BACKEND = ChromaBackend()
+
+
+def get_collection(
+    palace_path: str,
+    collection_name: str = "mempalace_drawers",
+    create: bool = True,
+):
+    """Get the palace collection through the backend layer."""
+    return _DEFAULT_BACKEND.get_collection(
+        palace_path,
+        collection_name=collection_name,
+        create=create,
+    )
+
+
+def file_already_mined(collection, source_file: str, check_mtime: bool = False) -> bool:
+    """Check if a file has already been filed in the palace.
+
+    When check_mtime=True (used by project miner), returns False if the file
+    has been modified since it was last mined, so it gets re-mined.
+    When check_mtime=False (used by convo miner), just checks existence.
+    """
+    try:
+        results = collection.get(where={"source_file": source_file}, limit=1)
+        if not results.get("ids"):
+            return False
+        if check_mtime:
+            stored_meta = results.get("metadatas", [{}])[0]
+            stored_mtime = stored_meta.get("source_mtime")
+            if stored_mtime is None:
+                return False
+            current_mtime = os.path.getmtime(source_file)
+            return abs(float(stored_mtime) - current_mtime) < 0.001
+        return True
+    except Exception:
+        return False

mempalace/palace_graph.py

@@ -16,16 +16,19 @@ No external graph DB needed — built from ChromaDB metadata.
 """
 
 from collections import defaultdict, Counter
-from .config import MempalaceConfig
 
-import chromadb
+from .config import MempalaceConfig
+from .palace import get_collection as _get_palace_collection
 
 
 def _get_collection(config=None):
     config = config or MempalaceConfig()
     try:
-        client = chromadb.PersistentClient(path=config.palace_path)
-        return client.get_collection(config.collection_name)
+        return _get_palace_collection(
+            config.palace_path,
+            collection_name=config.collection_name,
+            create=False,
+        )
     except Exception:
         return None
 

mempalace/query_sanitizer.py

@@ -0,0 +1,188 @@
+"""
+query_sanitizer.py — Mitigate system prompt contamination in search queries.
+
+Problem: AI agents sometimes prepend system prompts (2000+ chars) to search queries.
+Embedding models represent the concatenated string as a single vector where the
+system prompt overwhelms the actual question (typically 10-50 chars), causing
+near-total retrieval failure (89.8% → 1.0% R@10). See Issue #333.
+
+Approach: "Mitigation" (減災) — not perfect prevention, but prevents the cliff.
+
+Expected recovery:
+  Step 1 passthrough (≤200 chars)     → no degradation, ~89.8%
+  Step 2 question extraction (？found) → near-full recovery, ~85-89%
+  Step 3 tail sentence extraction      → moderate recovery, ~80-89%
+  Step 4 tail truncation (fallback)    → minimum viable, ~70-80%
+
+  Without sanitizer: 1.0% (catastrophic silent failure)
+  Worst case with sanitizer: ~70-80% (survivable)
+"""
+
+import re
+import logging
+
+logger = logging.getLogger("mempalace_mcp")
+
+# --- Constants ---
+MAX_QUERY_LENGTH = 250  # Above this, prompt contamination increasingly dominates
+SAFE_QUERY_LENGTH = 200  # Below this, query is almost certainly clean
+MIN_QUERY_LENGTH = 10  # Extracted result shorter than this = extraction failed
+QUOTE_CHARS = {"'", '"'}
+
+# Sentence splitter: split on . ! ? (including fullwidth) and newlines
+_SENTENCE_SPLIT = re.compile(r"[.!?。！？\n]+")
+
+# Question detector: ends with ? or ？ (possibly with trailing whitespace/quotes)
+_QUESTION_MARK = re.compile(r'[?？]\s*["\']?\s*$')
+
+
+def sanitize_query(raw_query: str) -> dict:
+    """
+    Extract the actual search intent from a potentially contaminated query.
+
+    Args:
+        raw_query: The raw query string from the AI agent, possibly containing
+                   system prompt content prepended to the actual question.
+
+    Returns:
+        dict with keys:
+            clean_query (str): The sanitized query to use for embedding search
+            was_sanitized (bool): Whether any sanitization was applied
+            original_length (int): Length of the raw input
+            clean_length (int): Length of the sanitized output
+            method (str): Which extraction method was used
+                - "passthrough": query was short enough, no action taken
+                - "question_extraction": found and extracted a question sentence
+                - "tail_sentence": extracted the last meaningful sentence
+                - "tail_truncation": fallback — took the last MAX_QUERY_LENGTH chars
+    """
+    if not raw_query or not raw_query.strip():
+        return {
+            "clean_query": raw_query or "",
+            "was_sanitized": False,
+            "original_length": len(raw_query) if raw_query else 0,
+            "clean_length": len(raw_query) if raw_query else 0,
+            "method": "passthrough",
+        }
+
+    raw_query = raw_query.strip()
+    original_length = len(raw_query)
+
+    def _strip_wrapping_quotes(candidate: str) -> str:
+        candidate = candidate.strip()
+        while (
+            len(candidate) >= 2 and candidate[:1] in QUOTE_CHARS and candidate[:1] == candidate[-1:]
+        ):
+            candidate = candidate[1:-1].strip()
+            if not candidate:
+                return ""
+        if candidate[:1] in QUOTE_CHARS:
+            candidate = candidate[1:].strip()
+        if candidate[-1:] in QUOTE_CHARS:
+            candidate = candidate[:-1].strip()
+        return candidate
+
+    def _trim_candidate(candidate: str) -> str:
+        candidate = _strip_wrapping_quotes(candidate)
+        if len(candidate) <= MAX_QUERY_LENGTH:
+            return candidate
+
+        nested_fragments = [
+            _strip_wrapping_quotes(frag)
+            for frag in _SENTENCE_SPLIT.split(candidate)
+            if frag.strip()
+        ]
+        for frag in reversed(nested_fragments):
+            if MIN_QUERY_LENGTH <= len(frag) <= MAX_QUERY_LENGTH:
+                return frag
+
+        return candidate[-MAX_QUERY_LENGTH:].strip()
+
+    # --- Step 1: Short query passthrough ---
+    if original_length <= SAFE_QUERY_LENGTH:
+        return {
+            "clean_query": raw_query,
+            "was_sanitized": False,
+            "original_length": original_length,
+            "clean_length": original_length,
+            "method": "passthrough",
+        }
+
+    # --- Step 2: Question extraction ---
+    # Split into sentences and find ones ending with ?
+    sentences = [s.strip() for s in _SENTENCE_SPLIT.split(raw_query) if s.strip()]
+
+    # Also split on newlines to catch questions on their own line
+    all_segments = []
+    for s in raw_query.split("\n"):
+        s = s.strip()
+        if s:
+            all_segments.append(s)
+
+    # Look for question marks in segments (prefer later ones = more likely the actual query)
+    question_sentences = []
+    for seg in reversed(all_segments):
+        if _QUESTION_MARK.search(seg):
+            question_sentences.append(seg)
+
+    if not question_sentences:
+        # Also check the sentence-split results
+        for sent in reversed(sentences):
+            if "?" in sent or "？" in sent:
+                question_sentences.append(sent)
+
+    if question_sentences:
+        # Take the last (most recent) question found
+        candidate = question_sentences[0].strip()
+        if len(candidate) >= MIN_QUERY_LENGTH:
+            # Apply length guard
+            if len(candidate) > MAX_QUERY_LENGTH:
+                candidate = _trim_candidate(candidate)
+            logger.warning(
+                "Query sanitized: %d → %d chars (method=question_extraction)",
+                original_length,
+                len(candidate),
+            )
+            return {
+                "clean_query": candidate,
+                "was_sanitized": True,
+                "original_length": original_length,
+                "clean_length": len(candidate),
+                "method": "question_extraction",
+            }
+
+    # --- Step 3: Tail sentence extraction ---
+    # System prompts are prepended, so the actual query is near the end.
+    # Walk backwards through segments to find the last meaningful sentence.
+    for seg in reversed(all_segments):
+        seg = seg.strip()
+        if len(seg) >= MIN_QUERY_LENGTH:
+            candidate = _trim_candidate(seg)
+            if len(candidate) < MIN_QUERY_LENGTH:
+                continue
+            logger.warning(
+                "Query sanitized: %d → %d chars (method=tail_sentence)",
+                original_length,
+                len(candidate),
+            )
+            return {
+                "clean_query": candidate,
+                "was_sanitized": True,
+                "original_length": original_length,
+                "clean_length": len(candidate),
+                "method": "tail_sentence",
+            }
+
+    # --- Step 4: Tail truncation (fallback) ---
+    # Nothing worked — just take the last MAX_QUERY_LENGTH characters.
+    candidate = raw_query[-MAX_QUERY_LENGTH:].strip()
+    logger.warning(
+        "Query sanitized: %d → %d chars (method=tail_truncation)", original_length, len(candidate)
+    )
+    return {
+        "clean_query": candidate,
+        "was_sanitized": True,
+        "original_length": original_length,
+        "clean_length": len(candidate),
+        "method": "tail_truncation",
+    }

mempalace/repair.py

@@ -0,0 +1,299 @@
+"""
+repair.py — Scan, prune corrupt entries, and rebuild HNSW index
+================================================================
+
+When ChromaDB's HNSW index accumulates duplicate entries (from repeated
+add() calls with the same ID), link_lists.bin can grow unbounded —
+terabytes on large palaces — eventually causing segfaults.
+
+This module provides three operations:
+
+  scan    — find every corrupt/unfetchable ID in the palace
+  prune   — delete only the corrupt IDs (surgical)
+  rebuild — extract all drawers, delete the collection, recreate with
+            correct HNSW settings, and upsert everything back
+
+The rebuild backs up ONLY chroma.sqlite3 (the source of truth), not the
+full palace directory — so it works even when link_lists.bin is bloated.
+
+Usage (standalone):
+    python -m mempalace.repair scan [--wing X]
+    python -m mempalace.repair prune --confirm
+    python -m mempalace.repair rebuild
+
+Usage (from CLI):
+    mempalace repair
+    mempalace repair-scan [--wing X]
+    mempalace repair-prune --confirm
+"""
+
+import argparse
+import os
+import shutil
+import time
+
+import chromadb
+
+
+COLLECTION_NAME = "mempalace_drawers"
+
+
+def _get_palace_path():
+    """Resolve palace path from config."""
+    try:
+        from .config import MempalaceConfig
+
+        return MempalaceConfig().palace_path
+    except Exception:
+        default = os.path.join(os.path.expanduser("~"), ".mempalace", "palace")
+        return default
+
+
+def _paginate_ids(col, where=None):
+    """Pull all IDs in a collection using pagination."""
+    ids = []
+    page = 1000
+    offset = 0
+    while True:
+        try:
+            r = col.get(where=where, include=[], limit=page, offset=offset)
+        except Exception:
+            try:
+                r = col.get(where=where, include=[], limit=page)
+                new_ids = [i for i in r["ids"] if i not in set(ids)]
+                if not new_ids:
+                    break
+                ids.extend(new_ids)
+                offset += len(new_ids)
+                continue
+            except Exception:
+                break
+        n = len(r["ids"]) if r["ids"] else 0
+        if n == 0:
+            break
+        ids.extend(r["ids"])
+        offset += n
+        if n < page:
+            break
+    return ids
+
+
+def scan_palace(palace_path=None, only_wing=None):
+    """Scan the palace for corrupt/unfetchable IDs.
+
+    Probes in batches of 100, falls back to per-ID on failure.
+    Writes corrupt_ids.txt to the palace directory for the prune step.
+
+    Returns (good_set, bad_set).
+    """
+    palace_path = palace_path or _get_palace_path()
+    print(f"\n  Palace: {palace_path}")
+    print("  Loading...")
+
+    client = chromadb.PersistentClient(path=palace_path)
+    col = client.get_collection(COLLECTION_NAME)
+
+    where = {"wing": only_wing} if only_wing else None
+    total = col.count()
+    print(f"  Collection: {COLLECTION_NAME}, total: {total:,}")
+    if only_wing:
+        print(f"  Scanning wing: {only_wing}")
+
+    print("\n  Step 1: listing all IDs...")
+    t0 = time.time()
+    all_ids = _paginate_ids(col, where=where)
+    print(f"  Found {len(all_ids):,} IDs in {time.time() - t0:.1f}s\n")
+
+    if not all_ids:
+        print("  Nothing to scan.")
+        return set(), set()
+
+    print("  Step 2: probing each ID (batches of 100)...")
+    t0 = time.time()
+    good_set = set()
+    bad_set = set()
+    batch = 100
+
+    for i in range(0, len(all_ids), batch):
+        chunk = all_ids[i : i + batch]
+        try:
+            r = col.get(ids=chunk, include=["documents"])
+            for got in r["ids"]:
+                good_set.add(got)
+            for mid in chunk:
+                if mid not in good_set:
+                    bad_set.add(mid)
+        except Exception:
+            for sid in chunk:
+                try:
+                    r = col.get(ids=[sid], include=["documents"])
+                    if r["ids"]:
+                        good_set.add(sid)
+                    else:
+                        bad_set.add(sid)
+                except Exception:
+                    bad_set.add(sid)
+
+        if (i // batch) % 50 == 0:
+            elapsed = time.time() - t0
+            rate = (i + batch) / max(elapsed, 0.01)
+            eta = (len(all_ids) - i - batch) / max(rate, 0.01)
+            print(
+                f"    {i + batch:>6}/{len(all_ids):>6}  "
+                f"good={len(good_set):>6}  bad={len(bad_set):>6}  "
+                f"eta={eta:.0f}s"
+            )
+
+    print(f"\n  Scan complete in {time.time() - t0:.1f}s")
+    print(f"  GOOD: {len(good_set):,}")
+    print(f"  BAD:  {len(bad_set):,}  ({len(bad_set) / max(len(all_ids), 1) * 100:.1f}%)")
+
+    bad_file = os.path.join(palace_path, "corrupt_ids.txt")
+    with open(bad_file, "w") as f:
+        for bid in sorted(bad_set):
+            f.write(bid + "\n")
+    print(f"\n  Bad IDs written to: {bad_file}")
+    return good_set, bad_set
+
+
+def prune_corrupt(palace_path=None, confirm=False):
+    """Delete corrupt IDs listed in corrupt_ids.txt."""
+    palace_path = palace_path or _get_palace_path()
+    bad_file = os.path.join(palace_path, "corrupt_ids.txt")
+
+    if not os.path.exists(bad_file):
+        print("  No corrupt_ids.txt found — run scan first.")
+        return
+
+    with open(bad_file) as f:
+        bad_ids = [line.strip() for line in f if line.strip()]
+    print(f"  {len(bad_ids):,} corrupt IDs queued for deletion")
+
+    if not confirm:
+        print("\n  DRY RUN — no deletions performed.")
+        print("  Re-run with --confirm to actually delete.")
+        return
+
+    client = chromadb.PersistentClient(path=palace_path)
+    col = client.get_collection(COLLECTION_NAME)
+    before = col.count()
+    print(f"  Collection size before: {before:,}")
+
+    batch = 100
+    deleted = 0
+    failed = 0
+    for i in range(0, len(bad_ids), batch):
+        chunk = bad_ids[i : i + batch]
+        try:
+            col.delete(ids=chunk)
+            deleted += len(chunk)
+        except Exception:
+            for sid in chunk:
+                try:
+                    col.delete(ids=[sid])
+                    deleted += 1
+                except Exception:
+                    failed += 1
+        if (i // batch) % 20 == 0:
+            print(f"    deleted {deleted}/{len(bad_ids)}  (failed: {failed})")
+
+    after = col.count()
+    print(f"\n  Deleted: {deleted:,}")
+    print(f"  Failed:  {failed:,}")
+    print(f"  Collection size: {before:,} → {after:,}")
+
+
+def rebuild_index(palace_path=None):
+    """Rebuild the HNSW index from scratch.
+
+    1. Extract all drawers via ChromaDB get()
+    2. Back up ONLY chroma.sqlite3 (not the bloated HNSW files)
+    3. Delete and recreate the collection with hnsw:space=cosine
+    4. Upsert all drawers back
+    """
+    palace_path = palace_path or _get_palace_path()
+
+    if not os.path.isdir(palace_path):
+        print(f"\n  No palace found at {palace_path}")
+        return
+
+    print(f"\n{'=' * 55}")
+    print("  MemPalace Repair — Index Rebuild")
+    print(f"{'=' * 55}\n")
+    print(f"  Palace: {palace_path}")
+
+    client = chromadb.PersistentClient(path=palace_path)
+    try:
+        col = client.get_collection(COLLECTION_NAME)
+        total = col.count()
+    except Exception as e:
+        print(f"  Error reading palace: {e}")
+        print("  Palace may need to be re-mined from source files.")
+        return
+
+    print(f"  Drawers found: {total}")
+
+    if total == 0:
+        print("  Nothing to repair.")
+        return
+
+    # Extract all drawers in batches
+    print("\n  Extracting drawers...")
+    batch_size = 5000
+    all_ids = []
+    all_docs = []
+    all_metas = []
+    offset = 0
+    while offset < total:
+        batch = col.get(limit=batch_size, offset=offset, include=["documents", "metadatas"])
+        if not batch["ids"]:
+            break
+        all_ids.extend(batch["ids"])
+        all_docs.extend(batch["documents"])
+        all_metas.extend(batch["metadatas"])
+        offset += len(batch["ids"])
+    print(f"  Extracted {len(all_ids)} drawers")
+
+    # Back up ONLY the SQLite database, not the bloated HNSW files
+    sqlite_path = os.path.join(palace_path, "chroma.sqlite3")
+    if os.path.exists(sqlite_path):
+        backup_path = sqlite_path + ".backup"
+        print(f"  Backing up chroma.sqlite3 ({os.path.getsize(sqlite_path) / 1e6:.0f} MB)...")
+        shutil.copy2(sqlite_path, backup_path)
+        print(f"  Backup: {backup_path}")
+
+    # Rebuild with correct HNSW settings
+    print("  Rebuilding collection with hnsw:space=cosine...")
+    client.delete_collection(COLLECTION_NAME)
+    new_col = client.create_collection(COLLECTION_NAME, metadata={"hnsw:space": "cosine"})
+
+    filed = 0
+    for i in range(0, len(all_ids), batch_size):
+        batch_ids = all_ids[i : i + batch_size]
+        batch_docs = all_docs[i : i + batch_size]
+        batch_metas = all_metas[i : i + batch_size]
+        new_col.upsert(documents=batch_docs, ids=batch_ids, metadatas=batch_metas)
+        filed += len(batch_ids)
+        print(f"  Re-filed {filed}/{len(all_ids)} drawers...")
+
+    print(f"\n  Repair complete. {filed} drawers rebuilt.")
+    print("  HNSW index is now clean with cosine distance metric.")
+    print(f"\n{'=' * 55}\n")
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description="MemPalace repair tools")
+    p.add_argument("command", choices=["scan", "prune", "rebuild"])
+    p.add_argument("--palace", default=None, help="Palace directory path")
+    p.add_argument("--wing", default=None, help="Scan only this wing")
+    p.add_argument("--confirm", action="store_true", help="Actually delete corrupt IDs")
+    args = p.parse_args()
+
+    path = os.path.expanduser(args.palace) if args.palace else None
+
+    if args.command == "scan":
+        scan_palace(palace_path=path, only_wing=args.wing)
+    elif args.command == "prune":
+        prune_corrupt(palace_path=path, confirm=args.confirm)
+    elif args.command == "rebuild":
+        rebuild_index(palace_path=path)

mempalace/room_detector_local.py

@@ -9,12 +9,15 @@ Two ways to define rooms without calling any AI:
 No internet. No API key. Your files stay on your machine.
 """
 
+import logging
 import os
 import sys
 import yaml
 from pathlib import Path
 from collections import defaultdict
 
+logger = logging.getLogger(__name__)
+
 # Common room patterns — detected from folder names and filenames
 # Format: {folder_keyword: room_name}
 FOLDER_ROOM_MAP = {
@@ -118,7 +121,12 @@ def detect_rooms_from_folders(project_dir: str) -> list:
 
     # Check top-level directories first (most reliable signal)
     for item in project_path.iterdir():
-        if item.is_dir() and item.name not in SKIP_DIRS:
+        try:
+            is_dir = item.is_dir()  # WinError 448 — reparse point / untrusted mount point
+        except OSError as exc:
+            logger.debug("Skipping %s: %s", item, exc)
+            continue
+        if is_dir and item.name not in SKIP_DIRS:
             name_lower = item.name.lower().replace("-", "_")
             if name_lower in FOLDER_ROOM_MAP:
                 room_name = FOLDER_ROOM_MAP[name_lower]
@@ -132,9 +140,28 @@ def detect_rooms_from_folders(project_dir: str) -> list:
 
     # Walk one level deeper for nested patterns
     for item in project_path.iterdir():
-        if item.is_dir() and item.name not in SKIP_DIRS:
-            for subitem in item.iterdir():
-                if subitem.is_dir() and subitem.name not in SKIP_DIRS:
+        try:
+            item_is_dir = item.is_dir()  # WinError 448 — reparse point / untrusted mount point
+        except OSError as exc:
+            logger.debug("Skipping %s: %s", item, exc)
+            continue
+        if item_is_dir and item.name not in SKIP_DIRS:
+            try:
+                subitems = list(
+                    item.iterdir()
+                )  # WinError 448 — iterdir can also fail on some reparse points
+            except OSError as exc:
+                logger.debug("Skipping contents of %s: %s", item, exc)
+                continue
+            for subitem in subitems:
+                try:
+                    subitem_is_dir = (
+                        subitem.is_dir()
+                    )  # WinError 448 — reparse point / untrusted mount point
+                except OSError as exc:
+                    logger.debug("Skipping %s: %s", subitem, exc)
+                    continue
+                if subitem_is_dir and subitem.name not in SKIP_DIRS:
                     name_lower = subitem.name.lower().replace("-", "_")
                     if name_lower in FOLDER_ROOM_MAP:
                         room_name = FOLDER_ROOM_MAP[name_lower]
@@ -255,7 +282,14 @@ def get_user_approval(rooms: list) -> list:
 def save_config(project_dir: str, project_name: str, rooms: list):
     config = {
         "wing": project_name,
-        "rooms": [{"name": r["name"], "description": r["description"]} for r in rooms],
+        "rooms": [
+            {
+                "name": r["name"],
+                "description": r["description"],
+                "keywords": r.get("keywords", [r["name"]]),
+            }
+            for r in rooms
+        ],
     }
     config_path = Path(project_dir).expanduser().resolve() / "mempalace.yaml"
     with open(config_path, "w") as f:
@@ -267,7 +301,7 @@ def save_config(project_dir: str, project_name: str, rooms: list):
     print(f"\n{'=' * 55}\n")
 
 
-def detect_rooms_local(project_dir: str):
+def detect_rooms_local(project_dir: str, yes: bool = False):
     """Main entry point for local setup."""
     project_path = Path(project_dir).expanduser().resolve()
     project_name = project_path.name.lower().replace(" ", "_").replace("-", "_")
@@ -296,5 +330,8 @@ def detect_rooms_local(project_dir: str):
         source = "fallback (flat project)"
 
     print_proposed_structure(project_name, rooms, len(files), source)
-    approved_rooms = get_user_approval(rooms)
+    if yes:
+        approved_rooms = rooms
+    else:
+        approved_rooms = get_user_approval(rooms)
     save_config(project_dir, project_name, approved_rooms)

mempalace/searcher.py

@@ -6,10 +6,27 @@ Semantic search against the palace.
 Returns verbatim text — the actual words, never summaries.
 """
 
-import sys
+import logging
 from pathlib import Path
 
-import chromadb
+from .palace import get_collection
+
+logger = logging.getLogger("mempalace_mcp")
+
+
+class SearchError(Exception):
+    """Raised when search cannot proceed (e.g. no palace found)."""
+
+
+def build_where_filter(wing: str = None, room: str = None) -> dict:
+    """Build ChromaDB where filter for wing/room filtering."""
+    if wing and room:
+        return {"$and": [{"wing": wing}, {"room": room}]}
+    elif wing:
+        return {"wing": wing}
+    elif room:
+        return {"room": room}
+    return {}
 
 
 def search(query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5):
@@ -18,21 +35,13 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
     Optionally filter by wing (project) or room (aspect).
     """
     try:
-        client = chromadb.PersistentClient(path=palace_path)
-        col = client.get_collection("mempalace_drawers")
+        col = get_collection(palace_path, create=False)
     except Exception:
         print(f"\n  No palace found at {palace_path}")
         print("  Run: mempalace init <dir> then mempalace mine <dir>")
-        sys.exit(1)
+        raise SearchError(f"No palace found at {palace_path}")
 
-    # Build where filter
-    where = {}
-    if wing and room:
-        where = {"$and": [{"wing": wing}, {"room": room}]}
-    elif wing:
-        where = {"wing": wing}
-    elif room:
-        where = {"room": room}
+    where = build_where_filter(wing, room)
 
     try:
         kwargs = {
@@ -47,7 +56,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
 
     except Exception as e:
         print(f"\n  Search error: {e}")
-        sys.exit(1)
+        raise SearchError(f"Search error: {e}") from e
 
     docs = results["documents"][0]
     metas = results["metadatas"][0]
@@ -66,7 +75,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
     print(f"{'=' * 60}\n")
 
     for i, (doc, meta, dist) in enumerate(zip(docs, metas, dists), 1):
-        similarity = round(1 - dist, 3)
+        similarity = round(max(0.0, 1 - dist), 3)
         source = Path(meta.get("source_file", "?")).name
         wing_name = meta.get("wing", "?")
         room_name = meta.get("room", "?")
@@ -85,26 +94,38 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
 
 
 def search_memories(
-    query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5
+    query: str,
+    palace_path: str,
+    wing: str = None,
+    room: str = None,
+    n_results: int = 5,
+    max_distance: float = 0.0,
 ) -> dict:
-    """
-    Programmatic search — returns a dict instead of printing.
+    """Programmatic search — returns a dict instead of printing.
+
     Used by the MCP server and other callers that need data.
+
+    Args:
+        query: Natural language search query.
+        palace_path: Path to the ChromaDB palace directory.
+        wing: Optional wing filter.
+        room: Optional room filter.
+        n_results: Max results to return.
+        max_distance: Max cosine distance threshold. The palace collection uses
+            cosine distance (hnsw:space=cosine) — 0 = identical, 2 = opposite.
+            Results with distance > this value are filtered out. A value of
+            0.0 disables filtering. Typical useful range: 0.3–1.0.
     """
     try:
-        client = chromadb.PersistentClient(path=palace_path)
-        col = client.get_collection("mempalace_drawers")
+        col = get_collection(palace_path, create=False)
     except Exception as e:
-        return {"error": f"No palace found at {palace_path}: {e}"}
+        logger.error("No palace found at %s: %s", palace_path, e)
+        return {
+            "error": "No palace found",
+            "hint": "Run: mempalace init <dir> && mempalace mine <dir>",
+        }
 
-    # Build where filter
-    where = {}
-    if wing and room:
-        where = {"$and": [{"wing": wing}, {"room": room}]}
-    elif wing:
-        where = {"wing": wing}
-    elif room:
-        where = {"room": room}
+    where = build_where_filter(wing, room)
 
     try:
         kwargs = {
@@ -125,18 +146,23 @@ def search_memories(
 
     hits = []
     for doc, meta, dist in zip(docs, metas, dists):
+        # Filter on raw distance before rounding to avoid precision loss
+        if max_distance > 0.0 and dist > max_distance:
+            continue
         hits.append(
             {
                 "text": doc,
                 "wing": meta.get("wing", "unknown"),
                 "room": meta.get("room", "unknown"),
                 "source_file": Path(meta.get("source_file", "?")).name,
-                "similarity": round(1 - dist, 3),
+                "similarity": round(max(0.0, 1 - dist), 3),
+                "distance": round(dist, 4),
             }
         )
 
     return {
         "query": query,
         "filters": {"wing": wing, "room": room},
+        "total_before_filter": len(docs),
         "results": hits,
     }

mempalace/split_mega_files.py

@@ -34,20 +34,39 @@ LUMI_DIR = Path(os.environ.get("MEMPALACE_SOURCE_DIR", str(HOME / "Desktop/trans
 # People we know about (for name detection in content)
 # Loaded from ~/.mempalace/known_names.json if it exists, otherwise generic fallback.
 _KNOWN_NAMES_PATH = HOME / ".mempalace" / "known_names.json"
+_FALLBACK_KNOWN_PEOPLE = ["Alice", "Ben", "Riley", "Max", "Sam", "Devon", "Jordan"]
+_KNOWN_NAMES_CACHE = None
+
+
+def _load_known_names_config(force_reload: bool = False):
+    """Load and cache the optional known-names config file."""
+    global _KNOWN_NAMES_CACHE
+
+    if force_reload:
+        _KNOWN_NAMES_CACHE = None
+
+    if _KNOWN_NAMES_CACHE is not None:
+        return _KNOWN_NAMES_CACHE
+
+    if _KNOWN_NAMES_PATH.exists():
+        try:
+            _KNOWN_NAMES_CACHE = json.loads(_KNOWN_NAMES_PATH.read_text())
+            return _KNOWN_NAMES_CACHE
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    _KNOWN_NAMES_CACHE = None
+    return None
 
 
 def _load_known_people() -> list:
     """Load known names from config file, falling back to a generic list."""
-    if _KNOWN_NAMES_PATH.exists():
-        try:
-            data = json.loads(_KNOWN_NAMES_PATH.read_text())
-            if isinstance(data, list):
-                return data
-            return data.get("names", [])
-        except (json.JSONDecodeError, OSError):
-            pass
-    # Generic fallback — override by creating ~/.mempalace/known_names.json
-    return ["Alice", "Ben", "Riley", "Max", "Sam", "Devon", "Jordan"]
+    data = _load_known_names_config()
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        return data.get("names", [])
+    return list(_FALLBACK_KNOWN_PEOPLE)
 
 
 KNOWN_PEOPLE = _load_known_people()
@@ -55,13 +74,9 @@ KNOWN_PEOPLE = _load_known_people()
 
 def _load_username_map() -> dict:
     """Load username-to-name mapping from config file."""
-    if _KNOWN_NAMES_PATH.exists():
-        try:
-            data = json.loads(_KNOWN_NAMES_PATH.read_text())
-            if isinstance(data, dict):
-                return data.get("username_map", {})
-        except (json.JSONDecodeError, OSError):
-            pass
+    data = _load_known_names_config()
+    if isinstance(data, dict):
+        return data.get("username_map", {})
     return {}
 
 
@@ -167,6 +182,10 @@ def split_file(filepath, output_dir, dry_run=False):
     Returns list of output paths written (or would be written if dry_run).
     """
     path = Path(filepath)
+    max_size = 500 * 1024 * 1024  # 500 MB safety limit
+    if path.stat().st_size > max_size:
+        print(f"  SKIP: {path.name} exceeds {max_size // (1024*1024)} MB limit")
+        return []
     lines = path.read_text(errors="replace").splitlines(keepends=True)
 
     boundaries = find_session_boundaries(lines)
@@ -204,7 +223,7 @@ def split_file(filepath, output_dir, dry_run=False):
         if dry_run:
             print(f"  [{i + 1}/{len(boundaries) - 1}] {name}  ({len(chunk)} lines)")
         else:
-            out_path.write_text("".join(chunk))
+            out_path.write_text("".join(chunk), encoding="utf-8")
             print(f"  ✓ {name}  ({len(chunk)} lines)")
 
         written.append(out_path)
@@ -242,7 +261,7 @@ def main():
     )
     args = parser.parse_args()
 
-    src_dir = Path(args.source) if args.source else LUMI_DIR
+    src_dir = Path(args.source).expanduser().resolve() if args.source else LUMI_DIR
     output_dir = args.output_dir or None  # None = same dir as file
 
     if args.file:
@@ -251,7 +270,11 @@ def main():
         files = sorted(src_dir.glob("*.txt"))
 
     mega_files = []
+    max_scan_size = 500 * 1024 * 1024  # 500 MB
     for f in files:
+        if f.stat().st_size > max_scan_size:
+            print(f"  SKIP: {f.name} exceeds {max_scan_size // (1024*1024)} MB limit")
+            continue
         lines = f.read_text(errors="replace").splitlines(keepends=True)
         boundaries = find_session_boundaries(lines)
         if len(boundaries) >= args.min_sessions:

mempalace/version.py

@@ -0,0 +1,3 @@
+"""Single source of truth for the MemPalace package version."""
+
+__version__ = "3.2.0"

pyproject.toml

@@ -1,10 +1,6 @@
-[build-system]
-requires = ["setuptools>=64"]
-build-backend = "setuptools.build_meta"
-
 [project]
 name = "mempalace"
-version = "3.0.0"
+version = "3.2.0"
 description = "Give your AI a memory — mine projects and conversations into a searchable palace. No API key required."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -29,8 +25,8 @@ classifiers = [
     "Topic :: Utilities",
 ]
 dependencies = [
-    "chromadb>=0.4.0",
-    "pyyaml>=6.0",
+    "chromadb>=0.5.0",
+    "pyyaml>=6.0,<7",
 ]
 
 [project.urls]
@@ -38,25 +34,55 @@ Homepage = "https://github.com/milla-jovovich/mempalace"
 Repository = "https://github.com/milla-jovovich/mempalace"
 "Bug Tracker" = "https://github.com/milla-jovovich/mempalace/issues"
 
-[tool.setuptools.packages.find]
-include = ["mempalace*"]
-
 [project.scripts]
 mempalace = "mempalace:main"
 
 [project.optional-dependencies]
-dev = ["pytest>=7.0", "build>=1.0", "twine>=4.0"]
+dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff>=0.4.0", "psutil>=5.9"]
+spellcheck = ["autocorrect>=2.0"]
+
+[dependency-groups]
+dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff>=0.4.0", "psutil>=5.9"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["mempalace"]
 
 [tool.ruff]
 line-length = 100
 target-version = "py39"
+extend-exclude = ["benchmarks"]
 
 [tool.ruff.lint]
-select = ["E", "F", "W"]
+select = ["E", "F", "W", "C901"]
 ignore = ["E501"]
 
+[tool.ruff.lint.mccabe]
+max-complexity = 25
+
 [tool.ruff.format]
 quote-style = "double"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+pythonpath = ["."]
+addopts = "-m 'not benchmark and not slow and not stress'"
+markers = [
+    "benchmark: scale/performance benchmark tests",
+    "slow: tests that take more than 30 seconds",
+    "stress: destructive scale tests (100K+ drawers)",
+]
+
+[tool.coverage.run]
+source = ["mempalace"]
+
+[tool.coverage.report]
+fail_under = 85
+show_missing = true
+exclude_lines = [
+    "if __name__",
+    "pragma: no cover",
+]

requirements.txt

@@ -1,2 +0,0 @@
-chromadb>=0.4.0
-pyyaml>=6.0

tests/benchmarks/README.md

@@ -0,0 +1,138 @@
+# MemPalace Scale Benchmark Suite
+
+106 tests that benchmark mempalace at scale to validate real-world performance limits.
+
+## Why
+
+MemPalace has strong academic scores (96.6% R@5 on LongMemEval) but no empirical data on how it behaves at scale. Key unknowns:
+
+- `tool_status()` loads ALL metadata into memory — at what palace size does this OOM?
+- `PersistentClient` is re-instantiated on every MCP call — what's the overhead?
+- Modified files are never re-ingested — what's the skip-check cost at scale?
+- How does query latency degrade as the palace grows from 1K to 100K drawers?
+- Does wing/room filtering actually improve retrieval, and by how much?
+- At what per-room drawer count does recall break regardless of filtering?
+
+This suite finds those answers.
+
+## Quick Start
+
+```bash
+# Fast smoke test (~2 min)
+uv run pytest tests/benchmarks/ -v --bench-scale=small -m "benchmark and not slow"
+
+# Full small scale (~35 min)
+uv run pytest tests/benchmarks/ -v --bench-scale=small
+
+# Medium scale with JSON report
+uv run pytest tests/benchmarks/ -v --bench-scale=medium --bench-report=results.json
+
+# Stress test (local only, very slow)
+uv run pytest tests/benchmarks/ -v --bench-scale=stress -m stress
+```
+
+## Scale Levels
+
+| Level   | Drawers | Wings | Rooms/Wing | KG Triples | Use case            |
+|---------|---------|-------|------------|------------|---------------------|
+| small   | 1,000   | 3     | 5          | 200        | CI, quick checks    |
+| medium  | 10,000  | 8     | 12         | 2,000      | Pre-release testing |
+| large   | 50,000  | 15    | 20         | 10,000     | Scale limit finding |
+| stress  | 100,000 | 25    | 30         | 50,000     | Breaking point      |
+
+## Test Modules
+
+### Critical Path
+
+| File | What it tests |
+|------|--------------|
+| `test_mcp_bench.py` | MCP tool response times, unbounded metadata fetch, client re-instantiation overhead |
+| `test_chromadb_stress.py` | ChromaDB breaking point, query degradation curve, batch vs sequential insert |
+| `test_memory_profile.py` | RSS/heap growth over repeated operations, leak detection |
+
+### Performance Baselines
+
+| File | What it tests |
+|------|--------------|
+| `test_ingest_bench.py` | Mining throughput (files/sec, drawers/sec), peak RSS, chunking speed, re-ingest skip overhead |
+| `test_search_bench.py` | Query latency vs palace size, recall@k with planted needles, concurrent queries, n_results scaling |
+
+### Architectural Validation
+
+| File | What it tests |
+|------|--------------|
+| `test_palace_boost.py` | Retrieval improvement from wing/room filtering at different scales |
+| `test_recall_threshold.py` | Per-room recall ceiling — isolates embedding model limit with all drawers in one bucket |
+| `test_knowledge_graph_bench.py` | Triple insertion rate, temporal query accuracy, SQLite concurrent access |
+| `test_layers_bench.py` | MemoryStack wake-up cost, Layer1 unbounded fetch, token budget compliance |
+
+## Architecture
+
+```
+tests/benchmarks/
+  conftest.py              # --bench-scale / --bench-report CLI options, fixtures, markers
+  data_generator.py        # Deterministic data factory (seeded RNG, planted needles)
+  report.py                # JSON report writer + regression checker
+  test_*.py                # 9 test modules (106 tests total)
+```
+
+### Data Generator
+
+`PalaceDataGenerator(seed=42, scale="small")` produces deterministic, realistic test data:
+
+- **`generate_project_tree()`** — writes real files + `mempalace.yaml` for `mine()` to ingest
+- **`populate_palace_directly()`** — bypasses mining, inserts directly into ChromaDB (10-100x faster for search/MCP benchmarks)
+- **`generate_kg_triples()`** — entity-relationship triples with temporal validity
+- **`generate_search_queries()`** — queries with known-good answers for recall measurement
+
+**Planted needles**: Unique identifiable content (e.g., `NEEDLE_0042: PostgreSQL vacuum autovacuum threshold...`) seeded into specific wings/rooms. Search queries target these needles, enabling recall@k measurement without an LLM judge.
+
+### JSON Reports
+
+When run with `--bench-report=path.json`, produces machine-readable output:
+
+```json
+{
+  "timestamp": "2026-04-07T...",
+  "git_sha": "abc123",
+  "scale": "small",
+  "system": {"os": "linux", "cpu_count": 8},
+  "results": {
+    "mcp_status": {"latency_ms_at_1000": 45.2, "rss_delta_mb_at_5000": 12.3},
+    "search": {"avg_latency_ms_at_5000": 23.1, "recall_at_5": 0.92},
+    "chromadb_insert": {"sequential_ms": 8500, "batched_ms": 1200, "speedup_ratio": 7.1}
+  }
+}
+```
+
+### Regression Detection
+
+```python
+from tests.benchmarks.report import check_regression
+
+regressions = check_regression("current.json", "baseline.json", threshold=0.2)
+# Returns list of metric descriptions that degraded beyond 20%
+```
+
+## CI Integration
+
+The GitHub Actions workflow runs benchmarks on PRs at small scale:
+
+```yaml
+benchmark:
+  runs-on: ubuntu-latest
+  if: github.event_name == 'pull_request'
+  # Runs: pytest tests/benchmarks/ -m "benchmark and not stress and not slow" --bench-scale=small
+```
+
+Existing unit tests are isolated with `--ignore=tests/benchmarks`.
+
+## Markers
+
+- `@pytest.mark.benchmark` — all benchmark tests
+- `@pytest.mark.slow` — tests taking >30s even at small scale
+- `@pytest.mark.stress` — tests that should only run at large/stress scale
+
+## Dependencies
+
+Only one new dependency beyond the existing dev stack: `psutil` (for cross-platform RSS measurement). `tracemalloc` and `resource` are stdlib.

tests/benchmarks/__init__.py

@@ -0,0 +1 @@
+# MemPalace scale benchmark suite

tests/benchmarks/conftest.py

@@ -0,0 +1,144 @@
+"""Benchmark-specific pytest configuration, fixtures, and CLI options."""
+
+import json
+import os
+import tempfile
+
+import pytest
+
+
+SCALE_OPTIONS = ["small", "medium", "large", "stress"]
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--bench-scale",
+        default="small",
+        choices=SCALE_OPTIONS,
+        help="Scale level for benchmark tests: small (1K), medium (10K), large (50K), stress (100K)",
+    )
+    parser.addoption(
+        "--bench-report",
+        default=None,
+        help="Path for JSON benchmark report output",
+    )
+
+
+@pytest.fixture(scope="session")
+def bench_scale(request):
+    """The configured benchmark scale level."""
+    return request.config.getoption("--bench-scale")
+
+
+@pytest.fixture(scope="session")
+def bench_report_path(request):
+    """Path for JSON report output, or None."""
+    return request.config.getoption("--bench-report")
+
+
+@pytest.fixture
+def palace_dir(tmp_path):
+    """Isolated palace directory for a single test."""
+    p = tmp_path / "palace"
+    p.mkdir()
+    return str(p)
+
+
+@pytest.fixture
+def kg_db(tmp_path):
+    """Isolated KG SQLite path for a single test."""
+    return str(tmp_path / "test_kg.sqlite3")
+
+
+@pytest.fixture
+def config_dir(tmp_path):
+    """Isolated config directory for monkeypatching MempalaceConfig."""
+    d = tmp_path / "config"
+    d.mkdir()
+    config = {"palace_path": str(tmp_path / "palace"), "collection_name": "mempalace_drawers"}
+    with open(d / "config.json", "w") as f:
+        json.dump(config, f)
+    return str(d)
+
+
+@pytest.fixture
+def project_dir(tmp_path):
+    """Temporary project directory for mining tests."""
+    d = tmp_path / "project"
+    d.mkdir()
+    return d
+
+
+# ── Session-scoped result collector ──────────────────────────────────────
+
+
+class BenchmarkResults:
+    """Collect benchmark metrics across all tests in a session."""
+
+    def __init__(self):
+        self.results = {}
+
+    def record(self, category: str, metric: str, value):
+        if category not in self.results:
+            self.results[category] = {}
+        self.results[category][metric] = value
+
+
+@pytest.fixture(scope="session")
+def bench_results():
+    """Session-scoped results collector shared by all benchmark tests."""
+    return BenchmarkResults()
+
+
+def pytest_terminal_summary(terminalreporter, config):
+    """Write JSON benchmark report after all tests complete."""
+    report_path = config.getoption("--bench-report", default=None)
+    if not report_path:
+        return
+
+    # Collect results written by individual tests via record_metric()
+    import platform
+    import subprocess
+
+    try:
+        git_sha = subprocess.check_output(
+            ["git", "rev-parse", "--short", "HEAD"], text=True, stderr=subprocess.DEVNULL
+        ).strip()
+    except Exception:
+        git_sha = "unknown"
+
+    try:
+        import chromadb
+
+        chromadb_version = chromadb.__version__
+    except Exception:
+        chromadb_version = "unknown"
+
+    report = {
+        "timestamp": __import__("datetime").datetime.now().isoformat(),
+        "git_sha": git_sha,
+        "python_version": platform.python_version(),
+        "chromadb_version": chromadb_version,
+        "scale": config.getoption("--bench-scale", default="small"),
+        "system": {
+            "os": platform.system().lower(),
+            "cpu_count": os.cpu_count(),
+            "platform": platform.platform(),
+        },
+        "results": {},
+    }
+
+    # Read results from the temp file written by record_metric()
+    results_file = os.path.join(tempfile.gettempdir(), "mempalace_bench_results.json")
+    if os.path.exists(results_file):
+        try:
+            with open(results_file) as f:
+                report["results"] = json.load(f)
+            os.unlink(results_file)
+        except Exception:
+            pass
+
+    os.makedirs(os.path.dirname(os.path.abspath(report_path)), exist_ok=True)
+    with open(report_path, "w") as f:
+        json.dump(report, f, indent=2)
+    terminalreporter.write_line(f"\nBenchmark report written to: {report_path}")

tests/benchmarks/data_generator.py

@@ -0,0 +1,568 @@
+"""
+Deterministic data factory for MemPalace scale benchmarks.
+
+Generates realistic project files, conversations, and KG triples at
+configurable scale levels. All randomness uses seeded RNG for reproducibility.
+
+Planted "needle" drawers enable recall measurement without an LLM judge.
+"""
+
+import hashlib
+import os
+import random
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import chromadb
+import yaml
+
+
+# ── Scale configurations ─────────────────────────────────────────────────
+
+SCALE_CONFIGS = {
+    "small": {
+        "drawers": 1_000,
+        "wings": 3,
+        "rooms_per_wing": 5,
+        "kg_entities": 50,
+        "kg_triples": 200,
+        "needles": 20,
+        "search_queries": 20,
+    },
+    "medium": {
+        "drawers": 10_000,
+        "wings": 8,
+        "rooms_per_wing": 12,
+        "kg_entities": 200,
+        "kg_triples": 2_000,
+        "needles": 50,
+        "search_queries": 50,
+    },
+    "large": {
+        "drawers": 50_000,
+        "wings": 15,
+        "rooms_per_wing": 20,
+        "kg_entities": 500,
+        "kg_triples": 10_000,
+        "needles": 100,
+        "search_queries": 100,
+    },
+    "stress": {
+        "drawers": 100_000,
+        "wings": 25,
+        "rooms_per_wing": 30,
+        "kg_entities": 1_000,
+        "kg_triples": 50_000,
+        "needles": 200,
+        "search_queries": 200,
+    },
+}
+
+# ── Vocabulary banks for realistic content ───────────────────────────────
+
+WING_NAMES = [
+    "webapp",
+    "backend_api",
+    "mobile_app",
+    "data_pipeline",
+    "ml_platform",
+    "devops",
+    "auth_service",
+    "payments",
+    "analytics",
+    "docs_site",
+    "cli_tool",
+    "dashboard",
+    "notification_service",
+    "search_engine",
+    "user_mgmt",
+    "inventory",
+    "reporting",
+    "testing_infra",
+    "monitoring",
+    "email_service",
+    "chat_bot",
+    "file_storage",
+    "scheduler",
+    "gateway",
+    "marketplace",
+]
+
+ROOM_NAMES = [
+    "backend",
+    "frontend",
+    "api",
+    "database",
+    "auth",
+    "tests",
+    "docs",
+    "config",
+    "deployment",
+    "models",
+    "views",
+    "controllers",
+    "middleware",
+    "utils",
+    "schemas",
+    "migrations",
+    "fixtures",
+    "scripts",
+    "styles",
+    "components",
+    "hooks",
+    "services",
+    "routes",
+    "templates",
+    "static",
+    "media",
+    "logging",
+    "cache",
+    "queue",
+    "workers",
+]
+
+TECH_TERMS = [
+    "authentication",
+    "authorization",
+    "middleware",
+    "endpoint",
+    "REST API",
+    "GraphQL",
+    "WebSocket",
+    "database migration",
+    "ORM",
+    "query optimization",
+    "caching strategy",
+    "load balancer",
+    "rate limiting",
+    "pagination",
+    "serialization",
+    "validation",
+    "error handling",
+    "logging framework",
+    "monitoring",
+    "deployment pipeline",
+    "CI/CD",
+    "containerization",
+    "microservice",
+    "event sourcing",
+    "message queue",
+    "pub/sub",
+    "connection pooling",
+    "session management",
+    "token refresh",
+    "CORS",
+    "SSL termination",
+    "health check",
+    "circuit breaker",
+    "retry logic",
+    "batch processing",
+    "stream processing",
+    "data pipeline",
+    "ETL",
+    "feature flag",
+    "A/B testing",
+    "blue-green deployment",
+    "canary release",
+]
+
+CODE_SNIPPETS = [
+    "def process_request(data):\n    validated = schema.validate(data)\n    result = handler.execute(validated)\n    return Response(result, status=200)\n",
+    "class UserRepository:\n    def __init__(self, db):\n        self.db = db\n    def find_by_id(self, user_id):\n        return self.db.query(User).filter(User.id == user_id).first()\n",
+    "async def fetch_data(url, timeout=30):\n    async with aiohttp.ClientSession() as session:\n        async with session.get(url, timeout=timeout) as resp:\n            return await resp.json()\n",
+    "const handleSubmit = async (formData) => {\n  try {\n    const response = await api.post('/users', formData);\n    dispatch({ type: 'USER_CREATED', payload: response.data });\n  } catch (error) {\n    setError(error.message);\n  }\n};\n",
+    "SELECT u.name, COUNT(o.id) as order_count\nFROM users u\nLEFT JOIN orders o ON u.id = o.user_id\nWHERE u.created_at > '2025-01-01'\nGROUP BY u.name\nHAVING COUNT(o.id) > 5\nORDER BY order_count DESC;\n",
+]
+
+PROSE_TEMPLATES = [
+    "The {component} module handles {task}. It was refactored in {month} to improve {quality}. Key design decision: {decision}.",
+    "Bug report: {component} fails when {condition}. Root cause: {cause}. Fixed by {fix}. Regression test added in {test_file}.",
+    "Architecture decision: switched from {old_tech} to {new_tech} for {reason}. Migration completed {date}. Performance improved by {percent}%.",
+    "Meeting notes: discussed {topic} with {person}. Agreed to {action}. Deadline: {deadline}. Follow-up: {followup}.",
+    "Feature spec: {feature_name} allows users to {capability}. Dependencies: {deps}. Estimated effort: {effort} days.",
+]
+
+ENTITY_NAMES = [
+    "Alice",
+    "Bob",
+    "Carol",
+    "Dave",
+    "Eve",
+    "Frank",
+    "Grace",
+    "Heidi",
+    "Ivan",
+    "Judy",
+    "Karl",
+    "Linda",
+    "Mike",
+    "Nina",
+    "Oscar",
+    "Pat",
+    "Quinn",
+    "Rita",
+    "Steve",
+    "Tina",
+    "Ursula",
+    "Victor",
+    "Wendy",
+    "Xander",
+]
+
+ENTITY_TYPES = ["person", "project", "tool", "concept", "team", "service"]
+
+PREDICATES = [
+    "works_on",
+    "manages",
+    "reports_to",
+    "collaborates_with",
+    "created",
+    "maintains",
+    "uses",
+    "depends_on",
+    "replaced",
+    "reviewed",
+    "deployed",
+    "tested",
+    "documented",
+    "mentors",
+    "leads",
+    "contributes_to",
+]
+
+
+class PalaceDataGenerator:
+    """Generate deterministic, realistic test data at configurable scale."""
+
+    def __init__(self, seed=42, scale="small"):
+        self.rng = random.Random(seed)
+        self.scale = scale
+        self.cfg = SCALE_CONFIGS[scale]
+        self.wings = WING_NAMES[: self.cfg["wings"]]
+        self.rooms_by_wing = {}
+        for wing in self.wings:
+            n = self.cfg["rooms_per_wing"]
+            rooms = self.rng.sample(ROOM_NAMES, min(n, len(ROOM_NAMES)))
+            self.rooms_by_wing[wing] = rooms
+        # Planted needles for recall measurement
+        self.needles = []
+        self._generate_needles()
+
+    def _generate_needles(self):
+        """Create unique needle content for recall testing."""
+        topics = [
+            "Fibonacci sequence optimization uses memoization with O(n) space complexity",
+            "PostgreSQL vacuum autovacuum threshold set to 50 percent for table users",
+            "Redis cluster failover timeout configured at 30 seconds with sentinel monitoring",
+            "Kubernetes horizontal pod autoscaler targets 70 percent CPU utilization",
+            "GraphQL subscription uses WebSocket transport with heartbeat interval 25 seconds",
+            "JWT token rotation policy requires refresh every 15 minutes with sliding window",
+            "Elasticsearch index sharding strategy uses 5 primary shards with 1 replica each",
+            "Docker multi-stage build reduces image size from 1.2GB to 180MB for production",
+            "Apache Kafka consumer group rebalance timeout set to 45 seconds",
+            "MongoDB change streams resume token persisted every 100 operations",
+            "gRPC streaming uses bidirectional flow control with 64KB window size",
+            "Prometheus alerting rule fires when p99 latency exceeds 500ms for 5 minutes",
+            "Terraform state locking uses DynamoDB with consistent reads enabled",
+            "Nginx rate limiting configured at 100 requests per second with burst of 50",
+            "SQLAlchemy connection pool size set to 20 with max overflow of 10 connections",
+            "React concurrent mode uses startTransition for non-urgent state updates",
+            "AWS Lambda cold start mitigation uses provisioned concurrency of 10 instances",
+            "Git bisect automated with custom test script for regression hunting",
+            "OpenTelemetry trace sampling rate set to 10 percent in production environment",
+            "Celery worker prefetch multiplier set to 1 for fair task distribution",
+        ]
+        for i in range(self.cfg["needles"]):
+            topic = topics[i % len(topics)]
+            wing = self.rng.choice(self.wings)
+            room = self.rng.choice(self.rooms_by_wing[wing])
+            needle_id = f"NEEDLE_{i:04d}"
+            content = f"{needle_id}: {topic}. This is a unique planted needle for recall benchmarking at scale."
+            self.needles.append(
+                {
+                    "id": needle_id,
+                    "content": content,
+                    "wing": wing,
+                    "room": room,
+                    "query": topic.split(" uses ")[0]
+                    if " uses " in topic
+                    else topic.split(" set to ")[0]
+                    if " set to " in topic
+                    else topic[:60],
+                }
+            )
+
+    def _random_text(self, min_chars=600, max_chars=900):
+        """Generate a random text block of realistic content."""
+        parts = []
+        total = 0
+        target = self.rng.randint(min_chars, max_chars)
+        while total < target:
+            choice = self.rng.random()
+            if choice < 0.3:
+                text = self.rng.choice(CODE_SNIPPETS)
+            elif choice < 0.7:
+                template = self.rng.choice(PROSE_TEMPLATES)
+                text = template.format(
+                    component=self.rng.choice(ROOM_NAMES),
+                    task=self.rng.choice(TECH_TERMS),
+                    month=self.rng.choice(["January", "February", "March", "April", "May"]),
+                    quality=self.rng.choice(
+                        ["performance", "readability", "test coverage", "latency"]
+                    ),
+                    decision=self.rng.choice(TECH_TERMS),
+                    condition=self.rng.choice(TECH_TERMS) + " is null",
+                    cause=self.rng.choice(["race condition", "null pointer", "timeout", "OOM"]),
+                    fix="adding " + self.rng.choice(TECH_TERMS),
+                    test_file=f"test_{self.rng.choice(ROOM_NAMES)}.py",
+                    old_tech=self.rng.choice(["MySQL", "Flask", "REST", "Jenkins"]),
+                    new_tech=self.rng.choice(
+                        ["PostgreSQL", "FastAPI", "GraphQL", "GitHub Actions"]
+                    ),
+                    reason=self.rng.choice(TECH_TERMS),
+                    date=f"2025-{self.rng.randint(1, 12):02d}-{self.rng.randint(1, 28):02d}",
+                    percent=self.rng.randint(10, 80),
+                    topic=self.rng.choice(TECH_TERMS),
+                    person=self.rng.choice(ENTITY_NAMES),
+                    action=self.rng.choice(["refactor", "migrate", "optimize", "test"]),
+                    deadline=f"2025-{self.rng.randint(1, 12):02d}-{self.rng.randint(1, 28):02d}",
+                    followup=self.rng.choice(TECH_TERMS),
+                    feature_name=self.rng.choice(TECH_TERMS),
+                    capability=self.rng.choice(TECH_TERMS),
+                    deps=", ".join(self.rng.sample(TECH_TERMS, 2)),
+                    effort=self.rng.randint(1, 15),
+                )
+            else:
+                words = self.rng.sample(TECH_TERMS, min(5, len(TECH_TERMS)))
+                text = (
+                    " ".join(words)
+                    + ". "
+                    + self.rng.choice(TECH_TERMS)
+                    + " implementation details follow.\n"
+                )
+            parts.append(text)
+            total += len(text)
+        return "\n".join(parts)[:max_chars]
+
+    # ── Project tree generation (for mine() tests) ───────────────────────
+
+    def generate_project_tree(self, base_path, wing=None, rooms=None, n_files=50):
+        """
+        Write realistic project files + mempalace.yaml to base_path.
+
+        Returns the project path suitable for passing to mine().
+        """
+        base = Path(base_path)
+        base.mkdir(parents=True, exist_ok=True)
+        wing = wing or self.rng.choice(self.wings)
+        rooms = rooms or self.rooms_by_wing.get(wing, ["general"])
+
+        # Write mempalace.yaml
+        room_defs = [{"name": r, "description": f"{r} code and docs"} for r in rooms]
+        with open(base / "mempalace.yaml", "w") as f:
+            yaml.dump({"wing": wing, "rooms": room_defs}, f)
+
+        # Write files distributed across room directories
+        files_written = 0
+        for i in range(n_files):
+            room = rooms[i % len(rooms)]
+            room_dir = base / room
+            room_dir.mkdir(parents=True, exist_ok=True)
+
+            ext = self.rng.choice([".py", ".js", ".md", ".ts", ".yaml"])
+            filename = f"file_{i:04d}{ext}"
+            content = self._random_text(400, 2000)
+            (room_dir / filename).write_text(content, encoding="utf-8")
+            files_written += 1
+
+        return str(base), wing, rooms, files_written
+
+    # ── Conversation file generation (for mine_convos() tests) ───────────
+
+    def generate_conversation_files(self, base_path, wing=None, n_files=20):
+        """Write conversation transcript files for convo_miner tests."""
+        base = Path(base_path)
+        base.mkdir(parents=True, exist_ok=True)
+        wing = wing or self.rng.choice(self.wings)
+
+        for i in range(n_files):
+            lines = []
+            n_exchanges = self.rng.randint(5, 20)
+            for j in range(n_exchanges):
+                user_msg = f"> User: {self.rng.choice(TECH_TERMS)}? How does {self.rng.choice(TECH_TERMS)} work with {self.rng.choice(TECH_TERMS)}?"
+                ai_msg = self._random_text(200, 600)
+                lines.append(user_msg)
+                lines.append(ai_msg)
+                lines.append("")
+
+            (base / f"convo_{i:04d}.txt").write_text("\n".join(lines), encoding="utf-8")
+
+        return str(base), wing
+
+    # ── Direct palace population (bypasses mining for speed) ─────────────
+
+    def populate_palace_directly(self, palace_path, n_drawers=None, include_needles=True):
+        """
+        Insert drawers directly into ChromaDB, bypassing the mining pipeline.
+
+        Much faster than mining for benchmarks that only care about
+        search/MCP behavior on a pre-populated palace.
+
+        Returns (client, collection, needle_info).
+        """
+        n_drawers = n_drawers or self.cfg["drawers"]
+        os.makedirs(palace_path, exist_ok=True)
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_or_create_collection("mempalace_drawers")
+
+        batch_size = 500
+        docs = []
+        ids = []
+        metas = []
+
+        # Insert needles first
+        needle_info = []
+        if include_needles:
+            for needle in self.needles:
+                needle_id = f"drawer_{needle['wing']}_{needle['room']}_{hashlib.md5(needle['id'].encode()).hexdigest()[:16]}"
+                docs.append(needle["content"])
+                ids.append(needle_id)
+                metas.append(
+                    {
+                        "wing": needle["wing"],
+                        "room": needle["room"],
+                        "source_file": f"needle_{needle['id']}.txt",
+                        "chunk_index": 0,
+                        "added_by": "benchmark",
+                        "filed_at": datetime.now().isoformat(),
+                    }
+                )
+                needle_info.append(
+                    {
+                        "id": needle_id,
+                        "query": needle["query"],
+                        "wing": needle["wing"],
+                        "room": needle["room"],
+                    }
+                )
+
+        # Fill remaining drawers with realistic content
+        remaining = n_drawers - len(docs)
+        for i in range(remaining):
+            wing = self.wings[i % len(self.wings)]
+            rooms = self.rooms_by_wing[wing]
+            room = rooms[i % len(rooms)]
+            content = self._random_text(400, 800)
+            drawer_id = f"drawer_{wing}_{room}_{hashlib.md5(f'gen_{i}'.encode()).hexdigest()[:16]}"
+
+            docs.append(content)
+            ids.append(drawer_id)
+            metas.append(
+                {
+                    "wing": wing,
+                    "room": room,
+                    "source_file": f"generated_{i:06d}.txt",
+                    "chunk_index": i % 10,
+                    "added_by": "benchmark",
+                    "filed_at": datetime.now().isoformat(),
+                }
+            )
+
+            # Flush in batches
+            if len(docs) >= batch_size:
+                col.add(documents=docs, ids=ids, metadatas=metas)
+                docs, ids, metas = [], [], []
+
+        # Flush remainder
+        if docs:
+            col.add(documents=docs, ids=ids, metadatas=metas)
+
+        return client, col, needle_info
+
+    # ── KG triple generation ─────────────────────────────────────────────
+
+    def generate_kg_triples(self, n_entities=None, n_triples=None):
+        """
+        Generate realistic entity-relationship triples.
+
+        Returns (entities, triples) where:
+          entities = [(name, type), ...]
+          triples = [(subject, predicate, object, valid_from, valid_to), ...]
+        """
+        n_entities = n_entities or self.cfg["kg_entities"]
+        n_triples = n_triples or self.cfg["kg_triples"]
+
+        # Generate entities
+        entities = []
+        entity_names = []
+        for i in range(n_entities):
+            if i < len(ENTITY_NAMES):
+                name = ENTITY_NAMES[i]
+            else:
+                name = f"Entity_{i:04d}"
+            etype = self.rng.choice(ENTITY_TYPES)
+            entities.append((name, etype))
+            entity_names.append(name)
+
+        # Generate triples
+        triples = []
+        base_date = datetime(2024, 1, 1)
+        for i in range(n_triples):
+            subject = self.rng.choice(entity_names)
+            obj = self.rng.choice(entity_names)
+            while obj == subject:
+                obj = self.rng.choice(entity_names)
+            predicate = self.rng.choice(PREDICATES)
+            days_offset = self.rng.randint(0, 730)
+            valid_from = (base_date + timedelta(days=days_offset)).strftime("%Y-%m-%d")
+            # 30% chance of having a valid_to
+            valid_to = None
+            if self.rng.random() < 0.3:
+                end_offset = self.rng.randint(30, 365)
+                valid_to = (base_date + timedelta(days=days_offset + end_offset)).strftime(
+                    "%Y-%m-%d"
+                )
+            triples.append((subject, predicate, obj, valid_from, valid_to))
+
+        return entities, triples
+
+    # ── Search query generation ──────────────────────────────────────────
+
+    def generate_search_queries(self, n_queries=None):
+        """
+        Generate search queries with expected results.
+
+        Returns list of {"query": str, "expected_wing": str|None, "expected_room": str|None, "is_needle": bool}.
+        Needle queries have known-good answers for recall measurement.
+        """
+        n_queries = n_queries or self.cfg["search_queries"]
+        queries = []
+
+        # Half are needle queries (known-good answers)
+        n_needle = min(n_queries // 2, len(self.needles))
+        for needle in self.needles[:n_needle]:
+            queries.append(
+                {
+                    "query": needle["query"],
+                    "expected_wing": needle["wing"],
+                    "expected_room": needle["room"],
+                    "needle_id": needle["id"],
+                    "is_needle": True,
+                }
+            )
+
+        # Other half are generic queries (measure latency, not recall)
+        n_generic = n_queries - n_needle
+        for _ in range(n_generic):
+            queries.append(
+                {
+                    "query": self.rng.choice(TECH_TERMS) + " " + self.rng.choice(TECH_TERMS),
+                    "expected_wing": None,
+                    "expected_room": None,
+                    "needle_id": None,
+                    "is_needle": False,
+                }
+            )
+
+        self.rng.shuffle(queries)
+        return queries

tests/benchmarks/report.py

@@ -0,0 +1,117 @@
+"""
+Benchmark report utilities — JSON output and regression detection.
+
+Each test records metrics via record_metric(). At session end, the
+conftest.py pytest_terminal_summary hook writes the collected results.
+"""
+
+import json
+import os
+import tempfile
+
+
+RESULTS_FILE = os.path.join(tempfile.gettempdir(), "mempalace_bench_results.json")
+
+
+def record_metric(category: str, metric: str, value):
+    """Append a metric to the session results file (JSON on disk)."""
+    results = {}
+    if os.path.exists(RESULTS_FILE):
+        try:
+            with open(RESULTS_FILE) as f:
+                results = json.load(f)
+        except (json.JSONDecodeError, OSError):
+            results = {}
+
+    if category not in results:
+        results[category] = {}
+    results[category][metric] = value
+
+    with open(RESULTS_FILE, "w") as f:
+        json.dump(results, f, indent=2)
+
+
+def check_regression(current_report: str, baseline_report: str, threshold: float = 0.2):
+    """
+    Compare current benchmark results against a baseline.
+
+    Returns a list of regression descriptions. Empty list = no regressions.
+
+    threshold: fractional degradation allowed (0.2 = 20% worse is OK).
+    """
+    with open(current_report) as f:
+        current = json.load(f)
+    with open(baseline_report) as f:
+        baseline = json.load(f)
+
+    regressions = []
+    # Keywords for metric direction — checked in order, first match wins.
+    # "improvement" is checked before "latency" so that composite names
+    # like "latency_improvement_pct" are classified correctly.
+    _higher_is_better_kw = [
+        "improvement",
+        "recall",
+        "throughput",
+        "per_sec",
+        "files_per_sec",
+        "drawers_per_sec",
+        "triples_per_sec",
+        "speedup",
+    ]
+    _higher_is_worse_kw = [
+        "latency",
+        "rss",
+        "memory",
+        "oom",
+        "lock_failures",
+        "elapsed",
+        "p50_ms",
+        "p95_ms",
+        "p99_ms",
+        "rss_delta_mb",
+        "peak_rss_mb",
+        "errors",
+        "failures",
+    ]
+
+    def _metric_direction(name: str) -> str:
+        """Return 'higher_better', 'higher_worse', or 'unknown'."""
+        low = name.lower()
+        for kw in _higher_is_better_kw:
+            if kw in low:
+                return "higher_better"
+        for kw in _higher_is_worse_kw:
+            if kw in low:
+                return "higher_worse"
+        return "unknown"
+
+    for category in baseline.get("results", {}):
+        if category not in current.get("results", {}):
+            continue
+        for metric, base_val in baseline["results"][category].items():
+            if metric not in current["results"][category]:
+                continue
+            curr_val = current["results"][category][metric]
+            if not isinstance(base_val, (int, float)) or not isinstance(curr_val, (int, float)):
+                continue
+            if base_val == 0:
+                continue
+
+            direction = _metric_direction(metric)
+
+            if direction == "higher_worse":
+                # Higher is worse — check if current exceeds baseline by threshold
+                if curr_val > base_val * (1 + threshold):
+                    pct = ((curr_val - base_val) / base_val) * 100
+                    regressions.append(
+                        f"{category}/{metric}: {base_val:.2f} -> {curr_val:.2f} ({pct:+.1f}%, threshold {threshold * 100:.0f}%)"
+                    )
+            elif direction == "higher_better":
+                # Lower is worse — check if current is below baseline by threshold
+                if curr_val < base_val * (1 - threshold):
+                    pct = ((curr_val - base_val) / base_val) * 100
+                    regressions.append(
+                        f"{category}/{metric}: {base_val:.2f} -> {curr_val:.2f} ({pct:+.1f}%, threshold {threshold * 100:.0f}%)"
+                    )
+
+    return regressions

tests/benchmarks/test_chromadb_stress.py

@@ -0,0 +1,206 @@
+"""
+ChromaDB stress tests — find the breaking point.
+
+Tests the raw ChromaDB patterns used by mempalace to determine:
+  - At what collection size does col.get(include=["metadatas"]) become dangerous?
+  - How does query latency degrade as collection grows?
+  - How much faster is batched insertion vs sequential?
+"""
+
+import os
+import time
+
+import chromadb
+import pytest
+
+from tests.benchmarks.data_generator import PalaceDataGenerator
+from tests.benchmarks.report import record_metric
+
+
+def _get_rss_mb():
+    try:
+        import psutil
+
+        return psutil.Process().memory_info().rss / (1024 * 1024)
+    except ImportError:
+        import resource
+        import platform
+
+        usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        if platform.system() == "Darwin":
+            return usage / (1024 * 1024)
+        return usage / 1024
+
+
+@pytest.mark.benchmark
+class TestGetAllMetadatasOOM:
+    """
+    The specific pattern causing finding #3:
+    col.get(include=["metadatas"]) with NO limit.
+
+    Measures RSS growth to find when this becomes dangerous.
+    """
+
+    SIZES = [1_000, 2_500, 5_000, 10_000]
+
+    @pytest.mark.parametrize("n_drawers", SIZES)
+    def test_get_all_metadatas_rss(self, n_drawers, tmp_path, bench_scale):
+        """RSS growth from fetching all metadata at once."""
+        gen = PalaceDataGenerator(seed=42, scale=bench_scale)
+        palace_path = str(tmp_path / "palace")
+        gen.populate_palace_directly(palace_path, n_drawers=n_drawers, include_needles=False)
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+
+        rss_before = _get_rss_mb()
+        start = time.perf_counter()
+        all_meta = col.get(include=["metadatas"])["metadatas"]
+        elapsed_ms = (time.perf_counter() - start) * 1000
+        rss_after = _get_rss_mb()
+
+        assert len(all_meta) == n_drawers
+        rss_delta = rss_after - rss_before
+
+        record_metric("chromadb_get_all", f"rss_delta_mb_at_{n_drawers}", round(rss_delta, 2))
+        record_metric("chromadb_get_all", f"latency_ms_at_{n_drawers}", round(elapsed_ms, 1))
+
+
+@pytest.mark.benchmark
+class TestQueryDegradation:
+    """Measure query latency as collection grows."""
+
+    SIZES = [1_000, 2_500, 5_000, 10_000]
+
+    @pytest.mark.parametrize("n_drawers", SIZES)
+    def test_query_latency_at_size(self, n_drawers, tmp_path, bench_scale):
+        gen = PalaceDataGenerator(seed=42, scale=bench_scale)
+        palace_path = str(tmp_path / "palace")
+        gen.populate_palace_directly(palace_path, n_drawers=n_drawers, include_needles=False)
+
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_collection("mempalace_drawers")
+
+        queries = [
+            "authentication middleware optimization",
+            "database connection pooling strategy",
+            "error handling retry logic",
+            "deployment pipeline configuration",
+            "load balancer health check",
+        ]
+
+        latencies = []
+        for q in queries:
+            start = time.perf_counter()
+            results = col.query(query_texts=[q], n_results=5, include=["documents", "distances"])
+            elapsed_ms = (time.perf_counter() - start) * 1000
+            latencies.append(elapsed_ms)
+            assert results["documents"][0]  # got results
+
+        avg_ms = sum(latencies) / len(latencies)
+        p95_ms = sorted(latencies)[int(len(latencies) * 0.95)]
+
+        record_metric("chromadb_query", f"avg_latency_ms_at_{n_drawers}", round(avg_ms, 1))
+        record_metric("chromadb_query", f"p95_latency_ms_at_{n_drawers}", round(p95_ms, 1))
+
+
+@pytest.mark.benchmark
+class TestBulkInsertPerformance:
+    """Compare batch insertion vs sequential add_drawer pattern."""
+
+    def test_sequential_vs_batched(self, tmp_path):
+        """The current miner uses single-document add(). How much faster is batching?"""
+        n_docs = 500
+        gen = PalaceDataGenerator(seed=42)
+
+        # Generate content
+        contents = [gen._random_text(400, 800) for _ in range(n_docs)]
+
+        # Sequential insertion (mimics add_drawer pattern)
+        palace_seq = str(tmp_path / "seq")
+        os.makedirs(palace_seq)
+        client_seq = chromadb.PersistentClient(path=palace_seq)
+        col_seq = client_seq.get_or_create_collection("mempalace_drawers")
+
+        start = time.perf_counter()
+        for i, content in enumerate(contents):
+            col_seq.add(
+                documents=[content],
+                ids=[f"seq_{i}"],
+                metadatas=[{"wing": "test", "room": "bench", "chunk_index": i}],
+            )
+        sequential_ms = (time.perf_counter() - start) * 1000
+
+        # Batched insertion
+        palace_batch = str(tmp_path / "batch")
+        os.makedirs(palace_batch)
+        client_batch = chromadb.PersistentClient(path=palace_batch)
+        col_batch = client_batch.get_or_create_collection("mempalace_drawers")
+
+        batch_size = 100
+        start = time.perf_counter()
+        for batch_start in range(0, n_docs, batch_size):
+            batch_end = min(batch_start + batch_size, n_docs)
+            batch_docs = contents[batch_start:batch_end]
+            batch_ids = [f"batch_{i}" for i in range(batch_start, batch_end)]
+            batch_metas = [
+                {"wing": "test", "room": "bench", "chunk_index": i}
+                for i in range(batch_start, batch_end)
+            ]
+            col_batch.add(documents=batch_docs, ids=batch_ids, metadatas=batch_metas)
+        batched_ms = (time.perf_counter() - start) * 1000
+
+        speedup = sequential_ms / max(batched_ms, 0.01)
+
+        assert col_seq.count() == n_docs
+        assert col_batch.count() == n_docs
+
+        record_metric("chromadb_insert", "sequential_ms", round(sequential_ms, 1))
+        record_metric("chromadb_insert", "batched_ms", round(batched_ms, 1))
+        record_metric("chromadb_insert", "speedup_ratio", round(speedup, 2))
+        record_metric("chromadb_insert", "n_docs", n_docs)
+        record_metric("chromadb_insert", "batch_size", batch_size)
+
+
+@pytest.mark.benchmark
+@pytest.mark.slow
+class TestMaxCollectionSize:
+    """Incrementally grow collection to find practical limits."""
+
+    def test_incremental_growth(self, tmp_path, bench_scale):
+        """Add drawers in batches, measure latency per batch."""
+        gen = PalaceDataGenerator(seed=42, scale=bench_scale)
+        cfg = gen.cfg
+        target = min(cfg["drawers"], 10_000)  # cap at 10K for this test
+
+        palace_path = str(tmp_path / "palace")
+        os.makedirs(palace_path)
+        client = chromadb.PersistentClient(path=palace_path)
+        col = client.get_or_create_collection("mempalace_drawers")
+
+        batch_size = 500
+        batch_times = []
+        total_inserted = 0
+
+        for batch_num in range(0, target, batch_size):
+            n = min(batch_size, target - batch_num)
+            docs = [gen._random_text(400, 800) for _ in range(n)]
+            ids = [f"growth_{batch_num + i}" for i in range(n)]
+            metas = [
+                {"wing": gen.wings[i % len(gen.wings)], "room": "bench", "chunk_index": i}
+                for i in range(batch_num, batch_num + n)
+            ]
+
+            start = time.perf_counter()
+            col.add(documents=docs, ids=ids, metadatas=metas)
+            batch_ms = (time.perf_counter() - start) * 1000
+            total_inserted += n
+            batch_times.append({"at_size": total_inserted, "batch_ms": round(batch_ms, 1)})
+
+        assert col.count() == total_inserted
+
+        # Record first and last batch times to show degradation
+        record_metric("chromadb_growth", "first_batch_ms", batch_times[0]["batch_ms"])
+        record_metric("chromadb_growth", "last_batch_ms", batch_times[-1]["batch_ms"])
+        record_metric("chromadb_growth", "total_inserted", total_inserted)
+        record_metric("chromadb_growth", "batch_times", batch_times)