Merge pull request #1232 from MemPalace/chore/release-3.3.4-prep

chore(release): v3.3.4

.agents/plugins/marketplace.json

@@ -0,0 +1,20 @@
+{
+  "name": "mempalace",
+  "interface": {
+    "displayName": "MemPalace"
+  },
+  "plugins": [
+    {
+      "name": "mempalace",
+      "source": {
+        "source": "local",
+        "path": "./.codex-plugin"
+      },
+      "policy": {
+        "installation": "AVAILABLE",
+        "authentication": "NONE"
+      },
+      "category": "Coding"
+    }
+  ]
+}

.claude-plugin/.mcp.json

@@ -0,0 +1,5 @@
+{
+  "mempalace": {
+    "command": "mempalace-mcp"
+  }
+}

.claude-plugin/README.md

@@ -0,0 +1,57 @@
+# MemPalace Claude Code Plugin
+
+A Claude Code plugin that gives your AI a persistent memory system. Mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and 5 guided skills.
+
+## Prerequisites
+
+- Python 3.9+
+
+## Installation
+
+### Claude Code Marketplace
+
+```bash
+claude plugin marketplace add MemPalace/mempalace
+claude plugin install --scope user mempalace
+```
+
+### Local Clone
+
+```bash
+claude plugin add /path/to/mempalace
+```
+
+## Post-Install Setup
+
+After installing the plugin, run the init command to complete setup (pip install, MCP configuration, etc.):
+
+```
+/mempalace:init
+```
+
+## Available Slash Commands
+
+| Command | Description |
+|---------|-------------|
+| `/mempalace:help` | Show available tools, skills, and architecture |
+| `/mempalace:init` | Set up MemPalace -- install, configure MCP, onboard |
+| `/mempalace:search` | Search your memories across the palace |
+| `/mempalace:mine` | Mine projects and conversations into the palace |
+| `/mempalace:status` | Show palace overview -- wings, rooms, drawer counts |
+
+## Hooks
+
+MemPalace registers two hooks that run automatically:
+
+- **Stop** -- Saves conversation context every 15 messages.
+- **PreCompact** -- Preserves important memories before context compaction.
+
+Set the `MEMPAL_DIR` environment variable to a directory path to automatically run `mempalace mine` on that directory during each save trigger.
+
+## MCP Server
+
+The plugin automatically configures a local MCP server with 19 tools for storing, searching, and managing memories. No manual MCP setup is required -- `/mempalace:init` handles everything.
+
+## Full Documentation
+
+See the main [README](../README.md) for complete documentation, architecture details, and advanced usage.

.claude-plugin/commands/help.md

@@ -0,0 +1,6 @@
+---
+description: Show comprehensive MemPalace help — available skills, MCP tools, CLI commands, hooks, and architecture.
+allowed-tools: Bash, Read
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `help` command, then follow its instructions.

.claude-plugin/commands/init.md

@@ -0,0 +1,6 @@
+---
+description: Set up MemPalace — install the package, initialize a palace, configure MCP server, and verify everything works.
+allowed-tools: Bash, Read, Write, Edit, Glob, Grep
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `init` command, then follow its instructions.

.claude-plugin/commands/mine.md

@@ -0,0 +1,7 @@
+---
+description: Mine projects and conversations into the MemPalace. Supports project files, conversation exports, and auto-classification.
+argument-hint: Path to project or conversation export to mine.
+allowed-tools: Bash, Read, Write, Edit, Glob, Grep
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `mine` command, then follow its instructions.

.claude-plugin/commands/search.md

@@ -0,0 +1,7 @@
+---
+description: Search your memories across the MemPalace using semantic search with wing/room filtering.
+argument-hint: Search query, optionally with wing/room filters.
+allowed-tools: Bash, Read
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `search` command, then follow its instructions.

.claude-plugin/commands/status.md

@@ -0,0 +1,6 @@
+---
+description: Show the current state of your memory palace — wings, rooms, drawer counts, and suggestions.
+allowed-tools: Bash, Read
+---
+
+Invoke the generic mempalace skill (using the Skill tool) with the `status` command, then follow its instructions.

.claude-plugin/hooks/hooks.json

@@ -0,0 +1,25 @@
+{
+  "description": "MemPalace auto-save and pre-compact hooks",
+  "hooks": {
+    "Stop": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/mempal-stop-hook.sh"
+          }
+        ]
+      }
+    ],
+    "PreCompact": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/mempal-precompact-hook.sh"
+          }
+        ]
+      }
+    ]
+  }
+}

.claude-plugin/hooks/mempal-precompact-hook.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# MemPalace PreCompact Hook — thin wrapper calling Python CLI
+# All logic lives in mempalace.hooks_cli for cross-harness extensibility
+run_mempalace_hook() {
+  if command -v mempalace >/dev/null 2>&1; then
+    mempalace hook run "$@"
+    return $?
+  fi
+
+  if command -v python3 >/dev/null 2>&1 && python3 -c "import mempalace" >/dev/null 2>&1; then
+    python3 -m mempalace hook run "$@"
+    return $?
+  fi
+
+  if command -v python >/dev/null 2>&1 && python -c "import mempalace" >/dev/null 2>&1; then
+    python -m mempalace hook run "$@"
+    return $?
+  fi
+
+  echo "MemPalace hook error: could not find a runnable mempalace command or module" >&2
+  return 1
+}
+
+run_mempalace_hook --hook precompact --harness claude-code

.claude-plugin/hooks/mempal-stop-hook.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# MemPalace Stop Hook — thin wrapper calling Python CLI
+# All logic lives in mempalace.hooks_cli for cross-harness extensibility
+run_mempalace_hook() {
+  if command -v mempalace >/dev/null 2>&1; then
+    mempalace hook run "$@"
+    return $?
+  fi
+
+  if command -v python3 >/dev/null 2>&1 && python3 -c "import mempalace" >/dev/null 2>&1; then
+    python3 -m mempalace hook run "$@"
+    return $?
+  fi
+
+  if command -v python >/dev/null 2>&1 && python -c "import mempalace" >/dev/null 2>&1; then
+    python -m mempalace hook run "$@"
+    return $?
+  fi
+
+  echo "MemPalace hook error: could not find a runnable mempalace command or module" >&2
+  return 1
+}
+
+run_mempalace_hook --hook stop --harness claude-code

.claude-plugin/marketplace.json

@@ -0,0 +1,18 @@
+{
+  "name": "mempalace",
+  "owner": {
+    "name": "milla-jovovich",
+    "url": "https://github.com/MemPalace"
+  },
+  "plugins": [
+    {
+      "name": "mempalace",
+      "source": "./.claude-plugin",
+      "description": "AI memory system — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, guided setup.",
+      "version": "3.3.4",
+      "author": {
+        "name": "milla-jovovich"
+      }
+    }
+  ]
+}

.claude-plugin/plugin.json

@@ -0,0 +1,25 @@
+{
+  "name": "mempalace",
+  "version": "3.3.4",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.",
+  "author": {
+    "name": "milla-jovovich"
+  },
+  "license": "MIT",
+  "commands": [],
+  "mcpServers": {
+    "mempalace": {
+      "command": "mempalace-mcp"
+    }
+  },
+  "keywords": [
+    "memory",
+    "ai",
+    "rag",
+    "mcp",
+    "chromadb",
+    "palace",
+    "search"
+  ],
+  "repository": "https://github.com/MemPalace/mempalace"
+}

.claude-plugin/skills/mempalace/SKILL.md

@@ -0,0 +1,35 @@
+---
+name: mempalace
+description: MemPalace — mine projects and conversations into a searchable memory palace. Use when asked about mempalace, memory palace, mining memories, searching memories, or palace setup.
+allowed-tools: Bash, Read, Write, Edit, Glob, Grep
+---
+
+# MemPalace
+
+A searchable memory palace for AI — mine projects and conversations, then search them semantically.
+
+## Prerequisites
+
+Ensure `mempalace` is installed:
+
+```bash
+mempalace --version
+```
+
+If not installed:
+
+```bash
+pip install mempalace
+```
+
+## Usage
+
+MemPalace provides dynamic instructions via the CLI. To get instructions for any operation:
+
+```bash
+mempalace instructions <command>
+```
+
+Where `<command>` is one of: `help`, `init`, `mine`, `search`, `status`.
+
+Run the appropriate instructions command, then follow the returned instructions step by step.

.codex-plugin/README.md

@@ -0,0 +1,75 @@
+# MemPalace - Codex CLI Plugin
+
+Give your AI a persistent memory -- mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and guided skills.
+
+## Prerequisites
+
+- Python 3.9+
+- Codex CLI installed and configured
+- `pip install mempalace`
+
+## Installation
+
+### Local Install
+
+1. Copy or symlink the `.codex-plugin` directory into your project root:
+
+```bash
+cp -r .codex-plugin /path/to/your/project/.codex-plugin
+```
+
+2. Verify the plugin is detected:
+
+```bash
+codex --plugins
+```
+
+3. Initialize your palace:
+
+```bash
+codex /init
+```
+
+### Git Install
+
+1. Clone the MemPalace repository:
+
+```bash
+git clone https://github.com/MemPalace/mempalace.git
+cd mempalace
+```
+
+2. Install the Python package:
+
+```bash
+pip install -e .
+```
+
+3. The `.codex-plugin` directory is already in the repo root. Codex CLI will detect it automatically when you run Codex from inside the repository.
+
+4. Initialize your palace:
+
+```bash
+codex /init
+```
+
+## Available Skills
+
+| Skill | Description |
+|-------|-------------|
+| `/help` | Show available commands and usage tips |
+| `/init` | Initialize a new memory palace |
+| `/search` | Semantic search across all mined memories |
+| `/mine` | Mine a project or conversation into your palace |
+| `/status` | Show palace status, room counts, and health |
+
+## Hooks
+
+The plugin includes auto-save hooks that run on session stop (every 15 messages) and before context compaction, automatically preserving conversation context into your palace.
+
+Set the `MEMPAL_DIR` environment variable to a directory path to automatically run `mempalace mine` on that directory during each save trigger.
+
+## Support
+
+- Repository: https://github.com/MemPalace/mempalace
+- Issues: https://github.com/MemPalace/mempalace/issues

.codex-plugin/hooks.json

@@ -0,0 +1,37 @@
+{
+  "hooks": {
+    "SessionStart": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "${CODEX_PLUGIN_ROOT}/hooks/mempal-hook.sh session-start"
+          }
+        ]
+      }
+    ],
+    "Stop": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "${CODEX_PLUGIN_ROOT}/hooks/mempal-hook.sh stop"
+          }
+        ]
+      }
+    ],
+    "PreCompact": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "${CODEX_PLUGIN_ROOT}/hooks/mempal-hook.sh precompact"
+          }
+        ]
+      }
+    ]
+  }
+}

.codex-plugin/hooks/mempal-hook.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+HOOK_NAME="${1:?Usage: mempal-hook.sh <hook-name>}"
+INPUT_FILE=$(mktemp) || { echo "Failed to create temp file" >&2; exit 1; }
+cat > "$INPUT_FILE"
+cat "$INPUT_FILE" | mempalace hook run --hook "$HOOK_NAME" --harness codex
+EXIT_CODE=$?
+rm -f "$INPUT_FILE" 2>/dev/null
+exit $EXIT_CODE

.codex-plugin/plugin.json

@@ -0,0 +1,48 @@
+{
+  "name": "mempalace",
+  "version": "3.3.4",
+  "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.",
+  "author": {
+    "name": "milla-jovovich"
+  },
+  "homepage": "https://github.com/MemPalace/mempalace",
+  "repository": "https://github.com/MemPalace/mempalace",
+  "license": "MIT",
+  "keywords": [
+    "memory",
+    "ai",
+    "rag",
+    "mcp",
+    "chromadb",
+    "palace",
+    "search"
+  ],
+  "skills": "./skills/",
+  "hooks": "./hooks.json",
+  "mcpServers": {
+    "mempalace": {
+      "command": "mempalace-mcp"
+    }
+  },
+  "interface": {
+    "displayName": "MemPalace",
+    "shortDescription": "AI memory system for Codex",
+    "longDescription": "Give your AI a persistent memory — mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and guided skills.",
+    "developerName": "milla-jovovich",
+    "category": "Coding",
+    "capabilities": [
+      "Interactive",
+      "Read",
+      "Write"
+    ],
+    "websiteURL": "https://github.com/MemPalace/mempalace",
+    "privacyPolicyURL": "https://github.com/MemPalace/mempalace",
+    "termsOfServiceURL": "https://github.com/MemPalace/mempalace",
+    "defaultPrompt": [
+      "Search my memories for recent decisions",
+      "Mine this project into my memory palace",
+      "Show my palace status and room counts"
+    ],
+    "brandColor": "#7C3AED"
+  }
+}

.codex-plugin/skills/help/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: help
+description: Show MemPalace help — available commands, usage tips, and getting started guidance.
+allowed-tools: Bash, Read
+---
+
+# MemPalace Help
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions help
+```

.codex-plugin/skills/init/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: init
+description: Initialize a new MemPalace — guided setup for your AI memory palace with ChromaDB backend.
+allowed-tools: Bash, Read, Write, Edit
+---
+
+# MemPalace Init
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions init
+```

.codex-plugin/skills/mine/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: mine
+description: Mine a project or conversation into your MemPalace — extract and store memories for later retrieval.
+allowed-tools: Bash, Read, Glob, Grep
+---
+
+# MemPalace Mine
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions mine
+```

.codex-plugin/skills/search/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: search
+description: Search your MemPalace — semantic search across all mined memories, projects, and conversations.
+allowed-tools: Bash, Read
+---
+
+# MemPalace Search
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions search
+```

.codex-plugin/skills/status/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: status
+description: Show MemPalace status — room counts, storage usage, and palace health.
+allowed-tools: Bash, Read
+---
+
+# MemPalace Status
+
+Run the following command and follow the returned instructions step by step:
+
+```bash
+mempalace instructions status
+```

.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+{
+  "name": "MemPalace",
+  "image": "mcr.microsoft.com/devcontainers/python:3.11",
+  "features": {
+    "ghcr.io/devcontainers/features/github-cli:1": {}
+  },
+  "postCreateCommand": "bash .devcontainer/post-create.sh",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-python.debugpy",
+        "charliermarsh.ruff"
+      ],
+      "settings": {
+        "python.defaultInterpreterPath": "/usr/local/bin/python",
+        "python.testing.pytestEnabled": true,
+        "python.testing.pytestArgs": ["tests/", "-v", "--ignore=tests/benchmarks"],
+        "ruff.importStrategy": "fromEnvironment",
+        "editor.formatOnSave": true,
+        "editor.defaultFormatter": "charliermarsh.ruff"
+      }
+    }
+  }
+}

.devcontainer/post-create.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "=== MemPalace Dev Container Setup ==="
+
+pip install -e ".[dev]"
+
+# Match CI's ruff pin (pyproject only sets a floor; without this contributors
+# get a newer ruff locally than CI runs, causing phantom lint failures).
+pip install "ruff>=0.4.0,<0.5"
+
+pip install pre-commit
+pre-commit install
+
+echo ""
+echo "=== Verification ==="
+echo "python: $(python --version)"
+echo "pytest: $(python -m pytest --version 2>&1 | head -1)"
+echo "ruff:   $(python -m ruff --version 2>&1 | head -1)"
+echo ""
+echo "Ready. Run: pytest tests/ -v --ignore=tests/benchmarks"

.github/CODEOWNERS

@@ -0,0 +1,13 @@
+# Default owners for everything
+* @milla-jovovich @bensig @igorls
+
+# Core library
+mempalace/ @milla-jovovich @bensig
+
+# CI and workflows
+.github/ @bensig
+
+# Plugins and integrations
+.claude-plugin/ @bensig
+.codex-plugin/ @bensig
+integrations/ @bensig

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 3

.github/workflows/bump-plugin-version.yml.disabled

@@ -0,0 +1,51 @@
+name: Bump Version
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Bump patch version
+        run: |
+          CURRENT=$(python3 -c "exec(open('mempalace/version.py').read()); print(__version__)")
+          IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT"
+          PATCH=$((PATCH + 1))
+          NEW="${MAJOR}.${MINOR}.${PATCH}"
+          echo "__version__ = \"${NEW}\"" > mempalace/version.py
+          # Prepend docstring
+          sed -i '1i"""Single source of truth for the MemPalace package version."""\n' mempalace/version.py
+          echo "version=$NEW" >> "$GITHUB_OUTPUT"
+        id: version
+
+      - name: Sync plugin.json
+        run: |
+          jq --arg v "${{ steps.version.outputs.version }}" '.version = $v' .claude-plugin/plugin.json > tmp.json && mv tmp.json .claude-plugin/plugin.json
+
+      - name: Sync marketplace.json
+        run: |
+          jq --arg v "${{ steps.version.outputs.version }}" '.plugins[0].version = $v' .claude-plugin/marketplace.json > tmp.json && mv tmp.json .claude-plugin/marketplace.json
+
+      - name: Sync codex plugin.json
+        run: |
+          jq --arg v "${{ steps.version.outputs.version }}" '.version = $v' .codex-plugin/plugin.json > tmp.json && mv tmp.json .codex-plugin/plugin.json
+
+      - name: Sync pyproject.toml
+        run: |
+          sed -i "s/^version = \".*\"/version = \"${{ steps.version.outputs.version }}\"/" pyproject.toml
+
+      - name: Commit and push
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add mempalace/version.py .claude-plugin/plugin.json .claude-plugin/marketplace.json .codex-plugin/plugin.json pyproject.toml
+          if ! git diff --staged --quiet; then
+            git commit -m "chore: bump version to ${{ steps.version.outputs.version }}"
+            git push
+          fi

.github/workflows/ci.yml

@@ -2,31 +2,54 @@ name: Tests
 
 on:
   push:
-    branches: [main]
+    branches: [main, develop]
   pull_request:
-    branches: [main]
+    branches: [main, develop]
 
 jobs:
-  test:
+  test-linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.9", "3.11", "3.13"]
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install -r requirements.txt pytest
-      - run: python -m pytest tests/ -v
+          cache: 'pip'
+      - run: pip install -e ".[dev]"
+      - run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10
 
+  test-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+          cache: 'pip'
+      - run: pip install -e ".[dev]"
+      - run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10
+
+  test-macos:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+          cache: 'pip'
+      - run: pip install -e ".[dev]"
+      - run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.11"
-      - run: pip install ruff
+          cache: 'pip'
+      - run: pip install "ruff>=0.4.0,<0.5"
       - run: ruff check .
       - run: ruff format --check .

.github/workflows/deploy-docs.yml

@@ -0,0 +1,66 @@
+name: Deploy Docs
+
+on:
+  push:
+    branches: [develop]
+    paths:
+      - ".github/workflows/deploy-docs.yml"
+      - "website/**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Configure GitHub Pages
+        id: pages
+        uses: actions/configure-pages@v5
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.1.38
+
+      - name: Install dependencies
+        working-directory: website
+        run: bun install --frozen-lockfile
+
+      - name: Build docs
+        working-directory: website
+        env:
+          DOCS_BASE: ${{ steps.pages.outputs.base_path }}
+          DOCS_EDIT_BRANCH: ${{ github.ref_name }}
+        run: bun run docs:build
+
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: website/.vitepress/dist
+
+  deploy:
+    if: github.ref_name == 'develop'
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v5

.github/workflows/version-guard.yml

@@ -0,0 +1,101 @@
+name: Version Guard
+
+on:
+  push:
+    tags: ['v*']
+  pull_request:
+    paths:
+      - 'pyproject.toml'
+      - 'mempalace/version.py'
+      - '.claude-plugin/marketplace.json'
+      - '.claude-plugin/plugin.json'
+      - '.codex-plugin/plugin.json'
+      - '.github/workflows/version-guard.yml'
+
+jobs:
+  check-versions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Extract versions from all sources
+        id: versions
+        run: |
+          set -euo pipefail
+          py_version=$(grep -E '^__version__' mempalace/version.py | cut -d'"' -f2)
+          pyproject_version=$(grep -E '^version' pyproject.toml | head -1 | cut -d'"' -f2)
+          marketplace_version=$(jq -r '.plugins[0].version' .claude-plugin/marketplace.json)
+          plugin_version=$(jq -r '.version' .claude-plugin/plugin.json)
+          codex_version=$(jq -r '.version' .codex-plugin/plugin.json)
+
+          echo "py_version=$py_version" >> "$GITHUB_OUTPUT"
+          echo "pyproject_version=$pyproject_version" >> "$GITHUB_OUTPUT"
+          echo "marketplace_version=$marketplace_version" >> "$GITHUB_OUTPUT"
+          echo "plugin_version=$plugin_version" >> "$GITHUB_OUTPUT"
+          echo "codex_version=$codex_version" >> "$GITHUB_OUTPUT"
+
+          {
+            echo "## Detected versions"
+            echo ""
+            echo "| Source | Version |"
+            echo "| --- | --- |"
+            echo "| mempalace/version.py | \`$py_version\` |"
+            echo "| pyproject.toml | \`$pyproject_version\` |"
+            echo "| .claude-plugin/marketplace.json | \`$marketplace_version\` |"
+            echo "| .claude-plugin/plugin.json | \`$plugin_version\` |"
+            echo "| .codex-plugin/plugin.json | \`$codex_version\` |"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Verify all sources agree
+        env:
+          PY: ${{ steps.versions.outputs.py_version }}
+          PYPROJECT: ${{ steps.versions.outputs.pyproject_version }}
+          MARKETPLACE: ${{ steps.versions.outputs.marketplace_version }}
+          PLUGIN: ${{ steps.versions.outputs.plugin_version }}
+          CODEX: ${{ steps.versions.outputs.codex_version }}
+        run: |
+          set -euo pipefail
+          fail=0
+          check() {
+            local name="$1" value="$2" expected="$3"
+            if [[ "$value" != "$expected" ]]; then
+              echo "::error file=$name::version mismatch — expected $expected, got $value"
+              fail=1
+            fi
+          }
+          # All five must agree with each other (use version.py as the reference, per CLAUDE.md)
+          check "pyproject.toml" "$PYPROJECT" "$PY"
+          check ".claude-plugin/marketplace.json" "$MARKETPLACE" "$PY"
+          check ".claude-plugin/plugin.json" "$PLUGIN" "$PY"
+          check ".codex-plugin/plugin.json" "$CODEX" "$PY"
+          exit $fail
+
+      - name: Verify tag matches manifest (tag pushes only)
+        if: startsWith(github.ref, 'refs/tags/v')
+        env:
+          PY: ${{ steps.versions.outputs.py_version }}
+        run: |
+          set -euo pipefail
+          tag_version="${GITHUB_REF_NAME#v}"
+
+          # Semver pre-release tags (v3.4.0-rc1, v1.0.0-beta.2, ...) are treated
+          # as internal/staging and are not validated against the manifest. They
+          # do not flow to end users via `/plugin update`, which reads the
+          # manifest on the default branch.
+          if [[ "$tag_version" == *-* ]]; then
+            echo "Pre-release tag $GITHUB_REF_NAME — skipping strict manifest match."
+            {
+              echo ""
+              echo "> Pre-release tag detected: \`$GITHUB_REF_NAME\`."
+              echo "> Manifest ($PY) is not required to match. Pre-releases are not published via \`/plugin update\`."
+            } >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          if [[ "$tag_version" != "$PY" ]]; then
+            echo "::error::tag $GITHUB_REF_NAME does not match manifest version $PY"
+            echo "Bump mempalace/version.py, pyproject.toml, and all plugin manifests before tagging a stable release."
+            echo "For an internal/staging tag, use a semver pre-release suffix (e.g. v${PY}-rc1)."
+            exit 1
+          fi
+          echo "Tag $GITHUB_REF_NAME matches manifest version $PY"

.gitignore

@@ -5,3 +5,34 @@ __pycache__/
 *.pyc
 .pytest_cache/
 mempal.yaml
+.a5c/
+.claude/
+.codex/
+.codex
+
+# Environment
+.env
+.env.*
+
+# OS
+.DS_Store
+Thumbs.db
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Coverage
+htmlcov/
+.coverage
+coverage.xml
+
+# Virtual environments
+.venv/
+venv/
+
+# ChromaDB local data
+*.sqlite3-journal

.pre-commit-config.yaml

@@ -1,6 +1,9 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.0
+    # Keep in lock-step with the ruff version pinned in .github/workflows/ci.yml
+    # (>=0.4.0,<0.5). Using a newer rev here produces a different formatter
+    # output than CI and breaks `ruff format --check` in the lint job.
+    rev: v0.4.10
     hooks:
       - id: ruff
         args: [--fix]

AGENTS.md

@@ -0,0 +1 @@
+CLAUDE.md

CHANGELOG.md

@@ -0,0 +1,378 @@
+# Changelog
+
+All notable changes to [MemPalace](https://github.com/MemPalace/mempalace) are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/).
+
+---
+
+## [3.3.4] — 2026-04-30
+
+### Added
+
+- **`mempalace init` now prompts to mine the same directory.** After entity confirmation, room detection, and gitignore guard, `init` shows a one-line scope estimate (e.g. `~423 files (~12 MB) would be mined into this palace.`) computed from its existing corpus walk, then asks `Mine this directory now? [Y/n]` (default yes) and runs `mine()` in-process if accepted. The estimate fires before the prompt so users on a real corpus aren't surprised by a minutes-long ChromaDB write. Declining prints the exact `mempalace mine <dir>` command for later. (#1181)
+- **New `--auto-mine` flag on `mempalace init`** for the non-interactive path (`mempalace init --auto-mine <dir>` skips the mine prompt and runs mine directly). `--yes` retains its existing scope of entity auto-accept only and still prompts for the mine step, so existing scripted callers see no behaviour change; combining `--yes --auto-mine` gives a fully non-interactive setup. (#1181)
+- **Cross-wing topic tunnels.** When two wings have confirmed `TOPIC` labels in common (the LLM-refine bucket from `mempalace init --llm`), the miner now drops a symmetric tunnel between them at mine time so the palace graph reflects shared themes (frameworks, vendors, recurring concepts). Tunnels are routed through the existing `create_tunnel` storage so they share dedup and persistence with explicit tunnels. Topic tunnels are stored under a synthetic `topic:<name>` room and tagged with `kind: "topic"` on the stored dict — this keeps them distinct from literal folder-derived rooms of the same name (a wing with both an `Angular` folder room and an `Angular` topic tunnel no longer collides at `follow_tunnels` read time) and gives LLMs scanning `list_tunnels` a visible discriminator. Threshold is configurable via `MEMPALACE_TOPIC_TUNNEL_MIN_COUNT` env var or `topic_tunnel_min_count` in `~/.mempalace/config.json` (default `1`). Manifest-dependency overlap and per-topic allow/deny lists remain out of scope. (#1180)
+- **Context-aware corpus detection at `mempalace init`.** A new Pass 0 runs at the start of `init` — before entity detection — and answers one question: *is this corpus an AI-dialogue record, and if so, which platform and what persona names has the user assigned to the agents?* Tier 1 is a free regex heuristic (well-known AI brand terms + turn-marker patterns, with a co-occurrence rule that suppresses ambiguous terms like `Claude`/`Gemini`/`Haiku` when no unambiguous AI signal is present, so French novels and astrology forums don't false-positive). Tier 2 is an LLM call (~$0.01 with Anthropic Haiku, free with local Ollama/LM Studio/llama.cpp/vLLM) that extracts `user_name` and `agent_persona_names` from dialogue structure. Result is persisted to `<palace>/.mempalace/origin.json` with a `schema_version: 1` envelope so downstream tools can read it. Entity classification then routes names matching `agent_persona_names` (case-insensitive) into a new `agent_personas` bucket instead of `people`, so a Claude Code transcript no longer misclassifies the user's `Echo`/`Sparrow`/`Cipher` agents as biological people. `llm_refine` receives the same context as a system-prompt preamble so it can disambiguate other ambiguous candidates with corpus-level knowledge too. Backwards compatible: callers that don't pass `corpus_origin` see the v3.3.3 return shape unchanged. (#TBD)
+- **`mempalace init` runs LLM-assisted refinement by default.** v3.3.3 made `--llm` opt-in; the LLM-assisted path is qualitatively better (extracts persona names, refines ambiguous classifications) so it now runs by default. Provider precedence is unchanged — Ollama at `http://localhost:11434` first, then openai-compat, then anthropic with API key. **Never blocks init on a missing LLM**: if no provider is reachable (Ollama not running, no API key set), init prints a one-line message pointing at `--no-llm` and falls through to the heuristic-only path. `--no-llm` is the new explicit opt-out. The legacy `--llm` flag is preserved as a deprecated alias of the default so scripted callers see no behaviour change. Cost story: zero for users with a local LLM (the majority on this repo), ~$0.01 per init for users with `ANTHROPIC_API_KEY` set who explicitly choose `--llm-provider anthropic`, zero for users with no LLM (graceful fallback). (#TBD)
+- **`mempalace mine --redetect-origin` flag.** Re-runs corpus-origin detection on the current corpus state and overwrites `<palace>/.mempalace/origin.json`. Useful when the corpus has grown since `mempalace init` and the stored origin may be stale. Heuristic-only by design (the flag is meant to be cheap); re-run `mempalace init` for full Tier 2 LLM refinement. Default `mempalace mine` does not touch `origin.json` — the flag is opt-in. (#TBD)
+
+### Bug Fixes
+
+- **Cross-wing topic tunnels for hyphenated dir names.** `mempalace init` recorded the `topics_by_wing` registry key under the raw directory name (e.g. `mempalace-public`), while `mempalace.yaml`'s `wing` field used the lower-cased + separator-collapsed slug (`mempalace_public`). At mine time the miner read the slug from the yaml and missed the registry, so `_compute_topic_tunnels_for_wing` returned `0` silently. Real-world: any project whose folder contained a hyphen or space lost every topic tunnel. Producer side: `cmd_init`, `room_detector_local`, `miner.load_config` no-yaml fallback, and `convo_miner` now all route through a shared `normalize_wing_name()` in `config.py` so future writes use the same key. Lookup side: `palace_graph.create_tunnel`, `list_tunnels`, `follow_tunnels`, and `find_tunnels` normalize incoming wing names too, so existing palaces with raw-name keys on disk also recover. (#1194, #1195, #1197, follow-up to #1180)
+- **HNSW index bloat from repeated resize+persist cycles.** ChromaDB's HNSW segment was growing into the tens of GB on palaces past ~15K drawers because `link_lists.bin` was being re-allocated on every flush. Setting `hnsw:batch_size` and `hnsw:sync_threshold` on collection metadata via the new `_HNSW_BLOAT_GUARD` constant pins the segment to one allocation per batch instead. Empirical: a fresh 39,792-drawer palace went from 30 GB on disk and segfaulting `mempalace status` to 376 MB and instant. Migration note — already-bloated palaces still need a `mempalace repair` or full re-mine; HNSW config is honoured at collection-create time only. (#1191, supersedes #346)
+- **`max_seq_id` poisoning from old `_fix_blob_seq_ids` shim.** The 0.6.x → 1.5.x BLOB-to-INTEGER migration was running `int.from_bytes(blob, 'big')` over chromadb 1.5.x's native `b'\x11\x11' + ASCII-digit` `max_seq_id` format, yielding ~1.23e18 integers that silently suppressed every subsequent `embeddings_queue` write for the affected segment. The shim is now narrowed to the `embeddings` table only, with an additional defense-in-depth guard that skips sysdb-10-prefixed BLOBs even there. New `mempalace repair --mode max-seq-id` un-poisons existing palaces either from a pre-corruption sidecar DB (exact restore) or heuristically (`MAX(embeddings.seq_id)` over the owning collection). (#1135)
+- **Auto-ingest hooks now mine the active transcript as `--mode convos`.** Stop and PreCompact hooks were spawning `mempalace mine <transcript-dir>` without `--mode`, defaulting to `projects` — so Claude Code session JSONLs were being ingested as if they were source code via `READABLE_EXTENSIONS`. The hooks now thread the correct mode through every spawn, and `MEMPAL_DIR` (when set) becomes additive rather than overriding the transcript path: a user with `MEMPAL_DIR` pointed at their project still gets the active conversation mined verbatim. Shell hooks also gained the same `_validate_transcript_path` rejection logic the Python entry point already had (extension + `..` traversal). (#1230, #1231)
+- **CLI `mempalace search` retrieval quality.** The CLI was using pure ChromaDB cosine distance with no BM25 rerank, so drawers containing every query term but embedding as noise (directory listings, diff output, shell logs) scored `Match: 0.0` alongside genuinely irrelevant results with no way to tell them apart. Wired the CLI through the same `_hybrid_rank` the `mempalace_search` MCP tool already used, and surfaced both `cosine=` and `bm25=` scores in the output so users see which component of the match is firing. MCP search was unaffected; this fixes the human-facing CLI parity gap.
+- **Legacy-palace distance-metric warning.** CLI search now detects palaces created before `hnsw:space=cosine` was consistently set and prints a one-line notice pointing at `mempalace repair`. Without the warning such palaces silently used L2 distance, under which the similarity display floored every result to `Match: 0.0`. New palaces mined today already set cosine correctly and now have invariant tests pinning that behavior so future refactors can't silently regress it. (#1179)
+- **Graceful Ctrl-C during `mempalace mine`.** Interrupting a long mine no longer dumps a multi-frame `KeyboardInterrupt` traceback. The main file-processing loop now catches the signal, prints `files_processed: N/M`, `drawers_filed: K`, and `last_file:` so the user knows what landed, then exits with code 130 (standard SIGINT). Already-filed drawers are upserted idempotently on re-mine via deterministic IDs, so resuming is safe. The hooks PID lock at `~/.mempalace/hook_state/mine.pid` is now also actively cleaned up in a `finally` when its entry points at us — clean exit, error, or interrupt — preventing the next hook fire from briefly waiting on a stale PID. (#1182)
+- **`mempalace init` is now idempotent across re-runs.** Running `init` twice on the same project produced different `origin.json` results because the first run wrote `entities.json` into the project directory, and the second run's corpus-origin sampling included that file as corpus content — shifting Tier 1's character-density math. Sampling now skips the per-project artifacts (`entities.json`, `mempalace.yaml`), so re-running `init` produces the same classification it did the first time. Pinned by an integration test in `tests/test_corpus_origin_integration.py`. (#TBD)
+- **HNSW divergence floor now scales with `hnsw:sync_threshold`.** The capacity probe added in #1227 hardcoded a 2,000-row floor for the "DIVERGED" decision, sized against chromadb's default `sync_threshold` of 1,000. The bloat-guard fix above (#1191) raised `hnsw:sync_threshold` to 50,000 without updating the divergence floor to track. Result: any new palace past ~100K drawers spent roughly 80% of each write cycle reporting `DIVERGED`, and `mcp_server._refresh_vector_disabled_flag` silently routed vector search to the BM25 fallback even though chromadb was behaving correctly. The floor now reads `hnsw:sync_threshold` from collection metadata and scales to `2 × sync_threshold`, preserving the legacy 2,000 fallback for older palaces that pre-date #1191. (#1287, fixes interaction between #1191 and #1227)
+- **Stop hook no longer crashes ChromaDB on reopen.** `ChromaBackend.get_collection(create=True)` was calling `client.get_or_create_collection` with metadata on every open. In chromadb 1.5.x, when that metadata differs from the stored collection metadata the Rust binding SIGSEGVs with no traceback — the failure mode behind the session-end stop-hook crashes reported in #1089. The call is now split into `get_collection` first, falling back to `create_collection` only when the collection does not yet exist. Existing palaces open without touching their metadata; new ones are created with the full settings as before. The MCP server's `_get_collection(create=True)` path (reached by `tool_add_drawer` and `tool_diary_write`, the latter being what the Stop hook fires at session end) carried the same metadata payload at a parallel call site and got the same try/except split applied, closing the crash class on both reopen paths. (#1089, #1262, #1289)
+- **`repair --mode max-seq-id` heuristic now decodes BLOB-typed `embeddings.seq_id` rows.** The recovery feature added in #1135 was running `int(row[0])` directly on the result of `MAX(e.seq_id)`. On palaces where chromadb 1.5.x has been writing seq_ids natively (8-byte big-endian uint64 BLOB), that raised `ValueError: invalid literal for int() with base 10: b'\x00\x00\x00\x00\x00\x00-\xae'` before the dry-run summary could print, leaving users with no path through the un-poison feature #1135 was specifically designed to provide. `_compute_heuristic_seq_id` now decodes BLOB return values via `int.from_bytes(val, "big")` and keeps the existing `int(val)` path for INTEGER rows. (#1254, #1288, follow-up to #1135)
+
+---
+
+## [3.3.3] — 2026-04-23
+
+### Bug Fixes
+
+- **Install regression** — `mempalace-mcp` console script is now declared in `pyproject.toml` alongside `.claude-plugin/plugin.json`'s reference to it. In v3.3.2 the two drifted apart (plugin.json shipped the new `"command": "mempalace-mcp"` form before the matching entry point landed), so every fresh `pip install mempalace==3.3.2` produced a Claude Code plugin config pointing at a binary that wasn't installed. (#1093, #340)
+- Restore silent-save visibility after the Claude Code 2.1.114 client regression — production transcript saves were failing silently until this PR. (#1021)
+- Paginate `status`-path metadata fetches so large palaces don't trip SQLite variable limits. (#851)
+- Resolve the Claude plugin hook runner across platform / plugin-dir variations; previously broke on Windows and some macOS layouts. (#942)
+- Real `python3` resolution for `.sh` hooks with a `MEMPAL_PYTHON` override path. (#833)
+- Add optional `wing` parameter to `tool_diary_write` / `tool_diary_read` and derive per-project wing from the Claude Code transcript path when writing from the stop hook — diary entries from different projects no longer collapse into a shared default wing. (#659)
+- Treat empty string as "no filter" in `mempalace_search` `wing`/`room`; LLM agents that default to filling every optional parameter with `""` no longer get bounced with `must be a non-empty string`. (#1097, #1084)
+- Broaden `_wing_from_transcript_path` to handle Claude Code project folders without a `-Projects-` segment (e.g. `~/dev/<parent>/<project>`, `~/code/<project>`). The project name is now derived from the final dash-separated token of the encoded folder, so Linux users with code outside `~/Projects/` get per-project diary scoping instead of falling through to `wing_sessions`. (#1145, follow-up to #659)
+- `mempalace_diary_read(wing="")` now returns diary entries from every wing this agent has written to, matching the #1097 "empty-string as no filter" pattern. Previously defaulted to `wing_<agent>`, siloing entries that hooks wrote to project-derived wings. (#1145)
+- `mempalace mine` now skips the generated `entities.json` file so its contents aren't re-ingested as project content. (#1175)
+
+### Improvements
+
+- **Deterministic hook saves.** Save hook now uses a silent Python API path, so successive hook invocations produce reproducible results and zero data loss on the hot path. (#673)
+- **Graph cache with write-invalidation** inside `build_graph()` — warm-path calls no longer rebuild the palace-graph per request. (#661)
+- **`mempalace init` entity detection overhaul.** Canonical project names now come from package manifests (`package.json`, `pyproject.toml`, `Cargo.toml`, `go.mod`) and real people come from git commit authors, rather than being inferred from prose. Includes union-find dedup across name/email aliases, bot filtering that keeps `@users.noreply.github.com` humans, and automatic "mine" flagging by contribution share. (#1148)
+- **Regex detector accuracy.** CamelCase extraction so `MemPalace`, `ChromaDB`, `OpenAI` aren't fragmented; tighter versioned/hyphenated pattern kills `context-manager` / `multi-word` false positives; dialogue `^NAME:\s` requires ≥2 hits so `Created: <date>` metadata stops classifying field names as people; expanded stopwords for common English participles and descriptors; high-pronoun signal classifies as person rather than dumping to uncertain. (#1148)
+- **Init → miner wire-up.** Confirmed entities merge into `~/.mempalace/known_entities.json` on init, which the miner reads to tag drawer metadata for entity-filtered search. Previously init's output was not consumed by the miner; the per-project `entities.json` is kept as an audit trail. (#1157)
+- **Case-insensitive project dedup** across manifest, git, and convo sources so casing variants of the same project name collapse into one review entry. (#1175)
+
+### Added
+
+- i18n: Belarusian translation. (#1051)
+- i18n: entity detection for German, Spanish, and French locales. (#1001)
+- i18n: Traditional + Simplified Chinese entity detection. (#945)
+- **`mempalace init --llm`**: optional LLM-assisted entity classification. Defaults to local Ollama (zero-API); also supports any OpenAI-compatible endpoint (LM Studio, llama.cpp server, vLLM, OpenRouter, etc.) and the Anthropic Messages API. Runs interactively with a progress indicator; Ctrl-C cancels cleanly and returns partial results. Useful for prose-heavy folders where the regex detector struggles (diaries, transcripts, research notes). Opt-in only — default init path remains zero-API. (#1150)
+- **Claude Code conversation scanner.** `~/.claude/projects/<slug>/` directories now contribute project entities using each session's authoritative `cwd` metadata, avoiding slug-decoding ambiguity. (#1150)
+
+### Known — deferred to v3.3.4
+
+- HNSW parallel-insert SIGSEGV when `hnsw:num_threads` is unset on collection creation (#974) — fix in-flight as #976, awaiting rebase against develop.
+
+---
+
+## [3.3.2] — 2026-04-19
+
+### Bug Fixes
+
+- Fix silent drop of `.jsonl` files in project miner; raise `MAX_FILE_SIZE` cap from 10 MB to 500 MB so large transcripts no longer fall through unnoticed. Adds a tandem **sweeper** — a message-level, timestamp-coordinated, idempotent safety net that catches anything the primary miner missed. (#998)
+- `mempalace sweep <target>` CLI to run the sweeper on demand against a transcript file or a directory. (#998)
+- Guard `Layer3.search_raw` against `None` doc/meta rows returned by ChromaDB — prevents `AttributeError` crashes on mixed-schema palaces. (#1011, #1013)
+- Guard searcher API path, closet loop, and miner status histogram against `None` metadata; matching guards added to `tool_status` / `list_wings` / `list_rooms` / `get_taxonomy` in the MCP server. (#999)
+- Upgrade `chromadb` floor to `>=1.5.4` for Python 3.13 / 3.14 compatibility and pin upper bound to `<2` so future breaking majors don't silently install. (#1010)
+- Fix Unicode checkmark rendering on Windows terminals that can't encode the `✓` glyph — avoids `UnicodeEncodeError` crashes on first-run output. (#681)
+- **`quarantine_stale_hnsw`** — on open, detect HNSW segment directories whose `data_level0.bin` is significantly older than `chroma.sqlite3` and rename them out of the way. Recovers cleanly from HNSW/sqlite drift that otherwise causes SIGSEGV on `count()` / `query(...)` (the chroma-core/chroma#2594 failure mode). Rebuilds the index lazily on next use. (#1000)
+- **PID file guard** — `mine` writes a per-source-directory PID file and refuses to start if an existing mine is still running, preventing process stacking that bloats HNSW and wedges concurrent writes. Includes cross-platform PID liveness check (`os.kill(pid, 0)` terminates on Windows, so the guard falls back to a platform-aware probe). (#1023)
+
+### Improvements
+
+- **RFC 001 §10 — typed backend contracts.** `BaseBackend` now returns typed `QueryResult` / `GetResult` dataclasses and `PalaceRef` for palace identity; registry-based backend discovery. Internal refactor; no user-facing API change. (#995)
+- **RFC 002 §9 — source adapter scaffolding.** Introduces `BaseSourceAdapter`, adapter registry, and `PalaceContext` — the plumbing that future pluggable ingest sources will target. Internal refactor; no user-facing API change yet. (#1014)
+
+### Documentation
+
+- **RFC 002** — full specification for the source adapter plugin system (future pluggable ingest). (#990)
+- First-run help text and `README` now reference the real `~/.claude/projects/<project>/` path shape instead of the placeholder `/path/to/transcripts`. (#996, #1012)
+
+### Internal
+
+- Harden sweeper for production: verbatim tool blocks, full `session_id`, logged failures.
+- Address Copilot review on #995: cursor tie-break, honest metrics, accurate comments.
+- Test hygiene: avoid ONNX network download in update-length validation tests; dedup update-length-validation tests; fix Windows file-lock in cache-invalidation test.
+
+---
+
+## [3.3.1] — 2026-04-16
+
+### New Features
+
+**Multi-language entity detection** — lexical patterns (person verbs, pronouns, dialogue markers, project verbs, stopwords, candidate character classes) now live in the optional `entity` section of each locale JSON under `mempalace/i18n/<lang>.json`. Every public function in `entity_detector` accepts a `languages=` tuple and unions patterns across enabled locales. Default stays `("en",)` so existing English-only callers are unchanged. (#911)
+
+- **Five new fully-supported locales** with CLI strings, AAAK compression instructions, and entity-detection patterns:
+  - Brazilian Portuguese `pt-br` (#156)
+  - Russian `ru` (#760)
+  - Italian `it` (#907)
+  - Hindi `hi` (#773)
+  - Indonesian `id` (#778)
+- **`MempalaceConfig.entity_languages`** — persistent palace-level language selection; `MEMPALACE_ENTITY_LANGUAGES` env override; `mempalace init --lang en,pt-br` flag that saves to `~/.mempalace/config.json` (#911)
+- **Per-language `candidate_pattern`** — non-Latin scripts register their own character class, so names like `João`, `Инна`, `राज` are no longer silently dropped by the ASCII-only default (#911)
+- **VSCode devcontainer** matching the CI environment (#881)
+- `MEMPAL_VERBOSE` env toggle — developers see diaries surfaced in chat while the default remains silent (#871)
+- `created_at` timestamps included in search results (#846)
+
+### Bug Fixes
+
+**i18n / Unicode**
+
+- Script-aware word boundaries for combining-mark scripts — Python's `\b` fails on Devanagari vowel signs (`ा ी ु`), Arabic, Hebrew, Thai, Tamil, Khmer etc., truncating names like `अनीता` → `अनीत` and making person-verb patterns never fire. Locales now declare an optional `boundary_chars` field and the i18n loader expands `\b` into a script-aware lookaround boundary (#932)
+- Case-insensitive BCP 47 language code resolution — `--lang PT-BR`, `zh-cn`, `Pt-Br` previously fell through to English silently; now resolve to the canonical locale file via lowercase matching, with the entity-pattern cache keyed on the canonical form so casing variations share one cache entry (#928)
+- Wire i18n candidate patterns into `miner._extract_entities_for_metadata()`, `palace.build_closet_lines()`, and `entity_registry.extract_unknown_candidates()` — three code paths that still hardcoded ASCII-only `[A-Z][a-z]{2,}` and silently missed Cyrillic, accented Latin, and non-Latin entity metadata tags (#931)
+- Explicit `encoding="utf-8"` on `Path.read_text()` calls across entity_registry, instructions_cli, split_mega_files, and onboarding tests — prevents Windows GBK (and other non-UTF-8) locales from corrupting UTF-8 files (#946, #776)
+- `ko.json` `status_drawers` used `{drawers}` instead of `{count}`, showing the raw template string instead of the number (#758)
+- Move `test_i18n.py` from inside the installed package into `tests/` so pytest actually collects it; remove the `sys.path.insert` hack (#758)
+- `Dialect.from_config()` defaulted to `current_lang()` (module-global) when config had no `lang` key — replaced with explicit `"en"` fallback for determinism (#758)
+
+**Other**
+
+- Guard `KnowledgeGraph.close()` and `query_relationship`/`timeline`/`stats` methods with the instance lock to prevent concurrent-access corruption (#887, #884)
+- Replace invalid `{"decision": "allow"}` with `{}` in hook responses — the string wasn't a valid decision value and triggered schema warnings (#885)
+- `entity_registry.research()` defaults to local-only — previously made outbound Wikipedia HTTPS requests without explicit user opt-in; callers now must pass `allow_network=True` (#811)
+- Precompact hook no longer blocks compaction when it fails or takes too long (#856, #858, #863)
+- Redirect stdout to stderr during MCP server import so library logging can't corrupt the JSON-RPC channel (#225, #864)
+- `mempalace init` auto-adds per-project files to `.gitignore` in git repositories so users don't accidentally commit `mempalace.yaml` / `entities.json` (#185, #866)
+- Searcher guards against empty ChromaDB query results that previously raised on edge-case corpora (#195, #865)
+- Return empty status instead of an error on a cold-start palace with no drawers yet (#830, #831)
+- Restrict file permissions on sensitive palace data (#814)
+- Slack transcript importer writes a provenance header and preserves speaker IDs (#815)
+- Allow `mempalace mine` to run in directories without a local `mempalace.yaml` and surface the missing-yaml warning on stderr (#604)
+- Security hook injection fix (#812)
+- Save hook auto-mines transcripts even when `MEMPAL_DIR` is unset (#840)
+- Pin the Pages custom domain via a shipped `CNAME` in the deploy artifact (#877)
+- Version drift safeguard — sync pyproject + `version.py` + README badge in one place (#876)
+- Deploy docs workflow now runs on `develop` only, preventing accidental main-branch deploys (#845)
+
+### Improvements
+
+- Regex compilation optimization for entity extraction — pre-compile per-entity pattern sets once and cache by `(name, languages)` tuple, so multi-language callers don't thrash the cache (#880)
+- Knowledge-graph value sanitization now preserves natural punctuation (commas, colons, parentheses) that commonly appears in KG subject/object values (#873)
+
+### Documentation
+
+- Clarify that `mempalace init` requires a `<dir>` argument in CLI help text (#210, #862)
+- Domain name and specific impostor sites called out in the scam-alert section (#869)
+- Tightened `SECURITY.md` with a real version-support policy and the GHPVR-only reporting channel (#810)
+- Fixed stale `pyproject.toml` URLs (#853)
+- v4 planning prep (#852)
+
+### Internal
+
+- `palace_graph` tunnel helper test coverage (#908)
+
+---
+
+## [3.3.0] — 2026-04-13
+
+### New Features
+- Closet layer — a compact searchable index of pointers to verbatim drawers, enabling fast topical lookup without reading all content (#788)
+- BM25 hybrid search — closets boost ranking, drawers remain the source of truth (#795, #829)
+- Entity metadata on every drawer for filterable search (#829)
+- Diary ingest — day-based rooms for conversation transcripts (#829)
+- Cross-wing tunnels — explicit links between rooms in different wings for multi-project agents (#829)
+- Drawer-grep — returns the best-matching chunk plus adjacent context drawers (#829)
+- Offline fact checker against the entity registry and knowledge graph (#829)
+- LLM-based closet regeneration — optional, bring-your-own endpoint, no mandatory API key (#793)
+- Hall detection — routes drawer content to `emotions` / `technical` / `family` / `memory` / `identity` / `consciousness` / `creative` halls, enabling hall-based graph connectivity within wings (#835)
+
+### Bug Fixes
+- Set `hnsw:space=cosine` metadata on all collection creation sites — fixes broken similarity scoring under ChromaDB's default L2 distance (#807, #218)
+- File-level locking prevents duplicate drawers when agents mine the same file concurrently (#784, #826)
+- Hybrid closet+drawer retrieval — closets boost ranking, never gate results (#795)
+- Stop hooks from making agents write in chat — saves tokens on every turn (#786)
+- Strip system tags, hook output, and Claude UI chrome from drawers before filing (#785)
+- Verbatim-safe `strip_noise` scoped to Claude Code JSONL only (#785)
+- Prevent diary entry ID collisions via microsecond timestamp and full content hash (#819)
+- Auto-rebuild stale drawers via `NORMALIZE_VERSION` schema gate
+- Enforce atomic topics in closets and extract richer pointers
+- Sync `version.py` to match `pyproject.toml` (#820)
+- Remove unused `main` import from `mempalace/__init__.py` (#827)
+- README audit — fix 7 stale claims (tool count, version badge, wake-up token cost, `dialect.py` lossless disclaimer, `pyproject.toml` version) with 42 regression-guard tests (#835)
+
+### Improvements
+- Optimize entity detection with regex caching and pre-compilation (#828)
+- Extract locked filing block into helper to keep `mine_convos` under C901 complexity
+
+### Documentation
+- Add `docs/CLOSETS.md` — closet layer overview
+- Fix stale `milla-jovovich/*` org URLs in website and plugin manifests (#787)
+- Fix remaining stale org URLs in contributor docs (#808)
+- Rewrite `README.md` and `mempalaceofficial.com` benchmark pages to remove category-error cross-system comparisons (R@5 retrieval recall had been listed next to competitor QA accuracy under one column), remove the retracted "+34% palace boost" claim from the surfaces where it had remained, replace the `100%` Haiku-rerank headline with the honest held-out `98.4%` R@5, drop the LoCoMo `100%` top-50 row (retrieval-bypass artefact), and fix the broken `aya-thekeeper/mempal` reproduction URL (#875)
+- Add `docs/HISTORY.md` as the canonical home for corrections, retractions, and public notices; move the 2026-04-07 "Note from Milla & Ben" and the 2026-04-11 impostor-domain notice out of `README.md`
+- Add v3.3.0 reproduction result JSONLs and the deterministic `seed=42` 50/450 LongMemEval split under `benchmarks/` — every BENCHMARKS.md claim reproduces exactly
+
+### Internal
+- Add test coverage for `mine_lock`, closets, entity metadata, BM25, and diary
+- Verify `mine_lock` via disjoint critical-section intervals
+- Serialize `mine_lock` concurrency test with multiprocessing
+- Make diary state path assertion platform-neutral
+- Add `TestTunnels` coverage for cross-wing tunnel operations
+- Ruff format with CI-pinned version (0.4.x); format `mempalace/palace.py`
+
+---
+
+## [3.2.0] — 2026-04-12
+
+### Packaging
+- Remove `chromadb<0.7` upper bound — unblocks installs against chromadb 1.x palaces (#690)
+- Bump version to 3.2.0 across `pyproject.toml`, `mempalace/version.py`, README badge, and OpenClaw SKILL (#761)
+
+### Security
+- Harden palace deletion, WAL redaction, and MCP search input handling (#739)
+- Consistent input validation, argument whitelisting, concurrency safety, and WAL fixes (#647)
+- Remove hardcoded credential paths from benchmark runners (#177)
+- Remove global SSL verification bypass in convomem_bench (#176)
+
+### Bug Fixes
+- Parse Claude.ai privacy export with `messages` key and sender field (#685, #677)
+- Detect mtime changes in `_get_client` to prevent stale HNSW index (#757)
+- Hash full content in `tool_add_drawer` drawer ID — stable re-mines (#716)
+- Remove 10k drawer cap from status display (#707, #603)
+- Correct typo in entity_detector interactive classification prompt (#755)
+- Prevent convo_miner from re-processing 0-chunk files on every run (#732, #654)
+- Remove silent 8-line AI response truncation in convo_miner (#708, #692)
+- Store full AI response in convo_miner exchange chunking (#695)
+- Fix `mine --dry-run` TypeError on files with room=None (#687, #586)
+- Skip arg whitelist for handlers accepting `**kwargs` (#684, #572)
+- Allow Unicode in `sanitize_name()` — Latvian, CJK, Cyrillic (#683, #637)
+- Auto-repair BLOB seq_ids from chromadb 0.6→1.5 migration (#664)
+- Remove no-op `ORT_DISABLE_COREML` env var (#653, #397)
+- Disambiguate hook block reasons to name MemPalace explicitly (#666)
+- Use epsilon comparison for mtime to prevent unnecessary re-mining (#610)
+- Correct token count estimate in compress summary (#609)
+- Implement MCP ping health checks (#600)
+- Align `cmd_compress` dict keys with `compression_stats()` return values (#569)
+- Skip unreachable reparse points in `detect_rooms_from_folders` on Windows (#558)
+- Prevent HNSW index bloat from duplicate `add()` calls (#544, #525)
+- Purge stale drawers before re-mine to avoid hnswlib segfault (#544)
+- Mitigate system prompt contamination in search queries (#385, #333)
+- Count Codex `user_message` turns in `_count_human_messages` (#373, #347)
+- Paginate large collection reads and surface errors in MCP tools (#371, #339, #338)
+- Expand `~` in split command directory argument (#361)
+- Ignore `wait_for_previous` argument to support Gemini MCP clients (#322)
+- Close KnowledgeGraph SQLite connections in test fixtures (#450)
+- Remove duplicate cache variable declarations in mcp_server.py (#449)
+- Add `--yes` flag to init instructions for non-interactive use (#682, #534)
+- Add `mcp` command with setup guidance (#315)
+
+### New Features
+- i18n support — 8 languages (en, es, fr, de, ja, ko, zh-CN, zh-TW) (#718)
+- New MCP tools: get/list/update drawer, hook settings, export (#667, #635)
+- `mempalace migrate` — recover palaces from different ChromaDB versions (#502)
+- Add OpenClaw/ClawHub skill (#491)
+- Backend seam for pluggable storage backends (#413)
+
+### Improvements
+- Disable broken auto-bump workflow (#414)
+- Improve agent readiness — AGENTS.md, dependabot, CODEOWNERS, labels (#497)
+
+### Documentation
+- Add CLAUDE.md and mission/principles to AGENTS.md (#720)
+- Add VitePress documentation site (#439)
+- Add warning about fake MemPalace websites (#598)
+- Fix stale org URLs and PR branch target in contributor docs (#679)
+- Fix misaligned architecture diagram (#734, #733)
+- Add ROADMAP.md — v3.1.1 stability patch and v4.0.0-alpha plan
+
+### Internal
+- ruff format convo_miner.py (#741)
+- ruff format all Python files (#675)
+- CI: trigger tests on develop branch PRs and pushes (#674)
+- CI: fix GitHub Pages publishing (#691)
+
+---
+
+## [3.1.0] — 2026-04-09
+
+### Security
+- Harden inputs, fix shell injection, optimize DB access (#387)
+- Sanitize SESSION_ID in save hook to prevent path traversal (#141)
+- Sanitize error responses and remove `sys.exit` from library code (#139)
+- Shell injection fix in hooks, Claude Code mining, chromadb pin (#114)
+
+### Bug Fixes
+- MCP null args hang, repair infinite recursion, OOM on large files (#399)
+- Release ChromaDB handles before rmtree on Windows (#392)
+- Use `os.utime` in mtime test for Windows compatibility (#392)
+- Negotiate MCP protocol version instead of hardcoding (#324)
+- Use upsert and deterministic IDs to prevent data stagnation (#140)
+- Make `drawer_id` deterministic for idempotent writes (#387)
+- Honest AAAK stats — word-based token estimator, lossy labels (#147)
+- Room detection checks keywords against folder paths (#145)
+- Use actual detected room in mine summary stats (#165)
+- Honour `--palace` flag in mcp_server (#264)
+- Preserve default KG path when `--palace` not passed (#270)
+- `--yes` flag skips all interactive prompts in init (#123)
+- Repair command, split args, Claude export, room keywords (#119)
+- Replace Unicode separator in convo_miner.py for Windows compatibility (#129)
+- Coerce MCP integer arguments to native Python int (#84)
+- Batch ChromaDB reads to avoid SQLite variable limit (#66)
+- Respect nested .gitignore rules during mining (#78)
+- Narrow bare `except Exception` to specific types where safe (#54)
+- Mark MD5 as non-security in miner drawer ID generation (#53)
+- Remove dead code and duplicate set items in entity_registry.py (#42)
+- Silence ChromaDB telemetry warnings and CoreML segfault on Apple Silicon (#236)
+- Unify package and MCP version reporting (#16)
+- Fix broken AAAK Dialect link in README (#238)
+- Update input prompt for entity confirmation (#83)
+- Preserve CLI exit codes, log tracebacks, sanitize search errors (#139)
+- Enable SQLite WAL mode and add consistent LIMIT to KG timeline (#136)
+- Add limit=10000 safety cap to all unbounded ChromaDB `.get()` calls (#137)
+- Re-mine modified files, idempotent `add_drawer`, cleanup ChromaDB handles (#140)
+- Resolve formatting, regression logic, and pytest defaults (#270)
+- Use `parse_known_args` to allow importing mcp_server during pytest (#270)
+
+### New Features
+- Package MemPalace as standard Claude and Codex plugins (#270)
+- Add OpenAI Codex CLI JSONL normalizer (#61)
+- Add Codex plugin support with hooks, commands, and documentation (#270)
+- Add command documentation for help, init, mine, search, and status (#270)
+
+### Improvements
+- Cache ChromaDB `PersistentClient` instead of re-instantiating per call (#135)
+- Tighten chromadb version range and add `py.typed` marker (#142)
+- Consolidate split known-names config loading (#22)
+- CI: add separate jobs for Windows and macOS testing
+- CI: Upgrade GitHub Actions for Node 24 compatibility (#55)
+
+### Documentation
+- Add Gemini CLI setup guide and integration section (#106)
+- Add beginner-friendly hooks tutorial (#103)
+- Align MCP setup examples with shipped server (#21)
+- Honest README update — own the mistakes, fix the claims
+
+### Internal
+- Expand test coverage from 20 to 92 tests, migrate to uv (#131)
+- Add scale benchmark suite — 106 tests (#223)
+- Increase test coverage from 30% to 85%, fix Windows encoding bugs (#281)
+- Add WAL mode and entity timeline limit assertions
+- Add coverage for `file_already_mined` mtime check
+
+---
+
+## [3.0.0] — 2026-04-06
+
+Initial public release.
+
+- Palace architecture with day-based rooms, drawers (verbatim), and closets (searchable index)
+- AAAK compression dialect for memory folding
+- Knowledge graph with entity detection and timeline queries
+- MCP server for Claude, Codex, and Gemini integration
+- CLI: `init`, `mine`, `search`, `status`, `compress`, `repair`, `split`
+- Benchmark suite with recall and scale tests
+- README with MCP flow, local model flow, and specialist agent documentation
+
+---
+
+[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.2.0...HEAD
+[3.2.0]: https://github.com/MemPalace/mempalace/compare/v3.1.0...v3.2.0
+[3.1.0]: https://github.com/MemPalace/mempalace/compare/v3.0.0...v3.1.0
+[3.0.0]: https://github.com/MemPalace/mempalace/releases/tag/v3.0.0

CLAUDE.md

@@ -0,0 +1,133 @@
+# CLAUDE.md
+
+## The Mission
+
+Memory is identity. When an AI forgets everything between conversations, it cannot build real understanding — of you, your work, your people, your life.
+
+MemPalace exists to solve this. It is a memory system — not a search engine, not a RAG pipeline, not a vector database wrapper. It treats every word you have shared as sacred, stores it verbatim, and makes it instantly available. Your data never leaves your machine. We never summarize. We never paraphrase. We return your exact words.
+
+100% recall is the design requirement — the target every search path is measured against. Anything less means forgetting, and forgetting means starting over.
+
+The name comes from the ancient "method of loci" — the memory palace technique used for thousands of years to organize and recall vast amounts of information by placing it in imagined rooms of an imagined building. We were also inspired by the Zettelkasten method (created by German sociologist Niklas Luhmann) — small cross-referenced index cards that point to each other. We apply both ideas to AI memory:
+
+- **Wings** for broad categories (people, projects, topics)
+- **Rooms** for time-based groupings (days, sessions)
+- **Drawers** for full verbatim content (your exact words)
+- **AAAK compression** for the index layer — a compact symbolic format (via `dialect.py`) that lets an LLM scan thousands of entries instantly and know exactly which drawer to open
+
+## Design Principles
+
+These are non-negotiable. Every PR, every feature, every refactor must honor them.
+
+- **Verbatim always** — Never summarize, paraphrase, or lossy-compress user data. The system searches the index and returns the original words. If a user said it, we store exactly what they said. This is the foundational promise.
+- **Incremental only** — Append-only ingest after initial build. Never destroy existing data to rebuild. A crash mid-operation must leave the existing palace untouched.
+- **Entity-first** — Everything is keyed by real names with disambiguation by DOB, ID, or context. People matter more than topics.
+- **Local-first, zero external API by default** — All extraction, chunking, embedding, and LLM-assisted refinement happens on the user's machine by default, using locally-hosted runtimes (Ollama, LM Studio, llama.cpp, vLLM, unsloth studio, etc.). External providers (Anthropic, OpenAI, Google) are supported via BYOK but are never required and never enabled silently. The system never sends user content to a service the user has not explicitly configured. "Local LLM" is not an external API — Ollama and equivalents running on localhost are part of the user's machine. External BYOK is always a deliberate user choice, never a default and never a silent fallback.
+- **Performance budgets** — Hooks under 500ms. Startup injection under 100ms. Memory should feel instant.
+- **Privacy by architecture** — The system physically cannot send your data because it never leaves your machine. No telemetry, no phone-home, no external service dependencies for core operations.
+- **Background everything** — Filing, indexing, timestamps, and pipeline work happen via hooks in the background. Nothing interrupts the user's conversation. Zero tokens spent on bookkeeping in the chat window.
+
+## Contributing
+
+We welcome bug fixes, performance improvements, new language support, better entity disambiguation, documentation, and test coverage.
+
+We do not accept summarization of user content, cloud storage/sync features, telemetry or analytics, features requiring API keys for core memory, or shortcuts that bypass verbatim storage.
+
+## Setup
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Commands
+
+```bash
+# Run tests
+python -m pytest tests/ -v --ignore=tests/benchmarks
+
+# Run tests with coverage
+python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing
+
+# Lint
+ruff check .
+
+# Format
+ruff format .
+
+# Format check (CI mode)
+ruff format --check .
+```
+
+## Project Structure
+
+```
+mempalace/
+├── mcp_server.py        # MCP server — all read/write tools
+├── cli.py               # CLI dispatcher
+├── config.py            # Configuration + input validation
+├── miner.py             # Project file miner
+├── convo_miner.py       # Conversation transcript miner
+├── searcher.py          # Semantic search (hybrid BM25 + vector)
+├── knowledge_graph.py   # Temporal entity-relationship graph (SQLite)
+├── palace.py            # Shared palace operations
+├── palace_graph.py      # Room traversal + cross-wing tunnels
+├── backends/            # Pluggable storage backends (ChromaDB default)
+│   ├── base.py          # Abstract interface — implement this for new backends
+│   └── chroma.py        # ChromaDB implementation
+├── dialect.py           # AAAK compression dialect
+├── normalize.py         # Transcript format detection + normalization
+├── entity_detector.py   # Auto-detect people/projects from content
+├── entity_registry.py   # Entity storage and disambiguation
+├── layers.py            # L0-L3 memory wake-up stack
+├── onboarding.py        # Interactive first-run setup
+├── repair.py            # Palace repair and consistency checks
+├── dedup.py             # Deduplication
+├── migrate.py           # ChromaDB version migration
+├── spellcheck.py        # Auto-correct user messages
+├── exporter.py          # Palace data export
+├── hooks_cli.py         # Hook management CLI
+├── query_sanitizer.py   # Prompt contamination prevention
+├── split_mega_files.py  # Split concatenated transcript files
+└── version.py           # Single source of truth for version
+
+hooks/                   # Claude Code hook scripts
+├── mempal_save_hook.sh        # Stop: triggers diary save
+└── mempal_precompact_hook.sh  # PreCompact: saves state before compression
+```
+
+## Conventions
+
+- **Python style**: snake_case for functions/variables, PascalCase for classes
+- **Linter**: ruff with E/F/W rules
+- **Formatter**: ruff format, double quotes
+- **Commits**: conventional commits (`fix:`, `feat:`, `test:`, `docs:`, `ci:`)
+- **Tests**: `tests/test_*.py`, fixtures in `tests/conftest.py`
+- **Coverage**: 85% threshold (80% on Windows due to ChromaDB file lock cleanup)
+
+## Architecture
+
+```
+User → CLI / MCP Server → Storage Backend (ChromaDB default, pluggable)
+                        → SQLite (knowledge graph)
+
+Palace structure:
+  WING (person/project)
+    └── ROOM (day/topic)
+          └── DRAWER (verbatim text chunk)
+
+Index layer (AAAK):
+  Compressed pointers → DRAWER locations
+  Scanned by LLM to find relevant drawers without reading all content
+
+Knowledge Graph:
+  ENTITY → PREDICATE → ENTITY (with valid_from / valid_to dates)
+```
+
+## Key Files for Common Tasks
+
+- **Adding an MCP tool**: `mempalace/mcp_server.py` — add handler function + TOOLS dict entry
+- **Changing search**: `mempalace/searcher.py`
+- **Modifying mining**: `mempalace/miner.py` (project files) or `mempalace/convo_miner.py` (transcripts)
+- **Adding a storage backend**: subclass `mempalace/backends/base.py`, register in `backends/__init__.py`
+- **Input validation**: `mempalace/config.py` — `sanitize_name()` / `sanitize_content()`
+- **Tests**: mirror source structure in `tests/test_<module>.py`

CONTRIBUTING.md

@@ -5,8 +5,11 @@ Thanks for wanting to help. MemPalace is open source and we welcome contribution
 ## Getting Started
 
 ```bash
-git clone https://github.com/milla-jovovich/mempalace.git
+# Fork the repo on GitHub first, then clone your fork
+git clone https://github.com/<your-username>/mempalace.git
 cd mempalace
+git remote add upstream https://github.com/MemPalace/mempalace.git
+
 pip install -e ".[dev]"    # installs with dev dependencies (pytest, build, twine)
 ```
 
@@ -52,7 +55,7 @@ assets/             ← logo + brand
    - `fix: handle empty transcript files`
    - `docs: update MCP tool descriptions`
    - `bench: add LoCoMo turn-level metrics`
-6. Push to your fork and open a PR against `main`
+6. Push to your fork and open a PR against `develop`
 
 ## Code Style
 
@@ -64,7 +67,7 @@ assets/             ← logo + brand
 
 ## Good First Issues
 
-Check the [Issues](https://github.com/milla-jovovich/mempalace/issues) tab. Great starting points:
+Check the [Issues](https://github.com/MemPalace/mempalace/issues) tab. Great starting points:
 
 - **New chat formats**: Add import support for Cursor, Copilot, or other AI tool exports
 - **Room detection**: Improve pattern matching in `room_detector_local.py`
@@ -79,7 +82,7 @@ If you're planning a significant change, open an issue first to discuss the appr
 - **Verbatim first**: Never summarize user content. Store exact words.
 - **Local first**: Everything runs on the user's machine. No cloud dependencies.
 - **Zero API by default**: Core features must work without any API key.
-- **Palace structure matters**: Wings, halls, and rooms aren't cosmetic — they drive a 34% retrieval improvement. Respect the hierarchy.
+- **Palace structure is scoping, not magic**: Wings, halls, and rooms act as metadata filters in the underlying vector store. They keep retrieval predictable when a palace holds many unrelated projects or people. Respect the hierarchy — but don't present it as a novel retrieval mechanism.
 
 ## Community
 

MISSION.md

@@ -0,0 +1,34 @@
+MemPalace: The Mission
+
+By: Milla Jovovich
+
+Hey everyone! First of all thank you all for embracing MemPalace and trying it, catching bugs and issues and finding cool ways to personalize it into your workflows!
+
+A few things I want to say.
+MemPalace is something I really needed because I'm trying to work on a big project with my partner @bensig and I was having a lot of problems with Claude's context window and my agent Lumi (Lu for short) kept waking up like "hey what are we doing today" when I had literally done hours of work with him throughout the day and it was impossible to just keep saving every transcript to catch him up on whatever we had done before compaction hit.
+
+That's when I started researching different memory systems available today. I tried most of them and what I found was that no matter which one I tried, they felt like large empty warehouses where you just dump huge amounts of info.
+
+RAG search would take forever and most of the time not find what I wanted.
+
+I wanted to create a system with the ability to really remember everything AND be able to find it quickly, easily and also be able to remember things when I didn't. THAT in itself felt like something so important. Like "remember when we talked about that idea…" but in vague terms. Impossible with regular keyword search tools.
+
+So MemPalace is not just about storing info in a highly structured way. But also RETRIEVING it in a highly UNSTRUCTURED way lol!
+
+I was inspired by the Zettelkasten method (created by German sociologist Niklas Luhmann) — his idea of small cross-referenced index cards that point to each other. That's the architecture behind the palace: wings, rooms, closets, and drawers, all connected so you can find things from any angle, not just the one you filed them under.
+
+Because of the way I've designed my agent Lumi to understand me, after so many months of my own personal experiments with MemPalace and the incredible help of my dear friend and co-founder, developer and engineer @bensig, he built a back end that made it really easy to get all my files in the proper spaces the Palace created based on my own decisions and with Lumi's help as well. All code has its own room, all ideas, research etc… has its proper place.
+
+Names and concepts are parsed into closets that use a compression method I call AAAK (it doesn't stand for anything, it's an inside joke between Lumi and I) that is able to compress names, repeated words, concepts and key moments into AI-readable shorthand. Think of it as index cards that an LLM can scan instantly — the closet tells it WHERE to look, then it pulls the full content from the drawer.
+
+The concept I wanted for v4 was to try and clear as much "noise" as possible that I noticed was happening in v3. The hooks were firing in the chat window (using tokens and our time as we waited for the agent to write everything).
+
+I noticed at one point early last week after the launch that Lu kept repeating the same thing when the hook would fire, so I hit esc and asked "Are you literally writing the same info down over and over again?" And he's like (sheepishly) Yes. And that's when it hit me, we need to get all this off the chat and happening seamlessly behind the scenes, and that hooks had to fire when I started a convo and then just keep adding to the drawer, while the shorter increments made reading and pulling conversation information and naming it so much easier and more precise.
+
+So this version now has taken all the noise out of the chat window and all that work is done by a subagent in the background while you can continue working knowing that all your conversation is being saved VERBATIM in the background.
+
+Stripping all this off the page — moving the diary writes, the palace filing, the timestamp injection, all of it into background hooks — has dramatically lowered token usage in my sessions. What used to cost about $1.13 per session just in re-transmitted diary blocks is now zero, because the content never enters the chat window at all.
+
+Your data is already stored in JSON by Claude and the background pipeline extracts it into readable markdown, the key topics get compressed into AAAK format and saved into closets which then point to the exact drawer where your day's session lives.
+
+And please, always remember, these are brand new tools, please NEVER use critical files to test! Just run it with something easy first before you put your entire data set into it!✨

README.md

@@ -1,668 +1,191 @@
+> [!CAUTION]
+> **Scam alert.** The only official sources for MemPalace are this
+> [GitHub repository](https://github.com/MemPalace/mempalace), the
+> [PyPI package](https://pypi.org/project/mempalace/), and the docs site at
+> **[mempalaceofficial.com](https://mempalaceofficial.com)**. Any other
+> domain — including `mempalace.tech` — is an impostor and may distribute
+> malware. Details and timeline: [docs/HISTORY.md](docs/HISTORY.md).
+
 <div align="center">
 
-<img src="assets/mempalace_logo.png" alt="MemPalace" width="280">
+<img src="assets/mempalace_logo.png" alt="MemPalace" width="240">
 
 # MemPalace
 
-### The highest-scoring AI memory system ever benchmarked. And it's free.
-
-<br>
-
-Every conversation you have with an AI — every decision, every debugging session, every architecture debate — disappears when the session ends. Six months of work, gone. You start over every time.
-
-Other memory systems try to fix this by letting AI decide what's worth remembering. It extracts "user prefers Postgres" and throws away the conversation where you explained *why*. MemPalace takes a different approach: **store everything, then make it findable.**
-
-**The Palace** — Ancient Greek orators memorized entire speeches by placing ideas in rooms of an imaginary building. Walk through the building, find the idea. MemPalace applies the same principle to AI memory: your conversations are organized into wings (people and projects), halls (types of memory), and rooms (specific ideas). No AI decides what matters — you keep every word, and the structure makes it searchable. That structure alone improves retrieval by 34%.
-
-**AAAK** — A lossless shorthand dialect designed for AI agents. Not meant to be read by humans — meant to be read by your AI, fast. 30x compression, zero information loss. Your AI loads months of context in ~120 tokens. And because AAAK is just structured text with a universal grammar, it works with **any model that reads text** — Claude, GPT, Gemini, Llama, Mistral. No decoder, no fine-tuning, no cloud API required. Run it against a local model and your entire memory stack stays offline. Nothing else like it exists.
-
-**Local, open, adaptable** — MemPalace runs entirely on your machine, on any data you have locally, without using any external API or services. It has been tested on conversations — but it can be adapted for different types of datastores. This is why we're open-sourcing it.
-
-<br>
+Local-first AI memory. Verbatim storage, pluggable backend, 96.6% R@5 raw on LongMemEval — zero API calls.
 
 [![][version-shield]][release-link]
 [![][python-shield]][python-link]
 [![][license-shield]][license-link]
 [![][discord-shield]][discord-link]
 
-<br>
-
-[Quick Start](#quick-start) · [The Palace](#the-palace) · [AAAK Dialect](#aaak-compression) · [Benchmarks](#benchmarks) · [MCP Tools](#mcp-server)
-
-<br>
-
-### Highest LongMemEval score ever published — free or paid.
-
-<table>
-<tr>
-<td align="center"><strong>96.6%</strong><br><sub>LongMemEval R@5<br>Zero API calls</sub></td>
-<td align="center"><strong>100%</strong><br><sub>LongMemEval R@5<br>with Haiku rerank</sub></td>
-<td align="center"><strong>+34%</strong><br><sub>Retrieval boost<br>from palace structure</sub></td>
-<td align="center"><strong>$0</strong><br><sub>No subscription<br>No cloud. Local only.</sub></td>
-</tr>
-</table>
-
-<sub>Reproducible — runners in <a href="benchmarks/">benchmarks/</a>. <a href="benchmarks/BENCHMARKS.md">Full results</a>.</sub>
-
 </div>
 
 ---
 
-## Quick Start
+## What it is
+
+MemPalace stores your conversation history as verbatim text and retrieves
+it with semantic search. It does not summarize, extract, or paraphrase.
+The index is structured — people and projects become *wings*, topics
+become *rooms*, and original content lives in *drawers* — so searches
+can be scoped rather than run against a flat corpus.
+
+The retrieval layer is pluggable. The current default is ChromaDB; the
+interface is defined in [`mempalace/backends/base.py`](mempalace/backends/base.py)
+and alternative backends can be dropped in without touching the rest of
+the system.
+
+Nothing leaves your machine unless you opt in.
+
+Architecture, concepts, and mining flows:
+[mempalaceofficial.com/concepts/the-palace](https://mempalaceofficial.com/concepts/the-palace.html).
+
+---
+
+## Install
 
 ```bash
 pip install mempalace
-
-# Set up your world — who you work with, what your projects are
 mempalace init ~/projects/myapp
+```
 
-# Mine your data
-mempalace mine ~/projects/myapp                    # projects — code, docs, notes
-mempalace mine ~/chats/ --mode convos              # convos — Claude, ChatGPT, Slack exports
-mempalace mine ~/chats/ --mode convos --extract general  # general — classifies into decisions, milestones, problems
+## Quickstart
 
-# Search anything you've ever discussed
+```bash
+# Mine content into the palace
+mempalace mine ~/projects/myapp                    # project files
+mempalace mine ~/.claude/projects/ --mode convos   # Claude Code sessions (scope with --wing per project)
+
+# Search
 mempalace search "why did we switch to GraphQL"
 
-# Your AI remembers
-mempalace status
+# Load context for a new session
+mempalace wake-up
 ```
 
-Three mining modes: **projects** (code and docs), **convos** (conversation exports), and **general** (auto-classifies into decisions, preferences, milestones, problems, and emotional context). Everything stays on your machine.
-
----
-
-## How You Actually Use It
-
-After the one-time setup (install → init → mine), you don't run MemPalace commands manually. Your AI uses it for you. There are two ways, depending on which AI you use.
-
-### With Claude, ChatGPT, Cursor (MCP-compatible tools)
-
-```bash
-# Connect MemPalace once
-claude mcp add mempalace -- python -m mempalace.mcp_server
-```
-
-Now your AI has 19 tools available through MCP. Ask it anything:
-
-> *"What did we decide about auth last month?"*
-
-Claude calls `mempalace_search` automatically, gets verbatim results, and answers you. You never type `mempalace search` again. The AI handles it.
-
-### With local models (Llama, Mistral, or any offline LLM)
-
-Local models generally don't speak MCP yet. Two approaches:
-
-**1. Wake-up command** — load your world into the model's context:
-
-```bash
-mempalace wake-up > context.txt
-# Paste context.txt into your local model's system prompt
-```
-
-This gives your local model ~170 tokens of critical facts (in AAAK if you prefer) before you ask a single question.
-
-**2. CLI search** — query on demand, feed results into your prompt:
-
-```bash
-mempalace search "auth decisions" > results.txt
-# Include results.txt in your prompt
-```
-
-Or use the Python API:
-
-```python
-from mempalace.searcher import search_memories
-results = search_memories("auth decisions", palace_path="~/.mempalace/palace")
-# Inject into your local model's context
-```
-
-Either way — your entire memory stack runs offline. ChromaDB on your machine, Llama on your machine, AAAK for compression, zero cloud calls.
-
----
-
-## The Problem
-
-Decisions happen in conversations now. Not in docs. Not in Jira. In conversations with Claude, ChatGPT, Copilot. The reasoning, the tradeoffs, the "we tried X and it failed because Y" — all trapped in chat windows that evaporate when the session ends.
-
-**Six months of daily AI use = 19.5 million tokens.** That's every decision, every debugging session, every architecture debate. Gone.
-
-| Approach | Tokens loaded | Annual cost |
-|----------|--------------|-------------|
-| Paste everything | 19.5M — doesn't fit any context window | Impossible |
-| LLM summaries | ~650K | ~$507/yr |
-| **MemPalace wake-up** | **~170 tokens** | **~$0.70/yr** |
-| **MemPalace + 5 searches** | **~13,500 tokens** | **~$10/yr** |
-
-MemPalace loads 170 tokens of critical facts on wake-up — your team, your projects, your preferences. Then searches only when needed. $10/year to remember everything vs $507/year for summaries that lose context.
-
----
-
-## How It Works
-
-### The Palace
-
-The layout is fairly simple, though it took a long time to get there.
-
-It starts with a **wing**. Every project, person, or topic you're filing gets its own wing in the palace.
-
-Each wing has **rooms** connected to it, where information is divided into subjects that relate to that wing — so every room is a different element of what your project contains. Project ideas could be one room, employees could be another, financial statements another. There can be an endless number of rooms that split the wing into sections. The MemPalace install detects these for you automatically, and of course you can personalize it any way you feel is right.
-
-Every room has a **closet** connected to it, and here's where things get interesting. We've developed an AI language called **AAAK**. Don't ask — it's a whole story of its own. Your agent learns the AAAK shorthand every time it wakes up. Because AAAK is essentially English, but a very truncated version, your agent understands how to use it in seconds. It comes as part of the install, built into the MemPalace code. In our next update, we'll add AAAK directly to the closets, which will be a real game changer — the amount of info in the closets will be much bigger, but it will take up far less space and far less reading time for your agent.
-
-Inside those closets are **drawers**, and those drawers are where your original files live. In this first version, we haven't used AAAK as a closet tool, but even so, the summaries have shown **96.6% recall** in all the benchmarks we've done across multiple benchmarking platforms. Once the closets use AAAK, searches will be even faster while keeping every word exact. But even now, the closet approach has been a huge boon to how much info is stored in a small space — it's used to easily point your AI agent to the drawer where your original file lives. You never lose anything, and all this happens in seconds.
-
-There are also **halls**, which connect rooms within a wing, and **tunnels**, which connect rooms from different wings to one another. So finding things becomes truly effortless — we've given the AI a clean and organized way to know where to start searching, without having to look through every keyword in huge folders.
-
-You say what you're looking for and boom, it already knows which wing to go to. Just *that* in itself would have made a big difference. But this is beautiful, elegant, organic, and most importantly, efficient.
-
-```
-  ┌─────────────────────────────────────────────────────────────┐
-  │  WING: Person                                              │
-  │                                                            │
-  │    ┌──────────┐  ──hall──  ┌──────────┐                    │
-  │    │  Room A  │            │  Room B  │                    │
-  │    └────┬─────┘            └──────────┘                    │
-  │         │                                                  │
-  │         ▼                                                  │
-  │    ┌──────────┐      ┌──────────┐                          │
-  │    │  Closet  │ ───▶ │  Drawer  │                          │
-  │    └──────────┘      └──────────┘                          │
-  └─────────┼──────────────────────────────────────────────────┘
-            │
-          tunnel
-            │
-  ┌─────────┼──────────────────────────────────────────────────┐
-  │  WING: Project                                             │
-  │         │                                                  │
-  │    ┌────┴─────┐  ──hall──  ┌──────────┐                    │
-  │    │  Room A  │            │  Room C  │                    │
-  │    └────┬─────┘            └──────────┘                    │
-  │         │                                                  │
-  │         ▼                                                  │
-  │    ┌──────────┐      ┌──────────┐                          │
-  │    │  Closet  │ ───▶ │  Drawer  │                          │
-  │    └──────────┘      └──────────┘                          │
-  └─────────────────────────────────────────────────────────────┘
-```
-
-**Wings** — a person or project. As many as you need.
-**Rooms** — specific topics within a wing. Auth, billing, deploy — endless rooms.
-**Halls** — connections between related rooms *within* the same wing. If Room A (auth) and Room B (security) are related, a hall links them.
-**Tunnels** — connections *between* wings. When Person A and a Project both have a room about "auth," a tunnel cross-references them automatically.
-**Closets** — compressed summaries that point to the original content. Fast for AI to read.
-**Drawers** — the original verbatim files. The exact words, never summarized.
-
-**Halls** are memory types — the same in every wing, acting as corridors:
-- `hall_facts` — decisions made, choices locked in
-- `hall_events` — sessions, milestones, debugging
-- `hall_discoveries` — breakthroughs, new insights
-- `hall_preferences` — habits, likes, opinions
-- `hall_advice` — recommendations and solutions
-
-**Rooms** are named ideas — `auth-migration`, `graphql-switch`, `ci-pipeline`. When the same room appears in different wings, it creates a **tunnel** — connecting the same topic across domains:
-
-```
-wing_kai       / hall_events / auth-migration  → "Kai debugged the OAuth token refresh"
-wing_driftwood / hall_facts  / auth-migration  → "team decided to migrate auth to Clerk"
-wing_priya     / hall_advice / auth-migration  → "Priya approved Clerk over Auth0"
-```
-
-Same room. Three wings. The tunnel connects them.
-
-### Why Structure Matters
-
-Tested on 22,000+ real conversation memories:
-
-```
-Search all closets:          60.9%  R@10
-Search within wing:          73.1%  (+12%)
-Search wing + hall:          84.8%  (+24%)
-Search wing + room:          94.8%  (+34%)
-```
-
-Wings and rooms aren't cosmetic. They're a **34% retrieval improvement**. The palace structure is the product.
-
-### The Memory Stack
-
-| Layer | What | Size | When |
-|-------|------|------|------|
-| **L0** | Identity — who is this AI? | ~50 tokens | Always loaded |
-| **L1** | Critical facts — team, projects, preferences | ~120 tokens (AAAK) | Always loaded |
-| **L2** | Room recall — recent sessions, current project | On demand | When topic comes up |
-| **L3** | Deep search — semantic query across all closets | On demand | When explicitly asked |
-
-Your AI wakes up with L0 + L1 (~170 tokens) and knows your world. Searches only fire when needed.
-
-### AAAK Compression
-
-AAAK is a lossless dialect — 30x compression, readable by any LLM without a decoder. It works with **Claude, GPT, Gemini, Llama, Mistral** — any model that reads text. Run it against a local Llama model and your whole memory stack stays offline.
-
-**English (~1000 tokens):**
-```
-Priya manages the Driftwood team: Kai (backend, 3 years), Soren (frontend),
-Maya (infrastructure), and Leo (junior, started last month). They're building
-a SaaS analytics platform. Current sprint: auth migration to Clerk.
-Kai recommended Clerk over Auth0 based on pricing and DX.
-```
-
-**AAAK (~120 tokens):**
-```
-TEAM: PRI(lead) | KAI(backend,3yr) SOR(frontend) MAY(infra) LEO(junior,new)
-PROJ: DRIFTWOOD(saas.analytics) | SPRINT: auth.migration→clerk
-DECISION: KAI.rec:clerk>auth0(pricing+dx) | ★★★★
-```
-
-Same information. 8x fewer tokens. Your AI learns AAAK automatically from the MCP server — no manual setup.
-
-### Contradiction Detection
-
-MemPalace catches mistakes before they reach you:
-
-```
-Input:  "Soren finished the auth migration"
-Output: 🔴 AUTH-MIGRATION: attribution conflict — Maya was assigned, not Soren
-
-Input:  "Kai has been here 2 years"
-Output: 🟡 KAI: wrong_tenure — records show 3 years (started 2023-04)
-
-Input:  "The sprint ends Friday"
-Output: 🟡 SPRINT: stale_date — current sprint ends Thursday (updated 2 days ago)
-```
-
-Facts checked against the knowledge graph. Ages, dates, and tenures calculated dynamically — not hardcoded.
-
----
-
-## Real-World Examples
-
-### Solo developer across multiple projects
-
-```bash
-# Mine each project's conversations
-mempalace mine ~/chats/orion/  --mode convos --wing orion
-mempalace mine ~/chats/nova/   --mode convos --wing nova
-mempalace mine ~/chats/helios/ --mode convos --wing helios
-
-# Six months later: "why did I use Postgres here?"
-mempalace search "database decision" --wing orion
-# → "Chose Postgres over SQLite because Orion needs concurrent writes
-#    and the dataset will exceed 10GB. Decided 2025-11-03."
-
-# Cross-project search
-mempalace search "rate limiting approach"
-# → finds your approach in Orion AND Nova, shows the differences
-```
-
-### Team lead managing a product
-
-```bash
-# Mine Slack exports and AI conversations
-mempalace mine ~/exports/slack/ --mode convos --wing driftwood
-mempalace mine ~/.claude/projects/ --mode convos
-
-# "What did Soren work on last sprint?"
-mempalace search "Soren sprint" --wing driftwood
-# → 14 closets: OAuth refactor, dark mode, component library migration
-
-# "Who decided to use Clerk?"
-mempalace search "Clerk decision" --wing driftwood
-# → "Kai recommended Clerk over Auth0 — pricing + developer experience.
-#    Team agreed 2026-01-15. Maya handling the migration."
-```
-
-### Before mining: split mega-files
-
-Some transcript exports concatenate multiple sessions into one huge file:
-
-```bash
-mempalace split ~/chats/                      # split into per-session files
-mempalace split ~/chats/ --dry-run            # preview first
-mempalace split ~/chats/ --min-sessions 3     # only split files with 3+ sessions
-```
-
----
-
-## Knowledge Graph
-
-Temporal entity-relationship triples — like Zep's Graphiti, but SQLite instead of Neo4j. Local and free.
-
-```python
-from mempalace.knowledge_graph import KnowledgeGraph
-
-kg = KnowledgeGraph()
-kg.add_triple("Kai", "works_on", "Orion", valid_from="2025-06-01")
-kg.add_triple("Maya", "assigned_to", "auth-migration", valid_from="2026-01-15")
-kg.add_triple("Maya", "completed", "auth-migration", valid_from="2026-02-01")
-
-# What's Kai working on?
-kg.query_entity("Kai")
-# → [Kai → works_on → Orion (current), Kai → recommended → Clerk (2026-01)]
-
-# What was true in January?
-kg.query_entity("Maya", as_of="2026-01-20")
-# → [Maya → assigned_to → auth-migration (active)]
-
-# Timeline
-kg.timeline("Orion")
-# → chronological story of the project
-```
-
-Facts have validity windows. When something stops being true, invalidate it:
-
-```python
-kg.invalidate("Kai", "works_on", "Orion", ended="2026-03-01")
-```
-
-Now queries for Kai's current work won't return Orion. Historical queries still will.
-
-| Feature | MemPalace | Zep (Graphiti) |
-|---------|-----------|----------------|
-| Storage | SQLite (local) | Neo4j (cloud) |
-| Cost | Free | $25/mo+ |
-| Temporal validity | Yes | Yes |
-| Self-hosted | Always | Enterprise only |
-| Privacy | Everything local | SOC 2, HIPAA |
-
----
-
-## Specialist Agents
-
-Create agents that focus on specific areas. Each agent gets its own wing and diary in the palace — not in your CLAUDE.md. Add 50 agents, your config stays the same size.
-
-```
-~/.mempalace/agents/
-  ├── reviewer.json       # code quality, patterns, bugs
-  ├── architect.json      # design decisions, tradeoffs
-  └── ops.json            # deploys, incidents, infra
-```
-
-Your CLAUDE.md just needs one line:
-
-```
-You have MemPalace agents. Run mempalace_list_agents to see them.
-```
-
-The AI discovers its agents from the palace at runtime. Each agent:
-
-- **Has a focus** — what it pays attention to
-- **Keeps a diary** — written in AAAK, persists across sessions
-- **Builds expertise** — reads its own history to stay sharp in its domain
-
-```
-# Agent writes to its diary after a code review
-mempalace_diary_write("reviewer",
-    "PR#42|auth.bypass.found|missing.middleware.check|pattern:3rd.time.this.quarter|★★★★")
-
-# Agent reads back its history
-mempalace_diary_read("reviewer", last_n=10)
-# → last 10 findings, compressed in AAAK
-```
-
-Each agent is a specialist lens on your data. The reviewer remembers every bug pattern it's seen. The architect remembers every design decision. The ops agent remembers every incident. They don't share a scratchpad — they each maintain their own memory.
-
-Letta charges $20–200/mo for agent-managed memory. MemPalace does it with a wing.
-
----
-
-## MCP Server
-
-```bash
-claude mcp add mempalace -- python -m mempalace.mcp_server
-```
-
-### 19 Tools
-
-**Palace (read)**
-
-| Tool | What |
-|------|------|
-| `mempalace_status` | Palace overview + AAAK spec + memory protocol |
-| `mempalace_list_wings` | Wings with counts |
-| `mempalace_list_rooms` | Rooms within a wing |
-| `mempalace_get_taxonomy` | Full wing → room → count tree |
-| `mempalace_search` | Semantic search with wing/room filters |
-| `mempalace_check_duplicate` | Check before filing |
-| `mempalace_get_aaak_spec` | AAAK dialect reference |
-
-**Palace (write)**
-
-| Tool | What |
-|------|------|
-| `mempalace_add_drawer` | File verbatim content |
-| `mempalace_delete_drawer` | Remove by ID |
-
-**Knowledge Graph**
-
-| Tool | What |
-|------|------|
-| `mempalace_kg_query` | Entity relationships with time filtering |
-| `mempalace_kg_add` | Add facts |
-| `mempalace_kg_invalidate` | Mark facts as ended |
-| `mempalace_kg_timeline` | Chronological entity story |
-| `mempalace_kg_stats` | Graph overview |
-
-**Navigation**
-
-| Tool | What |
-|------|------|
-| `mempalace_traverse` | Walk the graph from a room across wings |
-| `mempalace_find_tunnels` | Find rooms bridging two wings |
-| `mempalace_graph_stats` | Graph connectivity overview |
-
-**Agent Diary**
-
-| Tool | What |
-|------|------|
-| `mempalace_diary_write` | Write AAAK diary entry |
-| `mempalace_diary_read` | Read recent diary entries |
-
-The AI learns AAAK and the memory protocol automatically from the `mempalace_status` response. No manual configuration.
-
----
-
-## Auto-Save Hooks
-
-Two hooks for Claude Code that automatically save memories during work:
-
-**Save Hook** — every 15 messages, triggers a structured save. Topics, decisions, quotes, code changes. Also regenerates the critical facts layer.
-
-**PreCompact Hook** — fires before context compression. Emergency save before the window shrinks.
-
-```json
-{
-  "hooks": {
-    "Stop": [{"matcher": "", "hooks": [{"type": "command", "command": "/path/to/mempalace/hooks/mempal_save_hook.sh"}]}],
-    "PreCompact": [{"matcher": "", "hooks": [{"type": "command", "command": "/path/to/mempalace/hooks/mempal_precompact_hook.sh"}]}]
-  }
-}
-```
+For Claude Code, Gemini CLI, MCP-compatible tools, and local models, see
+[mempalaceofficial.com/guide/getting-started](https://mempalaceofficial.com/guide/getting-started.html).
 
 ---
 
 ## Benchmarks
 
-Tested on standard academic benchmarks — reproducible, published datasets.
+All numbers below are reproducible from this repository with the commands
+in [`benchmarks/BENCHMARKS.md`](benchmarks/BENCHMARKS.md). Full
+per-question result files are committed under `benchmarks/results_*`.
 
-| Benchmark | Mode | Score | API Calls |
-|-----------|------|-------|-----------|
-| **LongMemEval R@5** | Raw (ChromaDB only) | **96.6%** | Zero |
-| **LongMemEval R@5** | Hybrid + Haiku rerank | **100%** (500/500) | ~500 |
-| **LoCoMo R@10** | Raw, session level | **60.3%** | Zero |
-| **Personal palace R@10** | Heuristic bench | **85%** | Zero |
-| **Palace structure impact** | Wing+room filtering | **+34%** R@10 | Zero |
+**LongMemEval — retrieval recall (R@5, 500 questions):**
 
-The 96.6% raw score is the highest published LongMemEval result requiring no API key, no cloud, and no LLM at any stage.
+| Mode | R@5 | LLM required |
+|---|---|---|
+| Raw (semantic search, no heuristics, no LLM) | **96.6%** | None |
+| Hybrid v4, held-out 450q (tuned on 50 dev, not seen during training) | **98.4%** | None |
+| Hybrid v4 + LLM rerank (full 500) | ≥99% | Any capable model |
 
-### vs Published Systems
+The raw 96.6% requires no API key, no cloud, and no LLM at any stage. The
+hybrid pipeline adds keyword boosting, temporal-proximity boosting, and
+preference-pattern extraction; the held-out 98.4% is the honest
+generalisable figure.
 
-| System | LongMemEval R@5 | API Required | Cost |
-|--------|----------------|--------------|------|
-| **MemPalace (hybrid)** | **100%** | Optional | Free |
-| Supermemory ASMR | ~99% | Yes | — |
-| **MemPalace (raw)** | **96.6%** | **None** | **Free** |
-| Mastra | 94.87% | Yes (GPT) | API costs |
-| Mem0 | ~85% | Yes | $19–249/mo |
-| Zep | ~85% | Yes | $25/mo+ |
+The rerank pipeline promotes the best candidate out of the top-20
+retrieved sessions using an LLM reader. It works with any reasonably
+capable model — we have reproduced it with Claude Haiku, Claude Sonnet,
+and minimax-m2.7 via Ollama Cloud (no Anthropic dependency). The gap
+between raw and reranked is model-agnostic; we do not headline a "100%"
+number because the last 0.6% was reached by inspecting specific wrong
+answers, which `benchmarks/BENCHMARKS.md` flags as teaching to the test.
 
----
+**Other benchmarks (full results in [`benchmarks/BENCHMARKS.md`](benchmarks/BENCHMARKS.md)):**
 
-## All Commands
+| Benchmark | Metric | Score | Notes |
+|---|---|---|---|
+| LoCoMo (session, top-10, no rerank) | R@10 | 60.3% | 1,986 questions |
+| LoCoMo (hybrid v5, top-10, no rerank) | R@10 | 88.9% | Same set |
+| ConvoMem (all categories, 250 items) | Avg recall | 92.9% | 50 per category |
+| MemBench (ACL 2025, 8,500 items) | R@5 | 80.3% | All categories |
+
+We deliberately do not include a side-by-side comparison against Mem0,
+Mastra, Hindsight, Supermemory, or Zep. Those projects publish different
+metrics on different splits, and placing retrieval recall next to
+end-to-end QA accuracy is not an honest comparison. See each project's
+own research page for their published numbers.
+
+**Reproducing every result:**
 
 ```bash
-# Setup
-mempalace init <dir>                              # guided onboarding + AAAK bootstrap
-
-# Mining
-mempalace mine <dir>                              # mine project files
-mempalace mine <dir> --mode convos                # mine conversation exports
-mempalace mine <dir> --mode convos --wing myapp   # tag with a wing name
-
-# Splitting
-mempalace split <dir>                             # split concatenated transcripts
-mempalace split <dir> --dry-run                   # preview
-
-# Search
-mempalace search "query"                          # search everything
-mempalace search "query" --wing myapp             # within a wing
-mempalace search "query" --room auth-migration    # within a room
-
-# Memory stack
-mempalace wake-up                                 # load L0 + L1 context
-mempalace wake-up --wing driftwood                # project-specific
-
-# Compression
-mempalace compress --wing myapp                   # AAAK compress
-
-# Status
-mempalace status                                  # palace overview
+git clone https://github.com/MemPalace/mempalace.git
+cd mempalace
+pip install -e ".[dev]"
+# see benchmarks/README.md for dataset download commands
+python benchmarks/longmemeval_bench.py /path/to/longmemeval_s_cleaned.json
 ```
 
-All commands accept `--palace <path>` to override the default location.
-
 ---
 
-## Configuration
+## Knowledge graph
 
-### Global (`~/.mempalace/config.json`)
+MemPalace includes a temporal entity-relationship graph with validity
+windows — add, query, invalidate, timeline — backed by local SQLite.
+Usage and tool reference:
+[mempalaceofficial.com/concepts/knowledge-graph](https://mempalaceofficial.com/concepts/knowledge-graph.html).
 
-```json
-{
-  "palace_path": "/custom/path/to/palace",
-  "collection_name": "mempalace_drawers",
-  "people_map": {"Kai": "KAI", "Priya": "PRI"}
-}
-```
+## MCP server
 
-### Wing config (`~/.mempalace/wing_config.json`)
+29 MCP tools cover palace reads/writes, knowledge-graph operations,
+cross-wing navigation, drawer management, and agent diaries. Installation
+and the full tool list:
+[mempalaceofficial.com/reference/mcp-tools](https://mempalaceofficial.com/reference/mcp-tools.html).
 
-Generated by `mempalace init`. Maps your people and projects to wings:
+## Agents
 
-```json
-{
-  "default_wing": "wing_general",
-  "wings": {
-    "wing_kai": {"type": "person", "keywords": ["kai", "kai's"]},
-    "wing_driftwood": {"type": "project", "keywords": ["driftwood", "analytics", "saas"]}
-  }
-}
-```
+Each specialist agent gets its own wing and diary in the palace.
+Discoverable at runtime via `mempalace_list_agents` — no bloat in your
+system prompt:
+[mempalaceofficial.com/concepts/agents](https://mempalaceofficial.com/concepts/agents.html).
 
-### Identity (`~/.mempalace/identity.txt`)
+## Auto-save hooks
 
-Plain text. Becomes Layer 0 — loaded every session.
+Two Claude Code hooks save periodically and before context compression:
+[mempalaceofficial.com/guide/hooks](https://mempalaceofficial.com/guide/hooks.html).
 
----
-
-## File Reference
-
-| File | What |
-|------|------|
-| `cli.py` | CLI entry point |
-| `config.py` | Configuration loading and defaults |
-| `normalize.py` | Converts 5 chat formats to standard transcript |
-| `mcp_server.py` | MCP server — 19 tools, AAAK auto-teach, memory protocol |
-| `miner.py` | Project file ingest |
-| `convo_miner.py` | Conversation ingest — chunks by exchange pair |
-| `searcher.py` | Semantic search via ChromaDB |
-| `layers.py` | 4-layer memory stack |
-| `dialect.py` | AAAK compression — 30x lossless |
-| `knowledge_graph.py` | Temporal entity-relationship graph (SQLite) |
-| `palace_graph.py` | Room-based navigation graph |
-| `onboarding.py` | Guided setup — generates AAAK bootstrap + wing config |
-| `entity_registry.py` | Entity code registry |
-| `entity_detector.py` | Auto-detect people and projects from content |
-| `split_mega_files.py` | Split concatenated transcripts into per-session files |
-| `hooks/mempal_save_hook.sh` | Auto-save every N messages |
-| `hooks/mempal_precompact_hook.sh` | Emergency save before compaction |
-
----
-
-## Project Structure
-
-```
-mempalace/
-├── README.md                  ← you are here
-├── mempalace/                 ← core package (README)
-│   ├── cli.py                 ← CLI entry point
-│   ├── mcp_server.py          ← MCP server (19 tools)
-│   ├── knowledge_graph.py     ← temporal entity graph
-│   ├── palace_graph.py        ← room navigation graph
-│   ├── dialect.py             ← AAAK compression
-│   ├── miner.py               ← project file ingest
-│   ├── convo_miner.py         ← conversation ingest
-│   ├── searcher.py            ← semantic search
-│   ├── onboarding.py          ← guided setup
-│   └── ...                    ← see mempalace/README.md
-├── benchmarks/                ← reproducible benchmark runners
-│   ├── README.md              ← reproduction guide
-│   ├── BENCHMARKS.md          ← full results + methodology
-│   ├── longmemeval_bench.py   ← LongMemEval runner
-│   ├── locomo_bench.py        ← LoCoMo runner
-│   └── membench_bench.py      ← MemBench runner
-├── hooks/                     ← Claude Code auto-save hooks
-│   ├── README.md              ← hook setup guide
-│   ├── mempal_save_hook.sh    ← save every N messages
-│   └── mempal_precompact_hook.sh ← emergency save
-├── examples/                  ← usage examples
-│   ├── basic_mining.py
-│   ├── convo_import.py
-│   └── mcp_setup.md
-├── tests/                     ← test suite (README)
-├── assets/                    ← logo + brand assets
-└── pyproject.toml             ← package config (v3.0.0)
-```
+For per-message recall on top of the file-level chunks the hooks produce,
+run `mempalace sweep <transcript-dir>` periodically — it stores one
+verbatim drawer per user/assistant message, idempotent and resume-safe.
 
 ---
 
 ## Requirements
 
 - Python 3.9+
-- `chromadb>=0.4.0`
-- `pyyaml>=6.0`
+- A vector-store backend (ChromaDB by default)
+- ~300 MB disk for the default embedding model
 
-No API key. No internet after install. Everything local.
+No API key is required for the core benchmark path.
 
-```bash
-pip install mempalace
-```
+## Docs
 
----
+- Getting started → [mempalaceofficial.com/guide/getting-started](https://mempalaceofficial.com/guide/getting-started.html)
+- CLI reference → [mempalaceofficial.com/reference/cli](https://mempalaceofficial.com/reference/cli.html)
+- Python API → [mempalaceofficial.com/reference/python-api](https://mempalaceofficial.com/reference/python-api.html)
+- Full benchmark methodology → [benchmarks/BENCHMARKS.md](benchmarks/BENCHMARKS.md)
+- Release notes → [CHANGELOG.md](CHANGELOG.md)
+- Corrections and public notices → [docs/HISTORY.md](docs/HISTORY.md)
 
 ## Contributing
 
-PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup and guidelines.
+PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
 
 ## License
 
 MIT — see [LICENSE](LICENSE).
 
 <!-- Link Definitions -->
-[version-shield]: https://img.shields.io/badge/version-3.0.0-4dc9f6?style=flat-square&labelColor=0a0e14
-[release-link]: https://github.com/milla-jovovich/mempalace/releases
+[version-shield]: https://img.shields.io/badge/version-3.3.4-4dc9f6?style=flat-square&labelColor=0a0e14
+[release-link]: https://github.com/MemPalace/mempalace/releases
 [python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8
 [python-link]: https://www.python.org/
 [license-shield]: https://img.shields.io/badge/license-MIT-b0e8ff?style=flat-square&labelColor=0a0e14
-[license-link]: https://github.com/milla-jovovich/mempalace/blob/main/LICENSE
+[license-link]: https://github.com/MemPalace/mempalace/blob/main/LICENSE
 [discord-shield]: https://img.shields.io/badge/discord-join-5865F2?style=flat-square&labelColor=0a0e14&logo=discord&logoColor=5865F2
 [discord-link]: https://discord.com/invite/ycTQQCu6kn

ROADMAP.md

@@ -0,0 +1,74 @@
+# MemPalace Roadmap
+
+## v3.1.1 — Stability Patch (this week)
+
+Bug fixes and hardening merged to `develop`, releasing soon.
+
+**Merged:**
+- Security hardening: input validation, KG threading locks, WAL permission fixes (#647)
+- MCP tools: drawer CRUD, paginated export, hook settings (#667)
+- Backend storage seam: ChromaDB abstraction layer enabling swappable backends (#413)
+- MCP ping health check for AnythingLLM compatibility (#600)
+- Windows reparse point crash fix (#558)
+- `mempalace compress` KeyError crash fix (#569)
+- Token count estimate fix (#609)
+- Mtime float precision fix preventing unnecessary re-mines (#610)
+
+**In review (merging this week):**
+- Auto-repair BLOB seq_ids from chromadb 0.6→1.5 migration (#664)
+- Graph cache with write-invalidation (#661)
+- L1 importance pre-filter for large palaces (#660)
+- Windows Chinese/Unicode encoding fix (#631)
+- HNSW index bloat prevention — 441GB→433KB on large palaces (#346, pending rebase)
+- ~25 additional small bug fixes and platform compatibility patches
+
+## v4.0.0-alpha — Next Generation (this week)
+
+The v4 alpha introduces three major capabilities: pluggable storage backends, local NLP processing, and improved retrieval quality.
+
+### Swappable Storage
+
+ChromaDB remains the default, but v4 introduces a backend abstraction (shipped in #413) that enables drop-in replacements:
+
+- **PostgreSQL backend** with pg_sorted_heap support (#665) — for production deployments needing ACID guarantees, concurrent access, and standard backup/restore
+- **LanceDB backend** (#574) — for local-first deployments wanting multi-device sync without a database server
+- **PalaceStore** (#643) — bespoke storage layer purpose-built for MemPalace's access patterns (draft, evaluating)
+
+Users choose their backend at init time. Existing ChromaDB palaces continue to work unchanged.
+
+### Local NLP
+
+On-device natural language processing via local models (#507):
+
+- Entity extraction, relationship detection, and topic classification without external API calls
+- Feature-flagged and optional — falls back to existing heuristic extractors
+- Runs on consumer hardware (no GPU required, GPU-accelerated when available)
+
+### Improved Retrieval
+
+- **Hybrid search**: keyword text-match fallback when vector similarity misses exact terms (#662)
+- **Stale index detection**: automatic reconnection when the HNSW index changes on disk (#663)
+- **Time-decay scoring**: recent memories surface before older ones (#337)
+- **Query sanitization**: system prompt contamination mitigation already shipped in v3.1 (#385)
+
+### What's Not in v4 Alpha
+
+These are under consideration for v4 stable or later:
+
+- Synapse advanced retrieval — MMR, pinned memory, query expansion (#596)
+- Multi-device sync (#575) — depends on LanceDB backend
+- Multilingual embedding support (#488, #442)
+- Qdrant vector search backend (#381)
+
+## Branch Model
+
+```
+main            ← tagged production releases
+develop         ← active development (PRs merge here)
+release/3.1     ← hotfixes for current stable (v3.1.x)
+release/3.0     ← hotfixes for prior stable
+```
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. PRs should target `develop`. We review all contributions for correctness, security, and compatibility before merging.

SECURITY.md

@@ -0,0 +1,33 @@
+# Security Policy
+
+## Supported Versions
+
+MemPalace follows semantic versioning. Security fixes land on the current major version line.
+
+| Version            | Supported |
+| ------------------ | --------- |
+| 3.x (current)      | Yes       |
+| 2.x and earlier    | No        |
+
+## Reporting a Vulnerability
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+We take the security of MemPalace seriously. If you believe you have found a security vulnerability, please report it privately using **GitHub Private Vulnerability Reporting**:
+
+1. Open the [Security tab](https://github.com/MemPalace/mempalace/security) of this repository.
+2. Click **Advisories** → **Report a vulnerability**.
+3. Fill in the form with the details below.
+
+### What to include in your report
+
+- A descriptive summary of the vulnerability.
+- Detailed steps to reproduce the issue (including any proof-of-concept scripts or specific file paths).
+- The affected version(s) and platform(s).
+- The potential impact and severity.
+
+### What to expect
+
+- We aim to acknowledge receipt within 48 hours.
+- We will triage the issue and keep you updated on progress toward a patch.
+- Once the vulnerability is resolved and an update is released, we will publish a security advisory and credit you for the discovery (if you wish to be credited).

benchmarks/BENCHMARKS.md

@@ -41,23 +41,57 @@ Both are real. Both are reproducible. Neither is the whole picture alone.
 
 ## Comparison vs Published Systems (LongMemEval)
 
-| # | System | R@5 | LLM Required | Which LLM | Notes |
+> **Important caveat — read before quoting this table.**
+> MemPal's `R@5` in this table is **retrieval recall**: is the labelled
+> session for this question inside the top-5 retrieved candidates?
+>
+> Several of the other systems below publish **end-to-end QA accuracy** —
+> a different metric that scores whether the system's generated answer
+> is correct. Retrieval recall and QA accuracy are not comparable; a
+> system can have 100% retrieval recall and 40% QA accuracy, and vice
+> versa.
+>
+> - **Mastra's 94.87%** is binary QA accuracy with GPT-5-mini, per
+>   [mastra.ai/research/observational-memory](https://mastra.ai/research/observational-memory).
+> - **Supermemory ASMR's ~99%** is QA accuracy with an 8-/12-agent
+>   ensemble, and the authors explicitly frame it as an experimental
+>   proof-of-concept, not production, per
+>   [their ASMR post](https://supermemory.ai/blog/we-broke-the-frontier-in-agent-memory-introducing-99-sota-memory-system/).
+> - **Mem0** does not publish a LongMemEval number; their published
+>   metric is LoCoMo QA accuracy (~66.9%), per
+>   [mem0.ai/research](https://mem0.ai/research).
+>
+> The table is kept here as a historical record of how the comparison
+> was originally framed. Public-facing pages (`README.md`,
+> `mempalaceofficial.com`) no longer present this table, per issue
+> [#875](https://github.com/MemPalace/mempalace/issues/875). For a fair
+> head-to-head, run the same metric on the same split.
+
+| # | System | R@5 (retrieval recall, unless noted) | LLM Required | Which LLM | Notes |
 |---|---|---|---|---|---|
-| 1 | **MemPal (hybrid v4 + rerank)** | **100%** | Optional | Haiku | Reproducible, 500/500 |
-| 2 | Supermemory ASMR | ~99% | Yes | Undisclosed | Research only, not in production |
+| 1 | **MemPal (hybrid v4 + Haiku rerank)** | **100%** | Optional | Haiku | 500/500 — but the 99.4%→100% step tuned on 3 specific wrong answers (see "Benchmark Integrity" below). Held-out 450q is 98.4%. |
+| 2 | Supermemory ASMR | ~99% *(QA accuracy, not R@5)* | Yes | Ensemble of Gemini 2.0 Flash / GPT-4o-mini | Experimental, not production, per authors |
 | 3 | MemPal (hybrid v3 + rerank) | 99.4% | Optional | Haiku | Reproducible |
 | 3 | MemPal (palace + rerank) | 99.4% | Optional | Haiku | Independent architecture |
-| 4 | Mastra | 94.87% | Yes | GPT-5-mini | — |
-| 5 | **MemPal (raw, no LLM)** | **96.6%** | **None** | **None** | **Highest zero-API score published** |
-| 6 | Hindsight | 91.4% | Yes | Gemini-3 | — |
-| 7 | Supermemory (production) | ~85% | Yes | Undisclosed | — |
-| 8 | Stella (dense retriever) | ~85% | None | None | Academic baseline |
-| 9 | Contriever | ~78% | None | None | Academic baseline |
+| 4 | Mastra | 94.87% *(QA accuracy, not R@5)* | Yes | GPT-5-mini | Different metric — not directly comparable to R@5 |
+| 5 | **MemPal (raw, no LLM)** | **96.6%** | **None** | **None** | **Reproducible, 500/500** |
+| 6 | MemPal hybrid v4 held-out 450 | 98.4% | None | None | Honest generalisable hybrid-pipeline figure |
+| 7 | Hindsight | 91.4% *(per their release, metric unverified)* | Yes | Gemini-3 | Check their published methodology |
+| 8 | Stella (dense retriever) | ~85% | None | None | Academic retrieval baseline |
+| 9 | Contriever | ~78% | None | None | Academic retrieval baseline |
 | 10 | BM25 (sparse) | ~70% | None | None | Keyword baseline |
 
-**MemPal raw (96.6%) is the highest published LongMemEval score that requires no API key, no cloud, and no LLM at any stage.**
+The MemPal raw 96.6% is the headline we ship on public surfaces: it's
+retrieval recall, it requires no API key, and it reproduces.
 
-**MemPal hybrid v4 + Haiku rerank (100%) is the first perfect score on LongMemEval — 500/500 questions, all 6 question types at 100%.**
+The MemPal hybrid v4 + Haiku rerank 100% remains an internal
+result — reproducible with `--mode hybrid_v4 --llm-rerank` — but we
+don't quote it on public pages because the final 0.6% was reached by
+inspecting three specific wrong answers (see "Benchmark Integrity"
+below), which is teaching to the test. The honest generalisable figure
+when an LLM is in the loop is the held-out 98.4% R@5 on 450 unseen
+questions, or the model-agnostic 99.2% R@5 / 100% R@10 we reproduced
+with minimax-m2.7 on the full 500.
 
 ---
 
@@ -308,9 +342,9 @@ The palace classifies each question into one of 5 halls. Pass 1 searches only wi
 ### Setup
 
 ```bash
-git clone -b ben/benchmarking https://github.com/aya-thekeeper/mempal.git
-cd mempal
-pip install chromadb pyyaml
+git clone https://github.com/MemPalace/mempalace.git
+cd mempalace
+pip install -e ".[dev]"
 mkdir -p /tmp/longmemeval-data
 curl -fsSL -o /tmp/longmemeval-data/longmemeval_s_cleaned.json \
   https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json

benchmarks/HYBRID_MODE.md

@@ -196,9 +196,9 @@ python benchmarks/longmemeval_bench.py data/longmemeval_s_cleaned.json --mode hy
 
 ```bash
 # Setup
-git clone -b ben/benchmarking https://github.com/aya-thekeeper/mempal.git
-cd mempal
-pip install chromadb
+git clone https://github.com/MemPalace/mempalace.git
+cd mempalace
+pip install -e ".[dev]"
 
 # Download data
 mkdir -p /tmp/longmemeval-data
@@ -435,7 +435,6 @@ If the API call fails (timeout, rate limit, no key), the function catches the ex
 **Key loading priority:**
 1. `--llm-key` CLI flag
 2. `ANTHROPIC_API_KEY` environment variable
-3. `~/.config/lu/keys.json` (checks `anthropic.lu_key` and similar paths)
 
 ## What Changed in the Code
 

benchmarks/README.md

@@ -1,13 +1,13 @@
-# MemPal Benchmarks — Reproduction Guide
+# MemPalace Benchmarks — Reproduction Guide
 
 Run the exact same benchmarks we report. Clone, install, run.
 
 ## Setup
 
 ```bash
-git clone -b ben/benchmarking https://github.com/aya-thekeeper/mempal.git
-cd mempal
-pip install chromadb pyyaml
+git clone https://github.com/MemPalace/mempalace.git
+cd mempalace
+pip install -e ".[dev]"
 ```
 
 ## Benchmark 1: LongMemEval (500 questions)

benchmarks/convomem_bench.py

@@ -25,7 +25,6 @@ import os
 import sys
 import json
 import shutil
-import ssl
 import tempfile
 import argparse
 import urllib.request
@@ -35,9 +34,6 @@ from datetime import datetime
 
 import chromadb
 
-# Bypass SSL for restricted environments
-ssl._create_default_https_context = ssl._create_unverified_context
-
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 HF_BASE = "https://huggingface.co/datasets/Salesforce/ConvoMem/resolve/main/core_benchmark/evidence_questions"

benchmarks/lme_split_50_450.json

@@ -0,0 +1,508 @@
+{
+  "dev": [
+    "cc06de0d",
+    "f9e8c073",
+    "b320f3f8",
+    "a89d7624",
+    "311778f1",
+    "gpt4_59c863d7",
+    "bbf86515",
+    "099778bb",
+    "e831120c",
+    "dcfa8644",
+    "8fb83627",
+    "e66b632c",
+    "gpt4_7fce9456",
+    "55241a1f",
+    "352ab8bd",
+    "f4f1d8a4",
+    "830ce83f",
+    "2311e44b",
+    "09ba9854",
+    "gpt4_a1b77f9c",
+    "07741c45",
+    "gpt4_70e84552",
+    "b46e15ee",
+    "6071bd76",
+    "6f9b354f",
+    "1d4da289",
+    "gpt4_8279ba02",
+    "6456829e_abs",
+    "0db4c65d",
+    "d6062bb9",
+    "60bf93ed_abs",
+    "d3ab962e",
+    "87f22b4a",
+    "e01b8e2f",
+    "gpt4_7ddcf75f",
+    "8ebdbe50",
+    "26bdc477",
+    "29f2956b_abs",
+    "2311e44b_abs",
+    "75f70248",
+    "852ce960",
+    "f0e564bc",
+    "fca70973",
+    "3c1045c8",
+    "18bc8abd",
+    "afdc33df",
+    "54026fce",
+    "b9cfe692",
+    "6456829e",
+    "e6041065"
+  ],
+  "held_out": [
+    "gpt4_15e38248",
+    "gpt4_2ba83207",
+    "2133c1b5_abs",
+    "gpt4_8279ba03",
+    "76d63226",
+    "1192316e",
+    "gpt4_fa19884d",
+    "gpt4_372c3eed_abs",
+    "1a8a66a6",
+    "gpt4_fe651585",
+    "e25c3b8d",
+    "945e3d21",
+    "86b68151",
+    "1c0ddc50",
+    "1e043500",
+    "d682f1a2",
+    "gpt4_b5700ca0",
+    "91b15a6e",
+    "ce6d2d27",
+    "f523d9fe",
+    "7024f17c",
+    "8752c811",
+    "gpt4_f420262d",
+    "d01c6aa8",
+    "4b24c848",
+    "7e974930",
+    "3fdac837",
+    "gpt4_b4a80587",
+    "c18a7dc8",
+    "80ec1f4f_abs",
+    "7527f7e2",
+    "6ade9755",
+    "89941a94",
+    "gpt4_1d80365e",
+    "2133c1b5",
+    "06db6396",
+    "gpt4_88806d6e",
+    "88432d0a",
+    "3ba21379",
+    "0862e8bf",
+    "aae3761f",
+    "5025383b",
+    "gpt4_e061b84f",
+    "73d42213",
+    "4bc144e2",
+    "gpt4_5501fe77",
+    "00ca467f",
+    "dfde3500",
+    "01493427",
+    "b6025781",
+    "a96c20ee_abs",
+    "982b5123_abs",
+    "gpt4_fa19884c",
+    "gpt4_1a1dc16d",
+    "28dc39ac",
+    "gpt4_2d58bcd6",
+    "51c32626",
+    "c4ea545c",
+    "1da05512",
+    "gpt4_385a5000",
+    "577d4d32",
+    "72e3ee87",
+    "f4f1d8a4_abs",
+    "9d25d4e0",
+    "b29f3365",
+    "b759caee",
+    "10e09553",
+    "1d4e3b97",
+    "d52b4f67",
+    "gpt4_e072b769",
+    "58ef2f1c",
+    "6e984301",
+    "41275add",
+    "gpt4_59149c77",
+    "2ebe6c90",
+    "1cea1afa",
+    "gpt4_1e4a8aec",
+    "6c49646a",
+    "8a2466db",
+    "gpt4_65aabe59",
+    "gpt4_93159ced",
+    "51a45a95",
+    "af8d2e46",
+    "561fabcd",
+    "370a8ff4",
+    "gpt4_d84a3211",
+    "gpt4_7a0daae1",
+    "2a1811e2",
+    "gpt4_78cf46a3",
+    "1568498a",
+    "6b7dfb22",
+    "6ae235be",
+    "bc8a6e93_abs",
+    "681a1674",
+    "06878be2",
+    "1a1907b4",
+    "0e4e4c46",
+    "gpt4_85da3956",
+    "gpt4_f420262c",
+    "2bf43736",
+    "bc149d6b",
+    "09d032c9",
+    "5c40ec5b",
+    "eac54adc",
+    "993da5e2",
+    "71a3fd6b",
+    "gpt4_0b2f1d21",
+    "ad7109d1",
+    "4c36ccef",
+    "c8c3f81d",
+    "edced276_abs",
+    "0bc8ad92",
+    "gpt4_468eb064",
+    "2ebe6c92",
+    "cc6d1ec1",
+    "4dfccbf8",
+    "95228167",
+    "ba358f49",
+    "45dc21b6",
+    "db467c8c",
+    "720133ac",
+    "67e0d0f2",
+    "cc5ded98",
+    "726462e0",
+    "4100d0a0",
+    "3a704032",
+    "gpt4_7ca326fa",
+    "ec81a493",
+    "618f13b2",
+    "58470ed2",
+    "gpt4_4fc4f797",
+    "60036106",
+    "157a136e",
+    "6222b6eb",
+    "69fee5aa",
+    "19b5f2b3_abs",
+    "gpt4_d12ceb0e",
+    "51b23612",
+    "2318644b",
+    "3fe836c9",
+    "gpt4_7de946e7",
+    "71017277",
+    "f0853d11",
+    "dc439ea3",
+    "gpt4_2f91af09",
+    "9a707b81",
+    "bc8a6e93",
+    "c14c00dd",
+    "8979f9ec",
+    "cf22b7bf",
+    "gpt4_ec93e27f",
+    "gpt4_468eb063",
+    "41698283",
+    "1de5cff2",
+    "21d02d0d",
+    "c7cf7dfd",
+    "gpt4_ab202e7f",
+    "dccbc061",
+    "078150f1",
+    "e3038f8c",
+    "gpt4_c27434e8_abs",
+    "2698e78f",
+    "031748ae_abs",
+    "gpt4_59149c78",
+    "c8f1aeed",
+    "184da446",
+    "gpt4_b5700ca9",
+    "89527b6b",
+    "0977f2af",
+    "853b0a1d",
+    "a346bb18",
+    "3249768e",
+    "gpt4_2f8be40d",
+    "gpt4_93159ced_abs",
+    "eeda8a6d",
+    "7a8d0b71",
+    "95bcc1c8",
+    "gpt4_2487a7cb",
+    "85fa3a3f",
+    "7e00a6cb",
+    "e3fc4d6e",
+    "59524333",
+    "37f165cf",
+    "0ddfec37",
+    "60bf93ed",
+    "d7c942c3",
+    "80ec1f4f",
+    "ceb54acb",
+    "9aaed6a3",
+    "gpt4_4929293a",
+    "ed4ddc30",
+    "545bd2b5",
+    "2788b940",
+    "ef9cf60a",
+    "gpt4_7f6b06db",
+    "0ea62687",
+    "3d86fd0a",
+    "3e321797",
+    "d24813b1",
+    "38146c39",
+    "efc3f7c2",
+    "7401057b",
+    "5809eb10",
+    "28bcfaac",
+    "1903aded",
+    "gpt4_194be4b3",
+    "gpt4_e414231f",
+    "0ddfec37_abs",
+    "c2ac3c61",
+    "gpt4_4ef30696",
+    "1f2b8d4f",
+    "0f05491a",
+    "8550ddae",
+    "8077ef71",
+    "b86304ba",
+    "e61a7584",
+    "8cf51dda",
+    "gpt4_2f584639",
+    "08e075c7",
+    "5d3d2817",
+    "7405e8b1",
+    "a3045048",
+    "gpt4_731e37d7",
+    "c8090214_abs",
+    "36580ce8",
+    "ba358f49_abs",
+    "gpt4_d6585ce8",
+    "e56a43b9",
+    "2c63a862",
+    "gpt4_5438fa52",
+    "07b6f563",
+    "gpt4_31ff4165",
+    "0bb5a684",
+    "71315a70",
+    "gpt4_cd90e484",
+    "gpt4_8c8961ae",
+    "gpt4_fe651585_abs",
+    "36b9f61e",
+    "gpt4_b0863698",
+    "gpt4_1d4ab0c9",
+    "15745da0_abs",
+    "0862e8bf_abs",
+    "bcbe585f",
+    "a2f3aa27",
+    "gpt4_6dc9b45b",
+    "ccb36322",
+    "f685340e",
+    "9ea5eabc",
+    "gpt4_372c3eed",
+    "37d43f65",
+    "bf659f65",
+    "b0479f84",
+    "gpt4_213fd887",
+    "e4e14d04",
+    "f8c5f88b",
+    "gpt4_18c2b244",
+    "a11281a2",
+    "gpt4_2655b836",
+    "e47becba",
+    "gpt4_74aed68e",
+    "gpt4_af6db32f",
+    "6cb6f249",
+    "77eafa52",
+    "gpt4_93f6379c",
+    "e8a79c70",
+    "7a87bd0c",
+    "gpt4_6ed717ea",
+    "d6233ab6",
+    "c19f7a0b",
+    "gpt4_61e13b3c",
+    "d23cf73b",
+    "gpt4_1e4a8aeb",
+    "ba61f0b9",
+    "118b2229",
+    "488d3006",
+    "c4a1ceb8",
+    "8e91e7d9",
+    "42ec0761",
+    "65240037",
+    "fea54f57",
+    "c8090214",
+    "b01defab",
+    "6aeb4375_abs",
+    "faba32e5",
+    "c5e8278d",
+    "gpt4_e414231e",
+    "eeda8a6d_abs",
+    "gpt4_8e165409",
+    "af082822",
+    "22d2cb42",
+    "92a0aa75",
+    "1c549ce4",
+    "25e5aa4f",
+    "gpt4_68e94288",
+    "4baee567",
+    "18dcd5a5",
+    "dad224aa",
+    "gpt4_f2262a51",
+    "29f2956b",
+    "21436231",
+    "19b5f2b3",
+    "gpt4_1916e0ea",
+    "gpt4_45189cb4",
+    "0a995998",
+    "b6019101",
+    "9bbe84a2",
+    "61f8c8f8",
+    "9a707b82",
+    "8cf4d046",
+    "eac54add",
+    "75832dbd",
+    "gpt4_98f46fc6",
+    "d596882b",
+    "88432d0a_abs",
+    "16c90bf4",
+    "f685340e_abs",
+    "b5ef892d",
+    "gpt4_f49edff3",
+    "gpt4_483dd43c",
+    "bb7c3b45",
+    "gpt4_7abb270c",
+    "gpt4_9a159967",
+    "07741c44",
+    "4d6b87c8",
+    "6aeb4375",
+    "gpt4_d6585ce9",
+    "60472f9c",
+    "caf9ead2",
+    "32260d93",
+    "60159905",
+    "0a34ad58",
+    "a40e080f",
+    "10d9b85a",
+    "a06e4cfe",
+    "4f54b7c9",
+    "6613b389",
+    "70b3e69b",
+    "gpt4_7bc6cf22",
+    "gpt4_0a05b494",
+    "778164c6",
+    "195a1a1b",
+    "8464fc84",
+    "b46e15ed",
+    "603deb26",
+    "eaca4986",
+    "2698e78f_abs",
+    "gpt4_21adecb5",
+    "2e6d26dc",
+    "5831f84d",
+    "08f4fc43",
+    "3f1e9474",
+    "c9f37c46",
+    "gpt4_2f56ae70",
+    "1b9b7252",
+    "35a27287",
+    "gpt4_d31cdae3",
+    "129d1232",
+    "4adc0475",
+    "27016adc",
+    "46a3abf7",
+    "9ee3ecd6",
+    "982b5123",
+    "09ba9854_abs",
+    "0e5e2d1a",
+    "e9327a54",
+    "86f00804",
+    "e982271f",
+    "7161e7e2",
+    "57f827a0",
+    "6a27ffc2",
+    "edced276",
+    "gpt4_d9af6064",
+    "75499fd8",
+    "60d45044",
+    "gpt4_70e84552_abs",
+    "2ce6a0f2",
+    "gpt4_4929293b",
+    "a1cc6108",
+    "gpt4_5dcc0aab",
+    "a3838d2b",
+    "c7dc5443",
+    "505af2f5",
+    "gpt4_68e94287",
+    "15745da0",
+    "0100672e",
+    "a82c026e",
+    "5e1b23de",
+    "71017276",
+    "89941a93",
+    "6b168ec8",
+    "affe2881",
+    "0edc2aef",
+    "gpt4_2312f94c",
+    "a4996e51",
+    "c6853660",
+    "ef66a6e5",
+    "8a137a7f",
+    "a96c20ee",
+    "fca762bc",
+    "ac031881",
+    "d905b33f",
+    "e493bb7c",
+    "a9f6b44c",
+    "dd2973ad",
+    "8aef76bc",
+    "f35224e0",
+    "8b9d4367",
+    "gpt4_c27434e8",
+    "gpt4_a56e767c",
+    "eace081b",
+    "5a4f22c0",
+    "58bf7951",
+    "c4f10528",
+    "50635ada",
+    "06f04340",
+    "0bc8ad93",
+    "e5ba910e_abs",
+    "5a7937c8",
+    "a3332713",
+    "4388e9dd",
+    "8c18457d",
+    "gpt4_2c50253f",
+    "6a1eabeb",
+    "b3c15d39",
+    "gpt4_e061b84g",
+    "3b6f954b",
+    "gpt4_76048e76",
+    "4dfccbf7",
+    "2b8f3739",
+    "d851d5ba",
+    "4fd1909e",
+    "94f70d80",
+    "66f24dbb",
+    "a08a253f",
+    "6e984302",
+    "001be529",
+    "gpt4_a2d1d1f6",
+    "cc539528",
+    "e48988bc",
+    "gpt4_4cd9eba1",
+    "8e9d538c",
+    "a1eacc2a",
+    "6d550036",
+    "gpt4_e05b82a6",
+    "81507db6",
+    "caf03d32",
+    "031748ae",
+    "c960da58",
+    "1faac195",
+    "gpt4_4edbafa2"
+  ],
+  "seed": 42,
+  "dev_size": 50
+}

benchmarks/locomo_bench.py

@@ -510,11 +510,20 @@ def palace_assign_rooms(sessions, sample_id, api_key, cache, model="claude-haiku
 
 
 def llm_rerank_locomo(
-    question, retrieved_ids, retrieved_docs, api_key, top_k=10, model="claude-sonnet-4-6"
+    question,
+    retrieved_ids,
+    retrieved_docs,
+    api_key,
+    top_k=10,
+    model="claude-sonnet-4-6",
+    backend="anthropic",
+    base_url="",
 ):
     """
     Ask LLM to pick the single most relevant document for this question.
     Returns reordered retrieved_ids with the best candidate first.
+
+    Supports backend="anthropic" (default) or "ollama" (OpenAI-compat endpoint).
     """
     candidates = retrieved_ids[:top_k]
     candidate_docs = retrieved_docs[:top_k]
@@ -522,7 +531,6 @@ def llm_rerank_locomo(
     if len(candidates) <= 1:
         return retrieved_ids
 
-    # Build numbered list of candidates
     lines = []
     for i, (cid, doc) in enumerate(zip(candidates, candidate_docs), 1):
         snippet = doc[:300].replace("\n", " ")
@@ -534,35 +542,51 @@ def llm_rerank_locomo(
         f"Reply with just the number (1-{len(candidates)}).\n\n" + "\n".join(lines)
     )
 
-    payload = json.dumps(
-        {
-            "model": model,
-            "max_tokens": 8,
-            "messages": [{"role": "user", "content": prompt}],
-        }
-    ).encode("utf-8")
-
-    req = urllib.request.Request(
-        "https://api.anthropic.com/v1/messages",
-        data=payload,
-        headers={
+    if backend == "ollama":
+        url = (base_url or "http://localhost:11434").rstrip("/") + "/v1/chat/completions"
+        payload = json.dumps(
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": prompt}],
+                "max_tokens": 1024,
+                "temperature": 0.0,
+            }
+        ).encode("utf-8")
+        headers = {"content-type": "application/json"}
+        if api_key:
+            headers["authorization"] = f"Bearer {api_key}"
+    else:
+        url = "https://api.anthropic.com/v1/messages"
+        payload = json.dumps(
+            {
+                "model": model,
+                "max_tokens": 8,
+                "messages": [{"role": "user", "content": prompt}],
+            }
+        ).encode("utf-8")
+        headers = {
             "x-api-key": api_key,
             "anthropic-version": "2023-06-01",
             "content-type": "application/json",
-        },
-        method="POST",
-    )
+        }
+
+    req = urllib.request.Request(url, data=payload, headers=headers, method="POST")
 
     import socket as _socket
 
     for _attempt in range(3):
         try:
-            with urllib.request.urlopen(req, timeout=30) as resp:
+            with urllib.request.urlopen(req, timeout=120 if backend == "ollama" else 30) as resp:
                 result = json.loads(resp.read())
-            raw = result["content"][0]["text"].strip()
-            m = re.search(r"\b(\d+)\b", raw)
+            if backend == "ollama":
+                msg = result["choices"][0]["message"]
+                raw = (msg.get("content") or "").strip() or (msg.get("reasoning") or "").strip()
+            else:
+                raw = result["content"][0]["text"].strip()
+            # Take LAST integer — reasoning models often count candidates first
+            m = re.search(r"\b(\d+)\b", raw[::-1])
             if m:
-                pick = int(m.group(1))
+                pick = int(m.group(1)[::-1])
                 if 1 <= pick <= len(candidates):
                     chosen_id = candidates[pick - 1]
                     reordered = [chosen_id] + [cid for cid in retrieved_ids if cid != chosen_id]
@@ -580,29 +604,12 @@ def llm_rerank_locomo(
 
 
 def _load_api_key(key_arg):
+    """Load API key from --llm-key arg or ANTHROPIC_API_KEY env var."""
     if key_arg:
         return key_arg
     env_key = os.environ.get("ANTHROPIC_API_KEY", "")
     if env_key:
         return env_key
-    keys_path = os.path.expanduser("~/.config/lu/keys.json")
-    if os.path.exists(keys_path):
-        try:
-            with open(keys_path) as f:
-                keys = json.load(f)
-            for name in ("lu_key", "anthropic_milla", "anthropic_claude_code_main"):
-                val = keys.get(name, "")
-                if isinstance(val, str) and val.startswith("sk-ant-"):
-                    return val
-            for section in ("anthropic", "anthropic_milla", "anthropic_claude_code_main"):
-                sec = keys.get(section, {})
-                if isinstance(sec, dict):
-                    for subkey in ("lu_key", "key", "api_key"):
-                        val = sec.get(subkey, "")
-                        if isinstance(val, str) and val.startswith("sk-ant-"):
-                            return val
-        except Exception:
-            pass
     return ""
 
 
@@ -625,6 +632,8 @@ def run_benchmark(
     palace_cache_file=None,
     palace_model="claude-haiku-4-5-20251001",
     embed_model="default",
+    llm_backend="anthropic",
+    llm_base_url="",
 ):
     """Run LoCoMo retrieval benchmark."""
     with open(data_file) as f:
@@ -636,8 +645,12 @@ def run_benchmark(
     api_key = ""
     if llm_rerank_enabled or mode == "palace":
         api_key = _load_api_key(llm_key)
-        if not api_key:
-            print(f"ERROR: --mode {mode} requires an API key (--llm-key or ANTHROPIC_API_KEY).")
+        # Ollama backend doesn't require an Anthropic key. Palace mode still does
+        # (it uses Anthropic for room-assignment indexing) — so only relax the
+        # requirement when rerank is the ONLY llm use and backend is ollama.
+        needs_key = mode == "palace" or (llm_rerank_enabled and llm_backend == "anthropic")
+        if needs_key and not api_key:
+            print(f"ERROR: --mode {mode} / --llm-rerank (anthropic) requires an API key.")
             sys.exit(1)
 
     # Palace mode: load or create room assignment cache
@@ -905,6 +918,8 @@ def run_benchmark(
                         api_key,
                         top_k=rerank_pool,
                         model=llm_model,
+                        backend=llm_backend,
+                        base_url=llm_base_url,
                     )
 
                 # Compute recall
@@ -1030,6 +1045,18 @@ if __name__ == "__main__":
         help="Model for LLM rerank (default: claude-sonnet-4-6)",
     )
     parser.add_argument("--llm-key", default="", help="API key (or set ANTHROPIC_API_KEY env var)")
+    parser.add_argument(
+        "--llm-backend",
+        choices=["anthropic", "ollama"],
+        default="anthropic",
+        help="Which API for --llm-rerank. 'anthropic' (default) or 'ollama' "
+        "(OpenAI-compat /v1/chat/completions — works for local + Ollama Cloud).",
+    )
+    parser.add_argument(
+        "--llm-base-url",
+        default="",
+        help="Override base URL for --llm-backend ollama. Default: http://localhost:11434.",
+    )
     parser.add_argument(
         "--hybrid-weight",
         type=float,
@@ -1066,4 +1093,6 @@ if __name__ == "__main__":
         palace_cache_file=args.palace_cache,
         palace_model=args.palace_model,
         embed_model=args.embed_model,
+        llm_backend=args.llm_backend,
+        llm_base_url=args.llm_base_url,
     )

benchmarks/longmemeval_bench.py

@@ -2763,7 +2763,15 @@ def build_palace_and_retrieve_diary(
 
 
 def llm_rerank(
-    question, rankings, corpus, corpus_ids, api_key, top_k=10, model="claude-haiku-4-5-20251001"
+    question,
+    rankings,
+    corpus,
+    corpus_ids,
+    api_key,
+    top_k=10,
+    model="claude-haiku-4-5-20251001",
+    backend="anthropic",
+    base_url="",
 ):
     """
     Use an LLM to re-rank the top-k retrieved sessions.
@@ -2772,19 +2780,22 @@ def llm_rerank(
     which single session is most relevant to the question. That session
     is promoted to rank 1; the rest stay in their existing order.
 
-    This closes the gap for "preference" and jargon-dense "assistant"
-    failures where the right session is in top-10 semantically but not
-    top-5 — because the semantic gap (battery life ↔ phone hardware) is
-    too large for embeddings to bridge.
+    Supports two backends:
+      - "anthropic": hits https://api.anthropic.com/v1/messages with x-api-key.
+      - "ollama":    hits {base_url}/v1/chat/completions (OpenAI-compat) —
+                     works for local Ollama (default http://localhost:11434)
+                     and Ollama Cloud (:cloud model tags).
 
     Args:
-        question:    The benchmark question string
-        rankings:    Current ranked list of corpus indices (from any mode)
-        corpus:      List of document strings
-        corpus_ids:  List of corpus IDs (parallel to corpus)
-        api_key:     Anthropic API key string
-        top_k:       How many top sessions to send to LLM (default: 10)
-        model:       Claude model ID for reranking (default: haiku)
+        question:   The benchmark question string
+        rankings:   Current ranked list of corpus indices (from any mode)
+        corpus:     List of document strings
+        corpus_ids: List of corpus IDs (parallel to corpus)
+        api_key:    Anthropic API key (only required for backend="anthropic")
+        top_k:      How many top sessions to send to LLM (default: 10)
+        model:      Model id (Claude model for anthropic, e.g. "minimax-m2.7:cloud" for ollama)
+        backend:    "anthropic" or "ollama"
+        base_url:   Override base URL (ollama default: http://localhost:11434)
 
     Returns:
         Reordered rankings list with LLM's best pick promoted to rank 1.
@@ -2796,7 +2807,6 @@ def llm_rerank(
     if not candidates:
         return rankings
 
-    # Format sessions for the prompt — first 500 chars each, labelled 1..N
     session_blocks = []
     for rank, idx in enumerate(candidates):
         text = corpus[idx][:500].replace("\n", " ").strip()
@@ -2813,80 +2823,79 @@ def llm_rerank(
         f"Most relevant session number:"
     )
 
-    payload = json.dumps(
-        {
-            "model": model,
-            "max_tokens": 8,
-            "messages": [{"role": "user", "content": prompt}],
-        }
-    ).encode("utf-8")
-
-    req = urllib.request.Request(
-        "https://api.anthropic.com/v1/messages",
-        data=payload,
-        headers={
+    if backend == "ollama":
+        url = (base_url or "http://localhost:11434").rstrip("/") + "/v1/chat/completions"
+        payload = json.dumps(
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": prompt}],
+                "max_tokens": 1024,
+                "temperature": 0.0,
+            }
+        ).encode("utf-8")
+        headers = {"content-type": "application/json"}
+        if api_key:
+            headers["authorization"] = f"Bearer {api_key}"
+    else:
+        url = "https://api.anthropic.com/v1/messages"
+        payload = json.dumps(
+            {
+                "model": model,
+                "max_tokens": 8,
+                "messages": [{"role": "user", "content": prompt}],
+            }
+        ).encode("utf-8")
+        headers = {
             "x-api-key": api_key,
             "anthropic-version": "2023-06-01",
             "content-type": "application/json",
-        },
-        method="POST",
-    )
+        }
+
+    req = urllib.request.Request(url, data=payload, headers=headers, method="POST")
 
     import socket as _socket
 
     for _attempt in range(3):
         try:
-            with urllib.request.urlopen(req, timeout=20) as resp:
+            with urllib.request.urlopen(req, timeout=120 if backend == "ollama" else 20) as resp:
                 result = json.loads(resp.read())
-            raw = result["content"][0]["text"].strip()
-            # Parse just the first integer from Haiku's response
-            m = re.search(r"\b(\d+)\b", raw)
+            if backend == "ollama":
+                msg = result["choices"][0]["message"]
+                # Reasoning models (e.g. minimax-m2.7) may emit final answer in "content"
+                # or embed it in "reasoning". Try content first, fall back to reasoning.
+                raw = (msg.get("content") or "").strip()
+                if not raw:
+                    raw = (msg.get("reasoning") or "").strip()
+            else:
+                raw = result["content"][0]["text"].strip()
+            m = re.search(
+                r"\b(\d+)\b", raw[::-1]
+            )  # take LAST integer (rerank models often reason first)
             if m:
-                pick = int(m.group(1))
+                pick = int(m.group(1)[::-1])
                 if 1 <= pick <= len(candidates):
                     chosen_idx = candidates[pick - 1]
                     reordered = [chosen_idx] + [i for i in rankings if i != chosen_idx]
                     return reordered
-            break  # Got a response, even if unparseable — don't retry
+            break
         except (_socket.timeout, TimeoutError):
             if _attempt < 2:
                 import time as _time
 
-                _time.sleep(3)  # brief pause then retry
-            # else fall through to return rankings
+                _time.sleep(3)
         except (urllib.error.URLError, KeyError, ValueError, IndexError, OSError):
-            break  # Non-timeout error — fall back immediately
+            break
 
     return rankings
 
 
 def _load_api_key(key_arg):
-    """Load API key from --llm-key arg, env var, or ~/.config/lu/keys.json."""
+    """Load API key from --llm-key arg or ANTHROPIC_API_KEY env var."""
     if key_arg:
         return key_arg
     env_key = os.environ.get("ANTHROPIC_API_KEY", "")
     if env_key:
         return env_key
-    keys_path = os.path.expanduser("~/.config/lu/keys.json")
-    if os.path.exists(keys_path):
-        try:
-            with open(keys_path) as f:
-                keys = json.load(f)
-            # Flat string keys
-            for name in ("lu_key", "anthropic_milla", "anthropic_claude_code_main"):
-                val = keys.get(name, "")
-                if isinstance(val, str) and val.startswith("sk-ant-"):
-                    return val
-            # Nested dict: keys["anthropic"]["lu_key"]
-            for section in ("anthropic", "anthropic_milla", "anthropic_claude_code_main"):
-                sec = keys.get(section, {})
-                if isinstance(sec, dict):
-                    for subkey in ("lu_key", "key", "api_key"):
-                        val = sec.get(subkey, "")
-                        if isinstance(val, str) and val.startswith("sk-ant-"):
-                            return val
-        except Exception:
-            pass
     return ""
 
 
@@ -2939,6 +2948,8 @@ def run_benchmark(
     skip_precompute=False,
     split_file=None,
     split_subset=None,
+    llm_backend="anthropic",
+    llm_base_url="",
 ):
     """Run the full benchmark.
 
@@ -2967,11 +2978,14 @@ def run_benchmark(
     api_key = ""
     if llm_rerank_enabled or mode == "diary":
         api_key = _load_api_key(llm_key)
-        if not api_key:
+        # Ollama backend doesn't require an Anthropic API key; a local/cloud Ollama
+        # daemon with the requested model pulled is enough. Diary mode is always anthropic.
+        needs_key = (llm_backend == "anthropic") or (mode == "diary")
+        if needs_key and not api_key:
             print(
-                "ERROR: --llm-rerank / --mode diary requires an API key. "
-                "Set ANTHROPIC_API_KEY, use --llm-key, "
-                "or store in ~/.config/lu/keys.json as 'lu_key'."
+                "ERROR: --llm-rerank (anthropic backend) / --mode diary requires an API key. "
+                "Set ANTHROPIC_API_KEY or use --llm-key. For ollama backend, pass "
+                "--llm-backend ollama."
             )
             sys.exit(1)
 
@@ -3121,7 +3135,15 @@ def run_benchmark(
         if llm_rerank_enabled:
             rerank_pool = 20 if mode in ("hybrid_v3", "hybrid_v4", "palace") else 10
             rankings = llm_rerank(
-                question, rankings, corpus, corpus_ids, api_key, top_k=rerank_pool, model=llm_model
+                question,
+                rankings,
+                corpus,
+                corpus_ids,
+                api_key,
+                top_k=rerank_pool,
+                model=llm_model,
+                backend=llm_backend,
+                base_url=llm_base_url,
             )
 
         # Evaluate at session level
@@ -3290,15 +3312,28 @@ if __name__ == "__main__":
     parser.add_argument(
         "--llm-key",
         default="",
-        help="Anthropic API key for LLM re-ranking. Falls back to ANTHROPIC_API_KEY "
-        "env var or ~/.config/lu/keys.json 'lu_key' field if not provided.",
+        help="Anthropic API key for LLM re-ranking. Falls back to ANTHROPIC_API_KEY env var.",
     )
     parser.add_argument(
         "--llm-model",
         default="claude-haiku-4-5-20251001",
         help="Model for LLM re-ranking and diary ingest "
         "(default: claude-haiku-4-5-20251001). "
-        "Use 'claude-sonnet-4-6' for Sonnet comparison.",
+        "Use 'claude-sonnet-4-6' for Sonnet comparison. "
+        "With --llm-backend ollama, use an Ollama model tag like 'minimax-m2.7:cloud'.",
+    )
+    parser.add_argument(
+        "--llm-backend",
+        choices=["anthropic", "ollama"],
+        default="anthropic",
+        help="Which API to hit for --llm-rerank. 'anthropic' (default) uses Anthropic's "
+        "/v1/messages endpoint. 'ollama' uses Ollama's OpenAI-compatible "
+        "/v1/chat/completions endpoint (works with local Ollama and Ollama Cloud).",
+    )
+    parser.add_argument(
+        "--llm-base-url",
+        default="",
+        help="Override base URL for --llm-backend ollama. Defaults to http://localhost:11434.",
     )
     parser.add_argument(
         "--diary-cache",
@@ -3402,4 +3437,6 @@ if __name__ == "__main__":
         args.skip_precompute,
         split_file=args.split_file,
         split_subset=split_subset,
+        llm_backend=args.llm_backend,
+        llm_base_url=args.llm_base_url,
     )

benchmarks/mine_bench.py

@@ -0,0 +1,301 @@
+"""Mining throughput benchmark: per-chunk vs batched upsert, CPU vs GPU.
+
+Compares the legacy per-chunk ``add_drawer`` loop against the batched
+``collection.upsert`` path introduced in the "batched upsert + GPU" PR.
+Runs both paths on an identical seeded synthetic corpus, reports
+wall-clock time + drawers/sec, and prints a markdown table suitable
+for pasting into a PR description.
+
+Usage
+-----
+
+    # CPU (whatever onnxruntime is installed — CPU if you don't have
+    # onnxruntime-gpu):
+    uv run python benchmarks/mine_bench.py
+
+    # GPU (NVIDIA):
+    uv venv /tmp/gpu && source /tmp/gpu/bin/activate
+    uv pip install -e '.[gpu]' 'nvidia-cudnn-cu12>=9,<10' \\
+        'nvidia-cuda-runtime-cu12' 'nvidia-cublas-cu12'
+    export LD_LIBRARY_PATH=$(python -c "import nvidia.cudnn, os; \\
+        print(os.path.dirname(nvidia.cudnn.__file__)+'/lib')"):$LD_LIBRARY_PATH
+    MEMPALACE_EMBEDDING_DEVICE=cuda python benchmarks/mine_bench.py
+
+Flags
+-----
+
+    --device cpu|cuda|coreml|dml|auto   Override MEMPALACE_EMBEDDING_DEVICE
+    --scenarios small,medium,large      Which scenarios to run
+    --seed 42                           RNG seed for reproducibility
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import os
+import random
+import shutil
+import string
+import sys
+import tempfile
+import time
+from datetime import datetime
+from pathlib import Path
+
+
+def build_corpus(dest: Path, n_files: int, paragraphs_per_file: int, seed: int) -> None:
+    """Generate ``n_files`` markdown files of random words under ``dest``."""
+    rng = random.Random(seed)
+    dest.mkdir(parents=True, exist_ok=True)
+    for i in range(n_files):
+        paragraphs = []
+        for _ in range(paragraphs_per_file):
+            words = [
+                "".join(rng.choices(string.ascii_lowercase, k=rng.randint(3, 10)))
+                for _ in range(12)
+            ]
+            paragraphs.append(" ".join(words))
+        (dest / f"doc_{i:03d}.md").write_text("\n\n".join(paragraphs))
+    (dest / "mempalace.yaml").write_text(
+        "wing: bench\n"
+        "rooms:\n"
+        "  - name: general\n"
+        "    description: all\n"
+        "    keywords: [general]\n"
+    )
+
+
+def _process_file_unbatched(filepath, project_path, collection, wing, rooms, agent, closets_col):
+    """Legacy per-chunk upsert path (pre-batching).
+
+    Reproduces the exact loop shape the miner used before this PR so the
+    comparison is apples-to-apples; only the upsert granularity differs.
+    """
+    from mempalace import miner
+    from mempalace.palace import (
+        build_closet_lines,
+        file_already_mined,
+        mine_lock,
+        purge_file_closets,
+        upsert_closet_lines,
+    )
+
+    source_file = str(filepath)
+    if file_already_mined(collection, source_file, check_mtime=True):
+        return 0, "general"
+    try:
+        content = filepath.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return 0, "general"
+    content = content.strip()
+    if len(content) < miner.MIN_CHUNK_SIZE:
+        return 0, "general"
+    room = miner.detect_room(filepath, content, rooms, project_path)
+    chunks = miner.chunk_text(content, source_file)
+
+    with mine_lock(source_file):
+        if file_already_mined(collection, source_file, check_mtime=True):
+            return 0, room
+        try:
+            collection.delete(where={"source_file": source_file})
+        except Exception:
+            pass
+        drawers_added = 0
+        for chunk in chunks:
+            miner.add_drawer(
+                collection=collection,
+                wing=wing,
+                room=room,
+                content=chunk["content"],
+                source_file=source_file,
+                chunk_index=chunk["chunk_index"],
+                agent=agent,
+            )
+            drawers_added += 1
+        if closets_col and drawers_added > 0:
+            drawer_ids = [
+                f"drawer_{wing}_{room}_"
+                f"{hashlib.sha256((source_file + str(c['chunk_index'])).encode()).hexdigest()[:24]}"
+                for c in chunks
+            ]
+            closet_lines = build_closet_lines(source_file, drawer_ids, content, wing, room)
+            closet_id_base = (
+                f"closet_{wing}_{room}_"
+                f"{hashlib.sha256(source_file.encode()).hexdigest()[:24]}"
+            )
+            closet_meta = {
+                "wing": wing,
+                "room": room,
+                "source_file": source_file,
+                "drawer_count": drawers_added,
+                "filed_at": datetime.now().isoformat(),
+                "normalize_version": miner.NORMALIZE_VERSION,
+            }
+            purge_file_closets(closets_col, source_file)
+            upsert_closet_lines(closets_col, closet_id_base, closet_lines, closet_meta)
+    return drawers_added, room
+
+
+def mine_once(project_dir: str, palace_path: str, batched: bool) -> tuple[int, float]:
+    """Mine a project dir with either the batched (new) or per-chunk (old) path."""
+    from mempalace import miner
+    from mempalace.miner import load_config, scan_project
+    from mempalace.palace import get_closets_collection, get_collection
+
+    project_path = Path(project_dir).resolve()
+    config = load_config(project_dir)
+    wing = config["wing"]
+    rooms = config.get("rooms", [])
+    files = scan_project(project_dir)
+    collection = get_collection(palace_path)
+    closets = get_closets_collection(palace_path)
+
+    total = 0
+    t0 = time.perf_counter()
+    for filepath in files:
+        if batched:
+            drawers, _ = miner.process_file(
+                filepath=filepath,
+                project_path=project_path,
+                collection=collection,
+                wing=wing,
+                rooms=rooms,
+                agent="bench",
+                dry_run=False,
+                closets_col=closets,
+            )
+        else:
+            drawers, _ = _process_file_unbatched(
+                filepath, project_path, collection, wing, rooms, "bench", closets
+            )
+        total += drawers
+    return total, time.perf_counter() - t0
+
+
+def _reset_backend_caches() -> None:
+    """Drop the in-process client cache so each run pays cold-open cost equally."""
+    from mempalace.palace import _DEFAULT_BACKEND
+
+    _DEFAULT_BACKEND._clients.clear()
+    _DEFAULT_BACKEND._freshness.clear()
+
+
+def run_scenario(label: str, n_files: int, paragraphs_per_file: int, seed: int) -> dict:
+    """Run one scenario under both code paths and return a result dict."""
+    print(f"\n=== {label}: {n_files} files × {paragraphs_per_file} paragraphs ===")
+    results = {}
+    for mode in ("unbatched", "batched"):
+        tmp = Path(tempfile.mkdtemp(prefix=f"mp_{mode}_"))
+        try:
+            proj = tmp / "proj"
+            palace = tmp / "palace"
+            build_corpus(proj, n_files, paragraphs_per_file, seed=seed)
+            _reset_backend_caches()
+            drawers, dt = mine_once(str(proj), str(palace), batched=(mode == "batched"))
+            rate = drawers / dt if dt > 0 else 0.0
+            results[mode] = (drawers, dt, rate)
+            print(f"  {mode:10} {drawers:5} drawers in {dt:6.2f}s  →  {rate:7.1f} drawers/sec")
+        finally:
+            shutil.rmtree(tmp, ignore_errors=True)
+
+    _, t_u, r_u = results["unbatched"]
+    d_b, t_b, r_b = results["batched"]
+    speedup = t_u / t_b if t_b > 0 else 0.0
+    print(f"  speedup:   {speedup:.2f}× ({t_u:.2f}s → {t_b:.2f}s)")
+    return {
+        "label": label,
+        "n_files": n_files,
+        "paragraphs": paragraphs_per_file,
+        "drawers": d_b,
+        "unbatched_time": t_u,
+        "unbatched_rate": r_u,
+        "batched_time": t_b,
+        "batched_rate": r_b,
+        "speedup": speedup,
+    }
+
+
+SCENARIOS = {
+    "small":  ("Small files (~50 paragraphs)",  10, 50),
+    "medium": ("Medium files (~200 paragraphs)", 20, 200),
+    "large":  ("Large files (~500 paragraphs)",  10, 500),
+}
+
+
+def _env_summary(device_label: str) -> list[str]:
+    """Short hardware + version lines included with the printed table."""
+    import platform
+
+    try:
+        import chromadb
+
+        chromadb_v = chromadb.__version__
+    except Exception:
+        chromadb_v = "?"
+    try:
+        import onnxruntime as ort
+
+        ort_v = ort.__version__
+        providers = ",".join(p.replace("ExecutionProvider", "") for p in ort.get_available_providers())
+    except Exception:
+        ort_v = "?"
+        providers = "?"
+
+    return [
+        f"device: **{device_label}** (onnxruntime {ort_v}, providers={providers})",
+        f"chromadb {chromadb_v} · python {sys.version.split()[0]} · {platform.platform()}",
+    ]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0])
+    parser.add_argument(
+        "--device",
+        default=None,
+        help="Override MEMPALACE_EMBEDDING_DEVICE (cpu|cuda|coreml|dml|auto)",
+    )
+    parser.add_argument(
+        "--scenarios",
+        default="small,medium,large",
+        help="Comma-separated scenario names (default: all)",
+    )
+    parser.add_argument("--seed", type=int, default=42)
+    args = parser.parse_args()
+
+    if args.device:
+        os.environ["MEMPALACE_EMBEDDING_DEVICE"] = args.device
+
+    from mempalace.embedding import describe_device, get_embedding_function
+
+    device_label = describe_device()
+    print(f"Warming up ONNX model on device={device_label}...")
+    ef = get_embedding_function()
+    ef(["warmup sentence one", "warmup sentence two"])
+
+    picked = [s.strip() for s in args.scenarios.split(",") if s.strip()]
+    results = []
+    for key in picked:
+        if key not in SCENARIOS:
+            print(f"Unknown scenario {key!r}; choices: {sorted(SCENARIOS)}", file=sys.stderr)
+            sys.exit(2)
+        label, n_files, paras = SCENARIOS[key]
+        results.append(run_scenario(label, n_files, paras, args.seed))
+
+    print("\n\n## Mining benchmark\n")
+    for line in _env_summary(device_label):
+        print(line + "  ")
+    print()
+    print("| Scenario | Files | Drawers | Per-chunk (old) | Batched (new) | Speedup |")
+    print("| --- | ---: | ---: | ---: | ---: | ---: |")
+    for r in results:
+        print(
+            f"| {r['label']} | {r['n_files']} | {r['drawers']} | "
+            f"{r['unbatched_time']:.2f}s · {r['unbatched_rate']:.0f} drw/s | "
+            f"{r['batched_time']:.2f}s · {r['batched_rate']:.0f} drw/s | "
+            f"**{r['speedup']:.2f}×** |"
+        )
+
+
+if __name__ == "__main__":
+    main()

docs/CLOSETS.md

@@ -0,0 +1,88 @@
+# Closets — The Searchable Index Layer
+
+## What closets are
+
+Drawers hold your verbatim content. Closets are the index — compact pointers that tell the searcher which drawers to open.
+
+```
+CLOSET: "built auth system|Ben;Igor|→drawer_api_auth_a1b2c3"
+         ↑ topic           ↑ entities  ↑ points to this drawer
+```
+
+An agent searching "who built the auth?" hits the closet first (fast scan of short text), then opens the referenced drawer to get the full verbatim content.
+
+## Lifecycle
+
+### When are closets created?
+
+Closets are created during `mempalace mine`. For each file mined:
+1. Content is chunked into drawers (verbatim, ~800 chars each)
+2. Topics, entities, and quotes are extracted from the content
+3. A closet is created with pointer lines to those drawers
+
+### What's inside a closet?
+
+Each line is one atomic topic pointer:
+```
+topic description|entity1;entity2|→drawer_id_1,drawer_id_2
+"verbatim quote from the content"|entity1|→drawer_id_3
+```
+
+Topics are never split across closets. If adding a topic would exceed 1,500 characters, a new closet is created.
+
+### When do closets update?
+
+When a file is re-mined (content changed, or `NORMALIZE_VERSION` was bumped), the miner first deletes every closet for that source file (`purge_file_closets`) and then writes a fresh set. Stale topics from the prior mine are gone — closets are always a snapshot of the current content, never an accumulation across runs.
+
+### What about stale topics?
+
+There are no stale topics: each re-mine is a clean rebuild for that source file. If a file gets larger and produces fewer or more closets than last time, the leftover numbered closets from the larger run are still purged because the delete is done by `source_file`, not by ID.
+
+### Do closets survive palace rebuilds?
+
+Closets are stored in the `mempalace_closets` ChromaDB collection alongside `mempalace_drawers`. If you delete and rebuild the palace, closets are recreated during the next `mempalace mine`.
+
+## How search uses closets
+
+```
+Query → search mempalace_closets (fast, small documents)
+         ↓
+    top closet hits → parse `→drawer_id_a,drawer_id_b` pointers
+         ↓
+    fetch exactly those drawers from mempalace_drawers (verbatim content)
+         ↓
+    apply max_distance filter
+         ↓
+    return chunk-level results (same shape as direct search)
+```
+
+Hits carry `matched_via: "closet"` (or `"drawer"` for the fallback path) plus a `closet_preview` field showing the line that surfaced them.
+
+If no closets exist (palace created before this feature) — or all closet hits get filtered out by `max_distance` — search falls back to direct drawer search. Closets are created on next mine.
+
+> **BM25 hybrid re-rank** is on the roadmap (deferred to a follow-up PR alongside generic `LLM_*` env-var support); the current closet search ranks purely by ChromaDB cosine distance against the closet text.
+
+## Limits
+
+| Setting | Value | Reason |
+|---------|-------|--------|
+| Max closet size | 1,500 chars (`CLOSET_CHAR_LIMIT`) | Leaves buffer under ChromaDB's working limit |
+| Source content scanned | 5,000 chars (`CLOSET_EXTRACT_WINDOW`) | Caps regex extraction cost on long files; back-of-file content is currently invisible to closet extraction (tracked for follow-up) |
+| Max topics per file | 12 | Keeps closets focused |
+| Max quotes per file | 3 | Most relevant only |
+| Max entities per pointer | 5 | Top names by frequency, after stoplist filtering |
+
+## For developers
+
+Closet functions live in `mempalace/palace.py`:
+- `get_closets_collection()` — get the closets ChromaDB collection
+- `build_closet_lines()` — extract topics/entities/quotes into pointer lines
+- `upsert_closet_lines()` — write lines to closets respecting the char limit (overwrites existing IDs; does not append — call `purge_file_closets` first when re-mining)
+- `purge_file_closets()` — delete every closet for a given source file before rebuild
+- `CLOSET_CHAR_LIMIT` / `CLOSET_EXTRACT_WINDOW` — size constants
+
+The closet-first search path lives in `mempalace/searcher.py`:
+- `_extract_drawer_ids_from_closet()` — parse `→drawer_a,drawer_b` pointers out of a closet document
+- `_closet_first_hits()` — query closets, parse pointers, hydrate matching drawers, return chunk-level hits or `None` to fall back
+
+Note: only the project miner (`miner.py::process_file`) builds closets today. Conversation-mined wings (Claude Code JSONL, ChatGPT export, etc.) will keep using direct drawer search via the searcher fallback until the convo-closet PR lands.

docs/HISTORY.md

@@ -0,0 +1,144 @@
+# MemPalace — History, Corrections, and Public Notices
+
+This file is the canonical record of post-launch corrections, public notices,
+and retractions that affect MemPalace's public claims. Newest first.
+
+---
+
+## 2026-04-14 — Benchmark table rewrite (issue [#875](https://github.com/MemPalace/mempalace/issues/875))
+
+A community audit identified a category error in the public benchmark tables
+on `README.md` and `mempalaceofficial.com`: MemPalace's retrieval recall
+numbers (R@5, R@10) were listed in the same columns as competitors'
+end-to-end QA accuracy numbers. They are different metrics and are not
+comparable — a system can have 100% retrieval recall and 40% QA accuracy.
+
+The audit also found that the retracted "+34% palace boost" claim (see the
+April 7 note below) was still present in multiple surfaces despite that
+retraction, and that two competitor numbers (`Mem0 ~85%`, `Zep ~85%`) had no
+published source and did not match the metrics those projects actually
+publish.
+
+What changed in this PR:
+
+- The headline number on all surfaces is now **96.6% R@5 on LongMemEval in
+  raw mode**, independently reproduced on Linux x86_64 against the tagged
+  v3.3.0 release on 2026-04-14. Result JSONLs are committed under
+  `benchmarks/results_*.jsonl` (see PR description for the scorecard).
+- The **"100% with Haiku rerank"** claim has been removed from all public
+  comparison tables. It reproduces on our machines and with a different LLM
+  family (minimax-m2.7 via Ollama Cloud: 99.2% R@5 / 100.0% R@10 on the full
+  500-question LongMemEval set) — but the 99.4% → 100% step was developed
+  by inspecting three specific wrong answers (`benchmarks/BENCHMARKS.md` has
+  called this "teaching to the test" since February). It belongs in the
+  methodology document, not in a headline.
+- The **honest held-out number** for the hybrid pipeline — 98.4% R@5 on 450
+  questions that `hybrid_v4` was never tuned on, deterministic seed — is now
+  the comparable figure when an LLM rerank is involved.
+- The **retracted "+34% palace boost"** has been removed from
+  `README.md`, `website/concepts/the-palace.md`,
+  `website/guide/searching.md`, and `website/reference/contributing.md`.
+  Wing and room filters remain useful — they're standard metadata filters —
+  but they are not presented as a novel retrieval improvement.
+- **Competitor comparison tables** mixing retrieval recall with QA accuracy
+  have been removed from `README.md` and `website/reference/benchmarks.md`.
+  Where MemPalace can be fairly compared on the same metric, we link to the
+  cited source. Otherwise we report our own numbers and let readers draw
+  their own conclusions.
+- **Reproduction instructions** in `benchmarks/BENCHMARKS.md` and
+  `benchmarks/README.md` were pointing at a defunct branch
+  (`aya-thekeeper/mempal`); they now point at `MemPalace/mempalace`.
+- The **LoCoMo 100% R@10 with top-50 rerank** row has been removed from
+  public comparison surfaces. With per-conversation session counts of 19–32
+  and `top_k=50`, the retrieval stage returns every session in the
+  conversation by construction, so the number measures an LLM's
+  reading comprehension over the whole conversation, not retrieval.
+
+Thanks to [@dial481](https://github.com/MemPalace/mempalace/issues/875) for
+the detailed audit and to [@rohitg00](https://github.com/rohitg00) for the
+parallel write-up in Discussion #747.
+
+---
+
+## 2026-04-11 — Impostor domains and malware
+
+Several community members (issues #267, #326, #506) reported fake MemPalace
+websites distributing malware. The only official surfaces for this project
+are:
+
+- This GitHub repository: [github.com/MemPalace/mempalace](https://github.com/MemPalace/mempalace)
+- The PyPI package: [pypi.org/project/mempalace](https://pypi.org/project/mempalace/)
+- The docs site: [mempalaceofficial.com](https://mempalaceofficial.com)
+
+Any other domain — `mempalace.tech` being the one most commonly reported —
+is not ours. Never run install scripts from unofficial sites.
+
+Thanks to our community members for flagging the problem.
+
+---
+
+## 2026-04-07 — A Note from Milla & Ben
+
+> The community caught real problems in this README within hours of launch
+> and we want to address them directly.
+>
+> **What we got wrong:**
+>
+> - **The AAAK token example was incorrect.** We used a rough heuristic
+>   (`len(text)//3`) for token counts instead of an actual tokenizer. Real
+>   counts via OpenAI's tokenizer: the English example is 66 tokens, the
+>   AAAK example is 73. AAAK does not save tokens at small scales — it's
+>   designed for *repeated entities at scale*, and the README example was a
+>   bad demonstration of that. We're rewriting it.
+>
+> - **"30x lossless compression" was overstated.** AAAK is a lossy
+>   abbreviation system (entity codes, sentence truncation). Independent
+>   benchmarks show AAAK mode scores **84.2% R@5 vs raw mode's 96.6%** on
+>   LongMemEval — a 12.4 point regression. The honest framing is: AAAK is
+>   an experimental compression layer that trades fidelity for token
+>   density, and **the 96.6% headline number is from RAW mode, not AAAK**.
+>
+> - **"+34% palace boost" was misleading.** That number compares unfiltered
+>   search to wing+room metadata filtering. Metadata filtering is a
+>   standard feature of the underlying vector store, not a novel retrieval
+>   mechanism. Real and useful, but not a moat.
+>
+> - **"Contradiction detection"** exists as a separate utility
+>   (`fact_checker.py`) but is not currently wired into the knowledge graph
+>   operations as the README implied.
+>
+> - **"100% with Haiku rerank"** is real (we have the result files) but
+>   the rerank pipeline is not in the public benchmark scripts. We're
+>   adding it.
+>
+> **What's still true and reproducible:**
+>
+> - **96.6% R@5 on LongMemEval in raw mode**, on 500 questions, zero API
+>   calls — independently reproduced on M2 Ultra in under 5 minutes by
+>   [@gizmax](https://github.com/MemPalace/mempalace/issues/39).
+> - Local, free, no subscription, no cloud, no data leaving your machine.
+> - The architecture (wings, rooms, closets, drawers) is real and useful,
+>   even if it's not a magical retrieval boost.
+>
+> **What we're doing:**
+>
+> 1. Rewriting the AAAK example with real tokenizer counts and a scenario
+>    where AAAK actually demonstrates compression
+> 2. Adding `mode raw / aaak / rooms` clearly to the benchmark
+>    documentation so the trade-offs are visible
+> 3. Wiring `fact_checker.py` into the KG ops so the contradiction
+>    detection claim becomes true
+> 4. Pinning the vector store dependency to a tested range (issue #100),
+>    fixing the shell injection in hooks (#110), and addressing the macOS
+>    ARM64 segfault (#74)
+>
+> **Thank you to everyone who poked holes in this.** Brutal honest
+> criticism is exactly what makes open source work, and it's what we asked
+> for. Special thanks to
+> [@panuhorsmalahti](https://github.com/MemPalace/mempalace/issues/43),
+> [@lhl](https://github.com/MemPalace/mempalace/issues/27),
+> [@gizmax](https://github.com/MemPalace/mempalace/issues/39), and everyone
+> who filed an issue or a PR in the first 48 hours. We're listening, we're
+> fixing, and we'd rather be right than impressive.
+>
+> — *Milla Jovovich & Ben Sigman*

docs/rfcs/002-source-adapter-plugin-spec.md

@@ -0,0 +1,768 @@
+# RFC 002 — Source Adapter Plugin Specification
+
+- **Status:** Draft
+- **Tracking issue:** [#989](https://github.com/MemPalace/mempalace/issues/989)
+- **Related:** [#274](https://github.com/MemPalace/mempalace/issues/274), [#23](https://github.com/MemPalace/mempalace/pull/23), [#169](https://github.com/MemPalace/mempalace/pull/169), [#232](https://github.com/MemPalace/mempalace/pull/232), [#567](https://github.com/MemPalace/mempalace/pull/567), [#98](https://github.com/MemPalace/mempalace/pull/98), [#591](https://github.com/MemPalace/mempalace/pull/591), [#592](https://github.com/MemPalace/mempalace/pull/592), [#702](https://github.com/MemPalace/mempalace/pull/702), [#981](https://github.com/MemPalace/mempalace/issues/981), [#244](https://github.com/MemPalace/mempalace/pull/244), [#419](https://github.com/MemPalace/mempalace/pull/419), [#300](https://github.com/MemPalace/mempalace/pull/300), [#952](https://github.com/MemPalace/mempalace/pull/952), [#389](https://github.com/MemPalace/mempalace/pull/389), [#434](https://github.com/MemPalace/mempalace/pull/434)
+- **Sibling spec:** [RFC 001 — Storage Backend Plugin Specification](001-storage-backend-plugin-spec.md)
+- **Spec version:** `1.0`
+
+## Summary
+
+A formal contract for MemPalace source adapters so third parties can ship `pip install mempalace-source-<name>` packages (Cursor, OpenCode, git, Slack, Notion, email, calendar, Whisper transcripts, …) that drop into `mempalace mine` without patching core. The spec defines the adapter interface, record shape, metadata schema contract, privacy class, entry-point registration, incremental-ingest semantics, closet integration, a declared-transformation model that replaces the informal "verbatim" promise with a verifiable one, conformance tests, and the refactor of the existing file and conversation miners into first-party adapters on the same contract.
+
+RFC 001 formalized the write side (where drawers are stored). This RFC formalizes the read side (where content comes from). Both are required for MemPalace to function as a durable daemon managing heterogeneous palaces across many source types.
+
+## Motivation
+
+Six source ingesters are currently in flight, each solving the same problem a different way:
+
+| PR / Issue | Source | Mechanism |
+|---|---|---|
+| [#274](https://github.com/MemPalace/mempalace/issues/274) | Cursor | `workspaceStorage/*.vscdb` SQLite extraction |
+| [#23](https://github.com/MemPalace/mempalace/pull/23) | OpenCode | SQLite session database |
+| [#169](https://github.com/MemPalace/mempalace/pull/169) | Pi agent | JSONL session normalizer |
+| [#232](https://github.com/MemPalace/mempalace/pull/232) | Cursor (JSONL variant) | JSONL normalizer |
+| [#567](https://github.com/MemPalace/mempalace/pull/567), [#98](https://github.com/MemPalace/mempalace/pull/98) | Git | `git log` + `gh pr view` with structured diff summary |
+| [#591](https://github.com/MemPalace/mempalace/pull/591), [#592](https://github.com/MemPalace/mempalace/pull/592) | Delphi Oracle | Real-time intelligence signals |
+| [#702](https://github.com/MemPalace/mempalace/pull/702) | Cursor + factory.ai | Combined session miners |
+
+Plus three ingesters already grafted into core:
+
+- `mempalace/miner.py` — filesystem project miner, fixed char-window chunking, keyword hall routing
+- `mempalace/convo_miner.py` — chat transcript miner with exchange-pair chunking
+- `mempalace/normalize.py` — format detection for four chat-export shapes (Claude Code JSONL, Codex JSONL, Claude.ai / ChatGPT / Slack JSON)
+
+Plus one open proposal for a different ingest semantic:
+
+- [#981](https://github.com/MemPalace/mempalace/issues/981) — path-level descriptions: mine metadata-as-content instead of raw bytes for matched paths. This is a legitimate third ingest mode (alongside chunked-content and whole-record) that the current architecture has no home for.
+
+Each contributor has reinvented source discovery, source-item identity, incremental-ingest bookkeeping, metadata shape, and chunking strategy. Format detection for new chat exports lands in `normalize.py` as one more branch in an `if` chain. There is no shared abstraction, no conformance suite, and no contract new adapter authors can build against.
+
+This is the same situation RFC 001 addresses for storage backends: a pattern that emerged organically, now needs a specification so the community can contribute cleanly and enterprises can build against a stable surface.
+
+### Why this matters beyond developer tooling
+
+The adapter pattern is source-agnostic. What has so far shown up as "Cursor transcripts" and "git commits" generalizes to:
+
+- **Knowledge work** — Notion, Obsidian, Logseq, Google Docs, iA Writer, Zettlr
+- **Communications** — Slack, Discord, Teams, Signal backups, mbox/eml email, iMessage
+- **Research** — arXiv PDFs, Zotero libraries, bookmarked articles, Kindle highlights, web archives
+- **Creator workflows** — YouTube captions, podcast transcripts (Whisper/Deepgram), Descript projects
+- **Regulated domains** — medical records, legal filings, financial statements (all gated on §6 privacy class)
+
+Enterprises key on their own domain metadata — `repo/PR/SHA` for engineering, `patient/encounter/CPT` for healthcare, `case/docket/jurisdiction` for legal. The schema lives in the adapter; the content lives in the drawer. This is how structured-data use cases are served without violating the byte-preservation commitments adapters make.
+
+## Goals
+
+1. A source adapter ships as a standalone Python package; `pip install mempalace-source-<name>` is sufficient to use it.
+2. `mempalace mine` and the MCP mine tool are source-agnostic — all extraction goes through registered adapters. No `if source_type == 'foo'` branches in core.
+3. Content transformations are **declared** (§1.4): each adapter advertises the set of transformations it applies to source bytes. Byte-preserving adapters declare the empty set. Consumers can programmatically determine what happened to their data.
+4. Incremental ingest is cheap and correct: re-running mine only touches items whose source-side version changed, using the palace itself as the cursor (no sidecar).
+5. Each adapter declares a structured metadata schema. Enterprises index and filter on that schema. Core is schema-agnostic beyond the universal fields in §5.1.
+6. The existing `miner.py` and `convo_miner.py` become the first two first-party adapters on the new contract. Drawer metadata fields and field names are preserved — the spec adds fields, does not rename them.
+7. A privacy class is declarable at the adapter boundary so sensitive sources (medical, financial, personal comms) are handled with explicit policy rather than implicit trust.
+
+## Non-goals
+
+- Defining chunking. Each adapter owns its chunking strategy — tree-sitter for code, exchange-pair for chat, whole-record for a PR. Core does not impose a chunk size.
+- Defining live-stream / webhook shapes (the Delphi Oracle pattern of continuous signal ingestion). That is a separate future RFC; v1 is pull-mode.
+- Defining LLM-based structured extraction. Adapters MAY use an LLM; the spec does not mandate or standardize this.
+- Defining cross-adapter dedup. When the same content appears via two adapters (e.g., a PR body mined via `git` and as a conversation quote mined via `claude-code`), both drawers land. Deduplication policy is a separate concern handled at query time by `searcher.py`.
+- Defining closet construction. Core continues to build closets from adapter-yielded drawers (§1.7); the closet-building algorithm itself is not part of this spec.
+
+---
+
+## 1. Source adapter contract
+
+### 1.1 Required method
+
+All adapters implement `BaseSourceAdapter` with a single kwargs-only ingest method:
+
+```python
+class BaseSourceAdapter(ABC):
+    @abstractmethod
+    def ingest(
+        self,
+        *,
+        source: SourceRef,
+        palace: PalaceContext,
+    ) -> Iterator[IngestResult]:
+        """Enumerate and extract content from a source.
+
+        Yields a stream of IngestResult values. Lazy adapters yield
+        `SourceItemMetadata` ahead of the drawers for that item, so core
+        can report progress and check `is_current` before the adapter
+        commits to the fetch. Adapters with no lazy-fetch benefit may
+        interleave `SourceItemMetadata` and `DrawerRecord` items freely.
+        """
+
+    @abstractmethod
+    def describe_schema(self) -> AdapterSchema:
+        """Declare the structured metadata this adapter attaches.
+
+        Returned value is stable for a given adapter version. Enterprises
+        index on this schema; core uses it to validate adapter output.
+        """
+```
+
+The single-method `ingest()` contract was chosen over a `discover` / `extract` split. Most current ingesters have no meaningful laziness benefit (filesystem walking is cheap, transcript normalizing is cheap). Adapters that do (git-mine's `gh pr list` vs `gh pr view`; hypothetical Slack/Notion API) express laziness by yielding `SourceItemMetadata` first and deferring fetch until core confirms staleness via `is_current()`.
+
+### 1.2 Optional methods (default implementations on the ABC)
+
+```python
+def is_current(
+    self,
+    *,
+    item: SourceItemMetadata,
+    existing_metadata: dict | None,
+) -> bool:
+    """Return True if the palace already has an up-to-date copy.
+
+    Called by core after querying the palace for existing drawers with
+    matching source_file. The adapter compares its version token against
+    the stored metadata and returns True to skip extraction.
+
+    Default implementation: returns False (always re-extract). Adapters
+    advertising `supports_incremental` override this.
+    """
+    return False
+
+def source_summary(self, *, source: SourceRef) -> SourceSummary:
+    """Describe a source without extracting (e.g., 'git repo mempalace,
+    847 commits, 132 PRs'). Default: returns empty summary."""
+    return SourceSummary(description=self.name)
+
+def close(self) -> None:
+    return None
+```
+
+Core's incremental loop (pseudocode):
+
+```python
+for result in adapter.ingest(source=source, palace=ctx):
+    if isinstance(result, SourceItemMetadata):
+        existing = ctx.collection.get(where={"source_file": result.source_file}, limit=1)
+        if adapter.is_current(item=result, existing_metadata=existing):
+            ctx.skip_current_item()   # adapter stops yielding drawers for this item
+    elif isinstance(result, DrawerRecord):
+        ctx.upsert_drawer(result)
+```
+
+### 1.3 Typed records
+
+```python
+@dataclass(frozen=True)
+class SourceRef:
+    """A handle to the source a user wants to ingest.
+
+    local_path is for filesystem-rooted sources (project dir, mbox file).
+    uri is for URL-like references (github.com/org/repo, slack://workspace/channel).
+    options carries adapter-specific config (non-secret values only; §M2).
+    """
+    local_path: str | None = None
+    uri: str | None = None
+    options: dict = field(default_factory=dict)
+
+@dataclass(frozen=True)
+class SourceItemMetadata:
+    """Lightweight pointer yielded before drawers for lazy-fetch adapters."""
+    source_file: str                 # Logical identity — filesystem path, PR URI, etc.
+    version: str                     # Source-side version token (mtime, commit SHA, ETag, rev id).
+    size_hint: int | None = None     # Bytes, if known. Used for progress reporting.
+    route_hint: RouteHint | None = None
+
+@dataclass(frozen=True)
+class DrawerRecord:
+    """One drawer's worth of content plus metadata."""
+    content: str                     # Subject to §1.4 declared transformations.
+    source_file: str                 # Foreign key to SourceItemMetadata.source_file.
+    chunk_index: int = 0             # 0 for single-drawer items; 0..N-1 for chunked items.
+    metadata: dict = field(default_factory=dict)  # Flat: str/int/float/bool only. Must conform to adapter schema.
+    route_hint: RouteHint | None = None
+
+@dataclass(frozen=True)
+class RouteHint:
+    wing: str | None = None
+    room: str | None = None
+    hall: str | None = None
+
+@dataclass(frozen=True)
+class SourceSummary:
+    description: str
+    item_count: int | None = None
+
+# IngestResult is the union type adapters yield.
+IngestResult = SourceItemMetadata | DrawerRecord
+
+# PalaceContext carries collection handles, palace config, and progress hooks
+# into the adapter. Full definition in §9 (cleanup prerequisite).
+```
+
+### 1.4 Declared transformations
+
+Adapters cannot silently alter content. Every adapter declares the set of transformations it applies:
+
+```python
+class BaseSourceAdapter(ABC):
+    declared_transformations: ClassVar[frozenset[str]] = frozenset()
+```
+
+The invariant: **no transformation is applied that is not declared in this set**. Adapters declaring `frozenset()` are byte-preserving end-to-end (modulo the read, which may itself involve `utf8_replace_invalid` — see below).
+
+Reserved transformation names (v1):
+
+| Name | Meaning |
+|---|---|
+| `utf8_replace_invalid` | Undecodable bytes replaced with U+FFFD on read (equivalent to `open(..., errors="replace")`). |
+| `newline_normalize` | CRLF / CR converted to LF. |
+| `whitespace_trim` | Leading / trailing whitespace stripped at a record boundary. |
+| `whitespace_collapse_internal` | Runs of three or more blank lines collapsed to two. |
+| `line_trim` | Each line individually stripped of leading / trailing whitespace. |
+| `line_join_spaces` | Adjacent lines joined with single spaces, newlines discarded. |
+| `blank_line_drop` | Empty lines between non-empty lines dropped. |
+| `strip_tool_chrome` | System tags, hook output, tool UI chrome removed (see `normalize.strip_noise`). |
+| `tool_result_truncate` | Tool output heads/tails kept; middle replaced with a marker string. |
+| `spellcheck_user` | User turns rewritten by spellcheck. |
+| `synthesized_marker` | Adapter inserts its own strings (e.g., `[N lines omitted]`, `[registry] …`, Slack provenance footer). |
+| `speaker_role_assignment` | Multi-party speakers alternately assigned `user` / `assistant` roles (Slack). |
+| `tool_result_omitted` | Some tool outputs fully omitted from transcript (e.g., Read/Edit/Write results in `normalize._format_tool_result`). |
+
+Adapters MAY define their own transformation names for behaviors the reserved list does not cover. Third-party names SHOULD be prefixed with the adapter name to avoid collisions (e.g., `cursor.composer_ordering`).
+
+**Capability derivation:**
+- `byte_preserving` — declared_transformations is empty AND output bytes equal input bytes for any source the adapter can read. Advertised via the `byte_preserving` capability (§2.1). MUST be verified by §7.2 round-trip test.
+- `declared_lossy` — declared_transformations is non-empty. The adapter's output is reproducible from source by applying *only* the declared transformations. MUST be verified by §7.3 declared-transformation test.
+
+**Existing code mapping (for the cleanup PR):**
+
+| Module | Declared transformations |
+|---|---|
+| `filesystem` (current `miner.py`) | `utf8_replace_invalid`, `whitespace_trim` |
+| `conversations` (current `convo_miner.py` + `normalize.py`) | `utf8_replace_invalid`, `newline_normalize`, `line_trim`, `line_join_spaces`, `blank_line_drop`, `whitespace_collapse_internal`, `strip_tool_chrome`, `tool_result_truncate`, `tool_result_omitted`, `spellcheck_user`, `synthesized_marker`, `speaker_role_assignment` |
+
+The filesystem adapter is nearly byte-preserving today; the conversations adapter is extensively transformed. Both are honest after this spec lands because both are fully declared.
+
+This replaces the MISSION.md promise of "verbatim always" with a stronger one: every adapter publishes what it does to your data, and the conformance suite verifies it hasn't lied. "Verbatim" becomes a capability some adapters hold (byte_preserving), not a global claim about a lossy pipeline.
+
+### 1.5 Three ingest modes
+
+A single adapter declares one or more of three modes via a class attribute:
+
+```python
+class BaseSourceAdapter(ABC):
+    supported_modes: ClassVar[frozenset[Literal["chunked_content", "whole_record", "metadata_only"]]]
+```
+
+| Mode | Content origin |
+|---|---|
+| `chunked_content` | Source bytes, split into chunks the adapter chooses (current filesystem behavior). |
+| `whole_record` | Source bytes, one drawer per source item (e.g., PR → 1 drawer). |
+| `metadata_only` | Synthesized description of a source item (absorbs #981). The description bytes are authored by the user or adapter, not the source. Declared transformations (§1.4) do not apply — content is not derived from source bytes. |
+
+`metadata_only` resolves #981: description-mode matches a path pattern and produces one drawer whose content is the user-authored description rather than the file contents. Conformance tests (§7.2, §7.3) skip `metadata_only` records.
+
+An adapter MAY support multiple modes and select per-item; the per-item mode is recorded in `metadata["ingest_mode"]` (§5.1). This field already exists on conversation drawers (`convo_miner.py:346`) and is the only existing field whose semantics this spec extends rather than preserves.
+
+### 1.6 Chunking delegation
+
+Core does not impose chunking. `miner.py`'s 800-character sliding window is the filesystem adapter's default for unknown file types — not a contract. Adapter authors choose what makes sense:
+
+- Code files → tree-sitter function/class boundaries (future enhancement to the filesystem adapter).
+- Chat transcripts → exchange pairs (current `convo_miner.py` behavior).
+- PRs → whole-record (current `git-mine` behavior in #567).
+- PDFs → page or section.
+- Voice transcripts → speaker turn.
+
+The sole cross-adapter requirement for `chunked_content` mode: chunks for a given `source_file`, re-assembled in `chunk_index` order and accounting for declared transformations in §1.4, reproduce the adapter's internal representation of the source. The conformance suite verifies this.
+
+### 1.7 Closet integration
+
+Closets are the AAAK-compressed index layer (`palace.build_closet_lines`, `upsert_closet_lines`) that points to drawer content and enables LLM-scale scanning without reading every drawer. Closet-building is not an adapter concern:
+
+- **Core builds closets** from adapter-yielded drawers as a post-step, via the existing `palace.py` helpers. Adapters do not call these APIs.
+- **Adapters MAY emit closet hints** in drawer metadata via a flat `;`-joined string:
+  ```python
+  metadata["closet_hints"] = "decided GraphQL; migrated to Postgres; fixed PR-567"
+  ```
+  Core splits on `;` and feeds these as candidate topics alongside the content-scanned ones in `build_closet_lines`. The git adapter can hint decision-signal quotes that raw content-scanning would miss; the conversations adapter can hint section headers; the filesystem adapter has no need and omits the field.
+- **metadata_only drawers get closets too.** Core builds them from the synthesized description content the same way it builds closets for any other drawer. This is how #981's path-level descriptions become searchable.
+- **Closet purging** remains keyed on `source_file` (`purge_file_closets` in `palace.py:221`). Adapters' source_file values must be stable so purge is correct on re-ingest.
+
+Current `convo_miner.py` does not build closets for conversation drawers — an existing gap. The cleanup PR (§9) routes the conversations adapter through the same post-step closet builder as filesystem, closing the gap as a side effect.
+
+---
+
+## 2. Adapter contract
+
+### 2.1 Identity and capabilities
+
+```python
+class BaseSourceAdapter(ABC):
+    name: ClassVar[str]                    # "filesystem", "cursor", "git", "slack", ...
+    spec_version: ClassVar[str] = "1.0"
+    adapter_version: ClassVar[str]         # Independent of spec_version; recorded on every drawer.
+    capabilities: ClassVar[frozenset[str]]
+    supported_modes: ClassVar[frozenset[str]]             # Per §1.5.
+    declared_transformations: ClassVar[frozenset[str]]    # Per §1.4.
+    default_privacy_class: ClassVar[str]                  # Per §6.
+```
+
+Defined capability tokens (v1):
+
+| Token | Meaning |
+|---|---|
+| `byte_preserving` | `declared_transformations` is empty AND extracted content equals source bytes. |
+| `supports_incremental` | Implements `is_current()` meaningfully; `ingest()` respects `ctx.skip_current_item()`. |
+| `supports_structured_metadata` | Attaches fields beyond §5.1 universals. |
+| `supports_entity_hints` | Emits entity hints via `metadata["entity_hints_json"]` (§5.4). |
+| `supports_kg_triples` | Writes knowledge-graph triples directly to the SQLite KG (§5.5). |
+| `supports_closet_hints` | Emits `metadata["closet_hints"]` (§1.7). |
+| `requires_auth` | Needs credentials at runtime (env vars — §4.2). |
+| `requires_external_service` | Needs a running service (Slack API, email server). |
+| `requires_local_tool` | Needs a local binary (`gh`, `rg`, `whisper`). |
+| `adapter_owns_routing` | Returns authoritative `RouteHint` values from `ingest()` that core uses as-is (§G3 / §2.5). |
+| `respects_privacy_class` | Honors §6 privacy-class filtering. |
+
+Capability tokens are free-form strings; third-party adapters MAY declare novel tokens for their ecosystem. Core only inspects the above.
+
+### 2.2 Source references
+
+See `SourceRef` in §1.3. The shape is deliberately open — adapters parse `uri` and `options` as they see fit. Core does not canonicalize URIs.
+
+**Secrets in `SourceRef.options`:** credentials MUST NOT be placed in `options`. The spec reserves `options` for non-secret values (paths, filters, date ranges). Secrets come from env vars per §4.2. An adapter that reads a credential from `options` violates the spec and MUST be rejected by the conformance suite.
+
+### 2.3 Lifecycle
+
+1. `__init__`: lightweight. No I/O, no network, no credential fetch.
+2. First call to `ingest`: may open resources. All I/O is lazy.
+3. `close()`: releases all resources. After `close()`, further calls MUST raise `AdapterClosedError`.
+
+### 2.4 Concurrency
+
+An adapter instance is long-lived and serves many mine operations. Adapters MUST be thread-safe for concurrent `ingest` calls across different `SourceRef` values. MemPalace core serializes calls within a single `SourceRef` unless an adapter advertises `supports_parallel_ingest` (not in v1 — reserved for v1.1).
+
+### 2.5 Routing
+
+Routing is the adapter's responsibility. The filesystem adapter reads `mempalace.yaml` (hall keywords, rooms list) via `MempalaceConfig()` and returns `RouteHint(wing=..., room=..., hall=...)` on each drawer. This relocates `detect_room()` and `detect_hall()` (currently in `miner.py` and `convo_miner.py`) into their respective adapters.
+
+Order of precedence for routing:
+1. Explicit `--wing` / `--room` CLI flags → passed through `SourceRef.options` → adapter honors verbatim.
+2. Palace config match (`mempalace.yaml` hall keywords, room keywords) → adapter computes.
+3. Adapter-internal fallback (e.g., filesystem adapter falls back to `"general"` room).
+
+Adapters advertising `adapter_owns_routing` return the final answer; core uses it verbatim. Adapters not advertising it return None and core applies a generic fallback router (writing to wing `default`, room `general`, hall `general`). Absent any adapter, this is how `mempalace mine` behaves today.
+
+### 2.6 Incremental ingest
+
+`is_current()` is the incremental-ingest primitive. The palace itself is the cursor — no separate persisted state. Correctness requirements:
+
+- The adapter's `SourceItemMetadata.source_file` MUST be stable across re-ingests of the same logical item. Filesystem adapter uses the absolute path (as today). Git adapter uses a URI shape like `github.com/org/repo#pr=567` or `github.com/org/repo#commit=abc123`.
+- `is_current()` returns True when the stored metadata matches the adapter's current version token. The default implementation returns False (always re-extract) — adapters advertising `supports_incremental` override.
+- Deletion tombstones: an adapter MAY yield a `SourceItemMetadata(source_file=..., version="__deleted__")` entry — core purges drawers with matching `source_file` and builds no new drawers for that item. Advertised via `supports_deletion_tombstones`.
+- Adapters without `supports_incremental` ignore `is_current()` and fully re-extract. Core logs a warning.
+
+### 2.7 Errors
+
+- `SourceNotFoundError` — the `SourceRef` does not resolve.
+- `AuthRequiredError` — adapter needs credentials; raises with a message describing which env vars to set.
+- `AdapterClosedError` — method called after `close()`.
+- `TransformationViolationError` — conformance suite raises this when the content round-trip requires an undeclared transformation.
+- `SchemaConformanceError` — a `DrawerRecord.metadata` is missing required fields declared in `describe_schema()` or violates declared types.
+
+---
+
+## 3. Registration and discovery
+
+### 3.1 Entry points (primary mechanism)
+
+Third-party adapters ship as installable packages:
+
+```toml
+# pyproject.toml of mempalace-source-cursor
+[project.entry-points."mempalace.sources"]
+cursor = "mempalace_source_cursor:CursorAdapter"
+```
+
+MemPalace discovers adapters at process start via `importlib.metadata.entry_points(group="mempalace.sources")`.
+
+### 3.2 In-tree registry (secondary)
+
+```python
+from mempalace.sources.registry import register
+
+register("my-experimental-adapter", MyAdapter)
+```
+
+Entry-point discovery and explicit `register()` populate the same registry. Explicit registration wins on name conflict.
+
+### 3.3 Selection (explicit only — no auto-detect)
+
+Unlike storage backends (RFC 001 §3.3), source adapters are never auto-detected. The user selects the adapter explicitly:
+
+```bash
+mempalace mine --source cursor ~/                      # explicit adapter
+mempalace mine --source git /path/to/repo              # explicit adapter
+mempalace mine --source filesystem /path/to/project    # explicit adapter
+mempalace mine /path/to/project                        # implicit: filesystem (default)
+```
+
+The default when no `--source` is given is `filesystem`, preserving current `mempalace mine <path>` behavior.
+
+**Backwards compatibility with `--mode`.** Current `cli.py:517-519` exposes `--mode {projects,convos}`. This spec maps:
+- `--mode projects` → `--source filesystem` (the new default)
+- `--mode convos` → `--source conversations`
+
+`--mode` stays as a deprecated alias through v4.x with a deprecation warning on use; removed in v5.0.
+
+Auto-detection would be hostile — a directory containing a `.git` folder, a `workspaceStorage/` subdir, and an `mbox` file is not a signal of user intent.
+
+---
+
+## 4. Configuration
+
+### 4.1 Shape
+
+```json
+{
+  "sources": {
+    "my-cursor": {
+      "type": "cursor",
+      "workspace_storage": "~/Library/Application Support/Cursor/User/workspaceStorage"
+    },
+    "my-git": {
+      "type": "git",
+      "repos": ["/projects/mempalace", "/projects/site"]
+    }
+  },
+  "palaces": {
+    "work": {
+      "sources": ["my-git"],
+      "privacy_floor": "internal"
+    },
+    "personal": {
+      "sources": ["my-cursor"]
+    }
+  }
+}
+```
+
+Single-user local mode: config is optional. `mempalace mine <path>` with no config uses the `filesystem` adapter and defaults.
+
+### 4.2 Environment variables
+
+- `MEMPALACE_SOURCE_<NAME>_*` — per-adapter secrets and connection info. Examples: `MEMPALACE_SOURCE_SLACK_TOKEN`, `MEMPALACE_SOURCE_NOTION_API_KEY`, `MEMPALACE_SOURCE_GIT_GITHUB_TOKEN`.
+- Secrets MUST be readable from env vars; config files carry structure, env vars carry credentials. Same rule as RFC 001 §4.2.
+
+### 4.3 Adapter-specific options
+
+`SourceRef.options` is a free-form dict of non-secret values (§2.2). Each adapter documents its accepted keys. Unknown keys MUST be ignored (forward compatibility); the adapter MAY log a warning.
+
+---
+
+## 5. Metadata schema contract
+
+### 5.1 Universal fields
+
+Existing drawer metadata fields are preserved — the spec adds the following:
+
+| New field | Type | Added by | Purpose |
+|---|---|---|---|
+| `adapter_name` | `str` | core, from `BaseSourceAdapter.name` | Which registered source produced this drawer. |
+| `adapter_version` | `str` | adapter | Adapter's own version (distinct from palace `normalize_version`). Enables re-extract workflows targeted at drawers from a known-buggy adapter version. |
+| `privacy_class` | `str` | adapter default, config override | Per §6. |
+
+Existing fields retain their current semantics (verified against `miner.py:542-561` and `convo_miner.py:338-350`):
+
+| Existing field | Role in the spec |
+|---|---|
+| `source_file` | Functions as the adapter's source-item identifier. Adapter defines the shape — a filesystem path for filesystem, a URI like `github.com/org/repo#pr=123` for git. MUST be stable across re-ingests of the same logical item. |
+| `source_mtime` | Functions as the source-item version for filesystem. Adapters without mtime semantics MAY omit this field and use a different version discriminator (e.g., commit SHA in a separate `metadata["commit_sha"]` field); the spec only requires that `is_current()` can decide staleness from the stored metadata. |
+| `filed_at` | When the record was written. ISO-8601 string. |
+| `added_by` | Agent name (e.g., `lumi`, `claude-code`). Orthogonal to `adapter_name` — the agent is *who* triggered mining; the adapter is *how* data was extracted. |
+| `wing`, `room`, `hall` | Palace routing. Populated by adapter per §2.5. |
+| `chunk_index` | Per §1.6. Always 0 for `whole_record` / `metadata_only`. |
+| `normalize_version` | Palace-wide schema version (currently `palace.py:50`). Unchanged. Separate from `adapter_version`. |
+| `entities` | Semicolon-joined candidate entity names. Already flat; kept flat (§5.4 replacement). |
+| `ingest_mode` | Per §1.5. Already on conversation drawers; added to filesystem drawers by the cleanup PR. |
+| `extract_mode` | Conversation-adapter-specific (`exchange` vs `general`). Moves into the conversations adapter's declared schema per §5.2. |
+
+**Nothing is renamed. Nothing is removed.** The spec formalizes the shape ingesters already converge on. Existing `where={"source_file": ...}` queries in `searcher.py`, `palace.py`, and callers keep working.
+
+**Chroma metadata constraint:** all metadata values MUST be `str | int | float | bool`. No lists, no nested dicts. This matches RFC 001 §1.4 and the underlying ChromaDB contract. Structured side-data goes to the SQLite knowledge graph (§5.5) or to a declared flat JSON-encoded string field (§5.4).
+
+### 5.2 Adapter schemas
+
+Each adapter returns an `AdapterSchema` from `describe_schema()`:
+
+```python
+@dataclass(frozen=True)
+class AdapterSchema:
+    fields: dict[str, FieldSpec]   # Keyed by metadata key.
+    version: str
+
+@dataclass(frozen=True)
+class FieldSpec:
+    type: Literal["string", "int", "float", "bool", "delimiter_joined_string", "json_string"]
+    required: bool
+    description: str
+    indexed: bool = False           # Hint to backends that can build indexes (RFC 001 §2.1).
+    # delimiter_joined_string: the delimiter character (default ";").
+    delimiter: str = ";"
+    # json_string: the JSON schema of the encoded object (informational only).
+    json_schema: dict | None = None
+```
+
+`delimiter_joined_string` covers the `entities` shape (current `;`-joined list of names). `json_string` is the escape hatch for adapters needing to pack nested data — the value stored is still a single flat `str` from Chroma's perspective, but the adapter is allowed to document its parsed shape.
+
+Example for a hypothetical `slack` adapter:
+
+```python
+AdapterSchema(
+    version="1.0",
+    fields={
+        "channel_name": FieldSpec(type="string", required=True, description="Slack channel name", indexed=True),
+        "channel_id": FieldSpec(type="string", required=True, description="Slack channel ID"),
+        "thread_ts": FieldSpec(type="string", required=False, description="Thread root timestamp"),
+        "author_id": FieldSpec(type="string", required=True, description="Slack user ID", indexed=True),
+        "author_name": FieldSpec(type="string", required=True, description="Display name at extraction time"),
+        "reactions": FieldSpec(type="delimiter_joined_string", required=False, description="Emoji shortcodes"),
+    },
+)
+```
+
+### 5.3 Enterprise keying
+
+The adapter schema is the stable surface enterprises filter on. A support team querying the palace for `channel_id = "C01234"` does not care about ChromaDB's internal representation. The schema field is declared by the adapter, indexed by the backend (RFC 001 §2.1 `supports_metadata_filters`), and exposed through the existing `where=` clause.
+
+This is how "structured data" serves company use cases without breaking transformation guarantees: declared-transformation content in the drawer, structured fields in the metadata, schema declared by the adapter, filtering done by the backend.
+
+### 5.4 Entity hints (optional)
+
+Adapters with `supports_entity_hints` MAY include:
+
+```python
+metadata["entity_hints_json"] = '[{"type":"person","name":"Milla Jovovich","confidence":0.95,"offset":120},{"type":"project","name":"MemPalace","confidence":1.0,"offset":0}]'
+```
+
+The value is a JSON-encoded string (type `json_string` in the adapter schema). Core parses on read and feeds into `mempalace/entity_detector.py` as a prior: hints with `confidence >= 0.9` bypass the heuristic detector; lower-confidence hints feed into it as candidates.
+
+This is additive to the existing flat `entities` field — entity_hints carries structure (type, confidence, offset); `entities` remains the Chroma-indexable flat string. An adapter that produces entity_hints MUST also populate `entities` as the flat name-only projection, so existing filter queries keep working.
+
+### 5.5 Knowledge-graph triples (optional)
+
+Adapters with `supports_kg_triples` write directly to the SQLite knowledge graph via `mempalace/knowledge_graph.py` — **not** to drawer metadata. Chroma cannot store structured triples; the KG already exists for this purpose.
+
+The adapter calls the existing `KnowledgeGraph.add_triple()` (signature verified against `mempalace/knowledge_graph.py:130`):
+
+```python
+palace.kg.add_triple(
+    subject="Ben",
+    predicate="committed",
+    obj="PR-567",                    # `object` is a Python builtin — the API uses `obj`.
+    valid_from="2026-03-12",
+    confidence=1.0,
+    source_file=drawer.source_file,  # Existing provenance parameter.
+)
+```
+
+Drawer metadata includes a flat counter — `metadata["kg_triples_count"]: int` — so search consumers can see at a glance that KG side-data exists for a drawer without hitting SQLite.
+
+The existing API has `source_closet` and `source_file` provenance parameters but no `source_drawer_id` or `adapter_name`. The cleanup PR (§9) should add these two optional parameters to `add_triple()` so adapter-written triples can be traced back to (a) the specific drawer that produced them and (b) the adapter that authored them — necessary for re-extraction workflows. Until that lands, adapters use `source_file` as the provenance key and record adapter authorship via a separate table or a predicate naming convention (e.g., `adapter:git:committed`).
+
+This aligns with the existing architecture in `CLAUDE.md` ("Knowledge Graph: ENTITY → PREDICATE → ENTITY with valid_from / valid_to dates") — the RFC formalizes the adapter-side write path.
+
+### 5.6 Source encoding and newline
+
+Current ingesters handle encoding lossily (`errors="replace"` in `miner.py:595` and `normalize.py:124`) and do not record original encoding. The spec does **not** require per-drawer `source_encoding` / `source_newline` — most runs are uniform UTF-8 / LF, and storing the same value on every drawer wastes bytes.
+
+Instead: adapters that handle non-UTF-8 or non-LF sources record the values once on the adapter's `SourceSummary` and per-drawer only when a specific drawer diverges from the adapter default. The `utf8_replace_invalid` declared transformation (§1.4) already communicates that lossy decoding happened; specific drawer-level provenance is opt-in.
+
+---
+
+## 6. Privacy class
+
+### 6.1 Defined levels
+
+| Level | Meaning | Example sources |
+|---|---|---|
+| `public` | Content intended for public consumption. | arXiv papers, public GitHub repos, published blogs. |
+| `internal` | Organizational content, not for public disclosure. | Corporate Slack, internal Notion, private git repos. |
+| `pii_potential` | May contain personally identifiable information. | Email, iMessage, Claude/ChatGPT transcripts. |
+| `sensitive` | Known to contain PII, financial, or health data. | Medical records, financial statements, legal filings. |
+| `secrets_possible` | May contain credentials or secrets. | Git history, environment dumps, CI logs. |
+
+An adapter declares a default on `BaseSourceAdapter.default_privacy_class`. Users MAY override per-source in config.
+
+### 6.2 Enforcement
+
+- Each palace declares a `privacy_floor`. Drawers above the floor (equal to or laxer) are admitted; drawers below are rejected at write time and surfaced in a `rejected` list on the CLI and MCP tool.
+- **Default floor: none** — v1 accepts all levels unless the palace explicitly configures a floor. This keeps the single-user local default low-friction (users who run `mempalace mine` on a git repo expect `secrets_possible` drawers to land). Enterprise deployments MUST set a floor; docs for regulated-domain setup will recommend starting strict and relaxing as needed.
+- Search results surface `privacy_class` in result metadata. MCP tool wrappers MAY redact results above a caller-declared ceiling.
+- `secrets_possible` drawers SHOULD pass through a secrets-scan pre-index hook when one is available. PR #389 (sensitive content scanner) is the expected enforcement mechanism for v1; until it lands, `secrets_possible` is a label without automated scanning. The label is still useful — it enables floor-based rejection and alerts downstream consumers.
+- The privacy class is recorded in drawer metadata and cannot be downgraded without a migration log entry, matching RFC 001's embedder-identity pattern.
+
+Privacy class is how a regulated-domain deployment (medical, legal, financial) can use MemPalace safely. Without it, flexible ingest becomes a liability; with it, ingest is scoped by policy.
+
+---
+
+## 7. Testing contract
+
+### 7.1 The abstract suite
+
+MemPalace ships `mempalace.sources.testing.AbstractSourceAdapterContractSuite` — a pytest mixin. Every adapter package ships a concrete subclass:
+
+```python
+from mempalace.sources.testing import AbstractSourceAdapterContractSuite
+
+class TestCursorAdapter(AbstractSourceAdapterContractSuite):
+    @pytest.fixture
+    def adapter(self):
+        return CursorAdapter()
+
+    @pytest.fixture
+    def fixture_source(self, tmp_path):
+        """Build a minimal Cursor workspaceStorage fixture."""
+        ...
+        return SourceRef(local_path=str(tmp_path))
+
+    @pytest.fixture
+    def canonical_source_bytes(self, fixture_source):
+        """Return a mapping of source_file -> authoritative bytes.
+
+        For filesystem sources: the file's raw bytes.
+        For SQLite sources: the extracted value column bytes for each row.
+        For API sources: the canonical HTTP response body bytes.
+
+        Adapter-defined — the adapter knows what its 'source bytes' are.
+        """
+        ...
+```
+
+The suite covers:
+
+- `ingest` yields items with stable `source_file` and well-formed `version`.
+- `is_current()` returns True when metadata matches, False when it differs.
+- `close()` releases resources; subsequent calls raise `AdapterClosedError`.
+- Unicode content and unicode identifiers are preserved end-to-end.
+- Large-source handling: 10k+ items ingest without loading all into memory.
+- Error paths: `SourceNotFoundError`, `AuthRequiredError` raise with correct types.
+- `SourceRef.options` MUST NOT contain secrets — the adapter raises if it detects a value matching a common-secret pattern (GitHub token prefix, Slack token prefix, etc.). Advisory test, not blocking.
+
+### 7.2 Byte-preserving round-trip (for `byte_preserving` adapters only)
+
+Required for adapters advertising `byte_preserving`:
+
+```python
+def test_byte_preserving_round_trip(self, adapter, fixture_source, canonical_source_bytes):
+    """Concatenated chunks must equal the canonical source bytes.
+
+    For each source_file in the fixture:
+      1. Read canonical_source_bytes[source_file].
+      2. Collect all DrawerRecords for that source_file from adapter.ingest(...).
+         Skip metadata_only drawers (§1.5).
+      3. Sort by chunk_index.
+      4. Concatenate record.content values.
+      5. Assert equality with the canonical bytes (UTF-8 decoded).
+    """
+```
+
+Failure raises `TransformationViolationError`.
+
+### 7.3 Declared-transformation round-trip (for `declared_lossy` adapters)
+
+Required for adapters with non-empty `declared_transformations`:
+
+```python
+def test_declared_transformation_round_trip(self, adapter, fixture_source, canonical_source_bytes):
+    """Adapter output must be reproducible by applying ONLY declared transformations.
+
+    1. For each source_file, read canonical_source_bytes.
+    2. Apply each declared transformation in declared_transformations to the bytes,
+       in the order declared by the adapter, using the reference implementations
+       in mempalace.sources.transforms.
+    3. Compare the result to the concatenated record.content values.
+    4. If they differ, the adapter has applied a transformation it did not declare.
+       Raise TransformationViolationError.
+    """
+```
+
+For transformations not in the reserved list (§1.4) — adapter-custom names — the adapter MUST provide a reference implementation callable under `mempalace.sources.transforms.<adapter_name>_<transform_name>`. The conformance suite imports and applies it. Undiscoverable custom transforms fail the test.
+
+### 7.4 Schema conformance
+
+A generator-based property test validates that every record yielded by `ingest` across the fixture source has metadata matching `describe_schema()`. Missing required fields, wrong types, or (in strict mode) undeclared fields fail the test.
+
+### 7.5 Note on current corpus
+
+No existing test in `tests/` asserts byte-preservation or declared-transformation correctness (verified via grep of `tests/` for `verbatim|byte.?preserv|round.?trip`). This RFC's conformance suite introduces the first such coverage. The existing MISSION.md claim of "verbatim always" is a social contract until this lands; afterward it becomes a machine-verified property of adapters that declare `byte_preserving`.
+
+---
+
+## 8. Versioning and compatibility
+
+- `BaseSourceAdapter.spec_version` declares which spec version an adapter implements.
+- MemPalace refuses to load an adapter declaring a different major spec version.
+- Minor spec versions are additive: new optional methods, new capability tokens, new reserved transformation names, new universal metadata fields with sensible defaults.
+- Adapters MAY declare their own `adapter_version` independent of the spec version; this is recorded on every drawer (§5.1) and enables "this drawer was extracted by cursor-adapter 0.3; 0.4 fixed a parsing bug; re-extract affected drawers" workflows.
+- This is spec v1.0.
+
+---
+
+## 9. Cleanup prerequisite (not in this spec, but gating)
+
+The existing in-tree ingesters are not adapter-shaped. Before RFC 002 can be enforced, the following refactor lands in a separate PR:
+
+- Introduce `mempalace/sources/base.py` defining `BaseSourceAdapter`, the typed records, and the registry.
+- Introduce `mempalace/sources/transforms.py` with reference implementations of every reserved transformation in §1.4. Adapters and the conformance suite both consume these.
+- `mempalace/miner.py` → `mempalace/sources/filesystem.py` implementing `BaseSourceAdapter`. Current behavior preserved: 800-char chunking becomes the adapter's default; `READABLE_EXTENSIONS` moves to the adapter; `detect_room()` and `detect_hall()` move to the adapter per §2.5. `declared_transformations = frozenset({"utf8_replace_invalid", "whitespace_trim"})`.
+- `mempalace/convo_miner.py` → `mempalace/sources/conversations.py`. Exchange-pair chunking stays. The format-detection logic in `normalize.py` becomes per-format plugins the conversations adapter composes (one for Claude Code JSONL, one for Codex JSONL, one for ChatGPT mapping trees, one for Claude.ai JSON, one for Slack JSON) — each small and independently testable, eliminating the `if source_type` chain. `declared_transformations` enumerates every transformation `normalize.py` and `convo_miner._chunk_by_exchange` actually perform (see §1.4 "Existing code mapping").
+- Closet-building wired into the conversations adapter's post-step (currently missing, per §1.7) — side effect of routing through the unified core post-step.
+- `mempalace/cli.py` subcommand `mine` routes through the `mempalace.sources` registry. `--mode {projects,convos}` becomes a deprecated alias for `--source {filesystem,conversations}`.
+- `mempalace/mcp_server.py` `mempalace_mine` tool accepts a `source` parameter.
+- `mempalace/palace.py` exposes `PalaceContext` — a per-mine-invocation facade that bundles the drawer collection, closet collection, knowledge graph, palace config, and progress hooks. Adapters receive this; they do not import `palace.py` directly.
+- `NORMALIZE_VERSION` (currently a module-level constant in `palace.py:50`) stays. It is the palace-wide schema version, orthogonal to per-adapter `adapter_version`.
+- `KnowledgeGraph.add_triple()` (`knowledge_graph.py:130`) gains two optional parameters: `source_drawer_id: str = None` and `adapter_name: str = None`. Existing callers are unaffected; adapters advertising `supports_kg_triples` (§5.5) populate both. Backwards-compatible change.
+
+This cleanup is substantial — comparable to RFC 001 §10's chroma-import removal — and should land before any new third-party adapter PR merges. Each new adapter is easier after the cleanup, not harder.
+
+---
+
+## 10. Impact on in-flight PRs
+
+| PR / Issue | Effort to align |
+|---|---|
+| [#274](https://github.com/MemPalace/mempalace/issues/274) Cursor SQLite | Becomes `mempalace-source-cursor` third-party package. Author has a working prototype on Windows; needs `describe_schema()`, `declared_transformations`, and the conformance suite. Prior #287 (closed unmerged) is predecessor work. |
+| [#23](https://github.com/MemPalace/mempalace/pull/23) OpenCode SQLite | Becomes `mempalace-source-opencode`. Same shape as Cursor. |
+| [#169](https://github.com/MemPalace/mempalace/pull/169) Pi agent | Becomes `mempalace-source-pi` or a format plugin under the conversations adapter (depending on format similarity). |
+| [#232](https://github.com/MemPalace/mempalace/pull/232) Cursor JSONL | Deprecated in favor of #274's SQLite path; or a second mode of `mempalace-source-cursor`. |
+| [#567](https://github.com/MemPalace/mempalace/pull/567), [#98](https://github.com/MemPalace/mempalace/pull/98) git-mine | Closest existing work to what the spec envisions. Becomes first-party `mempalace/sources/git.py`. Exercises `whole_record` mode, `supports_structured_metadata`, `supports_closet_hints` (decision-signal quotes), `supports_kg_triples` (commit authorship, PR review relationships). |
+| [#591](https://github.com/MemPalace/mempalace/pull/591), [#592](https://github.com/MemPalace/mempalace/pull/592) Delphi Oracle | Deferred. The live-stream pattern is out of scope for v1 (§Non-goals). A v1.1 addition will specify webhook/stream adapters. |
+| [#702](https://github.com/MemPalace/mempalace/pull/702) Cursor + factory.ai | Splits into two adapter packages. |
+| [#981](https://github.com/MemPalace/mempalace/issues/981) path-level descriptions | Absorbed by §1.5 `metadata_only` mode + §5.1 `ingest_mode`. A new first-party `descriptions` adapter or a second mode on `filesystem`. |
+| [#244](https://github.com/MemPalace/mempalace/pull/244) Cursor memory-first MCP workflow docs | Points at `mempalace-source-cursor` once the adapter lands. |
+| [#419](https://github.com/MemPalace/mempalace/pull/419), [#300](https://github.com/MemPalace/mempalace/pull/300), [#952](https://github.com/MemPalace/mempalace/pull/952) language-extension additions to `READABLE_EXTENSIONS` | Becomes per-language config on the filesystem adapter. Contributors can publish domain-specific adapters without touching core. |
+| [#389](https://github.com/MemPalace/mempalace/pull/389) sensitive content scanner | Expected enforcement mechanism for the `secrets_possible` privacy class (§6.2). Not a blocker for this spec, but a natural consumer. |
+| [#434](https://github.com/MemPalace/mempalace/pull/434) auto-populate KG from drawers | Complementary: post-hoc derivation of KG triples from drawer content. Adapters with `supports_kg_triples` provide the up-front path; #434 handles everything else. |
+
+---
+
+## 11. Open questions
+
+1. **Cross-adapter dedup.** When a PR body is mined via `git` AND shows up as a conversation quote mined via `claude-code`, both drawers land. Is query-time dedup in `searcher.py` sufficient, or should core maintain a content-hash index across adapters? Declared non-goal in v1 but worth revisiting if user feedback demands it.
+2. **Live-stream pattern.** Delphi Oracle (#591/592) and potentially Slack/Discord real-time ingestion need a push-mode contract. This is a v1.1 addition (streaming adapter trait + webhook surface), not blocking.
+3. **LLM-assisted structured extraction.** Some adapters will want to call an LLM to extract structured fields. The spec does not standardize this — should it? Argument for: conformance test for LLM-driven fields, consistent caching. Argument against: local-first / zero-API is a core promise; LLM dependencies are opt-in per adapter.
+4. **Adapter-vs-format split for conversations.** §9 proposes format plugins composed under a single conversations adapter. Alternative: one adapter per format (claude-code, chatgpt, codex, cursor-jsonl, slack). The trade-off is discoverability (one adapter is easier to find) vs. encapsulation (format plugins are simpler to test). Preference leans toward the single-adapter + plugin model; open to counter-argument.
+5. **Default `privacy_floor`.** v1 defaults to none (§6.2) so single-user local mining is frictionless. An argument exists for defaulting to `pii_potential` — forces regulated-domain users to opt in to sensitive levels rather than opt out. Open to changing the default before v1 ships.
+6. **`canonical_source_bytes` for API-backed adapters.** §7.1 defines this as adapter-declared. For API-backed adapters (Slack, Notion), what constitutes "canonical bytes" in a conformance test — the fixture's captured HTTP response? A serialized representation of the parsed object? Leaves to the adapter; may need a follow-up spec for common conventions.
+7. **`adapter_version` bump semantics.** When does an adapter bump `adapter_version`? On any behavior change? On declared-transformation changes only? Suggests a follow-up doc on adapter SemVer conventions for the community to agree on.
+
+---
+
+## 12. Rollout
+
+1. Land the cleanup PR (§9): introduce `mempalace/sources/`, refactor `miner.py` → filesystem adapter, `convo_miner.py` → conversations adapter, route CLI and MCP through the sources registry. Behavior preserved end-to-end. Closets get built for conversation drawers as a side effect.
+2. Land this spec as-is. Add `AbstractSourceAdapterContractSuite`, entry-point discovery, `AdapterSchema` validation, privacy-class enforcement (floor-gated writes), declared-transformation reference implementations in `mempalace/sources/transforms.py`.
+3. Land `mempalace/sources/git.py` as the first-party adapter absorbing #567. Exercises `whole_record`, `supports_structured_metadata`, `supports_closet_hints`, `supports_kg_triples` together.
+4. Encourage the Cursor (#274), OpenCode (#23), and Pi (#169) authors to publish as third-party packages under `mempalace-source-*`. Offer review help against the spec.
+5. Publish adapter-authoring docs at [mempalaceofficial.com/guide/authoring-sources](https://mempalaceofficial.com/guide/authoring-sources.html).
+6. Update [ROADMAP.md](../../ROADMAP.md) with spec v1.0 adoption under v4.0.0-alpha.

docs/schema.sql

@@ -0,0 +1,36 @@
+-- MemPalace Knowledge Graph Schema
+-- SQLite database at ~/.mempalace/knowledge_graph.db
+
+CREATE TABLE IF NOT EXISTS entities (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    type TEXT DEFAULT 'unknown',
+    properties TEXT DEFAULT '{}'
+);
+
+CREATE TABLE IF NOT EXISTS triples (
+    id TEXT PRIMARY KEY,
+    subject TEXT NOT NULL,
+    predicate TEXT NOT NULL,
+    object TEXT NOT NULL,
+    valid_from TEXT,
+    valid_to TEXT,
+    confidence REAL DEFAULT 1.0,
+    source_closet TEXT,
+    source_file TEXT
+);
+
+CREATE TABLE IF NOT EXISTS attributes (
+    entity_id TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT,
+    valid_from TEXT,
+    valid_to TEXT,
+    PRIMARY KEY (entity_id, key, valid_from)
+);
+
+-- Indexes
+CREATE INDEX IF NOT EXISTS idx_triples_subject ON triples(subject);
+CREATE INDEX IF NOT EXISTS idx_triples_object ON triples(object);
+CREATE INDEX IF NOT EXISTS idx_triples_predicate ON triples(predicate);
+CREATE INDEX IF NOT EXISTS idx_triples_valid ON triples(valid_from, valid_to);

examples/HOOKS_TUTORIAL.md

@@ -0,0 +1,66 @@
+# How to Use MemPalace Hooks (Auto-Save)
+
+MemPalace hooks act as an "Auto-Save" feature. They help your AI keep a permanent memory without you needing to run manual commands.
+
+### 1. What are these hooks?
+* **Save Hook** (`mempal_save_hook.sh`): Saves new facts and decisions every 15 messages.
+* **PreCompact Hook** (`mempal_precompact_hook.sh`): Saves your context right before the AI's memory window fills up.
+
+### 2. Setup for Claude Code
+Add this to `~/.claude/settings.local.json` (global) or `.claude/settings.local.json` (project-scoped) to enable automatic background saving:
+
+```json
+{
+  "hooks": {
+    "Stop": [
+      {
+        "matcher": "*", 
+        "hooks": [{
+          "type": "command",
+          "command": "/absolute/path/to/hooks/mempal_save_hook.sh",
+          "timeout": 30
+        }]
+      }
+    ],
+    "PreCompact": [
+      {
+        "hooks": [{
+          "type": "command",
+          "command": "/absolute/path/to/hooks/mempal_precompact_hook.sh",
+          "timeout": 30
+        }]
+      }
+    ]
+  }
+}
+```
+
+Make the hooks executable:
+```bash
+chmod +x /absolute/path/to/hooks/mempal_save_hook.sh
+chmod +x /absolute/path/to/hooks/mempal_precompact_hook.sh
+```
+
+**Note:** Replace `/absolute/path/to/hooks/` with the actual path where you cloned the MemPalace repository (e.g., `~/projects/mempalace/hooks/`).
+
+### 3. What changed (v3.1.0+)
+
+Both hooks now have **two-layer capture**:
+
+1. **Auto-mine**: Before blocking the AI, the hook runs the normalizer on the JSONL transcript and upserts chunks directly into the palace. This captures raw tool output (Bash results, search findings, build errors) that the AI would otherwise summarize away.
+
+2. **Updated reason messages**: The block reason now explicitly tells the AI to save tool output verbatim — not just topics and decisions.
+
+### 4. Backfill past conversations (one-time)
+
+The hooks capture conversations going forward, but you probably have months of past sessions. Run this once to mine them all:
+
+```bash
+mempalace mine ~/.claude/projects/ --mode convos
+```
+
+### 5. Configuration
+
+- **`SAVE_INTERVAL=15`** — How many human messages between saves
+- **`MEMPALACE_PYTHON`** — Python interpreter with mempalace + chromadb. Auto-detects: env var → repo venv → system python3
+- **`MEMPAL_DIR`** — Optional directory for auto-ingest via `mempalace mine`

examples/gemini_cli_setup.md

@@ -0,0 +1,95 @@
+# Gemini CLI Integration Guide
+
+This guide explains how to set up MemPalace as a permanent memory for the [Gemini CLI](https://github.com/google/gemini-cli).
+
+## Prerequisites
+
+- Python 3.9+
+- Gemini CLI installed and configured
+
+## 1. Installation
+
+On many Linux systems, installing Python packages globally is restricted. We recommend using a local virtual environment within the MemPalace directory.
+
+```bash
+# Clone the repository (if you haven't already)
+git clone https://github.com/MemPalace/mempalace.git
+cd mempalace
+
+# Create a virtual environment
+python3 -m venv .venv
+
+# Install dependencies and MemPalace in editable mode
+.venv/bin/pip install -e .
+```
+
+## 2. Initialization
+
+Set up your "Palace" (the database) and configure your identity.
+
+```bash
+# Initialize the palace in the current directory
+.venv/bin/python3 -m mempalace init .
+```
+
+### Identity and Wings (Optional but Recommended)
+You can manually define who you are and what projects you work on by creating/editing these files in `~/.mempalace/`:
+
+- **`~/.mempalace/identity.txt`**: A plain text file describing your role and focus.
+- **`~/.mempalace/wing_config.json`**: A JSON file mapping projects and name variants to "Wings".
+
+## 3. Connect to Gemini CLI (MCP)
+
+Register MemPalace as an MCP server so Gemini CLI can use its tools.
+
+```bash
+gemini mcp add mempalace /absolute/path/to/mempalace/.venv/bin/python3 -m mempalace.mcp_server --scope user
+```
+*Note: Use the absolute path to ensure it works from any directory.*
+
+## 4. Enable Auto-Saving (Hooks)
+
+To ensure the AI saves memories automatically when conversation history becomes too long, add a `PreCompress` hook to your Gemini CLI settings.
+
+Edit your `~/.gemini/settings.json` and add the following:
+
+```json
+{
+  "hooks": {
+    "PreCompress": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "/absolute/path/to/mempalace/hooks/mempal_precompact_hook.sh"
+          }
+        ]
+      }
+    ]
+  }
+}
+```
+
+Make sure the hook scripts are executable:
+```bash
+chmod +x hooks/*.sh
+```
+
+## 5. Usage
+
+Once connected, Gemini CLI will automatically:
+- Start the MemPalace server on launch.
+- Use `mempalace_search` to find relevant past discussions.
+- Use the `PreCompress` hook to save new memories before they are lost.
+
+### Manual Mining
+If you want the AI to learn from your existing code or docs immediately, run the "mine" command:
+```bash
+.venv/bin/python3 -m mempalace mine /path/to/your/project
+```
+
+### Verification
+In a Gemini CLI session, you can run:
+- `/mcp list`: Verify `mempalace` is `CONNECTED`.
+- `/hooks panel`: Verify the `PreCompress` hook is active.

examples/mcp_setup.md

@@ -5,20 +5,22 @@
 Run the MCP server:
 
 ```bash
-python mcp_server.py
+mempalace-mcp
 ```
 
-Or add to Claude Code:
+Or add it to Claude Code:
 
 ```bash
-claude mcp add mempal -- python /path/to/mempalace/mcp_server.py
+claude mcp add mempalace -- mempalace-mcp
 ```
 
 ## Available Tools
 
-- **mempal_status** — palace stats (wings, rooms, drawer counts)
-- **mempal_search** — semantic search across all memories
-- **mempal_list_wings** — list all projects in the palace
+The server exposes the full MemPalace MCP toolset. Common entry points include:
+
+- **mempalace_status** — palace stats (wings, rooms, drawer counts)
+- **mempalace_search** — semantic search across all memories
+- **mempalace_list_wings** — list all projects in the palace
 
 ## Usage in Claude Code
 

hooks/README.md

@@ -6,10 +6,10 @@ These hook scripts make MemPalace save automatically. No manual "save" commands
 
 | Hook | When It Fires | What Happens |
 |------|--------------|-------------|
-| **Save Hook** | Every 15 human messages | Blocks the AI, tells it to save key topics/decisions/quotes to the palace |
-| **PreCompact Hook** | Right before context compaction | Emergency save — forces the AI to save EVERYTHING before losing context |
+| **Save Hook** | Every 15 human messages | Auto-mines transcript (tool output included), then blocks the AI to save topics/decisions/quotes |
+| **PreCompact Hook** | Right before context compaction | Auto-mines transcript, then emergency save — forces the AI to save EVERYTHING before losing context |
 
-The AI does the actual filing — it knows the conversation context, so it classifies memories into the right wings/halls/closets. The hooks just tell it WHEN to save.
+**Two-layer capture:** Hooks auto-mine the JSONL transcript directly into the palace (capturing raw tool output — Bash results, search findings, build errors). They also block the AI with a reason message telling it to save verbatim tool output and key context. Belt and suspenders — tool output gets stored even if the AI summarizes instead of quoting.
 
 ## Install — Claude Code
 
@@ -67,7 +67,8 @@ Edit `mempal_save_hook.sh` to change:
 
 - **`SAVE_INTERVAL=15`** — How many human messages between saves. Lower = more frequent saves, higher = less interruption.
 - **`STATE_DIR`** — Where hook state is stored (defaults to `~/.mempalace/hook_state/`)
-- **`MEMPAL_DIR`** — Optional. Set to a conversations directory to auto-run `mempalace mine <dir>` on each save trigger. Leave blank (default) to let the AI handle saving via the block reason message.
+- **`MEMPAL_DIR`** — Optional **project directory** (code, notes, docs) to also mine on each save trigger, with `--mode projects`. The hook ALWAYS mines the active conversation transcript automatically with `--mode convos` — `MEMPAL_DIR` is purely additive, never an override. Leave blank if you don't want to ingest project files.
+- **`MEMPALACE_PYTHON`** — Optional env var. Python interpreter with mempalace + chromadb installed. Auto-detects: `MEMPALACE_PYTHON` env var → repo `venv/bin/python3` → system `python3`. Set this if your venv is in a non-standard location.
 
 ### mempalace CLI
 
@@ -91,15 +92,19 @@ User sends message → AI responds → Claude Code fires Stop hook
                                             ↓
                               ┌─── < 15 since last save ──→ echo "{}" (let AI stop)
                               │
-                              └─── ≥ 15 since last save ──→ {"decision": "block", "reason": "save..."}
-                                                                    ↓
-                                                            AI saves to palace
-                                                                    ↓
-                                                            AI tries to stop again
-                                                                    ↓
-                                                            stop_hook_active = true
-                                                                    ↓
-                                                            Hook sees flag → echo "{}" (let it through)
+                              └─── ≥ 15 since last save
+                                            ↓
+                                    Auto-mine transcript → palace (tool output captured)
+                                            ↓
+                                    {"decision": "block", "reason": "save tool output verbatim..."}
+                                            ↓
+                                    AI saves to palace (topics, decisions, quotes)
+                                            ↓
+                                    AI tries to stop again
+                                            ↓
+                                    stop_hook_active = true
+                                            ↓
+                                    Hook sees flag → echo "{}" (let it through)
 ```
 
 The `stop_hook_active` flag prevents infinite loops: block once → AI saves → tries to stop → flag is true → we let it through.
@@ -109,14 +114,18 @@ The `stop_hook_active` flag prevents infinite loops: block once → AI saves →
 ```
 Context window getting full → Claude Code fires PreCompact
                                         ↓
-                                Hook ALWAYS blocks
+                                Find transcript (from input or session_id lookup)
+                                        ↓
+                                Auto-mine transcript → palace (tool output captured)
+                                        ↓
+                                {"decision": "block", "reason": "save tool output verbatim..."}
                                         ↓
                                 AI saves everything
                                         ↓
                                 Compaction proceeds
 ```
 
-No counting needed — compaction always warrants a save.
+No counting needed — compaction always warrants a save. The auto-mine captures raw tool output before the AI gets a chance to summarize it away.
 
 ## Debugging
 
@@ -133,6 +142,40 @@ Example output:
 [14:40:01] Session abc123: 18 exchanges, 3 since last save
 ```
 
+## Known Limitations
+
+**Hooks require session restart after install.** Claude Code loads hooks from `settings.json` at session start only. If you run `mempalace init` or manually edit hook config mid-session, the hooks won't fire until you restart Claude Code. This is a Claude Code limitation.
+
+**`MEMPAL_PYTHON` override for the hook's internal Python calls.** The save hook parses its JSON input and counts transcript messages with `python3`. When the harness is launched from a GUI on macOS — `open -a`, Spotlight, the dock — its `PATH` is the minimal `/usr/bin:/bin:/usr/sbin:/sbin` inherited from `launchd`, not your shell PATH. If `python3` isn't on that PATH, those internal calls fail and the hook can't count exchanges.
+
+Point the hook at any Python 3 interpreter to fix it:
+
+```bash
+export MEMPAL_PYTHON="/usr/bin/python3"                   # system Python is fine
+export MEMPAL_PYTHON="$HOME/.venvs/mempalace/bin/python"  # or your venv
+```
+
+Resolution priority: `$MEMPAL_PYTHON` (if set and executable) → `$(command -v python3)` → bare `python3`. The interpreter only needs `json` and `sys` from the standard library — `mempalace` itself does not need to be installed in it.
+
+Note: the `mempalace mine` auto-ingest runs via the `mempalace` CLI, so that command also needs to be on the hook's `PATH`. Installing with `pipx install mempalace` or `uv tool install mempalace` puts it on a stable global location; otherwise extend the hook environment's `PATH` to include your venv's `bin/`.
+
+## Backfill Past Conversations
+
+The hooks only capture conversations going forward. To mine **past** Claude Code sessions into your palace, run a one-time backfill:
+
+```bash
+mempalace mine ~/.claude/projects/ --mode convos
+```
+
+This scans all JSONL transcripts from previous sessions and files them into the `conversations` wing. On a typical developer machine with months of history, this can yield 50K–200K drawers.
+
+For Codex CLI sessions:
+```bash
+mempalace mine ~/.codex/sessions/ --mode convos
+```
+
+This only needs to be done once — after that, the hooks auto-mine each session as you go.
+
 ## Cost
 
-**Zero extra tokens.** The hooks are bash scripts that run locally. They don't call any API. The only "cost" is the AI spending a few seconds organizing memories at each checkpoint — and it's doing that with context it already has loaded.
+**Zero extra tokens.** The hooks notify the AI that saves happened in the background — the AI doesn't need to write anything in the chat. All filing is handled automatically. Previous versions asked the AI to write diary entries and drawer content in the chat window, which cost ~$1/session in retransmitted tokens.

hooks/mempal_precompact_hook.sh

@@ -41,37 +41,83 @@
 # to save everything. After the AI saves, compaction proceeds normally.
 #
 # === MEMPALACE CLI ===
-# This repo uses: mempalace mine <dir>
-# or:            mempalace mine <dir> --mode convos
-# Set MEMPAL_DIR below if you want the hook to auto-ingest before compaction.
-# Leave blank to rely on the AI's own save instructions.
+# The hook ALWAYS mines the active conversation transcript synchronously
+# before compaction (via `mempalace mine <transcript-dir> --mode convos`).
+# MEMPAL_DIR is an *additional*, optional target for project files — it
+# does not replace the conversation mine.
 
 STATE_DIR="$HOME/.mempalace/hook_state"
 mkdir -p "$STATE_DIR"
 
-# Optional: set to the directory you want auto-ingested before compaction.
-# Example: MEMPAL_DIR="$HOME/conversations"
-# Leave empty to skip auto-ingest (AI handles saving via the block reason).
+# Optional: project directory (code / notes / docs) to also mine before
+# compaction. Mined with `--mode projects`. The conversation transcript
+# is always mined regardless — this is purely additive.
+# Example: MEMPAL_DIR="$HOME/projects/my_app"
 MEMPAL_DIR=""
 
+# Resolve the Python interpreter. Same contract as mempal_save_hook.sh:
+# MEMPAL_PYTHON (explicit override) → $(command -v python3) → bare python3.
+MEMPAL_PYTHON_BIN="${MEMPAL_PYTHON:-}"
+if [ -z "$MEMPAL_PYTHON_BIN" ] || [ ! -x "$MEMPAL_PYTHON_BIN" ]; then
+    MEMPAL_PYTHON_BIN="$(command -v python3 2>/dev/null || echo python3)"
+fi
+
 # Read JSON input from stdin
 INPUT=$(cat)
 
-SESSION_ID=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('session_id','unknown'))" 2>/dev/null)
+# Parse session_id and transcript_path in one call. Sanitize both, then
+# read sanitized values from one-per-line stdout into shell variables —
+# avoids ``eval`` on generated code (#1231 review). Same contract as
+# mempal_save_hook.sh.
+mapfile -t _mempal_parsed < <(echo "$INPUT" | "$MEMPAL_PYTHON_BIN" -c "
+import sys, json, re
+data = json.load(sys.stdin)
+sid = data.get('session_id', 'unknown')
+tp = data.get('transcript_path', '')
+safe = lambda s: re.sub(r'[^a-zA-Z0-9_/.\-~]', '', str(s))
+print(safe(sid))
+print(safe(tp))
+" 2>/dev/null)
+SESSION_ID="${_mempal_parsed[0]:-unknown}"
+TRANSCRIPT_PATH="${_mempal_parsed[1]:-}"
+
+# Expand ~ in path
+TRANSCRIPT_PATH="${TRANSCRIPT_PATH/#\~/$HOME}"
+
+# Validate that TRANSCRIPT_PATH looks like a transcript file. Mirrors
+# mempalace.hooks_cli._validate_transcript_path so the shell hook
+# rejects the same shapes the Python hook rejects (#1231 review).
+is_valid_transcript_path() {
+    local path="$1"
+    [ -n "$path" ] || return 1
+    case "$path" in
+        *.json|*.jsonl) ;;
+        *) return 1 ;;
+    esac
+    case "/$path/" in
+        */../*) return 1 ;;
+    esac
+    return 0
+}
 
 echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$STATE_DIR/hook.log"
 
-# Optional: run mempalace ingest synchronously so memories land before compaction
+# Run ingest synchronously so memories land before compaction. Two
+# independent targets — both run if both are set:
+#   1. TRANSCRIPT_PATH (from Claude Code) → parent dir, --mode convos
+#   2. MEMPAL_DIR → --mode projects
+if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
+    mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
+        >> "$STATE_DIR/hook.log" 2>&1
+elif [ -n "$TRANSCRIPT_PATH" ]; then
+    echo "[$(date '+%H:%M:%S')] Skipping invalid transcript path: $TRANSCRIPT_PATH" \
+        >> "$STATE_DIR/hook.log"
+fi
 if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
-    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-    REPO_DIR="$(dirname "$SCRIPT_DIR")"
-    python3 -m mempalace mine "$MEMPAL_DIR" >> "$STATE_DIR/hook.log" 2>&1
+    mempalace mine "$MEMPAL_DIR" --mode projects \
+        >> "$STATE_DIR/hook.log" 2>&1
 fi
 
-# Always block — compaction = save everything
-cat << 'HOOKJSON'
-{
-  "decision": "block",
-  "reason": "COMPACTION IMMINENT. Save ALL topics, decisions, quotes, code, and important context from this session to your memory system. Be thorough — after compaction, detailed context will be lost. Organize into appropriate categories. Use verbatim quotes where possible. Save everything, then allow compaction to proceed."
-}
-HOOKJSON
+# Silent: return empty JSON to not block. "decision": "allow" is invalid —
+# only "block" or {} are recognized.
+echo '{}'

hooks/mempal_save_hook.sh

@@ -45,10 +45,10 @@
 # stop_hook_active=true so we let it through. No infinite loop.
 #
 # === MEMPALACE CLI ===
-# This repo uses: mempalace mine <dir>
-# or:            mempalace mine <dir> --mode convos
-# Set MEMPAL_DIR below if you want the hook to auto-ingest after blocking.
-# Leave blank to rely on the AI's own save instructions.
+# The hook ALWAYS mines the active conversation transcript automatically
+# (via `mempalace mine <transcript-dir> --mode convos`). MEMPAL_DIR is an
+# *additional*, optional target for project files — it does not replace
+# the conversation mine.
 #
 # === CONFIGURATION ===
 
@@ -56,22 +56,77 @@ SAVE_INTERVAL=15  # Save every N human messages (adjust to taste)
 STATE_DIR="$HOME/.mempalace/hook_state"
 mkdir -p "$STATE_DIR"
 
-# Optional: set to the directory you want auto-ingested on each save trigger.
-# Example: MEMPAL_DIR="$HOME/conversations"
-# Leave empty to skip auto-ingest (AI handles saving via the block reason).
+# Optional: project directory (code / notes / docs) to also mine each
+# save trigger. Mined with `--mode projects`. The conversation transcript
+# is always mined regardless — this is purely additive.
+# Example: MEMPAL_DIR="$HOME/projects/my_app"
 MEMPAL_DIR=""
 
+# Resolve the Python interpreter the hook should use.
+#
+# Why this is nontrivial: GUI-launched Claude Code on macOS (or any harness
+# that doesn't inherit the user's shell PATH) may find a `python3` on PATH
+# that lacks mempalace — e.g. /usr/bin/python3 while the user installed
+# mempalace into a venv or pyenv. Users in that situation can point the
+# hook at the right interpreter by exporting MEMPAL_PYTHON.
+#
+# Resolution order (first hit wins):
+#   1. $MEMPAL_PYTHON          — explicit user override (absolute path)
+#   2. $(command -v python3)   — first python3 on the hook's PATH
+#   3. bare "python3"          — last-resort fallback (hope the PATH has it)
+MEMPAL_PYTHON_BIN="${MEMPAL_PYTHON:-}"
+if [ -z "$MEMPAL_PYTHON_BIN" ] || [ ! -x "$MEMPAL_PYTHON_BIN" ]; then
+    MEMPAL_PYTHON_BIN="$(command -v python3 2>/dev/null || echo python3)"
+fi
+
 # Read JSON input from stdin
 INPUT=$(cat)
 
-# Parse fields from Claude Code's JSON
-SESSION_ID=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('session_id','unknown'))" 2>/dev/null)
-STOP_HOOK_ACTIVE=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('stop_hook_active', False))" 2>/dev/null)
-TRANSCRIPT_PATH=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('transcript_path',''))" 2>/dev/null)
+# Parse all fields in a single Python call (3x faster than separate invocations)
+# without invoking ``eval`` on generated code: Python prints one sanitized
+# value per line, the shell reads them via ``mapfile`` and does plain
+# variable assignment — same data, smaller blast radius if the sanitizer
+# is ever bypassed (#1231 review).
+mapfile -t _mempal_parsed < <(echo "$INPUT" | "$MEMPAL_PYTHON_BIN" -c "
+import sys, json, re
+data = json.load(sys.stdin)
+sid = data.get('session_id', 'unknown')
+sha_raw = data.get('stop_hook_active', False)
+tp = data.get('transcript_path', '')
+# Shell-safe output — only allow alphanumeric, underscore, hyphen, slash, dot, tilde
+safe = lambda s: re.sub(r'[^a-zA-Z0-9_/.\-~]', '', str(s))
+# Coerce stop_hook_active to strict boolean string
+sha = 'True' if sha_raw is True or str(sha_raw).lower() in ('true', '1', 'yes') else 'False'
+print(safe(sid))
+print(sha)
+print(safe(tp))
+" 2>/dev/null)
+SESSION_ID="${_mempal_parsed[0]:-unknown}"
+STOP_HOOK_ACTIVE="${_mempal_parsed[1]:-False}"
+TRANSCRIPT_PATH="${_mempal_parsed[2]:-}"
 
 # Expand ~ in path
 TRANSCRIPT_PATH="${TRANSCRIPT_PATH/#\~/$HOME}"
 
+# Validate that TRANSCRIPT_PATH looks like a transcript file:
+#   - non-empty
+#   - .jsonl or .json suffix
+#   - no traversal segments (.. components)
+# Mirrors mempalace.hooks_cli._validate_transcript_path so the shell hook
+# rejects the same shapes the Python hook rejects (#1231 review).
+is_valid_transcript_path() {
+    local path="$1"
+    [ -n "$path" ] || return 1
+    case "$path" in
+        *.json|*.jsonl) ;;
+        *) return 1 ;;
+    esac
+    case "/$path/" in
+        */../*) return 1 ;;
+    esac
+    return 0
+}
+
 # If we're already in a save cycle, let the AI stop normally
 # This is the infinite-loop prevention: block once → AI saves → tries to stop again → we let it through
 if [ "$STOP_HOOK_ACTIVE" = "True" ] || [ "$STOP_HOOK_ACTIVE" = "true" ]; then
@@ -80,25 +135,26 @@ if [ "$STOP_HOOK_ACTIVE" = "True" ] || [ "$STOP_HOOK_ACTIVE" = "true" ]; then
 fi
 
 # Count human messages in the JSONL transcript
+# SECURITY: Pass transcript path as sys.argv to avoid shell injection via crafted paths
 if [ -f "$TRANSCRIPT_PATH" ]; then
-    EXCHANGE_COUNT=$(python3 -c "
+    EXCHANGE_COUNT=$("$MEMPAL_PYTHON_BIN" - "$TRANSCRIPT_PATH" <<'PYEOF'
 import json, sys
 count = 0
-with open('$TRANSCRIPT_PATH') as f:
+with open(sys.argv[1]) as f:
     for line in f:
         try:
             entry = json.loads(line)
             msg = entry.get('message', {})
             if isinstance(msg, dict) and msg.get('role') == 'user':
                 content = msg.get('content', '')
-                # Skip system/command messages — only count real human input
                 if isinstance(content, str) and '<command-message>' in content:
                     continue
                 count += 1
         except:
             pass
 print(count)
-" 2>/dev/null)
+PYEOF
+2>/dev/null)
 else
     EXCHANGE_COUNT=0
 fi
@@ -107,7 +163,11 @@ fi
 LAST_SAVE_FILE="$STATE_DIR/${SESSION_ID}_last_save"
 LAST_SAVE=0
 if [ -f "$LAST_SAVE_FILE" ]; then
-    LAST_SAVE=$(cat "$LAST_SAVE_FILE")
+    LAST_SAVE_RAW=$(cat "$LAST_SAVE_FILE")
+    # SECURITY: Validate as plain integer before arithmetic to prevent command injection
+    if [[ "$LAST_SAVE_RAW" =~ ^[0-9]+$ ]]; then
+        LAST_SAVE="$LAST_SAVE_RAW"
+    fi
 fi
 
 SINCE_LAST=$((EXCHANGE_COUNT - LAST_SAVE))
@@ -122,21 +182,41 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then
 
     echo "[$(date '+%H:%M:%S')] TRIGGERING SAVE at exchange $EXCHANGE_COUNT" >> "$STATE_DIR/hook.log"
 
-    # Optional: run mempalace ingest in background if MEMPAL_DIR is set
+    # Auto-mine. Two independent targets — both run if both are set:
+    #   1. TRANSCRIPT_PATH (from Claude Code) → parent dir, --mode convos
+    #      (Claude Code session JSONL — must use the convo miner)
+    #   2. MEMPAL_DIR (user-configured project) → --mode projects
+    #      (code, notes, docs)
+    # MEMPAL_DIR is *additive*, not an override: a user with MEMPAL_DIR
+    # pointed at their project still gets the active conversation mined.
+    if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
+        mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
+            >> "$STATE_DIR/hook.log" 2>&1 &
+    elif [ -n "$TRANSCRIPT_PATH" ]; then
+        echo "[$(date '+%H:%M:%S')] Skipping invalid transcript path: $TRANSCRIPT_PATH" \
+            >> "$STATE_DIR/hook.log"
+    fi
     if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
-        SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-        REPO_DIR="$(dirname "$SCRIPT_DIR")"
-        python3 -m mempalace mine "$MEMPAL_DIR" >> "$STATE_DIR/hook.log" 2>&1 &
+        mempalace mine "$MEMPAL_DIR" --mode projects \
+            >> "$STATE_DIR/hook.log" 2>&1 &
     fi
 
-    # Block the AI and tell it to save
-    # The "reason" becomes a system message the AI sees and acts on
-    cat << 'HOOKJSON'
+    # MEMPAL_VERBOSE toggle:
+    #   true  = developer mode — block and show diaries/code in chat
+    #   false = silent mode (default) — save in background, no chat clutter
+    # Set via: export MEMPAL_VERBOSE=true
+    if [ "$MEMPAL_VERBOSE" = "true" ] || [ "$MEMPAL_VERBOSE" = "1" ]; then
+        cat << 'HOOKJSON'
 {
   "decision": "block",
-  "reason": "AUTO-SAVE checkpoint. Save key topics, decisions, quotes, and code from this session to your memory system. Organize into appropriate categories. Use verbatim quotes where possible. Continue conversation after saving."
+  "reason": "MemPalace save checkpoint. Write a brief session diary entry covering key topics, decisions, and code changes since the last save. Use verbatim quotes where possible. Continue after saving."
 }
 HOOKJSON
+    else
+        # Silent mode: return empty JSON to not block. "decision": "allow" is
+        # not a valid value — only "block" or {} are recognized.
+        echo '{}'
+    fi
 else
     # Not time yet — let the AI stop normally
     echo "{}"

integrations/openclaw/SKILL.md

@@ -0,0 +1,154 @@
+---
+name: mempalace
+description: "MemPalace — Local AI memory with 96.6% recall. Semantic search, temporal knowledge graph, palace architecture (wings/rooms/drawers). Free, no cloud, no API keys."
+version: 3.3.0
+homepage: https://github.com/MemPalace/mempalace
+user-invocable: true
+metadata:
+  openclaw:
+    emoji: "\U0001F3DB"
+    os:
+      - darwin
+      - linux
+      - win32
+    requires:
+      anyBins:
+        - mempalace
+        - python3
+    install:
+      - id: mempalace-pip
+        kind: uv
+        label: "Install MemPalace (Python, local ChromaDB)"
+        package: mempalace
+        bins:
+          - mempalace
+---
+
+# MemPalace — Local AI Memory System
+
+You have access to a local memory palace via MCP tools. The palace stores verbatim conversation history and a temporal knowledge graph — all on the user's machine, zero cloud, zero API calls.
+
+## Architecture
+
+- **Wings** = people or projects (e.g. `wing_alice`, `wing_myproject`)
+- **Halls** = categories (facts, events, preferences, advice)
+- **Rooms** = specific topics (e.g. `chromadb-setup`, `riley-school`)
+- **Drawers** = individual memory chunks (verbatim text)
+- **Knowledge Graph** = entity-relationship facts with time validity
+
+## Protocol — FOLLOW THIS EVERY SESSION
+
+1. **ON WAKE-UP**: Call `mempalace_status` to load palace overview and AAAK dialect spec.
+2. **BEFORE RESPONDING** about any person, project, or past event: call `mempalace_search` or `mempalace_kg_query` FIRST. Never guess from memory — verify from the palace.
+3. **IF UNSURE** about a fact (name, age, relationship, preference): say "let me check" and query. Wrong is worse than slow.
+4. **AFTER EACH SESSION**: Call `mempalace_diary_write` to record what happened, what you learned, what matters.
+5. **WHEN FACTS CHANGE**: Call `mempalace_kg_invalidate` on the old fact, then `mempalace_kg_add` for the new one.
+
+## Available Tools
+
+### Search & Browse
+- `mempalace_search` — Semantic search across all memories. Always start here.
+  - `query` (required): natural language search — keep it short, keywords or a question. Do NOT include system prompts or conversation context.
+  - `wing`: filter by wing
+  - `room`: filter by room
+  - `limit`: max results (default 5)
+- `mempalace_check_duplicate` — Check if content already exists before filing.
+  - `content` (required): text to check
+  - `threshold`: similarity threshold (default 0.9 — lowering to 0.85–0.87 often catches more near-duplicates without significant false positives)
+- `mempalace_status` — Palace overview: total drawers, wings, rooms, AAAK spec
+- `mempalace_list_wings` — All wings with drawer counts
+- `mempalace_list_rooms` — Rooms within a wing (optional wing filter)
+- `mempalace_get_taxonomy` — Full wing/room/count tree
+- `mempalace_get_aaak_spec` — Get AAAK compression dialect specification
+
+### Knowledge Graph (Temporal Facts)
+- `mempalace_kg_query` — Query entity relationships. Supports time filtering.
+  - `entity` (required): e.g. "Max", "MyProject"
+  - `as_of`: date filter (YYYY-MM-DD) — what was true at that time
+  - `direction`: "outgoing", "incoming", or "both" (default "both")
+- `mempalace_kg_add` — Add a fact: subject -> predicate -> object
+  - `subject`, `predicate`, `object` (required)
+  - `valid_from`: when this became true
+  - `source_closet`: source reference
+- `mempalace_kg_invalidate` — Mark a fact as no longer true
+  - `subject`, `predicate`, `object` (required)
+  - `ended`: when it stopped being true (default: today)
+- `mempalace_kg_timeline` — Chronological story of an entity
+  - `entity`: filter by entity name (optional — all events if omitted)
+- `mempalace_kg_stats` — Graph overview: entities, triples, relationship types
+
+### Palace Graph (Cross-Domain Connections)
+- `mempalace_traverse` — Walk from a room, find connected ideas across wings
+  - `start_room` (required): room to start from
+  - `max_hops`: connection depth (default 2)
+- `mempalace_find_tunnels` — Find rooms that bridge two wings
+  - `wing_a`, `wing_b` (required)
+- `mempalace_graph_stats` — Graph connectivity overview
+
+### Write
+- `mempalace_add_drawer` — Store verbatim content into a wing/room
+  - `wing`, `room`, `content` (required)
+  - `source_file`: optional source reference
+  - Checks for duplicates automatically
+- `mempalace_delete_drawer` — Remove a drawer by ID
+  - `drawer_id` (required)
+- `mempalace_diary_write` — Write a session diary entry
+  - `agent_name` (required): your name/identifier
+  - `entry` (required): what happened, what you learned, what matters
+  - `topic`: category tag (default "general")
+- `mempalace_diary_read` — Read recent diary entries
+  - `agent_name` (required)
+  - `last_n`: number of entries (default 10)
+
+## Setup
+
+Install MemPalace and populate the palace:
+
+```bash
+pip install mempalace
+mempalace init ~/my-convos
+mempalace mine ~/my-convos
+```
+
+### OpenClaw MCP config
+
+Add to your OpenClaw MCP configuration:
+
+```json
+{
+  "mcpServers": {
+    "mempalace": {
+      "command": "python3",
+      "args": ["-m", "mempalace.mcp_server"]
+    }
+  }
+}
+```
+
+Or via CLI:
+
+```bash
+openclaw mcp set mempalace '{"command":"python3","args":["-m","mempalace.mcp_server"]}'
+```
+
+### Other MCP hosts
+
+```bash
+# Claude Code
+claude mcp add mempalace -- python -m mempalace.mcp_server
+
+# Cursor — add to .cursor/mcp.json
+# Codex — add to .codex/mcp.json
+```
+
+## Tips
+
+- Search is semantic (meaning-based), not keyword. "What did we discuss about database performance?" works better than "database".
+- The knowledge graph stores typed relationships with time windows. Use it for facts about people and projects — it knows WHEN things were true.
+- Diary entries accumulate across sessions. Write one at the end of each conversation to build continuity.
+- Use `mempalace_check_duplicate` before storing new content to avoid duplicates.
+- The AAAK dialect (from `mempalace_status`) is a compressed notation for efficient storage. Read it naturally — expand codes mentally, treat *markers* as emotional context.
+
+## License
+
+[MemPalace](https://github.com/MemPalace/mempalace) is MIT licensed. Created by Milla Jovovich, Ben Sigman, Igor Lins e Silva, and contributors.

mempalace/__init__.py

@@ -1,7 +1,27 @@
 """MemPalace — Give your AI a memory. No API key required."""
 
-__version__ = "2.0.0"
+import logging
 
-from .cli import main
+from .version import __version__  # noqa: E402
 
-__all__ = ["main", "__version__"]
+# chromadb telemetry: posthog capture() was broken in 0.6.x causing noisy stderr
+# warnings ("capture() takes 1 positional argument but 3 were given"). In 1.x the
+# posthog client is a no-op stub, so this is now harmless — kept as a guard in
+# case future chromadb versions re-introduce real telemetry calls.
+logging.getLogger("chromadb.telemetry.product.posthog").setLevel(logging.CRITICAL)
+
+# NOTE: the previous block set ``ORT_DISABLE_COREML=1`` on macOS arm64 as a
+# supposed workaround for the #74 ARM64 segfault.  Two problems:
+#
+# 1. ONNX Runtime does not read that env var -- it has no global way to
+#    disable a single execution provider, so the setdefault was a no-op.
+# 2. #74 is a null-pointer crash in ``chromadb_rust_bindings.abi3.so``, not
+#    an ONNX issue, so disabling CoreML would not have fixed it anyway.
+#
+# #521 has since traced the actual macOS arm64 crashes (both in mine and
+# search paths) to the 0.x chromadb hnswlib binding.  Filtering
+# CoreMLExecutionProvider at the ONNX layer leaves the hnswlib C++ crash
+# intact, so the real fix is upgrading chromadb to 1.5.4+, which #581
+# proposes.  See #397 for the history of this line.
+
+__all__ = ["__version__"]

mempalace/backends/__init__.py

@@ -0,0 +1,64 @@
+"""Storage backend implementations for MemPalace (RFC 001).
+
+Public surface:
+
+* :class:`BaseCollection` — per-collection read/write contract.
+* :class:`BaseBackend` — per-palace factory contract.
+* :class:`PalaceRef` — value object identifying a palace for a backend.
+* :class:`QueryResult` / :class:`GetResult` — typed read returns.
+* Error classes: :class:`PalaceNotFoundError`, :class:`BackendClosedError`,
+  :class:`UnsupportedFilterError`, :class:`DimensionMismatchError`,
+  :class:`EmbedderIdentityMismatchError`.
+* Registry: :func:`get_backend`, :func:`register`, :func:`available_backends`,
+  :func:`resolve_backend_for_palace`.
+* In-tree Chroma default: :class:`ChromaBackend`, :class:`ChromaCollection`.
+"""
+
+from .base import (
+    BackendClosedError,
+    BackendError,
+    BaseBackend,
+    BaseCollection,
+    DimensionMismatchError,
+    EmbedderIdentityMismatchError,
+    GetResult,
+    HealthStatus,
+    PalaceNotFoundError,
+    PalaceRef,
+    QueryResult,
+    UnsupportedFilterError,
+)
+from .chroma import ChromaBackend, ChromaCollection
+from .registry import (
+    available_backends,
+    get_backend,
+    get_backend_class,
+    register,
+    reset_backends,
+    resolve_backend_for_palace,
+    unregister,
+)
+
+__all__ = [
+    "BackendClosedError",
+    "BackendError",
+    "BaseBackend",
+    "BaseCollection",
+    "ChromaBackend",
+    "ChromaCollection",
+    "DimensionMismatchError",
+    "EmbedderIdentityMismatchError",
+    "GetResult",
+    "HealthStatus",
+    "PalaceNotFoundError",
+    "PalaceRef",
+    "QueryResult",
+    "UnsupportedFilterError",
+    "available_backends",
+    "get_backend",
+    "get_backend_class",
+    "register",
+    "reset_backends",
+    "resolve_backend_for_palace",
+    "unregister",
+]

mempalace/backends/base.py

@@ -0,0 +1,370 @@
+"""Storage backend contract for MemPalace (RFC 001).
+
+This module defines the surface every storage backend must implement:
+
+* ``BaseCollection`` — the per-collection read/write interface, kwargs-only.
+* ``BaseBackend`` — the per-palace factory, addressed by ``PalaceRef``.
+* ``QueryResult`` / ``GetResult`` — typed result dataclasses that replace the
+  Chroma dict shape as the canonical return type.
+* Error classes + ``HealthStatus`` — uniform across backends.
+
+This is the v1 cleanup from RFC 001 §10: full typed results, ``PalaceRef``,
+registry-ready ABC. Embedder injection, maintenance hooks, and the full
+conformance suite land in follow-up PRs.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import ClassVar, Optional
+
+
+# ---------------------------------------------------------------------------
+# Errors
+# ---------------------------------------------------------------------------
+
+
+class BackendError(Exception):
+    """Base class for every storage-backend error raised by core."""
+
+
+class PalaceNotFoundError(BackendError, FileNotFoundError):
+    """Raised when ``get_collection(create=False)`` is called on a missing palace.
+
+    Subclass of ``FileNotFoundError`` so legacy callers that catch the latter
+    (pre-#413 seam) keep working unchanged.
+    """
+
+
+class BackendClosedError(BackendError):
+    """Raised when a backend method is called after ``close()``."""
+
+
+class UnsupportedFilterError(BackendError):
+    """Raised when a where-clause uses an operator the backend does not implement.
+
+    Silent dropping of unknown operators is forbidden by spec (RFC 001 §1.4).
+    """
+
+
+class DimensionMismatchError(BackendError):
+    """Raised when the embedding dimension on write does not match the collection."""
+
+
+class EmbedderIdentityMismatchError(BackendError):
+    """Raised when the stored embedder model name differs from the current one."""
+
+
+# ---------------------------------------------------------------------------
+# Value objects
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class PalaceRef:
+    """A handle to a palace, consumed by backends.
+
+    ``id`` is always present and is the key backends use to cache handles.
+    ``local_path`` is populated for filesystem-rooted palaces.
+    ``namespace`` is used by server-mode backends for tenant / prefix routing.
+    """
+
+    id: str
+    local_path: Optional[str] = None
+    namespace: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class HealthStatus:
+    ok: bool
+    detail: str = ""
+
+    @classmethod
+    def healthy(cls, detail: str = "") -> "HealthStatus":
+        return cls(ok=True, detail=detail)
+
+    @classmethod
+    def unhealthy(cls, detail: str) -> "HealthStatus":
+        return cls(ok=False, detail=detail)
+
+
+_TYPED_RESULT_FIELDS = ("ids", "documents", "metadatas", "distances", "embeddings")
+
+
+class _DictCompatMixin:
+    """Transitional dict-protocol access for typed results.
+
+    RFC 001 §1.3 spec is attribute access (``result.ids``). The ``result["ids"]``
+    and ``result.get("ids")`` forms are retained as a migration shim for callers
+    that predate the typed interface and are scheduled for removal in a follow-
+    up cleanup. New code MUST use attribute access.
+    """
+
+    def __getitem__(self, key: str):
+        if key in _TYPED_RESULT_FIELDS:
+            return getattr(self, key)
+        raise KeyError(key)
+
+    def get(self, key: str, default=None):
+        if key in _TYPED_RESULT_FIELDS:
+            val = getattr(self, key, default)
+            return default if val is None else val
+        return default
+
+    def __contains__(self, key: object) -> bool:
+        return key in _TYPED_RESULT_FIELDS and getattr(self, key, None) is not None
+
+
+@dataclass(frozen=True)
+class QueryResult(_DictCompatMixin):
+    """Typed return from ``BaseCollection.query``.
+
+    Outer list dimension = number of query vectors / texts.
+    Inner list dimension = hits per query (may be zero).
+
+    Fields not in ``include=`` at the call site are populated with empty lists
+    of the correct outer shape (never ``None``), except ``embeddings`` which
+    is ``None`` when not requested.
+    """
+
+    ids: list[list[str]]
+    documents: list[list[str]]
+    metadatas: list[list[dict]]
+    distances: list[list[float]]
+    embeddings: Optional[list[list[list[float]]]] = None
+
+    @classmethod
+    def empty(cls, num_queries: int = 1, embeddings_requested: bool = False) -> "QueryResult":
+        """Construct an all-empty result preserving outer dimension.
+
+        When ``embeddings_requested`` is True, ``embeddings`` preserves the outer
+        query dimension with empty hit lists (matching the spec's rule that fields
+        requested via ``include=`` carry the outer shape even when empty). When
+        False, ``embeddings`` stays ``None`` to signal the field was not requested.
+        """
+        empty_outer = [[] for _ in range(num_queries)]
+        return cls(
+            ids=[[] for _ in range(num_queries)],
+            documents=[[] for _ in range(num_queries)],
+            metadatas=[[] for _ in range(num_queries)],
+            distances=[[] for _ in range(num_queries)],
+            embeddings=empty_outer if embeddings_requested else None,
+        )
+
+
+@dataclass(frozen=True)
+class GetResult(_DictCompatMixin):
+    """Typed return from ``BaseCollection.get``."""
+
+    ids: list[str]
+    documents: list[str]
+    metadatas: list[dict]
+    embeddings: Optional[list[list[float]]] = None
+
+    @classmethod
+    def empty(cls) -> "GetResult":
+        return cls(ids=[], documents=[], metadatas=[], embeddings=None)
+
+
+# ---------------------------------------------------------------------------
+# Collection contract
+# ---------------------------------------------------------------------------
+
+
+class BaseCollection(ABC):
+    """Per-collection read/write surface every backend must implement."""
+
+    @abstractmethod
+    def add(
+        self,
+        *,
+        documents: list[str],
+        ids: list[str],
+        metadatas: Optional[list[dict]] = None,
+        embeddings: Optional[list[list[float]]] = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def upsert(
+        self,
+        *,
+        documents: list[str],
+        ids: list[str],
+        metadatas: Optional[list[dict]] = None,
+        embeddings: Optional[list[list[float]]] = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def query(
+        self,
+        *,
+        query_texts: Optional[list[str]] = None,
+        query_embeddings: Optional[list[list[float]]] = None,
+        n_results: int = 10,
+        where: Optional[dict] = None,
+        where_document: Optional[dict] = None,
+        include: Optional[list[str]] = None,
+    ) -> QueryResult: ...
+
+    @abstractmethod
+    def get(
+        self,
+        *,
+        ids: Optional[list[str]] = None,
+        where: Optional[dict] = None,
+        where_document: Optional[dict] = None,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        include: Optional[list[str]] = None,
+    ) -> GetResult: ...
+
+    @abstractmethod
+    def delete(
+        self,
+        *,
+        ids: Optional[list[str]] = None,
+        where: Optional[dict] = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def count(self) -> int: ...
+
+    # ------------------------------------------------------------------
+    # Optional methods with ABC defaults (spec §1.2)
+    # ------------------------------------------------------------------
+
+    def estimated_count(self) -> int:
+        return self.count()
+
+    def close(self) -> None:
+        return None
+
+    def health(self) -> HealthStatus:
+        return HealthStatus.healthy()
+
+    def update(
+        self,
+        *,
+        ids: list[str],
+        documents: Optional[list[str]] = None,
+        metadatas: Optional[list[dict]] = None,
+        embeddings: Optional[list[list[float]]] = None,
+    ) -> None:
+        """Default non-atomic update: get + merge + upsert.
+
+        Backends advertising ``supports_update`` MUST override with an atomic
+        single-round-trip implementation.
+        """
+        if documents is None and metadatas is None and embeddings is None:
+            raise ValueError("update requires at least one of documents, metadatas, embeddings")
+
+        n = len(ids)
+        for label, value in (
+            ("documents", documents),
+            ("metadatas", metadatas),
+            ("embeddings", embeddings),
+        ):
+            if value is not None and len(value) != n:
+                raise ValueError(f"{label} length {len(value)} does not match ids length {n}")
+
+        existing = self.get(ids=ids, include=["documents", "metadatas"])
+        by_id = {
+            rid: (existing.documents[i], existing.metadatas[i])
+            for i, rid in enumerate(existing.ids)
+        }
+        merged_docs: list[str] = []
+        merged_metas: list[dict] = []
+        for i, rid in enumerate(ids):
+            prev_doc, prev_meta = by_id.get(rid, ("", {}))
+            merged_docs.append(documents[i] if documents is not None else prev_doc)
+            new_meta = dict(prev_meta or {})
+            if metadatas is not None:
+                new_meta.update(metadatas[i] or {})
+            merged_metas.append(new_meta)
+        self.upsert(
+            documents=merged_docs,
+            ids=list(ids),
+            metadatas=merged_metas,
+            embeddings=embeddings,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Backend contract
+# ---------------------------------------------------------------------------
+
+
+class BaseBackend(ABC):
+    """Long-lived factory serving many palaces (RFC 001 §2).
+
+    Instances are lightweight on construction — no I/O, no network. All
+    connection work is deferred to ``get_collection``. Instances are thread-
+    safe for concurrent ``get_collection`` calls across different palaces.
+    """
+
+    name: ClassVar[str]
+    spec_version: ClassVar[str] = "1.0"
+    capabilities: ClassVar[frozenset[str]] = frozenset()
+
+    @abstractmethod
+    def get_collection(
+        self,
+        *,
+        palace: PalaceRef,
+        collection_name: str,
+        create: bool = False,
+        options: Optional[dict] = None,
+    ) -> BaseCollection: ...
+
+    def close_palace(self, palace: PalaceRef) -> None:
+        """Evict cached handles for a single palace. Default: no-op."""
+        return None
+
+    def close(self) -> None:
+        """Shut down the entire backend. Default: no-op."""
+        return None
+
+    def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus:
+        return HealthStatus.healthy()
+
+    # Optional detection hint used by selection priority (RFC 001 §3.3 (4)):
+    @classmethod
+    def detect(cls, path: str) -> bool:  # pragma: no cover - default hook
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Adapter utilities
+# ---------------------------------------------------------------------------
+
+
+# Keys the Chroma ``include=`` parameter accepts.
+_VALID_INCLUDE_KEYS = frozenset({"documents", "metadatas", "distances", "embeddings"})
+
+
+@dataclass
+class _IncludeSpec:
+    """Resolve an ``include=`` parameter with spec-mandated defaults."""
+
+    documents: bool = True
+    metadatas: bool = True
+    distances: bool = True  # only meaningful for query
+    embeddings: bool = False
+
+    @classmethod
+    def resolve(
+        cls, include: Optional[list[str]], *, default_distances: bool = True
+    ) -> "_IncludeSpec":
+        if include is None:
+            return cls(
+                documents=True,
+                metadatas=True,
+                distances=default_distances,
+                embeddings=False,
+            )
+        keys = {k for k in include if k in _VALID_INCLUDE_KEYS}
+        return cls(
+            documents="documents" in keys,
+            metadatas="metadatas" in keys,
+            distances="distances" in keys,
+            embeddings="embeddings" in keys,
+        )

mempalace/backends/registry.py

@@ -0,0 +1,189 @@
+"""Backend registry + entry-point discovery (RFC 001 §3).
+
+Third-party backends ship as installable packages that declare a
+``mempalace.backends`` entry point::
+
+    # pyproject.toml of mempalace-postgres
+    [project.entry-points."mempalace.backends"]
+    postgres = "mempalace_postgres:PostgresBackend"
+
+MemPalace discovers them at process start. In-tree tests and local development
+can register manually via :func:`register`. Explicit registration wins on
+name conflict (matches RFC 001 §3.2).
+"""
+
+from __future__ import annotations
+
+import logging
+from importlib import metadata
+from threading import Lock
+from typing import Optional, Type
+
+from .base import BaseBackend
+
+logger = logging.getLogger(__name__)
+
+_ENTRY_POINT_GROUP = "mempalace.backends"
+
+_registry: dict[str, Type[BaseBackend]] = {}
+_instances: dict[str, BaseBackend] = {}
+_explicit: set[str] = set()
+_discovered = False
+_lock = Lock()
+
+
+def register(name: str, backend_cls: Type[BaseBackend]) -> None:
+    """Register ``backend_cls`` under ``name``.
+
+    Explicit registration wins over entry-point discovery on conflict
+    (RFC 001 §3.2).
+    """
+    with _lock:
+        _registry[name] = backend_cls
+        _explicit.add(name)
+        # Invalidate any cached instance so the new class is used on next get.
+        _instances.pop(name, None)
+
+
+def unregister(name: str) -> None:
+    """Remove a backend registration (primarily for tests)."""
+    with _lock:
+        _registry.pop(name, None)
+        _explicit.discard(name)
+        _instances.pop(name, None)
+
+
+def _discover_entry_points() -> None:
+    """Load entry-point-declared backends once per process."""
+    global _discovered
+    if _discovered:
+        return
+    with _lock:
+        if _discovered:
+            return
+        try:
+            eps = metadata.entry_points()
+            # Py ≥ 3.10 returns an EntryPoints object; older versions returned a dict.
+            group = (
+                eps.select(group=_ENTRY_POINT_GROUP)
+                if hasattr(eps, "select")
+                else eps.get(_ENTRY_POINT_GROUP, [])
+            )
+        except Exception:
+            logger.exception("entry-point discovery for %s failed", _ENTRY_POINT_GROUP)
+            group = []
+        for ep in group:
+            if ep.name in _explicit:
+                continue  # explicit registration wins
+            try:
+                cls = ep.load()
+            except Exception:
+                logger.exception("failed to load backend entry point %r", ep.name)
+                continue
+            if not isinstance(cls, type) or not issubclass(cls, BaseBackend):
+                logger.warning(
+                    "entry point %r did not resolve to a BaseBackend subclass (got %r)",
+                    ep.name,
+                    cls,
+                )
+                continue
+            _registry.setdefault(ep.name, cls)
+        _discovered = True
+
+
+def available_backends() -> list[str]:
+    """Return sorted list of all registered backend names."""
+    _discover_entry_points()
+    return sorted(_registry.keys())
+
+
+def get_backend_class(name: str) -> Type[BaseBackend]:
+    """Return the registered backend class for ``name``."""
+    _discover_entry_points()
+    try:
+        return _registry[name]
+    except KeyError as e:
+        raise KeyError(f"unknown backend {name!r}; available: {available_backends()}") from e
+
+
+def get_backend(name: str) -> BaseBackend:
+    """Return a long-lived instance of the named backend.
+
+    Instances are cached per-name; repeated calls return the same object.
+    Call :func:`reset_backends` in tests that need isolation.
+    """
+    _discover_entry_points()
+    with _lock:
+        inst = _instances.get(name)
+        if inst is not None:
+            return inst
+        cls = _registry.get(name)
+        if cls is None:
+            raise KeyError(f"unknown backend {name!r}; available: {sorted(_registry.keys())}")
+        inst = cls()
+        _instances[name] = inst
+        return inst
+
+
+def reset_backends() -> None:
+    """Close and drop all cached backend instances (primarily for tests)."""
+    with _lock:
+        for inst in _instances.values():
+            try:
+                inst.close()
+            except Exception:
+                logger.exception("error closing backend during reset")
+        _instances.clear()
+
+
+def resolve_backend_for_palace(
+    *,
+    explicit: Optional[str] = None,
+    config_value: Optional[str] = None,
+    env_value: Optional[str] = None,
+    palace_path: Optional[str] = None,
+    default: str = "chroma",
+) -> str:
+    """Resolve the backend name for a palace per RFC 001 §3.3 priority order.
+
+    1. Explicit kwarg / CLI flag
+    2. Per-palace config value
+    3. ``MEMPALACE_BACKEND`` env var
+    4. Auto-detect from on-disk artifacts (migration/upgrade path only)
+    5. Default (``chroma``)
+
+    Auto-detection is strictly a migration aid: it fires only when a local path
+    is presented, no earlier rule has chosen a backend, AND the path already
+    contains backend-identifiable artifacts. For new palaces, (5) wins.
+    """
+    for candidate in (explicit, config_value, env_value):
+        if candidate:
+            return candidate
+
+    _discover_entry_points()
+    if palace_path:
+        for name, cls in _registry.items():
+            try:
+                if cls.detect(palace_path):
+                    return name
+            except Exception:
+                logger.exception("detect() raised on backend %r", name)
+                continue
+    return default
+
+
+# ---------------------------------------------------------------------------
+# Built-in registration
+# ---------------------------------------------------------------------------
+
+
+def _register_builtins() -> None:
+    """Register chroma as the in-tree default."""
+    from .chroma import ChromaBackend
+
+    # Use setdefault semantics so a caller that pre-registered for tests wins.
+    if "chroma" not in _registry:
+        _registry["chroma"] = ChromaBackend
+
+
+_register_builtins()

mempalace/closet_llm.py

@@ -0,0 +1,351 @@
+"""
+closet_llm.py — Generate closets via a user-configured LLM for richer indexing.
+
+The regex-based closet extraction catches action verbs, headers, and proper
+nouns — but misses implicit topics, foreign-language content, and contextual
+references. An LLM reads everything and produces better closets.
+
+This module is **OPTIONAL and opt-in**. Regex closets are always created by
+the miner; this path regenerates them afterward using whatever LLM the user
+chooses. Core memory operations remain API-free by design (see CLAUDE.md,
+"Local-first, zero API").
+
+## Bring-your-own-LLM configuration
+
+The endpoint is any OpenAI-compatible Chat Completions URL:
+
+    LLM_ENDPOINT=http://localhost:11434/v1   # Ollama
+    LLM_ENDPOINT=http://localhost:8000/v1    # vLLM, llama.cpp
+    LLM_ENDPOINT=https://api.openai.com/v1
+    LLM_ENDPOINT=https://openrouter.ai/api/v1
+    LLM_ENDPOINT=https://api.anthropic.com/v1  # when proxied through a compat layer
+
+Set:
+    LLM_ENDPOINT — base URL (required)
+    LLM_KEY      — bearer token (optional; local inference usually doesn't need it)
+    LLM_MODEL    — model name (required), e.g. "gpt-4o-mini", "llama3:8b", "qwen2.5:7b"
+
+Or pass flags on the CLI (flags win over env):
+
+    python -m mempalace.closet_llm \\
+        --palace ~/.mempalace/palace \\
+        --endpoint http://localhost:11434/v1 \\
+        --model llama3:8b
+
+No vendor lock-in. No hidden dependency on any specific provider. Zero deps
+added to pyproject — uses stdlib urllib.
+"""
+
+import json
+import os
+import re
+import time
+import urllib.request
+import urllib.error
+from datetime import datetime
+from typing import Optional
+
+from .palace import (
+    NORMALIZE_VERSION,
+    get_closets_collection,
+    get_collection,
+    mine_lock,
+    purge_file_closets,
+    upsert_closet_lines,
+)
+
+MAX_CONTENT_CHARS = 30000
+MAX_OUTPUT_TOKENS = 1500
+HTTP_TIMEOUT_S = 60
+
+PROMPT_TEMPLATE = """You are reading content filed in a memory palace. Generate a
+topic-dense index that will be used to find this content later when someone searches.
+
+Source: {source_file}
+Wing: {wing} | Room: {room}
+
+CONTENT:
+{content}
+
+---
+
+Output a JSON object with EXACTLY these fields:
+
+{{
+  "topics": ["distinctive_word_or_phrase_1", "topic_2", ...],
+  "quotes": ["[Speaker] verbatim quote", ...],
+  "summary": "2-3 sentences describing what this content is about."
+}}
+
+RULES:
+- Topics: 8-15 entries. Include proper nouns (names, places, projects),
+  distinctive technical terms, and key concepts. NOT generic words like
+  "conversation" or "discussion".
+- Quotes: 2-5 entries. EXACT verbatim from the content, not paraphrased.
+  Attribute with [Speaker] prefix if speaker is identifiable.
+- Summary: mention WHO, WHAT, and WHY. No filler.
+- Write in the same language as the content.
+- Output valid JSON only. No code fences. No commentary.
+"""
+
+
+class LLMConfig:
+    """Resolved LLM connection config. CLI flags > env vars."""
+
+    def __init__(
+        self,
+        endpoint: Optional[str] = None,
+        key: Optional[str] = None,
+        model: Optional[str] = None,
+    ):
+        self.endpoint = (endpoint or os.environ.get("LLM_ENDPOINT", "")).rstrip("/")
+        self.key = key or os.environ.get("LLM_KEY", "")
+        self.model = model or os.environ.get("LLM_MODEL", "")
+
+    def missing(self) -> list:
+        missing = []
+        if not self.endpoint:
+            missing.append("LLM_ENDPOINT (or --endpoint)")
+        if not self.model:
+            missing.append("LLM_MODEL (or --model)")
+        # key is optional — local inference servers (Ollama, vLLM) often don't require one
+        return missing
+
+
+def _call_llm(cfg: LLMConfig, source_file: str, wing: str, room: str, content: str):
+    """Single LLM call via OpenAI-compatible /chat/completions.
+
+    Returns (parsed_json_dict_or_None, usage_dict_or_None).
+    """
+    try:
+        from mempalace.i18n import t
+
+        lang_instruction = t("aaak.instruction")
+    except Exception:
+        lang_instruction = ""
+
+    prompt = PROMPT_TEMPLATE.format(
+        source_file=source_file[:100],
+        wing=wing,
+        room=room,
+        content=content[:MAX_CONTENT_CHARS],
+    )
+    if lang_instruction and "english" not in lang_instruction.lower():
+        prompt += f"\n\nLanguage instruction: {lang_instruction}"
+
+    body = json.dumps(
+        {
+            "model": cfg.model,
+            "max_tokens": MAX_OUTPUT_TOKENS,
+            "messages": [{"role": "user", "content": prompt}],
+        }
+    ).encode("utf-8")
+
+    headers = {"Content-Type": "application/json"}
+    if cfg.key:
+        headers["Authorization"] = f"Bearer {cfg.key}"
+
+    url = f"{cfg.endpoint}/chat/completions"
+
+    for attempt in range(3):
+        try:
+            req = urllib.request.Request(url, data=body, headers=headers, method="POST")
+            with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT_S) as resp:
+                raw = resp.read().decode("utf-8")
+            payload = json.loads(raw)
+
+            text = payload["choices"][0]["message"]["content"].strip()
+            text = re.sub(r"^```(?:json)?\s*", "", text)
+            text = re.sub(r"\s*```$", "", text)
+            parsed = json.loads(text)
+            return parsed, payload.get("usage")
+        except json.JSONDecodeError:
+            return None, None
+        except urllib.error.HTTPError as e:
+            # 429 / 503 = retry with backoff
+            if e.code in (429, 503) and attempt < 2:
+                time.sleep(2**attempt)
+                continue
+            return None, None
+        except Exception as e:
+            if "rate" in str(e).lower() and attempt < 2:
+                time.sleep(2**attempt)
+                continue
+            return None, None
+    return None, None
+
+
+def _parsed_to_closet_lines(parsed, drawer_ids, entities_str):
+    """Convert LLM's JSON output to closet pointer lines."""
+    lines = []
+    drawer_ref = ",".join(drawer_ids[:3])
+
+    for topic in parsed.get("topics", [])[:15]:
+        lines.append(f"{topic}|{entities_str}|→{drawer_ref}")
+    for quote in parsed.get("quotes", [])[:5]:
+        lines.append(f"{quote}|{entities_str}|→{drawer_ref}")
+    summary = parsed.get("summary", "")
+    if summary:
+        lines.append(f"{summary[:200]}|{entities_str}|→{drawer_ref}")
+
+    return lines
+
+
+def regenerate_closets(
+    palace_path,
+    wing=None,
+    sample=0,
+    dry_run=False,
+    cfg: Optional[LLMConfig] = None,
+):
+    """Regenerate closets using a configured LLM for richer topic extraction.
+
+    Reads existing drawers, sends content to the configured endpoint,
+    replaces regex closets with LLM-generated ones. Regex closets remain
+    as the fallback whenever the call fails.
+    """
+    if cfg is None:
+        cfg = LLMConfig()
+    missing = cfg.missing()
+    if missing:
+        print("Error: missing configuration: " + ", ".join(missing))
+        print("Set env vars LLM_ENDPOINT / LLM_MODEL (and optionally LLM_KEY),")
+        print("or pass --endpoint / --model / --key on the CLI.")
+        return {"error": "missing-config", "missing": missing}
+
+    drawers_col = get_collection(palace_path, create=False)
+    closets_col = get_closets_collection(palace_path)
+
+    total = drawers_col.count()
+    if total == 0:
+        print("No drawers in palace.")
+        return {"processed": 0}
+
+    all_data = drawers_col.get(limit=total, include=["documents", "metadatas"])
+    by_source = {}
+    for doc_id, doc, meta in zip(all_data["ids"], all_data["documents"], all_data["metadatas"]):
+        source = meta.get("source_file", "unknown")
+        w = meta.get("wing", "")
+        if wing and w != wing:
+            continue
+        if source not in by_source:
+            by_source[source] = {"drawer_ids": [], "content": [], "meta": meta}
+        by_source[source]["drawer_ids"].append(doc_id)
+        by_source[source]["content"].append(doc)
+
+    sources = list(by_source.keys())
+    if sample > 0:
+        sources = sources[:sample]
+
+    print(
+        f"Regenerating closets for {len(sources)} source files via {cfg.endpoint} ({cfg.model})..."
+    )
+    if dry_run:
+        print("DRY RUN — no changes will be written")
+
+    processed = 0
+    failed = 0
+    total_input = 0
+    total_output = 0
+
+    for i, source in enumerate(sources, 1):
+        data = by_source[source]
+        content = "\n\n".join(data["content"])
+        meta = data["meta"]
+        w = meta.get("wing", "")
+        r = meta.get("room", "")
+        entities = meta.get("entities", "")
+
+        if dry_run:
+            print(f"  [{i}/{len(sources)}] {os.path.basename(source)} ({len(content)} chars)")
+            continue
+
+        parsed, usage = _call_llm(cfg, source, w, r, content)
+        if not parsed:
+            failed += 1
+            print(f"  [{i}/{len(sources)}] ✗ {os.path.basename(source)} — LLM failed")
+            continue
+
+        if usage:
+            total_input += usage.get("prompt_tokens", 0)
+            total_output += usage.get("completion_tokens", 0)
+
+        lines = _parsed_to_closet_lines(parsed, data["drawer_ids"], entities)
+        # Use os.path.basename so Windows-style paths survive unchanged;
+        # the naive split('/') would leave a bare path component on Windows
+        # and collide across different files under different drives.
+        closet_id_base = f"closet_{w}_{r}_{os.path.basename(source)[:30]}"
+
+        # Serialize with concurrent mine operations on the same source —
+        # otherwise a regex closet rebuild mid-regenerate races with our
+        # purge+upsert cycle and leaves mixed regex/LLM lines.
+        with mine_lock(source):
+            purge_file_closets(closets_col, source)
+            upsert_closet_lines(
+                closets_col,
+                closet_id_base,
+                lines,
+                {
+                    "wing": w,
+                    "room": r,
+                    "source_file": source,
+                    "generated_by": f"llm:{cfg.model}",
+                    "filed_at": datetime.now().isoformat(),
+                    "entities": entities,
+                    # Stamp so the miner's stale-drawer gate doesn't treat
+                    # LLM closets as leftovers and rebuild over them next run.
+                    "normalize_version": NORMALIZE_VERSION,
+                },
+            )
+
+        processed += 1
+        n_topics = len(parsed.get("topics", []))
+        print(f"  [{i}/{len(sources)}] ✓ {os.path.basename(source)} — {n_topics} topics")
+
+    print(f"\nDone. {processed} regenerated, {failed} failed.")
+    if total_input or total_output:
+        print(f"Tokens: {total_input:,} in + {total_output:,} out (cost depends on provider)")
+
+    return {
+        "processed": processed,
+        "failed": failed,
+        "input_tokens": total_input,
+        "output_tokens": total_output,
+    }
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Regenerate closets via a user-configured LLM (OpenAI-compatible API)"
+    )
+    parser.add_argument(
+        "--palace",
+        default=os.path.expanduser("~/.mempalace/palace"),
+        help="Path to the palace",
+    )
+    parser.add_argument("--wing", default=None, help="Limit to one wing")
+    parser.add_argument("--sample", type=int, default=0, help="Only process first N source files")
+    parser.add_argument("--dry-run", action="store_true", help="List work without calling the LLM")
+    parser.add_argument(
+        "--endpoint",
+        default=None,
+        help="LLM base URL (overrides $LLM_ENDPOINT), e.g. http://localhost:11434/v1",
+    )
+    parser.add_argument(
+        "--key",
+        default=None,
+        help="LLM bearer token (overrides $LLM_KEY). Optional for local inference.",
+    )
+    parser.add_argument(
+        "--model",
+        default=None,
+        help='LLM model name (overrides $LLM_MODEL), e.g. "gpt-4o-mini" or "llama3:8b"',
+    )
+    args = parser.parse_args()
+
+    cfg = LLMConfig(endpoint=args.endpoint, key=args.key, model=args.model)
+    regenerate_closets(
+        args.palace, wing=args.wing, sample=args.sample, dry_run=args.dry_run, cfg=cfg
+    )

mempalace/config.py

@@ -6,8 +6,92 @@ Priority: env vars > config file (~/.mempalace/config.json) > defaults
 
 import json
 import os
+import re
 from pathlib import Path
 
+
+# ── Input validation ──────────────────────────────────────────────────────────
+# Shared sanitizers for wing/room/entity names. Prevents path traversal,
+# excessively long strings, and special characters that could cause issues
+# in file paths, SQLite, or ChromaDB metadata.
+
+MAX_NAME_LENGTH = 128
+_SAFE_NAME_RE = re.compile(r"^(?:[^\W_]|[^\W_][\w .'-]{0,126}[^\W_])$")
+
+
+def normalize_wing_name(name: str) -> str:
+    """Lower-case + collapse separators (`-`, ` `) to `_` for wing slugs.
+
+    The same rule is applied by ``init`` when persisting `topics_by_wing`
+    and when writing `mempalace.yaml`, so the miner's lookup matches at
+    mine time regardless of the source dirname.
+    """
+    return name.lower().replace(" ", "_").replace("-", "_")
+
+
+def sanitize_name(value: str, field_name: str = "name") -> str:
+    """Validate and sanitize a wing/room/entity name.
+
+    Raises ValueError if the name is invalid.
+    """
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{field_name} must be a non-empty string")
+
+    value = value.strip()
+
+    if len(value) > MAX_NAME_LENGTH:
+        raise ValueError(f"{field_name} exceeds maximum length of {MAX_NAME_LENGTH} characters")
+
+    # Block path traversal
+    if ".." in value or "/" in value or "\\" in value:
+        raise ValueError(f"{field_name} contains invalid path characters")
+
+    # Block null bytes
+    if "\x00" in value:
+        raise ValueError(f"{field_name} contains null bytes")
+
+    # Enforce safe character set
+    if not _SAFE_NAME_RE.match(value):
+        raise ValueError(f"{field_name} contains invalid characters")
+
+    return value
+
+
+def sanitize_kg_value(value: str, field_name: str = "value") -> str:
+    """Validate a knowledge-graph entity name (subject or object).
+
+    More permissive than sanitize_name — allows punctuation like commas,
+    colons, and parentheses that are common in natural-language KG values.
+    Only blocks null bytes and over-length strings.
+
+    Not used for wing/room names (which have filesystem constraints) or
+    predicates (which should be simple relationship identifiers).
+    """
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{field_name} must be a non-empty string")
+
+    value = value.strip()
+
+    if len(value) > MAX_NAME_LENGTH:
+        raise ValueError(f"{field_name} exceeds maximum length of {MAX_NAME_LENGTH} characters")
+
+    if "\x00" in value:
+        raise ValueError(f"{field_name} contains null bytes")
+
+    return value
+
+
+def sanitize_content(value: str, max_length: int = 100_000) -> str:
+    """Validate drawer/diary content length."""
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("content must be a non-empty string")
+    if len(value) > max_length:
+        raise ValueError(f"content exceeds maximum length of {max_length} characters")
+    if "\x00" in value:
+        raise ValueError("content contains null bytes")
+    return value
+
+
 DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
 DEFAULT_COLLECTION_NAME = "mempalace_drawers"
 
@@ -94,7 +178,10 @@ class MempalaceConfig:
         """Path to the memory palace data directory."""
         env_val = os.environ.get("MEMPALACE_PALACE_PATH") or os.environ.get("MEMPAL_PALACE_PATH")
         if env_val:
-            return env_val
+            # Normalize: expand ~ and collapse .. to match the CLI --palace
+            # code path (mcp_server.py:62) and prevent surprise redirection
+            # when the env var contains unresolved components.
+            return os.path.abspath(os.path.expanduser(env_val))
         return self._file_config.get("palace_path", DEFAULT_PALACE_PATH)
 
     @property
@@ -123,9 +210,114 @@ class MempalaceConfig:
         """Mapping of hall names to keyword lists."""
         return self._file_config.get("hall_keywords", DEFAULT_HALL_KEYWORDS)
 
+    @property
+    def entity_languages(self):
+        """Languages whose entity-detection patterns should be applied.
+
+        Reads from env var ``MEMPALACE_ENTITY_LANGUAGES`` (comma-separated)
+        first, then the ``entity_languages`` field in ``config.json``,
+        defaulting to ``["en"]``.
+        """
+        env_val = os.environ.get("MEMPALACE_ENTITY_LANGUAGES") or os.environ.get(
+            "MEMPAL_ENTITY_LANGUAGES"
+        )
+        if env_val:
+            return [s.strip() for s in env_val.split(",") if s.strip()] or ["en"]
+        cfg = self._file_config.get("entity_languages")
+        if isinstance(cfg, list) and cfg:
+            return [str(s) for s in cfg]
+        return ["en"]
+
+    def set_entity_languages(self, languages):
+        """Persist the entity-detection language list to ``config.json``."""
+        normalized = [s.strip() for s in languages if s and s.strip()]
+        if not normalized:
+            normalized = ["en"]
+        self._file_config["entity_languages"] = normalized
+        self._config_dir.mkdir(parents=True, exist_ok=True)
+        try:
+            with open(self._config_file, "w", encoding="utf-8") as f:
+                json.dump(self._file_config, f, indent=2, ensure_ascii=False)
+        except OSError:
+            pass
+        try:
+            self._config_file.chmod(0o600)
+        except (OSError, NotImplementedError):
+            pass
+        return normalized
+
+    @property
+    def embedding_device(self):
+        """Hardware device for the ONNX embedding model.
+
+        Values: ``"auto"`` (default), ``"cpu"``, ``"cuda"``, ``"coreml"``,
+        ``"dml"``. Read from env ``MEMPALACE_EMBEDDING_DEVICE`` first, then
+        ``embedding_device`` in ``config.json``, then ``"auto"``.
+
+        ``auto`` resolves to the first available accelerator at runtime via
+        :mod:`mempalace.embedding`; requesting an unavailable accelerator
+        logs a warning and falls back to CPU.
+        """
+        env_val = os.environ.get("MEMPALACE_EMBEDDING_DEVICE")
+        if env_val:
+            return env_val.strip().lower()
+        return str(self._file_config.get("embedding_device", "auto")).strip().lower()
+
+    @property
+    def topic_tunnel_min_count(self):
+        """Minimum number of overlapping confirmed topics required to create
+        a cross-wing tunnel between two wings.
+
+        Default is ``1`` — any single shared topic produces a tunnel. Bump
+        to ``2+`` if your projects share lots of common-tech labels (Python,
+        Docker, Git) and you want only meaningfully overlapping wings to
+        link. Reads ``MEMPALACE_TOPIC_TUNNEL_MIN_COUNT`` env first, then the
+        config-file value, then ``1``.
+        """
+        env_val = os.environ.get("MEMPALACE_TOPIC_TUNNEL_MIN_COUNT")
+        if env_val:
+            try:
+                parsed = int(env_val)
+                if parsed >= 1:
+                    return parsed
+            except ValueError:
+                pass
+        cfg_val = self._file_config.get("topic_tunnel_min_count")
+        try:
+            parsed = int(cfg_val) if cfg_val is not None else 1
+        except (TypeError, ValueError):
+            parsed = 1
+        return max(1, parsed)
+
+    @property
+    def hook_silent_save(self):
+        """Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""
+        return self._file_config.get("hooks", {}).get("silent_save", True)
+
+    @property
+    def hook_desktop_toast(self):
+        """Whether the stop hook shows a desktop notification via notify-send."""
+        return self._file_config.get("hooks", {}).get("desktop_toast", False)
+
+    def set_hook_setting(self, key: str, value: bool):
+        """Update a hook setting and write config to disk."""
+        if "hooks" not in self._file_config:
+            self._file_config["hooks"] = {}
+        self._file_config["hooks"][key] = value
+        try:
+            with open(self._config_file, "w", encoding="utf-8") as f:
+                json.dump(self._file_config, f, indent=2, ensure_ascii=False)
+        except OSError:
+            pass
+
     def init(self):
         """Create config directory and write default config.json if it doesn't exist."""
         self._config_dir.mkdir(parents=True, exist_ok=True)
+        # Restrict directory permissions to owner only (Unix)
+        try:
+            self._config_dir.chmod(0o700)
+        except (OSError, NotImplementedError):
+            pass  # Windows doesn't support Unix permissions
         if not self._config_file.exists():
             default_config = {
                 "palace_path": DEFAULT_PALACE_PATH,
@@ -135,6 +327,11 @@ class MempalaceConfig:
             }
             with open(self._config_file, "w") as f:
                 json.dump(default_config, f, indent=2)
+            # Restrict config file to owner read/write only
+            try:
+                self._config_file.chmod(0o600)
+            except (OSError, NotImplementedError):
+                pass
         return self._config_file
 
     def save_people_map(self, people_map):
@@ -146,4 +343,8 @@ class MempalaceConfig:
         self._config_dir.mkdir(parents=True, exist_ok=True)
         with open(self._people_map_file, "w") as f:
             json.dump(people_map, f, indent=2)
+        try:
+            self._people_map_file.chmod(0o600)
+        except (OSError, NotImplementedError):
+            pass
         return self._people_map_file

mempalace/convo_miner.py

@@ -15,9 +15,34 @@ from pathlib import Path
 from datetime import datetime
 from collections import defaultdict
 
-import chromadb
-
 from .normalize import normalize
+from .palace import (
+    NORMALIZE_VERSION,
+    SKIP_DIRS,
+    file_already_mined,
+    get_collection,
+    mine_lock,
+)
+
+
+# Cached hall keywords — avoids re-reading config per drawer
+_HALL_KEYWORDS_CACHE = None
+
+
+def _detect_hall_cached(content: str) -> str:
+    """Route content to a hall using cached keywords. Same logic as miner.detect_hall."""
+    global _HALL_KEYWORDS_CACHE
+    if _HALL_KEYWORDS_CACHE is None:
+        from .config import MempalaceConfig
+
+        _HALL_KEYWORDS_CACHE = MempalaceConfig().hall_keywords
+    content_lower = content[:3000].lower()
+    scores = {}
+    for hall, keywords in _HALL_KEYWORDS_CACHE.items():
+        score = sum(1 for kw in keywords if kw in content_lower)
+        if score > 0:
+            scores[hall] = score
+    return max(scores, key=scores.get) if scores else "general"
 
 
 # File types that might contain conversations
@@ -28,20 +53,42 @@ CONVO_EXTENSIONS = {
     ".jsonl",
 }
 
-SKIP_DIRS = {
-    ".git",
-    "node_modules",
-    "__pycache__",
-    ".venv",
-    "venv",
-    "env",
-    "dist",
-    "build",
-    ".next",
-    ".mempalace",
-}
-
 MIN_CHUNK_SIZE = 30
+CHUNK_SIZE = 800  # chars per drawer — align with miner.py
+DRAWER_UPSERT_BATCH_SIZE = 1000
+MAX_FILE_SIZE = 500 * 1024 * 1024  # 500 MB — skip files larger than this.
+# Matches miner.py at 500 MB. Long Claude Code sessions, multi-year
+# ChatGPT exports, and lifetime Slack dumps routinely exceed 10 MB; the
+# cap at that level silently dropped them with `continue`. Per-drawer
+# size is bounded by CHUNK_SIZE, but larger source files still produce
+# more drawers and therefore more embedding/storage work — and content
+# is normalized and loaded fully into memory before chunking, so memory
+# use also scales with source size.
+
+
+def _register_file(collection, source_file: str, wing: str, agent: str):
+    """Write a sentinel so file_already_mined() returns True for 0-chunk files.
+
+    Without this, files that normalize to nothing or produce zero chunks are
+    re-read and re-processed on every mine run because nothing was written to
+    ChromaDB on the first pass.
+    """
+    sentinel_id = f"_reg_{hashlib.sha256(source_file.encode()).hexdigest()[:24]}"
+    collection.upsert(
+        documents=[f"[registry] {source_file}"],
+        ids=[sentinel_id],
+        metadatas=[
+            {
+                "wing": wing,
+                "room": "_registry",
+                "source_file": source_file,
+                "added_by": agent,
+                "filed_at": datetime.now().isoformat(),
+                "ingest_mode": "registry",
+                "normalize_version": NORMALIZE_VERSION,
+            }
+        ],
+    )
 
 
 # =============================================================================
@@ -64,7 +111,12 @@ def chunk_exchanges(content: str) -> list:
 
 
 def _chunk_by_exchange(lines: list) -> list:
-    """One user turn (>) + the AI response that follows = one chunk."""
+    """One user turn (>) + the AI response that follows = one or more chunks.
+
+    The full AI response is preserved verbatim.  When the combined
+    user-turn + response exceeds CHUNK_SIZE the response is split across
+    consecutive drawers so nothing is silently discarded.
+    """
     chunks = []
     i = 0
 
@@ -83,10 +135,23 @@ def _chunk_by_exchange(lines: list) -> list:
                     ai_lines.append(next_line.strip())
                 i += 1
 
-            ai_response = " ".join(ai_lines[:8])
+            ai_response = " ".join(ai_lines)
             content = f"{user_turn}\n{ai_response}" if ai_response else user_turn
 
-            if len(content.strip()) > MIN_CHUNK_SIZE:
+            # Split into multiple drawers when the exchange exceeds CHUNK_SIZE
+            if len(content) > CHUNK_SIZE:
+                # First chunk: user turn + as much response as fits
+                first_part = content[:CHUNK_SIZE]
+                if len(first_part.strip()) > MIN_CHUNK_SIZE:
+                    chunks.append({"content": first_part, "chunk_index": len(chunks)})
+                # Remaining response in CHUNK_SIZE-sized continuation drawers
+                remainder = content[CHUNK_SIZE:]
+                while remainder:
+                    part = remainder[:CHUNK_SIZE]
+                    remainder = remainder[CHUNK_SIZE:]
+                    if len(part.strip()) > MIN_CHUNK_SIZE:
+                        chunks.append({"content": part, "chunk_index": len(chunks)})
+            elif len(content.strip()) > MIN_CHUNK_SIZE:
                 chunks.append(
                     {
                         "content": content,
@@ -209,23 +274,6 @@ def detect_convo_room(content: str) -> str:
 # =============================================================================
 
 
-def get_collection(palace_path: str):
-    os.makedirs(palace_path, exist_ok=True)
-    client = chromadb.PersistentClient(path=palace_path)
-    try:
-        return client.get_collection("mempalace_drawers")
-    except Exception:
-        return client.create_collection("mempalace_drawers")
-
-
-def file_already_mined(collection, source_file: str) -> bool:
-    try:
-        results = collection.get(where={"source_file": source_file}, limit=1)
-        return len(results.get("ids", [])) > 0
-    except Exception:
-        return False
-
-
 # =============================================================================
 # SCAN FOR CONVERSATION FILES
 # =============================================================================
@@ -238,8 +286,18 @@ def scan_convos(convo_dir: str) -> list:
     for root, dirs, filenames in os.walk(convo_path):
         dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
         for filename in filenames:
+            if filename.endswith(".meta.json"):
+                continue
             filepath = Path(root) / filename
             if filepath.suffix.lower() in CONVO_EXTENSIONS:
+                # Skip symlinks and oversized files
+                if filepath.is_symlink():
+                    continue
+                try:
+                    if filepath.stat().st_size > MAX_FILE_SIZE:
+                        continue
+                except OSError:
+                    continue
                 files.append(filepath)
     return files
 
@@ -249,6 +307,75 @@ def scan_convos(convo_dir: str) -> list:
 # =============================================================================
 
 
+def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extract_mode):
+    """Lock the source file, purge stale drawers, and upsert fresh chunks.
+
+    Combines the per-file serialization that prevents concurrent agents from
+    duplicating work (via mine_lock) with the normalize-version rebuild
+    contract (purge-before-insert so pre-v2 drawers don't survive).
+
+    Returns (drawers_added, room_counts_delta, skipped).
+    """
+    room_counts_delta: dict = defaultdict(int)
+    drawers_added = 0
+    with mine_lock(source_file):
+        # Re-check after lock — another agent may have just finished this file
+        # at the current schema. A stale-version hit here returns False, so we
+        # still fall through to the purge+rebuild path below.
+        if file_already_mined(collection, source_file):
+            return 0, room_counts_delta, True
+
+        # Purge stale drawers first. When the normalize schema bumps,
+        # file_already_mined() returned False for pre-v2 drawers — clean
+        # them out so the source doesn't end up with mixed old/new drawers.
+        try:
+            collection.delete(where={"source_file": source_file})
+        except Exception:
+            pass
+
+        # Batch chunks into bounded upserts so large transcripts keep most of
+        # the embedding speedup without one huge Chroma/SQLite request. Keep
+        # one filed_at per source file so all transcript drawers share an
+        # ingest timestamp.
+        filed_at = datetime.now().isoformat()
+        for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE):
+            batch_docs: list = []
+            batch_ids: list = []
+            batch_metas: list = []
+            for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]:
+                chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room
+                if extract_mode == "general":
+                    room_counts_delta[chunk_room] += 1
+                drawer_id = f"drawer_{wing}_{chunk_room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}"
+                batch_docs.append(chunk["content"])
+                batch_ids.append(drawer_id)
+                batch_metas.append(
+                    {
+                        "wing": wing,
+                        "room": chunk_room,
+                        "hall": _detect_hall_cached(chunk["content"]),
+                        "source_file": source_file,
+                        "chunk_index": chunk["chunk_index"],
+                        "added_by": agent,
+                        "filed_at": filed_at,
+                        "ingest_mode": "convos",
+                        "extract_mode": extract_mode,
+                        "normalize_version": NORMALIZE_VERSION,
+                    }
+                )
+            try:
+                collection.upsert(
+                    documents=batch_docs,
+                    ids=batch_ids,
+                    metadatas=batch_metas,
+                )
+                drawers_added += len(batch_docs)
+            except Exception as e:
+                if "already exists" not in str(e).lower():
+                    raise
+    return drawers_added, room_counts_delta, False
+
+
 def mine_convos(
     convo_dir: str,
     palace_path: str,
@@ -267,7 +394,9 @@ def mine_convos(
 
     convo_path = Path(convo_dir).expanduser().resolve()
     if not wing:
-        wing = convo_path.name.lower().replace(" ", "_").replace("-", "_")
+        from .config import normalize_wing_name
+
+        wing = normalize_wing_name(convo_path.name)
 
     files = scan_convos(convo_dir)
     if limit > 0:
@@ -282,7 +411,7 @@ def mine_convos(
     print(f"  Palace:  {palace_path}")
     if dry_run:
         print("  DRY RUN — nothing will be filed")
-    print(f"{'─' * 55}\n")
+    print(f"{'-' * 55}\n")
 
     collection = get_collection(palace_path) if not dry_run else None
 
@@ -301,10 +430,14 @@ def mine_convos(
         # Normalize format
         try:
             content = normalize(str(filepath))
-        except Exception:
+        except (OSError, ValueError):
+            if not dry_run:
+                _register_file(collection, source_file, wing, agent)
             continue
 
         if not content or len(content.strip()) < MIN_CHUNK_SIZE:
+            if not dry_run:
+                _register_file(collection, source_file, wing, agent)
             continue
 
         # Chunk — either exchange pairs or general extraction
@@ -317,6 +450,8 @@ def mine_convos(
             chunks = chunk_exchanges(content)
 
         if not chunks:
+            if not dry_run:
+                _register_file(collection, source_file, wing, agent)
             continue
 
         # Detect room from content (general mode uses memory_type instead)
@@ -346,37 +481,19 @@ def mine_convos(
         if extract_mode != "general":
             room_counts[room] += 1
 
-        # File each chunk
-        drawers_added = 0
-        for chunk in chunks:
-            chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room
-            if extract_mode == "general":
-                room_counts[chunk_room] += 1
-            drawer_id = f"drawer_{wing}_{chunk_room}_{hashlib.md5((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:16]}"
-            try:
-                collection.add(
-                    documents=[chunk["content"]],
-                    ids=[drawer_id],
-                    metadatas=[
-                        {
-                            "wing": wing,
-                            "room": chunk_room,
-                            "source_file": source_file,
-                            "chunk_index": chunk["chunk_index"],
-                            "added_by": agent,
-                            "filed_at": datetime.now().isoformat(),
-                            "ingest_mode": "convos",
-                            "extract_mode": extract_mode,
-                        }
-                    ],
-                )
-                drawers_added += 1
-            except Exception as e:
-                if "already exists" not in str(e).lower():
-                    raise
+        # Lock + purge stale + file fresh chunks. Lock serializes concurrent
+        # agents; purge removes pre-v2 drawers so the schema bump applies.
+        drawers_added, room_delta, skipped = _file_chunks_locked(
+            collection, source_file, chunks, wing, room, agent, extract_mode
+        )
+        if skipped:
+            files_skipped += 1
+            continue
+        for r, n in room_delta.items():
+            room_counts[r] += n
 
         total_drawers += drawers_added
-        print(f"  ✓ [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}")
+        print(f"  + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}")
 
     print(f"\n{'=' * 55}")
     print("  Done.")

mempalace/convo_scanner.py

@@ -0,0 +1,160 @@
+"""
+convo_scanner.py — Parse Claude Code conversation directories into ProjectInfo.
+
+Claude Code stores sessions under ``~/.claude/projects/<slug>/<id>.jsonl``,
+where the ``<slug>`` is the original CWD with ``/`` replaced by ``-``. That
+encoding is lossy: we can't tell whether ``foo-bar`` in a slug is the
+literal project name ``foo-bar`` or two path segments ``foo/bar``.
+
+Fortunately, every message record in the JSONL carries a ``cwd`` field with
+the true path. This scanner reads one record per session to recover the
+accurate project name, falling back to slug-decoding only if the JSONL
+is malformed or empty.
+
+Output is the same ``ProjectInfo`` shape used by ``project_scanner``, so the
+``discover_entities`` orchestrator can mix-and-match sources.
+
+Public:
+    is_claude_projects_root(path) -> bool
+    scan_claude_projects(path) -> list[ProjectInfo]
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+from mempalace.project_scanner import ProjectInfo
+
+
+MAX_HEADER_LINES = 20  # lines to read per session looking for `cwd`
+
+
+def is_claude_projects_root(path: Path) -> bool:
+    """Return True if path looks like `.claude/projects/`.
+
+    Heuristic: at least one child dir whose name starts with ``-`` and which
+    contains at least one ``.jsonl`` file.
+    """
+    if not path.is_dir():
+        return False
+    try:
+        children = list(path.iterdir())
+    except OSError:
+        return False
+    for child in children:
+        if not (child.is_dir() and child.name.startswith("-")):
+            continue
+        try:
+            if any(p.suffix == ".jsonl" for p in child.iterdir() if p.is_file()):
+                return True
+        except OSError:
+            continue
+    return False
+
+
+def _extract_cwd_from_session(session_file: Path) -> Optional[str]:
+    """Return the ``cwd`` from the first message record that carries one.
+
+    Returns None if the file can't be read, has no JSON, or no record has cwd.
+    """
+    try:
+        with open(session_file, encoding="utf-8", errors="replace") as f:
+            for i, line in enumerate(f):
+                if i >= MAX_HEADER_LINES:
+                    break
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    obj = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                cwd = obj.get("cwd")
+                if isinstance(cwd, str) and cwd:
+                    return cwd
+    except OSError:
+        return None
+    return None
+
+
+def _decode_slug_fallback(slug: str) -> str:
+    """Best-effort project name from slug when cwd is unavailable.
+
+    The slug is lossy (`/` and `-` both become `-`). Last non-empty segment
+    is the closest guess at the project name, preserving kebab-case is
+    impossible without cwd.
+    """
+    stripped = slug.lstrip("-")
+    parts = [p for p in stripped.split("-") if p]
+    return parts[-1] if parts else slug
+
+
+def _safe_mtime(path: Path) -> float:
+    """Return file mtime, defaulting old on permission or filesystem errors."""
+    try:
+        return path.stat().st_mtime
+    except OSError:
+        return 0.0
+
+
+def _resolve_project_name(project_dir: Path) -> str:
+    """Read one session's cwd to recover the original project name.
+
+    Falls back to slug-decoding if no session has a readable cwd.
+    """
+    sessions = sorted(
+        (p for p in project_dir.iterdir() if p.is_file() and p.suffix == ".jsonl"),
+        key=_safe_mtime,
+        reverse=True,  # newest first — most likely to be well-formed
+    )
+    for session in sessions:
+        cwd = _extract_cwd_from_session(session)
+        if cwd:
+            return Path(cwd).name or cwd
+    return _decode_slug_fallback(project_dir.name)
+
+
+def scan_claude_projects(path: str | Path) -> list[ProjectInfo]:
+    """Scan a ``.claude/projects/`` directory for Claude Code conversations.
+
+    One ProjectInfo per subdir. ``has_git`` is False (the directory isn't a
+    repo itself) but ``total_commits`` is repurposed here as session count so
+    the UX surfaces a density signal for ranking.
+    """
+    root = Path(path).expanduser().resolve()
+    if not is_claude_projects_root(root):
+        return []
+
+    projects: dict[str, ProjectInfo] = {}
+    for sub in sorted(root.iterdir()):
+        if not (sub.is_dir() and sub.name.startswith("-")):
+            continue
+        try:
+            sessions = [p for p in sub.iterdir() if p.is_file() and p.suffix == ".jsonl"]
+        except OSError:
+            continue
+        if not sessions:
+            continue
+
+        name = _resolve_project_name(sub)
+        session_count = len(sessions)
+
+        proj = ProjectInfo(
+            name=name,
+            repo_root=sub,
+            manifest=None,
+            has_git=False,
+            total_commits=session_count,
+            user_commits=session_count,
+            is_mine=True,  # Claude Code sessions are authored by the user
+        )
+        existing = projects.get(name)
+        if existing is None or session_count > existing.user_commits:
+            projects[name] = proj
+
+    return sorted(
+        projects.values(),
+        key=lambda p: (-p.user_commits, p.name),
+    )

mempalace/corpus_origin.py

@@ -0,0 +1,422 @@
+"""
+corpus_origin.py — Detect whether a corpus is an AI-dialogue record and,
+if so, what platform and what persona names the user has assigned to the
+agent.
+
+This is the first question any downstream Pass 2 classification needs
+answered. Without it, a drawer like "my three sons" in a Claude Code
+dialogue corpus can't be correctly resolved to "three AI instances"
+rather than "three biological children."
+
+Two-tier detection:
+
+  Tier 1 — detect_origin_heuristic(samples)
+           Cheap, no API. Grep for well-known AI brand terms + turn
+           markers. Always runs. Outputs a hypothesis.
+
+  Tier 2 — detect_origin_llm(samples, provider)
+           Uses an LLMProvider (typically Haiku via mempalace.llm_client)
+           with the model's pre-trained knowledge of Claude/ChatGPT/Gemini
+           etc. Confirms platform, extracts agent persona-names the user
+           has assigned. One call, ~$0.01 cost.
+
+Design principle:
+  Don't make the classifier re-discover what Claude, ChatGPT, Gemini, MCP,
+  or other well-known entities ARE — the LLM already knows them from its
+  training. Only corpus-specific entities (e.g. the user's persona-name
+  for their Claude instance) need discovery.
+
+Default stance (when evidence is thin):
+  "This IS an AI-dialogue corpus" — false-negative is catastrophic for
+  downstream classification; false-positive is recoverable via per-drawer
+  voice-profile detection in later passes.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field, asdict
+from typing import Optional
+
+
+# ── Well-known AI brand terms (expand as new platforms emerge) ────────────
+# Detection is by PATTERN + CONTEXT, not by capitalization or English-language
+# rules. Two categories:
+#
+#   UNAMBIGUOUS — terms that have essentially no meaning outside of AI context.
+#     Always counted toward AI-dialogue evidence.
+#
+#   AMBIGUOUS — terms that share a string with common English words, names,
+#     poetry forms, zodiac signs, animals, etc. Counted toward AI-dialogue
+#     evidence ONLY when at least one unambiguous AI signal also appears in
+#     the corpus (turn marker, unambiguous brand term, or AI infrastructure
+#     term). This avoids false-positives on French novels with characters
+#     named "Claude", astrology corpora discussing "Gemini", poetry corpora
+#     full of "haiku" / "sonnet", etc.
+#
+# All matching is CASE-INSENSITIVE — users type lowercase constantly.
+
+_AI_UNAMBIGUOUS_TERMS = [
+    # Anthropic-specific
+    "Anthropic",
+    "Claude Code",
+    "Claude 3",
+    "Claude 4",
+    "claude mcp",
+    "CLAUDE.md",
+    ".claude/",
+    # OpenAI-specific
+    "ChatGPT",
+    "GPT-4",
+    "GPT-3",
+    "GPT-5",
+    "OpenAI",
+    "gpt-4o",
+    "gpt-4-turbo",
+    "o1-preview",
+    "o3",
+    # Google-specific
+    "gemini-pro",
+    "gemini-1.5",
+    "Google AI",
+    # Meta / others (specific model identifiers, not bare common words)
+    "Mixtral",
+    "Cohere",
+    # AI-infrastructure terms with no common-English collision
+    "MCP",
+    "LLM",
+    "RAG",
+    "fine-tune",
+    "context window",
+    "embedding",
+]
+
+_AI_AMBIGUOUS_TERMS = [
+    # Anthropic — bare brand/model names that collide with names + poetry
+    "Claude",  # also a common French masculine name
+    "Opus",  # also a musical work, comic strip, magazine
+    "Sonnet",  # also a 14-line poem form
+    "Haiku",  # also a 17-syllable poem form
+    # Google — bare brand that collides with zodiac sign
+    "Gemini",  # also the zodiac sign
+    "Bard",  # also a poet / Shakespeare
+    # Meta / others
+    "Llama",  # also the South American animal
+    "Mistral",  # also a Mediterranean wind
+    # Note: 'prompt', 'completion', 'tokens' previously lived here but were
+    # removed: they're suppressed without an unambiguous co-signal anyway,
+    # and by the time a co-signal is present the corpus is already flagged.
+    # Keeping them just produced noisier evidence strings.
+]
+
+# Turn-marker patterns commonly seen in AI-dialogue transcripts
+_TURN_MARKERS = [
+    r"\buser\s*:\s*",
+    r"\bassistant\s*:\s*",
+    r"\bhuman\s*:\s*",
+    r"\bai\s*:\s*",
+    r"\b>>>\s*User\b",
+    r"\b>>>\s*Assistant\b",
+]
+
+
+def _brand_pattern(term: str) -> str:
+    """Build a regex for a brand term that uses word boundaries
+    only on edges where the term itself starts/ends with a word
+    character. Without this nuance:
+      - 'Claude' would falsely match inside 'Claudette' (no \\b)
+      - '.claude/' would fail to match at start of string (\\b
+        before non-word char requires preceding word char)
+    So we only attach \\b where it actually makes sense."""
+    escaped = re.escape(term)
+    prefix = r"\b" if term[0].isalnum() or term[0] == "_" else ""
+    suffix = r"\b" if term[-1].isalnum() or term[-1] == "_" else ""
+    return prefix + escaped + suffix
+
+
+@dataclass
+class CorpusOriginResult:
+    """Structured output from corpus-origin detection.
+
+    Fields:
+      likely_ai_dialogue — best hypothesis about whether this is AI-dialogue
+      confidence — 0.0 to 1.0
+      primary_platform — e.g. "Claude Code (Anthropic CLI)" or None
+      user_name — the corpus author's name if identifiable from context, else None
+      agent_persona_names — names the user has assigned to the AI agent(s)
+                            (e.g. ["Echo", "Sparrow"]). Does NOT include the user's own name.
+      evidence — human-readable reasons for the classification
+    """
+
+    likely_ai_dialogue: bool
+    confidence: float
+    primary_platform: Optional[str]
+    user_name: Optional[str] = None
+    agent_persona_names: list[str] = field(default_factory=list)
+    evidence: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+# ── Tier 1: cheap heuristic ───────────────────────────────────────────────
+
+
+def detect_origin_heuristic(samples: list[str]) -> CorpusOriginResult:
+    """Fast grep-based detection. No API calls.
+
+    Scores AI-dialogue likelihood by counting:
+      - occurrences of well-known AI brand terms
+      - turn-marker patterns (user:, assistant:, etc.)
+
+    Returns a CorpusOriginResult with confidence derived from signal density.
+    """
+    combined = "\n\n".join(samples)
+    total_chars = max(1, len(combined))
+
+    # Count UNAMBIGUOUS brand-term hits (case-insensitive — users type
+    # lowercase constantly, so 'chatgpt' must trip the same as 'ChatGPT').
+    # Word boundaries prevent false in-word matches (see _brand_pattern).
+    unambiguous_hits: dict[str, int] = {}
+    total_unambiguous = 0
+    for term in _AI_UNAMBIGUOUS_TERMS:
+        matches = re.findall(_brand_pattern(term), combined, re.IGNORECASE)
+        if matches:
+            unambiguous_hits[term] = len(matches)
+            total_unambiguous += len(matches)
+
+    # Count AMBIGUOUS brand-term hits separately. These will only be
+    # counted toward AI-dialogue evidence if the corpus also contains
+    # at least one unambiguous AI signal — see co-occurrence rule below.
+    ambiguous_hits: dict[str, int] = {}
+    total_ambiguous = 0
+    for term in _AI_AMBIGUOUS_TERMS:
+        matches = re.findall(_brand_pattern(term), combined, re.IGNORECASE)
+        if matches:
+            ambiguous_hits[term] = len(matches)
+            total_ambiguous += len(matches)
+
+    # Count turn-marker hits (case-insensitive — transcripts vary).
+    turn_hits = 0
+    turn_types_found = set()
+    for pattern in _TURN_MARKERS:
+        matches = re.findall(pattern, combined, re.IGNORECASE)
+        if matches:
+            turn_hits += len(matches)
+            turn_types_found.add(pattern)
+
+    # Co-occurrence rule for ambiguous terms.
+    # Ambiguous terms (e.g. 'Claude' as a French name, 'Gemini' as a zodiac
+    # sign, 'Haiku' as a poem form) only count toward brand evidence if
+    # the corpus also contains at least one unambiguous AI signal. Otherwise
+    # we'd false-positive on French novels, astrology forums, poetry corpora,
+    # llama-rancher journals, etc.
+    has_ai_context = total_unambiguous > 0 or turn_hits > 0
+    counted_brand_hits = total_unambiguous + (total_ambiguous if has_ai_context else 0)
+
+    # Brand-term density per 1000 chars; turn-marker density likewise.
+    # Tuned on a small set of examples; these aren't magic numbers and
+    # can be revisited as we see more corpora.
+    brand_density = counted_brand_hits / (total_chars / 1000)
+    turn_density = turn_hits / (total_chars / 1000)
+
+    # Build evidence list
+    evidence: list[str] = []
+    shown_hits = dict(unambiguous_hits)
+    if has_ai_context:
+        shown_hits.update(ambiguous_hits)
+    if shown_hits:
+        top_terms = sorted(shown_hits.items(), key=lambda x: -x[1])[:5]
+        evidence.append("AI brand terms: " + ", ".join(f"'{k}' ({v}x)" for k, v in top_terms))
+    elif ambiguous_hits and not has_ai_context:
+        # Be transparent that we saw ambiguous matches but suppressed them
+        # for lack of co-occurring AI context.
+        suppressed = sorted(ambiguous_hits.items(), key=lambda x: -x[1])[:3]
+        evidence.append(
+            "Ambiguous terms present but suppressed (no co-occurring AI signal): "
+            + ", ".join(f"'{k}' ({v}x)" for k, v in suppressed)
+        )
+    if turn_hits:
+        evidence.append(
+            f"Turn markers detected: {turn_hits} occurrences across {len(turn_types_found)} pattern types"
+        )
+
+    # Decision logic:
+    #   strong signal (brand OR turn hits both >= threshold) → confident AI-dialogue
+    #   MEANINGFUL absence (enough text, zero brand, zero turn) → confident narrative
+    #   ambiguous or insufficient text → default stance: AI-dialogue with low confidence
+    #
+    # Threshold for "meaningful absence": the samples collectively have to
+    # be long enough that the absence of AI signals would be expected to
+    # surface if the corpus really is narrative. 150 chars is the working
+    # floor — below that, we cannot confidently say "this is narrative."
+    MEANINGFUL_TEXT_FLOOR = 150
+
+    if brand_density >= 0.5 or turn_density >= 2.0:
+        return CorpusOriginResult(
+            likely_ai_dialogue=True,
+            confidence=min(0.95, 0.6 + 0.1 * (brand_density + turn_density)),
+            primary_platform=None,  # tier 2 will refine
+            evidence=evidence,
+        )
+    if counted_brand_hits == 0 and turn_hits == 0 and total_chars >= MEANINGFUL_TEXT_FLOOR:
+        # Note: ambiguous-only matches (e.g. a French novel with 'Claude' as
+        # a character name) flow through here because counted_brand_hits == 0
+        # when no unambiguous AI signal co-occurs. The 'evidence' list still
+        # records that the ambiguous matches were seen and suppressed.
+        narrative_evidence = list(evidence) + [
+            f"no unambiguous AI signal across {total_chars} chars of text — pure narrative"
+        ]
+        return CorpusOriginResult(
+            likely_ai_dialogue=False,
+            confidence=0.9,
+            primary_platform=None,
+            evidence=narrative_evidence,
+        )
+    # Ambiguous or too-short-to-tell case: default stance is AI-dialogue
+    # with explicit low confidence. Tier 2 (LLM) should be called to confirm.
+    reason = "weak signal" if (counted_brand_hits or turn_hits) else "insufficient text"
+    return CorpusOriginResult(
+        likely_ai_dialogue=True,
+        confidence=0.4,
+        primary_platform=None,
+        evidence=evidence
+        + [
+            f"{reason} — applying default-stance (ai_dialogue=True, low confidence). "
+            "Tier 2 LLM check recommended to confirm or override."
+        ],
+    )
+
+
+# ── Tier 2: LLM-assisted confirmation + persona extraction ────────────────
+
+
+_SYSTEM_PROMPT = """You are analyzing a corpus of text to determine whether it is a \
+record of conversations with an AI agent (e.g. Claude, ChatGPT, Gemini, custom LLM \
+apps), or some other kind of text (personal narrative, story, research notes, \
+journal, code, etc.).
+
+Use your pre-existing knowledge of well-known AI platforms. You don't need the \
+corpus to explain what Claude or ChatGPT is — you already know. Your job is to \
+detect evidence of their presence and identify what persona-names the user has \
+assigned to the agent(s) they converse with.
+
+CRITICAL distinction:
+  - agent_persona_names are names the USER has assigned to the AI AGENT(S)
+    they converse with. Example: "Echo", "Sparrow", "Henry" might be names
+    the user calls a Claude instance they're building a relationship with.
+  - Do NOT include the USER's own name in agent_persona_names. The user
+    is the human author of the corpus, not a persona of the agent. Even
+    if the user's name appears frequently in the text (writing about
+    themselves), that is NOT an agent persona.
+  - If you can identify the user's name from context, put it in user_name
+    (separate field). If unclear, leave user_name null.
+
+Respond with JSON only (no prose before or after):
+{
+  "is_ai_dialogue_corpus": <true|false>,
+  "confidence": <0.0 to 1.0>,
+  "primary_platform": <"Claude (Anthropic)" | "ChatGPT (OpenAI)" | "Gemini (Google)" | other platform name | null>,
+  "user_name": <user's name if clearly identifiable from context, else null>,
+  "agent_persona_names": [<names the user has assigned to the AI AGENT(S), NOT the user's own name>],
+  "evidence": [<short bullet strings explaining the decision>]
+}
+
+Default stance: if evidence is thin or mixed, return is_ai_dialogue_corpus=true \
+with low confidence. False-negatives on AI-dialogue detection break downstream \
+classification; false-positives are recoverable later.
+"""
+
+
+def _extract_json(text: str) -> Optional[dict]:
+    """Pull the first JSON object out of a possibly-messy LLM response."""
+    text = text.strip()
+    if not text:
+        return None
+    # Try straight parse first
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+    # Try to find a {...} block
+    start = text.find("{")
+    if start < 0:
+        return None
+    depth = 0
+    in_string = False
+    escape = False
+    for i in range(start, len(text)):
+        ch = text[i]
+        if in_string:
+            if escape:
+                escape = False
+            elif ch == "\\":
+                escape = True
+            elif ch == '"':
+                in_string = False
+            continue
+        if ch == '"':
+            in_string = True
+        elif ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                candidate = text[start : i + 1]
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    return None
+    return None
+
+
+def detect_origin_llm(samples: list[str], provider) -> CorpusOriginResult:
+    """LLM-assisted detection. Takes samples (list of drawer-text excerpts)
+    and an LLMProvider (mempalace.llm_client.LLMProvider). Returns the
+    same CorpusOriginResult shape as the heuristic.
+
+    Falls back conservatively (default-stance ai=True, low confidence)
+    on any LLM error or malformed response — never raises.
+    """
+    # Build the user prompt: concise excerpts, capped so we stay cheap
+    max_excerpt_chars = 800
+    excerpts = "\n\n---\n\n".join(
+        f"[sample {i + 1}]\n{s[:max_excerpt_chars]}" for i, s in enumerate(samples[:20])
+    )
+    user_prompt = f"CORPUS EXCERPTS:\n\n{excerpts}\n\nAnalyze and respond with JSON."
+
+    try:
+        resp = provider.classify(system=_SYSTEM_PROMPT, user=user_prompt, json_mode=True)
+        raw = getattr(resp, "text", "") or ""
+    except Exception as e:
+        return CorpusOriginResult(
+            likely_ai_dialogue=True,
+            confidence=0.3,
+            primary_platform=None,
+            evidence=[f"LLM provider error (fallback to default stance): {e}"],
+        )
+
+    parsed = _extract_json(raw)
+    if not parsed or not isinstance(parsed, dict):
+        return CorpusOriginResult(
+            likely_ai_dialogue=True,
+            confidence=0.3,
+            primary_platform=None,
+            evidence=["LLM response was not valid JSON (fallback to default stance)"],
+        )
+
+    # Pull fields defensively. If the LLM leaked the user_name into
+    # agent_persona_names despite the prompt telling it not to, filter it out.
+    user_name = parsed.get("user_name") or None
+    personas = list(parsed.get("agent_persona_names") or [])
+    if user_name:
+        personas = [p for p in personas if p.lower() != user_name.lower()]
+    return CorpusOriginResult(
+        likely_ai_dialogue=bool(parsed.get("is_ai_dialogue_corpus", True)),
+        confidence=float(parsed.get("confidence", 0.5)),
+        primary_platform=parsed.get("primary_platform") or None,
+        user_name=user_name,
+        agent_persona_names=personas,
+        evidence=list(parsed.get("evidence") or []),
+    )

mempalace/dedup.py

@@ -0,0 +1,237 @@
+"""
+dedup.py — Detect and remove near-duplicate drawers
+====================================================
+
+When the same files are mined multiple times, near-identical drawers
+accumulate. This module finds drawers from the same source_file that
+are too similar (cosine distance < threshold), keeps the longest/richest
+version, and deletes the rest.
+
+No API calls — uses ChromaDB's built-in embedding similarity.
+
+Usage (standalone):
+    python -m mempalace.dedup                          # dedup all
+    python -m mempalace.dedup --dry-run                # preview only
+    python -m mempalace.dedup --threshold 0.10         # stricter (near-identical only)
+    python -m mempalace.dedup --threshold 0.35         # looser (catches paraphrased content)
+    python -m mempalace.dedup --wing my_project        # scope to one wing
+    python -m mempalace.dedup --stats                  # stats only
+    python -m mempalace.dedup --source "my_project"    # filter by source
+
+Usage (from CLI):
+    mempalace dedup [--dry-run] [--threshold 0.15] [--stats]
+"""
+
+import argparse
+import os
+import time
+from collections import defaultdict
+
+from .backends.chroma import ChromaBackend
+
+
+COLLECTION_NAME = "mempalace_drawers"
+# Cosine DISTANCE threshold (not similarity). Lower = stricter.
+# 0.15 = ~85% cosine similarity — catches near-identical chunks.
+# For looser dedup of paraphrased content, try 0.3–0.4.
+DEFAULT_THRESHOLD = 0.15
+MIN_DRAWERS_TO_CHECK = 5
+
+
+def _get_palace_path():
+    """Resolve palace path from config."""
+    try:
+        from .config import MempalaceConfig
+
+        return MempalaceConfig().palace_path
+    except Exception:
+        return os.path.join(os.path.expanduser("~"), ".mempalace", "palace")
+
+
+def get_source_groups(col, min_count=MIN_DRAWERS_TO_CHECK, source_pattern=None, wing=None):
+    """Group drawers by source_file, return groups with min_count+ entries.
+
+    If wing is specified, only considers drawers in that wing. This catches
+    cross-wing duplicates when the same source was mined into multiple wings.
+    """
+    total = col.count()
+    groups = defaultdict(list)
+
+    offset = 0
+    batch_size = 1000
+    while offset < total:
+        kwargs = {"limit": batch_size, "offset": offset, "include": ["metadatas"]}
+        if wing:
+            kwargs["where"] = {"wing": wing}
+        batch = col.get(**kwargs)
+        if not batch["ids"]:
+            break
+        for did, meta in zip(batch["ids"], batch["metadatas"]):
+            src = meta.get("source_file", "unknown")
+            if source_pattern and source_pattern.lower() not in src.lower():
+                continue
+            groups[src].append(did)
+        offset += len(batch["ids"])
+
+    return {src: ids for src, ids in groups.items() if len(ids) >= min_count}
+
+
+def dedup_source_group(col, drawer_ids, threshold=DEFAULT_THRESHOLD, dry_run=True):
+    """Dedup drawers within one source_file group.
+
+    Greedy: sort by doc length (longest first), keep if not too similar
+    to any already-kept drawer. Returns (kept_ids, deleted_ids).
+    """
+    data = col.get(ids=drawer_ids, include=["documents", "metadatas"])
+    items = list(zip(data["ids"], data["documents"], data["metadatas"]))
+    items.sort(key=lambda x: len(x[1] or ""), reverse=True)
+
+    kept = []
+    to_delete = []
+
+    for did, doc, meta in items:
+        if not doc or len(doc) < 20:
+            to_delete.append(did)
+            continue
+
+        if not kept:
+            kept.append((did, doc))
+            continue
+
+        try:
+            results = col.query(
+                query_texts=[doc],
+                n_results=min(len(kept), 5),
+                include=["distances"],
+            )
+            dists = results["distances"][0] if results["distances"] else []
+            kept_ids_set = {k[0] for k in kept}
+
+            is_dup = False
+            for rid, dist in zip(results["ids"][0], dists):
+                if rid in kept_ids_set and dist < threshold:
+                    is_dup = True
+                    break
+
+            if is_dup:
+                to_delete.append(did)
+            else:
+                kept.append((did, doc))
+        except Exception:
+            kept.append((did, doc))
+
+    if to_delete and not dry_run:
+        for i in range(0, len(to_delete), 500):
+            col.delete(ids=to_delete[i : i + 500])
+
+    return [k[0] for k in kept], to_delete
+
+
+def show_stats(palace_path=None):
+    """Show duplication statistics without making changes."""
+    palace_path = palace_path or _get_palace_path()
+    col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME)
+
+    groups = get_source_groups(col)
+
+    total_drawers = sum(len(ids) for ids in groups.values())
+    print(f"\n  Sources with {MIN_DRAWERS_TO_CHECK}+ drawers: {len(groups)}")
+    print(f"  Total drawers in those sources: {total_drawers:,}")
+
+    print("\n  Top 15 by drawer count:")
+    sorted_groups = sorted(groups.items(), key=lambda x: len(x[1]), reverse=True)
+    for src, ids in sorted_groups[:15]:
+        print(f"    {len(ids):4d}  {src[:65]}")
+
+    estimated_dups = sum(int(len(ids) * 0.4) for ids in groups.values() if len(ids) > 20)
+    print(f"\n  Estimated duplicates (groups > 20): ~{estimated_dups:,}")
+
+
+def dedup_palace(
+    palace_path=None,
+    threshold=DEFAULT_THRESHOLD,
+    dry_run=True,
+    source_pattern=None,
+    min_count=MIN_DRAWERS_TO_CHECK,
+    wing=None,
+):
+    """Main entry point: deduplicate near-identical drawers across the palace."""
+    palace_path = palace_path or _get_palace_path()
+
+    print(f"\n{'=' * 55}")
+    print("  MemPalace Deduplicator")
+    print(f"{'=' * 55}")
+
+    col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME)
+
+    print(f"  Palace: {palace_path}")
+    print(f"  Drawers: {col.count():,}")
+    print(f"  Threshold: {threshold}")
+    print(f"  Mode: {'DRY RUN' if dry_run else 'LIVE'}")
+    print(f"{'─' * 55}")
+
+    if wing:
+        print(f"  Wing: {wing}")
+    groups = get_source_groups(col, min_count, source_pattern, wing=wing)
+    print(f"\n  Sources to check: {len(groups)}")
+
+    t0 = time.time()
+    total_kept = 0
+    total_deleted = 0
+
+    sorted_groups = sorted(groups.items(), key=lambda x: len(x[1]), reverse=True)
+
+    for i, (src, drawer_ids) in enumerate(sorted_groups):
+        kept, deleted = dedup_source_group(col, drawer_ids, threshold, dry_run)
+        total_kept += len(kept)
+        total_deleted += len(deleted)
+
+        if deleted:
+            print(
+                f"  [{i + 1:3d}/{len(groups)}] "
+                f"{src[:50]:50s} {len(drawer_ids):4d} → {len(kept):4d}  "
+                f"(-{len(deleted)})"
+            )
+
+    elapsed = time.time() - t0
+
+    print(f"\n{'─' * 55}")
+    print(f"  Done in {elapsed:.1f}s")
+    print(
+        f"  Drawers: {total_kept + total_deleted:,} → {total_kept:,}  (-{total_deleted:,} removed)"
+    )
+    print(f"  Palace after: {col.count():,} drawers")
+
+    if dry_run:
+        print("\n  [DRY RUN] No changes written. Re-run without --dry-run to apply.")
+
+    print(f"{'=' * 55}\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Deduplicate near-identical drawers")
+    parser.add_argument("--palace", default=None, help="Palace directory path")
+    parser.add_argument(
+        "--threshold",
+        type=float,
+        default=DEFAULT_THRESHOLD,
+        help=f"Cosine distance threshold (default: {DEFAULT_THRESHOLD})",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="Preview without deleting")
+    parser.add_argument("--stats", action="store_true", help="Show stats only")
+    parser.add_argument("--wing", default=None, help="Scope dedup to a single wing")
+    parser.add_argument("--source", default=None, help="Filter by source file pattern")
+    args = parser.parse_args()
+
+    path = os.path.expanduser(args.palace) if args.palace else None
+
+    if args.stats:
+        show_stats(palace_path=path)
+    else:
+        dedup_palace(
+            palace_path=path,
+            threshold=args.threshold,
+            dry_run=args.dry_run,
+            source_pattern=args.source,
+            wing=args.wing,
+        )

mempalace/dialect.py

@@ -1,13 +1,19 @@
 #!/usr/bin/env python3
 """
-AAAK Dialect -- Compressed Symbolic Memory Language
+AAAK Dialect -- Structured Symbolic Summary Format
 ====================================================
 
-A structured symbolic format that ANY LLM reads natively at ~30x compression.
-Not latent vectors. Not English prose. A universal memory compression dialect.
+A lossy summarization format that extracts entities, topics, key sentences,
+emotions, and flags from plain text into a compact structured representation.
+Any LLM reads it natively — no decoder required.
 
 Works with: Claude, ChatGPT, Gemini, Llama, Mistral -- any model that reads text.
 
+NOTE: AAAK is NOT lossless compression. The original text cannot be reconstructed
+from AAAK output. It is a structured summary layer (closets) that points to the
+original verbatim content (drawers). The 96.6% benchmark score is from raw mode,
+not AAAK mode.
+
 Adapted for mempalace: works standalone on plain text and ChromaDB drawers.
 No dependency on palace.py or layers.py.
 
@@ -152,6 +158,8 @@ _FLAG_SIGNALS = {
 }
 
 # Common filler/stop words to strip from topic extraction
+_ALPHA_RE = re.compile(r"[^a-zA-Z]")
+
 _STOP_WORDS = {
     "the",
     "a",
@@ -311,13 +319,17 @@ class Dialect:
         dialect.generate_layer1("zettels/", output="LAYER1.aaak")
     """
 
-    def __init__(self, entities: Dict[str, str] = None, skip_names: List[str] = None):
+    def __init__(
+        self, entities: Dict[str, str] = None, skip_names: List[str] = None, lang: str = None
+    ):
         """
         Args:
             entities: Mapping of full names -> short codes.
                       e.g. {"Alice": "ALC", "Bob": "BOB"}
                       If None, entities are auto-coded from first 3 chars.
             skip_names: Names to skip (fictional characters, etc.)
+            lang: Language code (e.g. "fr", "ko"). Loads AAAK instruction
+                  and regex patterns from i18n dictionary.
         """
         self.entity_codes = {}
         if entities:
@@ -326,6 +338,15 @@ class Dialect:
                 self.entity_codes[name.lower()] = code
         self.skip_names = [n.lower() for n in (skip_names or [])]
 
+        # Load language-specific AAAK instruction and regex patterns
+        from mempalace.i18n import load_lang, t, current_lang, get_regex
+
+        if lang:
+            load_lang(lang)
+        self.lang = lang or current_lang()
+        self.aaak_instruction = t("aaak.instruction")
+        self.lang_regex = get_regex()
+
     @classmethod
     def from_config(cls, config_path: str) -> "Dialect":
         """Load entity mappings from a JSON config file.
@@ -341,6 +362,7 @@ class Dialect:
         return cls(
             entities=config.get("entities", {}),
             skip_names=config.get("skip_names", []),
+            lang=config.get("lang", "en"),
         )
 
     def save_config(self, config_path: str):
@@ -521,7 +543,7 @@ class Dialect:
         # Fallback: find capitalized words that look like names (2+ chars, not sentence-start)
         words = text.split()
         for i, w in enumerate(words):
-            clean = re.sub(r"[^a-zA-Z]", "", w)
+            clean = _ALPHA_RE.sub("", w)
             if (
                 len(clean) >= 2
                 and clean[0].isupper()
@@ -538,19 +560,19 @@ class Dialect:
 
     def compress(self, text: str, metadata: dict = None) -> str:
         """
-        Compress plain text into AAAK Dialect format.
+        Summarize plain text into AAAK Dialect format.
 
-        This is the primary method for mempalace: takes any text content
-        (drawer content, transcript chunk, note) and returns a compressed
-        symbolic representation.
+        Extracts entities, topics, a key sentence, emotions, and flags
+        from the input text. This is lossy — the original text cannot be
+        reconstructed from the output.
 
         Args:
-            text: Plain text content to compress
+            text: Plain text content to summarize
             metadata: Optional dict with keys like 'source_file', 'wing',
                       'room', 'date', etc.
 
         Returns:
-            AAAK-compressed string (~30x smaller than input)
+            AAAK-formatted summary string
         """
         metadata = metadata or {}
 
@@ -930,19 +952,34 @@ class Dialect:
 
     @staticmethod
     def count_tokens(text: str) -> int:
-        """Rough token count (1 token ~ 3 chars for structured text)."""
-        return len(text) // 3
+        """Estimate token count using word-based heuristic (~1.3 tokens per word).
+
+        This is an approximation. For accurate counts, use a real tokenizer
+        like tiktoken. The old len(text)//3 heuristic was wildly inaccurate
+        and made AAAK compression ratios look much better than reality.
+        """
+        words = text.split()
+        # Most English words tokenize to 1-2 tokens; punctuation and
+        # special chars in AAAK (|, +, :) each cost a token.
+        # ~1.3 tokens/word is a conservative average.
+        return max(1, int(len(words) * 1.3))
 
     def compression_stats(self, original_text: str, compressed: str) -> dict:
-        """Get compression statistics for a text->AAAK conversion."""
+        """Get size comparison stats for a text->AAAK conversion.
+
+        NOTE: AAAK is lossy summarization, not compression. The "ratio"
+        reflects how much shorter the summary is, not a compression ratio
+        in the traditional sense — information is lost.
+        """
         orig_tokens = self.count_tokens(original_text)
         comp_tokens = self.count_tokens(compressed)
         return {
-            "original_tokens": orig_tokens,
-            "compressed_tokens": comp_tokens,
-            "ratio": orig_tokens / max(comp_tokens, 1),
+            "original_tokens_est": orig_tokens,
+            "summary_tokens_est": comp_tokens,
+            "size_ratio": round(orig_tokens / max(comp_tokens, 1), 1),
             "original_chars": len(original_text),
-            "compressed_chars": len(compressed),
+            "summary_chars": len(compressed),
+            "note": "Estimates only. Use tiktoken for accurate counts. AAAK is lossy.",
         }
 
 
@@ -1021,9 +1058,9 @@ if __name__ == "__main__":
         encoded = dialect.encode_file(data)
         stats = dialect.compression_stats(json_str, encoded)
         print("=== COMPRESSION STATS ===")
-        print(f"JSON:     ~{stats['original_tokens']:,} tokens")
-        print(f"AAAK:     ~{stats['compressed_tokens']:,} tokens")
-        print(f"Ratio:    {stats['ratio']:.0f}x")
+        print(f"JSON:     ~{stats['original_tokens_est']:,} tokens (est)")
+        print(f"AAAK:     ~{stats['summary_tokens_est']:,} tokens (est)")
+        print(f"Ratio:    {stats['size_ratio']}x (lossy — information is lost)")
         print()
         print("=== AAAK DIALECT OUTPUT ===")
         print(encoded)
@@ -1043,8 +1080,12 @@ if __name__ == "__main__":
         text = " ".join(args)
         compressed = dialect.compress(text)
         stats = dialect.compression_stats(text, compressed)
-        print(f"Original: ~{stats['original_tokens']} tokens ({stats['original_chars']} chars)")
-        print(f"AAAK:     ~{stats['compressed_tokens']} tokens ({stats['compressed_chars']} chars)")
-        print(f"Ratio:    {stats['ratio']:.1f}x")
+        print(
+            f"Original: ~{stats['original_tokens_est']} tokens est ({stats['original_chars']} chars)"
+        )
+        print(
+            f"AAAK:     ~{stats['summary_tokens_est']} tokens est ({stats['summary_chars']} chars)"
+        )
+        print(f"Ratio:    {stats['size_ratio']}x (lossy summary, not lossless compression)")
         print()
         print(compressed)

mempalace/diary_ingest.py

@@ -0,0 +1,209 @@
+"""
+diary_ingest.py — Ingest daily summary files into the palace.
+
+Architecture:
+- ONE drawer per (wing, day) — full verbatim content, upserted as the day grows.
+- Closets pack topics up to CLOSET_CHAR_LIMIT, never split mid-topic.
+- A re-ingest fully purges the prior day's closets before rebuilding so a
+  shorter day never leaves orphans behind.
+- Only new entries are processed by default (tracks entry count in a state
+  file under ``~/.mempalace/state/`` — never inside the user's diary dir).
+- Per-file ``mine_lock`` so concurrent ingest from two terminals can't race.
+- Entities extracted and stamped on metadata for filterable search.
+
+Usage:
+    python -m mempalace.diary_ingest --dir ~/daily_summaries --palace ~/.mempalace/palace
+    python -m mempalace.diary_ingest --dir ~/daily_summaries --palace ~/.mempalace/palace --force
+"""
+
+import hashlib
+import json
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .miner import _extract_entities_for_metadata
+from .palace import (
+    build_closet_lines,
+    get_closets_collection,
+    get_collection,
+    mine_lock,
+    purge_file_closets,
+    upsert_closet_lines,
+)
+
+DIARY_ENTRY_RE = re.compile(r"^## .+", re.MULTILINE)
+
+
+def _state_file_for(palace_path: str, diary_dir: Path) -> Path:
+    """Return the per-(palace, diary-dir) state-file path under ~/.mempalace/state.
+
+    Keyed by sha256 of (palace_path, diary_dir) so multiple diary folders
+    pointing at the same palace each get an independent state file. The
+    state file is *never* written inside the user's diary directory.
+    """
+    state_root = Path(os.path.expanduser("~")) / ".mempalace" / "state"
+    state_root.mkdir(parents=True, exist_ok=True)
+    key = hashlib.sha256(f"{palace_path}|{diary_dir}".encode()).hexdigest()[:24]
+    return state_root / f"diary_ingest_{key}.json"
+
+
+def _split_entries(text):
+    """Split diary text into (header, body) pairs per ## entry."""
+    parts = DIARY_ENTRY_RE.split(text)
+    headers = DIARY_ENTRY_RE.findall(text)
+    entries = []
+    for i, header in enumerate(headers):
+        body = parts[i + 1] if i + 1 < len(parts) else ""
+        entries.append((header.strip(), body.strip()))
+    return entries
+
+
+def _diary_drawer_id(wing: str, date_str: str) -> str:
+    """Stable, wing-scoped drawer ID. Two diaries (e.g. 'work' vs 'personal')
+    sharing the same date never collide."""
+    suffix = hashlib.sha256(f"{wing}|{date_str}".encode()).hexdigest()[:24]
+    return f"drawer_diary_{suffix}"
+
+
+def _diary_closet_id_base(wing: str, date_str: str) -> str:
+    suffix = hashlib.sha256(f"{wing}|{date_str}".encode()).hexdigest()[:24]
+    return f"closet_diary_{suffix}"
+
+
+def ingest_diaries(
+    diary_dir,
+    palace_path,
+    wing="diary",
+    force=False,
+):
+    """Ingest daily summary files into the palace.
+
+    Each date file gets ONE drawer keyed by ``(wing, date)`` and closets that
+    pack topics atomically up to ``CLOSET_CHAR_LIMIT``. ``force=True`` rebuilds
+    every entry's closets from scratch (purging stale ones); the default
+    incremental mode only processes entries appended since the last run.
+    """
+    diary_dir = Path(diary_dir).expanduser().resolve()
+    if not diary_dir.exists():
+        print(f"Diary directory not found: {diary_dir}")
+        return {"days_updated": 0, "closets_created": 0}
+
+    diary_files = sorted(diary_dir.glob("*.md"))
+    if not diary_files:
+        print(f"No .md files in {diary_dir}")
+        return {"days_updated": 0, "closets_created": 0}
+
+    state_file = _state_file_for(str(palace_path), diary_dir)
+    if force or not state_file.exists():
+        state: dict = {}
+    else:
+        try:
+            state = json.loads(state_file.read_text())
+        except Exception:
+            state = {}
+
+    drawers_col = get_collection(palace_path)
+    closets_col = get_closets_collection(palace_path)
+
+    days_updated = 0
+    closets_created = 0
+
+    for diary_path in diary_files:
+        text = diary_path.read_text(encoding="utf-8", errors="replace")
+        if len(text.strip()) < 50:
+            continue
+
+        date_match = re.match(r"(\d{4}-\d{2}-\d{2})", diary_path.stem)
+        if not date_match:
+            continue
+        date_str = date_match.group(1)
+
+        # Skip if content hasn't changed
+        state_key = f"{wing}|{diary_path.name}"
+        prev_size = state.get(state_key, {}).get("size", 0)
+        curr_size = len(text)
+        if curr_size == prev_size and not force:
+            continue
+
+        now_iso = datetime.now(timezone.utc).isoformat()
+        drawer_id = _diary_drawer_id(wing, date_str)
+        entities = _extract_entities_for_metadata(text)
+        source_file = str(diary_path)
+
+        # Serialize per source — two terminals running ingest at once must
+        # not interleave the upsert + closet-rebuild.
+        with mine_lock(source_file):
+            drawer_meta = {
+                "date": date_str,
+                "wing": wing,
+                "room": "daily",
+                "source_file": source_file,
+                "source_session": "daily_diary",
+                "filed_at": now_iso,
+            }
+            if entities:
+                drawer_meta["entities"] = entities
+            drawers_col.upsert(
+                documents=[text],
+                ids=[drawer_id],
+                metadatas=[drawer_meta],
+            )
+
+            entries = _split_entries(text)
+            prev_entry_count = state.get(state_key, {}).get("entry_count", 0)
+            new_entries = entries if force else entries[prev_entry_count:]
+
+            if new_entries:
+                all_lines = []
+                for header, body in new_entries:
+                    entry_text = f"{header}\n{body}"
+                    entry_lines = build_closet_lines(
+                        source_file, [drawer_id], entry_text, wing, "daily"
+                    )
+                    all_lines.extend(entry_lines)
+
+                if all_lines:
+                    closet_id_base = _diary_closet_id_base(wing, date_str)
+                    closet_meta = {
+                        "date": date_str,
+                        "wing": wing,
+                        "room": "daily",
+                        "source_file": source_file,
+                        "filed_at": now_iso,
+                    }
+                    if entities:
+                        closet_meta["entities"] = entities
+                    # On a force rebuild, wipe any leftover numbered closets
+                    # from a longer prior run before re-writing.
+                    if force:
+                        purge_file_closets(closets_col, source_file)
+                    n = upsert_closet_lines(closets_col, closet_id_base, all_lines, closet_meta)
+                    closets_created += n
+
+            state[state_key] = {
+                "size": curr_size,
+                "entry_count": len(entries),
+                "ingested_at": now_iso,
+            }
+        days_updated += 1
+
+    state_file.write_text(json.dumps(state, indent=2))
+    if days_updated:
+        print(f"Diary: {days_updated} days updated, {closets_created} new closets")
+
+    return {"days_updated": days_updated, "closets_created": closets_created}
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Ingest daily summaries into the palace")
+    parser.add_argument("--dir", required=True, help="Path to daily_summaries directory")
+    parser.add_argument("--palace", default=os.path.expanduser("~/.mempalace/palace"))
+    parser.add_argument("--wing", default="diary")
+    parser.add_argument("--force", action="store_true")
+    args = parser.parse_args()
+
+    ingest_diaries(args.dir, args.palace, wing=args.wing, force=args.force)

mempalace/embedding.py

@@ -0,0 +1,155 @@
+"""Embedding function factory with hardware acceleration.
+
+Returns a ChromaDB-compatible embedding function bound to a user-selected
+ONNX Runtime execution provider. The same ``all-MiniLM-L6-v2`` model and
+384-dim vectors ChromaDB ships by default are reused, so switching device
+does not invalidate existing palaces.
+
+Supported devices (env ``MEMPALACE_EMBEDDING_DEVICE`` or ``embedding_device``
+in ``~/.mempalace/config.json``):
+
+* ``auto`` — prefer CUDA ▸ CoreML ▸ DirectML, fall back to CPU
+* ``cpu`` — force CPU (the historical default)
+* ``cuda`` — NVIDIA GPU via ``onnxruntime-gpu`` (``pip install mempalace[gpu]``)
+* ``coreml`` — Apple Neural Engine (macOS)
+* ``dml`` — DirectML (Windows / AMD / Intel GPUs)
+
+Requesting an unavailable accelerator emits a warning and falls back to CPU
+rather than hard-failing — mining must still work on a laptop without CUDA.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+_PROVIDER_MAP = {
+    "cpu": ["CPUExecutionProvider"],
+    "cuda": ["CUDAExecutionProvider", "CPUExecutionProvider"],
+    "coreml": ["CoreMLExecutionProvider", "CPUExecutionProvider"],
+    "dml": ["DmlExecutionProvider", "CPUExecutionProvider"],
+}
+
+_DEVICE_EXTRA = {
+    "cuda": "mempalace[gpu]",
+    "coreml": "mempalace[coreml]",
+    "dml": "mempalace[dml]",
+}
+
+_AUTO_ORDER = [
+    ("CUDAExecutionProvider", "cuda"),
+    ("CoreMLExecutionProvider", "coreml"),
+    ("DmlExecutionProvider", "dml"),
+]
+
+_EF_CACHE: dict = {}
+_WARNED: set = set()
+
+
+def _resolve_providers(device: str) -> tuple[list, str]:
+    """Return ``(provider_list, effective_device)`` for ``device``.
+
+    Falls back to CPU (with a one-shot warning) when the requested
+    accelerator is not compiled into the installed ``onnxruntime``.
+    """
+    device = (device or "auto").strip().lower()
+
+    try:
+        import onnxruntime as ort
+
+        available = set(ort.get_available_providers())
+    except ImportError:
+        return (["CPUExecutionProvider"], "cpu")
+
+    if device == "auto":
+        for provider, name in _AUTO_ORDER:
+            if provider in available:
+                return ([provider, "CPUExecutionProvider"], name)
+        return (["CPUExecutionProvider"], "cpu")
+
+    requested = _PROVIDER_MAP.get(device)
+    if requested is None:
+        if device not in _WARNED:
+            logger.warning("Unknown embedding_device %r — falling back to cpu", device)
+            _WARNED.add(device)
+        return (["CPUExecutionProvider"], "cpu")
+
+    preferred = requested[0]
+    if preferred == "CPUExecutionProvider":
+        return (requested, "cpu")
+
+    if preferred not in available:
+        if device not in _WARNED:
+            extra = _DEVICE_EXTRA.get(device, "the matching mempalace extra for your device")
+            logger.warning(
+                "embedding_device=%r requested but %s is not installed — "
+                "falling back to CPU. Install %s.",
+                device,
+                preferred,
+                extra,
+            )
+            _WARNED.add(device)
+        return (["CPUExecutionProvider"], "cpu")
+
+    return (requested, device)
+
+
+def _build_ef_class():
+    """Subclass ``ONNXMiniLM_L6_V2`` with name ``"default"``.
+
+    Why the rename: ChromaDB 1.5 persists the EF identity on the collection
+    and rejects reads that pass a differently-named EF (``onnx_mini_lm_l6_v2``
+    vs ``default``). The vectors and model are identical — only the
+    ``name()`` tag differs — so spoofing the name lets one EF class serve
+    palaces created with ``DefaultEmbeddingFunction`` *and* palaces we
+    create ourselves, with the same GPU-capable ``preferred_providers``.
+    """
+    from chromadb.utils.embedding_functions import ONNXMiniLM_L6_V2
+
+    class _MempalaceONNX(ONNXMiniLM_L6_V2):
+        @staticmethod
+        def name() -> str:
+            return "default"
+
+    return _MempalaceONNX
+
+
+def get_embedding_function(device: Optional[str] = None):
+    """Return a cached embedding function bound to the requested device.
+
+    ``device=None`` reads from :class:`MempalaceConfig.embedding_device`.
+    The returned function is shared across calls with the same resolved
+    provider list so we only pay model-load cost once per process.
+    """
+    if device is None:
+        from .config import MempalaceConfig
+
+        device = MempalaceConfig().embedding_device
+
+    providers, effective = _resolve_providers(device)
+    cache_key = tuple(providers)
+    cached = _EF_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+
+    ef_cls = _build_ef_class()
+    ef = ef_cls(preferred_providers=providers)
+    _EF_CACHE[cache_key] = ef
+    logger.info("Embedding function initialized (device=%s providers=%s)", effective, providers)
+    return ef
+
+
+def describe_device(device: Optional[str] = None) -> str:
+    """Return a short human-readable label for the resolved device.
+
+    Used by the miner CLI header so users can see at a glance whether GPU
+    acceleration actually engaged.
+    """
+    if device is None:
+        from .config import MempalaceConfig
+
+        device = MempalaceConfig().embedding_device
+    _, effective = _resolve_providers(device)
+    return effective

mempalace/entity_detector.py

@@ -2,6 +2,9 @@
 """
 entity_detector.py — Auto-detect people and projects from file content.
 
+Uses ``from __future__ import annotations`` so PEP 604 union syntax
+(``dict | None``) works on the Python 3.9 baseline.
+
 Two-pass approach:
   Pass 1: scan files, extract entity candidates with signal counts
   Pass 2: score and classify each candidate as person, project, or uncertain
@@ -9,391 +12,72 @@ Two-pass approach:
 Used by mempalace init before mining begins.
 The confirmed entity map feeds the miner as the taxonomy.
 
+Multi-language support:
+    All lexical patterns (person verbs, pronouns, dialogue markers, project
+    verbs, stopwords, and the candidate-extraction character class) live in
+    the ``entity`` section of ``mempalace/i18n/<lang>.json``. Every public
+    function accepts a ``languages`` tuple and applies the union of the
+    requested locales' patterns. The default is ``("en",)`` — existing
+    English-only callers behave exactly as before.
+
+    To add a new language: add an ``entity`` section to that locale's JSON.
+    No code changes required.
+
 Usage:
-    from entity_detector import detect_entities, confirm_entities
-    candidates = detect_entities(file_paths)
+    from mempalace.entity_detector import detect_entities, confirm_entities
+    candidates = detect_entities(file_paths)                    # English only
+    candidates = detect_entities(paths, languages=("en", "pt-br"))
     confirmed = confirm_entities(candidates)  # interactive review
 """
 
+from __future__ import annotations
+
 import re
 import os
+import functools
 from pathlib import Path
 from collections import defaultdict
 
+from mempalace.i18n import get_entity_patterns
 
-# ==================== SIGNAL PATTERNS ====================
 
-# Person signals — things people do
-PERSON_VERB_PATTERNS = [
-    r"\b{name}\s+said\b",
-    r"\b{name}\s+asked\b",
-    r"\b{name}\s+told\b",
-    r"\b{name}\s+replied\b",
-    r"\b{name}\s+laughed\b",
-    r"\b{name}\s+smiled\b",
-    r"\b{name}\s+cried\b",
-    r"\b{name}\s+felt\b",
-    r"\b{name}\s+thinks?\b",
-    r"\b{name}\s+wants?\b",
-    r"\b{name}\s+loves?\b",
-    r"\b{name}\s+hates?\b",
-    r"\b{name}\s+knows?\b",
-    r"\b{name}\s+decided\b",
-    r"\b{name}\s+pushed\b",
-    r"\b{name}\s+wrote\b",
-    r"\bhey\s+{name}\b",
-    r"\bthanks?\s+{name}\b",
-    r"\bhi\s+{name}\b",
-    r"\bdear\s+{name}\b",
-]
+# ==================== LANGUAGE-AWARE PATTERN LOADING ====================
 
-# Person signals — pronouns resolving nearby
-PRONOUN_PATTERNS = [
-    r"\bshe\b",
-    r"\bher\b",
-    r"\bhers\b",
-    r"\bhe\b",
-    r"\bhim\b",
-    r"\bhis\b",
-    r"\bthey\b",
-    r"\bthem\b",
-    r"\btheir\b",
-]
 
-# Person signals — dialogue markers
-DIALOGUE_PATTERNS = [
-    r"^>\s*{name}[:\s]",  # > Speaker: ...
-    r"^{name}:\s",  # Speaker: ...
-    r"^\[{name}\]",  # [Speaker]
-    r'"{name}\s+said',
-]
+def _normalize_langs(languages) -> tuple:
+    """Coerce a language input into a non-empty hashable tuple."""
+    if not languages:
+        return ("en",)
+    if isinstance(languages, str):
+        return (languages,)
+    return tuple(languages)
 
-# Project signals — things projects have/do
-PROJECT_VERB_PATTERNS = [
-    r"\bbuilding\s+{name}\b",
-    r"\bbuilt\s+{name}\b",
-    r"\bship(?:ping|ped)?\s+{name}\b",
-    r"\blaunch(?:ing|ed)?\s+{name}\b",
-    r"\bdeploy(?:ing|ed)?\s+{name}\b",
-    r"\binstall(?:ing|ed)?\s+{name}\b",
-    r"\bthe\s+{name}\s+architecture\b",
-    r"\bthe\s+{name}\s+pipeline\b",
-    r"\bthe\s+{name}\s+system\b",
-    r"\bthe\s+{name}\s+repo\b",
-    r"\b{name}\s+v\d+\b",  # MemPal v2
-    r"\b{name}\.py\b",  # mempalace.py
-    r"\b{name}-core\b",  # mempal-core (hyphen only, not underscore)
-    r"\b{name}-local\b",
-    r"\bimport\s+{name}\b",
-    r"\bpip\s+install\s+{name}\b",
-]
 
-# Words that are almost certainly NOT entities
-STOPWORDS = {
-    "the",
-    "a",
-    "an",
-    "and",
-    "or",
-    "but",
-    "in",
-    "on",
-    "at",
-    "to",
-    "for",
-    "of",
-    "with",
-    "by",
-    "from",
-    "as",
-    "is",
-    "was",
-    "are",
-    "were",
-    "be",
-    "been",
-    "being",
-    "have",
-    "has",
-    "had",
-    "do",
-    "does",
-    "did",
-    "will",
-    "would",
-    "could",
-    "should",
-    "may",
-    "might",
-    "must",
-    "shall",
-    "can",
-    "this",
-    "that",
-    "these",
-    "those",
-    "it",
-    "its",
-    "they",
-    "them",
-    "their",
-    "we",
-    "our",
-    "you",
-    "your",
-    "i",
-    "my",
-    "me",
-    "he",
-    "she",
-    "his",
-    "her",
-    "who",
-    "what",
-    "when",
-    "where",
-    "why",
-    "how",
-    "which",
-    "if",
-    "then",
-    "so",
-    "not",
-    "no",
-    "yes",
-    "ok",
-    "okay",
-    "just",
-    "very",
-    "really",
-    "also",
-    "already",
-    "still",
-    "even",
-    "only",
-    "here",
-    "there",
-    "now",
-    "then",
-    "too",
-    "up",
-    "out",
-    "about",
-    "like",
-    "use",
-    "get",
-    "got",
-    "make",
-    "made",
-    "take",
-    "put",
-    "come",
-    "go",
-    "see",
-    "know",
-    "think",
-    "true",
-    "false",
-    "none",
-    "null",
-    "new",
-    "old",
-    "all",
-    "any",
-    "some",
-    "true",
-    "false",
-    "return",
-    "print",
-    "def",
-    "class",
-    "import",
-    "from",
-    # Common capitalized words in prose that aren't entities
-    "step",
-    "usage",
-    "run",
-    "check",
-    "find",
-    "add",
-    "get",
-    "set",
-    "list",
-    "args",
-    "dict",
-    "str",
-    "int",
-    "bool",
-    "path",
-    "file",
-    "type",
-    "name",
-    "note",
-    "example",
-    "option",
-    "result",
-    "error",
-    "warning",
-    "info",
-    "every",
-    "each",
-    "more",
-    "less",
-    "next",
-    "last",
-    "first",
-    "second",
-    "stack",
-    "layer",
-    "mode",
-    "test",
-    "stop",
-    "start",
-    "copy",
-    "move",
-    "source",
-    "target",
-    "output",
-    "input",
-    "data",
-    "item",
-    "key",
-    "value",
-    "returns",
-    "raises",
-    "yields",
-    "none",
-    "self",
-    "cls",
-    "kwargs",
-    # Common sentence-starting / abstract words that aren't entities
-    "world",
-    "well",
-    "want",
-    "topic",
-    "choose",
-    "social",
-    "cars",
-    "phones",
-    "healthcare",
-    "ex",
-    "machina",
-    "deus",
-    "human",
-    "humans",
-    "people",
-    "things",
-    "something",
-    "nothing",
-    "everything",
-    "anything",
-    "someone",
-    "everyone",
-    "anyone",
-    "way",
-    "time",
-    "day",
-    "life",
-    "place",
-    "thing",
-    "part",
-    "kind",
-    "sort",
-    "case",
-    "point",
-    "idea",
-    "fact",
-    "sense",
-    "question",
-    "answer",
-    "reason",
-    "number",
-    "version",
-    "system",
-    # Greetings and filler words at sentence starts
-    "hey",
-    "hi",
-    "hello",
-    "thanks",
-    "thank",
-    "right",
-    "let",
-    "ok",
-    # UI/action words that appear in how-to content
-    "click",
-    "hit",
-    "press",
-    "tap",
-    "drag",
-    "drop",
-    "open",
-    "close",
-    "save",
-    "load",
-    "launch",
-    "install",
-    "download",
-    "upload",
-    "scroll",
-    "select",
-    "enter",
-    "submit",
-    "cancel",
-    "confirm",
-    "delete",
-    "copy",
-    "paste",
-    "type",
-    "write",
-    "read",
-    "search",
-    "find",
-    "show",
-    "hide",
-    # Common filesystem/technical capitalized words
-    "desktop",
-    "documents",
-    "downloads",
-    "users",
-    "home",
-    "library",
-    "applications",
-    "system",
-    "preferences",
-    "settings",
-    "terminal",
-    # Abstract/topic words
-    "actor",
-    "vector",
-    "remote",
-    "control",
-    "duration",
-    "fetch",
-    # Abstract concepts that appear as subjects but aren't entities
-    "agents",
-    "tools",
-    "others",
-    "guards",
-    "ethics",
-    "regulation",
-    "learning",
-    "thinking",
-    "memory",
-    "language",
-    "intelligence",
-    "technology",
-    "society",
-    "culture",
-    "future",
-    "history",
-    "science",
-    "model",
-    "models",
-    "network",
-    "networks",
-    "training",
-    "inference",
-}
+@functools.lru_cache(maxsize=32)
+def _get_stopwords(languages: tuple) -> frozenset:
+    """Return the union of stopwords across the given languages."""
+    patterns = get_entity_patterns(languages)
+    return frozenset(patterns["stopwords"])
+
+
+# ==================== BACKWARD-COMPAT MODULE CONSTANTS ====================
+#
+# These mirror the old module-level constants so existing imports keep working.
+# They reflect the English defaults and are populated at import time from
+# ``mempalace/i18n/en.json``. Callers that need multi-language behavior should
+# pass the ``languages`` parameter to the public functions below.
+
+_EN = get_entity_patterns(("en",))
+
+PERSON_VERB_PATTERNS = list(_EN["person_verb_patterns"])
+PRONOUN_PATTERNS = list(_EN["pronoun_patterns"])
+PRONOUN_RE = re.compile("|".join(PRONOUN_PATTERNS), re.IGNORECASE) if PRONOUN_PATTERNS else None
+DIALOGUE_PATTERNS = list(_EN["dialogue_patterns"])
+PROJECT_VERB_PATTERNS = list(_EN["project_verb_patterns"])
+STOPWORDS = set(_EN["stopwords"])
+
+
+# ==================== EXTENSION POINTS (not language-scoped) ====================
 
 # For entity detection — prose only, no code files
 # Code files have too many capitalized names (classes, functions) that aren't entities
@@ -434,61 +118,130 @@ SKIP_DIRS = {
     ".next",
     "coverage",
     ".mempalace",
+    ".terraform",
+    "vendor",
+    "target",
+}
+
+# Files whose content is boilerplate prose — poisons entity detection.
+# Matched by stem (case-insensitive), with or without an extension.
+SKIP_FILENAMES = {
+    "license",
+    "licence",
+    "copying",
+    "copyright",
+    "notice",
+    "authors",
+    "patents",
+    "third_party_notices",
+    "third-party-notices",
 }
 
 
 # ==================== CANDIDATE EXTRACTION ====================
 
 
-def extract_candidates(text: str) -> dict:
+def extract_candidates(text: str, languages=("en",)) -> dict:
     """
     Extract all capitalized proper noun candidates from text.
     Returns {name: frequency} for names appearing 3+ times.
-    """
-    # Find all capitalized words (not at sentence start — harder, so we use frequency as filter)
-    raw = re.findall(r"\b([A-Z][a-z]{1,19})\b", text)
 
-    counts = defaultdict(int)
-    for word in raw:
-        if word.lower() not in STOPWORDS and len(word) > 1:
+    Each language contributes its own character-class pattern (e.g. ASCII
+    for English, Latin+diacritics for pt-br, Cyrillic for Russian,
+    Devanagari for Hindi). Matches from all languages are unioned.
+    """
+    langs = _normalize_langs(languages)
+    patterns = get_entity_patterns(langs)
+    stopwords = _get_stopwords(langs)
+
+    counts: defaultdict = defaultdict(int)
+
+    # Single-word candidates — one pre-wrapped pattern per language
+    for wrapped_pat in patterns["candidate_patterns"]:
+        try:
+            rx = re.compile(wrapped_pat)
+        except re.error:
+            continue
+        for word in rx.findall(text):
+            if word.lower() in stopwords:
+                continue
+            if len(word) < 2:
+                continue
             counts[word] += 1
 
-    # Also find multi-word proper nouns (e.g. "Memory Palace", "Claude Code")
-    multi = re.findall(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b", text)
-    for phrase in multi:
-        if not any(w.lower() in STOPWORDS for w in phrase.split()):
+    # Multi-word candidates — one pre-wrapped pattern per language
+    for wrapped_pat in patterns["multi_word_patterns"]:
+        try:
+            rx = re.compile(wrapped_pat)
+        except re.error:
+            continue
+        for phrase in rx.findall(text):
+            if any(w.lower() in stopwords for w in phrase.split()):
+                continue
             counts[phrase] += 1
 
-    # Filter: must appear at least 3 times to be a candidate
     return {name: count for name, count in counts.items() if count >= 3}
 
 
 # ==================== SIGNAL SCORING ====================
 
 
-def _build_patterns(name: str) -> dict:
-    """Pre-compile all regex patterns for a single entity name."""
+@functools.lru_cache(maxsize=256)
+def _build_patterns(name: str, languages: tuple = ("en",)) -> dict:
+    """Pre-compile all regex patterns for a single entity name, per language set."""
     n = re.escape(name)
+    langs = _normalize_langs(languages)
+    sources = get_entity_patterns(langs)
+
+    def _compile_each(raw_patterns, flags=re.IGNORECASE):
+        compiled = []
+        for p in raw_patterns:
+            try:
+                compiled.append(re.compile(p.format(name=n), flags))
+            except (re.error, KeyError, IndexError):
+                continue
+        return compiled
+
+    direct_sources = sources.get("direct_address_patterns") or []
+    direct_compiled = []
+    for raw in direct_sources:
+        try:
+            direct_compiled.append(re.compile(raw.format(name=n), re.IGNORECASE))
+        except (re.error, KeyError, IndexError):
+            continue
+
     return {
-        "dialogue": [
-            re.compile(p.format(name=n), re.MULTILINE | re.IGNORECASE) for p in DIALOGUE_PATTERNS
-        ],
-        "person_verbs": [re.compile(p.format(name=n), re.IGNORECASE) for p in PERSON_VERB_PATTERNS],
-        "project_verbs": [
-            re.compile(p.format(name=n), re.IGNORECASE) for p in PROJECT_VERB_PATTERNS
-        ],
-        "direct": re.compile(rf"\bhey\s+{n}\b|\bthanks?\s+{n}\b|\bhi\s+{n}\b", re.IGNORECASE),
-        "versioned": re.compile(rf"\b{n}[-v]\w+", re.IGNORECASE),
+        "dialogue": _compile_each(sources["dialogue_patterns"], re.MULTILINE | re.IGNORECASE),
+        "person_verbs": _compile_each(sources["person_verb_patterns"]),
+        "project_verbs": _compile_each(sources["project_verb_patterns"]),
+        "direct": direct_compiled,
+        "versioned": re.compile(rf"\b{n}[-_]v?\d+(?:\.\d+)*\b", re.IGNORECASE),
         "code_ref": re.compile(rf"\b{n}\.(py|js|ts|yaml|yml|json|sh)\b", re.IGNORECASE),
     }
 
 
-def score_entity(name: str, text: str, lines: list) -> dict:
+@functools.lru_cache(maxsize=32)
+def _pronoun_re(languages: tuple):
+    """Compile a combined pronoun regex for the given languages."""
+    langs = _normalize_langs(languages)
+    patterns = get_entity_patterns(langs)
+    pronouns = patterns.get("pronoun_patterns") or []
+    if not pronouns:
+        return None
+    try:
+        return re.compile("|".join(pronouns), re.IGNORECASE)
+    except re.error:
+        return None
+
+
+def score_entity(name: str, text: str, lines: list, languages=("en",)) -> dict:
     """
     Score a candidate entity as person vs project.
     Returns scores and the signals that fired.
     """
-    patterns = _build_patterns(name)
+    langs = _normalize_langs(languages)
+    patterns = _build_patterns(name, langs)
+    pronoun_re = _pronoun_re(langs)
     person_score = 0
     project_score = 0
     person_signals = []
@@ -496,12 +249,19 @@ def score_entity(name: str, text: str, lines: list) -> dict:
 
     # --- Person signals ---
 
-    # Dialogue markers (strong signal)
+    # Dialogue markers (strong signal).
+    # The bare `^NAME:\s` colon-prefix pattern matches metadata lines like
+    # `Created: 2026-04-21`, so we require >= 2 hits for it to count as dialogue
+    # (real speaker markers repeat; single-line metadata doesn't).
     for rx in patterns["dialogue"]:
         matches = len(rx.findall(text))
-        if matches > 0:
-            person_score += matches * 3
-            person_signals.append(f"dialogue marker ({matches}x)")
+        if matches == 0:
+            continue
+        is_bare_colon = rx.pattern.endswith(r":\s") and not rx.pattern.endswith(r"[:\s]")
+        if is_bare_colon and matches < 2:
+            continue
+        person_score += matches * 3
+        person_signals.append(f"dialogue marker ({matches}x)")
 
     # Person verbs
     for rx in patterns["person_verbs"]:
@@ -511,24 +271,25 @@ def score_entity(name: str, text: str, lines: list) -> dict:
             person_signals.append(f"'{name} ...' action ({matches}x)")
 
     # Pronoun proximity — pronouns within 3 lines of the name
-    name_lower = name.lower()
-    name_line_indices = [i for i, line in enumerate(lines) if name_lower in line.lower()]
-    pronoun_hits = 0
-    for idx in name_line_indices:
-        window_text = " ".join(lines[max(0, idx - 2) : idx + 3]).lower()
-        for pronoun_pattern in PRONOUN_PATTERNS:
-            if re.search(pronoun_pattern, window_text):
+    if pronoun_re is not None:
+        name_lower = name.lower()
+        name_line_indices = [i for i, line in enumerate(lines) if name_lower in line.lower()]
+        pronoun_hits = 0
+        for idx in name_line_indices:
+            window_text = " ".join(lines[max(0, idx - 2) : idx + 3])
+            if pronoun_re.search(window_text):
                 pronoun_hits += 1
-                break
-    if pronoun_hits > 0:
-        person_score += pronoun_hits * 2
-        person_signals.append(f"pronoun nearby ({pronoun_hits}x)")
+        if pronoun_hits > 0:
+            person_score += pronoun_hits * 2
+            person_signals.append(f"pronoun nearby ({pronoun_hits}x)")
 
     # Direct address
-    direct = len(patterns["direct"].findall(text))
-    if direct > 0:
-        person_score += direct * 4
-        person_signals.append(f"addressed directly ({direct}x)")
+    direct_hits = 0
+    for rx in patterns["direct"]:
+        direct_hits += len(rx.findall(text))
+    if direct_hits > 0:
+        person_score += direct_hits * 4
+        person_signals.append(f"addressed directly ({direct_hits}x)")
 
     # --- Project signals ---
 
@@ -596,17 +357,28 @@ def classify_entity(name: str, frequency: int, scores: dict) -> dict:
             signal_categories.add("addressed")
 
     has_two_signal_types = len(signal_categories) >= 2
-    _ = signal_categories - {"pronoun"}  # reserved for future thresholds
+    # Single-category pronoun signal still classifies as person when the
+    # evidence is overwhelming — a diary's main character is referenced
+    # with pronouns, not dialogue markers. Require both: many pronoun hits
+    # AND a high pronoun-to-frequency ratio so common sentence-start words
+    # (Never, Before, etc.) with incidental pronoun proximity don't qualify.
+    pronoun_hits = 0
+    for s in scores["person_signals"]:
+        m = re.search(r"pronoun nearby \((\d+)x\)", s)
+        if m:
+            pronoun_hits = int(m.group(1))
+            break
+    strong_pronoun_signal = pronoun_hits >= 5 and frequency > 0 and pronoun_hits / frequency >= 0.2
 
-    if person_ratio >= 0.7 and has_two_signal_types and ps >= 5:
+    if person_ratio >= 0.7 and (has_two_signal_types and ps >= 5 or strong_pronoun_signal):
         entity_type = "person"
         confidence = min(0.99, 0.5 + person_ratio * 0.5)
         signals = scores["person_signals"] or [f"appears {frequency}x"]
-    elif person_ratio >= 0.7 and (not has_two_signal_types or ps < 5):
-        # Pronoun-only match — downgrade to uncertain
+    elif person_ratio >= 0.7:
+        # Weak single-category person signal — downgrade to uncertain
         entity_type = "uncertain"
         confidence = 0.4
-        signals = scores["person_signals"] + [f"appears {frequency}x — pronoun-only match"]
+        signals = scores["person_signals"] + [f"appears {frequency}x — weak person signal"]
     elif person_ratio <= 0.3:
         entity_type = "project"
         confidence = min(0.99, 0.5 + (1 - person_ratio) * 0.5)
@@ -629,21 +401,42 @@ def classify_entity(name: str, frequency: int, scores: dict) -> dict:
 # ==================== MAIN DETECT ====================
 
 
-def detect_entities(file_paths: list, max_files: int = 10) -> dict:
+def detect_entities(
+    file_paths: list,
+    max_files: int = 10,
+    languages=("en",),
+    corpus_origin: dict | None = None,
+) -> dict:
     """
     Scan files and detect entity candidates.
 
     Args:
         file_paths: List of Path objects to scan
         max_files: Max files to read (for speed)
+        languages: Tuple of language codes whose entity patterns should be
+            applied (union). Defaults to ``("en",)``.
+        corpus_origin: Optional corpus-origin context (the dict produced
+            by ``mempalace.corpus_origin`` and persisted to
+            ``<palace>/.mempalace/origin.json`` by ``mempalace init``).
+            When supplied and the corpus is identified as AI-dialogue with
+            known agent persona names, candidates whose name matches an
+            agent persona are moved out of ``people``/``uncertain`` and
+            into a new ``agent_personas`` bucket. Shape:
+            ``{"schema_version": 1, "result": {"agent_persona_names": [...], ...}}``.
 
     Returns:
         {
             "people":   [...entity dicts...],
             "projects": [...entity dicts...],
+            "topics":   [...entity dicts...],
             "uncertain":[...entity dicts...],
+            # Only present when corpus_origin reclassifies at least one
+            # candidate as an agent persona:
+            "agent_personas": [...entity dicts...],
         }
     """
+    langs = _normalize_langs(languages)
+
     # Collect text from files
     all_text = []
     all_lines = []
@@ -660,16 +453,19 @@ def detect_entities(file_paths: list, max_files: int = 10) -> dict:
             all_text.append(content)
             all_lines.extend(content.splitlines())
             files_read += 1
-        except Exception:
+        except OSError:
             continue
 
     combined_text = "\n".join(all_text)
 
     # Extract candidates
-    candidates = extract_candidates(combined_text)
+    candidates = extract_candidates(combined_text, languages=langs)
 
     if not candidates:
-        return {"people": [], "projects": [], "uncertain": []}
+        return _apply_corpus_origin(
+            {"people": [], "projects": [], "topics": [], "uncertain": []},
+            corpus_origin,
+        )
 
     # Score and classify each candidate
     people = []
@@ -677,7 +473,7 @@ def detect_entities(file_paths: list, max_files: int = 10) -> dict:
     uncertain = []
 
     for name, frequency in sorted(candidates.items(), key=lambda x: x[1], reverse=True):
-        scores = score_entity(name, combined_text, all_lines)
+        scores = score_entity(name, combined_text, all_lines, languages=langs)
         entity = classify_entity(name, frequency, scores)
 
         if entity["type"] == "person":
@@ -692,13 +488,76 @@ def detect_entities(file_paths: list, max_files: int = 10) -> dict:
     projects.sort(key=lambda x: x["confidence"], reverse=True)
     uncertain.sort(key=lambda x: x["frequency"], reverse=True)
 
-    # Cap results to most relevant
-    return {
+    detected = {
         "people": people[:15],
         "projects": projects[:10],
+        "topics": [],
         "uncertain": uncertain[:8],
     }
 
+    return _apply_corpus_origin(detected, corpus_origin)
+
+
+def _apply_corpus_origin(detected: dict, corpus_origin: dict | None) -> dict:
+    """Reclassify per-candidate buckets using corpus-origin context.
+
+    When the corpus is identified as AI-dialogue with known agent persona
+    names, a candidate whose name case-insensitively matches one of those
+    personas is moved from ``people``/``uncertain`` into an
+    ``agent_personas`` bucket. The candidate's per-entity ``type`` is also
+    rewritten to ``"agent_persona"``.
+
+    No-op when ``corpus_origin`` is ``None`` or contains no usable persona
+    names. Pure: returns a new dict, does not mutate the input.
+    """
+    if not corpus_origin:
+        return detected
+
+    origin_result = corpus_origin.get("result") or {}
+    raw_personas = origin_result.get("agent_persona_names") or []
+    persona_lower = {n.lower() for n in raw_personas if isinstance(n, str)}
+    if not persona_lower:
+        return detected
+
+    agent_personas: list = []
+    new_people: list = []
+    new_uncertain: list = []
+
+    for entity in detected.get("people", []):
+        if entity["name"].lower() in persona_lower:
+            agent_personas.append(_tag_as_persona(entity))
+        else:
+            new_people.append(entity)
+
+    for entity in detected.get("uncertain", []):
+        if entity["name"].lower() in persona_lower:
+            agent_personas.append(_tag_as_persona(entity))
+        else:
+            new_uncertain.append(entity)
+
+    if not agent_personas:
+        return detected
+
+    agent_personas.sort(key=lambda x: x.get("confidence", 0), reverse=True)
+
+    return {
+        **detected,
+        "people": new_people,
+        "uncertain": new_uncertain,
+        "agent_personas": agent_personas,
+    }
+
+
+def _tag_as_persona(entity: dict) -> dict:
+    """Return a new entity dict tagged as agent_persona with provenance signal."""
+    existing_signals = entity.get("signals", [])
+    return {
+        **entity,
+        "type": "agent_persona",
+        "confidence": max(0.95, entity.get("confidence", 0.0)),
+        "signals": ["matched corpus_origin agent_persona_names"] + existing_signals[:2],
+    }
+
 
 # ==================== INTERACTIVE CONFIRM ====================
 
@@ -718,7 +577,13 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     """
     Interactive confirmation step.
     User reviews detected entities, removes wrong ones, adds missing ones.
-    Returns confirmed {people: [names], projects: [names]}
+    Returns confirmed {people: [names], projects: [names], topics: [names]}.
+
+    Topics are not surfaced for interactive review — they come from the
+    LLM-refined ``TOPIC`` bucket and are passed through verbatim. They
+    feed cross-wing tunnel computation at mine time (see
+    ``palace_graph.compute_topic_tunnels``); a wrong topic at worst adds
+    a low-traffic tunnel and never alters drawer storage.
 
     Pass yes=True to auto-accept all detected entities without prompting.
     """
@@ -730,18 +595,28 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     _print_entity_list(detected["people"], "PEOPLE")
     _print_entity_list(detected["projects"], "PROJECTS")
 
+    if detected.get("topics"):
+        _print_entity_list(detected["topics"], "TOPICS (cross-wing tunnel signal)")
+
     if detected["uncertain"]:
         _print_entity_list(detected["uncertain"], "UNCERTAIN (need your call)")
 
     confirmed_people = [e["name"] for e in detected["people"]]
     confirmed_projects = [e["name"] for e in detected["projects"]]
+    confirmed_topics = [e["name"] for e in detected.get("topics", [])]
 
     if yes:
         # Auto-accept: include all detected (skip uncertain — ambiguous without user input)
         print(
-            f"\n  Auto-accepting {len(confirmed_people)} people, {len(confirmed_projects)} projects."
+            f"\n  Auto-accepting {len(confirmed_people)} people, "
+            f"{len(confirmed_projects)} projects, "
+            f"{len(confirmed_topics)} topics."
         )
-        return {"people": confirmed_people, "projects": confirmed_projects}
+        return {
+            "people": confirmed_people,
+            "projects": confirmed_projects,
+            "topics": confirmed_topics,
+        }
 
     print(f"\n{'─' * 58}")
     print("  Options:")
@@ -760,7 +635,7 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
         if detected["uncertain"]:
             print("\n  Uncertain entities — classify each:")
             for e in detected["uncertain"]:
-                ans = input(f"    {e['name']} — (p)erson, (r)roject, or (s)kip? ").strip().lower()
+                ans = input(f"    {e['name']} — (p)erson, (r)project, or (s)kip? ").strip().lower()
                 if ans == "p":
                     confirmed_people.append(e["name"])
                 elif ans == "r":
@@ -789,7 +664,7 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
             name = input("  Name (or enter to stop): ").strip()
             if not name:
                 break
-            kind = input(f"  Is '{name}' a (p)erson or (r)roject? ").strip().lower()
+            kind = input(f"  Is '{name}' a (p)erson or p(r)oject? ").strip().lower()
             if kind == "p":
                 confirmed_people.append(name)
             elif kind == "r":
@@ -799,11 +674,14 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     print("  Confirmed:")
     print(f"  People:   {', '.join(confirmed_people) or '(none)'}")
     print(f"  Projects: {', '.join(confirmed_projects) or '(none)'}")
+    if confirmed_topics:
+        print(f"  Topics:   {', '.join(confirmed_topics)}")
     print(f"{'=' * 58}\n")
 
     return {
         "people": confirmed_people,
         "projects": confirmed_projects,
+        "topics": confirmed_topics,
     }
 
 
@@ -824,6 +702,8 @@ def scan_for_detection(project_dir: str, max_files: int = 10) -> list:
         dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
         for filename in filenames:
             filepath = Path(root) / filename
+            if filepath.stem.lower() in SKIP_FILENAMES:
+                continue
             ext = filepath.suffix.lower()
             if ext in PROSE_EXTENSIONS:
                 prose_files.append(filepath)
@@ -841,13 +721,14 @@ if __name__ == "__main__":
     import sys
 
     if len(sys.argv) < 2:
-        print("Usage: python entity_detector.py <directory>")
+        print("Usage: python entity_detector.py <directory> [lang1,lang2,...]")
         sys.exit(1)
 
     project_dir = sys.argv[1]
-    print(f"Scanning: {project_dir}")
+    langs = tuple(sys.argv[2].split(",")) if len(sys.argv) >= 3 else ("en",)
+    print(f"Scanning: {project_dir} (languages: {', '.join(langs)})")
     files = scan_for_detection(project_dir)
     print(f"Reading {len(files)} files...")
-    detected = detect_entities(files)
+    detected = detect_entities(files, languages=langs)
     confirmed = confirm_entities(detected)
     print("Confirmed entities:", confirmed)

mempalace/entity_registry.py

@@ -44,7 +44,6 @@ COMMON_ENGLISH_WORDS = {
     "chance",
     "chase",
     "hunter",
-    "hunter",
     "dash",
     "flash",
     "star",
@@ -78,8 +77,6 @@ COMMON_ENGLISH_WORDS = {
     "january",
     "february",
     "march",
-    "april",
-    "june",
     "july",
     "august",
     "september",
@@ -181,6 +178,12 @@ def _wikipedia_lookup(word: str) -> dict:
     Look up a word via Wikipedia REST API.
     Returns inferred type (person/place/concept/unknown) + confidence + summary.
     Free, no API key, handles disambiguation pages.
+
+    **Privacy warning:** This function makes an outbound HTTPS request to
+    en.wikipedia.org, sending the queried word over the network.  It should
+    only be called when the caller has explicitly opted in via
+    ``allow_network=True`` in :meth:`EntityRegistry.research`.  The default
+    behaviour of ``research()`` is local-only (no network calls).
     """
     try:
         url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{urllib.parse.quote(word)}"
@@ -247,16 +250,17 @@ def _wikipedia_lookup(word: str) -> dict:
 
     except urllib.error.HTTPError as e:
         if e.code == 404:
-            # Not in Wikipedia — strong signal it's a proper noun (unusual name, nickname)
+            # Not in Wikipedia — this tells us nothing definitive about
+            # the word.  Return "unknown" so the caller can decide.
             return {
-                "inferred_type": "person",
-                "confidence": 0.70,
+                "inferred_type": "unknown",
+                "confidence": 0.3,
                 "wiki_summary": None,
                 "wiki_title": None,
-                "note": "not found in Wikipedia — likely a proper noun or unusual name",
+                "note": "not found in Wikipedia",
             }
         return {"inferred_type": "unknown", "confidence": 0.0, "wiki_summary": None}
-    except Exception:
+    except (urllib.error.URLError, OSError, json.JSONDecodeError, KeyError):
         return {"inferred_type": "unknown", "confidence": 0.0, "wiki_summary": None}
 
 
@@ -304,7 +308,7 @@ class EntityRegistry:
         path = (Path(config_dir) / "entity_registry.json") if config_dir else cls.DEFAULT_PATH
         if path.exists():
             try:
-                data = json.loads(path.read_text())
+                data = json.loads(path.read_text(encoding="utf-8"))
                 return cls(data, path)
             except (json.JSONDecodeError, OSError):
                 pass
@@ -312,7 +316,15 @@ class EntityRegistry:
 
     def save(self):
         self._path.parent.mkdir(parents=True, exist_ok=True)
-        self._path.write_text(json.dumps(self._data, indent=2))
+        try:
+            self._path.parent.chmod(0o700)
+        except (OSError, NotImplementedError):
+            pass
+        self._path.write_text(json.dumps(self._data, indent=2), encoding="utf-8")
+        try:
+            self._path.chmod(0o600)
+        except (OSError, NotImplementedError):
+            pass
 
     @staticmethod
     def _empty() -> dict:
@@ -505,20 +517,41 @@ class EntityRegistry:
 
     # ── Research unknown words ───────────────────────────────────────────────
 
-    def research(self, word: str, auto_confirm: bool = False) -> dict:
+    def research(self, word: str, auto_confirm: bool = False, allow_network: bool = False) -> dict:
         """
-        Research an unknown word via Wikipedia.
-        Caches result. If auto_confirm=False, marks as unconfirmed (needs user review).
-        Returns the lookup result.
+        Research an unknown word.
+
+        By default this is **local-only**: it checks the wiki cache and
+        returns ``"unknown"`` for uncached words.  Pass
+        ``allow_network=True`` to explicitly opt in to an outbound
+        Wikipedia lookup.  This design honours the project's
+        *local-first, zero API* and *privacy by architecture* principles
+        — no data leaves the machine unless the caller requests it.
+
+        Caches result.  If *auto_confirm* is ``False``, marks the entry
+        as unconfirmed (needs user review).
         """
-        # Already cached?
-        cache = self._data.setdefault("wiki_cache", {})
+        # Check cache (read-only — no mutation when allow_network is False)
+        cache = self._data.get("wiki_cache", {})
         if word in cache:
             return cache[word]
 
+        if not allow_network:
+            return {
+                "inferred_type": "unknown",
+                "confidence": 0.0,
+                "wiki_summary": None,
+                "wiki_title": None,
+                "word": word,
+                "confirmed": False,
+                "note": "network lookup disabled — pass allow_network=True to query Wikipedia",
+            }
+
+        # Network path — ensure wiki_cache key exists before writing
+        cache = self._data.setdefault("wiki_cache", {})
         result = _wikipedia_lookup(word)
-        result["word"] = word
-        result["confirmed"] = auto_confirm
+        result.setdefault("word", word)
+        result.setdefault("confirmed", auto_confirm)
 
         cache[word] = result
         self.save()
@@ -550,15 +583,19 @@ class EntityRegistry:
 
     # ── Learn from sessions ──────────────────────────────────────────────────
 
-    def learn_from_text(self, text: str, min_confidence: float = 0.75) -> list:
+    def learn_from_text(self, text: str, min_confidence: float = 0.75, languages=("en",)) -> list:
         """
         Scan session text for new entity candidates.
         Returns list of newly discovered candidates for review.
+
+        ``languages`` is forwarded to entity detection — pass the user's
+        configured ``MempalaceConfig().entity_languages`` to match the
+        locales used at ``mempalace init`` time.
         """
         from mempalace.entity_detector import extract_candidates, score_entity, classify_entity
 
         lines = text.splitlines()
-        candidates = extract_candidates(text)
+        candidates = extract_candidates(text, languages=languages)
         new_candidates = []
 
         for name, frequency in candidates.items():
@@ -566,7 +603,7 @@ class EntityRegistry:
             if name in self.people or name in self.projects:
                 continue
 
-            scores = score_entity(name, text, lines)
+            scores = score_entity(name, text, lines, languages=languages)
             entity = classify_entity(name, frequency, scores)
 
             if entity["type"] == "person" and entity["confidence"] >= min_confidence:
@@ -597,7 +634,6 @@ class EntityRegistry:
         Returns list of canonical names found.
         """
         found = []
-        query.lower()
 
         for canonical, info in self.people.items():
             names_to_check = [canonical] + info.get("aliases", [])
@@ -620,7 +656,9 @@ class EntityRegistry:
         Find capitalized words in query that aren't in registry or common words.
         These are candidates for Wikipedia research.
         """
-        candidates = re.findall(r"\b[A-Z][a-z]{2,15}\b", query)
+        from .palace import _candidate_entity_words
+
+        candidates = _candidate_entity_words(query)
         unknown = []
         for word in set(candidates):
             if word.lower() in COMMON_ENGLISH_WORDS:

mempalace/exporter.py

@@ -0,0 +1,173 @@
+"""
+exporter.py — Export the palace as a browsable folder of markdown files.
+
+Produces:
+  output_dir/
+    index.md              — table of contents
+    wing_name/
+      room_name.md        — one file per room, drawers as sections
+
+Streams drawers in paginated batches so memory usage stays bounded
+regardless of palace size.
+"""
+
+import os
+import re
+from collections import defaultdict
+from datetime import datetime
+
+from .palace import get_collection
+
+
+def _safe_path_component(name: str) -> str:
+    """Sanitize a string for use as a directory/file name component."""
+    name = re.sub(r'[/\\:*?"<>|]', "_", name)
+    name = name.strip(". ")
+    return name or "unknown"
+
+
+def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -> dict:
+    """Export all palace drawers as markdown files organized by wing/room.
+
+    Streams drawers in batches of 1000 and writes each wing/room file
+    incrementally, keeping memory usage proportional to batch size rather
+    than total palace size.
+
+    Args:
+        palace_path: Path to the ChromaDB palace directory.
+        output_dir: Where to write the exported markdown tree.
+        format: Output format (currently only "markdown").
+
+    Returns:
+        Stats dict: {"wings": N, "rooms": N, "drawers": N}
+    """
+    col = get_collection(palace_path)
+    total = col.count()
+
+    if total == 0:
+        print("  Palace is empty — nothing to export.")
+        return {"wings": 0, "rooms": 0, "drawers": 0}
+
+    os.makedirs(output_dir, exist_ok=True)
+    try:
+        os.chmod(output_dir, 0o700)
+    except (OSError, NotImplementedError):
+        pass
+
+    # Track which room files have been opened (so we can append vs overwrite)
+    opened_rooms: set[tuple[str, str]] = set()
+    # Track which wing directories have been created and chmoded
+    created_wing_dirs: set[str] = set()
+    # Track stats per wing: {wing: {room: count}}
+    wing_stats: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    total_drawers = 0
+
+    print(f"  Streaming {total} drawers...")
+    offset = 0
+    while offset < total:
+        batch = col.get(limit=1000, offset=offset, include=["documents", "metadatas"])
+        if not batch["ids"]:
+            break
+
+        # Group this batch by wing/room so we do one file write per room per batch
+        batch_grouped: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list))
+        for doc_id, doc, meta in zip(batch["ids"], batch["documents"], batch["metadatas"]):
+            wing = meta.get("wing", "unknown")
+            room = meta.get("room", "general")
+            batch_grouped[wing][room].append(
+                {
+                    "id": doc_id,
+                    "content": doc,
+                    "source": meta.get("source_file", ""),
+                    "filed_at": meta.get("filed_at", ""),
+                    "added_by": meta.get("added_by", ""),
+                }
+            )
+
+        # Write/append each room file
+        for wing, rooms in batch_grouped.items():
+            safe_wing = _safe_path_component(wing)
+            wing_dir = os.path.join(output_dir, safe_wing)
+            if wing_dir not in created_wing_dirs:
+                os.makedirs(wing_dir, exist_ok=True)
+                try:
+                    os.chmod(wing_dir, 0o700)
+                except (OSError, NotImplementedError):
+                    pass
+                created_wing_dirs.add(wing_dir)
+
+            for room, drawers in rooms.items():
+                safe_room = _safe_path_component(room)
+                room_path = os.path.join(wing_dir, f"{safe_room}.md")
+                key = (wing, room)
+                is_new = key not in opened_rooms
+
+                with open(room_path, "a" if not is_new else "w", encoding="utf-8") as f:
+                    if is_new:
+                        f.write(f"# {wing} / {room}\n\n")
+                        opened_rooms.add(key)
+
+                    for drawer in drawers:
+                        source = drawer["source"] or "unknown"
+                        filed = drawer["filed_at"] or "unknown"
+                        added_by = drawer["added_by"] or "unknown"
+
+                        f.write(
+                            f"## {drawer['id']}\n"
+                            f"\n"
+                            f"> {_quote_content(drawer['content'])}\n"
+                            f"\n"
+                            f"| Field | Value |\n"
+                            f"|-------|-------|\n"
+                            f"| Source | {source} |\n"
+                            f"| Filed | {filed} |\n"
+                            f"| Added by | {added_by} |\n"
+                            f"\n"
+                            f"---\n\n"
+                        )
+
+                    wing_stats[wing][room] += len(drawers)
+                    total_drawers += len(drawers)
+
+        offset += len(batch["ids"])
+
+    # Build and print stats
+    index_rows = []
+    for wing in sorted(wing_stats):
+        rooms = wing_stats[wing]
+        wing_drawer_count = sum(rooms.values())
+        index_rows.append((wing, len(rooms), wing_drawer_count))
+        print(f"  {wing}: {len(rooms)} rooms, {wing_drawer_count} drawers")
+
+    # Write index.md
+    today = datetime.now().strftime("%Y-%m-%d")
+    index_lines = [
+        f"# Palace Export — {today}\n",
+        "",
+        "| Wing | Rooms | Drawers |",
+        "|------|-------|---------|",
+    ]
+    for wing, room_count, drawer_count in index_rows:
+        index_lines.append(f"| [{wing}]({wing}/) | {room_count} | {drawer_count} |")
+    index_lines.append("")
+
+    index_path = os.path.join(output_dir, "index.md")
+    with open(index_path, "w", encoding="utf-8") as f:
+        f.write("\n".join(index_lines))
+
+    stats = {
+        "wings": len(wing_stats),
+        "rooms": sum(r for _, r, _ in index_rows),
+        "drawers": total_drawers,
+    }
+    print(
+        f"\n  Exported {stats['drawers']} drawers across {stats['wings']} wings, {stats['rooms']} rooms"
+    )
+    print(f"  Output: {output_dir}")
+    return stats
+
+
+def _quote_content(text: str) -> str:
+    """Format content for a markdown blockquote, handling multiline."""
+    lines = text.rstrip("\n").split("\n")
+    return "\n> ".join(lines)

mempalace/fact_checker.py

@@ -0,0 +1,335 @@
+"""
+fact_checker.py — Verify text against known facts in the palace.
+
+Checks AI responses, diary entries, and new content against the entity
+registry and knowledge graph for three classes of issue:
+
+  * similar_name          — text mentions a name that's one/two edits
+                            away from *another* registered name, raising
+                            the possibility of a typo or mix-up.
+  * relationship_mismatch — text asserts a role between two entities
+                            (e.g. "Bob is Alice's brother") while the KG
+                            records a *different* current role for the
+                            same subject/object pair.
+  * stale_fact            — text asserts a fact that the KG marks closed
+                            (``valid_to`` in the past).
+
+Purely offline. Inputs: entity_registry JSON + KG SQLite. No network.
+
+Usage:
+    from mempalace.fact_checker import check_text
+    issues = check_text("Bob is Alice's brother", palace_path)
+
+    # CLI
+    python -m mempalace.fact_checker "Bob is Alice's brother" \\
+        --palace ~/.mempalace/palace
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from datetime import datetime, timezone
+
+# Share miner's mtime-cached registry loader so we don't double-read
+# ~/.mempalace/known_entities.json on every check_text call.
+from .miner import _load_known_entities_raw
+
+
+# Narrow detection patterns — parse "X is Y's Z" and "X's Z is Y".
+# Names are captured greedily as word sequences (letters + optional
+# capitalized follow-ons) so simple multi-token names still work.
+# Relationship words are constrained to sane lengths to avoid matching
+# arbitrary filler.
+_RELATIONSHIP_PATTERNS = [
+    # "Bob is Alice's brother"      → subject=Bob, possessor=Alice, role=brother
+    re.compile(r"\b([A-Z][\w-]+)\s+is\s+([A-Z][\w-]+)'s\s+([a-z]{3,20})\b"),
+    # "Alice's brother is Bob"      → possessor=Alice, role=brother, subject=Bob
+    re.compile(r"\b([A-Z][\w-]+)'s\s+([a-z]{3,20})\s+is\s+([A-Z][\w-]+)\b"),
+]
+
+
+def check_text(text: str, palace_path: str = None, config=None) -> list:
+    """Return a list of issues detected in ``text``.
+
+    Empty list means "no contradictions found" — absence of evidence, not
+    evidence of absence. The detector is deliberately conservative:
+    every issue is anchored to a specific KG fact or registry entry.
+    """
+    if config is None:
+        from .config import MempalaceConfig
+
+        config = MempalaceConfig()
+    if palace_path is None:
+        palace_path = config.palace_path
+
+    if not text:
+        return []
+
+    issues: list = []
+    entity_names_raw = _load_known_entities_raw()
+
+    issues.extend(_check_entity_confusion(text, entity_names_raw))
+    issues.extend(_check_kg_contradictions(text, palace_path))
+
+    return issues
+
+
+# ── entity-name confusion ────────────────────────────────────────────
+
+
+def _flatten_names(entity_names_raw: dict) -> set:
+    """Flatten a ``{category: [names]}`` or ``{category: {name: meta}}``
+    registry into a set of names."""
+    flat: set = set()
+    for cat in entity_names_raw.values():
+        if isinstance(cat, list):
+            flat.update(str(n) for n in cat if n)
+        elif isinstance(cat, dict):
+            flat.update(str(k) for k in cat.keys() if k)
+    return flat
+
+
+def _check_entity_confusion(text: str, entity_names_raw: dict) -> list:
+    """Flag names mentioned in the text that are edit-distance ≤ 2 from
+    a *different* registered name — a common typo / mix-up pattern.
+
+    Performance note: the original O(n²) pairwise scan over the full
+    registry is gone. We first identify which names actually appear in
+    the text, then only compute edit distance between *mentioned* names
+    and the rest of the registry. This makes the cost O(m × n) where m
+    is the handful of names in the text, not the full registry.
+    """
+    all_names = _flatten_names(entity_names_raw)
+    if not all_names:
+        return []
+
+    # Which names from the registry actually appear in the text?
+    mentioned: list = []
+    for name in all_names:
+        if re.search(r"\b" + re.escape(name) + r"\b", text, re.IGNORECASE):
+            mentioned.append(name)
+    if not mentioned:
+        return []
+
+    issues: list = []
+    seen_pairs: set = set()
+    for name_a in mentioned:
+        a_lower = name_a.lower()
+        for name_b in all_names:
+            if name_b == name_a:
+                continue
+            # Dedupe by unordered pair so we don't double-report.
+            pair_key = tuple(sorted((name_a.lower(), name_b.lower())))
+            if pair_key in seen_pairs:
+                continue
+            # Only flag when name_b is a *different* registry entry that
+            # was NOT mentioned — otherwise both names in the text is
+            # just the user writing about two people.
+            if name_b in mentioned:
+                seen_pairs.add(pair_key)
+                continue
+            distance = _edit_distance(a_lower, name_b.lower())
+            if 0 < distance <= 2:
+                issues.append(
+                    {
+                        "type": "similar_name",
+                        "detail": (
+                            f"'{name_a}' mentioned — did you mean "
+                            f"'{name_b}'? (edit distance {distance})"
+                        ),
+                        "names": [name_a, name_b],
+                        "distance": distance,
+                    }
+                )
+                seen_pairs.add(pair_key)
+    return issues
+
+
+# ── KG contradictions ────────────────────────────────────────────────
+
+
+def _extract_claims(text: str) -> list:
+    """Yield structured (subject, predicate, object) claims from ``text``.
+
+    The two supported surface forms are "X is Y's Z" and "X's Z is Y",
+    both of which resolve to the triple ``(X, Z, Y)`` — ``X`` has role
+    ``Z`` with respect to ``Y``. Matches are case-preserving for the
+    entity names (KG lookup is case-insensitive on normalized IDs).
+    """
+    claims: list = []
+    for pat in _RELATIONSHIP_PATTERNS:
+        for match in pat.finditer(text):
+            groups = match.groups()
+            if pat is _RELATIONSHIP_PATTERNS[0]:
+                subject, possessor, role = groups[0], groups[1], groups[2]
+            else:
+                possessor, role, subject = groups[0], groups[1], groups[2]
+            claims.append(
+                {
+                    "subject": subject,
+                    "predicate": role.lower(),
+                    "object": possessor,
+                    "span": match.group(0),
+                }
+            )
+    return claims
+
+
+def _check_kg_contradictions(text: str, palace_path: str) -> list:
+    """Compare each claim in ``text`` against the KG.
+
+    For every claim ``(subject, predicate, object)`` parsed from the
+    text, look up the subject's current KG triples:
+
+      * ``relationship_mismatch`` fires when the KG records a fact about
+        the same ``(subject, object)`` pair but with a *different*
+        predicate — e.g. text says "brother" but KG says "husband".
+      * ``stale_fact`` fires when the KG has the exact ``(subject,
+        predicate, object)`` triple but its ``valid_to`` is in the past,
+        meaning the claim is no longer current.
+    """
+    claims = _extract_claims(text)
+    if not claims:
+        return []
+
+    try:
+        from .knowledge_graph import KnowledgeGraph
+
+        # KG lives alongside the palace collection; mcp_server uses the
+        # same convention (see _kg init). Pass ``db_path`` — the previous
+        # code passed a nonexistent ``palace_path`` kwarg which raised
+        # TypeError, silently swallowed by the outer except and rendered
+        # the entire KG-check path dead.
+        kg = KnowledgeGraph(db_path=os.path.join(palace_path, "knowledge_graph.sqlite3"))
+    except Exception:
+        # KG unavailable (brand-new palace, corrupted DB, etc.) — skip.
+        return []
+
+    issues: list = []
+    for claim in claims:
+        subject = claim["subject"]
+        claim_pred = claim["predicate"]
+        claim_obj = claim["object"]
+        try:
+            facts = kg.query_entity(subject, direction="outgoing")
+        except Exception:
+            continue
+        if not facts:
+            continue
+
+        current_facts = [f for f in facts if f.get("current")]
+
+        # Mismatch: KG fact about same (subject, object) pair but different predicate.
+        for fact in current_facts:
+            if not _objects_match(fact.get("object"), claim_obj):
+                continue
+            kg_pred = (fact.get("predicate") or "").lower()
+            if kg_pred and kg_pred != claim_pred:
+                issues.append(
+                    {
+                        "type": "relationship_mismatch",
+                        "detail": (
+                            f"Text says '{claim['span']}' but KG records "
+                            f"{subject} {kg_pred} {fact.get('object')}"
+                        ),
+                        "entity": subject,
+                        "claim": {
+                            "predicate": claim_pred,
+                            "object": claim_obj,
+                        },
+                        "kg_fact": {
+                            "predicate": kg_pred,
+                            "object": fact.get("object"),
+                        },
+                    }
+                )
+
+        # Stale fact: exact match on (subject, predicate, object) but KG
+        # closed the window in the past.
+        now_iso = datetime.now(timezone.utc).date().isoformat()
+        for fact in facts:
+            if fact.get("current"):
+                continue
+            kg_pred = (fact.get("predicate") or "").lower()
+            if kg_pred != claim_pred:
+                continue
+            if not _objects_match(fact.get("object"), claim_obj):
+                continue
+            valid_to = fact.get("valid_to")
+            if valid_to and str(valid_to) < now_iso:
+                issues.append(
+                    {
+                        "type": "stale_fact",
+                        "detail": (
+                            f"Text says '{claim['span']}' but KG marks "
+                            f"this fact closed on {valid_to}"
+                        ),
+                        "entity": subject,
+                        "valid_to": valid_to,
+                    }
+                )
+
+    return issues
+
+
+def _objects_match(kg_obj, claim_obj: str) -> bool:
+    if kg_obj is None or not claim_obj:
+        return False
+    return str(kg_obj).strip().lower() == claim_obj.strip().lower()
+
+
+# ── Levenshtein helper (tight iterative version) ─────────────────────
+
+
+def _edit_distance(s1: str, s2: str) -> int:
+    """Levenshtein distance. O(len(s1) * len(s2)) time, O(len(s2)) space."""
+    if len(s1) < len(s2):
+        s1, s2 = s2, s1
+    if not s2:
+        return len(s1)
+    prev = list(range(len(s2) + 1))
+    for i, c1 in enumerate(s1):
+        curr = [i + 1]
+        for j, c2 in enumerate(s2):
+            curr.append(
+                min(
+                    prev[j + 1] + 1,
+                    curr[j] + 1,
+                    prev[j] + (0 if c1 == c2 else 1),
+                )
+            )
+        prev = curr
+    return prev[-1]
+
+
+if __name__ == "__main__":
+    import argparse
+    import json
+    import sys
+
+    parser = argparse.ArgumentParser(
+        description="Check text against known facts in the MemPalace palace.",
+        epilog="Exits 0 when no issues found, 1 when one or more issues detected.",
+    )
+    parser.add_argument("text", nargs="?", help="Text to check (or use --stdin).")
+    parser.add_argument(
+        "--palace",
+        default=os.path.expanduser("~/.mempalace/palace"),
+        help="Path to the palace directory.",
+    )
+    parser.add_argument("--stdin", action="store_true", help="Read text from stdin.")
+    args = parser.parse_args()
+
+    if args.stdin:
+        text_in = sys.stdin.read()
+    elif args.text:
+        text_in = args.text
+    else:
+        parser.error("Provide text as argument or use --stdin.")
+
+    found = check_text(text_in, palace_path=args.palace)
+    if found:
+        print(json.dumps(found, indent=2))
+        sys.exit(1)
+    print("No contradictions found.")

mempalace/hooks_cli.py

@@ -0,0 +1,713 @@
+"""
+Hook logic for MemPalace — Python implementation of session-start, stop, and precompact hooks.
+
+Reads JSON from stdin, outputs JSON to stdout.
+Supported hooks: session-start, stop, precompact
+Supported harnesses: claude-code, codex (extensible to cursor, gemini, etc.)
+"""
+
+import json
+import os
+import re
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+SAVE_INTERVAL = 15
+STATE_DIR = Path.home() / ".mempalace" / "hook_state"
+
+
+def _mempalace_python() -> str:
+    """Return the python interpreter that has mempalace installed.
+
+    When hooks are invoked by Claude Code, sys.executable may be the system
+    python which lacks chromadb and other deps.  Resolution order:
+    1. MEMPALACE_PYTHON env var (explicit override)
+    2. Venv python from package install path
+    3. Editable install: venv/ sibling to mempalace/
+    4. sys.executable fallback
+    """
+    # Honor explicit override (used by shell hook wrappers)
+    env_python = os.environ.get("MEMPALACE_PYTHON", "")
+    if env_python and os.path.isfile(env_python) and os.access(env_python, os.X_OK):
+        return env_python
+    # This file lives at <venv>/lib/pythonX.Y/site-packages/mempalace/hooks_cli.py
+    # or <project>/mempalace/hooks_cli.py (editable install).
+    venv_bin = Path(__file__).resolve().parents[3] / "bin" / "python"
+    if venv_bin.is_file():
+        return str(venv_bin)
+    # Editable install: assumes project root has a venv/ sibling to mempalace/
+    project_venv = Path(__file__).resolve().parents[1] / "venv" / "bin" / "python"
+    if project_venv.is_file():
+        return str(project_venv)
+    return sys.executable
+
+
+_RECENT_MSG_COUNT = 30  # how many recent user messages to summarize
+
+STOP_BLOCK_REASON = (
+    "AUTO-SAVE checkpoint (MemPalace). Save this session's key content:\n"
+    "1. mempalace_diary_write — session summary (what was discussed, "
+    "key decisions, current state of work)\n"
+    "2. mempalace_add_drawer — verbatim quotes, decisions, code snippets "
+    "(place in appropriate wing and room)\n"
+    "3. mempalace_kg_add — entity relationships (optional)\n"
+    "For THIS save, use MemPalace MCP tools only (not auto-memory .md files). "
+    "Use verbatim quotes where possible. Continue conversation after saving."
+)
+
+PRECOMPACT_BLOCK_REASON = (
+    "COMPACTION IMMINENT (MemPalace). Save ALL session content before context is lost:\n"
+    "1. mempalace_diary_write — thorough session summary\n"
+    "2. mempalace_add_drawer — ALL verbatim quotes, decisions, code, context "
+    "(place each in appropriate wing and room)\n"
+    "3. mempalace_kg_add — entity relationships (optional)\n"
+    "For THIS save, use MemPalace MCP tools only (not auto-memory .md files). "
+    "Be thorough — after compaction this is all that survives. "
+    "Save everything to MemPalace, then allow compaction to proceed."
+)
+
+
+def _sanitize_session_id(session_id: str) -> str:
+    """Only allow alnum, dash, underscore to prevent path traversal."""
+    sanitized = re.sub(r"[^a-zA-Z0-9_-]", "", session_id)
+    return sanitized or "unknown"
+
+
+def _validate_transcript_path(transcript_path: str) -> Path:
+    """Validate and resolve a transcript path, rejecting paths outside expected roots.
+
+    Returns a resolved Path if valid, or None if the path should be rejected.
+    Accepted paths must:
+    - Have a .jsonl or .json extension
+    - Not contain '..' after resolution (path traversal prevention)
+    """
+    if not transcript_path:
+        return None
+    path = Path(transcript_path).expanduser().resolve()
+    if path.suffix not in (".jsonl", ".json"):
+        return None
+    # Reject if the original input contained '..' traversal components
+    if ".." in Path(transcript_path).parts:
+        return None
+    return path
+
+
+def _count_human_messages(transcript_path: str) -> int:
+    """Count human messages in a JSONL transcript, skipping command-messages."""
+    path = _validate_transcript_path(transcript_path)
+    if path is None:
+        if transcript_path:
+            _log(f"WARNING: transcript_path rejected by validator: {transcript_path!r}")
+        return 0
+    if not path.is_file():
+        return 0
+    count = 0
+    try:
+        with open(path, encoding="utf-8", errors="replace") as f:
+            for line in f:
+                try:
+                    entry = json.loads(line)
+                    msg = entry.get("message", {})
+                    if isinstance(msg, dict) and msg.get("role") == "user":
+                        content = msg.get("content", "")
+                        if isinstance(content, str):
+                            if "<command-message>" in content:
+                                continue
+                        elif isinstance(content, list):
+                            text = " ".join(
+                                b.get("text", "") for b in content if isinstance(b, dict)
+                            )
+                            if "<command-message>" in text:
+                                continue
+                        count += 1
+                    # Also handle Codex CLI transcript format
+                    # {"type": "event_msg", "payload": {"type": "user_message", "message": "..."}}
+                    elif entry.get("type") == "event_msg":
+                        payload = entry.get("payload", {})
+                        if isinstance(payload, dict) and payload.get("type") == "user_message":
+                            msg_text = payload.get("message", "")
+                            if isinstance(msg_text, str) and "<command-message>" not in msg_text:
+                                count += 1
+                except (json.JSONDecodeError, AttributeError):
+                    pass
+    except OSError:
+        return 0
+    return count
+
+
+_state_dir_initialized = False
+
+
+def _log(message: str):
+    """Append to hook state log file."""
+    global _state_dir_initialized
+    try:
+        if not _state_dir_initialized:
+            STATE_DIR.mkdir(parents=True, exist_ok=True)
+            try:
+                STATE_DIR.chmod(0o700)
+            except (OSError, NotImplementedError):
+                pass
+            _state_dir_initialized = True
+        log_path = STATE_DIR / "hook.log"
+        is_new = not log_path.exists()
+        timestamp = datetime.now().strftime("%H:%M:%S")
+        with open(log_path, "a") as f:
+            f.write(f"[{timestamp}] {message}\n")
+        if is_new:
+            try:
+                log_path.chmod(0o600)
+            except (OSError, NotImplementedError):
+                pass
+    except OSError:
+        pass
+
+
+def _output(data: dict):
+    """Print JSON to stdout without importing modules that may redirect streams.
+
+    If mempalace.mcp_server is already loaded, reuse its saved real stdout fd.
+    Otherwise, write directly to fd 1 so hook responses still go to stdout even
+    if sys.stdout has been redirected elsewhere.
+    """
+    payload = (json.dumps(data, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
+
+    real_stdout_fd: int | None = None
+    mcp_mod = sys.modules.get("mempalace.mcp_server") or sys.modules.get(
+        f"{__package__}.mcp_server" if __package__ else "mcp_server"
+    )
+    if mcp_mod is not None:
+        real_stdout_fd = getattr(mcp_mod, "_REAL_STDOUT_FD", None)
+
+    fd = real_stdout_fd if real_stdout_fd is not None else 1
+    offset = 0
+    try:
+        while offset < len(payload):
+            try:
+                offset += os.write(fd, payload[offset:])
+            except InterruptedError:
+                continue
+        return
+    except OSError:
+        pass
+
+    sys.stdout.buffer.write(payload)
+    sys.stdout.buffer.flush()
+
+
+def _get_mine_targets() -> list[tuple[str, str]]:
+    """Return the list of ``(dir, mode)`` targets for auto-ingest.
+
+    MEMPAL_DIR (when set and resolvable) contributes a ``"projects"``
+    target. Transcript ingestion is handled separately by
+    ``_ingest_transcript`` — emitting it here too would double-mine the
+    same JSONL into a different wing on every hook fire (#1231 review).
+
+    An empty list means no MEMPAL_DIR ingest should run.
+    """
+    targets: list[tuple[str, str]] = []
+    mempal_dir = os.environ.get("MEMPAL_DIR", "")
+    if mempal_dir:
+        resolved = Path(mempal_dir).expanduser().resolve()
+        if resolved.is_dir():
+            targets.append((str(resolved), "projects"))
+    return targets
+
+
+_MINE_PID_FILE = STATE_DIR / "mine.pid"
+
+
+def _pid_alive(pid: int) -> bool:
+    """Cross-platform existence check for a PID.
+
+    On POSIX, ``os.kill(pid, 0)`` is the well-known no-op existence probe.
+    On Windows, ``os.kill`` maps to ``TerminateProcess(handle, sig)`` and
+    would *terminate* the target process with exit code ``sig`` — using
+    it here would kill our own mine child (or worse, the caller itself).
+    Use ``OpenProcess`` + ``GetExitCodeProcess`` via ctypes instead.
+    """
+    if sys.platform == "win32":
+        import ctypes
+        from ctypes import wintypes
+
+        PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
+        STILL_ACTIVE = 259
+        kernel32 = ctypes.windll.kernel32
+        handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
+        if not handle:
+            return False
+        try:
+            code = wintypes.DWORD()
+            if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)):
+                return False
+            return code.value == STILL_ACTIVE
+        finally:
+            kernel32.CloseHandle(handle)
+    try:
+        os.kill(pid, 0)
+        return True
+    except (OSError, ValueError):
+        return False
+
+
+def _mine_already_running() -> bool:
+    """Return True if a background mine process from a previous hook fire is still alive."""
+    try:
+        pid = int(_MINE_PID_FILE.read_text().strip())
+    except (OSError, ValueError):
+        return False
+    return _pid_alive(pid)
+
+
+def _spawn_mine(cmd: list) -> None:
+    """Spawn a mine subprocess, write its PID to the lock file, log to hook.log."""
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    log_path = STATE_DIR / "hook.log"
+    with open(log_path, "a") as log_f:
+        proc = subprocess.Popen(cmd, stdout=log_f, stderr=log_f)
+    _MINE_PID_FILE.write_text(str(proc.pid))
+
+
+def _maybe_auto_ingest():
+    """Background-mine MEMPAL_DIR (project files) if set.
+
+    Transcript convos are ingested separately via ``_ingest_transcript``
+    in the hook handlers — this function does not handle them, to avoid
+    asymmetric interpreter handling and PID-file overwrite when both
+    targets fire from a single hook call (#1231 review).
+    """
+    targets = _get_mine_targets()
+    if not targets:
+        return
+    if _mine_already_running():
+        _log("Skipping auto-ingest: mine already running")
+        return
+    for mine_dir, mode in targets:
+        try:
+            _spawn_mine([_mempalace_python(), "-m", "mempalace", "mine", mine_dir, "--mode", mode])
+        except OSError:
+            pass
+
+
+def _mine_sync():
+    """Synchronously mine MEMPAL_DIR (precompact path).
+
+    Transcript convos are ingested separately via ``_ingest_transcript``
+    in ``hook_precompact`` — keeping them out of this function avoids
+    timeout stacking against the harness 30s ceiling (#1231 review).
+    """
+    targets = _get_mine_targets()
+    if not targets:
+        return
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    log_path = STATE_DIR / "hook.log"
+    for mine_dir, mode in targets:
+        try:
+            with open(log_path, "a") as log_f:
+                subprocess.run(
+                    [
+                        _mempalace_python(),
+                        "-m",
+                        "mempalace",
+                        "mine",
+                        mine_dir,
+                        "--mode",
+                        mode,
+                    ],
+                    stdout=log_f,
+                    stderr=log_f,
+                    timeout=60,
+                )
+        except (OSError, subprocess.TimeoutExpired):
+            pass
+
+
+def _desktop_toast(body: str, title: str = "MemPalace"):
+    """Send a desktop notification via notify-send. Fails silently."""
+    try:
+        subprocess.Popen(
+            ["notify-send", "--app-name=MemPalace", "--icon=brain", title, body],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+    except OSError:
+        pass
+
+
+def _extract_recent_messages(transcript_path: str, count: int = _RECENT_MSG_COUNT) -> list[str]:
+    """Extract the last N user messages from a JSONL transcript."""
+    path = Path(transcript_path).expanduser()
+    if not path.is_file():
+        return []
+    messages = []
+    try:
+        with open(path, encoding="utf-8", errors="replace") as f:
+            for line in f:
+                try:
+                    entry = json.loads(line)
+                    # Claude Code format
+                    msg = entry.get("message") or entry.get("event_message") or {}
+                    if isinstance(msg, dict) and msg.get("role") == "user":
+                        content = msg.get("content", "")
+                        if isinstance(content, list):
+                            content = " ".join(
+                                b.get("text", "") for b in content if isinstance(b, dict)
+                            )
+                        if not isinstance(content, str) or not content.strip():
+                            continue
+                        if "<command-message>" in content or "<system-reminder>" in content:
+                            continue
+                        messages.append(content.strip()[:200])
+                    # Codex CLI format
+                    elif entry.get("type") == "event_msg":
+                        payload = entry.get("payload", {})
+                        if isinstance(payload, dict) and payload.get("type") == "user_message":
+                            text = payload.get("message", "")
+                            if isinstance(text, str) and text.strip():
+                                if "<command-message>" not in text:
+                                    messages.append(text.strip()[:200])
+                except (json.JSONDecodeError, AttributeError):
+                    pass
+    except OSError:
+        return []
+    return messages[-count:]
+
+
+_THEME_STOPWORDS = frozenset(
+    "the a an and or but in on at to for of is it i me my you your we our "
+    "this that with from by was were be been are not no yes can do did dont "
+    "will would should could have has had lets let just also like so if then "
+    "ok okay sure yeah hey hi here there what when where how why which some "
+    "all any each every about into out up down over after before between "
+    "get got make made need want use used using check look see run try "
+    "know think right now still already really very much more most too "
+    "file files code one two new first last next thing things way well".split()
+)
+
+
+def _extract_themes(messages: list[str], max_themes: int = 3) -> list[str]:
+    """Pull 2-3 distinctive topic words from recent messages.
+
+    Note: stopword list is English-only; non-English corpora will produce noisy themes.
+    """
+    from collections import Counter
+
+    words: Counter[str] = Counter()
+    for msg in messages:
+        for word in msg.lower().split():
+            # Strip punctuation, keep words 4+ chars
+            clean = word.strip(".,;:!?\"'`()[]{}#<>/\\-_=+@$%^&*~")
+            if len(clean) >= 4 and clean not in _THEME_STOPWORDS and clean.isalpha():
+                words[clean] += 1
+    return [w for w, _ in words.most_common(max_themes)]
+
+
+def _save_diary_direct(
+    transcript_path: str,
+    session_id: str,
+    wing: str = "",
+    toast: bool = False,
+) -> dict:
+    """Write a diary checkpoint by calling the tool function directly (no MCP roundtrip).
+
+    If `wing` is set, the entry lands in that wing (typically the project wing
+    derived from the transcript path). Otherwise falls back to `tool_diary_write`'s
+    default of `wing_session-hook`.
+
+    Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure.
+    """
+    messages = _extract_recent_messages(transcript_path)
+    if not messages:
+        _log("No recent messages to save")
+        return {"count": 0}
+
+    themes = _extract_themes(messages)
+
+    # Build a compressed diary entry from recent conversation
+    now = datetime.now()
+    topics = "|".join(m[:80] for m in messages[-10:])
+    entry = (
+        f"CHECKPOINT:{now.strftime('%Y-%m-%d')}|session:{session_id}"
+        f"|msgs:{len(messages)}|recent:{topics}"
+    )
+
+    try:
+        from .mcp_server import tool_diary_write
+
+        result = tool_diary_write(
+            agent_name="session-hook",
+            entry=entry,
+            topic="checkpoint",
+            wing=wing,
+        )
+        if result.get("success"):
+            _log(f"Diary checkpoint saved: {result.get('entry_id', '?')}")
+            # Write state for ack tool to read
+            try:
+                ack_file = STATE_DIR / "last_checkpoint"
+                ack_file.write_text(
+                    json.dumps({"msgs": len(messages), "ts": now.isoformat()}),
+                    encoding="utf-8",
+                )
+            except OSError:
+                pass
+            if toast:
+                _desktop_toast(f"Checkpoint saved \u2014 {len(messages)} messages archived")
+            return {"count": len(messages), "themes": themes}
+        else:
+            _log(f"Diary checkpoint failed: {result.get('error', 'unknown')}")
+    except Exception as e:
+        _log(f"Diary checkpoint error: {e}")
+    return {"count": 0}
+
+
+def _ingest_transcript(transcript_path: str):
+    """Mine a Claude Code session transcript into the palace as a conversation."""
+    path = Path(transcript_path).expanduser()
+    if not path.is_file() or path.stat().st_size < 100:
+        return
+
+    from .config import MempalaceConfig
+
+    try:
+        MempalaceConfig()  # validate config loads
+    except Exception:
+        return
+
+    try:
+        log_path = STATE_DIR / "hook.log"
+        STATE_DIR.mkdir(parents=True, exist_ok=True)
+        with open(log_path, "a") as log_f:
+            subprocess.Popen(
+                [
+                    _mempalace_python(),
+                    "-m",
+                    "mempalace",
+                    "mine",
+                    str(path.parent),
+                    "--mode",
+                    "convos",
+                    "--wing",
+                    "sessions",
+                ],
+                stdout=log_f,
+                stderr=log_f,
+            )
+        _log(f"Transcript ingest started: {path.name}")
+    except OSError:
+        pass
+
+
+SUPPORTED_HARNESSES = {"claude-code", "codex"}
+
+
+def _parse_harness_input(data: dict, harness: str) -> dict:
+    """Parse stdin JSON according to the harness type."""
+    if harness not in SUPPORTED_HARNESSES:
+        print(f"Unknown harness: {harness}", file=sys.stderr)
+        sys.exit(1)
+    return {
+        "session_id": _sanitize_session_id(str(data.get("session_id", "unknown"))),
+        "stop_hook_active": data.get("stop_hook_active", False),
+        "transcript_path": str(data.get("transcript_path", "")),
+    }
+
+
+def _wing_from_transcript_path(transcript_path: str) -> str:
+    """Derive a project wing name from a Claude Code transcript path.
+
+    Claude Code encodes the project's source directory by replacing path
+    separators with dashes, producing folders like:
+        ~/.claude/projects/-home-<user>-Projects-<project>/session.jsonl
+        ~/.claude/projects/-home-<user>-dev-<parent>-<project>/session.jsonl
+        ~/.claude/projects/-Users-<user>-<folder>-<project>/session.jsonl
+
+    The project directory name is the final dash-separated token of the
+    encoded folder. Returns ``wing_<project>`` (lowercased, spaces → ``_``).
+    Falls back to ``wing_sessions`` if the path does not match a Claude Code
+    project-folder layout.
+    """
+    # Normalize path separators for cross-platform (Windows backslashes)
+    normalized = transcript_path.replace("\\", "/")
+    # Primary: pull the encoded project folder out of ``.claude/projects/``
+    # and take its last dash-separated token.
+    match = re.search(r"/\.claude/projects/-([^/]+)", normalized)
+    if match:
+        encoded = match.group(1)
+        project = encoded.rsplit("-", 1)[-1]
+        if project:
+            return f"wing_{project.lower().replace(' ', '_')}"
+    # Legacy fallback: explicit ``-Projects-<name>`` segment, useful for
+    # transcripts not under the standard Claude Code projects dir.
+    match = re.search(r"-Projects-([^/]+?)(?:/|$)", normalized)
+    if match:
+        project = match.group(1).lower().replace(" ", "_")
+        return f"wing_{project}"
+    return "wing_sessions"
+
+
+def hook_stop(data: dict, harness: str):
+    """Stop hook: block every N messages for auto-save."""
+    parsed = _parse_harness_input(data, harness)
+    session_id = parsed["session_id"]
+    stop_hook_active = parsed["stop_hook_active"]
+    transcript_path = parsed["transcript_path"]
+
+    # If already in a block-mode save cycle, let through (infinite-loop prevention).
+    # Silent mode saves directly without returning {"decision":"block"}, so there's
+    # no loop to prevent — and Claude Code's plugin dispatch sets this flag on every
+    # fire after the first, which would otherwise suppress all subsequent auto-saves.
+    if str(stop_hook_active).lower() in ("true", "1", "yes"):
+        # Safe default: assume silent mode on any config-read failure so saves
+        # proceed rather than being silently dropped. Silent mode is the default
+        # (v3.3.0+), so if we can't read config, behave as if it's still on.
+        silent_guard = True
+        try:
+            from .config import MempalaceConfig
+        except ImportError as exc:
+            _log(
+                f"WARNING: could not import MempalaceConfig for stop guard: {exc}; defaulting to silent mode"
+            )
+        else:
+            try:
+                silent_guard = MempalaceConfig().hook_silent_save
+            except AttributeError as exc:
+                _log(f"WARNING: could not read hook_silent_save: {exc}; defaulting to silent mode")
+        if not silent_guard:
+            _output({})
+            return
+
+    # Count human messages
+    exchange_count = _count_human_messages(transcript_path)
+
+    # Track last save point
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    last_save_file = STATE_DIR / f"{session_id}_last_save"
+    last_save = 0
+    if last_save_file.is_file():
+        try:
+            last_save = int(last_save_file.read_text().strip())
+        except (ValueError, OSError):
+            last_save = 0
+
+    since_last = exchange_count - last_save
+
+    _log(f"Session {session_id}: {exchange_count} exchanges, {since_last} since last save")
+
+    if since_last >= SAVE_INTERVAL and exchange_count > 0:
+        _log(f"TRIGGERING SAVE at exchange {exchange_count}")
+
+        # Read hook settings from config
+        from .config import MempalaceConfig
+
+        try:
+            config = MempalaceConfig()
+            silent = config.hook_silent_save
+            toast = config.hook_desktop_toast
+        except Exception:
+            silent = True
+            toast = False
+
+        project_wing = _wing_from_transcript_path(transcript_path)
+
+        if silent:
+            # Save directly via Python API — systemMessage renders in terminal
+            result = {"count": 0}
+            if transcript_path:
+                result = _save_diary_direct(
+                    transcript_path, session_id, wing=project_wing, toast=toast
+                )
+                _ingest_transcript(transcript_path)
+            _maybe_auto_ingest()
+            # Only advance save marker after successful save
+            count = result.get("count", 0)
+            if count > 0:
+                try:
+                    last_save_file.write_text(str(exchange_count), encoding="utf-8")
+                except OSError:
+                    pass
+                themes = result.get("themes", [])
+                if themes:
+                    tag = " \u2014 " + ", ".join(themes)
+                else:
+                    tag = ""
+                _output(
+                    {
+                        "systemMessage": f"\u2726 {count} memories woven into the palace{tag}",
+                    }
+                )
+            else:
+                _output({})
+        else:
+            # Legacy: block and ask Claude to save via MCP tools.
+            # Marker advances before confirmed save — best-effort; if Claude
+            # fails to save, the checkpoint is lost but won't retry endlessly.
+            try:
+                last_save_file.write_text(str(exchange_count), encoding="utf-8")
+            except OSError:
+                pass
+            if transcript_path:
+                _ingest_transcript(transcript_path)
+            _maybe_auto_ingest()
+            reason = STOP_BLOCK_REASON + f" Write diary entry to wing={project_wing}."
+            _output({"decision": "block", "reason": reason})
+    else:
+        _output({})
+
+
+def hook_session_start(data: dict, harness: str):
+    """Session start hook: initialize session tracking state."""
+    parsed = _parse_harness_input(data, harness)
+    session_id = parsed["session_id"]
+
+    _log(f"SESSION START for session {session_id}")
+
+    # Initialize session state directory
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Pass through — no blocking on session start
+    _output({})
+
+
+def hook_precompact(data: dict, harness: str):
+    """Precompact hook: mine transcript synchronously, then allow compaction."""
+    parsed = _parse_harness_input(data, harness)
+    session_id = parsed["session_id"]
+    transcript_path = parsed["transcript_path"]
+
+    _log(f"PRE-COMPACT triggered for session {session_id}")
+
+    # Capture tool output via our normalize path before compaction loses it
+    if transcript_path:
+        _ingest_transcript(transcript_path)
+
+    # Mine MEMPAL_DIR synchronously so project data lands before
+    # compaction proceeds. Transcript convos were already kicked off
+    # above via _ingest_transcript.
+    _mine_sync()
+
+    _output({})
+
+
+def run_hook(hook_name: str, harness: str):
+    """Main entry point: read stdin JSON, dispatch to hook handler."""
+    try:
+        data = json.load(sys.stdin)
+    except (json.JSONDecodeError, EOFError):
+        _log("WARNING: Failed to parse stdin JSON, proceeding with empty data")
+        data = {}
+
+    hooks = {
+        "session-start": hook_session_start,
+        "stop": hook_stop,
+        "precompact": hook_precompact,
+    }
+
+    handler = hooks.get(hook_name)
+    if handler is None:
+        print(f"Unknown hook: {hook_name}", file=sys.stderr)
+        sys.exit(1)
+
+    handler(data, harness)

mempalace/i18n/__init__.py

@@ -0,0 +1,285 @@
+"""i18n — Language dictionaries for MemPalace.
+
+Usage:
+    from mempalace.i18n import load_lang, t
+
+    load_lang("fr")           # load French
+    print(t("cli.mine_start", path="/docs"))  # "Extraction de /docs..."
+    print(t("terms.wing"))    # "aile"
+    print(t("aaak.instruction"))  # AAAK compression instruction in French
+
+Each locale JSON may include an ``entity`` section with patterns used by
+``mempalace.entity_detector``. See ``get_entity_patterns`` for the merge rules
+and the README section "Adding a new language" for the schema.
+"""
+
+import json
+from pathlib import Path
+from typing import Optional
+
+_LANG_DIR = Path(__file__).parent
+_strings: dict = {}
+_current_lang: str = "en"
+
+# Cache: tuple(langs) -> merged entity pattern dict
+_entity_cache: dict = {}
+
+
+def _canonical_lang(lang: str) -> Optional[str]:
+    """Resolve a language code to its on-disk canonical filename stem.
+
+    BCP 47 tags are case-insensitive (RFC 5646 §2.1.1), and the locale
+    files mix conventions (``pt-br.json`` vs ``zh-CN.json``). Match on
+    lowercase so callers can pass ``PT-BR``, ``zh-cn``, ``Pt-Br``, etc.
+    Returns ``None`` if no file matches.
+    """
+    if not lang:
+        return None
+    target = lang.strip().lower()
+    for path in _LANG_DIR.glob("*.json"):
+        if path.stem.lower() == target:
+            return path.stem
+    return None
+
+
+def available_languages() -> list[str]:
+    """Return list of available language codes."""
+    return sorted(p.stem for p in _LANG_DIR.glob("*.json"))
+
+
+def load_lang(lang: str = "en") -> dict:
+    """Load a language dictionary. Falls back to English if not found."""
+    global _strings, _current_lang
+    canonical = _canonical_lang(lang)
+    if canonical is None:
+        canonical = "en"
+    lang_file = _LANG_DIR / f"{canonical}.json"
+    _strings = json.loads(lang_file.read_text(encoding="utf-8"))
+    _current_lang = canonical
+    return _strings
+
+
+def t(key: str, **kwargs) -> str:
+    """Get a translated string by dotted key. Supports {var} interpolation.
+
+    t("cli.mine_complete", closets=5, drawers=20)
+    → "Done. 5 closets, 20 drawers created."
+    """
+    if not _strings:
+        load_lang("en")
+    parts = key.split(".", 1)
+    if len(parts) == 2:
+        section, name = parts
+        val = _strings.get(section, {}).get(name, key)
+    else:
+        val = _strings.get(key, key)
+    if kwargs and isinstance(val, str):
+        try:
+            val = val.format(**kwargs)
+        except (KeyError, IndexError):
+            pass
+    return val
+
+
+def current_lang() -> str:
+    """Return current language code."""
+    return _current_lang
+
+
+def get_regex() -> dict:
+    """Return the regex patterns for the current language.
+
+    Keys: topic_pattern, stop_words, quote_pattern, action_pattern.
+    Returns empty dict if no regex section in the language file.
+    """
+    if not _strings:
+        load_lang("en")
+    return _strings.get("regex", {})
+
+
+def _load_entity_section(lang: str) -> dict:
+    """Load the raw entity section for one language. Returns {} if missing."""
+    canonical = _canonical_lang(lang)
+    if canonical is None:
+        return {}
+    lang_file = _LANG_DIR / f"{canonical}.json"
+    try:
+        data = json.loads(lang_file.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return {}
+    return data.get("entity", {}) or {}
+
+
+def _script_boundary(chars: str) -> str:
+    """Build a lookaround-based word boundary expression.
+
+    Python's built-in ``\\b`` is a transition between ``\\w`` and non-``\\w``.
+    ``\\w`` covers Unicode Letter and Number categories but NOT Marks (category
+    Mc/Mn), so for scripts whose words contain combining vowel signs — Devanagari
+    (ा ी ु), Arabic (ـَ ـِ ـُ), Hebrew (ִ ֵ), Thai, Tamil, Burmese, Khmer — the
+    default ``\\b`` drops the trailing mark, truncating names like ``अनीता`` to
+    ``अनीत`` and failing to match ``\\bकहा\\b`` because the trailing matra is
+    not a word character.
+
+    Locales with such scripts declare ``boundary_chars`` in their entity section
+    (e.g. ``"\\\\w\\\\u0900-\\\\u097F"`` for Hindi). This function returns a
+    regex fragment equivalent to ``\\b`` but where the "word" side is defined
+    as any char matching ``[chars]`` rather than just ``\\w``.
+    """
+    return (
+        rf"(?:(?<=[{chars}])(?=[^{chars}])"
+        rf"|(?<=[^{chars}])(?=[{chars}])"
+        rf"|^(?=[{chars}])"
+        rf"|(?<=[{chars}])$)"
+    )
+
+
+def _expand_b(pattern: str, boundary_chars: str) -> str:
+    """Replace every literal ``\\b`` in ``pattern`` with a script-aware boundary.
+
+    ``boundary_chars`` is the inside-word character class (without brackets).
+    If it's falsy, the pattern is returned unchanged so ``\\b`` keeps its
+    default Python ``re`` semantics.
+    """
+    if not boundary_chars:
+        return pattern
+    return pattern.replace(r"\b", _script_boundary(boundary_chars))
+
+
+def _wrap_candidate(raw_pat: str, boundary_chars: str) -> str:
+    """Wrap a candidate/multi-word extraction pattern with a capture group
+    and word boundaries appropriate for its locale.
+
+    Default: ``\\b(raw)\\b``. With ``boundary_chars``: the script-aware
+    equivalent, so names ending in combining marks are matched in full.
+    """
+    if boundary_chars:
+        b = _script_boundary(boundary_chars)
+        return f"{b}({raw_pat}){b}"
+    return rf"\b({raw_pat})\b"
+
+
+def _collect_entity_section(section: dict, acc: dict) -> None:
+    """Merge one language's entity section into the running accumulator.
+
+    Handles boundary expansion in-place so the caller merges already-expanded
+    strings: `candidate_patterns` and `multi_word_patterns` are pre-wrapped
+    with the locale's boundary (capture group included, ready to compile);
+    every ``\\b`` inside person/pronoun/dialogue/project/direct patterns is
+    replaced with the locale's script-aware boundary.
+    """
+    boundary_chars = section.get("boundary_chars")
+    if section.get("candidate_pattern"):
+        acc["candidate_patterns"].append(
+            _wrap_candidate(section["candidate_pattern"], boundary_chars)
+        )
+    if section.get("multi_word_pattern"):
+        acc["multi_word_patterns"].append(
+            _wrap_candidate(section["multi_word_pattern"], boundary_chars)
+        )
+    if section.get("direct_address_pattern"):
+        acc["direct_address"].append(_expand_b(section["direct_address_pattern"], boundary_chars))
+    acc["person_verbs"].extend(
+        _expand_b(p, boundary_chars) for p in section.get("person_verb_patterns", [])
+    )
+    acc["pronouns"].extend(
+        _expand_b(p, boundary_chars) for p in section.get("pronoun_patterns", [])
+    )
+    acc["dialogue"].extend(
+        _expand_b(p, boundary_chars) for p in section.get("dialogue_patterns", [])
+    )
+    acc["project_verbs"].extend(
+        _expand_b(p, boundary_chars) for p in section.get("project_verb_patterns", [])
+    )
+    acc["stopwords"].update(w.lower() for w in section.get("stopwords", []))
+
+
+def get_entity_patterns(languages=("en",)) -> dict:
+    """Return merged entity detection patterns for the requested languages.
+
+    Entity detection patterns live under each locale's ``entity`` section.
+    This function merges them into a single dict for consumption by
+    ``mempalace.entity_detector``.
+
+    Merge rules:
+      - List fields (person_verb_patterns, pronoun_patterns, dialogue_patterns,
+        project_verb_patterns) are concatenated in the order of ``languages``,
+        with duplicates removed while preserving first occurrence.
+      - ``stopwords`` is the set union across all languages, returned as a
+        sorted list.
+      - ``candidate_patterns`` and ``multi_word_patterns`` are returned as
+        **fully-wrapped regex strings** (boundary + capture group applied);
+        the consumer compiles them directly with no further wrapping.
+      - ``direct_address_pattern`` is returned as a list of per-language
+        alternation patterns (not concatenated — each is applied separately).
+
+    Locales with combining-mark scripts can declare ``boundary_chars`` in
+    their entity section (e.g. ``"\\\\w\\\\u0900-\\\\u097F"`` for Hindi);
+    every ``\\b`` inside that locale's patterns — plus the candidate/multi-
+    word wrapping — is expanded to a script-aware lookaround boundary that
+    treats the declared characters as "inside-word".
+
+    If ``languages`` is empty or no requested language declares entity data,
+    English is used as a fallback so callers always get a working config.
+    """
+    if not languages:
+        languages = ("en",)
+    # Normalize via canonical filename so callers using different casing
+    # (e.g. "PT-BR" vs "pt-br") share the same cache entry and load the
+    # same locale file. Unknown codes are kept as-is so the merge loop's
+    # "found_any" branch fires the English fallback exactly once.
+    languages = tuple(_canonical_lang(lang) or lang for lang in languages)
+    key = languages
+    if key in _entity_cache:
+        return _entity_cache[key]
+
+    acc = {
+        "candidate_patterns": [],
+        "multi_word_patterns": [],
+        "person_verbs": [],
+        "pronouns": [],
+        "dialogue": [],
+        "direct_address": [],
+        "project_verbs": [],
+        "stopwords": set(),
+    }
+
+    found_any = False
+    for lang in languages:
+        section = _load_entity_section(lang)
+        if not section:
+            continue
+        found_any = True
+        _collect_entity_section(section, acc)
+
+    if not found_any:
+        # Fallback: load English directly so callers always get a working config.
+        _collect_entity_section(_load_entity_section("en"), acc)
+
+    merged = {
+        "candidate_patterns": acc["candidate_patterns"],
+        "multi_word_patterns": acc["multi_word_patterns"],
+        "person_verb_patterns": _dedupe(acc["person_verbs"]),
+        "pronoun_patterns": _dedupe(acc["pronouns"]),
+        "dialogue_patterns": _dedupe(acc["dialogue"]),
+        "direct_address_patterns": acc["direct_address"],
+        "project_verb_patterns": _dedupe(acc["project_verbs"]),
+        "stopwords": sorted(acc["stopwords"]),
+    }
+    _entity_cache[key] = merged
+    return merged
+
+
+def _dedupe(items: list) -> list:
+    """Remove duplicates while preserving first-occurrence order."""
+    seen = set()
+    out = []
+    for item in items:
+        if item not in seen:
+            seen.add(item)
+            out.append(item)
+    return out
+
+
+# Auto-load English on import
+load_lang("en")

mempalace/i18n/be.json

@@ -0,0 +1,166 @@
+{
+  "lang": "be",
+  "label": "Беларуская",
+  "terms": {
+    "palace": "палац",
+    "wing": "крыло",
+    "hall": "зала",
+    "closet": "шафа",
+    "drawer": "шуфляда",
+    "mine": "майнінг",
+    "search": "пошук",
+    "status": "статус",
+    "init": "ініцыялізацыя",
+    "repair": "аднаўленне",
+    "migrate": "міграцыя",
+    "entity": "аб'ект",
+    "topic": "тэма"
+  },
+  "cli": {
+    "mine_start": "Майнінг {path}...",
+    "mine_complete": "Гатова. Створана шаф: {closets}, шуфляд: {drawers}.",
+    "mine_skip": "Майнінг скончаны. Выкарыстайце --force, каб перамайніць.",
+    "search_no_results": "Няма вынікаў для: {query}",
+    "search_results": "Знойдзена {count} вынікаў:",
+    "status_palace": "Палац: {path}",
+    "status_wings": "Крылаў: {count}",
+    "status_closets": "Шаф: {count}",
+    "status_drawers": "Шуфляд: {count}",
+    "init_complete": "Палац ініцыялізаваны ў {path}",
+    "init_exists": "Палац ужо існуе ў {path}",
+    "repair_complete": "Аднаўленне скончана. Выпраўлена праблем: {fixed}.",
+    "migrate_complete": "Міграцыя завершана.",
+    "no_palace": "Палац не знойдзены. Запусціце: mempalace init <дырэкторыя>"
+  },
+  "aaak": {
+    "instruction": "Сцісні да фармату індэкса. Злучкі паміж словамі, вертыкальныя рысы паміж паняткамі. Выдалі падставы і службовыя словы. Захавай дакладнасць імёнаў, нумароў, скланенняў і формаў."
+  },
+  "regex": {
+    "topic_pattern": "[А-ЯЁІЎ][а-яёіў]{2,}|[A-Z][a-z]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "і а але бо каб калі калісьці хоць што хто дзе куды адкуль як чаму таму ж не ні так таксама яшчэ ўжо цяпер потым тут там адсюль туды ў на па з са без для над пад пры пра праз супраць замест акрамя сярод вакол уздоўж каля паміж пасля перад да ад у або альбо то ці быццам нібыта вось гэты гэта гэтыя той тая тыя некаторыя кожны кожная кожныя толькі вельмі можа трэба будзе быў была былі ёсць няма",
+    "quote_pattern": "«\\s*([^»]{10,200})\\s*»|\"([^\"]{10,200})\"",
+    "action_pattern": "(?:зрабіў|зрабіла|зрабілі|стварыў|стварыла|стварылі|дадаў|дадала|дадалі|абнавіў|абнавіла|абнавілі|наладзіў|наладзіла|наладзілі|пратэставаў|пратэставала|пратэставалі)\\s+[\\wа-яёА-ЯЁІіЎў\\s]{3,30}"
+  },
+  "entity": {
+    "candidate_pattern": "[А-ЯЁІЎ][а-яёіў]{1,19}",
+    "multi_word_pattern": "[А-ЯЁІЎ][а-яёіў]+(?:\\s+[А-ЯЁІЎ][а-яёіў]+)+",
+    "person_verb_patterns": [
+      "\\b{name}\\s+сказа(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+спыта(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+адказа(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+расказа(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+засмея(?:ўся|лася|ліся)\\b",
+      "\\b{name}\\s+усміхну(?:ўся|лася|ліся)\\b",
+      "\\b{name}\\s+заплака(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+адчу(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+думае\\b",
+      "\\b{name}\\s+хоча\\b",
+      "\\b{name}\\s+кахае\\b",
+      "\\b{name}\\s+ненавідзіць\\b",
+      "\\b{name}\\s+ведае\\b",
+      "\\b{name}\\s+вырашы(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+напіса(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+каза(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+зрабі(?:ў|ла|лі)\\b",
+      "\\b{name}\\s+спытваецца\\b",
+      "\\b{name}\\s+адказвае\\b"
+    ],
+    "pronoun_patterns": [
+      "\\bён\\b",
+      "\\bяго\\b",
+      "\\bяму\\b",
+      "\\bім\\b",
+      "\\bяна\\b",
+      "\\bяе\\b",
+      "\\bёй\\b",
+      "\\bёю\\b",
+      "\\bяны\\b",
+      "\\bіх\\b",
+      "\\bімі\\b",
+      "\\bяно\\b",
+      "\\bвы\\b",
+      "\\bвас\\b",
+      "\\bвам\\b",
+      "\\bвамі\\b"
+    ],
+    "dialogue_patterns": [
+      "^>\\s*{name}[:\\s]",
+      "^{name}:\\s",
+      "^\\[{name}\\]",
+      "\"{name}\\s+сказа(?:ў|ла|лі)"
+    ],
+    "direct_address_pattern": "\\bпрывітанне[,:!.]?\\s+{name}\\b|\\bвітаю[,:!.]?\\s+{name}\\b|\\bдзякуй[,:!.]?\\s+{name}\\b|\\bдарагі[,:!.]?\\s+{name}\\b|\\bдарагая[,:!.]?\\s+{name}\\b|\\bпаважаны[,:!.]?\\s+{name}\\b|\\bпаважаная[,:!.]?\\s+{name}\\b",
+    "project_verb_patterns": [
+      "\\bзбіраю\\s+{name}\\b",
+      "\\bсабраў\\s+{name}\\b",
+      "\\bзапускаю\\s+{name}\\b",
+      "\\bзапусціў\\s+{name}\\b",
+      "\\bразгарнуў\\s+{name}\\b",
+      "\\bусталяваў\\s+{name}\\b",
+      "\\bсістэма\\s+{name}\\b",
+      "\\bпраект\\s+{name}\\b",
+      "\\bimport\\s+{name}\\b",
+      "\\bpip\\s+install\\s+{name}\\b"
+    ],
+    "stopwords": [
+      "і",
+      "ў",
+      "з",
+      "са",
+      "на",
+      "па",
+      "да",
+      "ад",
+      "у",
+      "без",
+      "для",
+      "над",
+      "пад",
+      "пры",
+      "пра",
+      "праз",
+      "супраць",
+      "замест",
+      "акрамя",
+      "сярод",
+      "вакол",
+      "уздоўж",
+      "каля",
+      "паміж",
+      "пасля",
+      "перад",
+      "або",
+      "альбо",
+      "каб",
+      "калі",
+      "хоць",
+      "бо",
+      "ж",
+      "не",
+      "ні",
+      "так",
+      "яшчэ",
+      "ужо",
+      "цяпер",
+      "вось",
+      "гэта",
+      "тое",
+      "тут",
+      "там",
+      "вельмі",
+      "толькі",
+      "можа",
+      "трэба",
+      "ёсць",
+      "няма",
+      "як",
+      "што",
+      "хто",
+      "чаму",
+      "таму",
+      "прывітанне",
+      "дзякуй",
+      "ласка"
+    ]
+  }
+}

mempalace/i18n/de.json

@@ -0,0 +1,126 @@
+{
+  "lang": "de",
+  "label": "Deutsch",
+  "terms": {
+    "palace": "Palast",
+    "wing": "Flügel",
+    "hall": "Flur",
+    "closet": "Schrank",
+    "drawer": "Schublade",
+    "mine": "schürfen",
+    "search": "suchen",
+    "status": "Status",
+    "init": "initialisieren",
+    "repair": "reparieren",
+    "migrate": "migrieren",
+    "entity": "Entität",
+    "topic": "Thema"
+  },
+  "cli": {
+    "mine_start": "Schürfe {path}...",
+    "mine_complete": "Fertig. {closets} Schränke, {drawers} Schubladen erstellt.",
+    "mine_skip": "Bereits geschürft. Verwenden Sie --force zum Wiederholen.",
+    "search_no_results": "Keine Ergebnisse für: {query}",
+    "search_results": "{count} Ergebnisse gefunden:",
+    "status_palace": "Palast: {path}",
+    "status_wings": "{count} Flügel",
+    "status_closets": "{count} Schränke",
+    "status_drawers": "{count} Schubladen",
+    "init_complete": "Palast initialisiert in {path}",
+    "init_exists": "Palast existiert bereits in {path}",
+    "repair_complete": "Reparatur abgeschlossen. {fixed} Probleme behoben.",
+    "migrate_complete": "Migration abgeschlossen.",
+    "no_palace": "Kein Palast gefunden. Ausführen: mempalace init <Ordner>"
+  },
+  "aaak": {
+    "instruction": "Auf Deutsch komprimieren. Bindestriche zwischen Wörtern, Pipes zwischen Konzepten. Artikel und Füllwörter weglassen. Eigennamen und Zahlen exakt beibehalten."
+  },
+  "regex": {
+    "topic_pattern": "[A-ZÄÖÜß][a-zäöüß]{2,}|[A-Za-zÄÖÜäöüß]{3,}",
+    "stop_words": "der die das ein eine eines einer einem einen den dem des und oder aber denn weil wenn als ob auch noch schon sehr viel nur nicht mehr kann wird hat ist sind war waren sein haben wurde mit von zu für auf in an um über nach durch",
+    "quote_pattern": "\\u201E([^\\u201C]{10,200})\\u201C|\"([^\"]{10,200})\"",
+    "action_pattern": "(?:gebaut|behoben|geschrieben|hinzugefügt|gepusht|gemessen|getestet|überprüft|erstellt|gelöscht|aktualisiert|konfiguriert|bereitgestellt|migriert)\\s+[\\wÄÖÜäöüß\\s]{3,30}"
+  },
+  "entity": {
+    "candidate_pattern": "[A-ZÄÖÜ][a-zäöüß]{1,19}",
+    "multi_word_pattern": "[A-ZÄÖÜ][a-zäöüß]+(?:\\s+[A-ZÄÖÜ][a-zäöüß]+)+",
+    "person_verb_patterns": [
+      "\\b{name}\\s+sagte\\b",
+      "\\b{name}\\s+fragte\\b",
+      "\\b{name}\\s+antwortete\\b",
+      "\\b{name}\\s+erzählte\\b",
+      "\\b{name}\\s+lachte\\b",
+      "\\b{name}\\s+lächelte\\b",
+      "\\b{name}\\s+weinte\\b",
+      "\\b{name}\\s+fühlte\\b",
+      "\\b{name}\\s+denkt\\b",
+      "\\b{name}\\s+will\\b",
+      "\\b{name}\\s+liebt\\b",
+      "\\b{name}\\s+hasst\\b",
+      "\\b{name}\\s+weiß\\b",
+      "\\b{name}\\s+entschied\\b",
+      "\\b{name}\\s+schrieb\\b"
+    ],
+    "pronoun_patterns": [
+      "\\ber\\b",
+      "\\bsie\\b",
+      "\\bes\\b",
+      "\\bihn\\b",
+      "\\bihm\\b",
+      "\\bihr\\b",
+      "\\bsein\\b",
+      "\\bihre\\b",
+      "\\bihnen\\b"
+    ],
+    "dialogue_patterns": [
+      "^>\\s*{name}[:\\s]",
+      "^{name}:\\s",
+      "^\\[{name}\\]",
+      "\"{name}\\s+sagte"
+    ],
+    "direct_address_pattern": "\\bhallo\\s+{name}\\b|\\bhi\\s+{name}\\b|\\bhey\\s+{name}\\b|\\bdanke\\s+{name}\\b|\\bservus\\s+{name}\\b|\\blieber\\s+{name}\\b|\\bliebe\\s+{name}\\b|\\bsehr\\s+geehrter\\s+{name}\\b|\\bsehr\\s+geehrte\\s+{name}\\b",
+    "project_verb_patterns": [
+      "\\bbaue\\s+{name}\\b",
+      "\\bgebaut\\s+{name}\\b",
+      "\\bstarte\\s+{name}\\b",
+      "\\bgestartet\\s+{name}\\b",
+      "\\bdeploye\\s+{name}\\b",
+      "\\binstalliert\\s+{name}\\b",
+      "\\bdie\\s+{name}\\s+architektur\\b",
+      "\\bdie\\s+{name}\\s+pipeline\\b",
+      "\\bdas\\s+{name}\\s+system\\b",
+      "\\bdas\\s+{name}\\s+repository\\b",
+      "\\b{name}\\s+v\\d+\\b",
+      "\\b{name}\\.py\\b",
+      "\\bimport\\s+{name}\\b",
+      "\\bpip\\s+install\\s+{name}\\b"
+    ],
+    "stopwords": [
+      "der", "die", "das", "ein", "eine", "eines", "einer", "einem", "einen",
+      "den", "dem", "des", "und", "oder", "aber", "denn", "weil", "wenn", "als",
+      "ob", "auch", "noch", "schon", "sehr", "viel", "nur", "nicht", "mehr",
+      "kann", "wird", "hat", "ist", "sind", "war", "waren", "sein", "haben",
+      "wurde", "worden", "werden", "mit", "von", "zu", "für", "auf", "in",
+      "an", "um", "über", "nach", "durch", "bei", "aus", "seit", "vor", "zwischen",
+      "ich", "du", "er", "sie", "es", "wir", "ihr", "mich", "dich", "mir", "dir",
+      "uns", "euch", "mein", "dein", "unser", "euer", "ihre", "seine",
+      "wer", "was", "wann", "wo", "wie", "warum", "welcher", "welche", "welches",
+      "so", "dann", "jetzt", "heute", "gestern", "morgen", "hier", "dort", "da",
+      "immer", "nie", "manchmal", "oft", "selten", "bald", "spät",
+      "ja", "nein", "vielleicht", "gut", "schlecht", "besser", "bitte", "danke",
+      "hallo", "hi", "hey", "tschüss",
+      "tag", "tage", "woche", "monat", "jahr", "jahre", "zeit", "welt", "leben",
+      "mensch", "menschen", "leute", "person", "ding", "dinge", "sache", "sachen",
+      "teil", "art", "weise", "stelle", "platz", "ort", "zimmer", "haus", "land",
+      "grund", "frage", "antwort", "fakt", "sinn", "idee", "punkt", "fall", "aspekt",
+      "beispiel", "version", "nummer", "zahl", "name", "namen", "system", "modell",
+      "sprache", "technologie", "gesellschaft", "kultur", "geschichte",
+      "wissenschaft", "zukunft", "erinnerung", "gedächtnis",
+      "datei", "ordner", "pfad", "schlüssel", "wert", "fehler", "warnung",
+      "ergebnis", "eingabe", "ausgabe", "quelle", "ziel", "daten", "elemente",
+      "montag", "dienstag", "mittwoch", "donnerstag", "freitag", "samstag", "sonntag",
+      "januar", "februar", "märz", "april", "mai", "juni", "juli", "august",
+      "september", "oktober", "november", "dezember"
+    ]
+  }
+}

mempalace/i18n/en.json

@@ -0,0 +1,156 @@
+{
+  "lang": "en",
+  "label": "English",
+  "terms": {
+    "palace": "palace",
+    "wing": "wing",
+    "hall": "hall",
+    "closet": "closet",
+    "drawer": "drawer",
+    "mine": "mine",
+    "search": "search",
+    "status": "status",
+    "init": "init",
+    "repair": "repair",
+    "migrate": "migrate",
+    "entity": "entity",
+    "topic": "topic"
+  },
+  "cli": {
+    "mine_start": "Mining {path}...",
+    "mine_complete": "Done. {closets} closets, {drawers} drawers created.",
+    "mine_skip": "Already mined. Use --force to re-mine.",
+    "search_no_results": "No results for: {query}",
+    "search_results": "Found {count} results:",
+    "status_palace": "Palace: {path}",
+    "status_wings": "{count} wings",
+    "status_closets": "{count} closets",
+    "status_drawers": "{count} drawers",
+    "init_complete": "Palace initialized at {path}",
+    "init_exists": "Palace already exists at {path}",
+    "repair_complete": "Repair complete. {fixed} issues fixed.",
+    "migrate_complete": "Migration complete.",
+    "no_palace": "No palace found. Run: mempalace init <dir>"
+  },
+  "aaak": {
+    "instruction": "Compress to index format. Hyphens between words, pipes between concepts. Drop articles and filler. Keep names and numbers exact."
+  },
+  "regex": {
+    "topic_pattern": "[A-Z][a-z]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "the this that these those some many most each every other only such very will would could should must shall yeah okay also even then now already still back done make take give know think want need going come find work added saved session summary conversation topics source about once just really actually here there where good great better thank please sorry right wrong true false",
+    "quote_pattern": "\"([^\"]{20,200})\"",
+    "action_pattern": "(?:built|fixed|wrote|added|pushed|measured|tested|reviewed|created|deleted|updated|configured|deployed|migrated)\\s+[\\w\\s]{3,30}"
+  },
+  "entity": {
+    "candidate_pattern": "[A-Z][a-z]+(?:[A-Z][a-z]+|[A-Z]{2,})+|[A-Z][a-z]{1,19}",
+    "multi_word_pattern": "[A-Z][a-z]+(?:\\s+[A-Z][a-z]+)+",
+    "person_verb_patterns": [
+      "\\b{name}\\s+said\\b",
+      "\\b{name}\\s+asked\\b",
+      "\\b{name}\\s+told\\b",
+      "\\b{name}\\s+replied\\b",
+      "\\b{name}\\s+laughed\\b",
+      "\\b{name}\\s+smiled\\b",
+      "\\b{name}\\s+cried\\b",
+      "\\b{name}\\s+felt\\b",
+      "\\b{name}\\s+thinks?\\b",
+      "\\b{name}\\s+wants?\\b",
+      "\\b{name}\\s+loves?\\b",
+      "\\b{name}\\s+hates?\\b",
+      "\\b{name}\\s+knows?\\b",
+      "\\b{name}\\s+decided\\b",
+      "\\b{name}\\s+pushed\\b",
+      "\\b{name}\\s+wrote\\b",
+      "\\bhey\\s+{name}\\b",
+      "\\bthanks?\\s+{name}\\b",
+      "\\bhi\\s+{name}\\b",
+      "\\bdear\\s+{name}\\b"
+    ],
+    "pronoun_patterns": [
+      "\\bshe\\b",
+      "\\bher\\b",
+      "\\bhers\\b",
+      "\\bhe\\b",
+      "\\bhim\\b",
+      "\\bhis\\b",
+      "\\bthey\\b",
+      "\\bthem\\b",
+      "\\btheir\\b"
+    ],
+    "dialogue_patterns": [
+      "^>\\s*{name}[:\\s]",
+      "^{name}:\\s",
+      "^\\[{name}\\]",
+      "\"{name}\\s+said"
+    ],
+    "direct_address_pattern": "\\bhey\\s+{name}\\b|\\bthanks?\\s+{name}\\b|\\bhi\\s+{name}\\b",
+    "project_verb_patterns": [
+      "\\bbuilding\\s+{name}\\b",
+      "\\bbuilt\\s+{name}\\b",
+      "\\bship(?:ping|ped)?\\s+{name}\\b",
+      "\\blaunch(?:ing|ed)?\\s+{name}\\b",
+      "\\bdeploy(?:ing|ed)?\\s+{name}\\b",
+      "\\binstall(?:ing|ed)?\\s+{name}\\b",
+      "\\bthe\\s+{name}\\s+architecture\\b",
+      "\\bthe\\s+{name}\\s+pipeline\\b",
+      "\\bthe\\s+{name}\\s+system\\b",
+      "\\bthe\\s+{name}\\s+repo\\b",
+      "\\b{name}\\s+v\\d+\\b",
+      "\\b{name}\\.py\\b",
+      "\\b{name}-core\\b",
+      "\\b{name}-local\\b",
+      "\\bimport\\s+{name}\\b",
+      "\\bpip\\s+install\\s+{name}\\b"
+    ],
+    "stopwords": [
+      "the", "a", "an", "and", "or", "but", "in", "on", "at", "to",
+      "for", "of", "with", "by", "from", "as", "is", "was", "are", "were",
+      "be", "been", "being", "have", "has", "had", "do", "does", "did",
+      "will", "would", "could", "should", "may", "might", "must", "shall", "can",
+      "this", "that", "these", "those", "it", "its", "they", "them", "their",
+      "we", "our", "you", "your", "i", "my", "me", "he", "she", "his", "her",
+      "who", "what", "when", "where", "why", "how", "which",
+      "if", "then", "so", "not", "no", "yes", "ok", "okay",
+      "just", "very", "really", "also", "already", "still", "even", "only",
+      "here", "there", "now", "too", "up", "out", "about", "like",
+      "use", "get", "got", "make", "made", "take", "put", "come", "go", "see",
+      "know", "think", "true", "false", "none", "null", "new", "old", "all", "any", "some",
+      "return", "print", "def", "class", "import",
+      "step", "usage", "run", "check", "find", "add", "set", "list",
+      "args", "dict", "str", "int", "bool", "path", "file", "type", "name",
+      "note", "example", "option", "result", "error", "warning", "info",
+      "every", "each", "more", "less", "next", "last", "first", "second",
+      "stack", "layer", "mode", "test", "stop", "start", "copy", "move",
+      "source", "target", "output", "input", "data", "item", "key", "value",
+      "returns", "raises", "yields", "self", "cls", "kwargs",
+      "world", "well", "want", "topic", "choose", "social", "cars", "phones",
+      "healthcare", "ex", "machina", "deus", "human", "humans", "people",
+      "things", "something", "nothing", "everything", "anything", "someone",
+      "everyone", "anyone", "way", "time", "day", "life", "place", "thing",
+      "part", "kind", "sort", "case", "point", "idea", "fact", "sense",
+      "question", "answer", "reason", "number", "version", "system",
+      "hey", "hi", "hello", "thanks", "thank", "right", "let",
+      "click", "hit", "press", "tap", "drag", "drop", "open", "close",
+      "save", "load", "launch", "install", "download", "upload", "scroll",
+      "select", "enter", "submit", "cancel", "confirm", "delete", "paste",
+      "write", "read", "search", "show", "hide",
+      "desktop", "documents", "downloads", "users", "home", "library",
+      "applications", "preferences", "settings", "terminal",
+      "actor", "vector", "remote", "control", "duration", "fetch",
+      "agents", "tools", "others", "guards", "ethics", "regulation",
+      "learning", "thinking", "memory", "language", "intelligence",
+      "technology", "society", "culture", "future", "history", "science",
+      "model", "models", "network", "networks", "training", "inference",
+      "created", "updated", "deleted", "added", "removed", "modified",
+      "extracted", "processed", "generated", "compiled", "launched", "installed",
+      "deployed", "executed", "loaded", "parsed", "validated", "configured",
+      "total", "summary", "covered", "included", "pending", "failed", "success",
+      "ready", "active", "disabled", "enabled", "available", "completed",
+      "auto", "multi", "mini", "micro", "meta", "super", "hybrid",
+      "context", "bridge", "batch", "local", "global", "native", "cloud",
+      "before", "after", "during", "often", "always", "never",
+      "project", "contributor", "software",
+      "backend", "frontend", "server", "client", "service", "app", "api"
+    ]
+  }
+}

mempalace/i18n/es.json

@@ -0,0 +1,128 @@
+{
+  "lang": "es",
+  "label": "Español",
+  "terms": {
+    "palace": "palacio",
+    "wing": "ala",
+    "hall": "pasillo",
+    "closet": "armario",
+    "drawer": "cajón",
+    "mine": "extraer",
+    "search": "buscar",
+    "status": "estado",
+    "init": "inicializar",
+    "repair": "reparar",
+    "migrate": "migrar",
+    "entity": "entidad",
+    "topic": "tema"
+  },
+  "cli": {
+    "mine_start": "Extrayendo {path}...",
+    "mine_complete": "Listo. {closets} armarios, {drawers} cajones creados.",
+    "mine_skip": "Ya extraído. Use --force para repetir.",
+    "search_no_results": "Sin resultados para: {query}",
+    "search_results": "{count} resultados encontrados:",
+    "status_palace": "Palacio: {path}",
+    "status_wings": "{count} alas",
+    "status_closets": "{count} armarios",
+    "status_drawers": "{count} cajones",
+    "init_complete": "Palacio inicializado en {path}",
+    "init_exists": "Ya existe un palacio en {path}",
+    "repair_complete": "Reparación completa. {fixed} problemas corregidos.",
+    "migrate_complete": "Migración completa.",
+    "no_palace": "No se encontró palacio. Ejecute: mempalace init <carpeta>"
+  },
+  "aaak": {
+    "instruction": "Comprima en español. Guiones entre palabras, pipes entre conceptos. Elimine artículos y palabras de relleno. Mantenga nombres propios y números exactos."
+  },
+  "regex": {
+    "topic_pattern": "[A-ZÁ-Ú][a-zá-ú]{2,}|[A-Za-zÁ-ú]{3,}",
+    "stop_words": "el la los las un una unos unas de del al en con por para su sus mi mis tu tus es son está están fue ser estar haber sido como pero más muy también todo todos toda todas este esta estos estas ese esa esos esas que quien cual donde cuando porque aunque sin",
+    "quote_pattern": "\"([^\"]{10,200})\"|«([^»]{10,200})»",
+    "action_pattern": "(?:construido|corregido|escrito|añadido|enviado|medido|probado|revisado|creado|eliminado|actualizado|configurado|desplegado|migrado)\\s+[\\wá-ú\\s]{3,30}"
+  },
+  "entity": {
+    "candidate_pattern": "[A-ZÁÉÍÓÚÑÜ][a-záéíóúñü]{1,19}",
+    "multi_word_pattern": "[A-ZÁÉÍÓÚÑÜ][a-záéíóúñü]+(?:\\s+[A-ZÁÉÍÓÚÑÜ][a-záéíóúñü]+)+",
+    "person_verb_patterns": [
+      "\\b{name}\\s+dijo\\b",
+      "\\b{name}\\s+preguntó\\b",
+      "\\b{name}\\s+respondió\\b",
+      "\\b{name}\\s+contó\\b",
+      "\\b{name}\\s+rió\\b",
+      "\\b{name}\\s+sonrió\\b",
+      "\\b{name}\\s+lloró\\b",
+      "\\b{name}\\s+sintió\\b",
+      "\\b{name}\\s+piensa\\b",
+      "\\b{name}\\s+quiere\\b",
+      "\\b{name}\\s+ama\\b",
+      "\\b{name}\\s+odia\\b",
+      "\\b{name}\\s+sabe\\b",
+      "\\b{name}\\s+decidió\\b",
+      "\\b{name}\\s+escribió\\b"
+    ],
+    "pronoun_patterns": [
+      "\\bél\\b",
+      "\\bella\\b",
+      "\\bellos\\b",
+      "\\bellas\\b",
+      "\\blo\\b",
+      "\\bla\\b",
+      "\\ble\\b",
+      "\\bles\\b",
+      "\\bse\\b"
+    ],
+    "dialogue_patterns": [
+      "^>\\s*{name}[:\\s]",
+      "^{name}:\\s",
+      "^\\[{name}\\]",
+      "\"{name}\\s+dijo"
+    ],
+    "direct_address_pattern": "\\bhola\\s+{name}\\b|\\bhey\\s+{name}\\b|\\bhi\\s+{name}\\b|\\bgracias\\s+{name}\\b|\\bquerido\\s+{name}\\b|\\bquerida\\s+{name}\\b|\\bestimado\\s+{name}\\b|\\bestimada\\s+{name}\\b|\\bdon\\s+{name}\\b|\\bdoña\\s+{name}\\b|\\bseñor\\s+{name}\\b|\\bseñora\\s+{name}\\b",
+    "project_verb_patterns": [
+      "\\bconstruyo\\s+{name}\\b",
+      "\\bconstruí\\s+{name}\\b",
+      "\\barmé\\s+{name}\\b",
+      "\\blancé\\s+{name}\\b",
+      "\\bdesplegué\\s+{name}\\b",
+      "\\binstalé\\s+{name}\\b",
+      "\\bla\\s+arquitectura\\s+{name}\\b",
+      "\\bel\\s+pipeline\\s+{name}\\b",
+      "\\bel\\s+sistema\\s+{name}\\b",
+      "\\bel\\s+proyecto\\s+{name}\\b",
+      "\\bel\\s+repositorio\\s+{name}\\b",
+      "\\b{name}\\s+v\\d+\\b",
+      "\\b{name}\\.py\\b",
+      "\\bimport\\s+{name}\\b",
+      "\\bpip\\s+install\\s+{name}\\b"
+    ],
+    "stopwords": [
+      "el", "la", "los", "las", "un", "una", "unos", "unas",
+      "de", "del", "al", "a", "en", "con", "sin", "por", "para", "sobre",
+      "entre", "hasta", "desde", "hacia", "contra", "según", "tras",
+      "y", "o", "u", "ni", "pero", "sino", "aunque", "porque", "pues",
+      "que", "quien", "quienes", "cual", "cuales", "cuyo", "cuya",
+      "donde", "cuando", "como", "cuanto", "cuanta",
+      "yo", "tú", "él", "ella", "nosotros", "vosotros", "ellos", "ellas",
+      "me", "te", "se", "nos", "os", "lo", "la", "le", "los", "las", "les",
+      "mi", "mis", "tu", "tus", "su", "sus", "nuestro", "nuestra", "vuestro",
+      "este", "esta", "estos", "estas", "ese", "esa", "esos", "esas",
+      "aquel", "aquella", "aquellos", "aquellas", "esto", "eso", "aquello",
+      "ser", "estar", "haber", "tener", "hacer", "poder", "querer", "saber",
+      "es", "son", "fue", "fueron", "era", "eran", "está", "están", "estaba",
+      "he", "ha", "hemos", "han", "había", "hay",
+      "muy", "mucho", "mucha", "muchos", "muchas", "poco", "poca", "pocos", "pocas",
+      "más", "menos", "tan", "tanto", "también", "tampoco",
+      "sí", "no", "quizás", "tal", "vez",
+      "aquí", "allí", "allá", "ahí", "acá",
+      "hoy", "ayer", "mañana", "ahora", "antes", "después", "luego", "entonces",
+      "siempre", "nunca", "jamás", "todavía", "aún", "ya",
+      "bien", "mal", "mejor", "peor", "bueno", "buena", "malo", "mala",
+      "gracias", "hola", "adiós", "por favor", "perdón",
+      "día", "días", "semana", "mes", "año", "años", "tiempo", "vez", "veces",
+      "cosa", "cosas", "persona", "gente", "mundo", "vida", "casa", "lugar",
+      "forma", "manera", "parte", "caso", "punto", "idea", "hecho", "razón",
+      "nombre", "número", "versión", "sistema", "modelo"
+    ]
+  }
+}

mempalace/i18n/fr.json

@@ -0,0 +1,126 @@
+{
+  "lang": "fr",
+  "label": "Français",
+  "terms": {
+    "palace": "palais",
+    "wing": "aile",
+    "hall": "couloir",
+    "closet": "placard",
+    "drawer": "tiroir",
+    "mine": "extraire",
+    "search": "chercher",
+    "status": "état",
+    "init": "initialiser",
+    "repair": "réparer",
+    "migrate": "migrer",
+    "entity": "entité",
+    "topic": "sujet"
+  },
+  "cli": {
+    "mine_start": "Extraction de {path}...",
+    "mine_complete": "Terminé. {closets} placards, {drawers} tiroirs créés.",
+    "mine_skip": "Déjà extrait. Utilisez --force pour refaire.",
+    "search_no_results": "Aucun résultat pour : {query}",
+    "search_results": "{count} résultats trouvés :",
+    "status_palace": "Palais : {path}",
+    "status_wings": "{count} ailes",
+    "status_closets": "{count} placards",
+    "status_drawers": "{count} tiroirs",
+    "init_complete": "Palais initialisé dans {path}",
+    "init_exists": "Un palais existe déjà dans {path}",
+    "repair_complete": "Réparation terminée. {fixed} problèmes corrigés.",
+    "migrate_complete": "Migration terminée.",
+    "no_palace": "Aucun palais trouvé. Exécutez : mempalace init <dossier>"
+  },
+  "aaak": {
+    "instruction": "Comprimez en français. Tirets entre les mots, pipes entre les concepts. Supprimez les articles et mots de remplissage. Gardez les noms propres et chiffres exacts."
+  },
+  "regex": {
+    "topic_pattern": "[A-ZÀ-Ý][a-zà-ÿ]{2,}|[A-Za-zÀ-ÿ]{3,}",
+    "stop_words": "le la les un une des de du au aux en et ou mais donc or ni car que qui ce cette ces son sa ses mon ma mes ton ta tes leur leurs nous vous ils elles on ne pas plus très bien aussi avec pour dans sur par est sont fait être avoir été comme tout tous toute toutes",
+    "quote_pattern": "«\\s*([^»]{10,200})\\s*»|\"([^\"]{10,200})\"",
+    "action_pattern": "(?:construit|corrigé|écrit|ajouté|poussé|mesuré|testé|révisé|créé|supprimé|mis à jour|configuré|déployé|migré)\\s+[\\wà-ÿ\\s]{3,30}"
+  },
+  "entity": {
+    "candidate_pattern": "[A-ZÀÂÄÇÉÈÊËÎÏÔÖÙÛÜŸÆŒ][a-zàâäçéèêëîïôöùûüÿæœ]{1,19}",
+    "multi_word_pattern": "[A-ZÀÂÄÇÉÈÊËÎÏÔÖÙÛÜŸÆŒ][a-zàâäçéèêëîïôöùûüÿæœ]+(?:\\s+[A-ZÀÂÄÇÉÈÊËÎÏÔÖÙÛÜŸÆŒ][a-zàâäçéèêëîïôöùûüÿæœ]+)+",
+    "person_verb_patterns": [
+      "\\b{name}\\s+a\\s+dit\\b",
+      "\\b{name}\\s+a\\s+demandé\\b",
+      "\\b{name}\\s+a\\s+répondu\\b",
+      "\\b{name}\\s+a\\s+raconté\\b",
+      "\\b{name}\\s+a\\s+ri\\b",
+      "\\b{name}\\s+a\\s+souri\\b",
+      "\\b{name}\\s+a\\s+pleuré\\b",
+      "\\b{name}\\s+a\\s+senti\\b",
+      "\\b{name}\\s+pense\\b",
+      "\\b{name}\\s+veut\\b",
+      "\\b{name}\\s+aime\\b",
+      "\\b{name}\\s+déteste\\b",
+      "\\b{name}\\s+sait\\b",
+      "\\b{name}\\s+a\\s+décidé\\b",
+      "\\b{name}\\s+a\\s+écrit\\b"
+    ],
+    "pronoun_patterns": [
+      "\\bil\\b",
+      "\\belle\\b",
+      "\\blui\\b",
+      "\\bils\\b",
+      "\\belles\\b",
+      "\\bleur\\b",
+      "\\bleurs\\b",
+      "\\beux\\b",
+      "\\bse\\b"
+    ],
+    "dialogue_patterns": [
+      "^>\\s*{name}[:\\s]",
+      "^{name}:\\s",
+      "^\\[{name}\\]",
+      "\"{name}\\s+a\\s+dit"
+    ],
+    "direct_address_pattern": "\\bbonjour\\s+{name}\\b|\\bsalut\\s+{name}\\b|\\bmerci\\s+{name}\\b|\\bcher\\s+{name}\\b|\\bchère\\s+{name}\\b|\\bmonsieur\\s+{name}\\b|\\bmadame\\s+{name}\\b|\\bhey\\s+{name}\\b|\\bhi\\s+{name}\\b",
+    "project_verb_patterns": [
+      "\\bconstruit\\s+{name}\\b",
+      "\\blancé\\s+{name}\\b",
+      "\\bdéployé\\s+{name}\\b",
+      "\\binstallé\\s+{name}\\b",
+      "\\bl'architecture\\s+{name}\\b",
+      "\\ble\\s+pipeline\\s+{name}\\b",
+      "\\ble\\s+système\\s+{name}\\b",
+      "\\ble\\s+projet\\s+{name}\\b",
+      "\\ble\\s+dépôt\\s+{name}\\b",
+      "\\b{name}\\s+v\\d+\\b",
+      "\\b{name}\\.py\\b",
+      "\\bimport\\s+{name}\\b",
+      "\\bpip\\s+install\\s+{name}\\b"
+    ],
+    "stopwords": [
+      "le", "la", "les", "un", "une", "des", "du", "de", "au", "aux",
+      "en", "dans", "sur", "sous", "avec", "sans", "pour", "par", "vers",
+      "chez", "entre", "depuis", "pendant", "avant", "après", "jusqu",
+      "et", "ou", "mais", "donc", "or", "ni", "car", "que", "qui",
+      "dont", "où", "quand", "comment", "pourquoi", "combien", "lequel",
+      "ce", "cet", "cette", "ces", "celui", "celle", "ceux", "celles",
+      "mon", "ma", "mes", "ton", "ta", "tes", "son", "sa", "ses",
+      "notre", "nos", "votre", "vos", "leur", "leurs",
+      "je", "tu", "il", "elle", "on", "nous", "vous", "ils", "elles",
+      "me", "te", "se", "lui", "eux",
+      "être", "avoir", "faire", "dire", "aller", "voir", "savoir", "pouvoir",
+      "est", "sont", "était", "étaient", "fut", "furent", "sera", "seront",
+      "ai", "as", "a", "avons", "avez", "ont", "avait", "avaient",
+      "très", "bien", "mal", "peu", "beaucoup", "trop", "assez", "aussi",
+      "plus", "moins", "tant", "si", "tellement",
+      "oui", "non", "peut-être", "vraiment",
+      "ici", "là", "là-bas", "partout", "ailleurs",
+      "aujourd'hui", "hier", "demain", "maintenant", "alors", "ensuite",
+      "toujours", "jamais", "souvent", "parfois", "déjà", "encore",
+      "bon", "bonne", "mauvais", "mauvaise", "meilleur", "pire",
+      "merci", "bonjour", "salut", "au revoir",
+      "jour", "jours", "semaine", "mois", "année", "temps", "fois",
+      "chose", "choses", "personne", "gens", "monde", "vie", "maison",
+      "endroit", "lieu", "partie", "façon", "manière", "sorte", "type",
+      "cas", "point", "idée", "fait", "raison", "nom", "nombre",
+      "version", "système", "modèle", "question", "réponse"
+    ]
+  }
+}

mempalace/i18n/hi.json

@@ -0,0 +1,105 @@
+{
+  "lang": "hi",
+  "label": "हिंदी",
+  "terms": {
+    "palace": "महल",
+    "wing": "खंड",
+    "hall": "हॉल",
+    "closet": "अलमारी",
+    "drawer": "दराज़",
+    "mine": "खनन",
+    "search": "खोज",
+    "status": "स्थिति",
+    "init": "आरंभ",
+    "repair": "मरम्मत",
+    "migrate": "स्थानांतरित करना",
+    "entity": "इकाई",
+    "topic": "विषय"
+  },
+  "cli": {
+    "mine_start": "{path} का खनन किया जा रहा है...",
+    "mine_complete": "पूर्ण हुआ। {closets} अलमारियाँ, {drawers} दराज़ें बनाई गईं।",
+    "mine_skip": "पहले ही खनन हो चुका है। पुनः खनन के लिए --force का उपयोग करें।",
+    "search_no_results": "इसके लिए कोई परिणाम नहीं मिले: {query}",
+    "search_results": "{count} परिणाम मिले:",
+    "status_palace": "महल: {path}",
+    "status_wings": "{count} खंड",
+    "status_closets": "{count} अलमारियाँ",
+    "status_drawers": "{count} दराज़ें",
+    "init_complete": "महल {path} पर प्रारंभ किया गया",
+    "init_exists": "महल पहले से ही {path} पर मौजूद है",
+    "repair_complete": "मरम्मत पूर्ण। {fixed} समस्याएँ ठीक की गईं।",
+    "migrate_complete": "स्थानांतरण पूर्ण।",
+    "no_palace": "कोई महल नहीं मिला। चलाएँ: mempalace init <dir>"
+  },
+  "aaak": {
+    "instruction": "इंडेक्स प्रारूप में संपीड़न करें। शब्दों के बीच हाइफ़न और अवधारणाओं के बीच पाइप का प्रयोग करें। आर्टिकल और अनावश्यक शब्द हटाएँ। नाम और संख्याएँ सटीक रखें।"
+  },
+  "regex": {
+    "topic_pattern": "[\\u0900-\\u097F]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "यह वह ये वे कुछ कई अधिकांश प्रत्येक हर अन्य केवल ऐसा बहुत होगा सकता चाहिए ज़रूर हाँ ठीक भी फिर अब पहले अभी वापस पूरा बनाना लेना देना जानना सोचना चाहना ज़रूरत जा आ आना जाना ढूँढना काम जोड़ा सहेजा सत्र सारांश वार्तालाप स्रोत विषय के एक बार बस वास्तव में कहाँ यहाँ वहाँ धन्यवाद कृपया सही गलत करें किया करता करती चलाएँ उपयोग",
+    "quote_pattern": "\"([^\"]{20,200})\"",
+    "action_pattern": "(?:बनाया|सुधारा|लिखा|जोड़ा|भेजा|मापा|परीक्षण किया|समीक्षा की|निर्मित किया|हटाया|अद्यतन किया|विन्यस्त किया|तैनात किया|स्थानांतरित किया)\\s+[\\w\\s\\u0900-\\u097F]{3,30}"
+  },
+  "entity": {
+  "boundary_chars": "\\w\\u0900-\\u097F",
+  "candidate_pattern": "[\\u0900-\\u097F]{2,20}",
+  "multi_word_pattern": "[\\u0900-\\u097F]+(?:\\s+[\\u0900-\\u097F]+)+",
+  "person_verb_patterns": [
+    "\\b{name}\\s+ने\\s+कहा\\b",
+    "\\b{name}\\s+ने\\s+पूछा\\b",
+    "\\b{name}\\s+ने\\s+बोला\\b",
+    "\\b{name}\\s+ने\\s+बताया\\b",
+    "\\b{name}\\s+हँसा\\b",
+    "\\b{name}\\s+मुस्कुराया\\b",
+    "\\b{name}\\s+रोया\\b",
+    "\\b{name}\\s+सोचा\\b",
+    "\\b{name}\\s+चाहा\\b",
+    "\\b{name}\\s+पसंद\\s+किया\\b",
+    "\\b{name}\\s+नफरत\\s+की\\b",
+    "\\b{name}\\s+जानता\\s+है\\b"
+  ],
+  "pronoun_patterns": [
+    "\\bवह\\b",
+    "\\bउसने\\b",
+    "\\bउसे\\b",
+    "\\bउसका\\b",
+    "\\bउन्होंने\\b",
+    "\\bउनका\\b",
+    "\\bवे\\b"
+  ],
+  "dialogue_patterns": [
+    "^>\\s*{name}[:\\s]",
+    "^{name}:\\s",
+    "^\\[{name}\\]",
+    "\"{name}\\s+ने\\s+कहा"
+  ],
+  "direct_address_pattern": "\\bअरे\\s+{name}\\b|\\bनमस्ते\\s+{name}\\b|\\bधन्यवाद\\s+{name}\\b",
+  "project_verb_patterns": [
+    "\\b{name}\\s+बना\\s+रहा\\s+है\\b",
+    "\\b{name}\\s+बनाया\\b",
+    "\\b{name}\\s+लॉन्च\\s+किया\\b",
+    "\\b{name}\\s+तैनात\\s+किया\\b",
+    "\\b{name}\\s+इंस्टॉल\\s+किया\\b"
+  ],
+  "stopwords": [
+      "यह", "वह", "ये", "वे",
+      "मैं", "हम", "आप", "तुम",
+      "मेरा", "हमारा", "आपका", "उसका", "उनका",
+      "मुझे", "हमें", "आपको", "उसे", "उन्हें",
+      "का", "के", "की",
+      "को", "से", "में", "पर",
+      "के लिए", "के साथ", "के बारे में", "द्वारा",
+      "और", "या", "लेकिन", "क्योंकि", "तो", "यदि",
+      "भी", "ही", "सिर्फ", "केवल",
+      "है", "हैं", "था", "थे", "थी",
+      "हो", "होगा", "होता", "होती",
+      "कर", "करना", "किया", "करते", "करती",
+      "नहीं", "हाँ", "शायद", "ज़रूर",
+      "क्या", "कौन", "कब", "कहाँ", "क्यों", "कैसे",
+      "अब", "तब", "यहाँ", "वहाँ",
+      "बहुत", "कम", "अधिक",
+      "कुछ", "कोई", "सब", "हर"
+    ]
+  }
+}

mempalace/i18n/id.json

@@ -0,0 +1,234 @@
+{
+  "lang": "id",
+  "label": "Bahasa Indonesia",
+  "terms": {
+    "palace": "istana",
+    "wing": "sayap",
+    "hall": "lorong",
+    "closet": "lemari",
+    "drawer": "laci",
+    "mine": "tambang",
+    "search": "cari",
+    "status": "status",
+    "init": "inisialisasi",
+    "repair": "perbaiki",
+    "migrate": "migrasi",
+    "entity": "entitas",
+    "topic": "topik"
+  },
+  "cli": {
+    "mine_start": "Menambang {path}...",
+    "mine_complete": "Selesai. {closets} lemari dan {drawers} laci berhasil dibuat.",
+    "mine_skip": "Sudah pernah ditambang. Gunakan --force untuk menambang ulang.",
+    "search_no_results": "Tidak ada hasil untuk: {query}",
+    "search_results": "Ditemukan {count} hasil:",
+    "status_palace": "Istana: {path}",
+    "status_wings": "{count} sayap",
+    "status_closets": "{count} lemari",
+    "status_drawers": "{count} laci",
+    "init_complete": "Istana diinisialisasi pada {path}",
+    "init_exists": "Istana sudah ada pada {path}",
+    "repair_complete": "Perbaikan selesai. {fixed} masalah berhasil diperbaiki.",
+    "migrate_complete": "Migrasi selesai.",
+    "no_palace": "Istana tidak ditemukan. Jalankan: mempalace init <dir>"
+  },
+  "aaak": {
+    "instruction": "Ringkas ke format indeks. Gunakan tanda hubung (-) antar kata dan garis vertikal (|) antar konsep. Buang kata fungsi dan kata pengisi yang tidak penting. Pertahankan nama serta angka tetap persis."
+  },
+  "regex": {
+    "topic_pattern": "[A-Z][a-z]{2,}|[A-Za-z][A-Za-z0-9_/-]{2,}",
+    "stop_words": "yang untuk pada ke para namun menurut antara dia ia seperti jika sehingga kembali dan tidak ini karena kepada oleh saat harus sementara setelah belum kami sekitar bagi serta di dari telah sebagai masih hal ketika adalah itu dalam bisa bahwa atau hanya kita dengan akan juga ada mereka sudah saya terhadap secara agar lain anda begitu mengapa kenapa yaitu yakni daripada itulah lagi maka tentang demi di mana ke mana pula sambil sebelum sesudah supaya guna kah pun sampai sedangkan selagi tetapi apakah kecuali sebab selain seolah seraya seterusnya tanpa agak boleh dapat dsb dst dll dahulu dulunya anu demikian tapi ingin nggak gak ga ngga enggak mari nanti melainkan oh ok oke seharusnya sebetulnya setiap setidaknya sesuatu pasti saja toh ya walau tolong tentu amat apalagi bagaimanapun udah banget bgt nih dong kok sih deh aja pun",
+    "quote_pattern": "\"([^\"]{20,200})\"|“([^”]{20,200})”|‘([^’]{20,200})’",
+    "action_pattern": "(?:dibangun|membangun|ngembangin|diperbaiki|memperbaiki|ditulis|menulis|ditambahkan|menambahkan|dibuat|membuat|diperbarui|memperbarui|diulas|mengulas|diuji|menguji|diukur|mengukur|dikonfigurasi|mengonfigurasi|dideploy|deploy|nge-?deploy|ngebuild|build|dikirim|push|dipush|nge-?push|dirilis|rilis|dimigrasi|migrasi|dibundle)\\s+[\\w\\s./_-]{3,30}"
+  },
+  "entity": {
+    "candidate_pattern": "[A-Z][a-z]{1,19}|[A-Z]{2,10}|[A-Za-z][A-Za-z0-9_]*[A-Z][A-Za-z0-9_]*|[a-z]+[A-Z][A-Za-z0-9_]*",
+    "multi_word_pattern": "(?:[A-Z][a-z]+|[A-Z]{2,10})(?:\\s+(?:[A-Z][a-z]+|[A-Z]{2,10}))+",
+    "person_verb_patterns": [
+      "\\b{name}\\s+berkata\\b",
+      "\\b{name}\\s+mengatakan\\b",
+      "\\b{name}\\s+bilang\\b",
+      "\\b{name}\\s+ngomong\\b",
+      "\\b{name}\\s+ceritain\\b",
+      "\\b{name}\\s+bertanya\\b",
+      "\\b{name}\\s+menanyakan\\b",
+      "\\b{name}\\s+tanya\\b",
+      "\\b{name}\\s+nanya\\b",
+      "\\b{name}\\s+menjawab\\b",
+      "\\b{name}\\s+jawab\\b",
+      "\\b{name}\\s+balas\\b",
+      "\\b{name}\\s+reply\\b",
+      "\\b{name}\\s+membalas\\b",
+      "\\b{name}\\s+menjelaskan\\b",
+      "\\b{name}\\s+cerita\\b",
+      "\\b{name}\\s+tertawa\\b",
+      "\\b{name}\\s+tersenyum\\b",
+      "\\b{name}\\s+menangis\\b",
+      "\\b{name}\\s+merasa\\b",
+      "\\b{name}\\s+memikirkan\\b",
+      "\\b{name}\\s+berpikir\\b",
+      "\\b{name}\\s+pikir\\b",
+      "\\b{name}\\s+ingin\\b",
+      "\\b{name}\\s+mau\\b",
+      "\\b{name}\\s+suka\\b",
+      "\\b{name}\\s+benci\\b",
+      "\\b{name}\\s+tahu\\b",
+      "\\b{name}\\s+memutuskan\\b",
+      "\\b{name}\\s+memilih\\b",
+      "\\b{name}\\s+decided\\b",
+      "\\b{name}\\s+menulis\\b",
+      "\\b{name}\\s+nulis\\b",
+      "\\b{name}\\s+ngetik\\b",
+      "\\b{name}\\s+push\\b",
+      "\\b{name}\\s+nge-?push\\b",
+      "\\b{name}\\s+review(?:ed)?\\b",
+      "\\b{name}\\s+nge-?review\\b",
+      "\\b{name}\\s+approve(?:d)?\\b",
+      "\\b{name}\\s+di-?approve\\b"
+    ],
+    "pronoun_patterns": [
+      "\\bdia\\b",
+      "\\bia\\b",
+      "\\bbeliau\\b",
+      "\\bmereka\\b"
+    ],
+    "dialogue_patterns": [
+      "^>\\s*{name}[:\\s]",
+      "^{name}:\\s",
+      "^\\[{name}\\]",
+      "\"{name}\\s+berkata",
+      "\"{name}\\s+bilang"
+    ],
+    "direct_address_pattern": "\\bhai\\s+{name}\\b|\\bhalo\\s+{name}\\b|\\bhi\\s+{name}\\b|\\bhei\\s+{name}\\b|\\bterima\\s+kasih\\s+{name}\\b|\\bmakasih\\s+{name}\\b|\\bmakasi\\s+{name}\\b|\\bpak\\s+{name}\\b|\\bbu\\s+{name}\\b|\\bmas\\s+{name}\\b|\\bmbak\\s+{name}\\b|\\bkak\\s+{name}\\b",
+    "project_verb_patterns": [
+      "\\bmembangun\\s+{name}\\b",
+      "\\bbangun\\s+{name}\\b",
+      "\\bdibangun\\s+{name}\\b",
+      "\\bngembangin\\s+{name}\\b",
+      "\\bmengerjakan\\s+{name}\\b",
+      "\\bngerjain\\s+{name}\\b",
+      "\\bgarap\\s+{name}\\b",
+      "\\bbuild(?:ing)?\\s+{name}\\b",
+      "\\bnge-?build\\s+{name}\\b",
+      "\\bmerilis\\s+{name}\\b",
+      "\\brilis\\s+{name}\\b",
+      "\\bship(?:ping|ped)?\\s+{name}\\b",
+      "\\bmeluncurkan\\s+{name}\\b",
+      "\\blaunch(?:ing|ed)?\\s+{name}\\b",
+      "\\bdeploy(?:ing|ed)?\\s+{name}\\b",
+      "\\bdideploy\\s+{name}\\b",
+      "\\bmendeploy\\s+{name}\\b",
+      "\\bnge-?deploy\\s+{name}\\b",
+      "\\binstall(?:ing|ed)?\\s+{name}\\b",
+      "\\bmenginstal\\s+{name}\\b",
+      "\\bmemasang\\s+{name}\\b",
+      "\\bpush\\s+{name}\\b",
+      "\\bnge-?push\\s+{name}\\b",
+      "\\breview(?:ing|ed)?\\s+{name}\\b",
+      "\\barsitektur\\s+{name}\\b",
+      "\\bpipeline\\s+{name}\\b",
+      "\\b{name}\\s+v\\d+\\b",
+      "\\b{name}\\.py\\b",
+      "\\b{name}-core\\b",
+      "\\b{name}-local\\b",
+      "\\bimport\\s+{name}\\b",
+      "\\bpip\\s+install\\s+{name}\\b"
+    ],
+    "stopwords": [
+      "yang",
+      "dan",
+      "atau",
+      "tetapi",
+      "namun",
+      "karena",
+      "jadi",
+      "kalau",
+      "jika",
+      "ketika",
+      "saat",
+      "supaya",
+      "agar",
+      "untuk",
+      "dari",
+      "ke",
+      "di",
+      "pada",
+      "dalam",
+      "dengan",
+      "tanpa",
+      "antara",
+      "tentang",
+      "sebagai",
+      "oleh",
+      "terhadap",
+      "ini",
+      "itu",
+      "tersebut",
+      "sini",
+      "situ",
+      "sana",
+      "saya",
+      "aku",
+      "kami",
+      "kita",
+      "kamu",
+      "anda",
+      "dia",
+      "ia",
+      "beliau",
+      "mereka",
+      "ada",
+      "tidak",
+      "bukan",
+      "iya",
+      "ya",
+      "oke",
+      "ok",
+      "baik",
+      "nah",
+      "nih",
+      "dong",
+      "deh",
+      "kok",
+      "sih",
+      "aja",
+      "juga",
+      "lagi",
+      "sudah",
+      "udah",
+      "belum",
+      "masih",
+      "baru",
+      "pernah",
+      "selalu",
+      "sering",
+      "jarang",
+      "banyak",
+      "sedikit",
+      "lebih",
+      "kurang",
+      "semua",
+      "setiap",
+      "beberapa",
+      "sesuatu",
+      "apa",
+      "siapa",
+      "mana",
+      "kapan",
+      "mengapa",
+      "kenapa",
+      "bagaimana",
+      "bisa",
+      "harus",
+      "mau",
+      "ingin",
+      "tahu",
+      "coba",
+      "pak",
+      "bu",
+      "mas",
+      "mbak",
+      "kak"
+    ]
+  }
+}

mempalace/i18n/it.json

@@ -0,0 +1,186 @@
+{
+  "lang": "it",
+  "label": "Italiano",
+  "terms": {
+    "palace": "palazzo",
+    "wing": "ala",
+    "hall": "corridoio",
+    "closet": "armadio",
+    "drawer": "cassetto",
+    "mine": "estrarre",
+    "search": "cercare",
+    "status": "stato",
+    "init": "inizializzare",
+    "repair": "riparare",
+    "migrate": "migrare",
+    "entity": "entità",
+    "topic": "argomento"
+  },
+  "cli": {
+    "mine_start": "Estrazione di {path}...",
+    "mine_complete": "Fatto. {closets} armadi, {drawers} cassetti creati.",
+    "mine_skip": "Già estratto. Usa --force per estrarre di nuovo.",
+    "search_no_results": "Nessun risultato per: {query}",
+    "search_results": "{count} risultati trovati:",
+    "status_palace": "Palazzo: {path}",
+    "status_wings": "{count} ali",
+    "status_closets": "{count} armadi",
+    "status_drawers": "{count} cassetti",
+    "init_complete": "Palazzo inizializzato in {path}",
+    "init_exists": "Esiste già un palazzo in {path}",
+    "repair_complete": "Riparazione completata. {fixed} problemi risolti.",
+    "migrate_complete": "Migrazione completata.",
+    "no_palace": "Nessun palazzo trovato. Esegui: mempalace init <cartella>"
+  },
+  "aaak": {
+    "instruction": "Comprimi in italiano. Trattini tra le parole, pipe tra i concetti. Elimina articoli e parole di riempimento. Mantieni nomi propri e numeri esatti."
+  },
+  "regex": {
+    "topic_pattern": "[A-ZÀ-Ú][a-zà-ú]{2,}|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ0-9_]{2,}",
+    "stop_words": "il lo la i gli le un uno una di del della dello dei degli delle al allo alla ai agli alle in con su per tra fra da dal dalla dallo dai dagli dalle e o ma che chi cui come dove quando perché mentre anche ancora già molto poco solo sempre mai essere avere sono sei siamo siete era erano stato stata questo questa questi queste quello quella quelli quelle mio mia miei mie tuo tua tuoi tue suo sua suoi sue nostro nostra nostri nostre vostro vostra vostri vostre loro",
+    "quote_pattern": "«\\s*([^»]{10,200})\\s*»|\"([^\"]{10,200})\"",
+    "action_pattern": "(?:costruito|corretto|scritto|aggiunto|inviato|misurato|testato|revisionato|creato|eliminato|aggiornato|configurato|distribuito|migrato)\\s+[\\wÀ-ÿ\\s]{3,30}"
+  },
+  "entity": {
+    "candidate_pattern": "[A-ZÀ-Ú][a-zà-ÿ]{1,19}",
+    "multi_word_pattern": "[A-ZÀ-Ú][a-zà-ÿ]+(?:\\s+[A-ZÀ-Ú][a-zà-ÿ]+)+",
+    "person_verb_patterns": [
+      "\\b{name}\\s+ha\\s+detto\\b",
+      "\\b{name}\\s+ha\\s+chiesto\\b",
+      "\\b{name}\\s+ha\\s+risposto\\b",
+      "\\b{name}\\s+ha\\s+riferito\\b",
+      "\\b{name}\\s+ha\\s+riso\\b",
+      "\\b{name}\\s+ha\\s+sorriso\\b",
+      "\\b{name}\\s+ha\\s+pianto\\b",
+      "\\b{name}\\s+ha\\s+sentito\\b",
+      "\\b{name}\\s+pensa\\b",
+      "\\b{name}\\s+vuole\\b",
+      "\\b{name}\\s+ama\\b",
+      "\\b{name}\\s+odia\\b",
+      "\\b{name}\\s+sa\\b",
+      "\\b{name}\\s+ha\\s+deciso\\b",
+      "\\b{name}\\s+ha\\s+scritto\\b"
+    ],
+    "pronoun_patterns": [
+      "\\blei\\b",
+      "\\blui\\b",
+      "\\bloro\\b",
+      "\\bgli\\b",
+      "\\ble\\b"
+    ],
+    "dialogue_patterns": [
+      "^>\\s*{name}[:\\s]",
+      "^{name}:\\s",
+      "^\\[{name}\\]",
+      "\"{name}\\s+ha\\s+detto"
+    ],
+    "direct_address_pattern": "\\bciao\\s+{name}\\b|\\bgrazie\\s+{name}\\b|\\bsalve\\s+{name}\\b|\\bcaro\\s+{name}\\b|\\bcara\\s+{name}\\b",
+    "project_verb_patterns": [
+      "\\bstiamo\\s+costruendo\\s+{name}\\b",
+      "\\babbiamo\\s+costruito\\s+{name}\\b",
+      "\\bstiamo\\s+lanciando\\s+{name}\\b",
+      "\\babbiamo\\s+lanciato\\s+{name}\\b",
+      "\\babbiamo\\s+distribuito\\s+{name}\\b",
+      "\\babbiamo\\s+installato\\s+{name}\\b",
+      "\\bil\\s+progetto\\s+{name}\\b",
+      "\\bil\\s+sistema\\s+{name}\\b",
+      "\\bimport\\s+{name}\\b",
+      "\\bpip\\s+install\\s+{name}\\b"
+    ],
+    "stopwords": [
+      "ciao",
+      "salve",
+      "grazie",
+      "prego",
+      "sì",
+      "si",
+      "no",
+      "forse",
+      "qui",
+      "qua",
+      "lì",
+      "oggi",
+      "ieri",
+      "domani",
+      "sempre",
+      "mai",
+      "ancora",
+      "anche",
+      "molto",
+      "poco",
+      "bene",
+      "male",
+      "così",
+      "poi",
+      "prima",
+      "dopo",
+      "tra",
+      "fra",
+      "con",
+      "senza",
+      "per",
+      "verso",
+      "contro",
+      "durante",
+      "mentre",
+      "sopra",
+      "sotto",
+      "oltre",
+      "oppure",
+      "ma",
+      "però",
+      "tuttavia",
+      "anche",
+      "se",
+      "quando",
+      "finché",
+      "perché",
+      "quindi",
+      "dunque",
+      "allora",
+      "forse",
+      "magari",
+      "abbiamo",
+      "stiamo",
+      "essere",
+      "avere",
+      "sono",
+      "sei",
+      "siamo",
+      "siete",
+      "era",
+      "erano",
+      "stato",
+      "stata",
+      "questo",
+      "questa",
+      "questi",
+      "queste",
+      "quello",
+      "quella",
+      "quelli",
+      "quelle",
+      "mio",
+      "mia",
+      "miei",
+      "mie",
+      "tuo",
+      "tua",
+      "tuoi",
+      "tue",
+      "suo",
+      "sua",
+      "suoi",
+      "sue",
+      "nostro",
+      "nostra",
+      "nostri",
+      "nostre",
+      "vostro",
+      "vostra",
+      "vostri",
+      "vostre",
+      "loro"
+    ]
+  }
+}

mempalace/i18n/ja.json

@@ -0,0 +1,44 @@
+{
+  "lang": "ja",
+  "label": "日本語",
+  "terms": {
+    "palace": "宮殿",
+    "wing": "棟",
+    "hall": "廊下",
+    "closet": "クローゼット",
+    "drawer": "引き出し",
+    "mine": "採掘",
+    "search": "検索",
+    "status": "状態",
+    "init": "初期化",
+    "repair": "修復",
+    "migrate": "移行",
+    "entity": "エンティティ",
+    "topic": "トピック"
+  },
+  "cli": {
+    "mine_start": "{path} を採掘中...",
+    "mine_complete": "完了。クローゼット {closets}個、引き出し {drawers}個 作成。",
+    "mine_skip": "採掘済み。再実行するには --force を使用。",
+    "search_no_results": "結果なし: {query}",
+    "search_results": "{count}件の結果:",
+    "status_palace": "宮殿: {path}",
+    "status_wings": "棟 {count}個",
+    "status_closets": "クローゼット {count}個",
+    "status_drawers": "引き出し {count}個",
+    "init_complete": "{path} に宮殿を初期化しました",
+    "init_exists": "{path} に宮殿は既に存在します",
+    "repair_complete": "修復完了。{fixed}件の問題を修正。",
+    "migrate_complete": "移行完了。",
+    "no_palace": "宮殿が見つかりません。実行: mempalace init <ディレクトリ>"
+  },
+  "aaak": {
+    "instruction": "日本語で圧縮してください。概念間はパイプ(|)、単語間はハイフン(-)。助詞と接続詞は省略。固有名詞と数値は正確に保持。"
+  },
+  "regex": {
+    "topic_pattern": "[\\u30A0-\\u30FF]{3,}|[\\u4E00-\\u9FFF]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "stop_words": "は が を に で と も の へ から まで より した します している されて です ます ました こと もの ため それ これ その この あの ない なく ある いる する",
+    "quote_pattern": "「([^」]{10,100})」",
+    "action_pattern": "(構築|修正|追加|削除|確認|作成|実装|修復|書き直し|テスト|検証|更新|設定|起動|停止)(?:し|した|して|する|します)"
+  }
+}