diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 8e8872e..8f4b0ba 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "mempalace", "source": "./.claude-plugin", "description": "AI memory system — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, guided setup.", - "version": "3.3.6", + "version": "3.4.0", "author": { "name": "milla-jovovich" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index ae122dd..639ede1 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.3.6", + "version": "3.4.0", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 8bbf7d8..97d84ab 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.3.6", + "version": "3.4.0", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..544641e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,55 @@ +# Keep the build context lean — only what `uv sync` + the package need. + +# VCS +.git/ +.gitignore + +# Python build artifacts / caches +*.egg-info/ +dist/ +build/ +__pycache__/ +*.pyc +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +htmlcov/ +.coverage +coverage.xml + +# Virtual environments (rebuilt inside the image) +.venv/ +venv/ + +# Local config / secrets +.env +.env.* +.envrc +mempal.yaml + +# Editor / OS cruft +.idea/ +.vscode/ +*.swp +*.swo +*~ +.DS_Store +Thumbs.db + +# Repo material not needed at runtime +.devcontainer/ +.github/ +.agents/ +.claude/ +.codex/ +.codex-plugin/ +.claude-plugin/ +benchmarks/ +docs/ +website/ +landing/ +assets/ +examples/ +tests/ +*.md +!README.md diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..6e6ccec --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,95 @@ +name: Docker + +on: + push: + # `main` is the release branch: pushes here publish the released image and + # update `latest`; `v*` tags publish versioned images. develop does not + # publish — it is validated via the pull_request trigger below. + branches: [main] + tags: ["v*"] + pull_request: + branches: [main, develop] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v6 + + # Needed for the emulated linux/arm64 build on real pushes. + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Only authenticate + push for in-repo events. Fork PRs lack the + # packages:write token, so they build (to validate the Dockerfile) but + # do not push. + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # latest -> main (the latest release); semver tags -> released versions. + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + # Publish multi-arch (amd64 + arm64 for Apple Silicon / ARM hosts) on + # real pushes; keep PRs amd64-only so the emulated arm64 build does not + # slow the PR check. + platforms: ${{ github.event_name != 'pull_request' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + # Fork PRs get a read-only Actions cache, so writing it just emits 403 + # noise — only export cache on in-repo events. + cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max' || '' }} + + # Build-only validation for the CUDA image so it cannot silently rot (CUDA base + # tag drift, cross-stage interpreter/venv copy paths, the `gpu` extra). The + # runner has no GPU, so this only proves the image *compiles*; it is never + # published (users build it themselves, per the README). + build-gpu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build GPU image (validation only — not published) + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile.gpu + platforms: linux/amd64 + push: false + cache-from: type=gha,scope=gpu + cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=gpu' || '' }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..7688ebe --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,124 @@ +name: Publish to PyPI + +# Publishes mempalace to PyPI via Trusted Publishing (OIDC — no stored token). +# +# Triggers: +# * release: published — the normal path (drafting + publishing a GitHub +# Release fires this). +# * workflow_dispatch — a manual escape hatch. GitHub does not reliably emit +# a `release` event when a *draft* tied to a pre-existing tag is published, +# so this lets a maintainer run the publish for any existing tag from the +# Actions tab ("Run workflow" → enter the tag, e.g. v3.4.0). +# +# Build + publish run in ONE job on purpose: a two-job split hands the wheel +# between jobs via upload/download-artifact, which failed repeatedly with +# `BlobNotFound` on the same-run download. Building and publishing in the same +# job removes that handoff entirely. The job is gated by the `pypi` environment +# (manual approval), and the checks below keep it self-contained — it never +# uploads a tag that isn't on main or doesn't match the version manifest. +# +# One-time setup required before the first run (see docs/RELEASING.md): +# 1. PyPI → Manage project `mempalace` → Publishing → Add a trusted publisher: +# Owner: MemPalace Repository: mempalace +# Workflow: publish.yml Environment: pypi +# 2. GitHub → repo Settings → Environments → New environment `pypi`, +# add yourself as a Required reviewer (this is the manual approval gate). + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Existing tag to build and publish (e.g. v3.4.0). Used when the release event didn't fire." + required: true + type: string + +permissions: + contents: read + +jobs: + publish: + name: Build + publish to PyPI + runs-on: ubuntu-latest + # The `pypi` environment gate (required reviewer) pauses the whole job until + # approved, and scopes the OIDC trust on the PyPI side. + environment: + name: pypi + url: https://pypi.org/p/mempalace + permissions: + contents: read # checkout + id-token: write # mint the short-lived OIDC token for Trusted Publishing + steps: + - name: Resolve the release tag + id: tag + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + INPUT_TAG: ${{ github.event.inputs.tag }} + run: | + set -euo pipefail + tag="${RELEASE_TAG:-${INPUT_TAG:-}}" + if [[ -z "$tag" ]]; then + echo "::error::no tag to publish — neither a release tag nor a workflow_dispatch input was provided" + exit 1 + fi + # Validate the shape before it is ever used as a git ref (ref-injection + # guard): require vMAJOR.MINOR.PATCH with an optional -/+ suffix, so + # loose values like 'v3' or 'v3foo' are rejected. + re='^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$' + if [[ ! "$tag" =~ $re ]]; then + echo "::error::tag '$tag' is not a valid release tag (expected vMAJOR.MINOR.PATCH[-suffix])" + exit 1 + fi + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "Resolved tag: $tag" + + - uses: actions/checkout@v6 + with: + # Fully-qualified refs/tags/ so an unqualified name can't resolve to a + # same-named *branch* instead of the tag (checkout prefers branches). + # steps.tag.outputs.tag is format-validated above (ref-injection guard). + ref: refs/tags/${{ steps.tag.outputs.tag }} + fetch-depth: 0 # full history for the ancestry check + + - name: Verify the tag is on main + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + set -euo pipefail + git fetch --no-tags origin main + tag_sha=$(git rev-parse HEAD) # the checked-out tag commit + main_sha=$(git rev-parse FETCH_HEAD) + if git merge-base --is-ancestor "$tag_sha" "$main_sha"; then + echo "Tag $TAG ($tag_sha) is reachable from main — OK." + else + echo "::error::tag $TAG is not an ancestor of main. Releases must be cut from main." + exit 1 + fi + + - name: Verify the tag matches the version manifest + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + set -euo pipefail + # version.py is the source of truth (per CLAUDE.md); version-guard.yml + # separately enforces that pyproject + the three plugin manifests agree. + py_version=$(grep -E '^__version__' mempalace/version.py | cut -d'"' -f2) + tag_version="${TAG#v}" + if [[ "$tag_version" != "$py_version" ]]; then + echo "::error::tag $TAG does not match mempalace/version.py ($py_version). Bump the version files before releasing." + exit 1 + fi + echo "Tag $TAG matches manifest version $py_version — OK." + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Build sdist + wheel + run: | + python -m pip install --upgrade build + python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c06ae..8169515 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - **First-class Antigravity IDE support.** New `.antigravity-plugin/` package + idempotent installer at `hooks/antigravity/install.sh` that registers MemPalace as a Google Antigravity plugin (MCP server, skill, two lifecycle hooks) at `~/.gemini/config/plugins/mempalace/`. The Stop hook background-mines the active conversation transcript every Nth fire (default 15, configurable via `MEMPAL_SAVE_INTERVAL`); the PreInvocation hook injects verbatim memory on the first model call only via Antigravity's `injectSteps[].ephemeralMessage` output, gated by `invocationNum == 1`. Both hooks are bash 3.2.57 compatible (macOS default), use the same `~/.mempalace/hook_state/` directory as the Claude Code / Codex / Cursor hooks (`antigravity_*`-namespaced state files), and respect every existing kill switch (`MEMPAL_DISABLE_HOOK`, `MEMPALACE_HOOKS_AUTO_SAVE`, `~/.mempalace/config.json` `hooks.auto_save`). Installer is `cmp`-gated (re-run produces a byte-identical install), uninstall is basename-guarded (refuses to wipe a directory whose basename isn't `mempalace`), and `--dry-run` is side-effect free. Full audit of which Antigravity surfaces we ship and which we deliberately don't is in [`hooks/antigravity/INVESTIGATION.md`](hooks/antigravity/INVESTIGATION.md). User-facing guide: [`website/guide/antigravity.md`](website/guide/antigravity.md). Standalone examples in [`examples/antigravity/`](examples/antigravity/). - **Zero-config interpreter resolution.** `mempal_resolve_python` now derives the Python interpreter from the `mempalace-mcp` / `mempalace` console-script shebang on `$PATH` before falling back to `python3`. The common `uv tool install mempalace` / `pipx install` layout installs the console scripts into an isolated environment whose interpreter is **not** system `python3`, so the previous `command -v python3` resolution landed on a Python that couldn't import `mempalace`, the `-m mempalace` probe failed, and mining silently never fired. Resolution is pure shebang parsing + `stat` (no Python subprocess at source time, preserving the hook performance budget). `MEMPAL_PYTHON` remains the explicit override. Documented under *How the hooks find your `mempalace` install* in the guide. +### Bug Fixes + +- **Backup retention to prevent unbounded disk usage.** `mempalace migrate` (full-palace `.pre-migrate.` copies) and `mempalace repair max-seq-id` (`chroma.sqlite3.max-seq-id-backup-` copies) each wrote a fresh, full-size, timestamped backup every run and never deleted the old ones. On a machine that mines or repairs on a schedule, those copies could silently accumulate until they filled the disk — one palace was found with hundreds of GB of stale backups beside a few hundred MB of live data, hidden from a normal `du` of the home directory. A new `max_backups` setting (default `10`, env `MEMPALACE_MAX_BACKUPS`, or `config.json`) now prunes the oldest backups after each new one is written. Set it to `0` to keep every backup. Pruning is keyed by filesystem mtime, scoped strictly to each backup's own naming pattern (live data is never touched), and best-effort so a deletion failure can never abort a migration or repair that already succeeded. + --- ## [3.3.6] — 2026-05-24 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..64013ea --- /dev/null +++ b/Dockerfile @@ -0,0 +1,102 @@ +# syntax=docker/dockerfile:1.7 + +# MemPalace — CPU image. +# +# Multi-stage build using uv (the project ships a uv.lock, so we install from +# the frozen lockfile for reproducible images). The default runtime is the MCP +# server over stdio; the CLI is reachable through the same entrypoint. +# +# Build: +# docker build -t mempalace . +# docker build -t mempalace --build-arg EXTRAS="extract,spellcheck" . +# +# Run (MCP server over stdio, palace persisted on the host): +# docker run -i --rm -v mempalace-data:/data mempalace +# +# Run (CLI): +# docker run --rm -v mempalace-data:/data mempalace search "why GraphQL" +# +# GPU acceleration lives in Dockerfile.gpu (it needs a CUDA base image). + +ARG PYTHON_VERSION=3.12 + +# --- builder ---------------------------------------------------------------- +FROM python:${PYTHON_VERSION}-slim AS builder + +# uv: fast, lockfile-driven installer. Pinned by digest-less tag for clarity; +# bump deliberately. +COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/ + +# Some transitive deps (grpcio, onnxruntime, tokenizers) ship manylinux wheels +# for cp312, but keep a compiler around so a missing wheel degrades to a source +# build instead of failing the image. Dropped from the final stage. +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && rm -rf /var/lib/apt/lists/* + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never + +WORKDIR /app + +# Optional extras baked into the image. CPU-safe by default; GPU is a separate +# Dockerfile. Pass a comma-separated list, e.g. EXTRAS="extract,spellcheck". +ARG EXTRAS="extract,spellcheck" + +# Layer 1: dependencies only (no project) — cached across source changes. +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=README.md,target=README.md \ + set -e; \ + flags=""; \ + for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \ + uv sync --frozen --no-install-project --no-dev ${flags} + +# Layer 2: the project itself. --no-editable installs mempalace into the venv's +# site-packages (instead of an .pth pointing at /app), so the runtime stage can +# copy only /app/.venv and drop the source tree. +COPY . /app +RUN --mount=type=cache,target=/root/.cache/uv \ + set -e; \ + flags=""; \ + for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \ + uv sync --frozen --no-dev --no-editable ${flags} + +# --- runtime ---------------------------------------------------------------- +FROM python:${PYTHON_VERSION}-slim AS runtime + +LABEL org.opencontainers.image.title="MemPalace" \ + org.opencontainers.image.description="Local-first AI memory — verbatim storage, MCP server + CLI." \ + org.opencontainers.image.source="https://github.com/MemPalace/mempalace" \ + org.opencontainers.image.licenses="MIT" + +# /data is the single persistence root: HOME points here, so the palace +# (~/.mempalace/palace), config (~/.mempalace), and the embedding-model cache +# all land under one mountable volume. The default `minilm` model caches under +# ~/.cache/chroma (~80 MB, from ChromaDB's S3); the optional `embeddinggemma` +# model caches under ~/.cache/huggingface (~300 MB). Both lazy-download on +# first use. +ENV HOME=/data \ + PATH="/app/.venv/bin:${PATH}" \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +# Non-root user owning the data volume. +RUN groupadd --gid 1000 mempalace \ + && useradd --uid 1000 --gid 1000 --home-dir /data --create-home mempalace + +WORKDIR /app + +# The resolved virtualenv from the builder — no build toolchain in this layer. +COPY --from=builder --chown=mempalace:mempalace /app/.venv /app/.venv +COPY --chown=mempalace:mempalace docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +USER mempalace +VOLUME ["/data"] + +# Default to the MCP server; `docker run` it with `-i` for stdio JSON-RPC. +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["mcp"] diff --git a/Dockerfile.gpu b/Dockerfile.gpu new file mode 100644 index 0000000..5c8358d --- /dev/null +++ b/Dockerfile.gpu @@ -0,0 +1,112 @@ +# syntax=docker/dockerfile:1.7 + +# MemPalace — GPU (NVIDIA CUDA) image. +# +# Multi-stage build using uv (the project ships a uv.lock, so we install from +# the frozen lockfile for reproducible images). The `gpu` extra pulls +# onnxruntime-gpu, which needs CUDA + cuDNN shared libraries at runtime, so +# this variant builds on an nvidia/cuda base instead of python:slim. +# +# The builder stage is dropped from the final image, so build tools +# (compilers, apt cache) never reach production — only the uv-managed +# interpreter and the resolved virtualenv do. +# +# Build: +# docker build -f Dockerfile.gpu -t mempalace:gpu . +# +# Run (requires the NVIDIA Container Toolkit on the host): +# docker run -i --rm --gpus all \ +# -e MEMPALACE_EMBEDDING_DEVICE=cuda \ +# -v mempalace-data:/data mempalace:gpu +# +# NOTE: onnxruntime-gpu ties itself to a CUDA major version. If embeddings +# fail to load on the GPU, align CUDA_IMAGE below with the CUDA release that +# the resolved onnxruntime-gpu wheel targets (see its release notes), then +# rebuild. + +ARG CUDA_IMAGE=nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04 + +# --- builder ---------------------------------------------------------------- +FROM ${CUDA_IMAGE} AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/ + +# Minimal toolchain for any source-built wheels; dropped before the runtime +# stage so compilers never ship in the production image. +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && rm -rf /var/lib/apt/lists/* + +ARG PYTHON_VERSION=3.12 +ARG EXTRAS="extract,spellcheck,gpu" + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python \ + UV_PYTHON_PREFERENCE=only-managed + +WORKDIR /app + +# Layer 1: dependencies only (no project source). Bind-mounted files keep the +# project tree out of this layer, so changing source code does not bust the +# deps cache. The uv-managed interpreter is also installed here, into a +# stable path that the runtime stage can copy verbatim. +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=README.md,target=README.md \ + set -e; \ + uv python install ${PYTHON_VERSION}; \ + flags=""; \ + for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \ + uv sync --frozen --no-install-project --no-dev --python ${PYTHON_VERSION} ${flags} + +# Layer 2: the project itself. --no-editable installs mempalace into the +# venv's site-packages (instead of an .pth pointing at /app), so the runtime +# stage can copy only /app/.venv and drop the source tree. +COPY . /app +RUN --mount=type=cache,target=/root/.cache/uv \ + set -e; \ + flags=""; \ + for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \ + uv sync --frozen --no-dev --no-editable ${flags} + +# --- runtime ---------------------------------------------------------------- +FROM ${CUDA_IMAGE} AS runtime + +LABEL org.opencontainers.image.title="MemPalace (GPU)" \ + org.opencontainers.image.description="Local-first AI memory with CUDA-accelerated embeddings." \ + org.opencontainers.image.source="https://github.com/MemPalace/mempalace" \ + org.opencontainers.image.licenses="MIT" + +# ca-certificates only — needed for the lazy HuggingFace model download on +# first use. No build toolchain in this stage. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +ENV HOME=/data \ + PATH="/app/.venv/bin:${PATH}" \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + MEMPALACE_EMBEDDING_DEVICE=cuda + +# Non-root user owning the data volume. +RUN groupadd --gid 1000 mempalace \ + && useradd --uid 1000 --gid 1000 --home-dir /data --create-home mempalace + +WORKDIR /app + +# Bring the uv-managed interpreter and the resolved venv across the stage +# boundary. /opt/uv/python must be copied alongside .venv: the venv's +# shebangs and binary launcher reference it. +COPY --from=builder /opt/uv/python /opt/uv/python +COPY --from=builder --chown=mempalace:mempalace /app/.venv /app/.venv +COPY --chown=mempalace:mempalace docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +USER mempalace +VOLUME ["/data"] + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["mcp"] diff --git a/README.md b/README.md index 70c46eb..780fddf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,3 @@ -> [!CAUTION] -> # 🚨 CRITICAL SECURITY WARNING: BEWARE OF SCAMS -> **MemPalace has NO other official websites.** -> -> The **ONLY** official sources are: -> 1. This **[GitHub repository](https://github.com/MemPalace/mempalace)** -> 2. The **[PyPI package](https://pypi.org/project/mempalace/)** -> 3. The docs at **[mempalaceofficial.com](https://mempalaceofficial.com)** -> -> **ANY other domain** (including `.tech`, `.net`, or other `.com` variants) is an **impostor** and may distribute **malware**. Do not download executables from untrusted sites. Details and timeline: [docs/HISTORY.md](docs/HISTORY.md). - -> [!IMPORTANT] -> **🚨 Claude Code sessions expire in 30 days w/out auto-save hooks wired!** **[Read this →](https://github.com/MemPalace/mempalace/discussions/1388)** -> -> Need the shortest recovery/setup path? Use the -> [Claude Code retention setup checklist](https://mempalaceofficial.com/guide/claude-code-retention.html). - -
MemPalace @@ -31,6 +13,14 @@ Local-first AI memory. Verbatim storage, pluggable backend, 96.6% R@5 raw on Lon
+> [!CAUTION] +> **Beware of impostor sites.** MemPalace has no other official websites. The **only** official sources are this **[GitHub repository](https://github.com/MemPalace/mempalace)**, the **[PyPI package](https://pypi.org/project/mempalace/)**, and the docs at **[mempalaceofficial.com](https://mempalaceofficial.com)**. Any other domain (including `.tech`, `.net`, or other `.com` variants) is an impostor and may distribute malware. Details and timeline: [docs/HISTORY.md](docs/HISTORY.md). + +> [!IMPORTANT] +> **Claude Code sessions expire in 30 days without auto-save hooks wired.** [Read this →](https://github.com/MemPalace/mempalace/discussions/1388) +> +> Need the shortest recovery/setup path? Use the [Claude Code retention setup checklist](https://mempalaceofficial.com/guide/claude-code-retention.html). + --- ## What it is @@ -79,6 +69,79 @@ python -m venv .venv && source .venv/bin/activate pip install mempalace ``` +### Docker + +A container image is also available for running the MCP server or the CLI +without a local Python toolchain. Everything persists under `/data` (palace, +config, and the cached embedding model), so mount a volume there. + +```bash +# Build the image (CPU; bundles the `extract` + `spellcheck` extras) +docker build -t mempalace . + +# MCP server over stdio — note the `-i` flag (JSON-RPC needs stdin) +docker run -i --rm -v mempalace-data:/data mempalace + +# Run any CLI command instead (mount the host directory you want to mine) +docker run --rm -v mempalace-data:/data -v /path/to/project:/work mempalace mine /work +docker run --rm -v mempalace-data:/data mempalace search "why GraphQL" +``` + +Wire it into an MCP client (e.g. Claude Code) as a stdio server: + +```json +{ + "mcpServers": { + "mempalace": { + "command": "docker", + "args": ["run", "-i", "--rm", "-v", "mempalace-data:/data", "mempalace"] + } + } +} +``` + +`docker compose run --rm mcp` works too (see `docker-compose.yml`). For +CUDA-accelerated embeddings, build the GPU variant with +`docker build -f Dockerfile.gpu -t mempalace:gpu .` and run it with +`--gpus all`. Customise the bundled extras at build time, e.g. +`docker build --build-arg EXTRAS="extract,spellcheck" -t mempalace .`. + +## Storage backends + +ChromaDB is the default. For the pluggable-backend preview, MemPalace also +ships `sqlite_exact` for local exact-vector correctness checks, and two opt-in +external service backends — `qdrant` (REST) and `pgvector` (Postgres). The two +external backends exercise the storage contract on different substrates (a +REST/dict store and a SQL/JSONB store), so it is not accidentally shaped around +one vendor. + +```bash +# local no-service backend +mempalace mine ~/projects/myapp --backend sqlite_exact + +# Qdrant backend, defaulting to http://localhost:6333 +MEMPALACE_QDRANT_URL=http://localhost:6333 \ + mempalace mine ~/projects/myapp --backend qdrant + +# Postgres + pgvector backend, defaulting to postgresql://localhost:5432/mempalace +# needs the optional driver: pip install mempalace[pgvector] +# and the `vector` extension available on the server +MEMPALACE_PGVECTOR_DSN=postgresql://localhost:5432/mempalace \ + mempalace mine ~/projects/myapp --backend pgvector +``` + +Qdrant can also be configured with `MEMPALACE_QDRANT_API_KEY`, +`MEMPALACE_QDRANT_NAMESPACE`, and `MEMPALACE_QDRANT_TIMEOUT`; pgvector with +`MEMPALACE_PGVECTOR_NAMESPACE`. Both external backends isolate tenants by +namespace (advertised via the `supports_namespace_isolation` capability) and +write a local marker (`qdrant_backend.json` / `pgvector_backend.json`) to guard +against silently opening a palace against the wrong server. + +When `MEMPALACE_QDRANT_URL` or `MEMPALACE_PGVECTOR_DSN` points anywhere other +than your own local or trusted self-hosted service, MemPalace will send and +store verbatim drawer text and metadata there. That is an explicit opt-in +backend choice, never the default. + ## Quickstart ```bash @@ -216,7 +279,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md). MIT — see [LICENSE](LICENSE). -[version-shield]: https://img.shields.io/badge/version-3.3.6-4dc9f6?style=flat-square&labelColor=0a0e14 +[version-shield]: https://img.shields.io/badge/version-3.4.0-4dc9f6?style=flat-square&labelColor=0a0e14 [release-link]: https://github.com/MemPalace/mempalace/releases [python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8 [python-link]: https://www.python.org/ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1a9f53d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,32 @@ +# MemPalace via Docker Compose. +# +# The MCP server speaks JSON-RPC over stdio, so the `mcp` service is meant to +# be driven interactively (`docker compose run`), not left running detached: +# +# docker compose build +# docker compose run --rm mcp # MCP server over stdio +# docker compose run --rm mcp cli search "GraphQL" # one-off CLI command +# +# The named volume `mempalace-data` is mounted at /data and holds the palace, +# config, and the cached embedding model across runs. + +services: + mcp: + build: + context: . + dockerfile: Dockerfile + image: mempalace:local + # stdio transport: keep STDIN open and allocate no TTY (raw JSON-RPC). + stdin_open: true + tty: false + volumes: + - mempalace-data:/data + environment: + # Everything defaults correctly from HOME=/data (palace -> + # /data/.mempalace/palace, config -> /data/.mempalace, model cache -> + # /data/.cache). Override here only for non-default locations, e.g.: + # - MEMPALACE_EMBEDDING_MODEL=embeddinggemma # multilingual (default: minilm) + # - MEMPALACE_PALACE_PATH=/data/custom/palace + +volumes: + mempalace-data: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..7c4601f --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env sh +# Flexible entrypoint: pick the MCP server or the CLI from the first argument. +# +# docker run -i mempalace -> MCP server over stdio (default) +# docker run -i mempalace mcp -> MCP server over stdio (explicit) +# docker run mempalace cli search "query" -> CLI passthrough (explicit) +# docker run mempalace search "query" -> CLI passthrough (implicit) +# +# `mcp` and `cli` are dispatch keywords; anything else is forwarded to the +# `mempalace` CLI verbatim so subcommands like `mine`, `search`, `wake-up` +# work without ceremony. +set -e + +case "${1:-mcp}" in + mcp) + if [ "$#" -gt 0 ]; then + shift + fi + exec mempalace-mcp "$@" + ;; + cli) + if [ "$#" -gt 0 ]; then + shift + fi + exec mempalace "$@" + ;; + *) + exec mempalace "$@" + ;; +esac diff --git a/docs/RELEASING.md b/docs/RELEASING.md index a88e9e3..afb1a56 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -29,3 +29,59 @@ If `pyproject.toml` has no match, **stop** — the entry point is missing and any fresh `pip install` will ship a broken plugin config. Investigate whether the release branch was cut before [#340](https://github.com/MemPalace/mempalace/pull/340) landed on `develop`. + +## Publishing to PyPI + +Releases publish automatically via the +[`publish.yml`](../.github/workflows/publish.yml) workflow, using PyPI +[Trusted Publishing](https://docs.pypi.org/trusted-publishers/) (OIDC). There +is **no API token** stored anywhere — GitHub mints a short-lived identity at +upload time. The workflow fires when a **GitHub Release is published**, builds +the sdist + wheel, and pauses for manual approval on the `pypi` environment +before uploading. + +### One-time setup (owners only) + +Done once per project; both steps require PyPI owner / GitHub admin rights. + +1. **PyPI trusted publisher** — on PyPI, go to **Manage project `mempalace` + → Publishing → Add a trusted publisher** and enter exactly: + + | Field | Value | + | --- | --- | + | Owner | `MemPalace` | + | Repository name | `mempalace` | + | Workflow filename | `publish.yml` | + | Environment name | `pypi` | + +2. **GitHub environment** — in the repo, **Settings → Environments → New + environment** named `pypi`. Add yourself (and any other release approvers) + under **Required reviewers**. This is the manual gate the workflow waits on + before the upload step runs. + +### Cutting a release + +1. Bump the version in **all five** sources on `develop` so `version-guard.yml` + stays green (it is the single source of truth at `mempalace/version.py`, + mirrored in `pyproject.toml`, `.claude-plugin/marketplace.json`, + `.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json`). +2. Land everything for the release on `develop`, then merge `develop → main`. + Releases publish **only from `main`** — the workflow refuses any tag whose + commit is not an ancestor of `main`. Don't commit the bump directly to + `main`: it bypasses branch protection and leaves `develop` behind. +3. Run the **entry-point alignment check** above. +4. On GitHub, **Releases → Draft a new release**: + - **Target:** `main` + - **Tag:** `vX.Y.Z` (must equal `mempalace/version.py`; the workflow and + `version-guard.yml` both reject a mismatch) + - Write the release notes, then **Publish release**. +5. The `publish.yml` run validates the tag (on `main`, matches the manifest), + builds, and then waits for approval on the `pypi` environment. Approve it to + upload to PyPI. Watch the run land the new version on + . + +To stage a release candidate without shipping to end users, tag a semver +pre-release (`vX.Y.Z-rc1`) — `version-guard.yml` skips the strict manifest +match for pre-release tags. (Note: a published GitHub Release still triggers +`publish.yml`; use a **draft** release, or a plain pushed tag, for dry runs you +don't want uploaded.) diff --git a/docs/recovery/wing-name-migration.md b/docs/recovery/wing-name-migration.md new file mode 100644 index 0000000..e63a34f --- /dev/null +++ b/docs/recovery/wing-name-migration.md @@ -0,0 +1,63 @@ +# Recovery: legacy wing names split after the normalization change + +**Companion to #1675.** `normalize_wing_name` now strips leading and trailing +separators, so a path-encoded project dir like `-home-user-proj` derives the +wing `home_user_proj` instead of `_home_user_proj`. Palaces mined before that +change filed drawers under the old, separator-padded name. New mining and diary +writes land on the new name, so the two no longer meet — the history is +**split**, not lost. `mempalace migrate-wings` re-unites them. + +## Symptom + +After upgrading, a project that used to surface its memories returns less than +expected, and `mempalace status` shows two wings for one project — e.g. both +`_home_user_proj` (old drawers) and `home_user_proj` (newly mined). MCP writes +to the padded wing may also have been rejected, since `sanitize_name` does not +accept a leading underscore. + +## Recovery + +Preview first — this never modifies anything: + +```bash +mempalace migrate-wings --dry-run +mempalace migrate-wings --dry-run --palace /path/to/palace +``` + +The plan lists each rename and flags collisions that will **merge** into an +existing wing: + +``` + Wing-name migration plan: + '_home_user_proj' -> 'home_user_proj': 1284 drawer(s), 96 closet(s) (MERGE into existing wing) +``` + +Apply it: + +```bash +mempalace migrate-wings # prompts for confirmation +mempalace migrate-wings --yes # no prompt +``` + +## What it does + +- Re-keys the `wing` **metadata field** on drawers and closets to the normalized + form, merging collisions into the existing wing. +- Re-keys the `topics_by_wing` registry (merging topic lists on collision). + +## What it leaves alone + +- **Drawer/closet IDs** are untouched. The wing in an ID (`drawer__…`) is + an opaque prefix that is never decoded back into a wing, so leaving it keeps + closet `→drawer_id` pointers valid and lets future mining still skip + already-mined files (no duplicates). The verbatim drawer content is never + read or rewritten. +- **Tunnels** already normalize wing names at read time, so they resolve under + the new name without a rewrite. + +## Notes + +- **Idempotent.** A second run reports "nothing to migrate" and changes nothing. +- **Backend-agnostic.** Works on any configured storage backend. +- Run it once per palace after upgrading. New palaces are born with normalized + wing names and never need it. diff --git a/mempalace/backends/__init__.py b/mempalace/backends/__init__.py index 4e8fdce..7432cbe 100644 --- a/mempalace/backends/__init__.py +++ b/mempalace/backends/__init__.py @@ -17,6 +17,7 @@ Public surface: from .base import ( BackendClosedError, BackendError, + BackendMismatchError, BaseBackend, BaseCollection, CollectionNotInitializedError, @@ -24,14 +25,24 @@ from .base import ( EmbedderIdentityMismatchError, GetResult, HealthStatus, + LexicalHit, + LexicalResult, + MaintenanceResult, PalaceNotFoundError, PalaceRef, QueryResult, + UnsupportedCapabilityError, UnsupportedFilterError, + UnsupportedMaintenanceKindError, ) from .chroma import ChromaBackend, ChromaCollection +from .pgvector import PgVectorBackend, PgVectorCollection +from .qdrant import QdrantBackend, QdrantCollection +from .sqlite_exact import SQLiteExactBackend, SQLiteExactCollection from .registry import ( available_backends, + detect_backend_for_path, + detect_backends_for_path, get_backend, get_backend_class, register, @@ -43,6 +54,7 @@ from .registry import ( __all__ = [ "BackendClosedError", "BackendError", + "BackendMismatchError", "BaseBackend", "BaseCollection", "ChromaBackend", @@ -52,11 +64,24 @@ __all__ = [ "EmbedderIdentityMismatchError", "GetResult", "HealthStatus", + "LexicalHit", + "LexicalResult", + "MaintenanceResult", "PalaceNotFoundError", "PalaceRef", + "PgVectorBackend", + "PgVectorCollection", + "QdrantBackend", + "QdrantCollection", "QueryResult", + "SQLiteExactBackend", + "SQLiteExactCollection", + "UnsupportedCapabilityError", "UnsupportedFilterError", + "UnsupportedMaintenanceKindError", "available_backends", + "detect_backend_for_path", + "detect_backends_for_path", "get_backend", "get_backend_class", "register", diff --git a/mempalace/backends/_sidecar.py b/mempalace/backends/_sidecar.py new file mode 100644 index 0000000..a0485da --- /dev/null +++ b/mempalace/backends/_sidecar.py @@ -0,0 +1,71 @@ +"""Shared embedder-identity sidecar (RFC 001). + +A small JSON file in the palace directory, keyed by collection name, recording +the embedder identity (``model_name`` / ``dimension``). It is deliberately +*separate* from a backend's mismatch marker: a marker's presence signals +"palace initialized" (reads raise ``CollectionNotInitializedError`` when the +marker exists but the store doesn't), so recording identity at first empty open +must not create one. The sidecar is unguarded, so a brand-new palace can record +identity immediately — the same approach the chroma backend uses. +""" + +import json +import os +from typing import Optional + +EMBEDDER_SIDECAR_FILENAME = "mempalace_embedder.json" + + +def read_embedder_sidecar(path: Optional[str], collection_name: Optional[str]): + """Return the recorded :class:`EmbedderIdentity` for ``collection_name``, or None. + + Robust to a missing, unreadable, or malformed (non-dict) sidecar — any of + those degrade to ``None`` (the ``unknown`` state) rather than raising. + """ + from .base import EmbedderIdentity + + if not path or not collection_name or not os.path.isfile(path): + return None + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(data, dict): + return None + entry = data.get(collection_name) + if not isinstance(entry, dict) or not entry.get("model_name"): + return None + return EmbedderIdentity( + model_name=str(entry["model_name"]), + dimension=int(entry.get("dimension") or 0), + ) + + +def write_embedder_sidecar(path: Optional[str], collection_name: Optional[str], identity) -> None: + """Record ``identity`` for ``collection_name`` in the sidecar, creating it if needed. + + No-ops for a missing path, missing collection name, or a nameless identity. + Preserves other collections' entries; never raises on I/O failure. + """ + if not path or not collection_name or not identity or not getattr(identity, "model_name", ""): + return + data: dict = {} + if os.path.isfile(path): + try: + with open(path, encoding="utf-8") as f: + loaded = json.load(f) + if isinstance(loaded, dict): + data = loaded + except (OSError, json.JSONDecodeError): + data = {} + data[collection_name] = { + "model_name": str(identity.model_name), + "dimension": int(identity.dimension or 0), + } + try: + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + os.chmod(path, 0o600) + except (OSError, NotImplementedError): + pass diff --git a/mempalace/backends/base.py b/mempalace/backends/base.py index a32f0a8..0c643b9 100644 --- a/mempalace/backends/base.py +++ b/mempalace/backends/base.py @@ -14,8 +14,8 @@ conformance suite land in follow-up PRs. """ from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import ClassVar, Optional +from dataclasses import dataclass, field +from typing import ClassVar, Optional, Protocol, runtime_checkable # --------------------------------------------------------------------------- @@ -58,6 +58,23 @@ class UnsupportedFilterError(BackendError): """ +class UnsupportedCapabilityError(BackendError): + """Raised when a backend does not implement an optional capability.""" + + +class UnsupportedMaintenanceKindError(BackendError): + """Raised when ``run_maintenance(kind)`` is called with an unadvertised kind. + + A backend MUST advertise a kind in ``maintenance_kinds`` before it accepts + it (RFC 001). Advertising a kind it does not implement is a conformance + failure; a kind it has no analogue for MUST be omitted, not no-op'd. + """ + + +class BackendMismatchError(BackendError): + """Raised when a selected backend does not match existing palace artifacts.""" + + class DimensionMismatchError(BackendError): """Raised when the embedding dimension on write does not match the collection.""" @@ -66,6 +83,15 @@ class EmbedderIdentityMismatchError(BackendError): """Raised when the stored embedder model name differs from the current one.""" +class EmbedderIdentityUnknownWarning(UserWarning): + """Emitted on first open of a collection with no recorded embedder identity. + + Legacy palaces created before identity tracking carry no model name. Per + RFC 001 the right behavior is warn-not-fail: the identity is recorded on + the next write and subsequent opens become strict. + """ + + # --------------------------------------------------------------------------- # Value objects # --------------------------------------------------------------------------- @@ -78,6 +104,29 @@ class PalaceRef: ``id`` is always present and is the key backends use to cache handles. ``local_path`` is populated for filesystem-rooted palaces. ``namespace`` is used by server-mode backends for tenant / prefix routing. + + Isolation contract (RFC 001 §2.1, conformance: ``tests/test_backend_conformance.py``) + ----------------------------------------------------------------------------------- + ``id`` is the *required* isolation key. Within a single backend instance: + + A record written for one ``PalaceRef.id`` MUST NOT be returned, + modified, or deleted by an operation issued for a different + ``PalaceRef.id``. Cross-palace access is a spec violation. + + ``namespace`` is *additional* partitioning, honored only by backends that + advertise the ``supports_namespace_isolation`` capability. For those + backends the same guarantee extends to namespaces: + + A record written under one ``namespace`` MUST NOT be returned, + modified, or deleted by an operation issued under a different + ``namespace`` within the same backend instance. Cross-namespace + access is a spec violation. + + Backends that do not advertise ``supports_namespace_isolation`` (e.g. + ``sqlite_exact``, whose isolation is the on-disk path alone) MAY ignore + ``namespace`` entirely; callers MUST NOT rely on it for tenant isolation + on such backends. Any conforming backend can self-check both guarantees by + running the shared assertions in ``tests/_backend_conformance.py``. """ id: str @@ -85,6 +134,114 @@ class PalaceRef: namespace: Optional[str] = None +@dataclass(frozen=True) +class EmbedderIdentity: + """Identity of the embedder that produced a collection's vectors (RFC 001). + + ``model_name`` is the stable identity persisted alongside a collection and + checked on subsequent opens. ``dimension`` is the vector width. A + ``dimension`` of ``0`` means *unknown / not probed* — comparisons treat it + as "no dimension signal" rather than a real zero-width vector, so a cheap + read-path check can compare model names without loading the model. + """ + + model_name: str + dimension: int = 0 + + +@dataclass(frozen=True) +class MaintenanceResult: + """Observable outcome of ``run_maintenance(kind)`` (RFC 001). + + Maintenance is *not* fire-and-forget: a backend MUST serialize concurrent + same-kind runs and report the outcome so a caller can learn it must not + re-trigger. ``status`` is one of: + + * ``"ran"`` — this call performed the maintenance. + * ``"already_running"`` — another caller holds the work; this call did + nothing and the caller MUST NOT re-trigger (the production index-build + wedge: concurrent writers each issuing the build stacked exclusive locks). + * ``"noop"`` — nothing needed doing (e.g. the index already exists). + + ``stats`` is free-form per kind (rows analyzed, bytes reclaimed, index + build time) for benchmark/operator reporting. + """ + + kind: str + status: str + stats: dict = field(default_factory=dict) + + +@runtime_checkable +class Embedder(Protocol): + """Minimal embedder contract (RFC 001, normative for identity checking). + + The fuller embedder RFC (batching/async/pooling) is additive; identity + enforcement depends only on these three members. + """ + + model_name: str + dimension: int + + def embed(self, texts: list[str]) -> list[list[float]]: ... + + +def check_embedder_identity( + stored: Optional[EmbedderIdentity], + current: Optional[EmbedderIdentity], + *, + force_model_swap: bool = False, +) -> str: + """Three-state embedder-identity check (RFC 001). + + Returns the resolved state and raises on a hard, unforced conflict: + + * ``"unknown"`` — no identity recorded yet (legacy collection), or the + current embedder is nameless. The caller warns and records on write. + * ``"known_match"`` — stored name (and dimension, when both known) equal + the current embedder. Proceed normally. + * ``"known_mismatch"`` — names or dimensions differ. Without + ``force_model_swap`` this raises (:class:`EmbedderIdentityMismatchError` + for a model swap, :class:`DimensionMismatchError` for a width change, + which is checked first because mismatched vectors are physically + unusable). With ``force_model_swap`` it returns the state so the caller + can re-record the identity and log the swap. + + A ``dimension`` of ``0`` on either side means "unknown" and is skipped, so + a model-name-only check (cheap read path) still works. + """ + if current is None or not current.model_name: + return "unknown" + if stored is None: + return "unknown" + + dim_conflict = bool(stored.dimension and current.dimension) and ( + stored.dimension != current.dimension + ) + name_conflict = stored.model_name != current.model_name + + if not dim_conflict and not name_conflict: + return "known_match" + + if force_model_swap: + return "known_mismatch" + + if dim_conflict: + raise DimensionMismatchError( + f"collection was built with a {stored.dimension}-dim embedder " + f"({stored.model_name!r}) but the current embedder is " + f"{current.dimension}-dim ({current.model_name!r}); the stored " + "vectors are incompatible. Re-embed the palace to switch models." + ) + raise EmbedderIdentityMismatchError( + f"collection was built with embedder {stored.model_name!r} but the " + f"current embedder is {current.model_name!r}. Searching across a model " + "swap silently degrades recall. Re-embed the palace, or run " + "`mempalace palace set-embedder --model --force` to record the " + "new identity if you know the vectors are compatible." + ) + + @dataclass(frozen=True) class HealthStatus: ok: bool @@ -177,6 +334,23 @@ class GetResult(_DictCompatMixin): return cls(ids=[], documents=[], metadatas=[], embeddings=None) +@dataclass(frozen=True) +class LexicalHit: + """One hit from backend lexical candidate search.""" + + id: str + document: str + metadata: dict + score: float + + +@dataclass(frozen=True) +class LexicalResult: + """Typed return from ``BaseCollection.lexical_search``.""" + + hits: list[LexicalHit] + + # --------------------------------------------------------------------------- # Collection contract # --------------------------------------------------------------------------- @@ -253,6 +427,77 @@ class BaseCollection(ABC): def health(self) -> HealthStatus: return HealthStatus.healthy() + @property + def distance_metric(self) -> str: + """The space this collection's ``distances`` are reported in. + + Defaults to the owning backend's declared metric (cosine for all + in-tree backends). Collections that can vary per-collection — e.g. a + legacy Chroma palace built without ``hnsw:space=cosine`` — override + this to report their actual space so core ranking converts correctly. + """ + return "cosine" + + def get_stored_embedder_identity(self) -> Optional[EmbedderIdentity]: + """Return the embedder identity recorded for this collection, if any. + + Returns ``None`` when nothing is recorded — a legacy collection, or a + backend that does not yet persist identity. Core treats ``None`` as the + ``unknown`` state (warn, do not fail). Backends override this and + :meth:`set_embedder_identity` against their own metadata store. + """ + return None + + def set_embedder_identity(self, identity: EmbedderIdentity) -> None: + """Persist this collection's embedder identity. Default: no-op. + + A backend without an identity slot inherits the no-op default and so + stays permanently ``unknown`` (safe — it simply never enforces). The + enforcement choke point calls this when recording on first write or + on an explicit, forced model swap. + """ + return None + + def effective_embedder_identity(self) -> Optional[EmbedderIdentity]: + """The identity of the embedder this collection actually uses. + + For ``server_embedder`` backends that ignore the injected embedder, + this reports the server-side embedder so the same identity rules apply + (RFC 001). Defaults to ``None`` — the collection is embedded by the + injected/core embedder, and the caller supplies the current identity. + """ + return None + + def maintenance_state(self) -> dict: + """Return a structured snapshot of this collection's maintenance state. + + Free-form per backend (e.g. row count, whether a vector index exists, + last-analyze age). Used by benchmark harnesses to record state + alongside each latency/recall measurement so an un-analyzed store is + not compared against a settled one (RFC 001). Defaults to empty. + """ + return {} + + def run_maintenance(self, kind: str) -> "MaintenanceResult": + """Run a maintenance ``kind`` and return an observable result (RFC 001). + + Backends advertise supported kinds in ``BaseBackend.maintenance_kinds`` + and override this. The default supports nothing, so every kind raises + :class:`UnsupportedMaintenanceKindError`. Implementations MUST serialize + concurrent same-kind runs and report ``already_running`` rather than + stacking the work. + """ + raise UnsupportedMaintenanceKindError(f"backend does not support maintenance kind {kind!r}") + + def lexical_search( + self, + *, + query: str, + n_results: int = 10, + where: Optional[dict] = None, + ) -> LexicalResult: + raise UnsupportedCapabilityError("backend does not support lexical_search") + def update( self, *, @@ -311,11 +556,33 @@ class BaseBackend(ABC): Instances are lightweight on construction — no I/O, no network. All connection work is deferred to ``get_collection``. Instances are thread- safe for concurrent ``get_collection`` calls across different palaces. + + Every backend MUST satisfy the per-``PalaceRef.id`` isolation guarantee in + :class:`PalaceRef`. Backends that additionally isolate by + ``PalaceRef.namespace`` (multi-tenant / hosted deployments) MUST advertise + the ``supports_namespace_isolation`` capability token; doing so is a + promise to satisfy the cross-namespace guarantee and to pass the namespace + arm of the conformance suite. Backends without the token MAY ignore + ``namespace``. """ name: ClassVar[str] spec_version: ClassVar[str] = "1.0" capabilities: ClassVar[frozenset[str]] = frozenset() + #: The space ``query()`` reports ``distances`` in (RFC 001 §2.1). + #: One of ``"cosine"`` | ``"l2"`` | ``"ip"``. The contract for the + #: ``distances`` field is *lower = closer* regardless of metric; core + #: search converts distance→similarity off this declaration rather than + #: assuming cosine. All in-tree backends are cosine today. + distance_metric: ClassVar[str] = "cosine" + #: Maintenance kinds this backend implements (RFC 001). Reserved names: + #: ``"analyze"`` (refresh planner/query statistics), ``"compact"`` (reclaim + #: space, rewrite storage), ``"reindex"`` (build/rebuild secondary indexes). + #: A backend with no analogue for a kind MUST omit it rather than declare a + #: no-op, so a benchmark harness can trust the set. Backends MAY add their + #: own kinds. ``run_maintenance`` raises ``UnsupportedMaintenanceKindError`` + #: for anything not listed here. + maintenance_kinds: ClassVar[frozenset[str]] = frozenset() @abstractmethod def get_collection( diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index 0730a7b..06b71ba 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -2,10 +2,14 @@ import contextlib import datetime as _dt +import json import logging +import math import os import pickle +import re import sqlite3 +from collections import defaultdict from numbers import Integral from pathlib import Path from typing import Any, Optional @@ -13,12 +17,15 @@ from typing import Any, Optional import chromadb from chromadb.errors import NotFoundError as _ChromaNotFoundError +from ._sidecar import EMBEDDER_SIDECAR_FILENAME, read_embedder_sidecar, write_embedder_sidecar from .base import ( BaseBackend, BaseCollection, CollectionNotInitializedError, GetResult, HealthStatus, + LexicalHit, + LexicalResult, PalaceNotFoundError, PalaceRef, QueryResult, @@ -32,6 +39,7 @@ logger = logging.getLogger(__name__) _REQUIRED_OPERATORS = frozenset({"$eq", "$ne", "$in", "$nin", "$and", "$or", "$contains"}) _OPTIONAL_OPERATORS = frozenset({"$gt", "$gte", "$lt", "$lte"}) _SUPPORTED_OPERATORS = _REQUIRED_OPERATORS | _OPTIONAL_OPERATORS +_TOKEN_RE = re.compile(r"\w{2,}", re.UNICODE) # A healthy HNSW payload should keep link_lists.bin proportional to # data_level0.bin. When link_lists.bin grows orders of magnitude larger than @@ -104,36 +112,38 @@ def _hnsw_payload_appears_sane(seg_dir: str) -> bool: return ratio is None or ratio <= _HNSW_LINK_TO_DATA_MAX_RATIO -# HNSW tuning to prevent link_lists.bin bloat on large mines (#344). +# HNSW batch/sync thresholds applied at collection creation. # -# With default params (batch_size=100, sync_threshold=1000, initial capacity -# 1000), inserting tens of thousands of drawers triggers ~30 index resizes -# and hundreds of persistDirty() calls. persistDirty uses relative seek -# positioning in link_lists.bin; accumulated seek drift across resize cycles -# causes the OS to extend the sparse file with zero-filled regions, each -# cycle compounding the next. Result: link_lists.bin grows into hundreds of -# GB sparse, after which `status`/`search`/`repair` segfault. +# chromadb's Rust HNSW segment writes index_metadata.pickle and +# link_lists.bin only when internal counters cross both thresholds +# (batch_size gates _apply_batch; sync_threshold gates _persist). +# Records below both thresholds stay in memory and are lost on exit. # -# Setting large batch and sync thresholds at collection creation defers -# persistence until a single large batch completes, breaking the resize+ -# persist feedback loop. Empirically validated on a 39,792-drawer rebuild -# (palace 376 MB, link_lists.bin 0 bytes, no segfault) in 2026-04. +# Previously 50k/50k to work around link_lists.bin sparse-file bloat +# in pre-1.5.x Python chromadb (#344). chromadb >=1.5.4 Rust bindings +# (the minimum mempalace supports) do not exhibit that bloat; verified +# at batch_size=2 with 20k records: link_lists.bin = 171 KB, no +# sparse-file inflation. # -# Note: chromadb 1.5.x exposes a `collection.modify(configuration={"hnsw": -# {"batch_size": ..., "sync_threshold": ...}})` retrofit path for already- -# created collections (`UpdateHNSWConfiguration` in chromadb's API), but -# this PR doesn't pursue that — once link_lists.bin has bloated, the index -# is already corrupt and the only known recovery is a fresh mine. +# The 50k guard caused #1579: mines under 50k drawers never triggered +# _persist(), leaving index_metadata.pickle absent and link_lists.bin +# empty. quarantine_stale_hnsw then renamed the segment on every cold +# open after a 300s mtime gap, accumulating .drift-* directories. +# +# Lowered to 2 (empirical Rust-side minimum for chromadb >=1.5.4; the +# Rust bindings reject 1 with InvalidArgumentError) so any mine of 2+ +# drawers triggers a natural persist. Existing palaces created under +# the old 50k guard keep those thresholds in their collection metadata +# until the user runs repair --mode from-sqlite --archive-existing. _HNSW_BLOAT_GUARD = { - "hnsw:batch_size": 50_000, - "hnsw:sync_threshold": 50_000, + "hnsw:batch_size": 2, + "hnsw:sync_threshold": 2, } -# Missing index_metadata.pickle is normal only while a segment is still fresh -# or effectively empty. Once data_level0.bin has non-trivial payload, a -# missing metadata pickle means the segment was interrupted after writing HNSW -# data but before writing its metadata. Letting Chroma open that shape can -# segfault or hang in native HNSW code. +# Below this size, data_level0.bin is too small for a meaningful HNSW graph. +# Used by _hnsw_link_lists_is_usable_for_payload (empty link_lists is fine +# when data is trivially small) and _missing_dimensionality_appears_recoverable +# (don't attempt recovery on segments with negligible data). _HNSW_MISSING_METADATA_DATA_FLOOR = 1024 @@ -158,6 +168,127 @@ def _validate_where(where: Optional[dict]) -> None: stack.extend(x for x in v if isinstance(x, dict)) +def _tokenize(text: str) -> list[str]: + if not text: + return [] + return _TOKEN_RE.findall(text.lower()) + + +def _bm25_scores( + query: str, + documents: list[str], + k1: float = 1.5, + b: float = 0.75, +) -> list[float]: + query_terms = set(_tokenize(query)) + n_docs = len(documents) + if not query_terms or n_docs == 0: + return [0.0] * n_docs + + tokenized = [_tokenize(doc) for doc in documents] + doc_lens = [len(toks) for toks in tokenized] + if not any(doc_lens): + return [0.0] * n_docs + avgdl = sum(doc_lens) / n_docs or 1.0 + + df = {term: 0 for term in query_terms} + for toks in tokenized: + for term in set(toks) & query_terms: + df[term] += 1 + + idf = { + term: math.log((n_docs - df[term] + 0.5) / (df[term] + 0.5) + 1.0) for term in query_terms + } + + scores = [] + for toks, dl in zip(tokenized, doc_lens): + if dl == 0: + scores.append(0.0) + continue + tf: dict[str, int] = {} + for token in toks: + if token in query_terms: + tf[token] = tf.get(token, 0) + 1 + score = 0.0 + for term, freq in tf.items(): + num = freq * (k1 + 1) + den = freq + k1 * (1 - b + b * dl / avgdl) + score += idf[term] * num / den + scores.append(score) + return scores + + +def _coerce_metadata_value(value: Any) -> Any: + if isinstance(value, bool): + return int(value) + return value + + +def _compare_metadata(actual: Any, op: str, expected: Any) -> bool: + actual = _coerce_metadata_value(actual) + expected = _coerce_metadata_value(expected) + if op == "$eq": + return actual == expected + if op == "$ne": + return actual != expected + if op == "$in": + return actual in (expected or []) + if op == "$nin": + return actual not in (expected or []) + if op == "$contains": + return str(expected) in str(actual or "") + try: + if op == "$gt": + return actual > expected + if op == "$gte": + return actual >= expected + if op == "$lt": + return actual < expected + if op == "$lte": + return actual <= expected + except TypeError: + return False + raise UnsupportedFilterError(f"operator {op!r} not supported by chroma backend") + + +def _matches_where(meta: dict, where: Optional[dict]) -> bool: + if not where: + return True + if not isinstance(where, dict): + return False + for key, expected in where.items(): + if key == "$and": + if not all(_matches_where(meta, clause) for clause in expected or []): + return False + continue + if key == "$or": + if not any(_matches_where(meta, clause) for clause in expected or []): + return False + continue + if key.startswith("$"): + raise UnsupportedFilterError(f"operator {key!r} not supported by chroma backend") + actual = meta.get(key) + if isinstance(expected, dict): + for op, operand in expected.items(): + if not _compare_metadata(actual, op, operand): + return False + elif actual != expected: + return False + return True + + +def _metadata_cell_value(sval, ival, fval, bval): + if sval is not None: + return sval + if ival is not None: + return ival + if fval is not None: + return fval + if bval is not None: + return bool(bval) + return None + + def _segment_appears_healthy(seg_dir: str) -> bool: """Return True if a chromadb HNSW segment dir looks intact. @@ -168,11 +299,14 @@ def _segment_appears_healthy(seg_dir: str) -> bool: ``0x2e`` (the protocol/terminator byte sequence chromadb serializes with). - Missing metadata is healthy only while the segment still looks fresh or - empty. If ``data_level0.bin`` already has non-trivial payload but - ``index_metadata.pickle`` is missing, the segment is partially flushed: - Chroma wrote vector data without the metadata it needs to reopen the - HNSW reader safely. + When metadata is missing, the segment is either *never-persisted* + (sub-threshold: fewer records than ``batch_size``, so chromadb never + triggered ``_persist()``) or *partially flushed* (persist started but + crashed). The two are distinguished by ``link_lists.bin``: chromadb + writes link data during persist, so an empty/absent ``link_lists.bin`` + together with absent metadata means no persist was ever attempted. + Note: ``data_level0.bin`` is pre-allocated at index creation and its + size does not indicate actual record count. Deliberately format-sniffs only; never deserializes. Deserialization can execute arbitrary code, and the byte-sniff is sufficient to @@ -185,28 +319,23 @@ def _segment_appears_healthy(seg_dir: str) -> bool: files and quarantine_stale_hnsw would conservatively rename them out of the way. """ + meta_path = os.path.join(seg_dir, "index_metadata.pickle") + + if not os.path.isfile(meta_path): + link_path = os.path.join(seg_dir, "link_lists.bin") + try: + link_has_data = os.path.isfile(link_path) and os.path.getsize(link_path) > 0 + except OSError: + return False + # Both absent → sub-threshold, never persisted. + # link_lists written but metadata not → interrupted persist. + return not link_has_data + if not _hnsw_payload_appears_sane(seg_dir): return False - meta_path = os.path.join(seg_dir, "index_metadata.pickle") - if not os.path.isfile(meta_path): - data_path = os.path.join(seg_dir, "data_level0.bin") - try: - if ( - os.path.isfile(data_path) - and os.path.getsize(data_path) > _HNSW_MISSING_METADATA_DATA_FLOOR - ): - return False - except OSError: - return False - - # No metadata and no meaningful vector payload yet: fresh/empty segment. - return True - try: size = os.path.getsize(meta_path) - # A real chromadb metadata file is at least tens of bytes; a - # smaller-than-floor file is almost certainly truncated. if size < 16: return False with open(meta_path, "rb") as f: @@ -466,14 +595,15 @@ def _hnsw_element_count(palace_path: str, segment_id: str) -> Optional[int]: # read the collection metadata (older palaces missing the row, sqlite # unreadable). 2000 = 2 × chromadb's default sync_threshold of 1000. # -# Why dynamic: PR #1191 set ``hnsw:sync_threshold = 50_000`` to prevent -# index bloat, which means flush-lag can grow up to 50K naturally. A -# fixed 2000 floor would flag every actively-written palace as DIVERGED -# the moment its queue exceeded 10% of sqlite_count, even though chromadb -# is behaving correctly. The floor must scale with sync_threshold to -# distinguish real corruption (#1222 was 176 613 missing of 192 997 — -# orders of magnitude past 2 × any reasonable sync_threshold) from -# expected steady-state lag. +# Why dynamic: legacy palaces may still carry ``sync_threshold = 50_000`` +# (the pre-#1579 guard), so flush-lag can grow up to 50K on those palaces. +# New palaces use sync_threshold=2 (#1579) and flush almost immediately. +# A fixed 2000 floor would flag actively-written legacy palaces as +# DIVERGED the moment their queue exceeded 10% of sqlite_count, even +# though chromadb is behaving correctly. The floor must scale with the +# per-collection sync_threshold to distinguish real corruption (#1222 was +# 176 613 missing of 192 997, orders of magnitude past any reasonable +# sync_threshold) from expected steady-state lag. _HNSW_DIVERGENCE_FALLBACK_FLOOR = 2000 _HNSW_DIVERGENCE_FRACTION = 0.10 @@ -635,6 +765,93 @@ def _sqlite_embedding_count(palace_path: str, collection_name: str) -> Optional[ return None +def _sqlite_wing_room_counts( + palace_path: str, collection_name: str +) -> Optional[tuple[int, dict[str, dict[str, int]]]]: + """Tally drawers by wing/room straight from ``chroma.sqlite3``. + + Returns ``(total, {wing: {room: count}})`` or ``None`` when the read + cannot be trusted — missing DB file, the collection has not been + bootstrapped, or any sqlite error (including a sustained writer lock). + ``None`` signals the caller to fall back to the ChromaDB client path + (which also emits the right state-specific guidance for absent/empty + palaces). + + The point of reading sqlite directly is to count drawers **without opening + the collection**, because opening it cold-loads the HNSW vector index. On + large palaces that load costs tens of seconds of CPU per call — a steep, + pointless tax for an inspection command that only needs metadata the + relational tables already hold (#1681). Wings/rooms live in plain + ``embedding_metadata`` rows, joined to ``embeddings`` on the + ``(id, key)`` primary key, so the tally is a bounded scan of the metadata + segment: sub-second warm, a few seconds cold on a multi-GB DB — versus the + ~60s the vector-index load costs. + + Sibling readers that count the same way: :func:`_sqlite_embedding_count` + (total only) and ``mcp_server._tool_status_via_sqlite`` (independent + wing/room histograms for the #1222 fallback). This one cross-tabulates + wing→room to match the ChromaDB ``status()`` output shape. + + Notes: + - ``busy_timeout`` lets a transient checkpoint lock resolve instead of + instantly demoting to the slow HNSW path; a *sustained* lock still + raises and falls back (slow but correct). + - ``s.scope = 'METADATA'`` makes the single-segment join explicit so a + future ChromaDB that also stored per-vector-segment rows could not + silently double every count. + - ``COALESCE`` over ``string_value``/``int_value``/``float_value`` matches + the ChromaDB path, which surfaces a numeric wing/room natively rather + than dropping it to ``"?"``. + """ + db_path = os.path.join(palace_path, "chroma.sqlite3") + if not os.path.isfile(db_path): + return None + try: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + try: + # Wait out a transient writer/checkpoint lock rather than falling + # straight back to the expensive vector-index path (#1681). + conn.execute("PRAGMA busy_timeout = 3000") + # Distinguish "collection never bootstrapped" (-> None, so the + # caller can show the 'initialized but empty' message) from + # "collection exists with zero drawers" (-> a real 0 tally). + if ( + conn.execute( + "SELECT 1 FROM collections WHERE name = ?", (collection_name,) + ).fetchone() + is None + ): + return None + rows = conn.execute( + """ + SELECT COALESCE(wm.string_value, CAST(wm.int_value AS TEXT), + CAST(wm.float_value AS TEXT), '?') AS wing, + COALESCE(rm.string_value, CAST(rm.int_value AS TEXT), + CAST(rm.float_value AS TEXT), '?') AS room, + COUNT(*) AS n + FROM embeddings e + JOIN segments s ON e.segment_id = s.id AND s.scope = 'METADATA' + JOIN collections c ON s.collection = c.id + LEFT JOIN embedding_metadata wm ON wm.id = e.id AND wm.key = 'wing' + LEFT JOIN embedding_metadata rm ON rm.id = e.id AND rm.key = 'room' + WHERE c.name = ? + GROUP BY wing, room + """, + (collection_name,), + ).fetchall() + finally: + conn.close() + except sqlite3.Error: + return None + + total = 0 + wing_rooms: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + for wing, room, n in rows: + wing_rooms[wing][room] += int(n) + total += int(n) + return total, wing_rooms + + def _pin_hnsw_threads(collection) -> None: """Best-effort retrofit: pin ``hnsw:num_threads=1`` on an existing collection. @@ -665,6 +882,7 @@ def _pin_hnsw_threads(collection) -> None: _BLOB_FIX_MARKER = ".blob_seq_ids_migrated" +_COLLECTION_TYPE_MARKER = ".collection_type_fixed" def _valid_dimensionality(value: object) -> bool: @@ -708,7 +926,12 @@ def _missing_dimensionality_appears_recoverable( return False label_count = len(id_to_label) - if int(total) != label_count or len(label_to_id) != label_count: + # total_elements_added is monotonic across every add, while id_to_label and + # label_to_id hold only live elements, so a segment that has had deletions + # carries total_elements_added > label_count. Require >= (not ==), otherwise + # every post-deletion dim-None segment is wrongly quarantined (#1710); the + # label-map size and bijection checks still reject inconsistent label maps. + if int(total) < label_count or len(label_to_id) != label_count: return False try: return all(label_to_id.get(label) == item_id for item_id, label in id_to_label.items()) @@ -883,6 +1106,69 @@ def _fix_blob_seq_ids(palace_path: str) -> None: logger.exception("Could not write migration marker %s", marker) +def _fix_missing_collection_type(palace_path: str) -> None: + """Add ``_type`` to ``collections.config_json_str`` where absent. + + chromadb <= 1.5.8 writes ``config_json_str = '{}'`` (empty JSON) when + creating collections. chromadb 1.5.9 switched from the permissive + ``load_collection_configuration_from_json_str`` to + ``CollectionConfigurationInternal.from_json`` which requires a ``_type`` + key — its absence raises ``KeyError: '_type'`` on palace open. + + This migration adds the missing marker so both old and new chromadb + versions can load the collection. The value + ``"CollectionConfigurationInternal"`` matches what ``to_json()`` writes + for freshly-created collections. + + Same lifecycle constraints as :func:`_fix_blob_seq_ids`: must run + BEFORE ``PersistentClient`` is created. + """ + db_path = os.path.join(palace_path, "chroma.sqlite3") + if not os.path.isfile(db_path): + return + marker = os.path.join(palace_path, _COLLECTION_TYPE_MARKER) + if os.path.isfile(marker): + return + conn = sqlite3.connect(db_path) + try: + try: + rows = conn.execute("SELECT id, config_json_str FROM collections").fetchall() + except sqlite3.OperationalError: + return + updates = [] + for coll_id, config_str in rows: + if not config_str: + config_str = "{}" + try: + config = json.loads(config_str) + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(config, dict): + continue + if "_type" not in config: + config["_type"] = "CollectionConfigurationInternal" + updates.append((json.dumps(config), coll_id)) + if updates: + conn.executemany( + "UPDATE collections SET config_json_str = ? WHERE id = ?", + updates, + ) + conn.commit() + logger.info( + "Fixed %d collection(s) missing _type in config_json_str", + len(updates), + ) + except Exception: + logger.exception("Could not fix collection config_json_str in %s", db_path) + return + finally: + conn.close() + try: + Path(marker).touch() + except OSError: + logger.exception("Could not write migration marker %s", marker) + + # --------------------------------------------------------------------------- # Collection adapter # --------------------------------------------------------------------------- @@ -973,8 +1259,40 @@ class ChromaCollection(BaseCollection): for m in metadatas ] + @staticmethod + def _sanitize_documents_for_chromadb(documents): + """Strip lone UTF-16 surrogates from every document before it reaches + the chromadb client. + + A single lone surrogate (U+D800–U+DFFF) raises ``UnicodeEncodeError`` + inside chromadb's encode path and aborts the *entire* add/upsert batch + with a ``-32000`` Internal Error, silently dropping every other row in + the same batch (#1235). + + #1235 fixed this for the MCP write tools via ``sanitize_content``, but + the bulk ingest paths (miner, convo_miner, sweeper, diary_ingest) build + documents without routing through that helper and reach this backend + directly. Sanitising here makes the chokepoint catch-all complete: the + sibling :meth:`_sanitize_metadatas_for_chromadb` already guarantees this + for metadata one method over; documents get the same guarantee. + """ + if documents is None: + return None + from ..config import strip_lone_surrogates + + # chromadb accepts OneOrMany[Document]: a bare str is a single document, + # not an iterable of characters. Handle it explicitly so we don't split + # it into per-character documents — that would be exactly the kind of + # silent corruption this method exists to prevent. + if isinstance(documents, str): + return strip_lone_surrogates(documents) + return [strip_lone_surrogates(d) if isinstance(d, str) else d for d in documents] + def add(self, *, documents, ids, metadatas=None, embeddings=None): - kwargs: dict[str, Any] = {"documents": documents, "ids": ids} + kwargs: dict[str, Any] = { + "documents": self._sanitize_documents_for_chromadb(documents), + "ids": ids, + } sanitized = self._sanitize_metadatas_for_chromadb(metadatas) if sanitized is not None: kwargs["metadatas"] = sanitized @@ -984,7 +1302,10 @@ class ChromaCollection(BaseCollection): self._collection.add(**kwargs) def upsert(self, *, documents, ids, metadatas=None, embeddings=None): - kwargs: dict[str, Any] = {"documents": documents, "ids": ids} + kwargs: dict[str, Any] = { + "documents": self._sanitize_documents_for_chromadb(documents), + "ids": ids, + } sanitized = self._sanitize_metadatas_for_chromadb(metadatas) if sanitized is not None: kwargs["metadatas"] = sanitized @@ -1005,7 +1326,7 @@ class ChromaCollection(BaseCollection): raise ValueError("update requires at least one of documents, metadatas, embeddings") kwargs: dict[str, Any] = {"ids": ids} if documents is not None: - kwargs["documents"] = documents + kwargs["documents"] = self._sanitize_documents_for_chromadb(documents) if metadatas is not None: kwargs["metadatas"] = metadatas if embeddings is not None: @@ -1161,6 +1482,242 @@ class ChromaCollection(BaseCollection): def count(self): return self._collection.count() + def lexical_search( + self, + *, + query: str, + n_results: int = 10, + where: Optional[dict] = None, + ) -> LexicalResult: + """Return lexical BM25 candidates for this collection. + + This is the normal healthy-Chroma implementation behind the optional + backend capability. The HNSW-disabled fallback in ``searcher.py`` still + reads ``chroma.sqlite3`` directly and remains Chroma-only. + """ + _validate_where(where) + sqlite_hits = self._lexical_search_via_sqlite(query=query, n_results=n_results, where=where) + if sqlite_hits is not None: + return LexicalResult(hits=sqlite_hits) + + # Directly-constructed ChromaCollection test doubles may not carry a + # palace path. Keep lexical_search usable in that shape, but normal + # MemPalace paths above use Chroma's FTS table instead of scanning every + # drawer through the Python client. + total = self.count() + docs: list[str] = [] + metas: list[dict] = [] + ids: list[str] = [] + offset = 0 + batch_size = 1000 + while offset < total: + kwargs: dict[str, Any] = { + "include": ["documents", "metadatas"], + "limit": batch_size, + "offset": offset, + } + if where: + kwargs["where"] = where + batch = self.get(**kwargs) + if not batch.ids: + break + ids.extend(batch.ids) + docs.extend(doc or "" for doc in batch.documents) + metas.extend(meta or {} for meta in batch.metadatas) + offset += len(batch.ids) + + scores = _bm25_scores(query, docs) + hits = [ + LexicalHit(id=doc_id, document=doc, metadata=meta, score=float(score)) + for doc_id, doc, meta, score in zip(ids, docs, metas, scores) + if score > 0 + ] + hits.sort(key=lambda hit: hit.score, reverse=True) + return LexicalResult(hits=hits[:n_results]) + + def _collection_name(self) -> Optional[str]: + name = getattr(self._collection, "name", None) + if callable(name): + try: + name = name() + except TypeError: + name = None + return str(name) if name else None + + def _lexical_search_via_sqlite( + self, + *, + query: str, + n_results: int, + where: Optional[dict], + max_candidates: int = 500, + ) -> Optional[list[LexicalHit]]: + if not self._palace_path: + return None + db_path = os.path.join(self._palace_path, "chroma.sqlite3") + if not os.path.isfile(db_path): + return [] + collection_name = self._collection_name() + if not collection_name: + return [] + + tokens = [t for t in _tokenize(query) if len(t) >= 3] + use_recency_fallback = not tokens + candidate_ids: list[int] = [] + # Map internal embeddings.id (rowid, used to join embedding_metadata) + # to the public embeddings.embedding_id so returned LexicalHit.id values + # round-trip through get(ids=...). The two differ: id is the integer + # rowid, embedding_id is the user-facing drawer id. + public_ids: dict[int, str] = {} + try: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + except sqlite3.Error: + logger.debug("Chroma lexical sqlite open failed", exc_info=True) + return [] + + try: + if tokens: + fts_query = " OR ".join(tokens) + # If a metadata filter is present, do not cap before filtering: + # otherwise a common term can fill the window with wrong-scope + # rows and hide valid scoped hits later in the FTS result set. + limit_sql = "" if where else "LIMIT ?" + params = [fts_query, collection_name] + if not where: + params.append(max(max_candidates, n_results)) + try: + rows = conn.execute( + f""" + SELECT e.id, e.embedding_id + FROM embedding_fulltext_search + JOIN embeddings e ON e.id = embedding_fulltext_search.rowid + JOIN segments s ON e.segment_id = s.id + JOIN collections c ON s.collection = c.id + WHERE embedding_fulltext_search MATCH ? + AND c.name = ? + {limit_sql} + """, + params, + ).fetchall() + candidate_ids = [int(row[0]) for row in rows] + public_ids.update({int(row[0]): str(row[1]) for row in rows}) + except sqlite3.Error: + logger.debug( + "Chroma lexical FTS query failed; using recency fallback", exc_info=True + ) + use_recency_fallback = True + + if not candidate_ids and use_recency_fallback: + order_expr = "e.created_at DESC" + try: + rows = conn.execute( + f""" + SELECT e.id, e.embedding_id + FROM embeddings e + JOIN segments s ON e.segment_id = s.id + JOIN collections c ON s.collection = c.id + WHERE c.name = ? + ORDER BY {order_expr} + LIMIT ? + """, + (collection_name, max(max_candidates, n_results)), + ).fetchall() + except sqlite3.Error: + logger.debug( + "Chroma lexical recency fallback failed; ordering by id", exc_info=True + ) + rows = conn.execute( + """ + SELECT e.id, e.embedding_id + FROM embeddings e + JOIN segments s ON e.segment_id = s.id + JOIN collections c ON s.collection = c.id + WHERE c.name = ? + ORDER BY e.id DESC + LIMIT ? + """, + (collection_name, max(max_candidates, n_results)), + ).fetchall() + candidate_ids = [int(row[0]) for row in rows] + public_ids.update({int(row[0]): str(row[1]) for row in rows}) + + if not candidate_ids: + return [] + + meta_columns = { + row["name"] + for row in conn.execute("PRAGMA table_info(embedding_metadata)").fetchall() + } + value_columns = [ + col + for col in ("string_value", "int_value", "float_value", "bool_value") + if col in meta_columns + ] + if not value_columns: + return [] + meta_rows = [] + for start in range(0, len(candidate_ids), 900): + chunk_ids = candidate_ids[start : start + 900] + placeholders = ",".join("?" for _ in chunk_ids) + meta_rows.extend( + conn.execute( + f""" + SELECT id, key, {", ".join(value_columns)} + FROM embedding_metadata + WHERE id IN ({placeholders}) + """, + chunk_ids, + ).fetchall() + ) + except sqlite3.Error: + logger.debug("Chroma lexical sqlite read failed", exc_info=True) + return [] + finally: + conn.close() + + drawers: dict[int, dict] = {} + for row in meta_rows: + emb_id = int(row["id"]) + key = row["key"] + values = {col: row[col] if col in row.keys() else None for col in value_columns} + value = _metadata_cell_value( + values.get("string_value"), + values.get("int_value"), + values.get("float_value"), + values.get("bool_value"), + ) + drawer = drawers.setdefault(emb_id, {"metadata": {}, "document": ""}) + if key == "chroma:document": + drawer["document"] = str(value or "") + else: + drawer["metadata"][key] = value + + ordered = [] + for emb_id in candidate_ids: + drawer = drawers.get(emb_id) + if drawer is None: + continue + meta = drawer["metadata"] + if not _matches_where(meta, where): + continue + ordered.append((emb_id, drawer["document"], meta)) + + docs = [doc for _, doc, _ in ordered] + scores = _bm25_scores(query, docs) + hits = [ + LexicalHit( + id=public_ids.get(emb_id, str(emb_id)), + document=doc, + metadata=meta, + score=float(score), + ) + for (emb_id, doc, meta), score in zip(ordered, scores) + if score > 0 + ] + hits.sort(key=lambda hit: hit.score, reverse=True) + return hits[:n_results] + @property def metadata(self) -> dict: """Pass-through to the underlying ChromaDB collection's metadata. @@ -1173,6 +1730,46 @@ class ChromaCollection(BaseCollection): """ return self._collection.metadata or {} + @property + def distance_metric(self) -> str: + """Report this collection's actual space from ``hnsw:space``. + + MemPalace sets ``hnsw:space=cosine`` on every creation path, so a + healthy palace reports ``"cosine"``. When the key is absent, empty, or + an unrecognized value, the collection is genuinely using Chroma's HNSW + default — **L2** (Euclidean) — because cosine was never set on it. We + report ``"l2"`` in that case so core ranking maps the distances + correctly; reporting ``"cosine"`` here would reintroduce the + floor-every-result-to-zero misranking this property exists to fix. + """ + space = str(self.metadata.get("hnsw:space", "") or "").lower() + if space in ("cosine", "l2", "ip"): + return space + return "l2" + + # ------------------------------------------------------------------ + # Embedder identity (RFC 001) + # + # Stored in a small sidecar JSON in the palace dir rather than the Chroma + # collection metadata: ``collection.modify(metadata=...)`` replaces the + # whole dict and some Chroma versions reject re-passing the immutable + # ``hnsw:*`` construction keys, so mutating it on every open is fragile. + # The sidecar is keyed by collection name (a palace may hold several). + # This is complementary to Chroma's own embedding-function-name check — + # the core check runs at open time and yields the clean cross-backend + # error before Chroma's read-time rejection fires. + # ------------------------------------------------------------------ + def _embedder_sidecar_path(self) -> Optional[str]: + if not self._palace_path: + return None + return os.path.join(self._palace_path, EMBEDDER_SIDECAR_FILENAME) + + def get_stored_embedder_identity(self): + return read_embedder_sidecar(self._embedder_sidecar_path(), self._collection_name()) + + def set_embedder_identity(self, identity) -> None: + write_embedder_sidecar(self._embedder_sidecar_path(), self._collection_name(), identity) + # --------------------------------------------------------------------------- # Backend @@ -1199,6 +1796,7 @@ class ChromaBackend(BaseBackend): "supports_embeddings_out", "supports_metadata_filters", "supports_contains_fast", + "supports_lexical_search", "local_mode", } ) @@ -1318,12 +1916,17 @@ class ChromaBackend(BaseBackend): ) if cached is None or inode_changed or mtime_changed or mtime_appeared: - # An inode swap means we are reopening a different physical DB - # (post-restore, fresh palace at the same path, etc.); drop the - # per-process gate so the quarantine pre-checks run again - # against the new disk state instead of trusting cached "we - # already cleaned this path" credit from the prior inode. - if inode_changed: + # Drop the per-process quarantine gate so the HNSW pre-checks + # run again against the new disk state. An inode swap means a + # different physical DB (post-restore, fresh palace at the same + # path); an mtime/appearance change means an external in-place + # write (closet_llm, mine, compress) that may have drifted the + # HNSW index while this process was running. + if ( + inode_changed + or mtime_changed + or (mtime_appeared and palace_path in self._freshness) + ): ChromaBackend._quarantined_paths.discard(palace_path) ChromaBackend._prepare_palace_for_open(palace_path) cached = chromadb.PersistentClient(path=palace_path) @@ -1340,17 +1943,12 @@ class ChromaBackend(BaseBackend): # Per-process record of palaces that have already had the cold-start # quarantine invoked at least once. The proactive HNSW checks are a - # *cold-start* protection — they catch segments that arrive stale relative + # *cold-start* protection -- they catch segments that arrive stale relative # to ``chroma.sqlite3`` or invalid on disk (e.g. cross-machine replication, - # partial restore, crashed-mid-write). Once a long-running process has - # opened the palace cleanly, re-firing the stale check on every reconnect - # is a *runtime thrash*: the daemon's own writes bump sqlite mtime but HNSW - # flushes batch on chromadb's internal cadence, so the mtime gap naturally - # exceeds the threshold under steady write load even though nothing is - # corrupt. - # Real runtime drift is still handled — palace-daemon's ``_auto_repair`` - # calls :func:`quarantine_stale_hnsw` directly on observed HNSW errors, - # which bypasses this gate. + # partial restore, crashed-mid-write). The gate is cleared whenever the + # palace changes on disk (inode swap, mtime bump, or file appearance), so + # external writes that drift HNSW segments are caught on the next open + # without requiring a full process restart. # # Thread-safety: this set is mutated without a lock. Two concurrent # ``make_client()`` calls for the same palace can both pass the @@ -1366,25 +1964,29 @@ class ChromaBackend(BaseBackend): """Run the pre-open safety pass shared by :meth:`make_client` and :meth:`_client`. - Three steps, all required before constructing a ``PersistentClient``: + Four steps, all required before constructing a ``PersistentClient``: - 1. ``_fix_blob_seq_ids`` — repairs the BLOB seq_id quirk that bites + 1. ``_fix_missing_collection_type`` — adds the ``_type`` marker to + ``collections.config_json_str`` that chromadb 1.5.9+ requires + but <= 1.5.8 never wrote (#1611). + 2. ``_fix_blob_seq_ids`` — repairs the BLOB seq_id quirk that bites certain chromadb migrations. - 2. ``quarantine_invalid_hnsw_metadata`` — renames aside any HNSW + 3. ``quarantine_invalid_hnsw_metadata`` — renames aside any HNSW ``index_metadata.pickle`` that fails to load, so chromadb opens against an empty index instead of crashing on the unloadable pickle (#1266 / PR #1285). - 3. ``quarantine_stale_hnsw`` — also gated by :attr:`_quarantined_paths` - so it fires once per palace per process. This is the SIGSEGV - prevention path for stale HNSW segments (see #1121, #1132, #1263); - wiring it through this helper means CLI mining, search, repair, - and status all benefit, not just the legacy ``make_client`` - callers. + 4. ``quarantine_stale_hnsw`` -- gated by :attr:`_quarantined_paths` + so it fires once per palace until the gate is re-armed by a + disk change. This is the SIGSEGV prevention path for stale + HNSW segments (see #1121, #1132, #1263); wiring it through + this helper means CLI mining, search, repair, and status all + benefit, not just the legacy ``make_client`` callers. Idempotent: safe to call from any code path that is about to open or re-open a palace. The ``_quarantined_paths`` gate prevents thrash on hot paths (e.g. ``_client()`` is called on every backend operation). """ + _fix_missing_collection_type(palace_path) _fix_blob_seq_ids(palace_path) if palace_path not in ChromaBackend._quarantined_paths: quarantine_invalid_hnsw_metadata(palace_path) @@ -1393,15 +1995,14 @@ class ChromaBackend(BaseBackend): @staticmethod def make_client(palace_path: str): - """Create a fresh ``PersistentClient`` (fixes BLOB seq_ids first). + """Create a fresh ``PersistentClient`` (runs pre-open safety pass first). Deprecated-ish: exposed for legacy long-lived callers that manage their own client cache. New code should obtain a collection through :meth:`get_collection` which manages caching internally. - Quarantines HNSW segments **once per palace per process**. See - :attr:`_quarantined_paths` for the rationale (cold-start protection - vs. runtime thrash on steady-write daemons). + Quarantines HNSW segments on first open and after any detected + disk change. See :attr:`_quarantined_paths` for the gate logic. """ ChromaBackend._prepare_palace_for_open(palace_path) return chromadb.PersistentClient(path=palace_path) diff --git a/mempalace/backends/embedding_wrapper.py b/mempalace/backends/embedding_wrapper.py new file mode 100644 index 0000000..46921df --- /dev/null +++ b/mempalace/backends/embedding_wrapper.py @@ -0,0 +1,171 @@ +"""Core-side embedding adapter for explicit-vector backends.""" + +from __future__ import annotations + +from typing import Optional + +from .base import BaseCollection + + +def _embed_texts(texts: list[str]) -> list[list[float]]: + """Embed ``texts`` with the configured local embedding function.""" + if not texts: + return [] + from ..embedding import get_embedding_function + + ef = get_embedding_function() + vectors = ef(input=texts) + return [list(v) for v in vectors] + + +def _as_list(value): + """Normalize ChromaDB's ``OneOrMany`` shape (``str`` | ``dict`` | sequence) to a list. + + A bare ``str`` (a document/id) or ``dict`` (a single metadata) must be + *wrapped*, not iterated: ``list("abc")`` yields ``['a', 'b', 'c']`` and + ``list({"k": 1})`` yields ``['k']`` — either desyncs embeddings/metadatas + from ``ids`` on explicit-vector backends (pgvector, sqlite_exact). A list is + returned unchanged (no copy); any other iterable is materialized once. + See PR #1706/#1707 review. + """ + if isinstance(value, (str, dict)): + return [value] + if isinstance(value, list): + return value + return list(value) + + +class EmbeddingCollection(BaseCollection): + """Wrap a collection that requires explicit vectors. + + Backends opt in with the ``requires_explicit_embeddings`` capability. + Core callers can keep using ``documents=`` and ``query_texts=``; this + wrapper computes vectors locally before delegating to the backend. + """ + + def __init__(self, inner: BaseCollection): + self._inner = inner + + def __getattr__(self, name): + return getattr(self._inner, name) + + @property + def distance_metric(self) -> str: + # Explicit delegation: ``BaseCollection`` defines ``distance_metric`` + # as a property, so it resolves on this subclass and ``__getattr__`` + # never fires — without this override the wrapper would report the + # base "cosine" default and mask a wrapped non-cosine backend. + return self._inner.distance_metric + + # Same shadowing reason as ``distance_metric``: these are concrete methods + # on ``BaseCollection``, so ``__getattr__`` never delegates them. Forward + # explicitly to the wrapped backend collection's identity store. + def get_stored_embedder_identity(self): + return self._inner.get_stored_embedder_identity() + + def set_embedder_identity(self, identity) -> None: + return self._inner.set_embedder_identity(identity) + + def effective_embedder_identity(self): + return self._inner.effective_embedder_identity() + + def maintenance_state(self) -> dict: + return self._inner.maintenance_state() + + def run_maintenance(self, kind: str): + return self._inner.run_maintenance(kind) + + def add(self, *, documents, ids, metadatas=None, embeddings=None): + documents = _as_list(documents) + ids = _as_list(ids) + if metadatas is not None: + metadatas = _as_list(metadatas) + if embeddings is None: + embeddings = _embed_texts(documents) + return self._inner.add( + documents=documents, + ids=ids, + metadatas=metadatas, + embeddings=embeddings, + ) + + def upsert(self, *, documents, ids, metadatas=None, embeddings=None): + documents = _as_list(documents) + ids = _as_list(ids) + if metadatas is not None: + metadatas = _as_list(metadatas) + if embeddings is None: + embeddings = _embed_texts(documents) + return self._inner.upsert( + documents=documents, + ids=ids, + metadatas=metadatas, + embeddings=embeddings, + ) + + def query( + self, + *, + query_texts: Optional[list[str] | str] = None, + query_embeddings: Optional[list[list[float]]] = None, + n_results: int = 10, + where: Optional[dict] = None, + where_document: Optional[dict] = None, + include: Optional[list[str]] = None, + ): + if query_texts is not None and query_embeddings is None: + query_embeddings = _embed_texts(_as_list(query_texts)) + query_texts = None + return self._inner.query( + query_texts=query_texts, + query_embeddings=query_embeddings, + n_results=n_results, + where=where, + where_document=where_document, + include=include, + ) + + def get( + self, *, ids=None, where=None, where_document=None, limit=None, offset=None, include=None + ): + return self._inner.get( + ids=ids, + where=where, + where_document=where_document, + limit=limit, + offset=offset, + include=include, + ) + + def delete(self, *, ids=None, where=None): + return self._inner.delete(ids=ids, where=where) + + def count(self) -> int: + return self._inner.count() + + def estimated_count(self) -> int: + return self._inner.estimated_count() + + def close(self) -> None: + return self._inner.close() + + def health(self): + return self._inner.health() + + def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None): + return self._inner.lexical_search(query=query, n_results=n_results, where=where) + + def update(self, *, ids, documents=None, metadatas=None, embeddings=None): + ids = _as_list(ids) + if documents is not None: + documents = _as_list(documents) + if embeddings is None: + embeddings = _embed_texts(documents) + if metadatas is not None: + metadatas = _as_list(metadatas) + return self._inner.update( + ids=ids, + documents=documents, + metadatas=metadatas, + embeddings=embeddings, + ) diff --git a/mempalace/backends/pgvector.py b/mempalace/backends/pgvector.py new file mode 100644 index 0000000..9789304 --- /dev/null +++ b/mempalace/backends/pgvector.py @@ -0,0 +1,1432 @@ +"""Postgres + pgvector backend for MemPalace. + +pgvector is an opt-in external-service backend, the SQL counterpart to the +Qdrant REST backend. Chroma remains the default; this adapter only runs when +the user explicitly selects ``pgvector`` via config, env, or CLI/MCP flag. +Embeddings are still produced locally by MemPalace through the core embedding +wrapper before vectors are written to Postgres. + +Why a second external backend: it exercises the storage contract on a +fundamentally different substrate (SQL + JSONB + the pgvector ``<=>`` operator) +than Qdrant's REST/dict model, proving the ``BaseBackend`` / ``BaseCollection`` +surface is not accidentally shaped around one vendor. + +Isolation model (RFC 001 isolation contract): one table per +``namespace`` + ``palace`` + ``collection``. The namespace contributes to the +table name, so this backend advertises ``supports_namespace_isolation`` and +satisfies the cross-namespace conformance arm. + +Dependency posture: the live client needs the optional ``psycopg`` dependency +(``pip install mempalace[pgvector]``), imported lazily so the package imports +fine without it. CI runs against an in-memory fake client; the live Postgres +round-trip is gated behind ``MEMPALACE_PGVECTOR_LIVE_URL``. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import threading +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +from typing import Any, Optional +from urllib import parse as urlparse + +import numpy as np + +from ._sidecar import EMBEDDER_SIDECAR_FILENAME, read_embedder_sidecar, write_embedder_sidecar +from .base import ( + BackendClosedError, + BackendError, + BackendMismatchError, + BaseBackend, + BaseCollection, + CollectionNotInitializedError, + DimensionMismatchError, + GetResult, + HealthStatus, + LexicalHit, + LexicalResult, + PalaceNotFoundError, + PalaceRef, + QueryResult, + UnsupportedFilterError, + _IncludeSpec, +) + +logger = logging.getLogger(__name__) + +_DEFAULT_DSN = "postgresql://localhost:5432/mempalace" +_MARKER_FILENAME = "pgvector_backend.json" +_MAX_IDENTIFIER = 63 # Postgres identifier byte limit. +_TOKEN_RE = re.compile(r"\w{2,}", re.UNICODE) +# Operators that translate to a JSONB containment predicate and so can be +# pushed down to SQL. Comparisons, $or and $contains stay on the local exact +# path (Python filtering), mirroring the Qdrant backend's local fallback. +_SUPPORTED_OPERATORS = frozenset( + {"$eq", "$ne", "$in", "$nin", "$and", "$or", "$contains", "$gt", "$gte", "$lt", "$lte"} +) +_PUSHDOWN_OPERATORS = frozenset({"$eq", "$ne", "$in", "$nin", "$and"}) + + +def _utcnow() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _json_dumps(obj: Any) -> str: + return json.dumps(obj or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _tokenize(text: str) -> list[str]: + if not text: + return [] + return _TOKEN_RE.findall(text.lower()) + + +def _bm25_scores(query: str, documents: list[str], k1: float = 1.5, b: float = 0.75) -> list[float]: + query_terms = set(_tokenize(query)) + n_docs = len(documents) + if not query_terms or n_docs == 0: + return [0.0] * n_docs + + tokenized = [_tokenize(d) for d in documents] + doc_lens = [len(toks) for toks in tokenized] + if not any(doc_lens): + return [0.0] * n_docs + avgdl = sum(doc_lens) / n_docs or 1.0 + + df = {term: 0 for term in query_terms} + for toks in tokenized: + for term in set(toks) & query_terms: + df[term] += 1 + + idf = {term: np.log((n_docs - df[term] + 0.5) / (df[term] + 0.5) + 1.0) for term in query_terms} + scores = [] + for toks, dl in zip(tokenized, doc_lens): + if dl == 0: + scores.append(0.0) + continue + tf: dict[str, int] = {} + for token in toks: + if token in query_terms: + tf[token] = tf.get(token, 0) + 1 + score = 0.0 + for term, freq in tf.items(): + num = freq * (k1 + 1) + den = freq + k1 * (1 - b + b * dl / avgdl) + score += float(idf[term]) * num / den + scores.append(score) + return scores + + +def _validate_where(where: Optional[dict]) -> None: + if not where: + return + stack = [where] + while stack: + node = stack.pop() + if not isinstance(node, dict): + continue + for key, value in node.items(): + if key.startswith("$") and key not in _SUPPORTED_OPERATORS: + raise UnsupportedFilterError(f"operator {key!r} not supported by pgvector") + if isinstance(value, dict): + stack.append(value) + elif isinstance(value, list): + stack.extend(item for item in value if isinstance(item, dict)) + + +def _coerce_comparable(value: Any): + if isinstance(value, bool): + return int(value) + return value + + +def _compare(actual: Any, op: str, expected: Any) -> bool: + actual = _coerce_comparable(actual) + expected = _coerce_comparable(expected) + if op == "$eq": + return actual == expected + if op == "$ne": + return actual != expected + if op == "$in": + return actual in (expected or []) + if op == "$nin": + return actual not in (expected or []) + if op == "$contains": + return str(expected) in str(actual or "") + try: + if op == "$gt": + return actual > expected + if op == "$gte": + return actual >= expected + if op == "$lt": + return actual < expected + if op == "$lte": + return actual <= expected + except TypeError: + return False + raise UnsupportedFilterError(f"operator {op!r} not supported by pgvector") + + +def _matches_where(meta: dict, where: Optional[dict]) -> bool: + if not where: + return True + if not isinstance(where, dict): + return False + for key, expected in where.items(): + if key == "$and": + if not all(_matches_where(meta, clause) for clause in expected or []): + return False + continue + if key == "$or": + if not any(_matches_where(meta, clause) for clause in expected or []): + return False + continue + if key.startswith("$"): + raise UnsupportedFilterError(f"operator {key!r} not supported by pgvector") + actual = meta.get(key) + if isinstance(expected, dict): + for op, operand in expected.items(): + if not _compare(actual, op, operand): + return False + elif actual != expected: + return False + return True + + +def _matches_where_document(document: str, where_document: Optional[dict]) -> bool: + if not where_document: + return True + if not isinstance(where_document, dict): + return False + for key, value in where_document.items(): + if key == "$contains": + if str(value) not in document: + return False + continue + if key == "$and": + if not all(_matches_where_document(document, clause) for clause in value or []): + return False + continue + if key == "$or": + if not any(_matches_where_document(document, clause) for clause in value or []): + return False + continue + raise UnsupportedFilterError(f"where_document operator {key!r} not supported") + return True + + +def _requires_local_filter(where: Optional[dict], where_document: Optional[dict] = None) -> bool: + """True when ``where``/``where_document`` cannot be fully pushed to SQL. + + Equality, ``$in``, ``$nin``, ``$ne`` and ``$and`` become JSONB containment + predicates; everything else ($or, $contains, comparisons, any + where_document) is evaluated on the local exact path so correctness never + depends on a hand-rolled SQL cast. + """ + if where_document: + return True + if not where: + return False + stack = [where] + while stack: + node = stack.pop() + if not isinstance(node, dict): + continue + for key, value in node.items(): + if key.startswith("$") and key not in _PUSHDOWN_OPERATORS: + return True + if isinstance(value, dict): + # A field mapping to an operator dict: only pushdown operators + # keep it on the fast path. + for op in value: + if op.startswith("$") and op not in _PUSHDOWN_OPERATORS: + return True + stack.append(value) + elif isinstance(value, list): + stack.extend(item for item in value if isinstance(item, dict)) + return False + + +def _validate_write_batch( + *, + documents: list[str], + ids: list[str], + metadatas: Optional[list[dict]], + embeddings: Optional[list[list[float]]], +) -> None: + n = len(ids) + if len(documents) != n: + raise ValueError(f"documents length {len(documents)} does not match ids length {n}") + if metadatas is not None and len(metadatas) != n: + raise ValueError(f"metadatas length {len(metadatas)} does not match ids length {n}") + if embeddings is not None and len(embeddings) != n: + raise ValueError(f"embeddings length {len(embeddings)} does not match ids length {n}") + + +def _as_vector_array(vector: list[float]) -> np.ndarray: + arr = np.asarray(vector, dtype=np.float32) + if arr.ndim != 1 or arr.size == 0: + raise ValueError("embedding must be a non-empty 1D vector") + return arr + + +def _normalize_vectors(embeddings: list[list[float]]) -> tuple[list[list[float]], int]: + vectors = [] + dims = set() + for embedding in embeddings: + arr = _as_vector_array(embedding) + vectors.append(arr.astype(float).tolist()) + dims.add(int(arr.size)) + if len(dims) > 1: + raise DimensionMismatchError( + f"pgvector batch cannot mix embedding dimensions {sorted(dims)}" + ) + return vectors, dims.pop() if dims else 0 + + +def _jsonable_metadata(meta: dict | None) -> dict: + try: + value = json.loads(json.dumps(meta or {}, ensure_ascii=False)) + except (TypeError, ValueError): + value = {} + return value if isinstance(value, dict) else {} + + +def _vector_distance(query: np.ndarray, vector: list[float] | None) -> Optional[float]: + if vector is None: + return None + vec = _as_vector_array(vector) + if vec.size != query.size: + return None + denom = float(np.linalg.norm(query)) * float(np.linalg.norm(vec)) + cos = 0.0 if denom <= 0 else float(np.dot(query, vec) / denom) + return 1.0 - max(-1.0, min(1.0, cos)) + + +def _vector_literal(vector: list[float]) -> str: + """Render a vector as the pgvector text literal ``[1,2,3]``. + + Using the text form keeps the optional dependency surface to ``psycopg`` + alone — no ``pgvector`` Python adapter is required, only the server-side + extension. + """ + return "[" + ",".join(repr(float(v)) for v in vector) + "]" + + +def _parse_vector(value: Any) -> Optional[list[float]]: + if value is None: + return None + if isinstance(value, (list, tuple)): + return [float(v) for v in value] + text = str(value).strip() + if not text: + return None + text = text.strip("[]") + if not text: + return [] + return [float(part) for part in text.split(",")] + + +def _slug(value: str, fallback: str = "palace") -> str: + safe = re.sub(r"[^A-Za-z0-9_]+", "_", value).strip("_") + safe = safe or fallback + if len(safe) <= 48: + return safe + digest = sha256(value.encode("utf-8", errors="surrogatepass")).hexdigest()[:12] + return f"{safe[:35]}_{digest}" + + +def _pg_identifier(name: str) -> str: + """Clamp an identifier to Postgres' 63-byte limit, hashing the overflow.""" + if len(name.encode("utf-8")) <= _MAX_IDENTIFIER: + return name + digest = sha256(name.encode("utf-8", errors="surrogatepass")).hexdigest()[:12] + return f"{name[:50]}_{digest}" + + +def _quote_identifier(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +# Session-level advisory-lock namespace for serializing HNSW index builds +# across daemon writers (RFC 001). classid is a fixed mempalace constant +# ("MEMP" in ASCII); objid is a stable per-table key. Both must fit a signed +# int4, which ``pg_advisory_lock(int4, int4)`` requires. +_MAINTENANCE_LOCK_CLASSID = 0x4D454D50 # "MEMP" — a positive, valid int4 + + +def _advisory_objid(table: str) -> int: + """Stable signed-int4 advisory key derived from the table name.""" + raw = int(sha256(table.encode("utf-8")).hexdigest()[:8], 16) # 0 .. 2**32-1 + return raw - 2**32 if raw >= 2**31 else raw + + +def _hnsw_index_name(table: str) -> str: + """Deterministic, collision-safe index name for ``table``. + + Routes through :func:`_pg_identifier`, which hashes the overflow when the + name exceeds Postgres' 63-byte limit. A naive ``[:63]`` truncation could + return the table name verbatim (tables and indexes share the ``pg_class`` + namespace), which would fail with "relation already exists". + """ + return _pg_identifier(f"{table}_hnsw_idx") + + +def _field_sql(field: str, expression: Any, params: list) -> str: + """Translate one field predicate to a JSONB containment expression.""" + if isinstance(expression, dict): + parts = [] + for op, operand in expression.items(): + if op == "$eq": + params.append(_json_dumps({field: operand})) + parts.append("metadata @> %s::jsonb") + elif op == "$ne": + params.append(_json_dumps({field: operand})) + parts.append("(NOT (metadata @> %s::jsonb))") + elif op == "$in": + ors = [] + for item in operand or []: + params.append(_json_dumps({field: item})) + ors.append("metadata @> %s::jsonb") + parts.append("(" + (" OR ".join(ors) if ors else "FALSE") + ")") + elif op == "$nin": + ors = [] + for item in operand or []: + params.append(_json_dumps({field: item})) + ors.append("metadata @> %s::jsonb") + parts.append("(NOT (" + (" OR ".join(ors) if ors else "FALSE") + "))") + else: # pragma: no cover - guarded by _requires_local_filter + raise UnsupportedFilterError(f"operator {op!r} not pushed down by pgvector") + return " AND ".join(parts) if parts else "TRUE" + params.append(_json_dumps({field: expression})) + return "metadata @> %s::jsonb" + + +def _where_to_sql(where: Optional[dict], params: list) -> str: + """Translate the pushdown filter subset to a JSONB SQL predicate. + + Appends bound parameters to ``params`` and returns a boolean SQL string. + Only operators allowed past :func:`_requires_local_filter` reach here. + """ + if not where: + return "TRUE" + clauses = [] + for key, expected in where.items(): + if key == "$and": + for clause in expected or []: + clauses.append(f"({_where_to_sql(clause, params)})") + continue + if key.startswith("$"): # pragma: no cover - guarded upstream + raise UnsupportedFilterError(f"operator {key!r} not pushed down by pgvector") + clauses.append(_field_sql(key, expected, params)) + return " AND ".join(clauses) if clauses else "TRUE" + + +@dataclass(frozen=True) +class _PgVectorConfig: + dsn: str = _DEFAULT_DSN + namespace: Optional[str] = None + + @classmethod + def from_options(cls, options: Optional[dict] = None) -> "_PgVectorConfig": + options = options or {} + try: + from ..config import MempalaceConfig + + cfg = MempalaceConfig() + except Exception: # pragma: no cover - config import should be boring + cfg = None + dsn = ( + options.get("dsn") + or options.get("url") + or os.environ.get("MEMPALACE_PGVECTOR_DSN") + or getattr(cfg, "pgvector_dsn", None) + or _DEFAULT_DSN + ) + namespace = ( + options.get("namespace") + or os.environ.get("MEMPALACE_PGVECTOR_NAMESPACE") + or getattr(cfg, "pgvector_namespace", None) + ) + return cls( + dsn=str(dsn).strip() or _DEFAULT_DSN, + namespace=str(namespace).strip() or None if namespace else None, + ) + + +class _PgVectorClient: + """Thin psycopg wrapper. ``psycopg`` is imported lazily on first connect.""" + + def __init__(self, config: _PgVectorConfig): + self._config = config + self._conn = None + self._lock = threading.RLock() + + def _connect(self): + if self._conn is not None and not getattr(self._conn, "closed", False): + return self._conn + try: + import psycopg + except ImportError as exc: # pragma: no cover - exercised only without the extra + raise BackendError( + "pgvector backend requires the optional 'psycopg' dependency; " + "install mempalace[pgvector]" + ) from exc + try: + self._conn = psycopg.connect(self._config.dsn) + except Exception as exc: # noqa: BLE001 - surface any driver failure uniformly + raise BackendError(f"pgvector connection failed: {exc}") from exc + return self._conn + + def _execute(self, sql: str, params=None, *, fetch: bool = False, many: bool = False): + conn = self._connect() + with self._lock: + try: + with conn.cursor() as cur: + if many: + cur.executemany(sql, params or []) + rows = None + else: + cur.execute(sql, params or []) + rows = cur.fetchall() if fetch else None + conn.commit() + except Exception as exc: # noqa: BLE001 - normalize to BackendError + try: + conn.rollback() + except Exception: # pragma: no cover - rollback best effort + pass + raise BackendError(f"pgvector query failed: {exc}") from exc + return rows + + def ping(self) -> None: + self._execute("SELECT 1", fetch=True) + + def ensure_extension(self) -> None: + try: + self._execute("CREATE EXTENSION IF NOT EXISTS vector") + except BackendError: + # Extension may already exist or require elevated privilege; the + # table create will fail loudly later if the vector type is absent. + logger.debug("pgvector CREATE EXTENSION skipped", exc_info=True) + + def table_exists(self, table: str) -> bool: + rows = self._execute( + "SELECT 1 FROM information_schema.tables " + "WHERE table_schema = current_schema() AND table_name = %s", + [table], + fetch=True, + ) + return bool(rows) + + def table_dimension(self, table: str) -> Optional[int]: + # Read the declared dimension via ``format_type`` (which invokes the + # type's own typmod_out and yields canonical ``vector(384)`` text) + # rather than the raw ``atttypmod``. On the pgvector versions tested + # (0.8.x) atttypmod already equals the bare dimension, so the direct + # read also worked — but format_type is the canonical, version-proof + # source of truth and avoids depending on the internal typmod encoding + # staying stable across pgvector releases. + try: + rows = self._execute( + "SELECT format_type(a.atttypid, a.atttypmod) FROM pg_attribute a " + "WHERE a.attrelid = %s::regclass AND a.attname = 'embedding'", + [_quote_identifier(table)], + fetch=True, + ) + except BackendError: + return None + if not rows or not rows[0] or not rows[0][0]: + return None + match = re.search(r"\((\d+)\)", str(rows[0][0])) + return int(match.group(1)) if match else None + + def create_table(self, table: str, dimension: int) -> None: + self.ensure_extension() + qi = _quote_identifier(table) + self._execute( + f"CREATE TABLE IF NOT EXISTS {qi} (" + "id text PRIMARY KEY, " + "document text NOT NULL DEFAULT '', " + "metadata jsonb NOT NULL DEFAULT '{}'::jsonb, " + f"embedding vector({int(dimension)}), " + "updated_at timestamptz)" + ) + + def upsert_rows(self, table: str, rows: list[dict]) -> None: + if not rows: + return + qi = _quote_identifier(table) + sql = ( + f"INSERT INTO {qi} (id, document, metadata, embedding, updated_at) " + "VALUES (%s, %s, %s::jsonb, %s::vector, %s) " + "ON CONFLICT (id) DO UPDATE SET " + "document = EXCLUDED.document, metadata = EXCLUDED.metadata, " + "embedding = EXCLUDED.embedding, updated_at = EXCLUDED.updated_at" + ) + params = [ + ( + row["id"], + row["document"], + _json_dumps(row.get("metadata")), + _vector_literal(row["embedding"]), + row.get("updated_at") or _utcnow(), + ) + for row in rows + ] + self._execute(sql, params, many=True) + + def query_rows( + self, + table: str, + *, + vector: list[float], + limit: int, + where: Optional[dict], + with_embedding: bool, + ) -> list[dict]: + qi = _quote_identifier(table) + params: list = [_vector_literal(vector)] + where_sql = _where_to_sql(where, params) if where else "TRUE" + cols = "id, document, metadata" + if with_embedding: + cols += ", embedding" + params.append(int(limit)) + # SQL text order — distance %s::vector, then WHERE params, then LIMIT %s + # — already matches positional binding order in ``params``. + sql = ( + f"SELECT {cols}, embedding <=> %s::vector AS distance " + f"FROM {qi} WHERE {where_sql} ORDER BY distance ASC LIMIT %s" + ) + rows = self._execute(sql, params, fetch=True) + return [ + self._row(record, with_embedding=with_embedding, with_distance=True) + for record in rows or [] + ] + + def scroll_rows( + self, + table: str, + *, + where: Optional[dict] = None, + with_embedding: bool = False, + ) -> list[dict]: + qi = _quote_identifier(table) + params: list = [] + where_sql = _where_to_sql(where, params) if where else "TRUE" + cols = "id, document, metadata" + if with_embedding: + cols += ", embedding" + sql = f"SELECT {cols} FROM {qi} WHERE {where_sql}" + rows = self._execute(sql, params, fetch=True) + return [ + self._row(record, with_embedding=with_embedding, with_distance=False) + for record in rows or [] + ] + + def delete_rows( + self, + table: str, + *, + ids: Optional[list[str]] = None, + where: Optional[dict] = None, + ) -> None: + qi = _quote_identifier(table) + if ids is not None: + self._execute(f"DELETE FROM {qi} WHERE id = ANY(%s)", [list(ids)]) + return + params: list = [] + where_sql = _where_to_sql(where, params) if where else "TRUE" + self._execute(f"DELETE FROM {qi} WHERE {where_sql}", params) + + def count_rows(self, table: str) -> int: + rows = self._execute(f"SELECT count(*) FROM {_quote_identifier(table)}", fetch=True) + return int(rows[0][0]) if rows and rows[0] else 0 + + def drop_table(self, table: str) -> None: + self._execute(f"DROP TABLE IF EXISTS {_quote_identifier(table)}") + + # ------------------------------------------------------------------ + # Maintenance (RFC 001) + # ------------------------------------------------------------------ + def has_vector_index(self, table: str) -> bool: + rows = self._execute( + "SELECT 1 FROM pg_indexes WHERE schemaname = current_schema() " + "AND tablename = %s AND indexdef ILIKE %s", + [table, "%using hnsw%"], + fetch=True, + ) + return bool(rows) + + def try_advisory_lock(self, classid: int, objid: int) -> bool: + rows = self._execute("SELECT pg_try_advisory_lock(%s, %s)", [classid, objid], fetch=True) + return bool(rows and rows[0] and rows[0][0]) + + def advisory_unlock(self, classid: int, objid: int) -> None: + self._execute("SELECT pg_advisory_unlock(%s, %s)", [classid, objid], fetch=True) + + def create_hnsw_index(self, table: str) -> None: + qi = _quote_identifier(table) + idx = _quote_identifier(_hnsw_index_name(table)) + # Non-concurrent build takes ACCESS EXCLUSIVE for the build duration; + # the advisory lock in the caller ensures only one session builds, so + # writes are blocked once rather than by every writer that crossed the + # threshold (the production wedge this serialization fixes). + self._execute( + f"CREATE INDEX IF NOT EXISTS {idx} ON {qi} USING hnsw (embedding vector_cosine_ops)" + ) + + def analyze_table(self, table: str) -> None: + self._execute(f"ANALYZE {_quote_identifier(table)}") + + def close(self) -> None: + with self._lock: + if self._conn is not None: + try: + self._conn.close() + except Exception: # pragma: no cover - close best effort + pass + self._conn = None + + @staticmethod + def _row(record, *, with_embedding: bool, with_distance: bool) -> dict: + record = list(record) + row = { + "id": str(record[0]), + "document": record[1] if record[1] is not None else "", + "metadata": record[2] + if isinstance(record[2], dict) + else (json.loads(record[2]) if record[2] else {}), + "embedding": None, + "distance": None, + } + idx = 3 + if with_embedding: + row["embedding"] = _parse_vector(record[idx]) + idx += 1 + if with_distance: + row["distance"] = float(record[idx]) if record[idx] is not None else None + return row + + +class PgVectorCollection(BaseCollection): + def __init__( + self, + *, + backend: "PgVectorBackend", + client: _PgVectorClient, + config: _PgVectorConfig, + palace: PalaceRef, + collection_name: str, + table: str, + ): + self._backend = backend + self._client = client + self._config = config + self._palace = palace + self._collection_name = collection_name + self._table = table + self._lock = threading.RLock() + self._closed = False + self._known_dimension: Optional[int] = None + + def _ensure_open(self) -> None: + if self._closed or self._backend._closed: + raise BackendClosedError("PgVectorCollection has been closed") + + def _table_exists(self) -> bool: + return self._client.table_exists(self._table) + + def _marker_exists(self) -> bool: + return self._backend._marker_exists(self._palace) + + def get_stored_embedder_identity(self): + return self._backend._get_embedder_identity(self._palace, self._collection_name) + + def set_embedder_identity(self, identity) -> None: + # Sidecar-backed (see PgVectorBackend), so this records even on a + # brand-new palace whose mismatch marker doesn't exist yet. + self._backend._set_embedder_identity(self._palace, self._collection_name, identity) + + def _ensure_table(self, dimension: int) -> None: + if dimension <= 0: + raise ValueError("embedding dimension must be positive") + with self._lock: + self._ensure_open() + if self._known_dimension is not None: + if self._known_dimension != dimension: + raise DimensionMismatchError( + f"pgvector collection {self._collection_name!r} expects " + f"embedding dimension {self._known_dimension}, got {dimension}" + ) + return + if not self._table_exists(): + self._client.create_table(self._table, dimension) + self._known_dimension = dimension + return + existing_dim = self._client.table_dimension(self._table) + if existing_dim is not None and existing_dim != dimension: + raise DimensionMismatchError( + f"pgvector collection {self._collection_name!r} expects " + f"embedding dimension {existing_dim}, got {dimension}" + ) + self._known_dimension = existing_dim or dimension + + def _scroll(self, *, where=None, with_embedding=False) -> list[dict]: + self._ensure_open() + if not self._table_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return [] + return self._client.scroll_rows(self._table, where=where, with_embedding=with_embedding) + + def _rows( + self, + *, + ids=None, + where=None, + where_document=None, + with_embedding=False, + ) -> list[dict]: + _validate_where(where) + _validate_where(where_document) + pushdown = None if _requires_local_filter(where, where_document) else where + rows = self._scroll(where=pushdown, with_embedding=with_embedding) + id_set = set(ids) if ids is not None else None + return [ + row + for row in rows + if (id_set is None or row["id"] in id_set) + and _matches_where(row["metadata"], where) + and _matches_where_document(row["document"], where_document) + ] + + def add(self, *, documents, ids, metadatas=None, embeddings=None): + _validate_write_batch( + documents=documents, ids=ids, metadatas=metadatas, embeddings=embeddings + ) + if embeddings is None: + raise ValueError("pgvector requires explicit embeddings") + if len(set(ids)) != len(ids): + raise ValueError("add ids must be unique") + existing = self.get(ids=list(ids), include=[]) + if existing.ids: + raise ValueError(f"ids already exist in pgvector collection: {existing.ids}") + self.upsert(documents=documents, ids=ids, metadatas=metadatas, embeddings=embeddings) + + def upsert(self, *, documents, ids, metadatas=None, embeddings=None): + _validate_write_batch( + documents=documents, ids=ids, metadatas=metadatas, embeddings=embeddings + ) + if embeddings is None: + raise ValueError("pgvector requires explicit embeddings") + vectors, dimension = _normalize_vectors(embeddings) + self._ensure_table(dimension) + metadatas = metadatas or [{} for _ in ids] + rows = [ + { + "id": str(doc_id), + "document": str(doc), + "metadata": _jsonable_metadata(meta), + "embedding": vector, + "updated_at": _utcnow(), + } + for doc_id, doc, meta, vector in zip(ids, documents, metadatas, vectors) + ] + self._client.upsert_rows(self._table, rows) + self._backend._write_marker(self._palace, self._config) + + def update(self, *, ids, documents=None, metadatas=None, embeddings=None): + if documents is None and metadatas is None and embeddings is None: + raise ValueError("update requires at least one of documents, metadatas, embeddings") + n = len(ids) + for label, value in ( + ("documents", documents), + ("metadatas", metadatas), + ("embeddings", embeddings), + ): + if value is not None and len(value) != n: + raise ValueError(f"{label} length {len(value)} does not match ids length {n}") + existing = self.get(ids=ids, include=["documents", "metadatas", "embeddings"]) + by_id = { + rid: (existing.documents[i], existing.metadatas[i], existing.embeddings[i]) + for i, rid in enumerate(existing.ids) + if existing.embeddings is not None + } + out_ids, out_docs, out_metas, out_embeddings = [], [], [], [] + for idx, doc_id in enumerate(ids): + if doc_id not in by_id: + continue + prev_doc, prev_meta, prev_embedding = by_id[doc_id] + out_ids.append(doc_id) + out_docs.append(documents[idx] if documents is not None else prev_doc) + meta = dict(prev_meta or {}) + if metadatas is not None: + meta.update(metadatas[idx] or {}) + out_metas.append(meta) + out_embeddings.append(embeddings[idx] if embeddings is not None else prev_embedding) + if out_ids: + self.upsert( + documents=out_docs, ids=out_ids, metadatas=out_metas, embeddings=out_embeddings + ) + + def _query_local_exact( + self, *, query_embeddings, n_results, where, where_document, include + ) -> QueryResult: + spec = _IncludeSpec.resolve(include, default_distances=True) + pushdown = None if _requires_local_filter(where, where_document) else where + rows = self._scroll(where=pushdown, with_embedding=True) + rows = [ + row + for row in rows + if _matches_where(row["metadata"], where) + and _matches_where_document(row["document"], where_document) + ] + outer_ids: list[list[str]] = [] + outer_docs: list[list[str]] = [] + outer_metas: list[list[dict]] = [] + outer_dists: list[list[float]] = [] + outer_embeds: list[list[list[float]]] = [] + for query_vector in query_embeddings: + q = _as_vector_array(query_vector) + scored = [] + for row in rows: + distance = _vector_distance(q, row["embedding"]) + if distance is not None: + scored.append((distance, row)) + scored.sort(key=lambda item: item[0]) + top = scored[:n_results] + outer_ids.append([row["id"] for _, row in top]) + outer_docs.append([row["document"] for _, row in top] if spec.documents else []) + outer_metas.append([row["metadata"] for _, row in top] if spec.metadatas else []) + outer_dists.append([float(dist) for dist, _ in top] if spec.distances else []) + if spec.embeddings: + outer_embeds.append([row["embedding"] or [] for _, row in top]) + return QueryResult( + ids=outer_ids, + documents=outer_docs, + metadatas=outer_metas, + distances=outer_dists, + embeddings=outer_embeds if spec.embeddings else None, + ) + + def query( + self, + *, + query_texts=None, + query_embeddings=None, + n_results=10, + where=None, + where_document=None, + include=None, + ) -> QueryResult: + if query_texts is not None: + raise ValueError( + "pgvector requires query_embeddings; use palace.get_collection wrapper" + ) + if query_embeddings is None: + raise ValueError("query requires query_embeddings") + if not query_embeddings: + raise ValueError("query input must be a non-empty list") + _validate_where(where) + _validate_where(where_document) + if _requires_local_filter(where, where_document): + return self._query_local_exact( + query_embeddings=query_embeddings, + n_results=n_results, + where=where, + where_document=where_document, + include=include, + ) + self._ensure_open() + if not self._table_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return QueryResult.empty( + num_queries=len(query_embeddings), + embeddings_requested=bool(include and "embeddings" in include), + ) + spec = _IncludeSpec.resolve(include, default_distances=True) + outer_ids: list[list[str]] = [] + outer_docs: list[list[str]] = [] + outer_metas: list[list[dict]] = [] + outer_dists: list[list[float]] = [] + outer_embeds: list[list[list[float]]] = [] + for query_vector in query_embeddings: + q = _as_vector_array(query_vector) + if self._known_dimension is None: + self._known_dimension = self._client.table_dimension(self._table) + if self._known_dimension is not None and int(q.size) != self._known_dimension: + raise DimensionMismatchError( + f"pgvector collection {self._collection_name!r} expects " + f"embedding dimension {self._known_dimension}, got {int(q.size)}" + ) + rows = self._client.query_rows( + self._table, + vector=q.astype(float).tolist(), + limit=n_results, + where=where, + with_embedding=spec.embeddings, + ) + outer_ids.append([row["id"] for row in rows]) + outer_docs.append([row["document"] for row in rows] if spec.documents else []) + outer_metas.append([row["metadata"] for row in rows] if spec.metadatas else []) + outer_dists.append( + [float(row["distance"]) if row["distance"] is not None else 1.0 for row in rows] + if spec.distances + else [] + ) + if spec.embeddings: + outer_embeds.append([row["embedding"] or [] for row in rows]) + return QueryResult( + ids=outer_ids, + documents=outer_docs, + metadatas=outer_metas, + distances=outer_dists, + embeddings=outer_embeds if spec.embeddings else None, + ) + + def get( + self, + *, + ids=None, + where=None, + where_document=None, + limit=None, + offset=None, + include=None, + ) -> GetResult: + spec = _IncludeSpec.resolve(include, default_distances=False) + rows = self._rows( + ids=ids, where=where, where_document=where_document, with_embedding=spec.embeddings + ) + if ids is not None: + by_id = {row["id"]: row for row in rows} + rows = [by_id[doc_id] for doc_id in ids if doc_id in by_id] + if offset: + rows = rows[offset:] + if limit is not None: + rows = rows[:limit] + return GetResult( + ids=[row["id"] for row in rows], + documents=[row["document"] for row in rows] if spec.documents else [], + metadatas=[row["metadata"] for row in rows] if spec.metadatas else [], + embeddings=[row["embedding"] or [] for row in rows] if spec.embeddings else None, + ) + + def delete(self, *, ids=None, where=None): + _validate_where(where) + if not self._table_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return + if ids is not None and where is None: + self._client.delete_rows(self._table, ids=list(ids)) + return + if ids is None and where is not None and not _requires_local_filter(where): + self._client.delete_rows(self._table, where=where) + return + rows = self._rows(ids=ids, where=where) + if rows: + self._client.delete_rows(self._table, ids=[row["id"] for row in rows]) + + def count(self) -> int: + self._ensure_open() + if not self._table_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return 0 + return self._client.count_rows(self._table) + + def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None): + _validate_where(where) + pushdown = None if _requires_local_filter(where) else where + rows = self._scroll(where=pushdown, with_embedding=False) + rows = [row for row in rows if _matches_where(row["metadata"], where)] + scores = _bm25_scores(query, [row["document"] for row in rows]) + hits = [ + LexicalHit( + id=row["id"], + document=row["document"], + metadata=row["metadata"], + score=score, + ) + for row, score in zip(rows, scores) + if score > 0 + ] + hits.sort(key=lambda hit: hit.score, reverse=True) + return LexicalResult(hits=hits[:n_results]) + + def close(self) -> None: + self._closed = True + + def health(self) -> HealthStatus: + if self._closed or self._backend._closed: + return HealthStatus.unhealthy("collection closed") + try: + if not self._table_exists(): + return HealthStatus.unhealthy("pgvector table not found") + except Exception as exc: # noqa: BLE001 - backend health should summarize + return HealthStatus.unhealthy(str(exc)) + return HealthStatus.healthy() + + def maintenance_state(self) -> dict: + empty = {"row_count": 0, "vector_index": None, "index_build_complete": False} + self._ensure_open() + try: + if not self._table_exists(): + return empty + rows = self._client.count_rows(self._table) + has_index = self._client.has_vector_index(self._table) + except Exception: # noqa: BLE001 - state report must not raise + logger.debug("pgvector maintenance state probe failed", exc_info=True) + return empty + return { + "row_count": rows, + "vector_index": "hnsw" if has_index else None, + "index_build_complete": has_index, + } + + def run_maintenance(self, kind: str): + from .base import MaintenanceResult, UnsupportedMaintenanceKindError + + if kind not in PgVectorBackend.maintenance_kinds: + raise UnsupportedMaintenanceKindError( + f"pgvector does not support maintenance kind {kind!r}" + ) + self._ensure_open() + # Nothing to maintain on a not-yet-materialized table (collection opened + # create=True but never written) — return noop rather than letting a + # raw "relation does not exist" error escape. + if not self._table_exists(): + return MaintenanceResult(kind=kind, status="noop", stats={"reason": "no table"}) + if kind == "analyze": + self._client.analyze_table(self._table) + return MaintenanceResult(kind="analyze", status="ran") + + # reindex → build the optional HNSW index. Opt-in: it makes search + # approximate, trading the exact-scan 100%-recall default for scale. + # Serialized with a session advisory lock so concurrent daemon writers + # learn "already_running" instead of each stacking an ACCESS EXCLUSIVE + # index build. + if self._client.has_vector_index(self._table): + return MaintenanceResult(kind="reindex", status="noop", stats={"vector_index": "hnsw"}) + classid, objid = _MAINTENANCE_LOCK_CLASSID, _advisory_objid(self._table) + if not self._client.try_advisory_lock(classid, objid): + return MaintenanceResult(kind="reindex", status="already_running") + try: + if self._client.has_vector_index(self._table): # re-check under lock + return MaintenanceResult( + kind="reindex", status="noop", stats={"vector_index": "hnsw"} + ) + self._client.create_hnsw_index(self._table) + return MaintenanceResult(kind="reindex", status="ran", stats={"vector_index": "hnsw"}) + finally: + self._client.advisory_unlock(classid, objid) + + +class PgVectorBackend(BaseBackend): + name = "pgvector" + capabilities = frozenset( + { + "requires_explicit_embeddings", + "supports_embeddings_in", + "supports_embeddings_passthrough", + "supports_embeddings_out", + "supports_metadata_filters", + "supports_lexical_search", + "supports_namespace_isolation", + "supports_server_side_indexes", + "server_mode", + } + ) + # "compact" is omitted: Postgres autovacuum reclaims space automatically, + # so a manual VACUUM kind would be redundant. "reindex" builds the optional + # HNSW index — an opt-in scale lever, NOT on by default, because it makes + # vector search approximate (the exact ``<=>`` scan is the 100%-recall path). + maintenance_kinds = frozenset({"analyze", "reindex"}) + + def __init__(self): + self._clients: dict[_PgVectorConfig, _PgVectorClient] = {} + self._collections_by_palace: dict[str, list[PgVectorCollection]] = {} + self._lock = threading.RLock() + self._closed = False + + # ------------------------------------------------------------------ + # Marker / mismatch protection (mirrors the Qdrant local marker). + # ------------------------------------------------------------------ + @staticmethod + def _marker_path(palace_path: str) -> str: + return os.path.join(palace_path, _MARKER_FILENAME) + + @staticmethod + def _palace_hash(palace: PalaceRef) -> str: + return sha256(palace.id.encode("utf-8", errors="surrogatepass")).hexdigest()[:16] + + def _table_prefix(self, *, palace: PalaceRef, config: _PgVectorConfig) -> str: + parts = ["mempalace"] + if config.namespace: + parts.append(_slug(config.namespace, "namespace")) + parts.append(self._palace_hash(palace)) + return "_".join(parts) + + def _table_name( + self, *, palace: PalaceRef, collection_name: str, config: _PgVectorConfig + ) -> str: + config = _PgVectorConfig( + dsn=config.dsn, + namespace=palace.namespace or config.namespace, + ) + prefix = self._table_prefix(palace=palace, config=config) + return _pg_identifier(f"{prefix}_{_slug(collection_name, 'collection')}") + + def _sanitized_dsn(self, dsn: str) -> dict: + try: + parsed = urlparse.urlparse(dsn) + except Exception: # pragma: no cover - defensive + return {"raw": ""} + return { + "host": parsed.hostname or "", + "port": parsed.port or 5432, + "dbname": (parsed.path or "").lstrip("/"), + } + + def _marker_target(self, palace: PalaceRef, config: _PgVectorConfig) -> dict: + target = self._sanitized_dsn(config.dsn) + target.update( + { + "namespace": config.namespace, + "palace_hash": self._palace_hash(palace), + "table_prefix": self._table_prefix(palace=palace, config=config), + } + ) + return target + + def _marker_exists(self, palace: PalaceRef) -> bool: + return bool(palace.local_path and os.path.isfile(self._marker_path(palace.local_path))) + + def _read_marker(self, palace: PalaceRef) -> Optional[dict]: + if not palace.local_path: + return None + marker_path = self._marker_path(palace.local_path) + if not os.path.isfile(marker_path): + return None + try: + with open(marker_path, encoding="utf-8") as f: + marker = json.load(f) + except (OSError, json.JSONDecodeError) as exc: + raise BackendMismatchError(f"pgvector marker is unreadable: {marker_path}") from exc + return marker if isinstance(marker, dict) else {} + + def _validate_marker_target(self, palace: PalaceRef, config: _PgVectorConfig) -> None: + marker = self._read_marker(palace) + if marker is None: + return + if marker.get("backend") != self.name: + raise BackendMismatchError("pgvector marker does not identify the pgvector backend") + expected = self._marker_target(palace, config) + actual = marker.get("pgvector") + if not isinstance(actual, dict): + raise BackendMismatchError("pgvector marker is missing target metadata") + mismatched = [ + key for key, expected_value in expected.items() if actual.get(key) != expected_value + ] + if mismatched: + details = ", ".join(mismatched) + raise BackendMismatchError( + "pgvector marker target does not match current configuration " + f"({details}); keep MEMPALACE_PGVECTOR_DSN and namespace consistent " + "or use a fresh palace directory" + ) + + def _write_marker(self, palace: PalaceRef, config: _PgVectorConfig) -> None: + if not palace.local_path: + return + os.makedirs(palace.local_path, exist_ok=True) + try: + os.chmod(palace.local_path, 0o700) + except (OSError, NotImplementedError): + pass + marker = { + "backend": self.name, + "schema_version": 1, + "created_at": _utcnow(), + "palace_id": palace.id, + "pgvector": self._marker_target(palace, config), + } + marker_path = self._marker_path(palace.local_path) + with open(marker_path, "w", encoding="utf-8") as f: + json.dump(marker, f, indent=2, ensure_ascii=False) + try: + os.chmod(marker_path, 0o600) + except (OSError, NotImplementedError): + pass + + # Embedder identity lives in a sidecar, NOT the backend marker: the marker's + # presence signals "palace initialized" (reads raise CollectionNotInitialized + # when the marker exists but the remote table doesn't), so recording identity + # at first empty open must not create it. The sidecar is unguarded — like the + # chroma sidecar — so a brand-new palace can record identity immediately. + @staticmethod + def _embedder_sidecar_path(palace: PalaceRef) -> Optional[str]: + if not palace.local_path: + return None + return os.path.join(palace.local_path, EMBEDDER_SIDECAR_FILENAME) + + def _get_embedder_identity(self, palace: PalaceRef, collection_name: str): + return read_embedder_sidecar(self._embedder_sidecar_path(palace), collection_name) + + def _set_embedder_identity(self, palace: PalaceRef, collection_name: str, identity) -> None: + write_embedder_sidecar(self._embedder_sidecar_path(palace), collection_name, identity) + + # ------------------------------------------------------------------ + def _client(self, config: _PgVectorConfig) -> _PgVectorClient: + if self._closed: + raise BackendClosedError("PgVectorBackend has been closed") + with self._lock: + client = self._clients.get(config) + if client is None: + client = _PgVectorClient(config) + self._clients[config] = client + return client + + def get_collection(self, *args, **kwargs) -> PgVectorCollection: + palace, collection_name, create, options = self._normalize_args(args, kwargs) + config = _PgVectorConfig.from_options(options) + if palace.namespace and palace.namespace != config.namespace: + config = _PgVectorConfig(dsn=config.dsn, namespace=palace.namespace) + client = self._client(config) + if palace.local_path: + marker_path = self._marker_path(palace.local_path) + if os.path.isfile(marker_path): + self._validate_marker_target(palace, config) + elif not create: + raise PalaceNotFoundError(marker_path) + else: + # The local marker is this backend's only mismatch-protection + # anchor. With no local_path (pure-remote / hosted mode) we can + # neither write nor validate it, so opening would silently drop + # protection against DSN/namespace drift. Refuse loudly. A remote + # marker store for pure-remote palaces is tracked as a follow-up. + raise BackendError( + "pgvector backend requires a local palace path to anchor mismatch " + "protection; pure-remote palaces (local_path=None) are not " + "supported yet" + ) + table = self._table_name(palace=palace, collection_name=collection_name, config=config) + if not create and not client.table_exists(table): + raise CollectionNotInitializedError(collection_name) + collection = PgVectorCollection( + backend=self, + client=client, + config=config, + palace=palace, + collection_name=collection_name, + table=table, + ) + with self._lock: + self._collections_by_palace.setdefault(palace.id, []).append(collection) + return collection + + @staticmethod + def _normalize_args(args, kwargs): + if "palace" in kwargs: + palace = kwargs.pop("palace") + if not isinstance(palace, PalaceRef): + raise TypeError("palace= must be a PalaceRef instance") + collection_name = kwargs.pop("collection_name") + create = bool(kwargs.pop("create", False)) + options = kwargs.pop("options", None) + if args or kwargs: + raise TypeError("unexpected arguments to get_collection") + return palace, collection_name, create, options + if args: + palace_path = args[0] + rest = list(args[1:]) + collection_name = kwargs.pop("collection_name", None) or (rest.pop(0) if rest else None) + if collection_name is None: + raise TypeError("collection_name is required") + create = kwargs.pop("create", False) + if rest: + create = rest.pop(0) + options = kwargs.pop("options", None) + if rest or kwargs: + raise TypeError("unexpected arguments to get_collection") + return ( + PalaceRef(id=palace_path, local_path=palace_path), + collection_name, + bool(create), + options, + ) + if "palace_path" in kwargs: + palace_path = kwargs.pop("palace_path") + collection_name = kwargs.pop("collection_name") + create = bool(kwargs.pop("create", False)) + options = kwargs.pop("options", None) + if kwargs: + raise TypeError("unexpected arguments to get_collection") + return ( + PalaceRef(id=palace_path, local_path=palace_path), + collection_name, + create, + options, + ) + raise TypeError("get_collection requires palace= or a positional palace_path") + + def close_palace(self, palace: PalaceRef | str) -> None: + palace_id = palace.id if isinstance(palace, PalaceRef) else palace + with self._lock: + collections = self._collections_by_palace.pop(palace_id, []) + for collection in collections: + collection.close() + + def close(self) -> None: + with self._lock: + collections = [ + collection + for palace_collections in self._collections_by_palace.values() + for collection in palace_collections + ] + clients = list(self._clients.values()) + self._collections_by_palace.clear() + self._clients.clear() + self._closed = True + for collection in collections: + collection.close() + for client in clients: + client.close() + + def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus: + if self._closed: + return HealthStatus.unhealthy("backend closed") + try: + self._client(_PgVectorConfig.from_options()).ping() + except Exception as exc: # noqa: BLE001 - user-facing health status + return HealthStatus.unhealthy(str(exc)) + if ( + palace + and palace.local_path + and not os.path.isfile(self._marker_path(palace.local_path)) + ): + return HealthStatus.unhealthy("pgvector marker not found") + return HealthStatus.healthy() + + @classmethod + def detect(cls, path: str) -> bool: + return os.path.isfile(os.path.join(path, _MARKER_FILENAME)) + + def create_collection(self, palace_path: str, collection_name: str) -> PgVectorCollection: + return self.get_collection(palace_path, collection_name, create=True) + + def get_or_create_collection(self, palace_path: str, collection_name: str): + return self.get_collection(palace_path, collection_name, create=True) + + def delete_collection(self, palace_path: str, collection_name: str) -> None: + palace = PalaceRef(id=palace_path, local_path=palace_path) + config = _PgVectorConfig.from_options() + table = self._table_name(palace=palace, collection_name=collection_name, config=config) + self._client(config).drop_table(table) diff --git a/mempalace/backends/qdrant.py b/mempalace/backends/qdrant.py new file mode 100644 index 0000000..bc516d5 --- /dev/null +++ b/mempalace/backends/qdrant.py @@ -0,0 +1,1386 @@ +"""Qdrant REST backend for MemPalace. + +Qdrant is an opt-in external-service backend. Chroma remains the default; this +adapter only runs when the user explicitly selects ``qdrant`` via config, env, +or CLI/MCP flag. Embeddings are still produced locally by MemPalace through the +core embedding wrapper before vectors are sent to Qdrant. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import threading +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +from typing import Any, Optional +from urllib import error as urlerror +from urllib import parse as urlparse +from urllib import request as urlrequest + +import numpy as np + +from ._sidecar import EMBEDDER_SIDECAR_FILENAME, read_embedder_sidecar, write_embedder_sidecar +from .base import ( + BackendClosedError, + BackendMismatchError, + BackendError, + BaseBackend, + BaseCollection, + CollectionNotInitializedError, + DimensionMismatchError, + GetResult, + HealthStatus, + LexicalHit, + LexicalResult, + PalaceNotFoundError, + PalaceRef, + QueryResult, + UnsupportedFilterError, + _IncludeSpec, +) + +logger = logging.getLogger(__name__) + +_DEFAULT_URL = "http://localhost:6333" +_MARKER_FILENAME = "qdrant_backend.json" +_PAYLOAD_ID = "mempalace_id" +_PAYLOAD_DOCUMENT = "document" +_PAYLOAD_METADATA = "metadata" +_POINT_NAMESPACE = uuid.UUID("c06c3fc7-5c14-4dc4-84c2-24a5f72d8dc1") +_TOKEN_RE = re.compile(r"\w{2,}", re.UNICODE) +_SUPPORTED_OPERATORS = frozenset( + {"$eq", "$ne", "$in", "$nin", "$and", "$or", "$contains", "$gt", "$gte", "$lt", "$lte"} +) + + +def _utcnow() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _tokenize(text: str) -> list[str]: + if not text: + return [] + return _TOKEN_RE.findall(text.lower()) + + +def _bm25_scores(query: str, documents: list[str], k1: float = 1.5, b: float = 0.75) -> list[float]: + query_terms = set(_tokenize(query)) + n_docs = len(documents) + if not query_terms or n_docs == 0: + return [0.0] * n_docs + + tokenized = [_tokenize(d) for d in documents] + doc_lens = [len(toks) for toks in tokenized] + if not any(doc_lens): + return [0.0] * n_docs + avgdl = sum(doc_lens) / n_docs or 1.0 + + df = {term: 0 for term in query_terms} + for toks in tokenized: + for term in set(toks) & query_terms: + df[term] += 1 + + idf = {term: np.log((n_docs - df[term] + 0.5) / (df[term] + 0.5) + 1.0) for term in query_terms} + scores = [] + for toks, dl in zip(tokenized, doc_lens): + if dl == 0: + scores.append(0.0) + continue + tf: dict[str, int] = {} + for token in toks: + if token in query_terms: + tf[token] = tf.get(token, 0) + 1 + score = 0.0 + for term, freq in tf.items(): + num = freq * (k1 + 1) + den = freq + k1 * (1 - b + b * dl / avgdl) + score += float(idf[term]) * num / den + scores.append(score) + return scores + + +def _validate_where(where: Optional[dict]) -> None: + if not where: + return + stack = [where] + while stack: + node = stack.pop() + if not isinstance(node, dict): + continue + for key, value in node.items(): + if key.startswith("$") and key not in _SUPPORTED_OPERATORS: + raise UnsupportedFilterError(f"operator {key!r} not supported by qdrant") + if isinstance(value, dict): + stack.append(value) + elif isinstance(value, list): + stack.extend(item for item in value if isinstance(item, dict)) + + +def _coerce_comparable(value: Any): + if isinstance(value, bool): + return int(value) + return value + + +def _compare(actual: Any, op: str, expected: Any) -> bool: + actual = _coerce_comparable(actual) + expected = _coerce_comparable(expected) + if op == "$eq": + return actual == expected + if op == "$ne": + return actual != expected + if op == "$in": + return actual in (expected or []) + if op == "$nin": + return actual not in (expected or []) + if op == "$contains": + return str(expected) in str(actual or "") + try: + if op == "$gt": + return actual > expected + if op == "$gte": + return actual >= expected + if op == "$lt": + return actual < expected + if op == "$lte": + return actual <= expected + except TypeError: + return False + raise UnsupportedFilterError(f"operator {op!r} not supported by qdrant") + + +def _matches_where(meta: dict, where: Optional[dict]) -> bool: + if not where: + return True + if not isinstance(where, dict): + return False + for key, expected in where.items(): + if key == "$and": + if not all(_matches_where(meta, clause) for clause in expected or []): + return False + continue + if key == "$or": + if not any(_matches_where(meta, clause) for clause in expected or []): + return False + continue + if key.startswith("$"): + raise UnsupportedFilterError(f"operator {key!r} not supported by qdrant") + actual = meta.get(key) + if isinstance(expected, dict): + for op, operand in expected.items(): + if not _compare(actual, op, operand): + return False + elif actual != expected: + return False + return True + + +def _matches_where_document(document: str, where_document: Optional[dict]) -> bool: + if not where_document: + return True + if not isinstance(where_document, dict): + return False + for key, value in where_document.items(): + if key == "$contains": + if str(value) not in document: + return False + continue + if key == "$and": + if not all(_matches_where_document(document, clause) for clause in value or []): + return False + continue + if key == "$or": + if not any(_matches_where_document(document, clause) for clause in value or []): + return False + continue + raise UnsupportedFilterError(f"where_document operator {key!r} not supported") + return True + + +def _validate_write_batch( + *, + documents: list[str], + ids: list[str], + metadatas: Optional[list[dict]], + embeddings: Optional[list[list[float]]], +) -> None: + n = len(ids) + if len(documents) != n: + raise ValueError(f"documents length {len(documents)} does not match ids length {n}") + if metadatas is not None and len(metadatas) != n: + raise ValueError(f"metadatas length {len(metadatas)} does not match ids length {n}") + if embeddings is not None and len(embeddings) != n: + raise ValueError(f"embeddings length {len(embeddings)} does not match ids length {n}") + + +def _as_vector_array(vector: list[float]) -> np.ndarray: + arr = np.asarray(vector, dtype=np.float32) + if arr.ndim != 1 or arr.size == 0: + raise ValueError("embedding must be a non-empty 1D vector") + return arr + + +def _normalize_vectors(embeddings: list[list[float]]) -> tuple[list[list[float]], int]: + vectors = [] + dims = set() + for embedding in embeddings: + arr = _as_vector_array(embedding) + vectors.append(arr.astype(float).tolist()) + dims.add(int(arr.size)) + if len(dims) > 1: + raise DimensionMismatchError(f"qdrant batch cannot mix embedding dimensions {sorted(dims)}") + return vectors, dims.pop() if dims else 0 + + +def _jsonable_metadata(meta: dict | None) -> dict: + try: + value = json.loads(json.dumps(meta or {}, ensure_ascii=False)) + except (TypeError, ValueError): + value = {} + return value if isinstance(value, dict) else {} + + +def _point_id(doc_id: str) -> str: + return str(uuid.uuid5(_POINT_NAMESPACE, str(doc_id))) + + +def _slug(value: str, fallback: str = "palace") -> str: + safe = re.sub(r"[^A-Za-z0-9_-]+", "_", value).strip("_") + safe = safe or fallback + if len(safe) <= 64: + return safe + digest = sha256(value.encode("utf-8", errors="surrogatepass")).hexdigest()[:12] + return f"{safe[:51]}_{digest}" + + +def _payload_row(point: dict) -> dict: + payload = point.get("payload") or {} + meta = payload.get(_PAYLOAD_METADATA) or {} + if not isinstance(meta, dict): + meta = {} + vector = point.get("vector") + if isinstance(vector, dict): + vector = vector.get("") or vector.get("default") or next(iter(vector.values()), None) + return { + "id": str(payload.get(_PAYLOAD_ID) or point.get("id") or ""), + "document": str(payload.get(_PAYLOAD_DOCUMENT) or ""), + "metadata": meta, + "embedding": vector if isinstance(vector, list) else None, + "score": point.get("score"), + } + + +def _vector_distance(query: np.ndarray, vector: list[float] | None) -> Optional[float]: + if vector is None: + return None + vec = _as_vector_array(vector) + if vec.size != query.size: + return None + denom = float(np.linalg.norm(query)) * float(np.linalg.norm(vec)) + cos = 0.0 if denom <= 0 else float(np.dot(query, vec) / denom) + return 1.0 - max(-1.0, min(1.0, cos)) + + +def _qdrant_score_to_distance(score: Any) -> float: + try: + return 1.0 - max(-1.0, min(1.0, float(score))) + except (TypeError, ValueError): + return 1.0 + + +class _QdrantHTTPError(BackendError): + def __init__(self, status: int, detail: str): + super().__init__(f"Qdrant HTTP {status}: {detail}") + self.status = status + self.detail = detail + + +@dataclass(frozen=True) +class _QdrantConfig: + url: str = _DEFAULT_URL + api_key: Optional[str] = None + timeout: float = 10.0 + namespace: Optional[str] = None + + @classmethod + def from_options(cls, options: Optional[dict] = None) -> "_QdrantConfig": + options = options or {} + try: + from ..config import MempalaceConfig + + cfg = MempalaceConfig() + except Exception: # pragma: no cover - config import should be boring + cfg = None + url = ( + options.get("url") + or os.environ.get("MEMPALACE_QDRANT_URL") + or getattr(cfg, "qdrant_url", None) + or _DEFAULT_URL + ) + api_key = ( + options.get("api_key") + or os.environ.get("MEMPALACE_QDRANT_API_KEY") + or getattr(cfg, "qdrant_api_key", None) + ) + namespace = ( + options.get("namespace") + or os.environ.get("MEMPALACE_QDRANT_NAMESPACE") + or getattr(cfg, "qdrant_namespace", None) + ) + raw_timeout = ( + options.get("timeout") + or os.environ.get("MEMPALACE_QDRANT_TIMEOUT") + or getattr(cfg, "qdrant_timeout", None) + or 10.0 + ) + try: + timeout = float(raw_timeout) + except (TypeError, ValueError): + timeout = 10.0 + if timeout <= 0: + timeout = 10.0 + return cls( + url=str(url).rstrip("/") or _DEFAULT_URL, + api_key=str(api_key) if api_key else None, + timeout=timeout, + namespace=str(namespace).strip() or None if namespace else None, + ) + + +class _QdrantRESTClient: + def __init__(self, config: _QdrantConfig): + self._config = config + + def request( + self, + method: str, + path: str, + *, + body: Optional[dict] = None, + query: Optional[dict] = None, + ) -> dict: + url = f"{self._config.url}{path}" + if query: + url = f"{url}?{urlparse.urlencode(query)}" + data = None + headers = {"Content-Type": "application/json"} + if self._config.api_key: + headers["api-key"] = self._config.api_key + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + req = urlrequest.Request(url, data=data, method=method, headers=headers) + try: + with urlrequest.urlopen(req, timeout=self._config.timeout) as resp: + raw = resp.read() + except urlerror.HTTPError as exc: + raw = exc.read() + detail = raw.decode("utf-8", errors="replace") if raw else str(exc) + raise _QdrantHTTPError(exc.code, detail) from exc + except urlerror.URLError as exc: + raise BackendError(f"Qdrant request failed: {exc.reason}") from exc + if not raw: + return {} + try: + return json.loads(raw.decode("utf-8")) + except json.JSONDecodeError as exc: + raise BackendError("Qdrant returned invalid JSON") from exc + + def collection_exists(self, collection: str) -> bool: + try: + self.request("GET", f"/collections/{urlparse.quote(collection, safe='')}") + except _QdrantHTTPError as exc: + if exc.status == 404: + return False + raise + return True + + def get_collection_info(self, collection: str) -> dict: + return self.request("GET", f"/collections/{urlparse.quote(collection, safe='')}") + + def create_collection(self, collection: str, dimension: int) -> None: + self.request( + "PUT", + f"/collections/{urlparse.quote(collection, safe='')}", + body={"vectors": {"size": int(dimension), "distance": "Cosine"}}, + ) + + def create_payload_index(self, collection: str, field_name: str, field_schema: str) -> None: + try: + self.request( + "PUT", + f"/collections/{urlparse.quote(collection, safe='')}/index", + query={"wait": "true"}, + body={"field_name": field_name, "field_schema": field_schema}, + ) + except _QdrantHTTPError as exc: + if exc.status in (400, 409): + logger.debug("Qdrant payload index creation skipped: %s", exc) + return + raise + + def upsert_points(self, collection: str, points: list[dict]) -> None: + self.request( + "PUT", + f"/collections/{urlparse.quote(collection, safe='')}/points", + query={"wait": "true"}, + body={"points": points}, + ) + + def query_points( + self, + collection: str, + *, + vector: list[float], + limit: int, + qdrant_filter: Optional[dict], + with_vector: bool, + ) -> list[dict]: + body = { + "query": vector, + "limit": int(limit), + "with_payload": True, + "with_vector": bool(with_vector), + } + if qdrant_filter: + body["filter"] = qdrant_filter + try: + response = self.request( + "POST", + f"/collections/{urlparse.quote(collection, safe='')}/points/query", + body=body, + ) + except _QdrantHTTPError as exc: + if exc.status not in (404, 405): + raise + body = { + "vector": vector, + "limit": int(limit), + "with_payload": True, + "with_vector": bool(with_vector), + } + if qdrant_filter: + body["filter"] = qdrant_filter + response = self.request( + "POST", + f"/collections/{urlparse.quote(collection, safe='')}/points/search", + body=body, + ) + result = response.get("result") or {} + if isinstance(result, list): + return result + return list(result.get("points") or []) + + def scroll_points( + self, + collection: str, + *, + qdrant_filter: Optional[dict] = None, + limit: int = 256, + offset: Any = None, + with_vector: bool = False, + ) -> tuple[list[dict], Any]: + body: dict[str, Any] = { + "limit": int(limit), + "with_payload": True, + "with_vector": bool(with_vector), + } + if qdrant_filter: + body["filter"] = qdrant_filter + if offset is not None: + body["offset"] = offset + response = self.request( + "POST", + f"/collections/{urlparse.quote(collection, safe='')}/points/scroll", + body=body, + ) + result = response.get("result") or {} + return list(result.get("points") or []), result.get("next_page_offset") + + def delete_points( + self, + collection: str, + *, + point_ids: Optional[list[str]] = None, + qdrant_filter: Optional[dict] = None, + ) -> None: + selector = ( + {"points": point_ids or []} + if point_ids is not None + else {"filter": qdrant_filter or {}} + ) + self.request( + "POST", + f"/collections/{urlparse.quote(collection, safe='')}/points/delete", + query={"wait": "true"}, + body=selector, + ) + + def count_points(self, collection: str) -> int: + response = self.request( + "POST", + f"/collections/{urlparse.quote(collection, safe='')}/points/count", + body={"exact": True}, + ) + result = response.get("result") or {} + return int(result.get("count") or 0) + + def delete_collection(self, collection: str) -> None: + self.request("DELETE", f"/collections/{urlparse.quote(collection, safe='')}") + + +def _condition(field: str, expression: Any) -> tuple[Optional[dict], list[dict]]: + key = f"{_PAYLOAD_METADATA}.{field}" + if isinstance(expression, dict): + conditions = [] + must_not = [] + for op, operand in expression.items(): + if op == "$eq": + conditions.append({"key": key, "match": {"value": operand}}) + elif op == "$ne": + must_not.append({"key": key, "match": {"value": operand}}) + elif op == "$in": + conditions.append({"key": key, "match": {"any": operand or []}}) + elif op == "$nin": + must_not.append({"key": key, "match": {"any": operand or []}}) + elif op in ("$gt", "$gte", "$lt", "$lte"): + range_key = {"$gt": "gt", "$gte": "gte", "$lt": "lt", "$lte": "lte"}[op] + conditions.append({"key": key, "range": {range_key: operand}}) + else: + return None, [] + if len(conditions) == 1 and not must_not: + return conditions[0], [] + body: dict[str, Any] = {} + if conditions: + body["must"] = conditions + if must_not: + body["must_not"] = must_not + return body, [] + return {"key": key, "match": {"value": expression}}, [] + + +def _requires_local_filter(where: Optional[dict], where_document: Optional[dict] = None) -> bool: + if where_document: + return True + if not where: + return False + stack = [where] + while stack: + node = stack.pop() + if not isinstance(node, dict): + continue + for key, value in node.items(): + if key in ("$or", "$contains"): + return True + if isinstance(value, dict): + if "$contains" in value: + return True + stack.append(value) + elif isinstance(value, list): + stack.extend(item for item in value if isinstance(item, dict)) + return False + + +def _qdrant_filter(where: Optional[dict]) -> Optional[dict]: + if not where: + return None + _validate_where(where) + must = [] + must_not = [] + for key, expected in where.items(): + if key == "$and": + for clause in expected or []: + child = _qdrant_filter(clause) + if child: + must.append(child) + continue + if key == "$or": + return None + if key.startswith("$"): + return None + condition, not_conditions = _condition(key, expected) + if condition is None: + return None + must.append(condition) + must_not.extend(not_conditions) + out: dict[str, Any] = {} + if must: + out["must"] = must + if must_not: + out["must_not"] = must_not + return out or None + + +def _combine_filters(*filters: Optional[dict]) -> Optional[dict]: + present = [flt for flt in filters if flt] + if not present: + return None + if len(present) == 1: + return present[0] + return {"must": present} + + +def _text_any_filter(query: str) -> Optional[dict]: + tokens = _tokenize(query) + if not tokens: + return None + return {"must": [{"key": _PAYLOAD_DOCUMENT, "match": {"text_any": " ".join(tokens)}}]} + + +class QdrantCollection(BaseCollection): + def __init__( + self, + *, + backend: "QdrantBackend", + client: _QdrantRESTClient, + config: _QdrantConfig, + palace: PalaceRef, + collection_name: str, + remote_collection: str, + ): + self._backend = backend + self._client = client + self._config = config + self._palace = palace + self._collection_name = collection_name + self._remote_collection = remote_collection + self._lock = threading.RLock() + self._closed = False + self._known_dimension: Optional[int] = None + + def _ensure_open(self) -> None: + if self._closed or self._backend._closed: + raise BackendClosedError("QdrantCollection has been closed") + + def _remote_exists(self) -> bool: + return self._client.collection_exists(self._remote_collection) + + def _marker_exists(self) -> bool: + return self._backend._marker_exists(self._palace) + + def get_stored_embedder_identity(self): + return self._backend._get_embedder_identity(self._palace, self._collection_name) + + def set_embedder_identity(self, identity) -> None: + # Sidecar-backed (see QdrantBackend), so this records even on a + # brand-new palace whose mismatch marker doesn't exist yet. + self._backend._set_embedder_identity(self._palace, self._collection_name, identity) + + def _remote_dimension(self) -> Optional[int]: + try: + info = self._client.get_collection_info(self._remote_collection) + except _QdrantHTTPError as exc: + if exc.status == 404: + return None + raise + result = info.get("result") or info + params = (result.get("config") or {}).get("params") or {} + vectors = params.get("vectors") or params.get("vectors_config") or {} + if isinstance(vectors, dict) and "size" in vectors: + return int(vectors["size"]) + if isinstance(vectors, dict): + for value in vectors.values(): + if isinstance(value, dict) and "size" in value: + return int(value["size"]) + return None + + def _ensure_remote_collection(self, dimension: int) -> None: + if dimension <= 0: + raise ValueError("embedding dimension must be positive") + with self._lock: + self._ensure_open() + if self._known_dimension is not None: + if self._known_dimension != dimension: + raise DimensionMismatchError( + f"qdrant collection {self._collection_name!r} expects " + f"embedding dimension {self._known_dimension}, got {dimension}" + ) + return + if not self._remote_exists(): + self._client.create_collection(self._remote_collection, dimension) + self._client.create_payload_index( + self._remote_collection, _PAYLOAD_DOCUMENT, "text" + ) + self._known_dimension = dimension + return + remote_dim = self._remote_dimension() + if remote_dim is not None and remote_dim != dimension: + raise DimensionMismatchError( + f"qdrant collection {self._collection_name!r} expects " + f"embedding dimension {remote_dim}, got {dimension}" + ) + self._known_dimension = remote_dim or dimension + + def _scroll_all( + self, + *, + qdrant_filter: Optional[dict] = None, + with_vector: bool = False, + ) -> list[dict]: + self._ensure_open() + if not self._remote_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return [] + rows = [] + offset = None + while True: + points, offset = self._client.scroll_points( + self._remote_collection, + qdrant_filter=qdrant_filter, + limit=256, + offset=offset, + with_vector=with_vector, + ) + rows.extend(_payload_row(point) for point in points) + if offset is None: + return rows + + def _rows( + self, + *, + ids: Optional[list[str]] = None, + where: Optional[dict] = None, + where_document: Optional[dict] = None, + with_vector: bool = False, + ) -> list[dict]: + _validate_where(where) + _validate_where(where_document) + q_filter = None if _requires_local_filter(where, where_document) else _qdrant_filter(where) + if ids is not None: + id_filter = {"must": [{"has_id": [_point_id(doc_id) for doc_id in ids]}]} + q_filter = _combine_filters(q_filter, id_filter) + rows = self._scroll_all(qdrant_filter=q_filter, with_vector=with_vector) + rows = [ + row + for row in rows + if (ids is None or row["id"] in set(ids)) + and _matches_where(row["metadata"], where) + and _matches_where_document(row["document"], where_document) + ] + return rows + + def add(self, *, documents, ids, metadatas=None, embeddings=None): + _validate_write_batch( + documents=documents, + ids=ids, + metadatas=metadatas, + embeddings=embeddings, + ) + if embeddings is None: + raise ValueError("qdrant requires explicit embeddings") + if len(set(ids)) != len(ids): + raise ValueError("add ids must be unique") + existing = self.get(ids=list(ids), include=[]) + if existing.ids: + raise ValueError(f"ids already exist in qdrant collection: {existing.ids}") + self.upsert(documents=documents, ids=ids, metadatas=metadatas, embeddings=embeddings) + + def upsert(self, *, documents, ids, metadatas=None, embeddings=None): + _validate_write_batch( + documents=documents, + ids=ids, + metadatas=metadatas, + embeddings=embeddings, + ) + if embeddings is None: + raise ValueError("qdrant requires explicit embeddings") + vectors, dimension = _normalize_vectors(embeddings) + self._ensure_remote_collection(dimension) + metadatas = metadatas or [{} for _ in ids] + points = [] + for doc_id, doc, meta, vector in zip(ids, documents, metadatas, vectors): + points.append( + { + "id": _point_id(doc_id), + "vector": vector, + "payload": { + _PAYLOAD_ID: str(doc_id), + _PAYLOAD_DOCUMENT: str(doc), + _PAYLOAD_METADATA: _jsonable_metadata(meta), + "updated_at": _utcnow(), + }, + } + ) + self._client.upsert_points(self._remote_collection, points) + self._backend._write_marker(self._palace, self._config) + + def update(self, *, ids, documents=None, metadatas=None, embeddings=None): + if documents is None and metadatas is None and embeddings is None: + raise ValueError("update requires at least one of documents, metadatas, embeddings") + n = len(ids) + for label, value in ( + ("documents", documents), + ("metadatas", metadatas), + ("embeddings", embeddings), + ): + if value is not None and len(value) != n: + raise ValueError(f"{label} length {len(value)} does not match ids length {n}") + existing = self.get(ids=ids, include=["documents", "metadatas", "embeddings"]) + by_id = { + rid: (existing.documents[i], existing.metadatas[i], existing.embeddings[i]) + for i, rid in enumerate(existing.ids) + if existing.embeddings is not None + } + out_ids = [] + out_docs = [] + out_metas = [] + out_embeddings = [] + for idx, doc_id in enumerate(ids): + if doc_id not in by_id: + continue + prev_doc, prev_meta, prev_embedding = by_id[doc_id] + out_ids.append(doc_id) + out_docs.append(documents[idx] if documents is not None else prev_doc) + meta = dict(prev_meta or {}) + if metadatas is not None: + meta.update(metadatas[idx] or {}) + out_metas.append(meta) + out_embeddings.append(embeddings[idx] if embeddings is not None else prev_embedding) + if out_ids: + self.upsert( + documents=out_docs, + ids=out_ids, + metadatas=out_metas, + embeddings=out_embeddings, + ) + + def _query_local_exact( + self, + *, + query_embeddings: list[list[float]], + n_results: int, + where: Optional[dict], + where_document: Optional[dict], + include: Optional[list[str]], + ) -> QueryResult: + spec = _IncludeSpec.resolve(include, default_distances=True) + q_filter = None if _requires_local_filter(where, where_document) else _qdrant_filter(where) + rows = self._scroll_all(qdrant_filter=q_filter, with_vector=True) + rows = [ + row + for row in rows + if _matches_where(row["metadata"], where) + and _matches_where_document(row["document"], where_document) + ] + outer_ids: list[list[str]] = [] + outer_docs: list[list[str]] = [] + outer_metas: list[list[dict]] = [] + outer_dists: list[list[float]] = [] + outer_embeds: list[list[list[float]]] = [] + for query_vector in query_embeddings: + q = _as_vector_array(query_vector) + scored = [] + for row in rows: + distance = _vector_distance(q, row["embedding"]) + if distance is not None: + scored.append((distance, row)) + scored.sort(key=lambda item: item[0]) + top = scored[:n_results] + outer_ids.append([row["id"] for _, row in top]) + outer_docs.append([row["document"] for _, row in top] if spec.documents else []) + outer_metas.append([row["metadata"] for _, row in top] if spec.metadatas else []) + outer_dists.append([float(dist) for dist, _ in top] if spec.distances else []) + if spec.embeddings: + outer_embeds.append([row["embedding"] or [] for _, row in top]) + return QueryResult( + ids=outer_ids, + documents=outer_docs, + metadatas=outer_metas, + distances=outer_dists, + embeddings=outer_embeds if spec.embeddings else None, + ) + + def query( + self, + *, + query_texts=None, + query_embeddings=None, + n_results=10, + where=None, + where_document=None, + include=None, + ) -> QueryResult: + if query_texts is not None: + raise ValueError("qdrant requires query_embeddings; use palace.get_collection wrapper") + if query_embeddings is None: + raise ValueError("query requires query_embeddings") + if not query_embeddings: + raise ValueError("query input must be a non-empty list") + _validate_where(where) + _validate_where(where_document) + if _requires_local_filter(where, where_document): + return self._query_local_exact( + query_embeddings=query_embeddings, + n_results=n_results, + where=where, + where_document=where_document, + include=include, + ) + if not self._remote_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return QueryResult.empty( + num_queries=len(query_embeddings), + embeddings_requested=bool(include and "embeddings" in include), + ) + + spec = _IncludeSpec.resolve(include, default_distances=True) + q_filter = _qdrant_filter(where) + outer_ids: list[list[str]] = [] + outer_docs: list[list[str]] = [] + outer_metas: list[list[dict]] = [] + outer_dists: list[list[float]] = [] + outer_embeds: list[list[list[float]]] = [] + for query_vector in query_embeddings: + q = _as_vector_array(query_vector) + if self._known_dimension is None: + self._known_dimension = self._remote_dimension() + if self._known_dimension is not None and int(q.size) != self._known_dimension: + raise DimensionMismatchError( + f"qdrant collection {self._collection_name!r} expects " + f"embedding dimension {self._known_dimension}, got {int(q.size)}" + ) + points = self._client.query_points( + self._remote_collection, + vector=q.astype(float).tolist(), + limit=n_results, + qdrant_filter=q_filter, + with_vector=spec.embeddings, + ) + rows = [_payload_row(point) for point in points] + outer_ids.append([row["id"] for row in rows]) + outer_docs.append([row["document"] for row in rows] if spec.documents else []) + outer_metas.append([row["metadata"] for row in rows] if spec.metadatas else []) + outer_dists.append( + [_qdrant_score_to_distance(row["score"]) for row in rows] if spec.distances else [] + ) + if spec.embeddings: + outer_embeds.append([row["embedding"] or [] for row in rows]) + return QueryResult( + ids=outer_ids, + documents=outer_docs, + metadatas=outer_metas, + distances=outer_dists, + embeddings=outer_embeds if spec.embeddings else None, + ) + + def get( + self, + *, + ids=None, + where=None, + where_document=None, + limit=None, + offset=None, + include=None, + ) -> GetResult: + spec = _IncludeSpec.resolve(include, default_distances=False) + rows = self._rows( + ids=ids, + where=where, + where_document=where_document, + with_vector=spec.embeddings, + ) + if ids is not None: + by_id = {row["id"]: row for row in rows} + rows = [by_id[doc_id] for doc_id in ids if doc_id in by_id] + if offset: + rows = rows[offset:] + if limit is not None: + rows = rows[:limit] + return GetResult( + ids=[row["id"] for row in rows], + documents=[row["document"] for row in rows] if spec.documents else [], + metadatas=[row["metadata"] for row in rows] if spec.metadatas else [], + embeddings=[row["embedding"] or [] for row in rows] if spec.embeddings else None, + ) + + def delete(self, *, ids=None, where=None): + _validate_where(where) + if not self._remote_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return + if ids is not None and where is None: + self._client.delete_points( + self._remote_collection, + point_ids=[_point_id(doc_id) for doc_id in ids], + ) + return + if ids is None and where is not None and not _requires_local_filter(where): + q_filter = _qdrant_filter(where) + self._client.delete_points(self._remote_collection, qdrant_filter=q_filter) + return + rows = self._rows(ids=ids, where=where) + if rows: + self._client.delete_points( + self._remote_collection, + point_ids=[_point_id(row["id"]) for row in rows], + ) + + def count(self) -> int: + self._ensure_open() + if not self._remote_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return 0 + return self._client.count_points(self._remote_collection) + + def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None): + _validate_where(where) + q_filter = None if _requires_local_filter(where) else _qdrant_filter(where) + rows = [] + text_filter = _text_any_filter(query) + text_filter_success = False + if text_filter: + try: + rows = self._scroll_all( + qdrant_filter=_combine_filters(q_filter, text_filter), + with_vector=False, + ) + text_filter_success = True + except BackendError: + logger.debug( + "Qdrant text filter failed; falling back to lexical scan", exc_info=True + ) + rows = [] + if not text_filter_success: + rows = self._scroll_all(qdrant_filter=q_filter, with_vector=False) + rows = [row for row in rows if _matches_where(row["metadata"], where)] + scores = _bm25_scores(query, [row["document"] for row in rows]) + hits = [ + LexicalHit( + id=row["id"], + document=row["document"], + metadata=row["metadata"], + score=score, + ) + for row, score in zip(rows, scores) + if score > 0 + ] + hits.sort(key=lambda hit: hit.score, reverse=True) + return LexicalResult(hits=hits[:n_results]) + + def close(self) -> None: + self._closed = True + + def health(self) -> HealthStatus: + if self._closed or self._backend._closed: + return HealthStatus.unhealthy("collection closed") + try: + if not self._client.collection_exists(self._remote_collection): + return HealthStatus.unhealthy("qdrant collection not found") + except Exception as exc: # noqa: BLE001 - backend health should summarize + return HealthStatus.unhealthy(str(exc)) + return HealthStatus.healthy() + + +class QdrantBackend(BaseBackend): + name = "qdrant" + capabilities = frozenset( + { + "requires_explicit_embeddings", + "supports_embeddings_in", + "supports_embeddings_passthrough", + "supports_embeddings_out", + "supports_metadata_filters", + "supports_lexical_search", + "supports_namespace_isolation", + "server_mode", + } + ) + + def __init__(self): + self._clients: dict[_QdrantConfig, _QdrantRESTClient] = {} + self._collections_by_palace: dict[str, list[QdrantCollection]] = {} + self._lock = threading.RLock() + self._closed = False + + @staticmethod + def _marker_path(palace_path: str) -> str: + return os.path.join(palace_path, _MARKER_FILENAME) + + @staticmethod + def _palace_hash(palace: PalaceRef) -> str: + return sha256(palace.id.encode("utf-8", errors="surrogatepass")).hexdigest()[:16] + + def _remote_collection_prefix(self, *, palace: PalaceRef, config: _QdrantConfig) -> str: + parts = ["mempalace"] + if config.namespace: + parts.append(_slug(config.namespace, "namespace")) + parts.append(self._palace_hash(palace)) + return "_".join(parts) + + def _marker_target(self, palace: PalaceRef, config: _QdrantConfig) -> dict: + return { + "url": config.url, + "namespace": config.namespace, + "palace_hash": self._palace_hash(palace), + "remote_prefix": self._remote_collection_prefix(palace=palace, config=config), + } + + def _marker_exists(self, palace: PalaceRef) -> bool: + return bool(palace.local_path and os.path.isfile(self._marker_path(palace.local_path))) + + def _read_marker(self, palace: PalaceRef) -> Optional[dict]: + if not palace.local_path: + return None + marker_path = self._marker_path(palace.local_path) + if not os.path.isfile(marker_path): + return None + try: + with open(marker_path, encoding="utf-8") as f: + marker = json.load(f) + except (OSError, json.JSONDecodeError) as exc: + raise BackendMismatchError(f"qdrant marker is unreadable: {marker_path}") from exc + return marker if isinstance(marker, dict) else {} + + def _validate_marker_target(self, palace: PalaceRef, config: _QdrantConfig) -> None: + marker = self._read_marker(palace) + if marker is None: + return + if marker.get("backend") != self.name: + raise BackendMismatchError("qdrant marker does not identify the qdrant backend") + expected = self._marker_target(palace, config) + actual = marker.get("qdrant") + if not isinstance(actual, dict): + raise BackendMismatchError("qdrant marker is missing remote target metadata") + mismatched = [ + key for key, expected_value in expected.items() if actual.get(key) != expected_value + ] + if mismatched: + details = ", ".join(mismatched) + raise BackendMismatchError( + "qdrant marker remote target does not match current configuration " + f"({details}); keep MEMPALACE_QDRANT_URL and namespace consistent " + "or use a fresh palace directory" + ) + + def _write_marker(self, palace: PalaceRef, config: _QdrantConfig) -> None: + if not palace.local_path: + return + os.makedirs(palace.local_path, exist_ok=True) + try: + os.chmod(palace.local_path, 0o700) + except (OSError, NotImplementedError): + pass + marker = { + "backend": self.name, + "schema_version": 1, + "created_at": _utcnow(), + "palace_id": palace.id, + "qdrant": self._marker_target(palace, config), + } + marker_path = self._marker_path(palace.local_path) + with open(marker_path, "w", encoding="utf-8") as f: + json.dump(marker, f, indent=2, ensure_ascii=False) + try: + os.chmod(marker_path, 0o600) + except (OSError, NotImplementedError): + pass + + # Embedder identity lives in a sidecar, NOT the backend marker: the marker's + # presence signals "palace initialized" (reads raise CollectionNotInitialized + # when the marker exists but the remote collection doesn't), so recording + # identity at first empty open must not create it. The sidecar is unguarded, + # so a brand-new palace can record identity immediately. + @staticmethod + def _embedder_sidecar_path(palace: PalaceRef) -> Optional[str]: + if not palace.local_path: + return None + return os.path.join(palace.local_path, EMBEDDER_SIDECAR_FILENAME) + + def _get_embedder_identity(self, palace: PalaceRef, collection_name: str): + return read_embedder_sidecar(self._embedder_sidecar_path(palace), collection_name) + + def _set_embedder_identity(self, palace: PalaceRef, collection_name: str, identity) -> None: + write_embedder_sidecar(self._embedder_sidecar_path(palace), collection_name, identity) + + def _client(self, config: _QdrantConfig) -> _QdrantRESTClient: + if self._closed: + raise BackendClosedError("QdrantBackend has been closed") + with self._lock: + client = self._clients.get(config) + if client is None: + client = _QdrantRESTClient(config) + self._clients[config] = client + return client + + def _remote_collection_name( + self, + *, + palace: PalaceRef, + collection_name: str, + config: _QdrantConfig, + ) -> str: + config = _QdrantConfig( + url=config.url, + api_key=config.api_key, + timeout=config.timeout, + namespace=palace.namespace or config.namespace, + ) + prefix = self._remote_collection_prefix(palace=palace, config=config) + return f"{prefix}_{_slug(collection_name, 'collection')}" + + def get_collection( + self, + *args, + **kwargs, + ) -> QdrantCollection: + palace, collection_name, create, options = self._normalize_args(args, kwargs) + config = _QdrantConfig.from_options(options) + if palace.namespace and palace.namespace != config.namespace: + config = _QdrantConfig( + url=config.url, + api_key=config.api_key, + timeout=config.timeout, + namespace=palace.namespace, + ) + client = self._client(config) + if palace.local_path: + marker_path = self._marker_path(palace.local_path) + if os.path.isfile(marker_path): + self._validate_marker_target(palace, config) + elif not create: + raise PalaceNotFoundError(marker_path) + else: + # The qdrant marker is this backend's only mismatch-protection + # anchor, and it lives next to the palace on local disk. With no + # local_path (the pure-remote / hosted mode) we can neither write + # nor validate it, so opening would silently drop protection + # against URL/namespace drift. Refuse loudly instead. A remote + # marker store for pure-remote palaces is tracked as a follow-up. + raise BackendError( + "qdrant backend requires a local palace path to anchor mismatch " + "protection; pure-remote palaces (local_path=None) are not " + "supported yet" + ) + remote_collection = self._remote_collection_name( + palace=palace, + collection_name=collection_name, + config=config, + ) + if not create and not client.collection_exists(remote_collection): + raise CollectionNotInitializedError(collection_name) + collection = QdrantCollection( + backend=self, + client=client, + config=config, + palace=palace, + collection_name=collection_name, + remote_collection=remote_collection, + ) + with self._lock: + self._collections_by_palace.setdefault(palace.id, []).append(collection) + return collection + + @staticmethod + def _normalize_args(args, kwargs): + if "palace" in kwargs: + palace = kwargs.pop("palace") + if not isinstance(palace, PalaceRef): + raise TypeError("palace= must be a PalaceRef instance") + collection_name = kwargs.pop("collection_name") + create = bool(kwargs.pop("create", False)) + options = kwargs.pop("options", None) + if args or kwargs: + raise TypeError("unexpected arguments to get_collection") + return palace, collection_name, create, options + if args: + palace_path = args[0] + rest = list(args[1:]) + collection_name = kwargs.pop("collection_name", None) or (rest.pop(0) if rest else None) + if collection_name is None: + raise TypeError("collection_name is required") + create = kwargs.pop("create", False) + if rest: + create = rest.pop(0) + options = kwargs.pop("options", None) + if rest or kwargs: + raise TypeError("unexpected arguments to get_collection") + return ( + PalaceRef(id=palace_path, local_path=palace_path), + collection_name, + bool(create), + options, + ) + if "palace_path" in kwargs: + palace_path = kwargs.pop("palace_path") + collection_name = kwargs.pop("collection_name") + create = bool(kwargs.pop("create", False)) + options = kwargs.pop("options", None) + if kwargs: + raise TypeError("unexpected arguments to get_collection") + return ( + PalaceRef(id=palace_path, local_path=palace_path), + collection_name, + create, + options, + ) + raise TypeError("get_collection requires palace= or a positional palace_path") + + def close_palace(self, palace: PalaceRef | str) -> None: + palace_id = palace.id if isinstance(palace, PalaceRef) else palace + with self._lock: + collections = self._collections_by_palace.pop(palace_id, []) + for collection in collections: + collection.close() + + def close(self) -> None: + with self._lock: + collections = [ + collection + for palace_collections in self._collections_by_palace.values() + for collection in palace_collections + ] + self._collections_by_palace.clear() + self._clients.clear() + self._closed = True + for collection in collections: + collection.close() + + def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus: + if self._closed: + return HealthStatus.unhealthy("backend closed") + try: + client = self._client(_QdrantConfig.from_options()) + client.request("GET", "/collections") + except Exception as exc: # noqa: BLE001 - user-facing health status + return HealthStatus.unhealthy(str(exc)) + if ( + palace + and palace.local_path + and not os.path.isfile(self._marker_path(palace.local_path)) + ): + return HealthStatus.unhealthy("qdrant marker not found") + return HealthStatus.healthy() + + @classmethod + def detect(cls, path: str) -> bool: + return os.path.isfile(os.path.join(path, _MARKER_FILENAME)) + + def create_collection(self, palace_path: str, collection_name: str) -> QdrantCollection: + return self.get_collection(palace_path, collection_name, create=True) + + def get_or_create_collection(self, palace_path: str, collection_name: str): + return self.get_collection(palace_path, collection_name, create=True) + + def delete_collection(self, palace_path: str, collection_name: str) -> None: + palace = PalaceRef(id=palace_path, local_path=palace_path) + config = _QdrantConfig.from_options() + remote_collection = self._remote_collection_name( + palace=palace, + collection_name=collection_name, + config=config, + ) + client = self._client(config) + if client.collection_exists(remote_collection): + client.delete_collection(remote_collection) + + +__all__ = ["QdrantBackend", "QdrantCollection"] diff --git a/mempalace/backends/registry.py b/mempalace/backends/registry.py index 7551bd3..1aaa6b4 100644 --- a/mempalace/backends/registry.py +++ b/mempalace/backends/registry.py @@ -125,6 +125,38 @@ def get_backend(name: str) -> BaseBackend: return inst +def detect_backends_for_path(path: str) -> list[str]: + """Return all registered backend names whose artifacts are present at ``path``. + + Detection is a migration/protection aid for local palaces. Backends are + checked in registry-name order so callers get deterministic diagnostics if + a broken directory contains artifacts from more than one backend. + """ + _discover_entry_points() + detected = [] + for name in sorted(_registry): + cls = _registry[name] + try: + if cls.detect(path): + detected.append(name) + except Exception: + logger.exception("detect() raised on backend %r", name) + return detected + + +def detect_backend_for_path(path: str) -> Optional[str]: + """Return the single detected backend at ``path``, or ``None``. + + If multiple backend artifacts are present, the first name in registry order + is returned for backward compatibility. Callers that enforce mismatch + protection should use :func:`detect_backends_for_path`. + """ + detected = detect_backends_for_path(path) + if detected: + return detected[0] + return None + + def reset_backends() -> None: """Close and drop all cached backend instances (primarily for tests).""" with _lock: @@ -161,14 +193,9 @@ def resolve_backend_for_palace( return candidate _discover_entry_points() - if palace_path: - for name, cls in _registry.items(): - try: - if cls.detect(palace_path): - return name - except Exception: - logger.exception("detect() raised on backend %r", name) - continue + detected = detect_backend_for_path(palace_path) if palace_path else None + if detected: + return detected return default @@ -180,10 +207,19 @@ def resolve_backend_for_palace( def _register_builtins() -> None: """Register chroma as the in-tree default.""" from .chroma import ChromaBackend + from .pgvector import PgVectorBackend + from .qdrant import QdrantBackend + from .sqlite_exact import SQLiteExactBackend # Use setdefault semantics so a caller that pre-registered for tests wins. if "chroma" not in _registry: _registry["chroma"] = ChromaBackend + if "qdrant" not in _registry: + _registry["qdrant"] = QdrantBackend + if "sqlite_exact" not in _registry: + _registry["sqlite_exact"] = SQLiteExactBackend + if "pgvector" not in _registry: + _registry["pgvector"] = PgVectorBackend _register_builtins() diff --git a/mempalace/backends/sqlite_exact.py b/mempalace/backends/sqlite_exact.py new file mode 100644 index 0000000..f1b5ced --- /dev/null +++ b/mempalace/backends/sqlite_exact.py @@ -0,0 +1,1030 @@ +"""SQLite exact-vector backend for MemPalace. + +This backend is intentionally simple and local-first. It is a correctness +backend, not a high-throughput ANN backend: vectors are stored as float32 +blobs and query uses exact cosine distance over the matching collection. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import re +import sqlite3 +import threading +from datetime import datetime, timezone +from typing import Any, Optional + +import numpy as np + +from .base import ( + BackendClosedError, + BaseBackend, + BaseCollection, + CollectionNotInitializedError, + DimensionMismatchError, + GetResult, + HealthStatus, + LexicalHit, + LexicalResult, + PalaceNotFoundError, + PalaceRef, + QueryResult, + UnsupportedFilterError, + _IncludeSpec, +) + +logger = logging.getLogger(__name__) + +_DB_FILENAME = "sqlite_exact.sqlite3" +_TOKEN_RE = re.compile(r"\w{2,}", re.UNICODE) +_SUPPORTED_OPERATORS = frozenset( + {"$eq", "$ne", "$in", "$nin", "$and", "$or", "$contains", "$gt", "$gte", "$lt", "$lte"} +) + + +def _utcnow() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _json_dumps(obj: Any) -> str: + return json.dumps(obj or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _json_loads(text: str | None) -> dict: + if not text: + return {} + try: + value = json.loads(text) + except json.JSONDecodeError: + return {} + return value if isinstance(value, dict) else {} + + +def _encode_vector(vector: list[float]) -> bytes: + return _as_vector_array(vector).tobytes() + + +def _as_vector_array(vector: list[float]) -> np.ndarray: + arr = np.asarray(vector, dtype=np.float32) + if arr.ndim != 1 or arr.size == 0: + raise ValueError("embedding must be a non-empty 1D vector") + return arr + + +def _decode_vector(blob: bytes | None) -> list[float]: + if not blob: + return [] + return np.frombuffer(blob, dtype=np.float32).astype(float).tolist() + + +def _decode_array(blob: bytes | None) -> Optional[np.ndarray]: + if not blob: + return None + arr = np.frombuffer(blob, dtype=np.float32) + if arr.size == 0: + return None + return arr + + +def _tokenize(text: str) -> list[str]: + if not text: + return [] + return _TOKEN_RE.findall(text.lower()) + + +def _bm25_scores(query: str, documents: list[str], k1: float = 1.5, b: float = 0.75) -> list[float]: + query_terms = set(_tokenize(query)) + n_docs = len(documents) + if not query_terms or n_docs == 0: + return [0.0] * n_docs + + tokenized = [_tokenize(d) for d in documents] + doc_lens = [len(toks) for toks in tokenized] + if not any(doc_lens): + return [0.0] * n_docs + avgdl = sum(doc_lens) / n_docs or 1.0 + + df = {term: 0 for term in query_terms} + for toks in tokenized: + for term in set(toks) & query_terms: + df[term] += 1 + + idf = {term: np.log((n_docs - df[term] + 0.5) / (df[term] + 0.5) + 1.0) for term in query_terms} + + scores = [] + for toks, dl in zip(tokenized, doc_lens): + if dl == 0: + scores.append(0.0) + continue + tf: dict[str, int] = {} + for token in toks: + if token in query_terms: + tf[token] = tf.get(token, 0) + 1 + score = 0.0 + for term, freq in tf.items(): + num = freq * (k1 + 1) + den = freq + k1 * (1 - b + b * dl / avgdl) + score += float(idf[term]) * num / den + scores.append(score) + return scores + + +def _validate_where(where: Optional[dict]) -> None: + if not where: + return + stack = [where] + while stack: + node = stack.pop() + if not isinstance(node, dict): + continue + for key, value in node.items(): + if key.startswith("$") and key not in _SUPPORTED_OPERATORS: + raise UnsupportedFilterError(f"operator {key!r} not supported by sqlite_exact") + if isinstance(value, dict): + stack.append(value) + elif isinstance(value, list): + stack.extend(item for item in value if isinstance(item, dict)) + + +def _coerce_comparable(value: Any): + if isinstance(value, bool): + return int(value) + return value + + +def _compare(actual: Any, op: str, expected: Any) -> bool: + actual = _coerce_comparable(actual) + expected = _coerce_comparable(expected) + if op == "$eq": + return actual == expected + if op == "$ne": + return actual != expected + if op == "$in": + return actual in (expected or []) + if op == "$nin": + return actual not in (expected or []) + if op == "$contains": + return str(expected) in str(actual or "") + try: + if op == "$gt": + return actual > expected + if op == "$gte": + return actual >= expected + if op == "$lt": + return actual < expected + if op == "$lte": + return actual <= expected + except TypeError: + return False + raise UnsupportedFilterError(f"operator {op!r} not supported by sqlite_exact") + + +def _matches_where(meta: dict, where: Optional[dict]) -> bool: + if not where: + return True + if not isinstance(where, dict): + return False + for key, expected in where.items(): + if key == "$and": + if not all(_matches_where(meta, clause) for clause in expected or []): + return False + continue + if key == "$or": + if not any(_matches_where(meta, clause) for clause in expected or []): + return False + continue + if key.startswith("$"): + raise UnsupportedFilterError(f"operator {key!r} not supported by sqlite_exact") + actual = meta.get(key) + if isinstance(expected, dict): + for op, operand in expected.items(): + if not _compare(actual, op, operand): + return False + elif actual != expected: + return False + return True + + +def _matches_where_document(document: str, where_document: Optional[dict]) -> bool: + if not where_document: + return True + if not isinstance(where_document, dict): + return False + for key, value in where_document.items(): + if key == "$contains": + if str(value) not in document: + return False + continue + if key == "$and": + if not all(_matches_where_document(document, clause) for clause in value or []): + return False + continue + if key == "$or": + if not any(_matches_where_document(document, clause) for clause in value or []): + return False + continue + raise UnsupportedFilterError(f"where_document operator {key!r} not supported") + return True + + +def _validate_write_batch( + *, + documents: list[str], + ids: list[str], + metadatas: Optional[list[dict]], + embeddings: Optional[list[list[float]]], +) -> None: + n = len(ids) + if len(documents) != n: + raise ValueError(f"documents length {len(documents)} does not match ids length {n}") + if metadatas is not None and len(metadatas) != n: + raise ValueError(f"metadatas length {len(metadatas)} does not match ids length {n}") + if embeddings is not None and len(embeddings) != n: + raise ValueError(f"embeddings length {len(embeddings)} does not match ids length {n}") + + +class _SQLiteExactHandle: + def __init__(self, conn: sqlite3.Connection, lock: threading.RLock): + self.conn = conn + self.lock = lock + self.closed = False + + +class SQLiteExactCollection(BaseCollection): + def __init__(self, handle: _SQLiteExactHandle, collection_name: str): + self._handle = handle + self._collection_name = collection_name + self._closed = False + + def _ensure_open(self) -> None: + if self._closed or self._handle.closed: + raise BackendClosedError("SQLiteExactCollection has been closed") + + @contextlib.contextmanager + def _cursor(self): + with self._handle.lock: + self._ensure_open() + cur = self._handle.conn.cursor() + try: + yield cur + except Exception: + self._handle.conn.rollback() + raise + else: + self._handle.conn.commit() + finally: + cur.close() + + def _collection_id(self, cur) -> int: + row = cur.execute( + "SELECT id FROM collections WHERE name = ?", + (self._collection_name,), + ).fetchone() + if row is None: + raise CollectionNotInitializedError(self._collection_name) + return int(row[0]) + + def _collection_dimension(self, cur, collection_id: int) -> Optional[int]: + row = cur.execute( + "SELECT dimension FROM collections WHERE id = ?", + (collection_id,), + ).fetchone() + if row is None or row[0] is None: + return None + return int(row[0]) + + def _ensure_collection_dimension(self, cur, collection_id: int, dims: list[int]) -> None: + distinct = {int(dim) for dim in dims} + if not distinct: + return + if len(distinct) > 1: + raise DimensionMismatchError( + f"sqlite_exact collection {self._collection_name!r} cannot mix " + f"embedding dimensions {sorted(distinct)}" + ) + dim = distinct.pop() + stored = self._collection_dimension(cur, collection_id) + if stored is None: + cur.execute( + "UPDATE collections SET dimension = ? WHERE id = ?", + (dim, collection_id), + ) + elif stored != dim: + raise DimensionMismatchError( + f"sqlite_exact collection {self._collection_name!r} expects " + f"embedding dimension {stored}, got {dim}" + ) + + def _fts_available(self, cur) -> bool: + row = cur.execute("SELECT value FROM meta WHERE key = 'fts5_available'").fetchone() + return bool(row and row[0] == "1") + + def _embedder_meta_key(self) -> str: + return f"embedder_model:{self._collection_name}" + + def get_stored_embedder_identity(self): + from .base import EmbedderIdentity + + with self._cursor() as cur: + try: + cid = self._collection_id(cur) + except CollectionNotInitializedError: + return None + row = cur.execute( + "SELECT value FROM meta WHERE key = ?", + (self._embedder_meta_key(),), + ).fetchone() + if not row or not row[0]: + return None + dim = self._collection_dimension(cur, cid) or 0 + return EmbedderIdentity(model_name=str(row[0]), dimension=int(dim)) + + def set_embedder_identity(self, identity) -> None: + if not identity or not identity.model_name: + return + with self._cursor() as cur: + cur.execute( + "INSERT INTO meta(key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (self._embedder_meta_key(), str(identity.model_name)), + ) + + def _replace_fts(self, cur, collection_id: int, doc_id: str, document: str) -> None: + if not self._fts_available(cur): + return + cur.execute( + "DELETE FROM docs_fts WHERE collection_id = ? AND doc_id = ?", + (collection_id, doc_id), + ) + cur.execute( + "INSERT INTO docs_fts(collection_id, doc_id, document) VALUES (?, ?, ?)", + (collection_id, doc_id, document), + ) + + def add(self, *, documents, ids, metadatas=None, embeddings=None): + _validate_write_batch( + documents=documents, + ids=ids, + metadatas=metadatas, + embeddings=embeddings, + ) + if embeddings is None: + raise ValueError("sqlite_exact requires explicit embeddings") + metadatas = metadatas or [{} for _ in ids] + now = _utcnow() + with self._cursor() as cur: + collection_id = self._collection_id(cur) + prepared = [] + for doc_id, doc, meta, emb in zip(ids, documents, metadatas, embeddings): + arr = _as_vector_array(emb) + prepared.append((doc_id, doc, meta, arr.tobytes(), int(arr.size))) + self._ensure_collection_dimension(cur, collection_id, [item[4] for item in prepared]) + for doc_id, doc, meta, emb_blob, dim in prepared: + cur.execute( + """ + INSERT INTO documents + (collection_id, id, document, metadata_json, embedding, dim, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + collection_id, + doc_id, + doc, + _json_dumps(meta), + emb_blob, + dim, + now, + now, + ), + ) + self._replace_fts(cur, collection_id, doc_id, doc) + + def upsert(self, *, documents, ids, metadatas=None, embeddings=None): + _validate_write_batch( + documents=documents, + ids=ids, + metadatas=metadatas, + embeddings=embeddings, + ) + if embeddings is None: + raise ValueError("sqlite_exact requires explicit embeddings") + metadatas = metadatas or [{} for _ in ids] + now = _utcnow() + with self._cursor() as cur: + collection_id = self._collection_id(cur) + prepared = [] + for doc_id, doc, meta, emb in zip(ids, documents, metadatas, embeddings): + arr = _as_vector_array(emb) + prepared.append((doc_id, doc, meta, arr.tobytes(), int(arr.size))) + self._ensure_collection_dimension(cur, collection_id, [item[4] for item in prepared]) + for doc_id, doc, meta, emb_blob, dim in prepared: + cur.execute( + """ + INSERT INTO documents + (collection_id, id, document, metadata_json, embedding, dim, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(collection_id, id) DO UPDATE SET + document = excluded.document, + metadata_json = excluded.metadata_json, + embedding = excluded.embedding, + dim = excluded.dim, + updated_at = excluded.updated_at + """, + ( + collection_id, + doc_id, + doc, + _json_dumps(meta), + emb_blob, + dim, + now, + now, + ), + ) + self._replace_fts(cur, collection_id, doc_id, doc) + + def update(self, *, ids, documents=None, metadatas=None, embeddings=None): + if documents is None and metadatas is None and embeddings is None: + raise ValueError("update requires at least one of documents, metadatas, embeddings") + n = len(ids) + for label, value in ( + ("documents", documents), + ("metadatas", metadatas), + ("embeddings", embeddings), + ): + if value is not None and len(value) != n: + raise ValueError(f"{label} length {len(value)} does not match ids length {n}") + with self._cursor() as cur: + collection_id = self._collection_id(cur) + updates = [] + for idx, doc_id in enumerate(ids): + row = cur.execute( + """ + SELECT document, metadata_json, embedding, dim + FROM documents + WHERE collection_id = ? AND id = ? + """, + (collection_id, doc_id), + ).fetchone() + if row is None: + continue + doc = documents[idx] if documents is not None else row[0] + meta = _json_loads(row[1]) + if metadatas is not None: + meta.update(metadatas[idx] or {}) + if embeddings is not None: + arr = _as_vector_array(embeddings[idx]) + emb_blob = arr.tobytes() + dim = int(arr.size) + else: + emb_blob = row[2] + dim = row[3] + updates.append((doc_id, doc, meta, emb_blob, dim)) + if embeddings is not None: + self._ensure_collection_dimension(cur, collection_id, [item[4] for item in updates]) + for doc_id, doc, meta, emb_blob, dim in updates: + cur.execute( + """ + UPDATE documents + SET document = ?, metadata_json = ?, embedding = ?, dim = ?, updated_at = ? + WHERE collection_id = ? AND id = ? + """, + (doc, _json_dumps(meta), emb_blob, dim, _utcnow(), collection_id, doc_id), + ) + self._replace_fts(cur, collection_id, doc_id, doc) + + def _rows(self, cur, *, where=None, where_document=None) -> list[dict]: + _validate_where(where) + _validate_where(where_document) + collection_id = self._collection_id(cur) + rows = cur.execute( + """ + SELECT id, document, metadata_json, embedding + FROM documents + WHERE collection_id = ? + ORDER BY rowid + """, + (collection_id,), + ).fetchall() + out = [] + for doc_id, doc, meta_json, emb_blob in rows: + meta = _json_loads(meta_json) + if not _matches_where(meta, where): + continue + if not _matches_where_document(doc or "", where_document): + continue + out.append( + { + "id": doc_id, + "document": doc or "", + "metadata": meta, + "embedding": emb_blob, + } + ) + return out + + def query( + self, + *, + query_texts=None, + query_embeddings=None, + n_results=10, + where=None, + where_document=None, + include=None, + ) -> QueryResult: + if query_texts is not None: + raise ValueError( + "sqlite_exact requires query_embeddings; use palace.get_collection wrapper" + ) + if query_embeddings is None: + raise ValueError("query requires query_embeddings") + if not query_embeddings: + raise ValueError("query input must be a non-empty list") + + spec = _IncludeSpec.resolve(include, default_distances=True) + outer_ids: list[list[str]] = [] + outer_docs: list[list[str]] = [] + outer_metas: list[list[dict]] = [] + outer_dists: list[list[float]] = [] + outer_embeds: list[list[list[float]]] = [] + + with self._cursor() as cur: + collection_id = self._collection_id(cur) + expected_dim = self._collection_dimension(cur, collection_id) + rows = self._rows(cur, where=where, where_document=where_document) + row_vectors = [(row, _decode_array(row["embedding"])) for row in rows] + + for query_vector in query_embeddings: + q = _as_vector_array(query_vector) + if expected_dim is not None and int(q.size) != expected_dim: + raise DimensionMismatchError( + f"sqlite_exact collection {self._collection_name!r} expects " + f"embedding dimension {expected_dim}, got {int(q.size)}" + ) + q_norm = float(np.linalg.norm(q)) + scored = [] + for row, vec in row_vectors: + if vec is None or vec.size != q.size: + continue + denom = q_norm * float(np.linalg.norm(vec)) + cos = 0.0 if denom <= 0 else float(np.dot(q, vec) / denom) + distance = 1.0 - max(-1.0, min(1.0, cos)) + scored.append((distance, row, vec)) + scored.sort(key=lambda item: item[0]) + top = scored[:n_results] + + outer_ids.append([row["id"] for _, row, _ in top]) + outer_docs.append([row["document"] for _, row, _ in top] if spec.documents else []) + outer_metas.append([row["metadata"] for _, row, _ in top] if spec.metadatas else []) + outer_dists.append([float(dist) for dist, _, _ in top] if spec.distances else []) + if spec.embeddings: + outer_embeds.append([vec.astype(float).tolist() for _, _, vec in top]) + + return QueryResult( + ids=outer_ids, + documents=outer_docs, + metadatas=outer_metas, + distances=outer_dists, + embeddings=outer_embeds if spec.embeddings else None, + ) + + def get( + self, + *, + ids=None, + where=None, + where_document=None, + limit=None, + offset=None, + include=None, + ) -> GetResult: + spec = _IncludeSpec.resolve(include, default_distances=False) + with self._cursor() as cur: + rows = self._rows(cur, where=where, where_document=where_document) + if ids is not None: + by_id = {row["id"]: row for row in rows} + rows = [by_id[doc_id] for doc_id in ids if doc_id in by_id] + if offset: + rows = rows[offset:] + if limit is not None: + rows = rows[:limit] + return GetResult( + ids=[row["id"] for row in rows], + documents=[row["document"] for row in rows] if spec.documents else [], + metadatas=[row["metadata"] for row in rows] if spec.metadatas else [], + embeddings=( + [_decode_vector(row["embedding"]) for row in rows] if spec.embeddings else None + ), + ) + + def delete(self, *, ids=None, where=None): + with self._cursor() as cur: + collection_id = self._collection_id(cur) + if ids is None: + rows = self._rows(cur, where=where) + ids = [row["id"] for row in rows] + for doc_id in ids or []: + cur.execute( + "DELETE FROM documents WHERE collection_id = ? AND id = ?", + (collection_id, doc_id), + ) + if self._fts_available(cur): + cur.execute( + "DELETE FROM docs_fts WHERE collection_id = ? AND doc_id = ?", + (collection_id, doc_id), + ) + + def count(self) -> int: + with self._cursor() as cur: + collection_id = self._collection_id(cur) + row = cur.execute( + "SELECT COUNT(*) FROM documents WHERE collection_id = ?", + (collection_id,), + ).fetchone() + return int(row[0]) if row else 0 + + def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None): + _validate_where(where) + with self._cursor() as cur: + hits = self._lexical_search_fts(cur, query=query, n_results=n_results, where=where) + if hits is not None: + return LexicalResult(hits=hits) + rows = self._rows(cur, where=where) + scores = _bm25_scores(query, [row["document"] for row in rows]) + scored = [ + LexicalHit( + id=row["id"], + document=row["document"], + metadata=row["metadata"], + score=score, + ) + for row, score in zip(rows, scores) + if score > 0 + ] + scored.sort(key=lambda hit: hit.score, reverse=True) + return LexicalResult(hits=scored[:n_results]) + + def _lexical_search_fts(self, cur, *, query: str, n_results: int, where: Optional[dict]): + if not self._fts_available(cur): + return None + tokens = [t for t in _tokenize(query) if len(t) >= 2] + if not tokens: + return None + fts_query = " OR ".join(tokens) + collection_id = self._collection_id(cur) + try: + limit_sql = "" if where else "LIMIT ?" + params = (fts_query, collection_id) + if not where: + params = (*params, max(n_results * 5, n_results)) + rows = cur.execute( + f""" + SELECT doc_id, bm25(docs_fts) AS rank + FROM docs_fts + WHERE docs_fts MATCH ? AND collection_id = ? + ORDER BY rank + {limit_sql} + """, + params, + ).fetchall() + except sqlite3.Error: + logger.debug("sqlite_exact FTS query failed; using Python lexical scan", exc_info=True) + return None + if not rows: + return [] + ids = [row[0] for row in rows] + docs = [] + for start in range(0, len(ids), 900): + chunk_ids = ids[start : start + 900] + placeholders = ",".join("?" for _ in chunk_ids) + docs.extend( + cur.execute( + f""" + SELECT id, document, metadata_json + FROM documents + WHERE collection_id = ? AND id IN ({placeholders}) + """, + (collection_id, *chunk_ids), + ).fetchall() + ) + by_id = {doc_id: (doc or "", _json_loads(meta_json)) for doc_id, doc, meta_json in docs} + hits = [] + for doc_id, rank in rows: + doc_meta = by_id.get(doc_id) + if doc_meta is None: + continue + doc, meta = doc_meta + if not _matches_where(meta, where): + continue + hits.append( + LexicalHit( + id=doc_id, + document=doc, + metadata=meta, + score=-float(rank), + ) + ) + if len(hits) >= n_results: + break + return hits + + def close(self) -> None: + self._closed = True + + def health(self) -> HealthStatus: + if self._closed or self._handle.closed: + return HealthStatus.unhealthy("collection closed") + return HealthStatus.healthy() + + def maintenance_state(self) -> dict: + try: + rows = self.count() + except Exception: + rows = 0 + # vector_index is null by design — exact cosine over every row, no ANN. + state = {"row_count": rows, "vector_index": None} + try: + with self._cursor() as cur: + page_count = cur.execute("PRAGMA page_count").fetchone() + freelist = cur.execute("PRAGMA freelist_count").fetchone() + state["page_count"] = int(page_count[0]) if page_count else 0 + state["freelist_pages"] = int(freelist[0]) if freelist else 0 + except Exception: + pass + return state + + def run_maintenance(self, kind: str): + from .base import MaintenanceResult, UnsupportedMaintenanceKindError + + if kind not in SQLiteExactBackend.maintenance_kinds: + raise UnsupportedMaintenanceKindError( + f"sqlite_exact does not support maintenance kind {kind!r}" + ) + if kind == "analyze": + # Refresh planner stats. Concurrent runs serialize on the handle lock. + with self._cursor() as cur: + cur.execute("ANALYZE") + return MaintenanceResult(kind="analyze", status="ran") + + # compact → VACUUM. It cannot run inside a transaction, so flip the + # connection to autocommit for the duration. The handle lock serializes + # concurrent runs in-process; SQLite's own write lock serializes across + # processes. + before = self.maintenance_state() + with self._handle.lock: + self._ensure_open() + conn = self._handle.conn + prev_isolation = conn.isolation_level + try: + conn.commit() + conn.isolation_level = None + conn.execute("VACUUM") + finally: + conn.isolation_level = prev_isolation + after = self.maintenance_state() + reclaimed = max(0, before.get("page_count", 0) - after.get("page_count", 0)) + return MaintenanceResult( + kind="compact", + status="ran", + stats={ + "pages_before": before.get("page_count", 0), + "pages_after": after.get("page_count", 0), + "pages_reclaimed": reclaimed, + }, + ) + + +class SQLiteExactBackend(BaseBackend): + name = "sqlite_exact" + capabilities = frozenset( + { + "requires_explicit_embeddings", + "supports_embeddings_in", + "supports_embeddings_passthrough", + "supports_embeddings_out", + "supports_metadata_filters", + "supports_lexical_search", + "local_mode", + } + ) + # "reindex" is intentionally omitted: sqlite_exact does exact cosine over + # every row (no ANN index to build), so it has no analogue for it. + maintenance_kinds = frozenset({"analyze", "compact"}) + + def __init__(self): + self._clients: dict[str, _SQLiteExactHandle] = {} + self._clients_lock = threading.RLock() + self._closed = False + + @staticmethod + def _db_path(palace_path: str) -> str: + return os.path.join(palace_path, _DB_FILENAME) + + def _connect(self, palace_path: str, create: bool): + if self._closed: + raise BackendClosedError("SQLiteExactBackend has been closed") + db_path = self._db_path(palace_path) + if not create and not os.path.isfile(db_path): + raise PalaceNotFoundError(db_path) + if create: + os.makedirs(palace_path, exist_ok=True) + try: + os.chmod(palace_path, 0o700) + except (OSError, NotImplementedError): + pass + with self._clients_lock: + cached = self._clients.get(palace_path) + if cached is not None and not cached.closed: + return cached + conn = sqlite3.connect(db_path, check_same_thread=False) + conn.row_factory = sqlite3.Row + lock = threading.RLock() + handle = _SQLiteExactHandle(conn, lock) + with handle.lock: + self._init_schema(conn) + with self._clients_lock: + self._clients[palace_path] = handle + return handle + + def _init_schema(self, conn: sqlite3.Connection) -> None: + conn.executescript( + """ + PRAGMA journal_mode=WAL; + CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS collections ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + dimension INTEGER, + created_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS documents ( + collection_id INTEGER NOT NULL, + id TEXT NOT NULL, + document TEXT NOT NULL, + metadata_json TEXT NOT NULL, + embedding BLOB NOT NULL, + dim INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (collection_id, id), + FOREIGN KEY(collection_id) REFERENCES collections(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_documents_collection + ON documents(collection_id); + """ + ) + columns = {row[1] for row in conn.execute("PRAGMA table_info(collections)").fetchall()} + if "dimension" not in columns: + conn.execute("ALTER TABLE collections ADD COLUMN dimension INTEGER") + try: + conn.execute( + """ + CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts + USING fts5(collection_id UNINDEXED, doc_id UNINDEXED, document) + """ + ) + conn.execute( + """ + INSERT INTO meta(key, value) + VALUES ('fts5_available', '1') + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """ + ) + except sqlite3.OperationalError: + conn.execute( + """ + INSERT INTO meta(key, value) + VALUES ('fts5_available', '0') + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """ + ) + conn.commit() + + def get_collection( + self, + *args, + **kwargs, + ) -> SQLiteExactCollection: + palace, collection_name, create = self._normalize_args(args, kwargs) + palace_path = palace.local_path + if palace_path is None: + raise PalaceNotFoundError("SQLiteExactBackend requires PalaceRef.local_path") + if not create and not os.path.isdir(palace_path): + raise PalaceNotFoundError(palace_path) + handle = self._connect(palace_path, create=create) + with handle.lock: + row = handle.conn.execute( + "SELECT id FROM collections WHERE name = ?", + (collection_name,), + ).fetchone() + if row is None: + if not create: + raise CollectionNotInitializedError(collection_name) + handle.conn.execute( + "INSERT INTO collections(name, created_at) VALUES (?, ?)", + (collection_name, _utcnow()), + ) + handle.conn.commit() + return SQLiteExactCollection(handle, collection_name) + + @staticmethod + def _normalize_args(args, kwargs): + if "palace" in kwargs: + palace = kwargs.pop("palace") + if not isinstance(palace, PalaceRef): + raise TypeError("palace= must be a PalaceRef instance") + collection_name = kwargs.pop("collection_name") + create = bool(kwargs.pop("create", False)) + kwargs.pop("options", None) + if args or kwargs: + raise TypeError("unexpected arguments to get_collection") + return palace, collection_name, create + if args: + palace_path = args[0] + rest = list(args[1:]) + collection_name = kwargs.pop("collection_name", None) or (rest.pop(0) if rest else None) + if collection_name is None: + raise TypeError("collection_name is required") + create = kwargs.pop("create", False) + if rest: + create = rest.pop(0) + if rest or kwargs: + raise TypeError("unexpected arguments to get_collection") + return PalaceRef(id=palace_path, local_path=palace_path), collection_name, bool(create) + if "palace_path" in kwargs: + palace_path = kwargs.pop("palace_path") + collection_name = kwargs.pop("collection_name") + create = bool(kwargs.pop("create", False)) + if kwargs: + raise TypeError("unexpected arguments to get_collection") + return PalaceRef(id=palace_path, local_path=palace_path), collection_name, create + raise TypeError("get_collection requires palace= or a positional palace_path") + + def close_palace(self, palace: PalaceRef | str) -> None: + path = palace.local_path if isinstance(palace, PalaceRef) else palace + if path is None: + return + with self._clients_lock: + cached = self._clients.pop(path, None) + if cached is not None: + with cached.lock: + cached.closed = True + cached.conn.close() + + def close(self) -> None: + with self._clients_lock: + handles = list(self._clients.values()) + self._clients.clear() + for handle in handles: + with handle.lock: + handle.closed = True + handle.conn.close() + self._closed = True + + def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus: + if self._closed: + return HealthStatus.unhealthy("backend closed") + if palace and palace.local_path and not os.path.isfile(self._db_path(palace.local_path)): + return HealthStatus.unhealthy("sqlite_exact database not found") + return HealthStatus.healthy() + + @classmethod + def detect(cls, path: str) -> bool: + return os.path.isfile(os.path.join(path, _DB_FILENAME)) + + def create_collection(self, palace_path: str, collection_name: str) -> SQLiteExactCollection: + return self.get_collection(palace_path, collection_name, create=True) + + def get_or_create_collection(self, palace_path: str, collection_name: str): + return self.get_collection(palace_path, collection_name, create=True) + + def delete_collection(self, palace_path: str, collection_name: str) -> None: + handle = self._connect(palace_path, create=False) + with handle.lock: + row = handle.conn.execute( + "SELECT id FROM collections WHERE name = ?", + (collection_name,), + ).fetchone() + if row is None: + raise CollectionNotInitializedError(collection_name) + collection_id = int(row[0]) + handle.conn.execute("DELETE FROM documents WHERE collection_id = ?", (collection_id,)) + try: + handle.conn.execute( + "DELETE FROM docs_fts WHERE collection_id = ?", + (collection_id,), + ) + except sqlite3.OperationalError: + pass + handle.conn.execute("DELETE FROM collections WHERE id = ?", (collection_id,)) + handle.conn.commit() + + +__all__ = ["SQLiteExactBackend", "SQLiteExactCollection"] diff --git a/mempalace/backups.py b/mempalace/backups.py new file mode 100644 index 0000000..d995081 --- /dev/null +++ b/mempalace/backups.py @@ -0,0 +1,74 @@ +"""Retention pruning for timestamped palace backups. + +``mempalace migrate`` and ``mempalace repair max-seq-id`` each write a fresh, +timestamped backup every time they run and historically never deleted the old +ones. On a machine that mines or repairs on a schedule those full-size copies +accumulate silently — a real palace was found with hundreds of gigabytes of +backups sitting beside only a few hundred megabytes of live data, nearly +filling the disk. This module prunes the backup set down to a bounded count +after each new backup is written. + +The retention count comes from ``MempalaceConfig.max_backups`` (default 10). +""" + +import glob +import os +import shutil + + +def prune_backups(pattern, max_backups, *, log=None): + """Delete the oldest backups matching ``pattern`` so at most ``max_backups`` remain. + + Args: + pattern: A glob pattern matching the backup paths (files or + directories). The caller is responsible for ``glob.escape``-ing + any literal, non-wildcard portion that can contain glob + metacharacters — palace paths sometimes do (e.g. a ``[``). + max_backups: Number of most-recent backups to keep. ``None`` or any + value ``<= 0`` disables pruning and returns immediately, so a + backup set is never touched when the user has opted out. + log: Optional callable (e.g. ``print``) for human-readable progress. + + Returns: + The list of paths that were successfully removed. + + Recency is determined by filesystem mtime rather than by parsing the + timestamp out of the name, so it stays correct even when two backup + producers use different timestamp formats. Deletion failures are logged + and skipped: pruning is best-effort cleanup and must never abort the + migrate/repair operation that just completed successfully. + """ + if max_backups is None or max_backups <= 0: + return [] + + scored = [] + for path in glob.glob(pattern): + try: + scored.append((os.path.getmtime(path), path)) + except OSError: + # Vanished between glob and stat (concurrent prune / cleanup); + # nothing for us to remove. + continue + + if len(scored) <= max_backups: + return [] + + # Newest first; the path breaks mtime ties so ordering is deterministic. + scored.sort(key=lambda item: (item[0], item[1]), reverse=True) + + removed = [] + for _mtime, path in scored[max_backups:]: + try: + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + else: + os.remove(path) + except OSError as exc: + if log: + log(f" Backup prune: could not remove {path}: {exc}") + continue + removed.append(path) + if log: + log(f" Backup prune: removed old backup {path}") + + return removed diff --git a/mempalace/cli.py b/mempalace/cli.py index c17078f..974c4fa 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -51,6 +51,45 @@ _PASS_ZERO_PER_FILE_CAP = 100_000 # 100KB per file is generous for prose _PASS_ZERO_TOTAL_CAP = 5_000_000 # 5MB total ceiling — bounds memory _PASS_ZERO_LLM_PER_SAMPLE = 2_000 # for Tier 2 LLM call only _PASS_ZERO_LLM_MAX_SAMPLES = 20 # caps the LLM-tier sample count +_EXPLICIT_BACKEND_ENV = "MEMPALACE_BACKEND_EXPLICIT" + + +def _backend_arg(args): + """Return a CLI-selected backend from subcommand or global flags.""" + return getattr(args, "backend", None) or getattr(args, "global_backend", None) + + +def _apply_backend_arg(args) -> None: + backend = _backend_arg(args) + if not backend: + return + backend = str(backend).strip().lower() + from .backends import get_backend_class + + get_backend_class(backend) + os.environ[_EXPLICIT_BACKEND_ENV] = backend + os.environ["MEMPALACE_BACKEND"] = backend + + +def _selected_backend_for_palace(palace_path: str) -> str: + from .palace import resolve_backend_name + + return resolve_backend_name(palace_path, explicit=os.environ.get(_EXPLICIT_BACKEND_ENV)) + + +def _maintenance_requires_chroma(palace_path: str, command_name: str) -> bool: + try: + backend_name = _selected_backend_for_palace(palace_path) + except Exception as exc: # noqa: BLE001 - user-facing guard before maintenance imports + print(f"\n {command_name} cannot resolve the palace backend: {exc}", file=sys.stderr) + return False + if backend_name == "chroma": + return True + print( + f"\n {command_name} is Chroma-only in this release (selected backend: {backend_name}).", + file=sys.stderr, + ) + return False def _gather_origin_samples(project_dir) -> list: @@ -380,6 +419,9 @@ def cmd_init(args): # Pass 2: detect rooms from folder structure detect_rooms_local(project_dir=args.dir, yes=getattr(args, "yes", False)) cfg.init() + backend = _backend_arg(args) + if backend: + cfg.set_backend(backend) # Pass 3: protect git repos from accidentally committing per-project files _ensure_mempalace_files_gitignored(args.dir) @@ -615,6 +657,8 @@ def cmd_sync(args): """Prune drawers whose source files are gitignored, deleted, or moved (#1252).""" from .mcp_server import _wal_log from .palace import MineAlreadyRunning + from .backends import detect_backend_for_path + from .palace import _backend_artifact_label, resolve_backend_name from .sync import sync_palace palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path @@ -622,8 +666,16 @@ def cmd_sync(args): if not os.path.isdir(palace_path): print(f"\n No palace found at {palace_path}") return - if not os.path.isfile(os.path.join(palace_path, "chroma.sqlite3")): - print(f"\n Palace dir at {palace_path} exists but has no chroma.sqlite3 yet.") + try: + backend_name = resolve_backend_name(palace_path) + except Exception as exc: # noqa: BLE001 - user-facing CLI guard + print(f"\n Could not resolve palace backend: {exc}", file=sys.stderr) + return + if detect_backend_for_path(palace_path) is None: + print( + f"\n Palace dir at {palace_path} exists but has no " + f"{_backend_artifact_label(backend_name)} yet." + ) print(" Run: mempalace mine ") return @@ -748,9 +800,11 @@ def cmd_split(args): def cmd_migrate(args): """Migrate palace from a different ChromaDB version.""" + palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path + if not _maintenance_requires_chroma(palace_path, "migrate"): + raise SystemExit(2) from .migrate import migrate - palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path migrate( palace_path=palace_path, dry_run=args.dry_run, @@ -758,6 +812,18 @@ def cmd_migrate(args): ) +def cmd_migrate_wings(args): + """Normalize legacy wing names (strip leading/trailing separators).""" + palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path + from .migrate import migrate_wing_names + + migrate_wing_names( + palace_path=palace_path, + dry_run=args.dry_run, + confirm=getattr(args, "yes", False), + ) + + def cmd_status(args): from .miner import status @@ -765,16 +831,73 @@ def cmd_status(args): status(palace_path=palace_path) +def cmd_palace_set_embedder(args): + """Record (or force-override) a palace's embedder identity (RFC 001). + + Resolves the ``unknown`` state for a legacy palace, or records a specific + model with ``--model``. It records identity on the palace only; it does not + change the configured model — when the two differ it prints how to align + ``MEMPALACE_EMBEDDING_MODEL``. ``--force`` overwrites an existing, + differently-named identity. + """ + from .backends.base import EmbedderIdentityMismatchError + from .palace import set_palace_embedder_identity + + config = MempalaceConfig() + palace_path = os.path.abspath( + os.path.expanduser(args.palace) if args.palace else config.palace_path + ) + model = getattr(args, "model", None) + try: + old, new = set_palace_embedder_identity( + palace_path, + model=model, + force=getattr(args, "force", False), + backend=_backend_arg(args), + ) + except EmbedderIdentityMismatchError as exc: + print(f" ✗ {exc}") + raise SystemExit(2) from exc + if old is None: + print(f" ✓ recorded embedder identity: {new.model_name} (dim={new.dimension})") + elif old.model_name == new.model_name: + print(f" ✓ embedder identity unchanged: {new.model_name} (dim={new.dimension})") + else: + print( + f" ✓ embedder identity changed: {old.model_name} → {new.model_name} " + f"(dim={new.dimension})" + ) + # set-embedder records the palace's identity; it does not change the + # configured model. If they differ, the next normal open would mismatch — + # tell the user how to align them. + configured = config.embedding_model + if new.model_name and configured and new.model_name != configured: + print( + f" ⚠ configured model is {configured!r}; set MEMPALACE_EMBEDDING_MODEL=" + f"{new.model_name} (or run onboarding) so normal opens of this palace match." + ) + + def cmd_repair_status(args): """Read-only HNSW capacity health check (#1222).""" + palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path + if not _maintenance_requires_chroma(palace_path, "repair-status"): + raise SystemExit(2) from .repair import status as repair_status - palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path repair_status(palace_path=palace_path) def cmd_repair(args): """Rebuild palace vector index from SQLite metadata.""" + config = MempalaceConfig() + collection_name = config.collection_name + palace_path = os.path.abspath( + os.path.expanduser(args.palace) if args.palace else config.palace_path + ) + if not _maintenance_requires_chroma(palace_path, "repair"): + raise SystemExit(2) + import shutil from .backends.chroma import ChromaBackend from .migrate import confirm_destructive_action, contains_palace_database @@ -790,12 +913,6 @@ def cmd_repair(args): sqlite_integrity_errors, ) - config = MempalaceConfig() - collection_name = config.collection_name - palace_path = os.path.abspath( - os.path.expanduser(args.palace) if args.palace else config.palace_path - ) - if getattr(args, "mode", "legacy") == "max-seq-id": from .repair import repair_max_seq_id @@ -1004,12 +1121,15 @@ def cmd_instructions(args): def cmd_mcp(args): """Show how to wire MemPalace into MCP-capable hosts.""" base_server_cmd = "mempalace-mcp" + cmd_parts = [base_server_cmd] if args.palace: resolved_palace = str(Path(args.palace).expanduser()) - server_cmd = f"{base_server_cmd} --palace {shlex.quote(resolved_palace)}" - else: - server_cmd = base_server_cmd + cmd_parts.extend(["--palace", shlex.quote(resolved_palace)]) + backend = _backend_arg(args) + if backend: + cmd_parts.extend(["--backend", shlex.quote(str(backend).strip().lower())]) + server_cmd = " ".join(cmd_parts) print("MemPalace MCP quick setup:") print(f" claude mcp add mempalace -- {server_cmd}") @@ -1204,12 +1324,23 @@ def main(): default=None, help="Where the palace lives (default: from ~/.mempalace/config.json or ~/.mempalace/palace)", ) + parser.add_argument( + "--backend", + dest="global_backend", + default=None, + help="Storage backend to use for this command (default: config/env/detected/chroma)", + ) sub = parser.add_subparsers(dest="command") # init p_init = sub.add_parser("init", help="Detect rooms from your folder structure") p_init.add_argument("dir", help="Project directory to set up") + p_init.add_argument( + "--backend", + default=None, + help="Storage backend to persist for this palace (default: chroma)", + ) p_init.add_argument( "--yes", action="store_true", @@ -1292,6 +1423,11 @@ def main(): # mine p_mine = sub.add_parser("mine", help="Mine files into the palace") p_mine.add_argument("dir", help="Directory to mine") + p_mine.add_argument( + "--backend", + default=None, + help="Storage backend to use for this mine (default: config/env/detected/chroma)", + ) p_mine.add_argument( "--mode", choices=["projects", "convos", "extract"], @@ -1401,6 +1537,11 @@ def main(): # search p_search = sub.add_parser("search", help="Find anything, exact words") p_search.add_argument("query", help="What to search for") + p_search.add_argument( + "--backend", + default=None, + help="Storage backend to use for this search (default: config/env/detected/chroma)", + ) p_search.add_argument("--wing", default=None, help="Limit to one project") p_search.add_argument("--room", default=None, help="Limit to one room") p_search.add_argument("--results", type=int, default=5, help="Number of results") @@ -1555,10 +1696,15 @@ def main(): ) # mcp - sub.add_parser( + p_mcp = sub.add_parser( "mcp", help="Show MCP setup command for connecting MemPalace to your AI client", ) + p_mcp.add_argument( + "--backend", + default=None, + help="Storage backend to include in the MCP startup command", + ) # status # migrate @@ -1575,9 +1721,52 @@ def main(): "--yes", action="store_true", help="Skip confirmation for destructive changes" ) - sub.add_parser("status", help="Show what's been filed") + # migrate-wings + p_migrate_wings = sub.add_parser( + "migrate-wings", + help="Normalize legacy wing names (strip leading/trailing separators) so pre-#1675 palaces stay discoverable", + ) + p_migrate_wings.add_argument( + "--dry-run", + action="store_true", + help="Show what would change without modifying the palace", + ) + p_migrate_wings.add_argument("--yes", action="store_true", help="Skip the confirmation prompt") + + p_status = sub.add_parser("status", help="Show what's been filed") + p_status.add_argument( + "--backend", + default=None, + help="Storage backend to use for status (default: config/env/detected/chroma)", + ) + + p_palace = sub.add_parser("palace", help="Palace maintenance commands") + palace_sub = p_palace.add_subparsers(dest="palace_action") + p_set_embedder = palace_sub.add_parser( + "set-embedder", + help="Record/override the palace's embedder identity (resolve 'unknown', or switch models)", + ) + p_set_embedder.add_argument( + "--model", + default=None, + help="Embedder model to record (default: current configured model). " + "Records identity on the palace only; does not change the configured " + "model (prints how to align MEMPALACE_EMBEDDING_MODEL if they differ).", + ) + p_set_embedder.add_argument( + "--force", + action="store_true", + help="Overwrite an existing identity that names a different model " + "(only if you know the stored vectors are compatible)", + ) + p_set_embedder.add_argument( + "--backend", + default=None, + help="Storage backend (default: config/env/detected/chroma)", + ) args = parser.parse_args() + _apply_backend_arg(args) if not args.command: parser.print_help() @@ -1600,6 +1789,13 @@ def main(): cmd_instructions(args) return + if args.command == "palace": + if getattr(args, "palace_action", None) == "set-embedder": + cmd_palace_set_embedder(args) + else: + p_palace.print_help() + return + dispatch = { "init": cmd_init, "mine": cmd_mine, @@ -1613,6 +1809,7 @@ def main(): "repair": cmd_repair, "repair-status": cmd_repair_status, "migrate": cmd_migrate, + "migrate-wings": cmd_migrate_wings, "status": cmd_status, } dispatch[args.command](args) diff --git a/mempalace/closet_llm.py b/mempalace/closet_llm.py index a85d517..801f711 100644 --- a/mempalace/closet_llm.py +++ b/mempalace/closet_llm.py @@ -264,7 +264,7 @@ def regenerate_closets( f"Regenerating closets for {len(sources)} source files via {cfg.endpoint} ({cfg.model})..." ) if dry_run: - print("DRY RUN — no changes will be written") + print("DRY RUN - no changes will be written") processed = 0 failed = 0 @@ -286,7 +286,7 @@ def regenerate_closets( parsed, usage = _call_llm(cfg, source, w, r, content) if not parsed: failed += 1 - print(f" [{i}/{len(sources)}] ✗ {os.path.basename(source)} — LLM failed") + print(f" [{i}/{len(sources)}] [FAIL] {os.path.basename(source)} - LLM failed") continue if usage: @@ -323,7 +323,7 @@ def regenerate_closets( processed += 1 n_topics = len(parsed.get("topics", [])) - print(f" [{i}/{len(sources)}] ✓ {os.path.basename(source)} — {n_topics} topics") + print(f" [{i}/{len(sources)}] [OK] {os.path.basename(source)} - {n_topics} topics") print(f"\nDone. {processed} regenerated, {failed} failed.") if total_input or total_output: diff --git a/mempalace/collision_scan.py b/mempalace/collision_scan.py new file mode 100644 index 0000000..53819f2 --- /dev/null +++ b/mempalace/collision_scan.py @@ -0,0 +1,122 @@ +"""Pre-mining defense against drawer_id collisions. + +Runs immediately before a batched chromadb upsert. Computes the union of +incoming drawer_ids and existing drawer_ids that share a key with the +batch; raises ``CollisionError`` if any drawer_id appears more than once +in that union with conflicting ``(source_file, chunk_index)`` metadata. + +Under the v2 hash recipe (see :mod:`mempalace.ids`) accidental collisions +are vanishingly rare — SHA-256 truncated to 24 hex chars makes random +collision ~2^-96. The scan exists for two reasons: + +1. Catch upstream bugs that emit duplicate ``(source_file, chunk_index)`` + pairs in the same batch with conflicting content. ChromaDB would + silently let the last-write win; the scan surfaces it as an + actionable error naming both call sites. +2. Catch the astronomical-but-possible SHA-256 hash collision with a + clear message instead of a silent overwrite at upsert time. + +The scan does NOT fire on idempotent re-mines — when an incoming drawer +matches an existing one with the SAME ``(source_file, chunk_index)`` +metadata, that is normal re-write behavior, not collision. +""" + +from __future__ import annotations + +from collections import defaultdict + + +class CollisionError(Exception): + """Raised by :func:`assert_no_collisions` when the pre-mining scan + detects a drawer_id that would silently overwrite existing content + or duplicate within a batch with conflicting metadata. + + The exception message names every colliding ``drawer_id`` and the + full set of ``(source_file, chunk_index)`` pairs producing each one, + so a user fixing one collision does not have to rediscover the next + by re-running the mine. + """ + + +def _metadata_key(meta: dict) -> tuple: + """Reduce a drawer metadata dict to the tuple used for collision + discrimination. Two metadata dicts are 'the same chunk' iff their + key tuples match. Falls back to ``(source_file,)`` when + ``chunk_index`` is absent (diary entries, sentinels).""" + source_file = meta.get("source_file") + chunk_index = meta.get("chunk_index") + if chunk_index is None: + return (source_file,) + return (source_file, chunk_index) + + +def assert_no_collisions( + proposed: list[tuple[str, dict]], + collection, +) -> None: + """Abort the mine via ``CollisionError`` if any proposed drawer_id + collides with itself or with an existing drawer in ``collection``. + + Args: + proposed: list of ``(drawer_id, metadata)`` tuples for the + chunks about to be upserted. ``metadata`` must carry at + least ``source_file``; ``chunk_index`` is used when + present. + collection: a ChromaDB-shaped collection with ``get(ids=...)`` + returning a dict with ``ids`` and ``metadatas`` keys. + + Raises: + CollisionError: when a drawer_id maps to two or more distinct + ``(source_file, chunk_index)`` tuples in the union of + incoming and existing rows. + """ + if not proposed: + return + + # Build incoming map: drawer_id -> set of metadata key tuples. + # Using a set collapses duplicate-metadata cases (same chunk twice + # in the batch) without flagging them as collisions. + incoming: dict[str, set[tuple]] = defaultdict(set) + for drawer_id, meta in proposed: + incoming[drawer_id].add(_metadata_key(meta)) + + # Query existing rows for any incoming id. ChromaDB's get(ids=...) + # returns only the rows whose ids are present; missing ids are + # silently absent from the result, which is what we want. + incoming_ids = list(incoming.keys()) + result = collection.get(ids=incoming_ids, include=["metadatas"]) + existing_ids: list = result["ids"] if hasattr(result, "__getitem__") else [] + existing_metas: list = result["metadatas"] if existing_ids else [] + + # Merge existing metadata into the incoming map. A real collision is + # a drawer_id whose incoming + existing metadata key tuples are not + # all the same. + for drawer_id, meta in zip(existing_ids, existing_metas): + incoming[drawer_id].add(_metadata_key(meta or {})) + + collisions = {did: keys for did, keys in incoming.items() if len(keys) > 1} + if collisions: + raise CollisionError(_format_collisions(collisions)) + + +def _format_collisions(collisions: dict[str, set[tuple]]) -> str: + """Render a CollisionError message that enumerates every colliding + drawer_id and the metadata tuples producing it.""" + lines = [ + f"Pre-mining collision scan detected {len(collisions)} " + f"colliding drawer_id{'s' if len(collisions) != 1 else ''}:", + ] + for drawer_id, keys in sorted(collisions.items()): + lines.append(f" {drawer_id}:") + for key in sorted(keys, key=lambda k: tuple(str(part) for part in k)): + if len(key) == 1: + lines.append(f" source_file={key[0]!r}") + else: + lines.append(f" source_file={key[0]!r}, chunk_index={key[1]!r}") + lines.append( + "Each colliding drawer_id would cause the second ChromaDB upsert " + "to silently overwrite the first. Fix the upstream chunker / " + "miner to emit distinct keys, or investigate the SHA-256 hash " + "collision." + ) + return "\n".join(lines) diff --git a/mempalace/config.py b/mempalace/config.py index 752c918..87fdae3 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -37,8 +37,13 @@ def normalize_wing_name(name: str) -> str: The same rule is applied by ``init`` when persisting `topics_by_wing` and when writing `mempalace.yaml`, so the miner's lookup matches at mine time regardless of the source dirname. + + Leading/trailing separators are stripped so a path-encoded dirname like + ``-home-user-proj`` yields ``home_user_proj`` rather than a leading- + underscore slug that ``sanitize_name`` (and thus the MCP write tools) + would reject. """ - return name.lower().replace(" ", "_").replace("-", "_") + return name.lower().replace(" ", "_").replace("-", "_").strip("_") def sanitize_name(value: str, field_name: str = "name") -> str: @@ -191,6 +196,13 @@ def sanitize_content(value: str, max_length: int = 100_000) -> str: DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace") DEFAULT_COLLECTION_NAME = "mempalace_drawers" +DEFAULT_BACKEND = "chroma" + +# How many timestamped palace backups to retain before the oldest are +# pruned. Applies to the accumulating backups written by ``mempalace +# migrate`` and ``mempalace repair max-seq-id`` — see +# ``MempalaceConfig.max_backups``. +DEFAULT_MAX_BACKUPS = 10 @lru_cache(maxsize=1) @@ -325,6 +337,87 @@ class MempalaceConfig: """ChromaDB collection name.""" return self._file_config.get("collection_name", DEFAULT_COLLECTION_NAME) + @property + def backend(self): + """Storage backend name. + + Read from ``config.json`` first, then ``MEMPALACE_BACKEND``, then + ``"chroma"`` for backwards compatibility with existing palaces. + """ + cfg_val = self._file_config.get("backend") + if cfg_val: + return str(cfg_val).strip().lower() + env_val = os.environ.get("MEMPALACE_BACKEND") + if env_val: + return env_val.strip().lower() + return DEFAULT_BACKEND + + @property + def qdrant_url(self): + """Qdrant endpoint for the opt-in ``qdrant`` backend. + + Defaults to localhost so selecting Qdrant never silently sends memory + to a remote service. Users can point at a LAN or cloud endpoint via + config or ``MEMPALACE_QDRANT_URL`` when they deliberately choose that. + """ + env_val = os.environ.get("MEMPALACE_QDRANT_URL") + if env_val: + return env_val.strip() + return str(self._file_config.get("qdrant_url", "http://localhost:6333")).strip() + + @property + def qdrant_api_key(self): + """API key for the opt-in ``qdrant`` backend, if configured.""" + env_val = os.environ.get("MEMPALACE_QDRANT_API_KEY") + if env_val: + return env_val + value = self._file_config.get("qdrant_api_key") + return str(value) if value else None + + @property + def qdrant_namespace(self): + """Optional Qdrant collection namespace/prefix.""" + env_val = os.environ.get("MEMPALACE_QDRANT_NAMESPACE") + if env_val: + return env_val.strip() + value = self._file_config.get("qdrant_namespace") + return str(value).strip() if value else None + + @property + def qdrant_timeout(self): + """Qdrant HTTP timeout in seconds.""" + env_val = os.environ.get("MEMPALACE_QDRANT_TIMEOUT") + raw = env_val if env_val is not None else self._file_config.get("qdrant_timeout", 10.0) + try: + timeout = float(raw) + except (TypeError, ValueError): + timeout = 10.0 + return timeout if timeout > 0 else 10.0 + + @property + def pgvector_dsn(self): + """Postgres DSN for the opt-in ``pgvector`` backend. + + Defaults to a localhost DSN so selecting pgvector never silently sends + memory to a remote database. Point at a LAN or cloud Postgres via config + or ``MEMPALACE_PGVECTOR_DSN`` only when deliberately chosen. + """ + env_val = os.environ.get("MEMPALACE_PGVECTOR_DSN") + if env_val: + return env_val.strip() + return str( + self._file_config.get("pgvector_dsn", "postgresql://localhost:5432/mempalace") + ).strip() + + @property + def pgvector_namespace(self): + """Optional pgvector table namespace/prefix for multi-tenant isolation.""" + env_val = os.environ.get("MEMPALACE_PGVECTOR_NAMESPACE") + if env_val: + return env_val.strip() + value = self._file_config.get("pgvector_namespace") + return str(value).strip() if value else None + @property def people_map(self): """Mapping of name variants to canonical names.""" @@ -560,6 +653,24 @@ class MempalaceConfig: except (OSError, NotImplementedError): pass + def set_backend(self, backend: str) -> None: + """Persist the storage backend choice to ``config.json``.""" + backend = str(backend).strip().lower() + from .backends import get_backend_class + + get_backend_class(backend) + self._file_config["backend"] = backend + self._config_dir.mkdir(parents=True, exist_ok=True) + try: + with open(self._config_file, "w", encoding="utf-8") as f: + json.dump(self._file_config, f, indent=2, ensure_ascii=False) + except OSError: + pass + try: + self._config_file.chmod(0o600) + except (OSError, NotImplementedError): + pass + @property def topic_tunnel_min_count(self): """Minimum number of overlapping confirmed topics required to create @@ -586,6 +697,36 @@ class MempalaceConfig: parsed = 1 return max(1, parsed) + @property + def max_backups(self) -> int: + """Number of timestamped palace backups to retain before pruning. + + Applies to the accumulating, timestamped backups created by + ``mempalace migrate`` (``.pre-migrate.``) and + ``mempalace repair max-seq-id`` + (``chroma.sqlite3.max-seq-id-backup-``). Each of those + commands writes a fresh full-size copy every run and historically + never deleted the old ones, so on a machine that mines or repairs on + a schedule the backup set could silently grow until it filled the + disk. After each backup is written, copies beyond this count (oldest + first) are removed. + + Reads ``MEMPALACE_MAX_BACKUPS`` env first, then ``max_backups`` in + ``config.json``, then the default of ``10``. A value of ``0`` disables + pruning and keeps every backup (use when an external retention policy + manages cleanup). Negative or non-numeric values fall back to the + default rather than crashing migrate/repair. + """ + env_val = os.environ.get("MEMPALACE_MAX_BACKUPS") + if env_val is not None: + coerced = self._try_coerce_int(env_val, minimum=0) + if coerced is not None: + return coerced + coerced = self._try_coerce_int( + self._file_config.get("max_backups", DEFAULT_MAX_BACKUPS), minimum=0 + ) + return DEFAULT_MAX_BACKUPS if coerced is None else coerced + @property def hook_silent_save(self): """Whether the stop hook saves directly (True) or blocks for MCP calls (False).""" diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index ee82e36..ad80259 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -10,13 +10,14 @@ Same palace as project mining. Different ingest strategy. import os import sys -import hashlib import logging from pathlib import Path from datetime import datetime from collections import defaultdict from typing import Optional +from .collision_scan import assert_no_collisions +from .ids import ID_RECIPE, make_convo_drawer_id, make_convo_sentinel_id from .normalize import normalize from .palace import ( NORMALIZE_VERSION, @@ -83,8 +84,7 @@ def _register_file(collection, source_file: str, wing: str, agent: str, extract_ re-read and re-processed on every mine run because nothing was written to ChromaDB on the first pass. """ - sentinel_key = f"{source_file}:{extract_mode}" - sentinel_id = f"_reg_{hashlib.sha256(sentinel_key.encode()).hexdigest()[:24]}" + sentinel_id = make_convo_sentinel_id(source_file, extract_mode) collection.upsert( documents=[f"[registry] {source_file}"], ids=[sentinel_id], @@ -98,6 +98,7 @@ def _register_file(collection, source_file: str, wing: str, agent: str, extract_ "ingest_mode": "registry", "extract_mode": extract_mode, "normalize_version": NORMALIZE_VERSION, + "id_recipe": ID_RECIPE, } ], ) @@ -192,11 +193,15 @@ def _chunk_by_exchange(lines: list, chunk_size: int, min_chunk_size: int) -> lis next_line = lines[i] if next_line.strip().startswith(">") or next_line.strip().startswith("---"): break - if next_line.strip(): - ai_lines.append(next_line.strip()) + # Preserve the line as-is — blank lines and indentation carry meaning + # (paragraph breaks, list/code structure) and must survive verbatim. + ai_lines.append(next_line) i += 1 - ai_response = " ".join(ai_lines) + # Join on newline (not space) so line structure, blank lines, and + # indentation reach the drawer unchanged. Trim only trailing blank + # lines produced by the loop stopping at the next `>` turn. + ai_response = "\n".join(ai_lines).rstrip("\n") content = f"{user_turn}\n{ai_response}" if ai_response else user_turn _emit_bounded(chunks, content, chunk_size, min_chunk_size) @@ -419,10 +424,8 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room if extract_mode == "general": room_counts_delta[chunk_room] += 1 - drawer_key = f"{source_file}:{extract_mode}:{chunk['chunk_index']}" - drawer_id = ( - f"drawer_{wing}_{chunk_room}_" - f"{hashlib.sha256(drawer_key.encode()).hexdigest()[:24]}" + drawer_id = make_convo_drawer_id( + wing, chunk_room, source_file, extract_mode, chunk["chunk_index"] ) batch_docs.append(chunk["content"]) batch_ids.append(drawer_id) @@ -438,8 +441,10 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr "ingest_mode": "convos", "extract_mode": extract_mode, "normalize_version": NORMALIZE_VERSION, + "id_recipe": ID_RECIPE, } ) + assert_no_collisions(list(zip(batch_ids, batch_metas)), collection) try: collection.upsert( documents=batch_docs, @@ -589,15 +594,14 @@ def _mine_convos_impl( wing = _resolve_wing(convo_path, wing) files = scan_convos(convo_dir) - if limit > 0: - files = files[:limit] print(f"\n{'=' * 55}") print(" MemPalace Mine — Conversations") print(f"{'=' * 55}") print(f" Wing: {wing}") print(f" Source: {convo_path}") - print(f" Files: {len(files)}") + limit_suffix = f" (limit: {limit} new)" if limit > 0 else "" + print(f" Files: {len(files)}{limit_suffix}") print(f" Palace: {palace_path}") if dry_run: print(" DRY RUN — nothing will be filed") @@ -615,10 +619,13 @@ def _mine_convos_impl( ) total_drawers = 0 + files_mined = 0 files_skipped = 0 + files_processed = 0 room_counts = defaultdict(int) for i, filepath in enumerate(files, 1): + files_processed = i source_file = str(filepath) # Skip if already filed at current NORMALIZE_VERSION @@ -679,6 +686,9 @@ def _mine_convos_impl( room_counts[c.get("memory_type", "general")] += 1 else: room_counts[room] += 1 + files_mined += 1 + if limit > 0 and files_mined >= limit: + break continue if extract_mode != "general": @@ -696,14 +706,17 @@ def _mine_convos_impl( room_counts[r] += n total_drawers += drawers_added + files_mined += 1 print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") + if limit > 0 and files_mined >= limit: + break if not dry_run: _validate_palace_fts5_after_mine(palace_path) print(f"\n{'=' * 55}") print(" Done.") - print(f" Files processed: {len(files) - files_skipped}") + print(f" Files processed: {files_processed - files_skipped}") print(f" Files skipped (already filed): {files_skipped}") print(f" Drawers filed: {total_drawers}") if room_counts: diff --git a/mempalace/data/known_systems.json b/mempalace/data/known_systems.json new file mode 100644 index 0000000..6cdd480 --- /dev/null +++ b/mempalace/data/known_systems.json @@ -0,0 +1,68 @@ +{ + "schema_version": 1, + "description": "Multi-word product/system names that must be detected atomically — not decomposed into their constituent words. When any of these compounds appears in mined content, the entity detector counts the COMPOUND, not the parts. The single-word detection pass that runs after this compound pre-pass would otherwise decompose 'Claude Code' into 'Claude' + 'Code', and the COCA filter (Tier 2) would then drop 'Code' as a content word while attributing usage incorrectly to 'Claude' alone.", + "source": "Curated 2026-05-24 from common AI/development product names that mempalace users frequently mention. Each entry is a multi-word name (two-or-more whitespace-separated tokens, or hyphenated). Single-word product names like 'ChatGPT', 'Cursor', 'OrbStack' are NOT included — they have no decomposition risk and are handled by the existing single-word regex.", + "case_handling": "Matching is case-insensitive. Canonical form (the entry below) is what gets counted. Caller must use re.IGNORECASE when matching.", + "tier_2_interaction": "This file complements coca_content_words.json. Tier 2 filters single-word content nouns. Tier 3 protects multi-word product names. Run THIS pre-pass FIRST so the COCA filter doesn't see the decomposed words.", + "compounds": [ + "Claude Code", + "Claude Desktop", + "Claude Sonnet", + "Claude Opus", + "Claude Haiku", + "Claude Sonnet 4.5", + "Claude Sonnet 4.6", + "Claude Opus 4.5", + "Claude Opus 4.6", + "Claude Opus 4.7", + "Gemini Code Assist", + "Gemini 1.5 Pro", + "Gemini 2.5 Pro", + "Gemini 2.5 Flash", + "Gemini Pro", + "Gemini Flash", + "GitHub Copilot", + "GitHub Copilot CLI", + "Microsoft Copilot", + "Visual Studio Code", + "Visual Studio", + "IntelliJ IDEA", + "Android Studio", + "Sublime Text", + "Replit Agent", + "Docker Desktop", + "Docker Compose", + "GitHub Actions", + "GitHub Pages", + "GitHub Codespaces", + "GitHub Pull Request", + "Hugging Face", + "Stack Overflow", + "Stack Exchange", + "Hacker News", + "Y Combinator", + "Google Cloud Platform", + "Amazon Web Services", + "Microsoft Azure", + "Azure DevOps", + "Notion AI", + "OpenAI API", + "Anthropic Console", + "GPT-4", + "GPT-4o", + "GPT-5", + "Llama 3", + "Llama 3.1", + "Mistral Small", + "Mistral Large", + "Microsoft Word", + "Microsoft Excel", + "Microsoft PowerPoint", + "Google Docs", + "Google Sheets", + "Google Drive", + "Google Cloud", + "Apple Watch", + "Apple TV" + ] +} diff --git a/mempalace/dedup.py b/mempalace/dedup.py index 5e57aff..82afa28 100644 --- a/mempalace/dedup.py +++ b/mempalace/dedup.py @@ -7,7 +7,9 @@ accumulate. This module finds drawers from the same source_file that are too similar (cosine distance < threshold), keeps the longest/richest version, and deletes the rest. -No API calls — uses ChromaDB's built-in embedding similarity. +Uses the configured storage backend's similarity search. With the default +local backends (Chroma, sqlite_exact) this stays on-machine with no external +calls; with a remote backend (e.g. Qdrant) it issues queries to that backend. Usage (standalone): python -m mempalace.dedup # dedup all @@ -27,7 +29,7 @@ import os import time from collections import defaultdict -from .backends.chroma import ChromaBackend +from .palace import get_collection COLLECTION_NAME = "mempalace_drawers" @@ -130,7 +132,7 @@ def dedup_source_group(col, drawer_ids, threshold=DEFAULT_THRESHOLD, dry_run=Tru def show_stats(palace_path=None): """Show duplication statistics without making changes.""" palace_path = palace_path or _get_palace_path() - col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME) + col = get_collection(palace_path, COLLECTION_NAME) groups = get_source_groups(col) @@ -162,7 +164,7 @@ def dedup_palace( print(" MemPalace Deduplicator") print(f"{'=' * 55}") - col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME) + col = get_collection(palace_path, COLLECTION_NAME) print(f" Palace: {palace_path}") print(f" Drawers: {col.count():,}") diff --git a/mempalace/embedding.py b/mempalace/embedding.py index b22b1f5..930c6fb 100644 --- a/mempalace/embedding.py +++ b/mempalace/embedding.py @@ -191,6 +191,9 @@ class EmbeddinggemmaONNX: model_path = hf_hub_download( _EMBEDDINGGEMMA_REPO, subfolder="onnx", filename=_EMBEDDINGGEMMA_ONNX ) + hf_hub_download( + _EMBEDDINGGEMMA_REPO, subfolder="onnx", filename=_EMBEDDINGGEMMA_ONNX + "_data" + ) tok_path = hf_hub_download(_EMBEDDINGGEMMA_REPO, filename="tokenizer.json") self._session = ort.InferenceSession(model_path, providers=self._providers) @@ -223,6 +226,14 @@ class EmbeddinggemmaONNX: norms = np.linalg.norm(sent_emb, axis=1, keepdims=True) + 1e-12 return (sent_emb / norms).tolist() + def embed_query(self, input: list[str]) -> list[list[float]]: # noqa: A002 — ChromaDB EF protocol + """Embed query documents (ChromaDB EF protocol).""" + return self(input) + + def embed_documents(self, input: list[str]) -> list[list[float]]: # noqa: A002 + """Embed a batch of documents (ChromaDB EF protocol).""" + return self(input) + def get_embedding_function(device: Optional[str] = None, model: Optional[str] = None): """Return a cached embedding function for the requested device + model. @@ -276,3 +287,59 @@ def describe_device(device: Optional[str] = None) -> str: device = MempalaceConfig().embedding_device _, effective = _resolve_providers(device) return effective + + +# Probed vector widths, keyed by resolved model name. Populated once per +# process the first time an identity is resolved for a model. +_DIM_CACHE: dict = {} + + +def current_model_name(model: Optional[str] = None) -> str: + """Resolve the canonical embedder model name (cheap, no model load). + + This is the configured ``embedding_model`` (``"minilm"`` / + ``"embeddinggemma"`` / ...), not the embedding function's internal + ``name()`` (which is spoofed to ``"default"`` for ChromaDB compatibility). + """ + if model is not None: + return str(model).strip().lower() + from .config import MempalaceConfig + + return MempalaceConfig().embedding_model + + +def probe_dimension(device: Optional[str] = None, model: Optional[str] = None) -> int: + """Return the embedder's output dimension by embedding a short probe. + + Model-agnostic — works for any model without a hardcoded table — and + cached per resolved model name so the probe is paid at most once per + process. Returns ``0`` if the probe fails (treated as "dimension unknown" + by the identity check, so a probe failure never blocks normal operation). + """ + name = current_model_name(model) + cached = _DIM_CACHE.get(name) + if cached is not None: + return cached + try: + ef = get_embedding_function(device=device, model=model) + vectors = ef(input=["probe"]) + dim = len(vectors[0]) if vectors and vectors[0] is not None else 0 + except Exception: + logger.debug("Embedding dimension probe failed for model=%s", name, exc_info=True) + dim = 0 + _DIM_CACHE[name] = dim + return dim + + +def get_embedder_identity(device: Optional[str] = None, model: Optional[str] = None): + """Resolve the current embedder identity (RFC 001). + + ``model_name`` from config (cheap); ``dimension`` from a cached one-time + probe. Returns an :class:`~mempalace.backends.base.EmbedderIdentity`. + """ + from .backends.base import EmbedderIdentity + + return EmbedderIdentity( + model_name=current_model_name(model), + dimension=probe_dimension(device=device, model=model), + ) diff --git a/mempalace/entity_detector.py b/mempalace/entity_detector.py index 429b04d..d79509c 100644 --- a/mempalace/entity_detector.py +++ b/mempalace/entity_detector.py @@ -78,6 +78,98 @@ def _get_coca_filter() -> frozenset[str]: return frozenset() +# ==================== KNOWN-SYSTEMS COMPOUND LEXICON (Tier 3 linguistics cleanup) ==================== +# +# Multi-word product / system names that must be detected atomically — NOT +# decomposed into their constituent words. When "Claude Code" appears in +# content, the entity detector counts the compound, not the parts. Without +# this pre-pass, the single-word loop would split "Claude Code" into +# "Claude" + "Code", and the COCA filter (Tier 2) would drop "Code" as a +# content word — leaving "Claude" alone with the wrong attribution. +# +# Data file: ``mempalace/data/known_systems.json``. Loaded once on first +# call via ``_get_known_systems``. Matching is case-insensitive with word +# boundaries. + + +@functools.lru_cache(maxsize=1) +def _get_known_systems() -> tuple[tuple[str, "re.Pattern[str]"], ...]: + """Return the known-systems compound tuple — pairs of (canonical name, + pre-compiled case-insensitive word-bounded regex). + + Loads ``mempalace/data/known_systems.json`` on first call, compiles a + regex for each valid compound, and caches the resulting tuple of + pairs. Subsequent calls are O(1) and skip both the disk read AND the + regex compilation. Returns an empty tuple if the data file is missing + or malformed — extraction behavior then degrades gracefully + (compounds detected only by the existing multi-word regex) rather + than crashing. + + Entries are sorted by length descending so the compound matcher + prefers longer matches first (e.g. "Visual Studio Code" wins over + a hypothetical "Visual Studio" if both were in the lexicon). + """ + data_path = Path(__file__).parent / "data" / "known_systems.json" + try: + raw = json.loads(data_path.read_text(encoding="utf-8")) + compounds = raw.get("compounds", []) + valid = [c for c in compounds if isinstance(c, str) and c.strip()] + # Sort by length descending so longest-match-wins during the + # pre-pass scan (longer compounds get masked first, so a shorter + # compound contained within a longer one doesn't double-count). + sorted_compounds = sorted(valid, key=len, reverse=True) + + compiled: list[tuple[str, re.Pattern[str]]] = [] + for c in sorted_compounds: + # Word-boundary, case-insensitive. Compound may contain + # hyphens or spaces — re.escape handles special chars; word + # boundaries on each side prevent partial-word matches + # (e.g. "GPT-4" must not match "GPT-40"). + pattern = r"(? tuple[str, dict[str, int]]: + """Scan ``text`` for known-systems compounds, return a working copy + with matched spans masked to whitespace plus a dict of detected + compound counts. + + Returning the counts (instead of mutating a caller-supplied container) + lets the three call sites (``extract_candidates`` at init-time, + ``palace.build_closet_lines`` at closet construction, and + ``miner._extract_entities_for_metadata`` at per-drawer tagging) use + whichever container shape they already maintain. + + Compounds are matched case-insensitively with word boundaries; the + canonical (lexicon) casing is what gets counted, regardless of how + the compound appears in the source text. Regexes are pre-compiled + once in ``_get_known_systems`` so this function does no compilation. + """ + compounds = _get_known_systems() + if not compounds: + return text, {} + working = text + compound_counts: dict[str, int] = {} + for compound, rx in compounds: + matches = list(rx.finditer(working)) + if not matches: + continue + compound_counts[compound] = compound_counts.get(compound, 0) + len(matches) + # Mask matched spans with spaces so the subsequent regex passes + # don't re-decompose. Replacing right-to-left keeps earlier + # indices stable. + for m in reversed(matches): + start, end = m.span() + working = working[:start] + (" " * (end - start)) + working[end:] + return working, compound_counts + + # ==================== LANGUAGE-AWARE PATTERN LOADING ==================== @@ -194,13 +286,21 @@ def extract_candidates(text: str, languages=("en",)) -> dict: counts: defaultdict = defaultdict(int) + # Tier 3 — known-systems compound pre-pass. Find compound product names + # ("Claude Code", "GitHub Copilot", ...) FIRST and mask them out of the + # working text so the subsequent single-word + multi-word loops don't + # re-decompose them into their constituent tokens. + working_text, compound_counts = _apply_known_systems_prepass(text) + for compound, n in compound_counts.items(): + counts[compound] += n + # Single-word candidates — one pre-wrapped pattern per language for wrapped_pat in patterns["candidate_patterns"]: try: rx = re.compile(wrapped_pat) except re.error: continue - for word in rx.findall(text): + for word in rx.findall(working_text): wl = word.lower() if wl in stopwords: continue @@ -214,13 +314,16 @@ def extract_candidates(text: str, languages=("en",)) -> dict: continue counts[word] += 1 - # Multi-word candidates — one pre-wrapped pattern per language + # Multi-word candidates — one pre-wrapped pattern per language. + # Runs against the working_text (compounds already masked) so an + # unknown two-word phrase like "Jane Smith" still gets caught by + # the regex without competing with known compounds. for wrapped_pat in patterns["multi_word_patterns"]: try: rx = re.compile(wrapped_pat) except re.error: continue - for phrase in rx.findall(text): + for phrase in rx.findall(working_text): if any(w.lower() in stopwords for w in phrase.split()): continue counts[phrase] += 1 diff --git a/mempalace/format_miner.py b/mempalace/format_miner.py index adb1aee..296ca29 100644 --- a/mempalace/format_miner.py +++ b/mempalace/format_miner.py @@ -82,6 +82,8 @@ from .palace import ( # mempalace.format_miner.. Lazy imports inside functions would not # expose these as attributes of this module, breaking the test seams. from .config import MempalaceConfig, normalize_wing_name +from .collision_scan import assert_no_collisions +from .ids import ID_RECIPE, make_drawer_id_from_chunk from .miner import ( _compute_topic_tunnels_for_wing, chunk_text, @@ -505,7 +507,7 @@ def scan_formats(directory: Union[Path, str]) -> list[Path]: def _print_mine_summary( - files: list, + files_seen: int, files_with_text: int, files_skipped: int, files_errored: int, @@ -521,7 +523,7 @@ def _print_mine_summary( print(f"\n{'=' * 55}") print(" Summary") print(f"{'-' * 55}") - print(f" Files seen: {len(files)}") + print(f" Files seen: {files_seen}") print(f" Files extracted: {files_with_text}") print(f" Files skipped: {files_skipped}") print(f" Files errored: {files_errored}") @@ -640,8 +642,7 @@ def _file_chunks_locked( batch_ids: list = [] batch_metas: list = [] for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]: - key = (source_file + str(chunk["chunk_index"])).encode() - drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256(key).hexdigest()[:24]}" + drawer_id = make_drawer_id_from_chunk(wing, room, source_file, chunk["chunk_index"]) content = chunk["content"] meta: dict = { "wing": wing, @@ -654,6 +655,7 @@ def _file_chunks_locked( "extract_mode": "format", "normalize_version": NORMALIZE_VERSION, "hall": detect_hall(content), + "id_recipe": ID_RECIPE, } if source_mtime is not None: meta["source_mtime"] = source_mtime @@ -674,6 +676,7 @@ def _file_chunks_locked( batch_docs.append(content) batch_ids.append(drawer_id) batch_metas.append(meta) + assert_no_collisions(list(zip(batch_ids, batch_metas)), collection) try: collection.upsert( documents=batch_docs, @@ -786,9 +789,11 @@ def mine_formats( files: list = [] collection = None total_drawers = 0 + files_mined = 0 files_skipped = 0 files_with_text = 0 files_errored = 0 + files_processed = 0 status_counts: dict = defaultdict(int) try: @@ -796,15 +801,14 @@ def mine_formats( # ``~/docs`` and relative inputs work consistently. Per PR #1555 review # (Copilot #10). files = scan_formats(format_path) - if limit > 0: - files = files[:limit] print(f"\n{'=' * 55}") print(" MemPalace Mine — Format extraction") print(f"{'=' * 55}") print(f" Wing: {wing}") print(f" Source: {format_path}") - print(f" Files: {len(files)}") + limit_suffix = f" (limit: {limit} new)" if limit > 0 else "" + print(f" Files: {len(files)}{limit_suffix}") print(f" Palace: {palace_path}") if dry_run: print(" DRY RUN — nothing will be filed") @@ -813,6 +817,7 @@ def mine_formats( collection = get_collection(palace_path) if not dry_run else None for i, filepath in enumerate(files, 1): + files_processed = i source_file = str(filepath) # Per-file try/except so one bad file can't crash the whole mine. @@ -873,6 +878,9 @@ def mine_formats( if dry_run: print(f" [DRY RUN] {filepath.name} → {len(chunks)} drawers") total_drawers += len(chunks) + files_mined += 1 + if limit > 0 and files_mined >= limit: + break continue drawers_added, skipped = _file_chunks_locked( @@ -890,7 +898,10 @@ def mine_formats( continue total_drawers += drawers_added + files_mined += 1 print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") + if limit > 0 and files_mined >= limit: + break except Exception as exc: # Log and continue — one malformed file shouldn't kill the # whole mine. Mirrors miner.py's per-file recovery. @@ -970,7 +981,7 @@ def mine_formats( logger.debug("mine_formats: _cleanup_mine_pid_file failed", exc_info=True) _print_mine_summary( - files=files, + files_seen=files_processed, files_with_text=files_with_text, files_skipped=files_skipped, files_errored=files_errored, diff --git a/mempalace/hallways.py b/mempalace/hallways.py index 907527a..9bee6c6 100644 --- a/mempalace/hallways.py +++ b/mempalace/hallways.py @@ -179,7 +179,12 @@ def compute_hallways_for_wing( Args: wing: wing name to scan. - col: ChromaDB collection — must support ``.get(where=..., include=...)``. + col: ChromaDB collection — must support ``.count()`` and paginated + ``.get(limit=..., offset=..., include=...)``. The fetch is filtered + to ``wing`` client-side rather than via ``.get(where={"wing": ...})``, + which binds one SQL variable per matched id and overflows SQLite's + ``SQLITE_MAX_VARIABLE_NUMBER`` on large wings (#1619). Fake + collections and alternate backends must implement this shape. If ``None``, returns ``[]`` (caller didn't supply a backing store, so nothing to compute against). Tests pass a controlled MagicMock. @@ -198,16 +203,32 @@ def compute_hallways_for_wing( min_count = max(1, int(min_count)) - # 1. Query drawers for this wing. + # 1. Query drawers for this wing. Paginate the fetch and filter to the + # wing client-side: a single get(where={"wing": wing}) binds one SQL + # variable per matched id and overflows SQLite's + # SQLITE_MAX_VARIABLE_NUMBER (32766) on wings > ~32k drawers (#1619). + # Mirrors the established pagination in miner.status / palace / + # palace_graph. + metadatas: list = [] try: - results = col.get(where={"wing": wing}, include=["metadatas"]) + total = col.count() + batch_size = 5000 + offset = 0 + while offset < total: + batch = col.get(limit=batch_size, offset=offset, include=["metadatas"]) + batch_metas = (batch or {}).get("metadatas") or [] + if not batch_metas: + break + metadatas.extend( + m for m in batch_metas if isinstance(m, dict) and m.get("wing") == wing + ) + offset += len(batch_metas) except Exception: logger.warning( - "compute_hallways_for_wing: collection.get failed for %s", wing, exc_info=True + "compute_hallways_for_wing: collection fetch failed for %s", wing, exc_info=True ) return [] - metadatas = (results or {}).get("metadatas") or [] if not metadatas: return [] diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 7275ef4..3b86477 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -651,12 +651,16 @@ def _save_diary_direct( session_id: str, wing: str = "", toast: bool = False, + *, + agent_name: str, ) -> dict: """Write a diary checkpoint by calling the tool function directly (no MCP roundtrip). - If `wing` is set, the entry lands in that wing (typically the project wing - derived from the transcript path). Otherwise falls back to `tool_diary_write`'s - default of `wing_session-hook`. + The entry is filed under `agent_name` so the agent that later calls + `mempalace_diary_read(agent_name=...)` discovers it (#1693). If `wing` is + set, the entry lands in that wing (typically the project wing derived from + the transcript path); a `diary_read` with an empty wing spans every wing + the agent wrote to, so project-derived wings stay discoverable. Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure. """ @@ -679,7 +683,7 @@ def _save_diary_direct( from .mcp_server import tool_diary_write result = tool_diary_write( - agent_name="session-hook", + agent_name=agent_name, entry=entry, topic="checkpoint", wing=wing, @@ -741,6 +745,20 @@ def _ingest_transcript(transcript_path: str): SUPPORTED_HARNESSES = {"claude-code", "codex"} +def _diary_agent_for_harness(harness: str) -> str: + """Return the diary ``agent_name`` a session in ``harness`` reads under. + + Stop-hook checkpoints must be filed beside the agent's own entries so + ``mempalace_diary_read(agent_name=...)`` surfaces them. The old code filed + them under a fixed ``"session-hook"`` identity that no reader ever queried, + hiding every checkpoint (#1693). A ``claude-code`` session reads its diary + as ``"claude"``; every other harness already reads under its own name, so + returning the harness name keeps a newly supported harness discoverable + instead of silently invisible again. + """ + return "claude" if harness == "claude-code" else harness + + def _parse_harness_input(data: dict, harness: str) -> dict: """Parse stdin JSON according to the harness type.""" if harness not in SUPPORTED_HARNESSES: @@ -942,7 +960,11 @@ def hook_stop(data: dict, harness: str): result = {"count": 0} if transcript_path: result = _save_diary_direct( - transcript_path, session_id, wing=project_wing, toast=toast + transcript_path, + session_id, + wing=project_wing, + toast=toast, + agent_name=_diary_agent_for_harness(harness), ) _ingest_transcript(transcript_path) _maybe_auto_ingest() diff --git a/mempalace/ids.py b/mempalace/ids.py new file mode 100644 index 0000000..ff20946 --- /dev/null +++ b/mempalace/ids.py @@ -0,0 +1,128 @@ +"""Centralized drawer/triple ID construction with collision-safe delimiter. + +Drawer IDs and content-addressed identifiers built by concatenating strings +without a delimiter before hashing form a defect class that allows +``hash(s1 + str(i1)) == hash(s2 + str(i2))`` whenever +``s1 + str(i1) == s2 + str(i2)``. Under ChromaDB's primary-key constraint +the second upsert silently overwrites the first, losing content with no +error raised. The styleguide's partial-scope-key-migration rule names this +shape — every concat-into-hash site is a candidate that must be triaged. + +This module is the single source of truth for ID construction in mempalace. +All call sites use the named helpers below; no module should inline +``hashlib.sha256(a + b)`` patterns. +""" + +from __future__ import annotations + +import hashlib + +# Recipe tag written to every drawer's metadata under this module's helpers. +# Audits compare like-for-like: drawers without ``id_recipe`` are treated +# as legacy ``v1`` (pre-delimiter recipe), drawers with ``id_recipe="v2"`` +# are guaranteed collision-safe within the v2 generation. The constant is +# exported so call sites use ``ids.ID_RECIPE`` rather than a magic string. +ID_RECIPE: str = "v2" + +# '|' is reserved in Windows filenames and cannot appear in source paths +# on any supported platform, making it strictly safer than ':' (which +# appears in Windows drive letters and URL ports). Matches the existing +# diary_ingest precedent at diary_ingest.py:52,76,91,98. +_DELIM: str = "|" + +# SHA-256 hex truncation lengths. Drawer IDs historically truncate at 24 +# chars; knowledge-graph triple IDs at 12. Preserved per-recipe so existing +# fixture comparisons that hard-code truncation length still parse. +_HASH_TRUNC_DRAWER: int = 24 +_HASH_TRUNC_TRIPLE: int = 12 + + +def _delimited_sha256(parts: tuple[object, ...], truncate: int) -> str: + """Hash parts joined by the unambiguous delimiter, truncate to N hex chars. + + Internal helper. Call sites should use the named ``make_*`` wrappers + below so the per-site contract is documented in code, not derived + from caller arguments. + + Each part is coerced to ``str`` before joining so the helper mirrors + the pre-v2 behavior of ``f"{a}{b}"`` for ``None`` and numeric inputs — + e.g. ``valid_from=None`` joins as the literal string ``"None"`` rather + than crashing. + """ + key = _DELIM.join(str(p) for p in parts).encode() + return hashlib.sha256(key).hexdigest()[:truncate] + + +def make_drawer_id_from_chunk(wing: str, room: str, source_file: str, chunk_index: int) -> str: + """Drawer ID for the project / format miner paths. + + Hash input is ``f"{source_file}|{chunk_index}"`` — the '|' separator + prevents the classic ``"/a1" + "23" == "/a" + "123"`` collision. + + Returns ``drawer_{wing}_{room}_{hash24}`` where hash24 is the first + 24 hex chars of SHA-256 over the delimited input. + """ + return ( + f"drawer_{wing}_{room}_" + f"{_delimited_sha256((source_file, str(chunk_index)), _HASH_TRUNC_DRAWER)}" + ) + + +def make_drawer_id_from_content(wing: str, room: str, content: str) -> str: + """Drawer ID for the MCP ``add_drawer`` tool path. + + Hash input is ``f"{wing}|{room}|{content}"`` — the delimiters prevent + ``wing="foo" + room="bar"`` colliding with ``wing="fooba" + room="r"`` + (architecturally identical defect class to the chunk-index sites, + even though astronomically rare in practice since content is large + freeform text). + """ + return f"drawer_{wing}_{room}_{_delimited_sha256((wing, room, content), _HASH_TRUNC_DRAWER)}" + + +def make_convo_drawer_id( + wing: str, room: str, source_file: str, extract_mode: str, chunk_index: int +) -> str: + """Drawer ID for the conversation miner path. + + Pre-v2 the convo miner used ':' as delimiter; this helper migrates + to '|' for codebase-wide consistency and to remove the Windows-path + / URL-source edge case that ':' carried. + + Hash input is ``f"{source_file}|{extract_mode}|{chunk_index}"``. + """ + return ( + f"drawer_{wing}_{room}_" + f"{_delimited_sha256((source_file, extract_mode, str(chunk_index)), _HASH_TRUNC_DRAWER)}" + ) + + +def make_convo_sentinel_id(source_file: str, extract_mode: str) -> str: + """Sentinel registry ID for the conversation miner zero-chunk-file path. + + Pre-v2 the sentinel used ':' as delimiter; this helper migrates to + '|' for the same reasons as ``make_convo_drawer_id``. + + Hash input is ``f"{source_file}|{extract_mode}"``. + """ + return f"_reg_{_delimited_sha256((source_file, extract_mode), _HASH_TRUNC_DRAWER)}" + + +def make_triple_id( + sub_id: str, predicate: str, obj_id: str, valid_from: str, recorded_at: str +) -> str: + """Triple ID for knowledge-graph insertion. + + Pre-v2 the recorded_at hash input was + ``f"{valid_from}{datetime.now().isoformat()}"`` with no delimiter — + two ISO datetimes concatenated could collide in principle (e.g. + ``valid_from="2026-01-01" + isoformat "T12:..."`` vs + ``valid_from="2026-01-01T12" + isoformat ":..."``). + + Returns ``t_{sub_id}_{predicate}_{obj_id}_{hash12}`` where hash12 is + the first 12 hex chars of SHA-256 over ``f"{valid_from}|{recorded_at}"``. + """ + return ( + f"t_{sub_id}_{predicate}_{obj_id}_" + f"{_delimited_sha256((valid_from, recorded_at), _HASH_TRUNC_TRIPLE)}" + ) diff --git a/mempalace/knowledge_graph.py b/mempalace/knowledge_graph.py index 774ee79..e3f075c 100644 --- a/mempalace/knowledge_graph.py +++ b/mempalace/knowledge_graph.py @@ -35,7 +35,6 @@ Usage: kg.invalidate("Max", "has_issue", "sports_injury", ended="2026-02-15") """ -import hashlib import json import os import sqlite3 @@ -44,6 +43,7 @@ from datetime import date, datetime from pathlib import Path from typing import Optional from .config import sanitize_iso_temporal +from .ids import make_triple_id DEFAULT_KG_PATH = os.path.expanduser("~/.mempalace/knowledge_graph.sqlite3") @@ -302,7 +302,9 @@ class KnowledgeGraph: if existing: return existing["id"] # Already exists and still valid - triple_id = f"t_{sub_id}_{pred}_{obj_id}_{hashlib.sha256(f'{valid_from}{datetime.now().isoformat()}'.encode()).hexdigest()[:12]}" + triple_id = make_triple_id( + sub_id, pred, obj_id, valid_from, datetime.now().isoformat() + ) conn.execute( """INSERT INTO triples ( id, subject, predicate, object, valid_from, valid_to, diff --git a/mempalace/layers.py b/mempalace/layers.py index b92890a..6acf852 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -23,7 +23,12 @@ from collections import defaultdict from .config import MempalaceConfig from .palace import get_collection as _get_collection -from .searcher import _first_or_empty, build_where_filter +from .searcher import ( + _distance_to_similarity, + _first_or_empty, + _metric_for_collection, + build_where_filter, +) # --------------------------------------------------------------------------- @@ -283,11 +288,12 @@ class Layer3: if not docs: return "No results found." + metric = _metric_for_collection(col) lines = [f'## L3 — SEARCH RESULTS for "{query}"'] for i, (doc, meta, dist) in enumerate(zip(docs, metas, dists), 1): meta = meta or {} doc = doc or "" - similarity = round(max(0.0, 1 - dist), 3) + similarity = round(_distance_to_similarity(dist, metric), 3) wing_name = meta.get("wing", "?") room_name = meta.get("room", "?") source = Path(meta.get("source_file", "")).name if meta.get("source_file") else "" @@ -327,6 +333,7 @@ class Layer3: except Exception: return [] + metric = _metric_for_collection(col) hits = [] for doc, meta, dist in zip( _first_or_empty(results, "documents"), @@ -346,7 +353,7 @@ class Layer3: "wing": meta.get("wing", "unknown"), "room": meta.get("room", "unknown"), "source_file": Path(meta.get("source_file", "?")).name, - "similarity": round(1 - dist, 3), + "similarity": round(_distance_to_similarity(dist, metric), 3), "metadata": meta, } ) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 33dafd2..ca4ce3f 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -72,8 +72,13 @@ from .backends.chroma import ( # noqa: E402 _pin_hnsw_threads, hnsw_capacity_status, ) +from .backends import BackendMismatchError, PalaceRef, detect_backend_for_path # noqa: E402 from .query_sanitizer import sanitize_query # noqa: E402 -from .searcher import search_memories # noqa: E402 +from .searcher import ( # noqa: E402 + _distance_to_similarity, + _metric_for_collection, + search_memories, +) from .palace_graph import ( # noqa: E402 traverse, find_tunnels, @@ -83,8 +88,14 @@ from .palace_graph import ( # noqa: E402 delete_tunnel, follow_tunnels, ) +from .hallways import ( # noqa: E402 + list_hallways, + delete_hallway, +) from .knowledge_graph import KnowledgeGraph, DEFAULT_KG_PATH # noqa: E402 +from .collision_scan import assert_no_collisions # noqa: E402 +from .ids import ID_RECIPE, make_drawer_id_from_content # noqa: E402 def _init_logging() -> None: @@ -160,6 +171,11 @@ def _parse_args(): metavar="PATH", help="Path to the palace directory (overrides config file and env var)", ) + parser.add_argument( + "--backend", + metavar="NAME", + help="Storage backend to use (default: config/env/detected/chroma)", + ) args, unknown = parser.parse_known_args() if unknown: logger.debug("Ignoring unknown args: %s", unknown) @@ -170,6 +186,13 @@ _args = _parse_args() if _args.palace: os.environ["MEMPALACE_PALACE_PATH"] = os.path.abspath(_args.palace) +if _args.backend: + backend_name = str(_args.backend).strip().lower() + from .backends import get_backend_class # noqa: E402 + + get_backend_class(backend_name) + os.environ["MEMPALACE_BACKEND_EXPLICIT"] = backend_name + os.environ["MEMPALACE_BACKEND"] = backend_name _config = MempalaceConfig() @@ -289,6 +312,9 @@ def _call_kg(op): _client_cache = None _collection_cache = None +_collection_cache_backend = None +_collection_cache_palace = None +_collection_open_error = None _palace_db_inode = 0 # inode of chroma.sqlite3 at cache time _palace_db_mtime = 0.0 # mtime of chroma.sqlite3 at cache time @@ -301,7 +327,15 @@ def _is_transient_index_error(result) -> bool: if not isinstance(result, dict): return False err = result.get("error", "") - return isinstance(err, str) and ("Error finding id" in err or "Internal error" in err) + if not isinstance(err, str): + return False + err_l = err.lower() + return ( + "error finding id" in err_l + or "internal error" in err_l + or "stale-index" in err_l + or "stale index" in err_l + ) def _force_chroma_cache_reset() -> None: @@ -313,21 +347,27 @@ def _force_chroma_cache_reset() -> None: global \ _client_cache, \ _collection_cache, \ + _collection_cache_backend, \ + _collection_cache_palace, \ + _collection_open_error, \ _palace_db_inode, \ _palace_db_mtime, \ _metadata_cache, \ _metadata_cache_time _client_cache = None _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _collection_open_error = None _palace_db_inode = 0 _palace_db_mtime = 0.0 _metadata_cache = None _metadata_cache_time = 0 try: - from .palace import _DEFAULT_BACKEND + from .palace import get_backend_for_palace - _DEFAULT_BACKEND._clients.pop(_config.palace_path, None) - _DEFAULT_BACKEND._freshness.pop(_config.palace_path, None) + backend = get_backend_for_palace(_config.palace_path) + backend.close_palace(PalaceRef(id=_config.palace_path, local_path=_config.palace_path)) except Exception: pass @@ -356,6 +396,11 @@ def _refresh_vector_disabled_flag() -> None: would defeat the point. """ global _vector_disabled, _vector_disabled_reason, _vector_capacity_status + if not _is_chroma_backend(): + _vector_disabled = False + _vector_disabled_reason = "" + _vector_capacity_status = None + return try: info = hnsw_capacity_status(_config.palace_path, _config.collection_name) except Exception: @@ -387,21 +432,51 @@ def _refresh_vector_disabled_flag() -> None: # This provides an audit trail for detecting memory poisoning and # enables review/rollback of writes from external or untrusted sources. -_WAL_DIR = Path(os.path.expanduser("~/.mempalace/wal")) -_WAL_DIR.mkdir(parents=True, exist_ok=True) -try: - _WAL_DIR.chmod(0o700) -except (OSError, NotImplementedError): - pass -_WAL_FILE = _WAL_DIR / "write_log.jsonl" -# Atomically create WAL file with restricted permissions (no TOCTOU race). -# os.open with O_CREAT|O_WRONLY and mode 0o600 creates the file if absent -# or opens it if present, both in a single syscall. -try: - _fd = os.open(str(_WAL_FILE), os.O_CREAT | os.O_WRONLY, 0o600) - os.close(_fd) -except (OSError, NotImplementedError): - pass +_WAL_FILE = Path(os.path.expanduser("~/.mempalace/wal")) / "write_log.jsonl" +_WAL_INITIALIZED_DIR = None + + +def _ensure_wal() -> None: + """Create (and re-harden) the WAL directory lazily, on the first write. + + This must NOT run at import time: a user who removed ``~/.mempalace`` has + engaged the documented kill-switch (``hooks_cli._palace_root_exists()``, + #1305), and recreating the directory just by importing this module would + silently re-arm the autosave/mining hooks they disabled (#1676). Creating + it on the first real write keeps the kill-switch contract intact. + + It is deliberately not gated on ``_palace_root_exists()``: by the time a + write reaches here the palace is already being recreated by the ChromaDB/KG + layer regardless, so gating would only drop audit records, not prevent + recreation. Runtime kill-switch enforcement for MCP writes is the broader + question tracked in #504. + + Hardening is attempted once per directory and the path cached in + ``_WAL_INITIALIZED_DIR`` regardless of outcome (keyed on the path, so a + test repointing ``_WAL_FILE`` re-initialises), so a persistent failure on a + restricted filesystem does not retry on every write. ``mkdir`` runs only + when the initial ``chmod`` raises ``FileNotFoundError`` (EAFP). The parent + ``~/.mempalace`` keeps its umask mode, like the other palace directories; + the WAL file is created atomically with mode 0o600 by ``_wal_log``. + """ + global _WAL_INITIALIZED_DIR + wal_dir = _WAL_FILE.parent + if _WAL_INITIALIZED_DIR == wal_dir: + return + try: + wal_dir.chmod(0o700) + except FileNotFoundError: + try: + wal_dir.mkdir(parents=True, exist_ok=True) + wal_dir.chmod(0o700) + except (OSError, NotImplementedError): + pass + except (OSError, NotImplementedError): + pass + # Cache regardless of outcome: one attempt per directory, so a persistent + # chmod/mkdir failure (restricted FS) is not retried on every write. + _WAL_INITIALIZED_DIR = wal_dir + # Keys whose values should be redacted in WAL entries to avoid logging sensitive content _WAL_REDACT_KEYS = frozenset( @@ -425,6 +500,9 @@ def _wal_log(operation: str, params: dict, result: dict = None): "result": result, } try: + # Dir setup shares the append's exception handler below: any WAL + # failure is logged and non-fatal, never crashing the tool call. + _ensure_wal() fd = os.open(str(_WAL_FILE), os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) with os.fdopen(fd, "a", encoding="utf-8") as f: f.write(json.dumps(entry, default=str) + "\n") @@ -447,10 +525,15 @@ def _get_client(): global \ _client_cache, \ _collection_cache, \ + _collection_cache_backend, \ + _collection_cache_palace, \ + _collection_open_error, \ _palace_db_inode, \ _palace_db_mtime, \ _metadata_cache, \ _metadata_cache_time + if not _is_chroma_backend(): + raise RuntimeError("_get_client is only available for the Chroma backend") db_path = os.path.join(_config.palace_path, "chroma.sqlite3") try: st = os.stat(db_path) @@ -467,6 +550,9 @@ def _get_client(): if not os.path.isfile(db_path) and _collection_cache is not None: _client_cache = None _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _collection_open_error = None _palace_db_inode = 0 _palace_db_mtime = 0.0 # Fall through to normal reconnect which will handle missing DB @@ -475,13 +561,18 @@ def _get_client(): mtime_changed = current_mtime != 0.0 and abs(current_mtime - _palace_db_mtime) > 0.01 if _client_cache is None or inode_changed or mtime_changed: - # Run the HNSW capacity probe BEFORE chromadb opens the segment — + # Run the HNSW capacity probe BEFORE chromadb opens the segment -- # if the index is severely undersized, segment load can segfault # the whole MCP server (#1222). The probe is pure sqlite + - # metadata-pickle read; never touches the HNSW binary files. + # metadata read; never touches the HNSW binary files. _refresh_vector_disabled_flag() + if inode_changed or mtime_changed: + ChromaBackend._quarantined_paths.discard(_config.palace_path) _client_cache = ChromaBackend.make_client(_config.palace_path) _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _collection_open_error = None _metadata_cache = None _metadata_cache_time = 0 _palace_db_inode = current_inode @@ -490,20 +581,114 @@ def _get_client(): def _get_collection(create=False): - """Return the ChromaDB collection, caching the client between calls. + """Return the configured backend collection, caching handles between calls. On failure, log the exception and retry once after clearing the client and collection caches. Tools were silently returning ``None`` when a cached client/collection went stale — typically after the chromadb rust bindings invalidated a handle following an out-of-band write — leaving the LLM with no diagnostic and no recovery path. The retry - forces ``_get_client()`` to rebuild from scratch (which re-runs - ``quarantine_stale_hnsw`` per #1322), so the second attempt heals the - common stale-handle / stale-HNSW case automatically. + forces ``_get_client()`` to rebuild the chromadb client from + scratch, so the second attempt heals the common stale-handle case + automatically. """ - global _client_cache, _collection_cache, _metadata_cache, _metadata_cache_time + global \ + _client_cache, \ + _collection_cache, \ + _collection_cache_backend, \ + _collection_cache_palace, \ + _collection_open_error, \ + _palace_db_inode, \ + _palace_db_mtime, \ + _metadata_cache, \ + _metadata_cache_time + try: + backend_name = _selected_backend_name() + except (BackendMismatchError, KeyError) as exc: + logger.warning("backend resolution failed for %s: %s", _config.palace_path, exc) + _collection_open_error = { + "error": "Backend mismatch" + if isinstance(exc, BackendMismatchError) + else "Unknown backend", + "details": str(exc), + "hint": "Select the matching backend or use a fresh palace directory.", + } + _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + return None + + if backend_name != "chroma": + for attempt in range(2): + try: + if ( + _collection_cache is not None + and _collection_cache_backend == backend_name + and _collection_cache_palace == _config.palace_path + ): + _collection_open_error = None + return _collection_cache + _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + if _collection_cache is None: + from .palace import get_collection as palace_get_collection + + _collection_cache = palace_get_collection( + _config.palace_path, + collection_name=_config.collection_name, + create=create, + backend=backend_name, + ) + _collection_cache_backend = backend_name + _collection_cache_palace = _config.palace_path + _collection_open_error = None + _metadata_cache = None + _metadata_cache_time = 0 + return _collection_cache + except (BackendMismatchError, KeyError) as exc: + logger.warning("backend open failed for %s: %s", _config.palace_path, exc) + _collection_open_error = { + "error": "Backend mismatch" + if isinstance(exc, BackendMismatchError) + else "Unknown backend", + "details": str(exc), + "hint": "Select the matching backend or use a fresh palace directory.", + } + _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _metadata_cache = None + _metadata_cache_time = 0 + return None + except Exception: + logger.exception( + "_get_collection generic attempt %d/2 failed (palace=%s, create=%s)", + attempt + 1, + _config.palace_path, + create, + ) + _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _metadata_cache = None + _metadata_cache_time = 0 + _collection_open_error = { + "error": "Backend open failed", + "details": "Could not open the selected backend collection.", + "hint": "Run: mempalace status or mempalace repair-status for diagnostics.", + } + return None + for attempt in range(2): try: + if _collection_cache is not None and ( + _collection_cache_backend not in (None, "chroma") + or _collection_cache_palace not in (None, _config.palace_path) + ): + _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None client = _get_client() # ChromaDB 1.x persists the EF *identity* (its ``name()``) with the # collection but not the EF *instance/configuration*. So a reader or @@ -550,6 +735,9 @@ def _get_collection(create=False): ) _pin_hnsw_threads(raw) _collection_cache = ChromaCollection(raw, palace_path=_config.palace_path) + _collection_cache_backend = "chroma" + _collection_cache_palace = _config.palace_path + _collection_open_error = None _metadata_cache = None _metadata_cache_time = 0 elif _collection_cache is None: @@ -558,9 +746,29 @@ def _get_collection(create=False): raw = client.get_collection(_config.collection_name, **ef_kwargs) _pin_hnsw_threads(raw) _collection_cache = ChromaCollection(raw, palace_path=_config.palace_path) + _collection_cache_backend = "chroma" + _collection_cache_palace = _config.palace_path + _collection_open_error = None _metadata_cache = None _metadata_cache_time = 0 return _collection_cache + except (BackendMismatchError, KeyError) as exc: + _collection_open_error = { + "error": "Backend mismatch" + if isinstance(exc, BackendMismatchError) + else "Unknown backend", + "details": str(exc), + "hint": "Select the matching backend or use a fresh palace directory.", + } + _client_cache = None + _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _palace_db_inode = 0 + _palace_db_mtime = 0.0 + _metadata_cache = None + _metadata_cache_time = 0 + return None except Exception: logger.exception( "_get_collection attempt %d/2 failed (palace=%s, create=%s)", @@ -570,13 +778,35 @@ def _get_collection(create=False): ) if attempt == 0: # Reset all caches so the next attempt forces _get_client() - # to rebuild the chromadb client from scratch — that path - # re-runs quarantine_stale_hnsw (#1322) and reopens the - # collection cleanly, healing the common stale-handle case. + # to rebuild the chromadb client from scratch, reopening + # the collection cleanly and healing the common + # stale-handle case. _client_cache = None _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _palace_db_inode = 0 + _palace_db_mtime = 0.0 _metadata_cache = None _metadata_cache_time = 0 + _collection_open_error = { + "error": "Backend open failed", + "details": "Could not open the Chroma collection.", + "hint": "Run: mempalace repair-status for diagnostics.", + } + _client_cache = None + _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _palace_db_inode = 0 + _palace_db_mtime = 0.0 + _metadata_cache = None + _metadata_cache_time = 0 + _collection_open_error = _collection_open_error or { + "error": "Backend open failed", + "details": "Could not open the selected backend collection.", + "hint": "Run: mempalace status or mempalace repair-status for diagnostics.", + } return None @@ -587,6 +817,42 @@ def _no_palace(): } +def _collection_error_or_no_palace(): + if not _collection_open_error: + return _no_palace() + result = dict(_collection_open_error) + try: + result["backend"] = _selected_backend_name() + except Exception: + pass + return result + + +def _selected_backend_name() -> str: + from .palace import resolve_backend_name + + return resolve_backend_name( + _config.palace_path, + explicit=os.environ.get("MEMPALACE_BACKEND_EXPLICIT"), + ) + + +def _is_chroma_backend() -> bool: + try: + return _selected_backend_name() == "chroma" + except Exception: + logger.debug("backend resolution failed", exc_info=True) + return False + + +def _backend_db_exists() -> bool: + try: + return detect_backend_for_path(_config.palace_path) is not None + except Exception: + logger.debug("backend artifact detection failed", exc_info=True) + return False + + # ==================== HELPERS ==================== @@ -722,6 +988,7 @@ def _tool_status_via_sqlite() -> dict: "rooms": rooms, "protocol": PALACE_PROTOCOL, "aaak_dialect": AAAK_SPEC, + "backend": "chroma", "vector_disabled": True, "vector_disabled_reason": _vector_disabled_reason, } @@ -739,7 +1006,7 @@ def tool_status(): # #1222 failure mode, opening the persistent client to call .count() # can segfault — short-circuit to a pure-sqlite path when divergence # is detected so status stays reachable. - db_exists = os.path.isfile(os.path.join(_config.palace_path, "chroma.sqlite3")) + db_exists = _backend_db_exists() _refresh_vector_disabled_flag() if _vector_disabled: @@ -750,7 +1017,7 @@ def tool_status(): # accidentally creating a palace in a non-existent directory (#830). col = _get_collection(create=db_exists) if not col: - return _no_palace() + return _collection_error_or_no_palace() count = col.count() wings = {} rooms = {} @@ -760,6 +1027,7 @@ def tool_status(): "rooms": rooms, "protocol": PALACE_PROTOCOL, "aaak_dialect": AAAK_SPEC, + "backend": _selected_backend_name(), } try: all_meta = _get_cached_metadata(col) @@ -812,7 +1080,7 @@ When WRITING AAAK: use entity codes, mark emotions, keep structure tight.""" def tool_list_wings(): col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() wings = {} result = {"wings": wings} try: @@ -835,7 +1103,7 @@ def tool_list_rooms(wing: str = None): return {"error": str(e)} col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() rooms = {} result = {"wing": wing or "all", "rooms": rooms} try: @@ -855,7 +1123,7 @@ def tool_list_rooms(wing: str = None): def tool_get_taxonomy(): col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() taxonomy = {} result = {"taxonomy": taxonomy} try: @@ -925,6 +1193,7 @@ def tool_search( n_results=limit, max_distance=dist, vector_disabled=_vector_disabled, + collection_name=_config.collection_name, ) if not _is_transient_index_error(result): result["index_recovered"] = True @@ -963,7 +1232,7 @@ def tool_check_duplicate(content: str, threshold: float = 0.9): } col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() try: content = strip_lone_surrogates(content) results = col.query( @@ -973,9 +1242,10 @@ def tool_check_duplicate(content: str, threshold: float = 0.9): ) duplicates = [] if results["ids"] and results["ids"][0]: + metric = _metric_for_collection(col) for i, drawer_id in enumerate(results["ids"][0]): dist = results["distances"][0][i] - similarity = round(max(0.0, 1 - dist), 3) + similarity = round(_distance_to_similarity(dist, metric), 3) if similarity >= threshold: # Chroma 1.5.x can return None for partially-flushed rows; # coerce to empty sentinels so downstream .get() is safe. @@ -1009,7 +1279,7 @@ def tool_traverse_graph(start_room: str, max_hops: int = 2): max_hops = max(1, min(max_hops, 10)) col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() return traverse(start_room, col=col, max_hops=max_hops) @@ -1022,7 +1292,7 @@ def tool_find_tunnels(wing_a: str = None, wing_b: str = None): return {"error": str(e)} col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() return find_tunnels(wing_a, wing_b, col=col) @@ -1030,7 +1300,7 @@ def tool_graph_stats(): """Palace graph overview: nodes, tunnels, edges, connectivity.""" col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() return graph_stats(col=col) @@ -1088,6 +1358,22 @@ def tool_delete_tunnel(tunnel_id: str): return delete_tunnel(tunnel_id) +def tool_list_hallways(wing: str = None): + """List within-wing hallway records, optionally filtered by wing.""" + try: + wing = _sanitize_optional_name(wing, "wing") + except ValueError as e: + return {"error": str(e)} + return list_hallways(wing) + + +def tool_delete_hallway(hallway_id: str): + """Delete a hallway record by its ID.""" + if not hallway_id or not isinstance(hallway_id, str): + return {"error": "hallway_id is required"} + return {"deleted": delete_hallway(hallway_id)} + + def tool_follow_tunnels(wing: str, room: str): """Follow explicit tunnels from a room to see connected drawers in other wings.""" try: @@ -1096,6 +1382,8 @@ def tool_follow_tunnels(wing: str, room: str): except ValueError as e: return {"error": str(e)} col = _get_collection() + if not col: + return _collection_error_or_no_palace() return follow_tunnels(wing, room, col=col) @@ -1130,11 +1418,9 @@ def tool_add_drawer( col = _get_collection(create=True) if not col: - return _no_palace() + return _collection_error_or_no_palace() - drawer_id = ( - f"drawer_{wing}_{room}_{hashlib.sha256((wing + room + content).encode()).hexdigest()[:24]}" - ) + drawer_id = make_drawer_id_from_content(wing, room, content) _wal_log( "add_drawer", @@ -1155,6 +1441,7 @@ def tool_add_drawer( "source_file": source_file or "", "added_by": added_by, "filed_at": datetime.now().isoformat(), + "id_recipe": ID_RECIPE, } # Idempotency. Three cases to detect a prior committed write: @@ -1216,6 +1503,7 @@ def tool_add_drawer( chunk_metas.append( {**base_meta, "chunk_index": chunk_idx, "parent_drawer_id": drawer_id} ) + assert_no_collisions(list(zip(chunk_ids, chunk_metas)), col) col.upsert(ids=chunk_ids, documents=chunk_docs, metadatas=chunk_metas) # Probe the LAST chunk id, not the first — its presence confirms # the whole batch landed, not just the leading row. @@ -1244,7 +1532,7 @@ def tool_delete_drawer(drawer_id: str): global _metadata_cache col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() existing = col.get(ids=[drawer_id]) if not existing["ids"]: return {"success": False, "error": f"Drawer not found: {drawer_id}"} @@ -1272,6 +1560,213 @@ def tool_delete_drawer(drawer_id: str): return {"success": False, "error": str(e)} +def _capture_fd_stdout(fn): + """Run ``fn()`` with its stdout captured at both the Python and fd level. + + The mining engines (``miner.mine`` / ``convo_miner.mine_convos`` / + ``format_miner.mine_formats``) print progress and a summary to stdout. In + the MCP server stdout is the JSON-RPC channel (``_restore_stdout`` runs once + in ``main`` before the protocol loop), so that output would corrupt the + protocol. Two layers are needed: + + * ``contextlib.redirect_stdout`` captures Python-level ``print`` into a + buffer — this is what becomes the returned summary, and it works even when + ``sys.stdout`` has been swapped (e.g. under pytest capture). + * an ``os.dup2`` of fd 1 to a temp file contains C-level banners emitted by + onnxruntime / chromadb during embedding, which bypass ``sys.stdout`` + entirely (the same reason the module redirects fd 1 at import, #225), and + keeps any direct fd-1 write off the live JSON-RPC channel. + + Returns ``(result, captured_text)``. ``captured_text`` is handed back to the + caller verbatim as an opaque summary; it is never parsed into fields. Falls + back to Python-level capture alone on platforms without fd-level stdio + (embedded interpreters), matching the import-time fallback. + """ + import contextlib + import io + import tempfile + + buf = io.StringIO() + sys.stdout.flush() + sys.stderr.flush() + try: + saved_fd = os.dup(1) + except (OSError, AttributeError): + with contextlib.redirect_stdout(buf): + result = fn() + return result, buf.getvalue() + + try: + with tempfile.TemporaryFile() as tmp: + os.dup2(tmp.fileno(), 1) + try: + with contextlib.redirect_stdout(buf): + result = fn() + finally: + sys.stdout.flush() + os.dup2(saved_fd, 1) + tmp.seek(0) + fd_text = tmp.read().decode("utf-8", "replace") + return result, buf.getvalue() + fd_text + finally: + os.close(saved_fd) + + +def tool_mine( + source: str, + mode: str = "projects", + wing: str = None, + agent: str = "mempalace", + limit: int = 0, + dry_run: bool = False, + extract: str = "exchange", +): + """Mine a directory into the palace — the MCP equivalent of ``mempalace mine``. + + Lets MCP clients that cannot shell out (Claude Desktop, LM Studio, Aionui, + Desktop Commander) trigger indexing in-conversation (#1662). Wraps the same + in-process miners the CLI's ``cmd_mine`` calls; it adds no new ingestion + logic of its own. + + mode: + ``"projects"`` (default) — code/docs via ``miner.mine``. + ``"convos"`` — chat transcripts via ``convo_miner.mine_convos``. + ``"extract"`` — office documents (PDF/DOCX/RTF/…) via + ``format_miner.mine_formats``; requires the + optional ``mempalace[extract]`` dependency. + wing: target wing (default: derived from the source directory name). + agent: recorded on every drawer (default ``"mempalace"``). + limit: max files to process (0 = all). + dry_run: walk + chunk and report, but file nothing. + extract: convos extraction strategy — ``"exchange"`` (default) or + ``"general"``; ignored by the other modes. + + Runs synchronously and mirrors the :func:`tool_sync` contract: success + returns ``{success: True, mode, dry_run, output[, output_truncated]}`` where ``output`` is + the miner's human-readable summary (captured so it cannot corrupt the + JSON-RPC stream); failure returns ``{success: False, error[, error_class]}``. + The palace write lock is held by the miners themselves, so a concurrent mine + surfaces as a structured already-running error. Orphan cleanup is not part of + mining — use ``mempalace_sync`` for that. + """ + global _metadata_cache + from .palace import MineAlreadyRunning, MineValidationError + + if not _config.palace_path: + np = _no_palace() + return {"success": False, "error": np.get("error", "no palace"), "hint": np.get("hint")} + + valid_modes = ("projects", "convos", "extract") + if mode not in valid_modes: + return { + "success": False, + "error": f"invalid mode '{mode}'; expected one of: {', '.join(valid_modes)}", + } + + src = os.path.expanduser(source) if source else "" + if not src or not os.path.isdir(src): + return {"success": False, "error": f"source directory not found: {source!r}"} + + def _run(): + if mode == "convos": + from .convo_miner import mine_convos + + return mine_convos( + convo_dir=src, + palace_path=_config.palace_path, + wing=wing, + agent=agent, + limit=limit, + dry_run=dry_run, + extract_mode=extract, + ) + if mode == "extract": + from .format_miner import mine_formats + + return mine_formats( + format_dir=src, + palace_path=_config.palace_path, + wing=wing, + agent=agent, + limit=limit, + dry_run=dry_run, + ) + from .miner import mine + + return mine( + project_dir=src, + palace_path=_config.palace_path, + wing_override=wing, + agent=agent, + limit=limit, + dry_run=dry_run, + ) + + try: + try: + _result, output = _capture_fd_stdout(_run) + # Order matters: typed handlers precede the bare Exception (mirroring + # tool_sync) so MineAlreadyRunning / MineValidationError / ValueError + # don't fall into the generic "mine failed" branch. + except MineAlreadyRunning as exc: + return { + "success": False, + "error": f"another mine is in progress: {exc}", + "error_class": "LockHeldByOtherProcess", + } + except MineValidationError as exc: + return { + "success": False, + "error": f"palace integrity check failed after mine: {exc}", + "error_class": "MineValidationError", + } + except ImportError as exc: + # 'extract' mode pulls in the optional mempalace[extract] stack; + # name it so the caller knows to install the extra. Other modes have + # no optional imports, so an ImportError there is a real bug, not a + # missing extra — log the traceback and surface its type. + if mode == "extract": + return { + "success": False, + "error": f"mode 'extract' needs the mempalace[extract] extra: {exc}", + "error_class": "MissingDependency", + } + logger.exception("tool_mine: unexpected ImportError (mode=%s)", mode) + return {"success": False, "error": f"mine failed: {exc}", "error_class": "ImportError"} + except ValueError as exc: + return {"success": False, "error": str(exc), "error_class": "ValueError"} + except SystemExit as exc: + # A library mine() must never terminate the MCP server. miner.mine + # converts Ctrl-C into sys.exit(130) (CLI semantics); in-process + # that SystemExit is a BaseException that would slip past the + # protocol loop's `except Exception` and kill the server with no + # response. Convert it to a structured error instead. + return { + "success": False, + "error": f"mine exited early (code {exc.code})", + "error_class": "Interrupted", + } + except Exception as exc: + logger.exception("tool_mine: mine failed (mode=%s)", mode) + return { + "success": False, + "error": f"mine failed: {exc}", + "error_class": type(exc).__name__, + } + # Cap the echoed summary so a very large mine cannot return a multi-MB + # payload to the MCP client. The useful summary is at the tail, so keep + # the end and flag the truncation (never silently). + payload = {"success": True, "mode": mode, "dry_run": dry_run, "output": output} + cap = 4000 + if len(output) > cap: + payload["output"] = output[-cap:] + payload["output_truncated"] = True + return payload + finally: + if not dry_run: + _metadata_cache = None + + def tool_sync(project_dir: str = None, wing: str = None, apply: bool = False): """Prune drawers whose source files are gitignored, missing, or moved (#1252).""" global _metadata_cache @@ -1314,7 +1809,7 @@ def tool_get_drawer(drawer_id: str): """Fetch a single drawer by ID. Returns full content and metadata.""" col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() try: result = col.get(ids=[drawer_id], include=["documents", "metadatas"]) if not result["ids"]: @@ -1352,7 +1847,7 @@ def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offse return {"error": str(e)} col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() try: where = None conditions = [] @@ -1409,7 +1904,7 @@ def tool_update_drawer(drawer_id: str, content: str = None, wing: str = None, ro col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() try: existing = col.get(ids=[drawer_id], include=["documents", "metadatas"]) if not existing["ids"]: @@ -1428,14 +1923,22 @@ def tool_update_drawer(drawer_id: str, content: str = None, wing: str = None, ro new_meta = dict(old_meta) if wing is not None: try: - new_meta["wing"] = sanitize_name(wing, "wing") + wing = sanitize_name(wing, "wing") except ValueError as e: return {"success": False, "error": str(e)} + # Preserve existing casing when the caller passes a case-only + # variant (LLM clients often "autocorrect" acronyms like ps5→PS5). + if wing.lower() != str(old_meta.get("wing") or "").lower(): + new_meta["wing"] = wing if room is not None: try: - new_meta["room"] = sanitize_name(room, "room") + room = sanitize_name(room, "room") except ValueError as e: return {"success": False, "error": str(e)} + # Preserve existing casing when the caller passes a case-only + # variant (LLM clients often "autocorrect" acronyms like ps5→PS5). + if room.lower() != str(old_meta.get("room") or "").lower(): + new_meta["room"] = room _wal_log( "update_drawer", @@ -1627,7 +2130,7 @@ def tool_diary_write(agent_name: str, entry: str, topic: str = "general", wing: room = "diary" col = _get_collection(create=True) if not col: - return _no_palace() + return _collection_error_or_no_palace() now = datetime.now() entry_id = ( @@ -1750,7 +2253,7 @@ def tool_diary_read(agent_name: str, last_n: int = 10, wing: str = ""): last_n = max(1, min(last_n, 100)) col = _get_collection() if not col: - return _no_palace() + return _collection_error_or_no_palace() # Build filter: always scope by agent + room=diary. Wing is optional — # when empty, return entries across all wings for this agent (matches @@ -1884,6 +2387,9 @@ def tool_reconnect(): global \ _client_cache, \ _collection_cache, \ + _collection_cache_backend, \ + _collection_cache_palace, \ + _collection_open_error, \ _palace_db_inode, \ _palace_db_mtime, \ _vector_disabled, \ @@ -1891,31 +2397,63 @@ def tool_reconnect(): from . import palace as palace_module close_errors = [] + palace_ref = PalaceRef(id=_config.palace_path, local_path=_config.palace_path) + closed_backend_names = set() + cached_backend_name = _collection_cache_backend try: - palace_module._DEFAULT_BACKEND.close_palace(_config.palace_path) + backend = palace_module.get_backend_for_palace(_config.palace_path) + backend.close_palace(palace_ref) + if getattr(backend, "name", None): + closed_backend_names.add(backend.name) except Exception as exc: logger.debug("Failed to close shared palace backend during reconnect", exc_info=True) close_errors.append(f"backend close_palace failed: {exc}") - try: - from chromadb.api.client import SharedSystemClient + if cached_backend_name and cached_backend_name not in closed_backend_names: + try: + from .backends import get_backend - clear_system_cache = getattr(SharedSystemClient, "clear_system_cache", None) - if callable(clear_system_cache): - clear_system_cache() - else: + get_backend(cached_backend_name).close_palace(palace_ref) + closed_backend_names.add(cached_backend_name) + except Exception as exc: logger.debug( - "SharedSystemClient.clear_system_cache is unavailable; skipping shared Chroma cache clear during reconnect" + "Failed to close previously cached %s backend during reconnect", + cached_backend_name, + exc_info=True, ) - except Exception as exc: - logger.debug( - "Failed to clear Chroma shared system cache during reconnect", - exc_info=True, - ) - close_errors.append(f"shared Chroma cache clear failed: {exc}") + close_errors.append(f"cached {cached_backend_name} close_palace failed: {exc}") + if _client_cache is not None: + try: + close = getattr(_client_cache, "close", None) + if callable(close): + close() + except Exception as exc: + logger.debug("Failed to close MCP-local Chroma client during reconnect", exc_info=True) + close_errors.append(f"local Chroma client close failed: {exc}") + if _is_chroma_backend(): + try: + from chromadb.api.client import SharedSystemClient + + clear_system_cache = getattr(SharedSystemClient, "clear_system_cache", None) + if callable(clear_system_cache): + clear_system_cache() + else: + logger.debug( + "SharedSystemClient.clear_system_cache is unavailable; skipping shared Chroma cache clear during reconnect" + ) + except Exception as exc: + logger.debug( + "Failed to clear Chroma shared system cache during reconnect", + exc_info=True, + ) + close_errors.append(f"shared Chroma cache clear failed: {exc}") _client_cache = None _collection_cache = None + _collection_cache_backend = None + _collection_cache_palace = None + _collection_open_error = None _palace_db_inode = 0 _palace_db_mtime = 0.0 + ChromaBackend._quarantined_paths.discard(_config.palace_path) # Force probe re-run on next _get_client by clearing the flag now; # _refresh_vector_disabled_flag will re-set it if the divergence # still applies after the reconnect. @@ -1933,12 +2471,17 @@ def tool_reconnect(): try: col = _get_collection() if col is None: + open_error = _collection_error_or_no_palace() result = { "success": False, - "message": "No palace found after reconnect", + "message": open_error.get("error", "No palace found after reconnect"), "drawers": 0, "vector_disabled": _vector_disabled, } + if "details" in open_error: + result["details"] = open_error["details"] + if "hint" in open_error: + result["hint"] = open_error["hint"] if close_errors: result["error"] = "; ".join(close_errors) return result @@ -2169,6 +2712,30 @@ TOOLS = { }, "handler": tool_delete_tunnel, }, + "mempalace_list_hallways": { + "description": "List within-wing hallway records (entity-to-entity co-occurrence links built at mine time). Optionally filter by wing.", + "input_schema": { + "type": "object", + "properties": { + "wing": { + "type": "string", + "description": "Filter hallways by wing", + }, + }, + }, + "handler": tool_list_hallways, + }, + "mempalace_delete_hallway": { + "description": "Delete a hallway record by its ID. Returns {deleted: bool}.", + "input_schema": { + "type": "object", + "properties": { + "hallway_id": {"type": "string", "description": "Hallway ID to delete"}, + }, + "required": ["hallway_id"], + }, + "handler": tool_delete_hallway, + }, "mempalace_follow_tunnels": { "description": "Follow tunnels from a room to see what it connects to in other wings. Returns connected rooms with drawer previews.", "input_schema": { @@ -2259,6 +2826,60 @@ TOOLS = { }, "handler": tool_delete_drawer, }, + "mempalace_mine": { + "description": ( + "Mine a directory into the palace — the MCP equivalent of `mempalace mine`. " + "mode='projects' (default) ingests code/docs; mode='convos' ingests chat " + "transcripts; mode='extract' ingests office documents (PDF/DOCX/RTF, requires " + "the mempalace[extract] extra). Runs synchronously and returns the miner's " + "summary as `output`. The palace write lock is automatic; a concurrent mine " + "returns a structured already-running error. Orphan cleanup is separate — use " + "mempalace_sync." + ), + "input_schema": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Directory to mine.", + }, + "mode": { + "type": "string", + "enum": ["projects", "convos", "extract"], + "description": ( + "Ingest mode: projects (code/docs, default), convos (chat " + "transcripts), extract (office docs)." + ), + }, + "wing": { + "type": "string", + "description": "Target wing (default: source directory name).", + }, + "agent": { + "type": "string", + "description": "Recorded on every drawer (default: mempalace).", + }, + "limit": { + "type": "integer", + "description": "Max files to process (0 = all). Default: 0.", + }, + "dry_run": { + "type": "boolean", + "description": "Report what would be filed without writing. Default: false.", + }, + "extract": { + "type": "string", + "enum": ["exchange", "general"], + "description": ( + "Convos extraction strategy: exchange (default) or general. " + "Ignored by other modes." + ), + }, + }, + "required": ["source"], + }, + "handler": tool_mine, + }, "mempalace_sync": { "description": "Prune drawers whose source files are gitignored, deleted, or moved. Returns dry-run report by default; pass apply=true to commit deletions.", "input_schema": { @@ -2354,8 +2975,18 @@ TOOLS = { "type": "string", "description": "Target wing for this diary entry (optional). If omitted, uses wing_{agent_name}. Use this to write diary entries to a project wing instead of an agent-specific wing.", }, + "content": { + "type": "string", + "description": "Alias for 'entry' — accepted because add_drawer uses 'content'. Provide either 'entry' or 'content'; 'entry' wins if both are given.", + }, }, - "required": ["agent_name", "entry"], + # agent_name is always required; 'entry' or its alias 'content' must + # be present (the server remaps content->entry at dispatch). + "required": ["agent_name"], + "anyOf": [ + {"required": ["entry"]}, + {"required": ["content"]}, + ], }, "handler": tool_diary_write, }, @@ -2562,6 +3193,17 @@ def handle_request(request): "error": {"code": -32602, "message": f"Invalid value for parameter '{key}'"}, } tool_args.pop("wait_for_previous", None) + # 'content' is an accepted alias for diary_write's 'entry' (callers often + # reuse add_drawer's 'content' name). Map it in here, before dispatch, so a + # content-only call still satisfies the required 'entry' param while the + # signature-based missing-parameter diagnostic (-32602) keeps working. + # 'entry' wins if both are supplied. + if tool_name == "mempalace_diary_write" and "content" in tool_args: + content_val = tool_args.pop("content") + # Only fill from the alias when the caller did not supply 'entry' at + # all (or passed it as null). An explicit entry — even "" — wins. + if "entry" not in tool_args or tool_args["entry"] is None: + tool_args["entry"] = content_val try: result = TOOLS[tool_name]["handler"](**tool_args) return { @@ -2724,8 +3366,17 @@ def _maybe_eager_warmup_embedder() -> None: ) return palace_path = _config.palace_path - db_path = os.path.join(palace_path, "chroma.sqlite3") - if not os.path.isfile(db_path): + try: + backend_name = _selected_backend_name() + except Exception as exc: # fail-soft per docstring + logger.warning( + "MEMPALACE_EAGER_WARMUP=%s: backend resolution failed for %s (%s)", + raw, + palace_path, + exc, + ) + return + if not _backend_db_exists(): # Pre-check (NOT a try/except on _ChromaNotFoundError, which never # propagates out of _get_collection — see docstring). No palace # file means nothing to warm AND avoids the chromadb-client @@ -2769,9 +3420,11 @@ def _maybe_eager_warmup_embedder() -> None: type(exc).__name__, ) else: + warmed = "embedder + HNSW ready" if backend_name == "chroma" else "embedder + backend ready" logger.info( - "MEMPALACE_EAGER_WARMUP=%s: embedder + HNSW ready (palace=%s, device=%s)", + "MEMPALACE_EAGER_WARMUP=%s: %s (palace=%s, device=%s)", raw, + warmed, palace_path, device, ) diff --git a/mempalace/migrate.py b/mempalace/migrate.py index 9e809da..0814bf5 100644 --- a/mempalace/migrate.py +++ b/mempalace/migrate.py @@ -19,6 +19,7 @@ Usage: """ import errno +import glob import os import shutil import sqlite3 @@ -28,6 +29,9 @@ from collections import defaultdict from contextlib import closing from datetime import datetime +from .backups import prune_backups +from .config import MempalaceConfig + def _restore_stale_palace(palace_path: str, stale_path: str) -> None: """Roll back a failed swap. @@ -293,6 +297,16 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False): print(f"\n Backing up to {backup_path}...") shutil.copytree(palace_path, backup_path) + # Enforce backup retention so repeated migrations cannot fill the disk + # with full-palace copies. The backup we just created is the newest, so + # it survives; only older ``.pre-migrate.*`` siblings beyond the limit + # are removed. Best-effort — never let cleanup fail the migration. + prune_backups( + glob.escape(palace_path) + ".pre-migrate.*", + MempalaceConfig().max_backups, + log=print, + ) + # Build fresh palace in a temp directory (avoids chromadb reading old state). # Wrap the whole import-and-swap dance in try/finally so the temp dir is # cleaned up if any of the chromadb writes, the verify count, or the @@ -360,3 +374,218 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False): print(f"\n{'=' * 60}\n") return True + + +# --------------------------------------------------------------------------- +# Wing-name normalization migration (#1675 follow-up) +# --------------------------------------------------------------------------- +# +# normalize_wing_name now strips leading/trailing separators, so a path-encoded +# dirname like ``-home-user-proj`` derives ``home_user_proj`` instead of +# ``_home_user_proj``. Palaces built before that rule filed drawers under the +# old, leading-underscore wing, which the new derivation no longer matches — +# searches and diary reads under the new name miss the old memories. +# +# This migration re-keys the ``wing`` metadata field on drawers and closets to +# the normalized form, merging collisions. Drawer/closet IDs embed the wing as +# an opaque prefix that is never decoded back into a wing (verified: nothing +# splits a wing out of an ID; mining idempotency keys on ``source_file``), so +# the IDs are left untouched — closet ``→drawer_id`` pointers stay valid and +# future mining still skips already-mined files. Tunnels resolve via existing +# read-time normalization and need no rewrite. The pass is idempotent. + + +def _normalized_wing_target(wing): + """Return the normalized wing if it differs from ``wing``, else ``None``. + + ``None`` means "no migration needed" — either the value is not a non-empty + string, normalization is a no-op, or it would normalize to empty. + """ + from .config import normalize_wing_name + + if not isinstance(wing, str) or not wing: + return None + # Apply the full normalization and explicitly strip leading/trailing + # separators. The strip is this migration's whole purpose (#1675); doing it + # here rather than relying on normalize_wing_name keeps the migration correct + # even when run against a build whose normalize_wing_name predates #1675, and + # matches the post-#1675 derivation exactly. + target = normalize_wing_name(wing).strip("_") + if not target or target == wing: + return None + return target + + +def plan_wing_renames(items): + """Pure planner over ``(id, metadata)`` pairs. + + Returns ``(summary, updates)`` where ``summary`` is ``{(old, new): count}`` + and ``updates`` is ``[(id, new_metadata), ...]`` for only the records whose + wing changes. Metadata is copied; only the ``wing`` key is rewritten. + """ + summary = defaultdict(int) + updates = [] + for rec_id, meta in items: + meta = dict(meta or {}) + target = _normalized_wing_target(meta.get("wing")) + if target is None: + continue + summary[(meta["wing"], target)] += 1 + meta["wing"] = target + updates.append((rec_id, meta)) + return summary, updates + + +def _iter_collection_items(col, batch_size=1000): + """Yield ``(id, metadata)`` for every record in a backend collection.""" + total = col.count() + offset = 0 + while offset < total: + batch = col.get(limit=batch_size, offset=offset, include=["metadatas"]) + ids = batch.ids if hasattr(batch, "ids") else batch["ids"] + metas = batch.metadatas if hasattr(batch, "metadatas") else batch["metadatas"] + if not ids: + break + for rec_id, meta in zip(ids, metas): + yield rec_id, meta + offset += len(ids) + + +def _apply_wing_updates(col, updates, batch_size=500): + """Re-label the ``wing`` metadata field in place for the planned updates.""" + for i in range(0, len(updates), batch_size): + chunk = updates[i : i + batch_size] + col.update(ids=[u[0] for u in chunk], metadatas=[u[1] for u in chunk]) + + +def _plan_topics_by_wing_renames(): + """Return ``{old_wing: new_wing}`` for ``topics_by_wing`` keys to normalize.""" + try: + from .miner import _load_known_entities_raw + + reg = _load_known_entities_raw() + except Exception: + return {} + tbw = reg.get("topics_by_wing") + if not isinstance(tbw, dict): + return {} + renames = {} + for wing in list(tbw.keys()): + target = _normalized_wing_target(wing) + if target is not None: + renames[wing] = target + return renames + + +def _apply_topics_by_wing_renames(renames): + """Re-key ``topics_by_wing`` in known_entities.json, merging on collision.""" + if not renames: + return + import json + + from .miner import _ENTITY_REGISTRY_PATH, _load_known_entities_raw + + try: + reg = _load_known_entities_raw() + except Exception: + return + tbw = reg.get("topics_by_wing") + if not isinstance(tbw, dict): + return + for old, new in renames.items(): + if old not in tbw: + continue + old_topics = tbw.pop(old) or [] + if new in tbw: + merged = list(tbw[new]) + for topic in old_topics: + if topic not in merged: + merged.append(topic) + tbw[new] = merged + else: + tbw[new] = old_topics + reg["topics_by_wing"] = tbw + os.makedirs(os.path.dirname(_ENTITY_REGISTRY_PATH), exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(_ENTITY_REGISTRY_PATH), suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(reg, f, ensure_ascii=False, indent=2) + os.replace(tmp, _ENTITY_REGISTRY_PATH) + except Exception: + if os.path.exists(tmp): + os.remove(tmp) + raise + + +def migrate_wing_names(palace_path: str, dry_run: bool = False, confirm: bool = False) -> bool: + """Normalize legacy wing names in ``palace_path`` (strip leading/trailing + separators), so palaces built before #1675 keep their memories discoverable. + + Re-keys the ``wing`` metadata on drawers and closets in place (IDs untouched) + and the ``topics_by_wing`` registry, merging collisions. Idempotent. + + Returns True if anything was (or, in dry-run, would be) migrated. + """ + from .palace import get_closets_collection, get_collection + + try: + drawers = get_collection(palace_path, create=False) + except Exception as exc: + print(f" No drawer collection found at {palace_path} ({exc}).") + return False + + d_items = list(_iter_collection_items(drawers)) + all_wings = {(m or {}).get("wing") for _, m in d_items if (m or {}).get("wing")} + d_summary, d_updates = plan_wing_renames(d_items) + + closets = None + c_summary, c_updates = defaultdict(int), [] + try: + closets = get_closets_collection(palace_path, create=False) + c_summary, c_updates = plan_wing_renames(_iter_collection_items(closets)) + except Exception: + closets = None + + topic_renames = _plan_topics_by_wing_renames() + + if not d_updates and not c_updates and not topic_renames: + print(" All wing names are already normalized — nothing to migrate.") + return False + + print("\n Wing-name migration plan:") + merged = defaultdict(lambda: [0, 0]) + for key, count in d_summary.items(): + merged[key][0] = count + for key, count in c_summary.items(): + merged[key][1] = count + for (old, new), (d_count, c_count) in sorted(merged.items()): + note = " (MERGE into existing wing)" if new in all_wings else "" + print(f" {old!r} -> {new!r}: {d_count} drawer(s), {c_count} closet(s){note}") + if topic_renames: + print(f" topics_by_wing: {len(topic_renames)} key(s) re-keyed") + + if dry_run: + print("\n DRY RUN — no changes made.\n") + return True + + if not confirm: + try: + resp = input(" Apply this wing-name migration? [y/N] ").strip().lower() + except EOFError: + resp = "" + if resp not in ("y", "yes"): + print(" Aborted.") + return False + + _apply_wing_updates(drawers, d_updates) + if closets is not None and c_updates: + _apply_wing_updates(closets, c_updates) + _apply_topics_by_wing_renames(topic_renames) + + parts = [f"{len(d_updates)} drawer(s)"] + if c_updates: + parts.append(f"{len(c_updates)} closet(s)") + if topic_renames: + parts.append(f"{len(topic_renames)} topic key(s)") + print(f"\n Migrated {', '.join(parts)}.\n") + return True diff --git a/mempalace/miner.py b/mempalace/miner.py index 2937e66..aed1810 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -19,7 +19,7 @@ from datetime import datetime from collections import defaultdict from typing import Optional -from .entity_detector import _get_coca_filter +from .entity_detector import _apply_known_systems_prepass, _get_coca_filter from .palace import ( NORMALIZE_VERSION, SKIP_DIRS, @@ -41,7 +41,9 @@ from .palace import ( # ``mempalace.miner.compute_hallways_for_wing``. The integration call # lives at the end of _mine_impl, alongside the existing # ``_compute_topic_tunnels_for_wing`` post-mine block. +from .collision_scan import assert_no_collisions from .hallways import compute_hallways_for_wing +from .ids import ID_RECIPE, make_drawer_id_from_chunk logger = logging.getLogger("mempalace_mcp") @@ -871,8 +873,14 @@ def _extract_entities_for_metadata(content: str) -> str: coca_filter = _get_coca_filter() window = content[:_ENTITY_EXTRACT_WINDOW] - words = _candidate_entity_words(window) - freq: dict = {} + # Tier 3 linguistics cleanup — known-systems compound pre-pass. Detects + # multi-word product names atomically and masks them from the window so + # the single-word extraction below doesn't decompose them into their + # constituent tokens (which would then either get COCA-filtered or + # appear as wrongly-attributed standalone entities). + working_window, compound_counts = _apply_known_systems_prepass(window) + words = _candidate_entity_words(working_window) + freq: dict = dict(compound_counts) for w in words: if w in _ENTITY_STOPLIST: continue @@ -1219,6 +1227,7 @@ def _build_drawer_metadata( "added_by": agent, "filed_at": datetime.now().isoformat(), "normalize_version": NORMALIZE_VERSION, + "id_recipe": ID_RECIPE, } if source_mtime is not None: metadata["source_mtime"] = source_mtime @@ -1244,7 +1253,7 @@ def add_drawer( miner uses ``_build_drawer_metadata`` + a batched ``collection.upsert`` to amortize the embedding model's forward-pass cost across chunks. """ - drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(chunk_index)).encode()).hexdigest()[:24]}" + drawer_id = make_drawer_id_from_chunk(wing, room, source_file, chunk_index) try: source_mtime = os.path.getmtime(source_file) except OSError: @@ -1377,7 +1386,7 @@ def process_file( batch_ids: list = [] batch_metas: list = [] for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]: - drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}" + drawer_id = make_drawer_id_from_chunk(wing, room, source_file, chunk["chunk_index"]) batch_docs.append(chunk["content"]) batch_ids.append(drawer_id) batch_metas.append( @@ -1394,6 +1403,7 @@ def process_file( content_date=file_content_date, ) ) + assert_no_collisions(list(zip(batch_ids, batch_metas)), collection) collection.upsert( documents=batch_docs, ids=batch_ids, @@ -1407,8 +1417,7 @@ def process_file( # fully replace the prior closets, not append to them. if closets_col and drawers_added > 0: drawer_ids = [ - f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(c['chunk_index'])).encode()).hexdigest()[:24]}" - for c in chunks + make_drawer_id_from_chunk(wing, room, source_file, c["chunk_index"]) for c in chunks ] # Pass drawer_metas so build_closet_lines can emit the Tier 6a # 4-segment pointer (``topic|entities|YYYY-MM-DD:Lstart-Lend|→ids``) @@ -1614,9 +1623,6 @@ def _mine_impl( respect_gitignore=respect_gitignore, include_ignored=include_ignored, ) - if limit > 0: - files = files[:limit] - from .embedding import describe_device print(f"\n{'=' * 55}") @@ -1624,7 +1630,8 @@ def _mine_impl( print(f"{'=' * 55}") print(f" Wing: {wing}") print(f" Rooms: {', '.join(r['name'] for r in rooms)}") - print(f" Files: {len(files)}") + limit_suffix = f" (limit: {limit} new)" if limit > 0 else "" + print(f" Files: {len(files)}{limit_suffix}") print(f" Palace: {palace_path}") print(f" Device: {describe_device()}") if dry_run: @@ -1643,6 +1650,7 @@ def _mine_impl( closets_col = None total_drawers = 0 + files_mined = 0 files_skipped = 0 files_skipped_chunk_cap = 0 files_processed = 0 @@ -1690,8 +1698,11 @@ def _mine_impl( else: total_drawers += drawers room_counts[room] += 1 + files_mined += 1 if not dry_run: print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers}") + if limit > 0 and files_mined >= limit: + break if not dry_run: # Cross-wing topic tunnels: after every file in this wing has been @@ -1744,7 +1755,7 @@ def _mine_impl( print(f"\n{'=' * 55}") print(" Done.") - print(f" Files processed: {len(files) - files_skipped}") + print(f" Files processed: {files_processed - files_skipped}") # The residual skip bucket label depends on mode: dry-run bypasses # the already-mined check, so the only paths producing (0, room, # None) under dry_run are OSError / too-short / post-lock re-check @@ -1905,7 +1916,23 @@ def _compute_entity_tunnels_for_wing(wing: str) -> int: def status(palace_path: str): - """Show what's been filed in the palace.""" + """Show what's been filed in the palace. + + Tallies drawers by wing/room directly from ``chroma.sqlite3`` so a routine + status check never cold-loads the HNSW vector index — a load that costs + tens of seconds of CPU per call on large palaces (#1681). Falls back to the + ChromaDB client path when the sqlite read is unavailable (missing DB, + un-bootstrapped collection, or an unexpected schema); the fallback also + emits the state-specific guidance for absent/empty palaces. + """ + from .backends.chroma import _sqlite_wing_room_counts + + counts = _sqlite_wing_room_counts(palace_path, "mempalace_drawers") + if counts is not None: + total, wing_rooms = counts + _print_status(total, wing_rooms) + return + col = _open_collection_or_explain(palace_path) if col is None: return @@ -1926,6 +1953,11 @@ def status(palace_path: str): wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1 offset += len(batch) + _print_status(total, wing_rooms) + + +def _print_status(total: int, wing_rooms: dict[str, dict[str, int]]) -> None: + """Render the wing/room histogram shared by both status code paths.""" print(f"\n{'=' * 55}") print(f" MemPalace Status — {total} drawers") print(f"{'=' * 55}\n") diff --git a/mempalace/normalize.py b/mempalace/normalize.py index ca62cca..9f9fe0a 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -122,7 +122,7 @@ def normalize(filepath: str) -> str: if file_size > 500 * 1024 * 1024: # 500 MB safety limit raise IOError(f"File too large ({file_size // (1024 * 1024)} MB): {filepath}") try: - with open(filepath, "r", encoding="utf-8", errors="replace") as f: + with open(filepath, "r", encoding="utf-8-sig", errors="replace") as f: content = f.read() except OSError as e: raise IOError(f"Could not read {filepath}: {e}") from e @@ -523,6 +523,8 @@ def _format_tool_use(block: dict) -> str: """Format a tool_use block into a human-readable one-liner.""" name = block.get("name", "Unknown") inp = block.get("input", {}) + if isinstance(inp, list): + inp = {} if name == "Bash": cmd = inp.get("command", "") diff --git a/mempalace/palace.py b/mempalace/palace.py index bdab678..51603c9 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -13,9 +13,20 @@ import sys import threading from typing import Optional -from .backends import BackendClosedError, CollectionNotInitializedError, PalaceNotFoundError -from .backends.chroma import ChromaBackend -from .entity_detector import _get_coca_filter +from .backends import ( + BackendClosedError, + BackendMismatchError, + CollectionNotInitializedError, + PalaceNotFoundError, + PalaceRef, + detect_backend_for_path, + detect_backends_for_path, + get_backend, + get_backend_class, + resolve_backend_for_palace, +) +from .backends.embedding_wrapper import EmbeddingCollection +from .entity_detector import _apply_known_systems_prepass, _get_coca_filter logger = logging.getLogger("mempalace_mcp") @@ -45,7 +56,8 @@ SKIP_DIRS = { "target", } -_DEFAULT_BACKEND = ChromaBackend() +_DEFAULT_BACKEND = get_backend("chroma") +_EXPLICIT_BACKEND_ENV = "MEMPALACE_BACKEND_EXPLICIT" # Schema version for drawer normalization. Bump when the normalization # pipeline changes in a way that existing drawers should be rebuilt to pick up @@ -58,26 +70,283 @@ _DEFAULT_BACKEND = ChromaBackend() NORMALIZE_VERSION = 2 +# (palace_id, collection_name, model_name) tuples already validated this +# process, so the identity check (one metadata read) runs at most once per +# collection per run — keeps the hot get_collection path cheap. +_VALIDATED_IDENTITY: set = set() + + +def _enforce_embedder_identity(collection, palace_path, collection_name, *, create) -> None: + """Check (and, for a brand-new collection, record) embedder identity (RFC 001). + + Check at open so a model swap fails fast — before any query silently + returns degraded results. Record only when the collection is brand-new and + empty: recording the *current* model on a legacy palace that already holds + vectors from an unknown model would mislabel it, so populated-but-unrecorded + collections warn instead and are resolved with + ``mempalace palace set-embedder``. + + Bookkeeping must never break memory operations: only the deliberate + identity/dimension mismatch propagates; every other error is swallowed. + """ + import warnings + + from .backends.base import ( + DimensionMismatchError, + EmbedderIdentity, + EmbedderIdentityMismatchError, + EmbedderIdentityUnknownWarning, + check_embedder_identity, + ) + from .embedding import current_model_name + + # A server_embedder backend embeds with its own model and ignores the + # injected/core embedder, so its effective identity — not the configured + # model — is what must be checked and recorded. Fall back to the configured + # model name for the normal (core-embedder) case. + current: Optional[EmbedderIdentity] = None + try: + effective = collection.effective_embedder_identity() + except Exception: + effective = None + if effective is not None and getattr(effective, "model_name", ""): + current = effective + else: + try: + model_name = current_model_name() + except Exception: + return + if not model_name: + return # nameless embedder — cannot enforce identity + current = EmbedderIdentity(model_name=model_name, dimension=0) + + model_name = current.model_name + key = (str(palace_path), str(collection_name), model_name) + if key in _VALIDATED_IDENTITY: + return + + try: + stored = collection.get_stored_embedder_identity() + except Exception: + logger.debug("embedder-identity read failed for %s", collection_name, exc_info=True) + return + try: + state = check_embedder_identity(stored, current) + except (EmbedderIdentityMismatchError, DimensionMismatchError): + raise # deliberate, user-facing — the whole point of the contract + except Exception: + return + + if state == "unknown" and stored is None: + try: + count = collection.count() + except Exception: + count = None + if count == 0: + if create: + try: + collection.set_embedder_identity(current) + except Exception: + logger.debug("embedder-identity record failed", exc_info=True) + elif count: + warnings.warn( + f"palace collection {collection_name!r} has no recorded embedder " + f"identity; assuming the current model {model_name!r}. Run " + "`mempalace palace set-embedder --model ` to record it.", + EmbedderIdentityUnknownWarning, + stacklevel=2, + ) + + _VALIDATED_IDENTITY.add(key) + + def get_collection( palace_path: str, collection_name: Optional[str] = None, create: bool = True, + backend: Optional[str] = None, + _skip_identity_check: bool = False, ): - """Get the palace collection through the backend layer.""" + """Get the palace collection through the backend layer. + + ``_skip_identity_check`` bypasses the embedder-identity enforcement so the + ``set-embedder`` override path can open a palace whose recorded model + differs from the current one (the very state it exists to repair). + """ if collection_name is None: from .config import get_configured_collection_name collection_name = get_configured_collection_name() - return _DEFAULT_BACKEND.get_collection( + backend_obj = get_backend_for_palace(palace_path, explicit=backend) + palace_ref = PalaceRef(id=palace_path, local_path=palace_path) + try: + collection = backend_obj.get_collection( + palace=palace_ref, + collection_name=collection_name, + create=create, + ) + except TypeError as exc: + if "unexpected keyword argument 'palace'" not in str(exc): + raise + collection = backend_obj.get_collection( + palace_path, + collection_name=collection_name, + create=create, + ) + if "requires_explicit_embeddings" in getattr(backend_obj, "capabilities", frozenset()): + collection = EmbeddingCollection(collection) + if not _skip_identity_check: + _enforce_embedder_identity(collection, palace_path, collection_name, create=create) + return collection + + +def set_palace_embedder_identity( + palace_path: str, + model: Optional[str] = None, + *, + force: bool = False, + backend: Optional[str] = None, + collection_name: Optional[str] = None, +): + """Record (or force-override) a palace collection's embedder identity (RFC 001). + + Backs ``mempalace palace set-embedder``. Returns ``(old, new)`` identities. + Without ``force``, refuses to overwrite an existing identity that names a + different model (the user must confirm they know the vectors are + compatible). Opens with the identity check skipped so a mismatched palace — + the exact state being repaired — can be opened at all. + """ + from .backends.base import EmbedderIdentity, EmbedderIdentityMismatchError + from .config import MempalaceConfig + from .embedding import get_embedder_identity + + configured = MempalaceConfig().embedding_model + target = (model or configured or "").strip().lower() + if not target: + # No model given and none configured — there is nothing to record, and + # recording a nameless identity is a silent no-op in every backend. + raise ValueError( + "no embedder model to record: pass --model NAME or configure MEMPALACE_EMBEDDING_MODEL" + ) + if target == (configured or "").strip().lower(): + # Recording the in-use model — probe its dimension (already loaded). + new = get_embedder_identity() + else: + # Explicit override of a non-configured model: record the name only, + # never load a foreign model (which can be a large download) just to + # probe a dimension. The model-name check is the actual protection. + new = EmbedderIdentity(model_name=target, dimension=0) + collection = get_collection( palace_path, collection_name=collection_name, + create=True, + backend=backend, + _skip_identity_check=True, + ) + try: + old = collection.get_stored_embedder_identity() + except Exception: + old = None + if old is not None and old.model_name != new.model_name and not force: + raise EmbedderIdentityMismatchError( + f"palace already records embedder {old.model_name!r}; pass --force to " + f"overwrite it with {new.model_name!r} (only if the vectors are compatible)" + ) + collection.set_embedder_identity(new) + # Reset the per-process validation cache so a re-open re-checks against the + # newly recorded identity rather than a stale verdict. + _VALIDATED_IDENTITY.clear() + return old, new + + +def get_closets_collection( + palace_path: str, + create: bool = True, + backend: Optional[str] = None, +): + """Get the closets collection — the searchable index layer.""" + return get_collection( + palace_path, + collection_name="mempalace_closets", create=create, + backend=backend, ) -def get_closets_collection(palace_path: str, create: bool = True): - """Get the closets collection — the searchable index layer.""" - return get_collection(palace_path, collection_name="mempalace_closets", create=create) +def _config_backend_value(palace_path: str) -> Optional[str]: + try: + from .config import MempalaceConfig + + cfg = MempalaceConfig() + cfg_palace = os.path.abspath(os.path.expanduser(cfg.palace_path)) + target_palace = os.path.abspath(os.path.expanduser(palace_path)) + if cfg_palace != target_palace: + return None + value = cfg._file_config.get("backend") + return str(value).strip().lower() if value else None + except Exception: + return None + + +def _env_backend_value() -> Optional[str]: + value = os.environ.get("MEMPALACE_BACKEND") + return value.strip().lower() if value else None + + +def resolve_backend_name(palace_path: str, explicit: Optional[str] = None) -> str: + """Resolve and validate the selected backend for ``palace_path``. + + Public resolution order: + + 1. Explicit CLI/MCP flag or direct ``get_collection(..., backend=...)``. + 2. ``backend`` in ``~/.mempalace/config.json``. + 3. ``MEMPALACE_BACKEND``. + 4. Detected existing palace artifacts. + 5. ``chroma``. + + If artifacts for a different backend are already present, raise + ``BackendMismatchError`` so normal write paths cannot silently mix storage + formats in one palace directory. + """ + explicit = explicit or os.environ.get(_EXPLICIT_BACKEND_ENV) + selected = resolve_backend_for_palace( + explicit=explicit.strip().lower() if explicit else None, + config_value=_config_backend_value(palace_path), + env_value=_env_backend_value(), + palace_path=palace_path, + default="chroma", + ) + get_backend_class(selected) + detected_backends = detect_backends_for_path(palace_path) + if len(detected_backends) > 1: + raise BackendMismatchError( + f"palace at {palace_path!r} contains multiple backend artifacts: " + f"{', '.join(detected_backends)}" + ) + detected = detected_backends[0] if detected_backends else None + if detected and detected != selected: + raise BackendMismatchError( + f"palace at {palace_path!r} contains {detected!r} backend artifacts, " + f"but {selected!r} was selected" + ) + return selected + + +def get_backend_for_palace(palace_path: str, explicit: Optional[str] = None): + """Return the resolved backend instance for ``palace_path``.""" + return get_backend(resolve_backend_name(palace_path, explicit=explicit)) + + +def _backend_artifact_label(backend_name: Optional[str]) -> str: + if backend_name == "chroma": + return "chroma.sqlite3" + if backend_name == "qdrant": + return "qdrant_backend.json" + if backend_name == "pgvector": + return "pgvector_backend.json" + if backend_name == "sqlite_exact": + return "sqlite_exact.sqlite3" + return "backend database" def _open_collection_or_explain( @@ -85,6 +354,7 @@ def _open_collection_or_explain( *, collection_name: Optional[str] = None, out=None, + opener=None, ): """Open the palace collection or print a state-specific message and return ``None``. @@ -101,11 +371,11 @@ def _open_collection_or_explain( first when the vector path is disabled (see PR #831 / issue #830). State A: palace dir is absent. - State B: dir is present but ``chroma.sqlite3`` is absent. The helper - short-circuits to a message before reaching the backend, because - ``chromadb.PersistentClient`` lazily creates the DB file on first - open — calling the backend on this state would silently mutate - the filesystem for what should be a read-only inspection. + State B: dir is present but no backend database artifact is present. + The helper short-circuits to a message before reaching the backend, + because some backends lazily create their DB file on first open — + calling the backend on this state would silently mutate the filesystem + for what should be a read-only inspection. State C: DB is present but the ``mempalace_drawers`` collection has never been bootstrapped (``init`` ran, ``mine`` has not). State D: healthy — returns the opened collection. @@ -116,17 +386,41 @@ def _open_collection_or_explain( callable (e.g. a repair progress emitter) to route messages through it. """ emit = out if out is not None else print + open_collection = opener or get_collection if not os.path.isdir(palace_path): emit(f"\n No palace found at {palace_path}") emit(" Run: mempalace init then mempalace mine ") return None - if not os.path.isfile(os.path.join(palace_path, "chroma.sqlite3")): - emit(f"\n Palace dir at {palace_path} exists but has no chroma.sqlite3 yet.") + try: + backend_name = resolve_backend_name(palace_path) + except BackendMismatchError as e: + emit(f"\n Backend mismatch at {palace_path}: {e}") + emit(" Select the matching backend or use a fresh palace directory.") + return None + except KeyError as e: + # Unknown backend name (e.g. a typo in MEMPALACE_BACKEND/--backend): + # resolve_backend_name -> get_backend_class raises KeyError carrying the + # available-backend list. Surface it as a CLI state message rather than + # letting it escape as a stack trace. + emit(f"\n Unknown backend selected for {palace_path}: {e.args[0] if e.args else e}") + emit(" Set --backend or MEMPALACE_BACKEND to a registered backend.") + return None + detected = detect_backend_for_path(palace_path) + if detected is None: + emit( + f"\n Palace dir at {palace_path} exists but has no " + f"{_backend_artifact_label(backend_name)} yet." + ) emit(" Run: mempalace mine ") return None try: - return get_collection(palace_path, collection_name=collection_name, create=False) + return open_collection( + palace_path, + collection_name=collection_name, + create=False, + backend=backend_name, + ) except CollectionNotInitializedError: emit(f"\n Palace at {palace_path} is initialized but empty (no drawers yet).") emit(" Run: mempalace mine ") @@ -135,6 +429,10 @@ def _open_collection_or_explain( emit(f"\n No palace found at {palace_path}") emit(" Run: mempalace init then mempalace mine ") return None + except BackendMismatchError as e: + emit(f"\n Backend mismatch at {palace_path}: {e}") + emit(" Select the matching backend or use a fresh palace directory.") + return None except BackendClosedError: # Surface this as a programmer error, not a palace-state UX message: # a closed backend means the caller violated the backend lifecycle, @@ -265,9 +563,15 @@ def build_closet_lines(source_file, drawer_ids, content, wing, room, drawer_meta # Extract proper nouns (2+ occurrences). Uses i18n-aware patterns so # non-Latin names (Cyrillic, accented Latin, etc.) are also detected. + # Tier 3 linguistics cleanup — known-systems compound pre-pass. Detects + # multi-word product names ("Claude Code", "GitHub Copilot", …) atomically + # and masks them out of the working window so the single-word extraction + # below doesn't decompose them. + working_window, compound_counts = _apply_known_systems_prepass(window) + coca_filter = _get_coca_filter() - words = _candidate_entity_words(window) - word_freq = {} + words = _candidate_entity_words(working_window) + word_freq: dict = dict(compound_counts) for w in words: if w in _ENTITY_STOPLIST: continue @@ -478,6 +782,9 @@ def _validate_palace_fts5_after_mine(palace_path: str) -> None: operator sees the same recovery banner regardless of which command surfaces the bug. """ + if resolve_backend_name(palace_path) != "chroma": + return + # Defer-import: keeps the repair module graph out of mine's hot import path. from .repair import _close_chroma_handles, sqlite_integrity_errors @@ -735,47 +1042,51 @@ def file_already_mined( treated as exchange-mode drawers. """ try: - stored_meta = None - if extract_mode is None: - results = collection.get(where={"source_file": source_file}, limit=1) - if not results.get("ids"): - return False - stored_meta = results.get("metadatas", [{}])[0] or {} - else: - offset = 0 - while True: - results = collection.get( - where={"source_file": source_file}, - limit=1000, - offset=offset, - include=["metadatas"], - ) - ids = results.get("ids") or [] - metadatas = results.get("metadatas") or [] - stored_meta = next( - ( - meta or {} - for meta in metadatas - if _metadata_matches_extract_mode(meta or {}, extract_mode) - ), - None, - ) - if stored_meta is not None or not ids: - break - offset += len(ids) - if stored_meta is None: - return False - # Pre-v2 drawers have no version field — treat them as stale. - stored_version = stored_meta.get("normalize_version", 1) - if stored_version < NORMALIZE_VERSION: - return False - if check_mtime: - stored_mtime = stored_meta.get("source_mtime") - if stored_mtime is None: - return False - current_mtime = os.path.getmtime(source_file) - return abs(float(stored_mtime) - current_mtime) < 0.001 - return True + # Under the additive-mining model, a single ``source_file`` can have + # multiple ``parent_drawer_id`` groups in the palace — one per + # mining pass — each with its own stored ``source_mtime`` and + # ``normalize_version``. The function must return True if ANY stored + # group is current (matching version + matching mtime when checked), + # because ChromaDB's ``get(..., limit=1)`` has undefined ordering + # across multiple matching rows: a ``limit=1`` shortcut picks + # whichever row ChromaDB orders first and only checks that one, + # causing spurious re-mines whenever the stale group is returned. + # Iterating via the same paginated pattern used in the + # extract_mode-is-set branch lets the function short-circuit on the + # first matching group regardless of ordering. + current_mtime = os.path.getmtime(source_file) if check_mtime else None + offset = 0 + while True: + results = collection.get( + where={"source_file": source_file}, + limit=1000, + offset=offset, + include=["metadatas"], + ) + ids = results.get("ids") or [] + metadatas = results.get("metadatas") or [] + for meta in metadatas: + meta = meta or {} + # extract_mode scoping (was the existing ``else`` branch): + if extract_mode is not None and not _metadata_matches_extract_mode( + meta, extract_mode + ): + continue + # Pre-v2 drawers have no version field — treat them as stale. + stored_version = meta.get("normalize_version", 1) + if stored_version < NORMALIZE_VERSION: + continue + if not check_mtime: + return True + stored_mtime = meta.get("source_mtime") + if stored_mtime is None: + continue + if abs(float(stored_mtime) - current_mtime) < 0.001: + return True + if not ids: + break + offset += len(ids) + return False except Exception: return False diff --git a/mempalace/repair.py b/mempalace/repair.py index 7a4a28c..5a060b3 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -1529,12 +1529,26 @@ def repair_max_seq_id( return result if backup: + import glob + + from .backups import prune_backups + from .config import MempalaceConfig + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") backup_path = os.path.join(palace_path, f"chroma.sqlite3.max-seq-id-backup-{timestamp}") shutil.copy2(db_path, backup_path) result["backup"] = backup_path print(f" Backup: {backup_path}") + # Retain only the most recent N backups (the copy just written is the + # newest and is kept). Without this, every max-seq-id repair leaves a + # full chroma.sqlite3 copy behind that is never cleaned up. + prune_backups( + os.path.join(glob.escape(palace_path), "chroma.sqlite3.max-seq-id-backup-*"), + MempalaceConfig().max_backups, + log=print, + ) + _close_chroma_handles(palace_path) with sqlite3.connect(db_path) as conn: diff --git a/mempalace/searcher.py b/mempalace/searcher.py index db14c19..46402e7 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -16,8 +16,19 @@ import re import sqlite3 from pathlib import Path -from .backends import CollectionNotInitializedError, PalaceNotFoundError -from .palace import get_closets_collection, get_collection +from .backends import ( + BackendError, + BackendMismatchError, + CollectionNotInitializedError, + PalaceNotFoundError, + UnsupportedCapabilityError, +) +from .palace import ( + _open_collection_or_explain, + get_closets_collection, + get_collection, + resolve_backend_name, +) # Closet pointer line format: "topic|entities|→drawer_id_a,drawer_id_b" # Multiple lines may join with newlines inside one closet document. @@ -119,18 +130,70 @@ def _bm25_scores( return scores +def _distance_to_similarity(distance, metric: str = "cosine") -> float: + """Map a backend-reported ``distance`` to a [0, 1]-ish similarity. + + The backend contract for the ``distances`` field is *lower = closer* + regardless of metric (RFC 001, backend metric declaration), so every + mapping here is monotonic decreasing in ``distance``. The output stays + bounded so it is + commensurable with the min-max-normalized BM25 term in + :func:`_hybrid_rank`. + + * ``cosine`` — distance ∈ [0, 2], 0 = identical: ``max(0, 1 - d)``. + * ``l2`` — Euclidean ∈ [0, ∞): ``1 / (1 + d)`` (1 at d=0, →0 as d→∞). + * ``ip`` — inner-product distance (e.g. pgvector ``<#>`` = -dot, lower = + closer), unbounded and signed: logistic squash ``1 / (1 + e^d)``. + Provisional until a real ip backend exercises it; no in-tree backend + uses ip today. + + ``distance is None`` (vector-unknown, e.g. a BM25-only candidate) maps to + 0.0 so the candidate scores on its BM25 contribution alone. + """ + if distance is None: + return 0.0 + m = (metric or "cosine").lower() + if m == "l2": + return 1.0 / (1.0 + max(0.0, distance)) + if m == "ip": + # Clamp the exponent so a large positive distance can't overflow. + return 1.0 / (1.0 + math.exp(min(60.0, distance))) + # cosine (default) + return max(0.0, 1.0 - distance) + + +def _metric_for_collection(col) -> str: + """Resolve a collection's declared distance metric, defaulting to cosine. + + Reads the ``distance_metric`` exposed by the backend collection (the + RFC 001 backend metric declaration). ``EmbeddingCollection`` delegates the + attribute to its inner collection; legacy Chroma palaces report their + actual ``hnsw:space``. + Any failure falls back to ``"cosine"`` — the value all in-tree backends + use and the only metric MemPalace created palaces with historically. + """ + try: + metric = getattr(col, "distance_metric", "cosine") + except Exception: + return "cosine" + metric = str(metric or "cosine").lower() + return metric if metric in ("cosine", "l2", "ip") else "cosine" + + def _hybrid_rank( results: list, query: str, vector_weight: float = 0.6, bm25_weight: float = 0.4, + metric: str = "cosine", ) -> list: """Re-rank ``results`` by a convex combination of vector similarity and BM25. - * Vector similarity uses absolute cosine sim ``max(0, 1 - distance)`` — - ChromaDB's hnsw cosine distance lives in ``[0, 2]`` (0 = identical). - Absolute (not relative-to-max) means adding/removing a candidate - can't reshuffle the others. + * Vector similarity is derived from each candidate's backend-reported + ``distance`` via :func:`_distance_to_similarity`, interpreted in the + collection's declared ``metric`` (per RFC 001) rather than assuming + cosine. Absolute (not relative-to-max) means adding/removing a + candidate can't reshuffle the others. * BM25 is real Okapi-BM25 with corpus-relative IDF over the candidates themselves. Since the absolute scale is unbounded, BM25 is min-max normalized within the candidate set so weights are commensurable. @@ -153,11 +216,7 @@ def _hybrid_rank( scored = [] for r, raw, norm in zip(results, bm25_raw, bm25_norm): - distance = r.get("distance") - if distance is None: - vec_sim = 0.0 - else: - vec_sim = max(0.0, 1.0 - distance) + vec_sim = _distance_to_similarity(r.get("distance"), metric) r["bm25_score"] = round(raw, 3) scored.append((vector_weight * vec_sim + bm25_weight * norm, r)) @@ -296,32 +355,11 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r Search the palace. Returns verbatim drawer content. Optionally filter by wing (project) or room (aspect). """ - # Filesystem-first checks distinguish State A / State B before reaching - # chromadb. PersistentClient lazily creates chroma.sqlite3 on first open - # of an empty palace dir, so without these checks State B collapses into - # the "initialized but empty" State C message and mutates the dir as a - # side effect of a read-only search call (#1498). - if not os.path.isdir(palace_path): - print(f"\n No palace found at {palace_path}") - print(" Run: mempalace init then mempalace mine ") - raise SearchError(f"No palace found at {palace_path}") - if not os.path.isfile(os.path.join(palace_path, "chroma.sqlite3")): - print(f"\n Palace dir at {palace_path} exists but has no chroma.sqlite3 yet.") - print(" Run: mempalace mine ") + col = _open_collection_or_explain(palace_path, opener=get_collection) + if col is None: + if not os.path.isdir(palace_path): + raise SearchError(f"No palace found at {palace_path}") raise SearchError(f"No palace database at {palace_path}") - try: - col = get_collection(palace_path, create=False) - except CollectionNotInitializedError as e: - # State C from #1498: palace initialized but never mined. - print(f"\n Palace at {palace_path} is initialized but empty (no drawers yet).") - print(" Run: mempalace mine ") - raise SearchError(f"Palace at {palace_path} is initialized but empty") from e - except PalaceNotFoundError as e: - # Backend filesystem-race fallback: dir was deleted between our - # check above and the backend call. Same message as State A. - print(f"\n No palace found at {palace_path}") - print(" Run: mempalace init then mempalace mine ") - raise SearchError(f"No palace found at {palace_path}") from e # Alert the user if this palace predates hnsw:space=cosine being set on # creation — their similarity scores will be junk until they run repair. @@ -360,11 +398,12 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r # The MCP tool path already hybridizes BM25 with vector sim via # `_hybrid_rank`; do the same here so CLI results match what agents # see via `mempalace_search`. + metric = _metric_for_collection(col) hits = [ {"text": doc or "", "distance": float(dist), "metadata": meta or {}} for doc, meta, dist in zip(docs, metas, dists) ] - hits = _hybrid_rank(hits, query) + hits = _hybrid_rank(hits, query, metric=metric) print(f"\n{'=' * 60}") print(f' Results for: "{query}"') @@ -375,7 +414,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r print(f"{'=' * 60}\n") for i, hit in enumerate(hits, 1): - vec_sim = round(max(0.0, 1 - hit["distance"]), 3) + vec_sim = round(_distance_to_similarity(hit["distance"], metric), 3) bm25 = hit.get("bm25_score", 0.0) meta = hit["metadata"] source = Path(meta.get("source_file", "?")).name @@ -384,7 +423,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r print(f" [{i}] {wing_name} / {room_name}") print(f" Source: {source}") - print(f" Match: cosine={vec_sim} bm25={bm25}") + print(f" Match: {metric}_sim={vec_sim} bm25={bm25}") print() # Print the verbatim text, indented for line in hit["text"].strip().split("\n"): @@ -636,14 +675,14 @@ def _bm25_only_via_sqlite( def _merge_bm25_union_candidates( hits: list, + drawers_col, query: str, - palace_path: str, wing: str, room: str, n_results: int, max_distance: float = 0.0, ) -> None: - """Append top-K BM25-only candidates from sqlite into ``hits`` in place. + """Append top-K backend lexical candidates into ``hits`` in place. Used by ``search_memories(..., candidate_strategy="union")`` to widen the rerank pool's *source* (not just its size) — vector-only candidate @@ -668,19 +707,41 @@ def _merge_bm25_union_candidates( if max_distance > 0.0: return + where = build_where_filter(wing, room) try: - bm25_extra = _bm25_only_via_sqlite( - query, - palace_path, - wing=wing, - room=room, + lexical = drawers_col.lexical_search( + query=query, n_results=n_results * 3, - _include_internal=True, - ).get("results", []) + where=where or None, + ) + except UnsupportedCapabilityError: + raise except Exception: - logger.debug("candidate_strategy=union: BM25 fetch failed", exc_info=True) + logger.debug("candidate_strategy=union: lexical fetch failed", exc_info=True) return + bm25_extra = [] + for hit in lexical.hits: + meta = hit.metadata or {} + full_source = meta.get("source_file", "") or "" + bm25_extra.append( + { + "text": hit.document or "", + "wing": meta.get("wing", "unknown"), + "room": meta.get("room", "unknown"), + "source_file": Path(full_source).name if full_source else "?", + "created_at": meta.get("filed_at", "unknown"), + "similarity": None, + "distance": None, + "effective_distance": None, + "closet_boost": 0.0, + "matched_via": "bm25_backend", + "bm25_score": round(float(hit.score), 3), + "_source_file_full": full_source, + "_chunk_index": meta.get("chunk_index"), + } + ) + def _dedup_key(entry: dict): full = entry.get("_source_file_full") ci = entry.get("_chunk_index") @@ -728,8 +789,8 @@ def _validate_candidate_strategy(strategy: str) -> None: def _apply_candidate_strategy( strategy: str, hits: list, + drawers_col, query: str, - palace_path: str, wing: str, room: str, n_results: int, @@ -742,7 +803,163 @@ def _apply_candidate_strategy( """ merger = _CANDIDATE_MERGERS[strategy] if merger is not None: - merger(hits, query, palace_path, wing, room, n_results, max_distance=max_distance) + merger(hits, drawers_col, query, wing, room, n_results, max_distance=max_distance) + + +def _finalize_candidate_hits( + *, + candidate_strategy: str, + hits: list, + drawers_col, + query: str, + wing: str, + room: str, + n_results: int, + max_distance: float, +) -> tuple: + try: + _apply_candidate_strategy( + candidate_strategy, + hits, + drawers_col, + query, + wing, + room, + n_results, + max_distance=max_distance, + ) + except UnsupportedCapabilityError: + return [], { + "error": "candidate_strategy='union' requires a backend with lexical_search support", + "unsupported_capability": "supports_lexical_search", + "hint": "Use candidate_strategy='vector' or select a backend that supports lexical search.", + } + + hits = _hybrid_rank(hits, query, metric=_metric_for_collection(drawers_col))[:n_results] + for h in hits: + h.pop("_sort_key", None) + h.pop("_source_file_full", None) + h.pop("_chunk_index", None) + return hits, None + + +def _backend_mismatch_result(error: BackendMismatchError) -> dict: + return { + "error": "Backend mismatch", + "details": str(error), + "hint": "Select the matching backend or use a fresh palace directory.", + } + + +def _unknown_backend_result(error: KeyError) -> dict: + return { + "error": "Unknown backend", + "details": str(error), + "hint": "Check MEMPALACE_BACKEND or the configured backend name.", + } + + +def _vector_disabled_search( + *, + query: str, + palace_path: str, + wing: str, + room: str, + n_results: int, + collection_name: str, +) -> dict: + try: + backend_name = resolve_backend_name(palace_path) + except BackendMismatchError as e: + return _backend_mismatch_result(e) + except KeyError as e: + return _unknown_backend_result(e) + if backend_name != "chroma": + return { + "error": "vector_disabled fallback is Chroma-only", + "unsupported_capability": "chroma_hnsw_fallback", + "backend": backend_name, + "hint": "Disable vector_disabled for non-Chroma backends.", + } + return _bm25_only_via_sqlite( + query, + palace_path, + wing=wing, + room=room, + n_results=n_results, + collection_name=collection_name, + ) + + +def _open_search_collection(palace_path: str, collection_name: str): + try: + return get_collection(palace_path, collection_name=collection_name, create=False), None + except BackendMismatchError as e: + return None, _backend_mismatch_result(e) + except KeyError as e: + return None, _unknown_backend_result(e) + except (CollectionNotInitializedError, PalaceNotFoundError) as e: + logger.error("No palace found at %s: %s", palace_path, e) + return None, { + "error": "No palace found", + "hint": "Run: mempalace init && mempalace mine ", + } + except BackendError as e: + logger.error("Backend error opening palace at %s: %s", palace_path, e) + return None, { + "error": "Backend error", + "details": str(e), + "hint": "Check the selected backend configuration and availability.", + } + except Exception as e: + logger.error("No palace found at %s: %s", palace_path, e) + return None, { + "error": "No palace found", + "hint": "Run: mempalace init && mempalace mine ", + } + + +def _query_drawers_with_filter_fallback(drawers_col, dkwargs, query, n_results, wing, room): + """Run the filtered drawer query, falling back to an unfiltered query plus a + Python-side post-filter when ChromaDB raises on the filtered query. + + A ChromaDB HNSW/SQLite index mismatch makes filtered queries fail with + "Error finding id" even when unfiltered search works fine — it happens when + drawers are ingested via two different paths (e.g. bulk import vs MCP tool + calls), leaving the vector index inconsistent with the metadata store. We + retry unfiltered (over-fetching) and re-apply the wing/room filter in Python. + See #1245 / #1035. + """ + where = dkwargs.get("where") + try: + return drawers_col.query(**dkwargs) + except Exception as filter_err: + if not where: + raise + logger.warning( + "Filtered search failed (%s); falling back to unfiltered + post-filter", + filter_err, + ) + raw = drawers_col.query( + query_texts=[query], + n_results=min(n_results * 15, 500), + include=["documents", "metadatas", "distances"], + ) + fdocs, fmetas, fdists = [], [], [] + for doc, meta, dist in zip( + _first_or_empty(raw, "documents"), + _first_or_empty(raw, "metadatas"), + _first_or_empty(raw, "distances"), + ): + meta = meta or {} + if wing and meta.get("wing") != wing: + continue + if room and meta.get("room") != room: + continue + fdocs.append(doc) + fmetas.append(meta) + fdists.append(dist) + return {"documents": [fdocs], "metadatas": [fmetas], "distances": [fdists]} def search_memories( @@ -780,14 +997,12 @@ def search_memories( ``n_results * 3`` rows from the vector index are the rerank pool. Cheap; works well when query and target docs agree in the embedding space. - * ``"union"`` — also pull top ``n_results * 3`` BM25 candidates - from the sqlite FTS5 index and merge them into the rerank pool - (deduped by source_file). Catches docs with strong BM25 signal - that are vector-distant from the query (e.g. terminology guides - looked up by narrative-shaped queries; policy clauses surfaced - by scenario descriptions). Adds one sqlite open + FTS5 MATCH - per query; perf cost is small but unmeasured at corpus scale. - Opt in until the cost is characterized. + * ``"union"`` — also pull top ``n_results * 3`` lexical candidates + through the backend's ``lexical_search`` capability and merge + them into the rerank pool (deduped by source_file). Catches docs + with strong BM25 signal that are vector-distant from the query. + Perf depends on the selected backend; opt in until the cost is + characterized. When ``max_distance > 0.0`` is also set, BM25-only candidates are skipped — they have no vector distance and would silently @@ -799,24 +1014,20 @@ def search_memories( _validate_candidate_strategy(candidate_strategy) if vector_disabled: - return _bm25_only_via_sqlite( - query, - palace_path, + return _vector_disabled_search( + query=query, + palace_path=palace_path, wing=wing, room=room, n_results=n_results, collection_name=collection_name, ) - try: - drawers_col = get_collection(palace_path, collection_name=collection_name, create=False) - except Exception as e: - logger.error("No palace found at %s: %s", palace_path, e) - return { - "error": "No palace found", - "hint": "Run: mempalace init && mempalace mine ", - } + drawers_col, open_error = _open_search_collection(palace_path, collection_name) + if open_error: + return open_error + metric = _metric_for_collection(drawers_col) where = build_where_filter(wing, room) # Hybrid retrieval: always query drawers directly (the floor), then use @@ -834,7 +1045,9 @@ def search_memories( } if where: dkwargs["where"] = where - drawer_results = drawers_col.query(**dkwargs) + drawer_results = _query_drawers_with_filter_fallback( + drawers_col, dkwargs, query, n_results, wing, room + ) except Exception as e: return {"error": f"Search error: {e}"} @@ -907,7 +1120,7 @@ def search_memories( "room": meta.get("room", "unknown"), "source_file": Path(source).name if source else "?", "created_at": meta.get("filed_at", "unknown"), - "similarity": round(max(0.0, 1 - effective_dist), 3), + "similarity": round(_distance_to_similarity(effective_dist, metric), 3), "distance": round(dist, 4), "effective_distance": round(effective_dist, 4), "closet_boost": round(boost, 3), @@ -985,31 +1198,24 @@ def search_memories( # Candidate strategy hook: optionally widen the rerank pool's *source* # before ranking. Default ("vector") is a no-op; "union" merges top-K - # BM25 candidates from sqlite. See `_apply_candidate_strategy`. + # backend lexical candidates. See `_apply_candidate_strategy`. # ``max_distance`` is forwarded so union mode can refuse to inject # BM25-only (distance=None) candidates that would silently bypass the # caller's strict distance threshold. - _apply_candidate_strategy( - candidate_strategy, - hits, - query, - palace_path, - wing, - room, - n_results, + # The helper also runs the final BM25 hybrid re-rank and strips internal + # dedup fields before returning. + hits, strategy_error = _finalize_candidate_hits( + candidate_strategy=candidate_strategy, + hits=hits, + drawers_col=drawers_col, + query=query, + wing=wing, + room=room, + n_results=n_results, max_distance=max_distance, ) - - # BM25 hybrid re-rank within the final candidate set, then trim back - # to the requested size. Without the trim, ``candidate_strategy="union"`` - # would return up to 4× ``n_results`` (vector hits + BM25 union pool), - # breaking the existing ``search_memories`` size contract that the MCP - # ``limit`` parameter is built on. - hits = _hybrid_rank(hits, query)[:n_results] - for h in hits: - h.pop("_sort_key", None) - h.pop("_source_file_full", None) - h.pop("_chunk_index", None) + if strategy_error: + return strategy_error return { "query": query, diff --git a/mempalace/version.py b/mempalace/version.py index 8ea91e9..e5b9b45 100644 --- a/mempalace/version.py +++ b/mempalace/version.py @@ -1,3 +1,3 @@ """Single source of truth for the MemPalace package version.""" -__version__ = "3.3.6" +__version__ = "3.4.0" diff --git a/pyproject.toml b/pyproject.toml index fdf326b..29fac0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mempalace" -version = "3.3.6" +version = "3.4.0" description = "Give your AI a memory — mine projects and conversations into a searchable palace. No API key required." readme = "README.md" requires-python = ">=3.9" @@ -60,6 +60,9 @@ mempalace-mcp = "mempalace.mcp_server:main" [project.entry-points."mempalace.backends"] chroma = "mempalace.backends.chroma:ChromaBackend" +pgvector = "mempalace.backends.pgvector:PgVectorBackend" +qdrant = "mempalace.backends.qdrant:QdrantBackend" +sqlite_exact = "mempalace.backends.sqlite_exact:SQLiteExactBackend" # RFC 002 source-adapter entry-point group. Core publishes no first-party # adapters under this group yet; ``miner.py`` and ``convo_miner.py`` migrate @@ -71,7 +74,7 @@ chroma = "mempalace.backends.chroma:ChromaBackend" dev = [ "pytest>=7.0", "pytest-cov>=4.0", - "ruff==0.15.14", + "ruff==0.15.15", "psutil>=5.9", # Property-based testing — generates hundreds of random inputs per # test to find counterexamples the hand-written positive tests miss. @@ -92,6 +95,11 @@ dev = [ "mypy>=1.0", ] spellcheck = ["autocorrect>=2.0"] +# Opt-in Postgres + pgvector backend. Only the psycopg driver is needed on the +# client; the server must have the `vector` extension available. Selected via +# MEMPALACE_BACKEND=pgvector / --backend pgvector; never required for the +# default (Chroma) install. +pgvector = ["psycopg[binary]>=3.1"] # Hardware acceleration for the ONNX embedding model. Install exactly one: # pip install mempalace[gpu] — NVIDIA CUDA # pip install mempalace[dml] — DirectML (Windows AMD/Intel/NVIDIA) @@ -123,7 +131,7 @@ extract = [ dev = [ "pytest>=7.0", "pytest-cov>=4.0", - "ruff==0.15.14", + "ruff==0.15.15", "psutil>=5.9", "hypothesis>=6.0", "pre-commit>=3.0", @@ -161,6 +169,15 @@ markers = [ "slow: tests that take more than 30 seconds", "stress: destructive scale tests (100K+ drawers)", ] +filterwarnings = [ + # Many tests build raw-chromadb palaces directly (no recorded embedder + # identity), which correctly emits this on open. The behavior itself is + # asserted in tests/test_embedder_identity.py; ignore the fixture noise. + # Matched by message (not by class path) so pytest does not import the + # mempalace package at config time — importing it before coverage starts + # would drop module-level lines from the report. + "ignore:palace collection.*has no recorded embedder identity", +] [tool.coverage.run] source = ["mempalace"] diff --git a/tests/_backend_conformance.py b/tests/_backend_conformance.py new file mode 100644 index 0000000..9f3a3ac --- /dev/null +++ b/tests/_backend_conformance.py @@ -0,0 +1,61 @@ +"""Shared backend isolation conformance assertions (RFC 001 isolation contract). + +Any backend's test module can import these to prove the isolation guarantees +declared on :class:`mempalace.backends.PalaceRef`: + +* per-``PalaceRef.id`` isolation — required of every backend; +* per-``PalaceRef.namespace`` isolation — required of backends advertising the + ``supports_namespace_isolation`` capability. + +This module is intentionally not a ``test_*`` file: it ships assertions, not +test cases, so pytest does not collect it directly. +""" + +_PROBE_ID = "conformance-isolation-probe" +_PROBE_DOC = "partition isolation probe document" +_PROBE_EMBEDDING = [1.0, 0.0, 0.0, 0.0] + + +def assert_partition_isolation(backend, writer, other, *, embedding=None): + """Assert ``writer`` and ``other`` are isolated partitions of ``backend``. + + A record written to ``writer`` MUST NOT be returned, modified, or deleted + through ``other`` (query / get / count / delete), and ``writer`` MUST still + hold it afterwards. ``writer`` and ``other`` are two collections that the + isolation contract says must not see each other — distinct palace ids (the + universal guarantee) or distinct namespaces (the namespace guarantee). + + Embeddings are supplied only for backends that require explicit vectors, so + the same assertion works for text-embedding backends (Chroma) and + explicit-vector backends (qdrant, sqlite_exact) alike. + """ + explicit = "requires_explicit_embeddings" in backend.capabilities + vector = list(embedding if embedding is not None else _PROBE_EMBEDDING) + + baseline_other = other.count() + + add_kwargs = { + "ids": [_PROBE_ID], + "documents": [_PROBE_DOC], + "metadatas": [{"wing": "conformance"}], + } + if explicit: + add_kwargs["embeddings"] = [vector] + writer.add(**add_kwargs) + + if explicit: + leaked = other.query(query_embeddings=[vector], n_results=10) + else: + leaked = other.query(query_texts=[_PROBE_DOC], n_results=10) + hit_ids = leaked.ids[0] if leaked.ids else [] + assert _PROBE_ID not in hit_ids, "query() leaked a record across the isolation boundary" + + assert other.get(ids=[_PROBE_ID]).ids == [], ( + "get() leaked a record across the isolation boundary" + ) + assert other.count() == baseline_other, "count() leaked a record across the isolation boundary" + + # A delete issued against the other partition MUST NOT touch writer's record. + other.delete(ids=[_PROBE_ID]) + survivor = writer.get(ids=[_PROBE_ID]) + assert survivor.ids == [_PROBE_ID], "delete() crossed the isolation boundary" diff --git a/tests/conftest.py b/tests/conftest.py index bd65363..3c18ce7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,6 +79,12 @@ def _reset_mcp_cache(): mcp_server._client_cache = None mcp_server._collection_cache = None + if hasattr(mcp_server, "_collection_cache_backend"): + mcp_server._collection_cache_backend = None + if hasattr(mcp_server, "_collection_cache_palace"): + mcp_server._collection_cache_palace = None + if hasattr(mcp_server, "_collection_open_error"): + mcp_server._collection_open_error = None except AttributeError: pass diff --git a/tests/test_backend_conformance.py b/tests/test_backend_conformance.py new file mode 100644 index 0000000..a62e0ba --- /dev/null +++ b/tests/test_backend_conformance.py @@ -0,0 +1,43 @@ +"""Backend isolation conformance suite (RFC 001 isolation contract). + +Runs the shared isolation assertions from ``_backend_conformance`` against the +built-in local backends. Server-mode backends (qdrant) run the same assertions +under their own fake/live client in ``test_qdrant_backend.py``. +""" + +import pytest + +from _backend_conformance import assert_partition_isolation + +from mempalace.backends import PalaceRef +from mempalace.backends.chroma import ChromaBackend +from mempalace.backends.sqlite_exact import SQLiteExactBackend + +_LOCAL_BACKENDS = [ + pytest.param(ChromaBackend, id="chroma"), + pytest.param(SQLiteExactBackend, id="sqlite_exact"), +] + + +@pytest.mark.parametrize("backend_cls", _LOCAL_BACKENDS) +def test_cross_palace_isolation(backend_cls, tmp_path): + """One backend instance must isolate two distinct palaces (PalaceRef.id).""" + backend = backend_cls() + try: + cols = [] + for label in ("alpha", "beta"): + path = tmp_path / label + ref = PalaceRef(id=str(path), local_path=str(path)) + cols.append( + backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True) + ) + assert_partition_isolation(backend, cols[0], cols[1]) + finally: + backend.close() + + +def test_local_backends_do_not_claim_namespace_isolation(): + """Local backends isolate by on-disk path, not namespace; they must not + advertise the namespace-isolation capability (RFC 001 isolation contract).""" + assert "supports_namespace_isolation" not in ChromaBackend.capabilities + assert "supports_namespace_isolation" not in SQLiteExactBackend.capabilities diff --git a/tests/test_backends.py b/tests/test_backends.py index e0c3b2b..8af4f2b 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -23,12 +23,16 @@ from mempalace.backends.chroma import ( ChromaCollection, _HNSW_MISSING_METADATA_DATA_FLOOR, _fix_blob_seq_ids, + _fix_missing_collection_type, _pin_hnsw_threads, _segment_appears_healthy, quarantine_invalid_hnsw_metadata, quarantine_stale_hnsw, ) +# embeddinggemma-300m Matryoshka truncation (first 384 of 768 dims). +_TEST_EMBED_DIM = 384 + class _FakeCollection: """Stand-in for a chromadb.Collection returning raw chroma-shaped dicts.""" @@ -180,6 +184,103 @@ def test_chroma_detect_matches_palace_with_chroma_sqlite(tmp_path): assert ChromaBackend.detect(str(tmp_path.parent)) is False +def test_chroma_lexical_search_uses_sqlite_fts_not_full_collection_scan(tmp_path): + db_path = tmp_path / "chroma.sqlite3" + conn = sqlite3.connect(db_path) + conn.executescript( + """ + CREATE TABLE collections (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE segments (id INTEGER PRIMARY KEY, collection INTEGER NOT NULL); + CREATE TABLE embeddings ( + id INTEGER PRIMARY KEY, + segment_id INTEGER NOT NULL, + embedding_id TEXT, + created_at TEXT + ); + CREATE TABLE embedding_metadata ( + id INTEGER, + key TEXT, + string_value TEXT, + int_value INTEGER, + float_value REAL, + bool_value INTEGER + ); + CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value); + """ + ) + conn.execute("INSERT INTO collections(id, name) VALUES (1, 'mempalace_drawers')") + conn.execute("INSERT INTO segments(id, collection) VALUES (1, 1)") + ids = list(range(1, 14)) + for emb_id in ids: + wing = "target" if emb_id == 13 else "old" + doc = "needle shared lexical note" + conn.execute( + "INSERT INTO embeddings(id, segment_id, embedding_id, created_at) VALUES (?, 1, ?, ?)", + (emb_id, f"public-{emb_id}", f"2026-01-01T00:00:{emb_id:02d}"), + ) + conn.execute( + "INSERT INTO embedding_fulltext_search(rowid, string_value) VALUES (?, ?)", + (emb_id, doc), + ) + conn.execute( + "INSERT INTO embedding_metadata(id, key, string_value) VALUES (?, 'chroma:document', ?)", + (emb_id, doc), + ) + conn.execute( + "INSERT INTO embedding_metadata(id, key, string_value) VALUES (?, 'wing', ?)", + (emb_id, wing), + ) + conn.commit() + conn.close() + + class _NoScanCollection: + name = "mempalace_drawers" + + def count(self): + raise AssertionError("lexical_search should use Chroma sqlite FTS") + + def get(self, **_kwargs): + raise AssertionError("lexical_search should use Chroma sqlite FTS") + + collection = ChromaCollection(_NoScanCollection(), palace_path=str(tmp_path)) + + hits = collection.lexical_search(query="needle", n_results=1, where={"wing": "target"}).hits + + assert [hit.metadata["wing"] for hit in hits] == ["target"] + # Hit ids must be the public embedding_id (so lexical_search -> get(ids=...) + # round-trips), not the internal embeddings.id rowid. + assert [hit.id for hit in hits] == ["public-13"] + + +def test_chroma_lexical_search_ids_roundtrip_through_get(tmp_path): + """lexical_search must return public drawer ids that get(ids=...) accepts. + + Regression for the sqlite FTS path returning the internal rowid instead of + embeddings.embedding_id, which silently broke hybrid search id round-trips. + """ + backend = ChromaBackend() + palace = tmp_path / "palace" + ref = PalaceRef(id=str(palace), local_path=str(palace)) + col = backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True) + col.add( + ids=["drawer-alpha", "drawer-bravo", "drawer-charlie"], + documents=[ + "rareterm needle lexical note", + "unrelated content here", + "another rareterm needle entry", + ], + metadatas=[{"wing": "w"}, {"wing": "w"}, {"wing": "w"}], + ) + hits = col.lexical_search(query="rareterm needle", n_results=5).hits + hit_ids = [hit.id for hit in hits] + assert hit_ids, "expected lexical hits" + assert set(hit_ids) <= {"drawer-alpha", "drawer-bravo", "drawer-charlie"} + # Every returned id must resolve back through get() — proving it is a public id. + fetched = col.get(ids=hit_ids) + assert set(fetched.ids) == set(hit_ids) + backend.close() + + def test_query_rejects_missing_input(): fake = _FakeCollection() collection = ChromaCollection(fake) @@ -391,14 +492,12 @@ def test_chroma_backend_creates_collection_with_cosine_distance(tmp_path): def test_chroma_backend_sets_hnsw_bloat_guard_on_creation(tmp_path): - """The HNSW guard from #344 must land on freshly-created collection metadata. + """HNSW batch/sync thresholds must land on freshly-created collection metadata. - Without batch_size + sync_threshold, mining ~10K+ drawers triggers the - resize+persist drift that bloats link_lists.bin into hundreds of GB sparse - and segfaults `status` / `search` / `repair`. The guard belongs at - collection-creation time so every fresh palace gets it without needing - a runtime retrofit. Asserting both keys land on the persisted metadata - also covers the #1161 "config silently dropped" concern at CI time. + Low thresholds (2/2 per #1579) make chromadb's Rust HNSW segment + persist index_metadata and link_lists after any mine of 2+ drawers. + Asserting both keys land on the persisted metadata also covers the + #1161 "config silently dropped" concern at CI time. """ palace_path = tmp_path / "palace" @@ -410,8 +509,8 @@ def test_chroma_backend_sets_hnsw_bloat_guard_on_creation(tmp_path): client = chromadb.PersistentClient(path=str(palace_path)) col = client.get_collection("mempalace_drawers") - assert col.metadata.get("hnsw:batch_size") == 50_000 - assert col.metadata.get("hnsw:sync_threshold") == 50_000 + assert col.metadata.get("hnsw:batch_size") == 2 + assert col.metadata.get("hnsw:sync_threshold") == 2 def test_chroma_backend_create_collection_sets_hnsw_bloat_guard(tmp_path): @@ -422,8 +521,85 @@ def test_chroma_backend_create_collection_sets_hnsw_bloat_guard(tmp_path): client = chromadb.PersistentClient(path=str(palace_path)) col = client.get_collection("mempalace_drawers") - assert col.metadata.get("hnsw:batch_size") == 50_000 - assert col.metadata.get("hnsw:sync_threshold") == 50_000 + assert col.metadata.get("hnsw:batch_size") == 2 + assert col.metadata.get("hnsw:sync_threshold") == 2 + + +def test_sub_threshold_mine_persists_hnsw_metadata(tmp_path): + """Regression for #1579: small mines must persist HNSW metadata. + + _HNSW_BLOAT_GUARD sets batch_size=2 and sync_threshold=2 so that any + upsert of 2+ records crosses both thresholds, triggering chromadb's + _apply_batch and _persist. Without this, index_metadata and link_lists + stay empty and quarantine_stale_hnsw renames the segment on cold open. + """ + palace_path = str(tmp_path / "palace") + backend = ChromaBackend() + try: + col = backend.get_collection(palace_path, "mempalace_drawers", create=True) + + col.upsert( + ids=["a", "b", "c"], + documents=["doc a", "doc b", "doc c"], + embeddings=[[0.1] * _TEST_EMBED_DIM, [0.2] * _TEST_EMBED_DIM, [0.3] * _TEST_EMBED_DIM], + metadatas=[{"wing": "t"}, {"wing": "t"}, {"wing": "t"}], + ) + finally: + backend.close() + + found_healthy_segment = False + for entry in (tmp_path / "palace").iterdir(): + if not entry.is_dir() or entry.name.startswith("."): + continue + meta = entry / "index_metadata.pickle" + link = entry / "link_lists.bin" + data = entry / "data_level0.bin" + if data.exists() and data.stat().st_size > _HNSW_MISSING_METADATA_DATA_FLOOR: + assert meta.exists(), "index_metadata missing after sub-threshold upsert" + assert link.exists() and link.stat().st_size > 0, "link_lists empty" + assert _segment_appears_healthy(str(entry)) + found_healthy_segment = True + + assert found_healthy_segment, "no VECTOR segment with data found" + + # stale_seconds=0.0 forces the stage-2 integrity gate (_segment_appears_healthy) + # to run on every segment regardless of mtime delta, proving the fix directly. + moved = quarantine_stale_hnsw(palace_path, stale_seconds=0.0) + assert moved == [], f"quarantine fired on freshly-persisted segment: {moved}" + + +def test_single_record_upsert_not_quarantined(tmp_path): + """A single-record upsert must not trigger quarantine. + + With batch_size=2 chromadb only persists HNSW metadata after the second + record. A one-record segment has no index_metadata.pickle and no + link_lists.bin data; _segment_appears_healthy must treat that combination + as sub-threshold (never persisted), not as corruption. + """ + palace_path = str(tmp_path / "palace") + backend = ChromaBackend() + try: + col = backend.get_collection(palace_path, "mempalace_drawers", create=True) + col.upsert( + ids=["solo"], + documents=["only one drawer"], + embeddings=[[0.5] * _TEST_EMBED_DIM], + metadatas=[{"wing": "t"}], + ) + finally: + backend.close() + + for entry in (tmp_path / "palace").iterdir(): + if not entry.is_dir() or entry.name.startswith("."): + continue + data = entry / "data_level0.bin" + if data.exists() and data.stat().st_size > 0: + assert _segment_appears_healthy(str(entry)), ( + f"single-record segment flagged unhealthy: data={data.stat().st_size}B" + ) + + moved = quarantine_stale_hnsw(palace_path, stale_seconds=0.0) + assert moved == [], f"quarantine fired on single-record segment: {moved}" def test_get_collection_create_true_is_idempotent(tmp_path): @@ -449,7 +625,7 @@ def test_get_collection_create_true_preserves_existing_metadata(tmp_path): backend.get_collection(palace, collection_name="mempalace_drawers", create=True) col = backend.get_collection(palace, collection_name="mempalace_drawers", create=True) assert col._collection.metadata["hnsw:space"] == "cosine" - assert col._collection.metadata.get("hnsw:batch_size") == 50_000 + assert col._collection.metadata.get("hnsw:batch_size") == 2 def test_fix_blob_seq_ids_converts_blobs_to_integers(tmp_path): @@ -614,6 +790,186 @@ def test_fix_blob_seq_ids_skips_sqlite_when_marker_present(tmp_path): mock_connect.assert_not_called() +# ── _fix_missing_collection_type ───────────────────────────────────────── + + +def test_fix_collection_type_adds_type(tmp_path): + """Legacy config_json_str '{}' gets _type added.""" + import json + + db_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(db_path))) as conn: + conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)") + conn.execute( + "INSERT INTO collections (id, config_json_str) VALUES (?, ?)", + ("col-1", "{}"), + ) + conn.commit() + + _fix_missing_collection_type(str(tmp_path)) + + with closing(sqlite3.connect(str(db_path))) as conn: + row = conn.execute("SELECT config_json_str FROM collections WHERE id = 'col-1'").fetchone() + config = json.loads(row[0]) + assert config["_type"] == "CollectionConfigurationInternal" + + +def test_fix_collection_type_preserves_existing(tmp_path): + """Config that already has _type is left unchanged.""" + import json + + original = json.dumps({"_type": "CollectionConfigurationInternal", "extra": 1}) + db_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(db_path))) as conn: + conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)") + conn.execute( + "INSERT INTO collections (id, config_json_str) VALUES (?, ?)", + ("col-1", original), + ) + conn.commit() + + _fix_missing_collection_type(str(tmp_path)) + + with closing(sqlite3.connect(str(db_path))) as conn: + row = conn.execute("SELECT config_json_str FROM collections WHERE id = 'col-1'").fetchone() + assert row[0] == original + + +def test_fix_collection_type_noop_without_db(tmp_path): + """No error when palace has no chroma.sqlite3, no marker written.""" + from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER + + _fix_missing_collection_type(str(tmp_path)) + assert not (tmp_path / _COLLECTION_TYPE_MARKER).exists() + + +def test_fix_collection_type_writes_marker(tmp_path): + """Marker is written after a successful migration.""" + from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER + + db_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(db_path))) as conn: + conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)") + conn.execute( + "INSERT INTO collections (id, config_json_str) VALUES (?, ?)", + ("col-1", "{}"), + ) + conn.commit() + + marker = tmp_path / _COLLECTION_TYPE_MARKER + assert not marker.exists() + + _fix_missing_collection_type(str(tmp_path)) + + assert marker.is_file() + + +def test_fix_collection_type_skips_with_marker(tmp_path): + """When the marker exists, sqlite3 is not opened.""" + from unittest.mock import patch + + from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER + + db_path = tmp_path / "chroma.sqlite3" + db_path.write_bytes(b"sentinel") + (tmp_path / _COLLECTION_TYPE_MARKER).touch() + + with patch("mempalace.backends.chroma.sqlite3.connect") as mock_connect: + _fix_missing_collection_type(str(tmp_path)) + + mock_connect.assert_not_called() + + +def test_fix_collection_type_writes_marker_when_already_has_type(tmp_path): + """Marker written even when all collections already have _type (noop case).""" + import json + + from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER + + db_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(db_path))) as conn: + conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)") + conn.execute( + "INSERT INTO collections (id, config_json_str) VALUES (?, ?)", + ("col-1", json.dumps({"_type": "CollectionConfigurationInternal"})), + ) + conn.commit() + + marker = tmp_path / _COLLECTION_TYPE_MARKER + assert not marker.exists() + + _fix_missing_collection_type(str(tmp_path)) + + assert marker.is_file(), "marker must be written even when no collections needed fixing" + + +def test_fix_collection_type_multi_collection_mixed(tmp_path): + """Multiple collections: NULL, empty, and already-valid configs.""" + import json + + db_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(db_path))) as conn: + conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)") + conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-null", None)) + conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-empty", "{}")) + conn.execute( + "INSERT INTO collections VALUES (?, ?)", + ("col-ok", json.dumps({"_type": "CollectionConfigurationInternal"})), + ) + conn.commit() + + _fix_missing_collection_type(str(tmp_path)) + + with closing(sqlite3.connect(str(db_path))) as conn: + rows = { + r[0]: json.loads(r[1]) if r[1] else None + for r in conn.execute("SELECT id, config_json_str FROM collections") + } + assert rows["col-null"]["_type"] == "CollectionConfigurationInternal" + assert rows["col-empty"]["_type"] == "CollectionConfigurationInternal" + assert rows["col-ok"] == {"_type": "CollectionConfigurationInternal"} + + +def test_fix_collection_type_skips_non_dict_json(tmp_path): + """Non-dict JSON (array, null literal) is skipped without error.""" + import json + + db_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(db_path))) as conn: + conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)") + conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-arr", "[]")) + conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-null", "null")) + conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-ok", "{}")) + conn.commit() + + _fix_missing_collection_type(str(tmp_path)) + + with closing(sqlite3.connect(str(db_path))) as conn: + rows = dict(conn.execute("SELECT id, config_json_str FROM collections").fetchall()) + assert rows["col-arr"] == "[]" + assert rows["col-null"] == "null" + assert json.loads(rows["col-ok"])["_type"] == "CollectionConfigurationInternal" + + +def test_fix_collection_type_skips_malformed_json(tmp_path): + """Malformed JSON in one row does not prevent fixing other rows.""" + import json + + db_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(db_path))) as conn: + conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)") + conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-bad", "{corrupt")) + conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-ok", "{}")) + conn.commit() + + _fix_missing_collection_type(str(tmp_path)) + + with closing(sqlite3.connect(str(db_path))) as conn: + rows = dict(conn.execute("SELECT id, config_json_str FROM collections").fetchall()) + assert rows["col-bad"] == "{corrupt" + assert json.loads(rows["col-ok"])["_type"] == "CollectionConfigurationInternal" + + # ── quarantine_stale_hnsw ───────────────────────────────────────────────── @@ -698,17 +1054,18 @@ def test_quarantine_stale_hnsw_leaves_empty_segment_without_metadata_alone(tmp_p def test_segment_without_metadata_but_with_nontrivial_data_is_unhealthy(tmp_path): - """Data without index_metadata.pickle is a partial flush, not a fresh segment.""" + """Interrupted persist: link_lists written but metadata absent is unhealthy.""" seg = tmp_path / "abcd-1234-5678" seg.mkdir() (seg / "data_level0.bin").write_bytes(b"\0" * (_HNSW_MISSING_METADATA_DATA_FLOOR + 1)) + (seg / "link_lists.bin").write_bytes(b"\x01" * 128) assert not _segment_appears_healthy(str(seg)) def test_segment_without_metadata_and_tiny_data_is_still_treated_as_fresh(tmp_path): - """Tiny data payloads can occur before metadata has flushed; leave them alone.""" + """No metadata and no link_lists means no persist was attempted; treat as fresh.""" seg = tmp_path / "abcd-1234-5678" seg.mkdir() @@ -718,7 +1075,7 @@ def test_segment_without_metadata_and_tiny_data_is_still_treated_as_fresh(tmp_pa def test_quarantine_stale_hnsw_renames_missing_metadata_with_nontrivial_data(tmp_path): - """Regression for #1274: missing pickle + non-trivial data must quarantine.""" + """Regression for #1274: missing pickle + link data must quarantine.""" now = 1_700_000_000.0 palace, seg = _make_palace_with_segment( @@ -728,6 +1085,7 @@ def test_quarantine_stale_hnsw_renames_missing_metadata_with_nontrivial_data(tmp meta_bytes=None, ) (seg / "data_level0.bin").write_bytes(b"\0" * (_HNSW_MISSING_METADATA_DATA_FLOOR + 1)) + (seg / "link_lists.bin").write_bytes(b"\x01" * 128) os.utime(seg / "data_level0.bin", (now - 7200, now - 7200)) moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0) @@ -946,6 +1304,42 @@ def test_client_quarantines_only_on_first_call_per_palace(tmp_path, monkeypatch) ) +def test_client_rearms_quarantine_on_mtime_change(tmp_path, monkeypatch): + """When the DB file's mtime changes between ``_client()`` calls (external + in-place write), the quarantine gate re-arms so HNSW checks run again. + + Before #1573, the gate was only cleared on *inode* change (full palace + replacement); mtime-only changes left the gate armed, so long-running + processes were blind to external drift.""" + palace_path = str(tmp_path / "palace") + os.makedirs(palace_path, exist_ok=True) + db_file = Path(palace_path) / "chroma.sqlite3" + db_file.write_text("") + + monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set()) + + calls: list[str] = [] + + def _spy(path, stale_seconds=300.0): + calls.append(path) + return [] + + monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _spy) + + backend = ChromaBackend() + try: + backend._client(palace_path) + assert len(calls) == 1, "quarantine should fire on first open" + + _, cached_mtime = backend._freshness[palace_path] + os.utime(str(db_file), (cached_mtime + 1.0, cached_mtime + 1.0)) + + backend._client(palace_path) + assert len(calls) == 2, "quarantine should re-fire after mtime change (gate re-armed)" + finally: + backend.close() + + # ── _pin_hnsw_threads (per-process retrofit, separate from this PR's gate) ── @@ -1073,6 +1467,35 @@ def test_quarantine_invalid_hnsw_metadata_keeps_consistent_missing_dimensionalit assert seg.exists() +def test_quarantine_invalid_hnsw_metadata_keeps_post_deletion_missing_dimensionality(tmp_path): + """A deleted-from segment has total_elements_added > live label count (the + counter is monotonic); that dim-None shape is recoverable, not corruption (#1710). + """ + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + (seg / "data_level0.bin").write_bytes(b"x" * 2048) + (seg / "link_lists.bin").write_bytes(b"x" * 128) + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump( + { + "dimensionality": None, + "total_elements_added": 5, + "max_seq_id": None, + "id_to_label": {"a": 1, "b": 2}, + "label_to_id": {1: "a", 2: "b"}, + "id_to_seq_id": {}, + }, + f, + ) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert moved == [] + assert seg.exists() + + def test_quarantine_invalid_hnsw_metadata_renames_mismatched_missing_dimensionality(tmp_path): palace = tmp_path / "palace" palace.mkdir() @@ -1218,6 +1641,9 @@ def test_chroma_backend_preflights_metadata_before_persistent_client(tmp_path, m return inner + monkeypatch.setattr( + "mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type") + ) monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob")) monkeypatch.setattr( "mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid") @@ -1235,13 +1661,16 @@ def test_chroma_backend_preflights_metadata_before_persistent_client(tmp_path, m backend._client(str(palace)) assert calls == [ + ("collection_type", str(palace)), ("blob", str(palace)), ("invalid", str(palace)), ("stale", str(palace)), ] -def test_chroma_backend_stale_quarantine_is_cold_start_only_on_refresh(tmp_path, monkeypatch): +def test_chroma_backend_quarantine_rearms_on_mtime_refresh(tmp_path, monkeypatch): + """When the DB mtime changes between ``_client()`` calls, the quarantine + gate re-arms and the HNSW safety checks run again (#1573).""" palace = tmp_path / "palace" palace.mkdir() (palace / "chroma.sqlite3").write_text("") @@ -1255,6 +1684,9 @@ def test_chroma_backend_stale_quarantine_is_cold_start_only_on_refresh(tmp_path, return inner monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set()) + monkeypatch.setattr( + "mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type") + ) monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob")) monkeypatch.setattr( "mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid") @@ -1276,10 +1708,14 @@ def test_chroma_backend_stale_quarantine_is_cold_start_only_on_refresh(tmp_path, backend._client(str(palace)) assert calls == [ + ("collection_type", str(palace)), ("blob", str(palace)), ("invalid", str(palace)), ("stale", str(palace)), + ("collection_type", str(palace)), ("blob", str(palace)), + ("invalid", str(palace)), + ("stale", str(palace)), ] @@ -1297,6 +1733,9 @@ def test_chroma_backend_requarantines_after_inode_replacement(tmp_path, monkeypa return inner monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set()) + monkeypatch.setattr( + "mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type") + ) monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob")) monkeypatch.setattr( "mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid") @@ -1318,9 +1757,11 @@ def test_chroma_backend_requarantines_after_inode_replacement(tmp_path, monkeypa backend._client(str(palace)) assert calls == [ + ("collection_type", str(palace)), ("blob", str(palace)), ("invalid", str(palace)), ("stale", str(palace)), + ("collection_type", str(palace)), ("blob", str(palace)), ("invalid", str(palace)), ("stale", str(palace)), @@ -1371,7 +1812,7 @@ def test_get_collection_translates_ef_mismatch_to_helpful_error(tmp_path): return "embeddinggemma_300m" def __call__(self, input): - return [[0.0] * 384 for _ in input] + return [[0.0] * _TEST_EMBED_DIM for _ in input] original_resolver = backend._resolve_embedding_function backend._resolve_embedding_function = lambda: _ConflictingEF() diff --git a/tests/test_backups.py b/tests/test_backups.py new file mode 100644 index 0000000..b11769c --- /dev/null +++ b/tests/test_backups.py @@ -0,0 +1,157 @@ +"""Tests for backup retention pruning (mempalace.backups.prune_backups). + +These guard the fix for unbounded backup growth: ``mempalace migrate`` and +``mempalace repair max-seq-id`` each drop a fresh full-size, timestamped copy +every run, and used to never delete the old ones — a palace was found with +hundreds of GB of stale backups beside a few hundred MB of live data. +""" + +import os + +import pytest + +from mempalace.backups import prune_backups + + +def _make_backup_dir(parent, name, mtime): + """Create a directory backup with a fixed mtime.""" + path = parent / name + path.mkdir() + (path / "chroma.sqlite3").write_text("db") + os.utime(path, (mtime, mtime)) + return path + + +def _make_backup_file(parent, name, mtime): + """Create a file backup with a fixed mtime.""" + path = parent / name + path.write_text("db") + os.utime(path, (mtime, mtime)) + return path + + +def test_prune_keeps_newest_and_removes_oldest(tmp_path): + # 5 backups, mtimes 100..500; keep 2 newest (400, 500). + paths = [_make_backup_file(tmp_path, f"b.{i}", mtime=i * 100) for i in range(1, 6)] + + removed = prune_backups(str(tmp_path / "b.*"), max_backups=2) + + surviving = {p.name for p in tmp_path.iterdir()} + assert surviving == {"b.4", "b.5"} + assert set(removed) == {str(paths[0]), str(paths[1]), str(paths[2])} + + +def test_prune_removes_directory_backups(tmp_path): + """migrate writes directory backups (full copytree) — must rmtree them.""" + _make_backup_dir(tmp_path, "palace.pre-migrate.1", mtime=100) + _make_backup_dir(tmp_path, "palace.pre-migrate.2", mtime=200) + keep = _make_backup_dir(tmp_path, "palace.pre-migrate.3", mtime=300) + + removed = prune_backups(str(tmp_path / "palace.pre-migrate.*"), max_backups=1) + + assert keep.is_dir() + assert len(removed) == 2 + assert not (tmp_path / "palace.pre-migrate.1").exists() + assert not (tmp_path / "palace.pre-migrate.2").exists() + + +def test_prune_noop_when_under_limit(tmp_path): + _make_backup_file(tmp_path, "b.1", mtime=100) + _make_backup_file(tmp_path, "b.2", mtime=200) + + removed = prune_backups(str(tmp_path / "b.*"), max_backups=10) + + assert removed == [] + assert len(list(tmp_path.iterdir())) == 2 + + +def test_prune_noop_when_exactly_at_limit(tmp_path): + _make_backup_file(tmp_path, "b.1", mtime=100) + _make_backup_file(tmp_path, "b.2", mtime=200) + + removed = prune_backups(str(tmp_path / "b.*"), max_backups=2) + + assert removed == [] + + +@pytest.mark.parametrize("disabled", [0, -1, None]) +def test_prune_disabled_keeps_everything(tmp_path, disabled): + for i in range(1, 6): + _make_backup_file(tmp_path, f"b.{i}", mtime=i * 100) + + removed = prune_backups(str(tmp_path / "b.*"), max_backups=disabled) + + assert removed == [] + assert len(list(tmp_path.iterdir())) == 5 + + +def test_prune_no_matches(tmp_path): + assert prune_backups(str(tmp_path / "nope.*"), max_backups=3) == [] + + +def test_prune_only_touches_matching_pattern(tmp_path): + """Live data and unrelated files must never be swept up by a backup glob.""" + _make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-1", mtime=100) + _make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-2", mtime=200) + _make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-3", mtime=300) + # The live database and an unrelated file — must survive. + live = _make_backup_file(tmp_path, "chroma.sqlite3", mtime=400) + other = _make_backup_file(tmp_path, "tunnels.json", mtime=400) + + prune_backups( + str(tmp_path / "chroma.sqlite3.max-seq-id-backup-*"), + max_backups=1, + ) + + assert live.exists() + assert other.exists() + assert (tmp_path / "chroma.sqlite3.max-seq-id-backup-3").exists() + assert not (tmp_path / "chroma.sqlite3.max-seq-id-backup-1").exists() + assert not (tmp_path / "chroma.sqlite3.max-seq-id-backup-2").exists() + + +def test_prune_respects_glob_escape_for_metacharacter_paths(tmp_path): + """Palace paths can contain glob metacharacters like ``[``. + + Without ``glob.escape`` the pattern would silently match nothing (the + bracket is read as a character class), leaving backups unpruned. Callers + escape the literal prefix; this confirms the helper prunes correctly once + they do. + """ + import glob + + weird = tmp_path / "weird[name]" + weird.mkdir() + for i in range(1, 4): + _make_backup_file(weird, f"chroma.sqlite3.max-seq-id-backup-{i}", mtime=i * 100) + + pattern = os.path.join(glob.escape(str(weird)), "chroma.sqlite3.max-seq-id-backup-*") + removed = prune_backups(pattern, max_backups=1) + + assert len(removed) == 2 + assert (weird / "chroma.sqlite3.max-seq-id-backup-3").exists() + + +def test_prune_is_best_effort_on_delete_failure(tmp_path, monkeypatch): + """A failed deletion is logged and skipped, never raised — pruning must + not undo a migrate/repair that already succeeded.""" + for i in range(1, 5): + _make_backup_file(tmp_path, f"b.{i}", mtime=i * 100) + + real_remove = os.remove + + def flaky_remove(path): + if path.endswith("b.1"): + raise OSError("permission denied") + return real_remove(path) + + monkeypatch.setattr(os, "remove", flaky_remove) + + logs = [] + removed = prune_backups(str(tmp_path / "b.*"), max_backups=2, log=logs.append) + + # b.1 and b.2 were over the limit; b.1 failed, b.2 succeeded. + assert str(tmp_path / "b.2") in removed + assert str(tmp_path / "b.1") not in removed + assert (tmp_path / "b.1").exists() + assert any("could not remove" in line for line in logs) diff --git a/tests/test_clean_lone_surrogates.py b/tests/test_clean_lone_surrogates.py index 60b6c12..650f800 100644 --- a/tests/test_clean_lone_surrogates.py +++ b/tests/test_clean_lone_surrogates.py @@ -177,3 +177,88 @@ class TestToolsAcceptSurrogates: topic="log", ) assert result["success"] is True + + +# ── Backend chokepoint (bulk ingest paths) ────────────────────────────────── + + +class _CapturingCollection: + """Minimal chromadb.Collection stand-in that records the kwargs it receives, + so we can assert what actually reaches the chromadb client.""" + + def __init__(self): + self.calls = [] + + def add(self, **kwargs): + self.calls.append(("add", kwargs)) + + def upsert(self, **kwargs): + self.calls.append(("upsert", kwargs)) + + def update(self, **kwargs): + self.calls.append(("update", kwargs)) + + +class TestBackendChokepointStripsDocuments: + """#1235 sanitised the MCP write tools, but the bulk ingest paths + (miner, convo_miner, sweeper, diary_ingest) build documents without routing + through ``sanitize_content`` and reach ``ChromaCollection`` directly. A lone + surrogate in *document* text crashes the whole add/upsert batch (-32000), + silently dropping the other rows. The backend chokepoint must strip + documents, mirroring ``_sanitize_metadatas_for_chromadb``.""" + + @staticmethod + def _collection(): + from mempalace.backends.chroma import ChromaCollection + + fake = _CapturingCollection() + return fake, ChromaCollection(fake) + + def test_add_strips_lone_surrogate_in_document(self): + fake, col = self._collection() + col.add(documents=["clean\udc95doc"], ids=["1"]) + _, kwargs = fake.calls[0] + assert kwargs["documents"] == ["clean�doc"] + kwargs["documents"][0].encode("utf-8") # the crash path must not raise + + def test_upsert_strips_lone_surrogate_in_document(self): + fake, col = self._collection() + col.upsert(documents=["a\ud800b"], ids=["1"], metadatas=[{"wing": "w"}]) + _, kwargs = fake.calls[0] + assert kwargs["documents"] == ["a�b"] + + def test_update_strips_lone_surrogate_in_document(self): + fake, col = self._collection() + col.update(ids=["1"], documents=["x\udcffy"]) + _, kwargs = fake.calls[0] + assert kwargs["documents"] == ["x�y"] + + def test_one_poison_message_does_not_drop_the_batch(self): + """A single bad row used to abort the entire batch, silently dropping + the others. After sanitising, every row survives.""" + fake, col = self._collection() + col.upsert( + documents=["ok one", "poison\udc95row", "ok three"], + ids=["1", "2", "3"], + ) + _, kwargs = fake.calls[0] + assert kwargs["documents"] == ["ok one", "poison�row", "ok three"] + assert len(kwargs["ids"]) == 3 + for d in kwargs["documents"]: + d.encode("utf-8") # must not raise + + def test_real_emoji_and_none_metadata_preserved(self): + fake, col = self._collection() + col.add(documents=["ship \U0001f680"], ids=["1"]) + _, kwargs = fake.calls[0] + assert kwargs["documents"] == ["ship \U0001f680"] + + def test_single_string_document_is_not_split_into_chars(self): + """chromadb accepts a bare str as one document (OneOrMany[Document]). + The sanitiser must keep it whole and clean, not split it into + per-character documents.""" + fake, col = self._collection() + col.upsert(documents="one\udc95document", ids=["1"]) + _, kwargs = fake.calls[0] + assert kwargs["documents"] == "one�document" + assert isinstance(kwargs["documents"], str) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0caf75c..3346b5c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -713,6 +713,40 @@ def test_main_status_dispatches(): mock_cmd.assert_called_once() +def test_main_backend_flag_sets_explicit_backend(monkeypatch): + monkeypatch.delenv("MEMPALACE_BACKEND_EXPLICIT", raising=False) + monkeypatch.delenv("MEMPALACE_BACKEND", raising=False) + with ( + patch("sys.argv", ["mempalace", "status", "--backend", "sqlite_exact"]), + patch("mempalace.cli.cmd_status") as mock_cmd, + ): + main() + + mock_cmd.assert_called_once() + args = mock_cmd.call_args.args[0] + assert args.backend == "sqlite_exact" + assert os.environ["MEMPALACE_BACKEND_EXPLICIT"] == "sqlite_exact" + os.environ.pop("MEMPALACE_BACKEND_EXPLICIT", None) + os.environ.pop("MEMPALACE_BACKEND", None) + + +def test_main_backend_flag_accepts_qdrant(monkeypatch): + monkeypatch.delenv("MEMPALACE_BACKEND_EXPLICIT", raising=False) + monkeypatch.delenv("MEMPALACE_BACKEND", raising=False) + with ( + patch("sys.argv", ["mempalace", "status", "--backend", "qdrant"]), + patch("mempalace.cli.cmd_status") as mock_cmd, + ): + main() + + mock_cmd.assert_called_once() + args = mock_cmd.call_args.args[0] + assert args.backend == "qdrant" + assert os.environ["MEMPALACE_BACKEND_EXPLICIT"] == "qdrant" + os.environ.pop("MEMPALACE_BACKEND_EXPLICIT", None) + os.environ.pop("MEMPALACE_BACKEND", None) + + def test_main_search_dispatches(): with ( patch("sys.argv", ["mempalace", "search", "my query"]), @@ -790,6 +824,34 @@ def test_mcp_command_uses_custom_palace_path_when_provided(monkeypatch, capsys): assert captured.err == "" +def test_mcp_command_includes_backend_when_provided(monkeypatch, capsys): + monkeypatch.delenv("MEMPALACE_BACKEND_EXPLICIT", raising=False) + monkeypatch.delenv("MEMPALACE_BACKEND", raising=False) + monkeypatch.setattr(sys, "argv", ["mempalace", "mcp", "--backend", "sqlite_exact"]) + + main() + + captured = capsys.readouterr() + assert "mempalace-mcp --backend sqlite_exact" in captured.out + assert captured.err == "" + os.environ.pop("MEMPALACE_BACKEND_EXPLICIT", None) + os.environ.pop("MEMPALACE_BACKEND", None) + + +def test_mcp_command_includes_qdrant_backend(monkeypatch, capsys): + monkeypatch.delenv("MEMPALACE_BACKEND_EXPLICIT", raising=False) + monkeypatch.delenv("MEMPALACE_BACKEND", raising=False) + monkeypatch.setattr(sys, "argv", ["mempalace", "mcp", "--backend", "qdrant"]) + + main() + + captured = capsys.readouterr() + assert "mempalace-mcp --backend qdrant" in captured.out + assert captured.err == "" + os.environ.pop("MEMPALACE_BACKEND_EXPLICIT", None) + os.environ.pop("MEMPALACE_BACKEND", None) + + def test_main_hook_no_subcommand_prints_help(capsys): with patch("sys.argv", ["mempalace", "hook"]): main() diff --git a/tests/test_collision_scan.py b/tests/test_collision_scan.py new file mode 100644 index 0000000..3ac315d --- /dev/null +++ b/tests/test_collision_scan.py @@ -0,0 +1,228 @@ +"""Tests for mempalace.collision_scan — pre-mining defense against +drawer_id collisions. + +The scan runs immediately before batched chromadb upserts and aborts the +mine with an actionable error if any proposed drawer_id appears more than +once in the union of (incoming-vs-incoming) and (incoming-vs-existing). + +Under the v2 hash recipe these collisions are vanishingly rare in practice +— SHA-256 truncated to 24 hex chars makes accidental collision ~2^-96. +The scan's real value is (a) catching upstream bugs that emit duplicate +(source_file, chunk_index) pairs in the same batch, and (b) surfacing the +astronomical-but-possible SHA-256 collision with a clear error instead of +a silent overwrite at the ChromaDB upsert. +""" + +from __future__ import annotations + +from typing import Optional + +import pytest + +from mempalace.collision_scan import CollisionError, assert_no_collisions + + +class _MockGet: + """Stand-in for ChromaDB's get() result. Real ChromaDB returns a + dict-like with ``ids`` and ``metadatas`` keys; we mirror that shape.""" + + def __init__(self, ids: list[str], metadatas: list[dict]): + self._payload = {"ids": ids, "metadatas": metadatas} + + def __getitem__(self, key): + return self._payload[key] + + def get(self, key, default=None): + return self._payload.get(key, default) + + +class _MockCollection: + """Stand-in for a ChromaDB collection. Stores a fixed mapping of + drawer_id → metadata for the test to model 'existing palace state'. + ``get(ids=[...])`` returns only the rows whose ids are in storage. + """ + + def __init__(self, existing: Optional[dict[str, dict]] = None): + self._existing = existing or {} + + def get(self, ids=None, include=None, **kwargs): + ids = ids or [] + rows = [(did, self._existing[did]) for did in ids if did in self._existing] + return _MockGet( + ids=[did for did, _ in rows], + metadatas=[meta for _, meta in rows], + ) + + +# ── Happy path: no collisions ──────────────────────────────────────── + + +def test_assert_no_collisions_passes_for_clean_batch(): + """Distinct incoming ids + no overlap with existing = no error.""" + proposed = [ + ("drawer_a", {"source_file": "/file_a.md", "chunk_index": 0}), + ("drawer_b", {"source_file": "/file_a.md", "chunk_index": 1}), + ("drawer_c", {"source_file": "/file_b.md", "chunk_index": 0}), + ] + col = _MockCollection() + assert_no_collisions(proposed, col) is None + + +def test_assert_no_collisions_passes_for_clean_batch_with_existing_drawers(): + """Existing drawers with DIFFERENT ids than incoming = no error. + The scan only fires when an id appears more than once across the + union of incoming + existing.""" + proposed = [ + ("drawer_new_1", {"source_file": "/new.md", "chunk_index": 0}), + ] + col = _MockCollection( + existing={ + "drawer_old_1": {"source_file": "/old.md", "chunk_index": 0}, + "drawer_old_2": {"source_file": "/old.md", "chunk_index": 1}, + } + ) + assert_no_collisions(proposed, col) is None + + +def test_assert_no_collisions_treats_idempotent_re_mine_as_clean(): + """If incoming drawer_id matches an existing id AND the + (source_file, chunk_index) metadata also matches, that's a normal + re-mine of the same chunk — NOT a collision. The scan must let it + pass; otherwise re-mining a clean palace would always raise.""" + proposed = [ + ("drawer_same", {"source_file": "/file.md", "chunk_index": 5}), + ] + col = _MockCollection( + existing={ + "drawer_same": {"source_file": "/file.md", "chunk_index": 5}, + } + ) + assert_no_collisions(proposed, col) is None + + +# ── Incoming-vs-incoming collisions ────────────────────────────────── + + +def test_assert_no_collisions_raises_on_incoming_duplicate_with_different_metadata(): + """Two incoming chunks producing the same drawer_id with DIFFERENT + (source_file, chunk_index) pairs = an upstream bug or astronomical + SHA-256 hash collision. Either way, abort the mine.""" + proposed = [ + ("drawer_X", {"source_file": "/file_a.md", "chunk_index": 0}), + ("drawer_X", {"source_file": "/file_b.md", "chunk_index": 1}), + ] + col = _MockCollection() + with pytest.raises(CollisionError) as exc_info: + assert_no_collisions(proposed, col) + # Error message names the colliding (source_file, chunk_index) pairs + msg = str(exc_info.value) + assert "drawer_X" in msg + assert "/file_a.md" in msg + assert "/file_b.md" in msg + + +def test_assert_no_collisions_passes_on_incoming_duplicate_with_same_metadata(): + """Two incoming chunks with the SAME (source_file, chunk_index) + producing the same drawer_id = a duplicate chunk in the batch (still + an upstream bug, but the downstream collision damage is zero since + they'd write identical content). The scan does not fire on this + case because it's not the collision shape v2 was designed to + catch.""" + proposed = [ + ("drawer_X", {"source_file": "/file.md", "chunk_index": 5}), + ("drawer_X", {"source_file": "/file.md", "chunk_index": 5}), + ] + col = _MockCollection() + assert_no_collisions(proposed, col) is None + + +# ── Incoming-vs-existing collisions ────────────────────────────────── + + +def test_assert_no_collisions_raises_on_incoming_matching_existing_with_different_metadata(): + """Incoming chunk produces the same drawer_id as an existing drawer + whose stored (source_file, chunk_index) DIFFERS = SHA-256 collision + or recipe-version skew. Abort the mine so the upsert doesn't + silently overwrite the existing row.""" + proposed = [ + ("drawer_Y", {"source_file": "/incoming.md", "chunk_index": 3}), + ] + col = _MockCollection( + existing={ + "drawer_Y": {"source_file": "/existing.md", "chunk_index": 7}, + } + ) + with pytest.raises(CollisionError) as exc_info: + assert_no_collisions(proposed, col) + msg = str(exc_info.value) + assert "drawer_Y" in msg + assert "/incoming.md" in msg + assert "/existing.md" in msg + + +# ── Error message quality ──────────────────────────────────────────── + + +def test_collision_error_lists_all_collisions_not_just_first(): + """If a batch contains multiple distinct collisions, the error + surfaces ALL of them — a user fixing one and re-running shouldn't + rediscover the next one from scratch.""" + proposed = [ + ("drawer_A", {"source_file": "/f1.md", "chunk_index": 0}), + ("drawer_A", {"source_file": "/f2.md", "chunk_index": 0}), # collision 1 + ("drawer_B", {"source_file": "/f3.md", "chunk_index": 0}), + ("drawer_B", {"source_file": "/f4.md", "chunk_index": 0}), # collision 2 + ] + col = _MockCollection() + with pytest.raises(CollisionError) as exc_info: + assert_no_collisions(proposed, col) + msg = str(exc_info.value) + assert "drawer_A" in msg + assert "drawer_B" in msg + assert "/f1.md" in msg + assert "/f2.md" in msg + assert "/f3.md" in msg + assert "/f4.md" in msg + + +# ── Edge cases ─────────────────────────────────────────────────────── + + +def test_assert_no_collisions_passes_for_empty_batch(): + """An empty mining batch is trivially collision-free — the scan + must not raise on len(proposed) == 0 so callers don't have to guard + the call site.""" + assert_no_collisions([], _MockCollection()) is None + + +def test_assert_no_collisions_handles_metadata_without_chunk_index(): + """Some drawer types (e.g. diary entries, sentinels) don't carry + chunk_index. The scan should compare whatever metadata is present + without crashing on missing keys.""" + proposed = [ + ("drawer_diary_1", {"source_file": "/diary.md"}), + ] + col = _MockCollection( + existing={ + "drawer_diary_1": {"source_file": "/diary_other.md"}, + } + ) + with pytest.raises(CollisionError): + assert_no_collisions(proposed, col) + + +def test_assert_no_collisions_tolerates_chromadb_get_failure(): + """ChromaDB's get() can raise on transient backend errors. The + scan should NOT swallow those — the caller's broad-except can decide + whether to abort the mine or proceed. The scan's contract is + 'either confirms no collisions, or raises'. Hiding backend errors + behind a False is the silent-failure shape this PR was meant to + eliminate.""" + + class _RaisingCollection: + def get(self, **kwargs): + raise RuntimeError("chromadb is sad") + + proposed = [("drawer_X", {"source_file": "/f.md", "chunk_index": 0})] + with pytest.raises(RuntimeError, match="chromadb is sad"): + assert_no_collisions(proposed, _RaisingCollection()) diff --git a/tests/test_config.py b/tests/test_config.py index ff48934..06ecba1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -17,6 +17,7 @@ def test_default_config(): cfg = MempalaceConfig(config_dir=tempfile.mkdtemp()) assert "palace" in cfg.palace_path assert cfg.collection_name == "mempalace_drawers" + assert cfg.backend == "chroma" def test_config_from_file(): @@ -27,6 +28,54 @@ def test_config_from_file(): assert cfg.palace_path == "/custom/palace" +def test_backend_from_config_wins_over_env(tmp_path, monkeypatch): + with open(tmp_path / "config.json", "w") as f: + json.dump({"backend": "sqlite_exact"}, f) + monkeypatch.setenv("MEMPALACE_BACKEND", "chroma") + + cfg = MempalaceConfig(config_dir=str(tmp_path)) + assert cfg.backend == "sqlite_exact" + + +def test_backend_from_env_when_config_absent(tmp_path, monkeypatch): + monkeypatch.setenv("MEMPALACE_BACKEND", "SQLite_Exact") + + cfg = MempalaceConfig(config_dir=str(tmp_path)) + assert cfg.backend == "sqlite_exact" + + +def test_qdrant_config_from_env_and_file(tmp_path, monkeypatch): + with open(tmp_path / "config.json", "w") as f: + json.dump( + { + "qdrant_url": "http://config.example:6333", + "qdrant_api_key": "config-key", + "qdrant_namespace": "config-ns", + "qdrant_timeout": 2, + }, + f, + ) + monkeypatch.setenv("MEMPALACE_QDRANT_URL", "http://env.example:6333") + monkeypatch.setenv("MEMPALACE_QDRANT_API_KEY", "env-key") + monkeypatch.setenv("MEMPALACE_QDRANT_NAMESPACE", "env-ns") + monkeypatch.setenv("MEMPALACE_QDRANT_TIMEOUT", "3.5") + + cfg = MempalaceConfig(config_dir=str(tmp_path)) + + assert cfg.qdrant_url == "http://env.example:6333" + assert cfg.qdrant_api_key == "env-key" + assert cfg.qdrant_namespace == "env-ns" + assert cfg.qdrant_timeout == 3.5 + + +def test_set_backend_persists_choice(tmp_path): + cfg = MempalaceConfig(config_dir=str(tmp_path)) + cfg.set_backend("sqlite_exact") + + reloaded = MempalaceConfig(config_dir=str(tmp_path)) + assert reloaded.backend == "sqlite_exact" + + def test_embedding_device_defaults_to_auto(monkeypatch): monkeypatch.delenv("MEMPALACE_EMBEDDING_DEVICE", raising=False) cfg = MempalaceConfig(config_dir=tempfile.mkdtemp()) @@ -115,6 +164,17 @@ def test_init(): cfg = MempalaceConfig(config_dir=tmpdir) cfg.init() assert os.path.exists(os.path.join(tmpdir, "config.json")) + with open(os.path.join(tmpdir, "config.json")) as f: + saved = json.load(f) + assert "backend" not in saved + assert MempalaceConfig(config_dir=tmpdir).backend == "chroma" + + +def test_set_backend_rejects_unknown_backend(tmp_path): + cfg = MempalaceConfig(config_dir=str(tmp_path)) + + with pytest.raises(KeyError): + cfg.set_backend("does_not_exist") # --- normalize_wing_name --- @@ -136,6 +196,16 @@ def test_normalize_wing_name_mixed(): assert normalize_wing_name("My-Cool App") == "my_cool_app" +def test_normalize_wing_name_strips_leading_separator(): + # Claude Code path-encoded project dirs begin with a separator; the slug + # must not start with "_" or sanitize_name / MCP writes would reject it. + assert normalize_wing_name("-home-user-linux-book") == "home_user_linux_book" + + +def test_normalize_wing_name_strips_trailing_separator(): + assert normalize_wing_name("project-") == "project" + + # --- sanitize_name --- @@ -618,3 +688,63 @@ def test_hooks_auto_save_env_override_true(): assert cfg.hooks_auto_save is True finally: del os.environ["MEMPALACE_HOOKS_AUTO_SAVE"] + + +# --- max_backups (backup retention) --- + + +def test_max_backups_default(monkeypatch): + monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False) + cfg = MempalaceConfig(config_dir=tempfile.mkdtemp()) + assert cfg.max_backups == 10 + + +def test_max_backups_from_config(monkeypatch, tmp_path): + monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False) + with open(tmp_path / "config.json", "w") as f: + json.dump({"max_backups": 3}, f) + cfg = MempalaceConfig(config_dir=str(tmp_path)) + assert cfg.max_backups == 3 + + +def test_max_backups_zero_disables(monkeypatch, tmp_path): + """0 is a valid, explicit "keep everything" — not garbage.""" + monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False) + with open(tmp_path / "config.json", "w") as f: + json.dump({"max_backups": 0}, f) + cfg = MempalaceConfig(config_dir=str(tmp_path)) + assert cfg.max_backups == 0 + + +def test_max_backups_env_overrides_config(monkeypatch, tmp_path): + with open(tmp_path / "config.json", "w") as f: + json.dump({"max_backups": 3}, f) + monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "7") + cfg = MempalaceConfig(config_dir=str(tmp_path)) + assert cfg.max_backups == 7 + + +@pytest.mark.parametrize("bad", ["abc", "", "-5", "1.5", "true"]) +def test_max_backups_garbage_falls_back_to_default(monkeypatch, tmp_path, bad): + """A hand-edited bad value must never crash migrate/repair.""" + with open(tmp_path / "config.json", "w") as f: + json.dump({"max_backups": bad}, f) + monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False) + cfg = MempalaceConfig(config_dir=str(tmp_path)) + assert cfg.max_backups == 10 + + +def test_max_backups_negative_in_config_falls_back(monkeypatch, tmp_path): + monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False) + with open(tmp_path / "config.json", "w") as f: + json.dump({"max_backups": -3}, f) + cfg = MempalaceConfig(config_dir=str(tmp_path)) + assert cfg.max_backups == 10 + + +def test_max_backups_bad_env_falls_back_to_config(monkeypatch, tmp_path): + with open(tmp_path / "config.json", "w") as f: + json.dump({"max_backups": 4}, f) + monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "garbage") + cfg = MempalaceConfig(config_dir=str(tmp_path)) + assert cfg.max_backups == 4 diff --git a/tests/test_convo_miner.py b/tests/test_convo_miner.py index fd775ac..22eefcd 100644 --- a/tests/test_convo_miner.py +++ b/tests/test_convo_miner.py @@ -416,3 +416,41 @@ def test_resolve_wing_empty_string_treated_as_no_wing(tmp_path): target = tmp_path / ".gemini" / "tmp" target.mkdir(parents=True) assert _resolve_wing(target, wing="") == "wing_api" + + +def test_mine_convos_limit_skips_already_mined(capsys): + """--limit N counts only new work, not already-mined skips (#1535).""" + tmpdir = tempfile.mkdtemp() + try: + convo_text = ( + "> What is topic {i}?\n" + "Topic {i} is about something important and interesting enough " + "to produce at least one exchange chunk for the test.\n\n" + "> Tell me more about topic {i}.\n" + "Sure, topic {i} has many facets worth exploring in detail.\n" + ) + for i in range(4): + with open(os.path.join(tmpdir, f"chat_{i}.txt"), "w") as f: + f.write(convo_text.format(i=i)) + + palace_path = os.path.join(tmpdir, "palace") + + mine_convos(tmpdir, palace_path, wing="test") + capsys.readouterr() + + for i in range(4, 7): + with open(os.path.join(tmpdir, f"chat_{i}.txt"), "w") as f: + f.write(convo_text.format(i=i)) + + mine_convos(tmpdir, palace_path, wing="test", limit=2) + out = capsys.readouterr().out + + assert "Files processed: 2" in out + assert "Drawers filed:" in out + for line in out.split("\n"): + if "Drawers filed:" in line: + filed = int(line.split(":")[1].strip()) + assert filed > 0, f"limit=2 should mine new files, got {filed}" + break + finally: + shutil.rmtree(tmpdir, ignore_errors=True) diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index b14f8ab..2970cb5 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -186,6 +186,64 @@ class TestChunkExchanges: assert chunks[1]["content"] == "b" * CHUNK_SIZE assert chunks[2]["content"] == "c" * CHUNK_SIZE + def test_ai_response_preserves_blank_lines(self): + """Blank lines inside an AI response must survive ingestion (verbatim principle). + + A response with paragraph breaks separates distinct ideas; collapsing the + blank lines loses that boundary and fuses unrelated content. + """ + # Three `>` turns route through _chunk_by_exchange (the exchange-pair path). + content = ( + "> explain the architecture\n" + "First paragraph introducing the system.\n" + "\n" + "Second paragraph about the data layer.\n" + "\n" + "Third paragraph about retrieval.\n" + "\n" + "> what about caching?\n" + "Cache lives in memory and is invalidated on write.\n" + "\n" + "> and persistence?\n" + "Persistence lives on disk via SQLite and Chroma.\n" + ) + chunks = chunk_exchanges(content) + assert len(chunks) >= 1 + stored = "\n".join(c["content"] for c in chunks) + # Paragraph break between the three bodies must survive as `\n\n`. + assert "First paragraph introducing the system.\n\nSecond paragraph" in stored + assert "Second paragraph about the data layer.\n\nThird paragraph" in stored + + def test_ai_response_preserves_line_structure(self): + """Line-oriented content (lists, code fences, tables) must keep newlines. + + Joining lines with a single space fuses structurally distinct tokens, + breaks downstream search, and destroys code blocks. + """ + content = ( + "> show me the steps\n" + "1. First step\n" + "2. Second step\n" + "3. Third step\n" + "```python\n" + "def hello():\n" + " return 'world'\n" + "```\n" + "\n" + "> what next?\n" + "Run the test suite.\n" + "\n" + "> anything else?\n" + "Ship the feature.\n" + ) + chunks = chunk_exchanges(content) + assert len(chunks) >= 1 + stored = "\n".join(c["content"] for c in chunks) + # Each list item keeps its own line (not "1. First step 2. Second step"). + assert "1. First step\n2. Second step\n3. Third step" in stored + # Code fence survives intact, with indentation preserved. + assert "```python\ndef hello():\n return 'world'\n```" in stored + class TestEmitBounded: """Direct unit tests for the chunk-size-enforcement helper.""" @@ -385,6 +443,11 @@ class TestFileChunksLocked: def delete(self, *args, **kwargs): pass + def get(self, ids=None, include=None, **kwargs): + # Pre-mining collision scan probes the collection; empty + # palace under test, so nothing matches. + return {"ids": [], "metadatas": []} + def upsert(self, documents, ids, metadatas): self.batch_sizes.append(len(documents)) diff --git a/tests/test_dedup.py b/tests/test_dedup.py index dfdd3de..a3f7467 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -198,15 +198,13 @@ def test_dedup_source_group_query_failure_keeps(): # ── show_stats ──────────────────────────────────────────────────────── -def _install_mock_backend(mock_backend_cls, collection): - mock_backend = MagicMock() - mock_backend.get_collection.return_value = collection - mock_backend_cls.return_value = mock_backend - return mock_backend +def _install_mock_collection(mock_get_collection, collection): + mock_get_collection.return_value = collection + return collection -@patch("mempalace.dedup.ChromaBackend") -def test_show_stats(mock_backend_cls, tmp_path): +@patch("mempalace.dedup.get_collection") +def test_show_stats(mock_get_collection, tmp_path): mock_col = MagicMock() mock_col.count.return_value = 5 mock_col.get.side_effect = [ @@ -222,7 +220,7 @@ def test_show_stats(mock_backend_cls, tmp_path): }, {"ids": []}, ] - _install_mock_backend(mock_backend_cls, mock_col) + _install_mock_collection(mock_get_collection, mock_col) dedup.show_stats(palace_path=str(tmp_path)) # should not raise @@ -232,11 +230,11 @@ def test_show_stats(mock_backend_cls, tmp_path): @patch("mempalace.dedup.dedup_source_group") @patch("mempalace.dedup.get_source_groups") -@patch("mempalace.dedup.ChromaBackend") -def test_dedup_palace_dry_run(mock_backend_cls, mock_groups, mock_dedup_group, tmp_path): +@patch("mempalace.dedup.get_collection") +def test_dedup_palace_dry_run(mock_get_collection, mock_groups, mock_dedup_group, tmp_path): mock_col = MagicMock() mock_col.count.return_value = 10 - _install_mock_backend(mock_backend_cls, mock_col) + _install_mock_collection(mock_get_collection, mock_col) mock_groups.return_value = {"a.txt": ["d1", "d2", "d3", "d4", "d5"]} mock_dedup_group.return_value = (["d1", "d2", "d3"], ["d4", "d5"]) @@ -247,11 +245,11 @@ def test_dedup_palace_dry_run(mock_backend_cls, mock_groups, mock_dedup_group, t @patch("mempalace.dedup.dedup_source_group") @patch("mempalace.dedup.get_source_groups") -@patch("mempalace.dedup.ChromaBackend") -def test_dedup_palace_with_wing(mock_backend_cls, mock_groups, mock_dedup_group, tmp_path): +@patch("mempalace.dedup.get_collection") +def test_dedup_palace_with_wing(mock_get_collection, mock_groups, mock_dedup_group, tmp_path): mock_col = MagicMock() mock_col.count.return_value = 10 - _install_mock_backend(mock_backend_cls, mock_col) + _install_mock_collection(mock_get_collection, mock_col) mock_groups.return_value = {} dedup.dedup_palace(palace_path=str(tmp_path), wing="test_wing", dry_run=True) @@ -260,11 +258,11 @@ def test_dedup_palace_with_wing(mock_backend_cls, mock_groups, mock_dedup_group, @patch("mempalace.dedup.dedup_source_group") @patch("mempalace.dedup.get_source_groups") -@patch("mempalace.dedup.ChromaBackend") -def test_dedup_palace_no_groups(mock_backend_cls, mock_groups, mock_dedup_group, tmp_path): +@patch("mempalace.dedup.get_collection") +def test_dedup_palace_no_groups(mock_get_collection, mock_groups, mock_dedup_group, tmp_path): mock_col = MagicMock() mock_col.count.return_value = 3 - _install_mock_backend(mock_backend_cls, mock_col) + _install_mock_collection(mock_get_collection, mock_col) mock_groups.return_value = {} dedup.dedup_palace(palace_path=str(tmp_path), dry_run=True) diff --git a/tests/test_distance_metric.py b/tests/test_distance_metric.py new file mode 100644 index 0000000..c4334b0 --- /dev/null +++ b/tests/test_distance_metric.py @@ -0,0 +1,227 @@ +"""Tests for backend-declared distance metrics (RFC 001) and the +metric-aware distance→similarity conversion in the searcher. + +Before this, the searcher hard-coded ``max(0, 1 - distance)`` everywhere, +which is correct only for cosine. A backend reporting L2 or inner-product +distances (or a legacy Chroma palace built without ``hnsw:space=cosine``) +was silently mis-ranked — L2 distances routinely exceed 1.0 and floored +every result's similarity to 0. The contract now lets a backend declare its +metric and the searcher converts accordingly. +""" + +import math +import types + +import pytest + +from mempalace.backends.base import BaseBackend, BaseCollection +from mempalace.backends.chroma import ChromaCollection +from mempalace.searcher import ( + _distance_to_similarity, + _hybrid_rank, + _metric_for_collection, +) + + +# --------------------------------------------------------------------------- +# Contract surface +# --------------------------------------------------------------------------- + + +def test_basebackend_declares_cosine_default(): + assert BaseBackend.distance_metric == "cosine" + + +def test_basecollection_reports_cosine_default(): + # A minimal concrete collection inherits the cosine default. + class _Col(BaseCollection): + def add(self, **k): ... + def upsert(self, **k): ... + def query(self, **k): ... + def get(self, **k): ... + def delete(self, **k): ... + def count(self): + return 0 + + assert _Col().distance_metric == "cosine" + + +# --------------------------------------------------------------------------- +# _distance_to_similarity — per-metric math +# --------------------------------------------------------------------------- + + +def test_cosine_conversion(): + assert _distance_to_similarity(0.0, "cosine") == 1.0 + assert _distance_to_similarity(2.0, "cosine") == 0.0 + # cosine distance > 1 must floor at 0, never go negative. + assert _distance_to_similarity(1.5, "cosine") == 0.0 + + +def test_l2_conversion_is_monotonic_and_bounded(): + assert _distance_to_similarity(0.0, "l2") == 1.0 + assert _distance_to_similarity(1.0, "l2") == pytest.approx(0.5) + # Strictly decreasing, and a large L2 distance does NOT floor to 0 the + # way the old cosine formula did — that was the bug. + far = _distance_to_similarity(5.0, "l2") + near = _distance_to_similarity(1.0, "l2") + assert 0.0 < far < near < 1.0 + + +def test_l2_distance_above_one_keeps_signal(): + # The regression this fixes: under cosine, d=1.7 -> 0.0 (no signal). + # Under a correctly-declared L2 metric, it stays positive and ordered. + assert _distance_to_similarity(1.7, "cosine") == 0.0 + assert _distance_to_similarity(1.7, "l2") > 0.0 + + +def test_ip_conversion_monotonic_decreasing(): + # Inner-product distance is signed/unbounded (lower = closer). Logistic + # squash keeps it in (0, 1) and monotonic. + assert _distance_to_similarity(-5.0, "ip") > _distance_to_similarity(0.0, "ip") + assert _distance_to_similarity(0.0, "ip") == pytest.approx(0.5) + assert _distance_to_similarity(0.0, "ip") > _distance_to_similarity(5.0, "ip") + + +def test_ip_does_not_overflow_on_large_distance(): + # Exponent is clamped so a huge positive distance can't raise OverflowError. + val = _distance_to_similarity(1e6, "ip") + assert val == pytest.approx(0.0, abs=1e-9) + assert not math.isinf(val) and not math.isnan(val) + + +def test_none_distance_maps_to_zero(): + # BM25-only candidates carry distance=None -> no vector signal. + assert _distance_to_similarity(None, "cosine") == 0.0 + assert _distance_to_similarity(None, "l2") == 0.0 + + +def test_unknown_metric_falls_back_to_cosine(): + assert _distance_to_similarity(0.3, "weird") == _distance_to_similarity(0.3, "cosine") + assert _distance_to_similarity(0.3, None) == _distance_to_similarity(0.3, "cosine") + + +# --------------------------------------------------------------------------- +# _metric_for_collection — resolution + delegation + safety +# --------------------------------------------------------------------------- + + +def test_metric_resolver_reads_declared_metric(): + col = types.SimpleNamespace(distance_metric="l2") + assert _metric_for_collection(col) == "l2" + + +def test_metric_resolver_normalizes_case_and_garbage(): + assert _metric_for_collection(types.SimpleNamespace(distance_metric="L2")) == "l2" + assert _metric_for_collection(types.SimpleNamespace(distance_metric="nonsense")) == "cosine" + assert _metric_for_collection(types.SimpleNamespace(distance_metric=None)) == "cosine" + + +def test_metric_resolver_defaults_when_absent(): + assert _metric_for_collection(object()) == "cosine" + + +def test_metric_resolver_follows_embeddingcollection_delegation(): + inner = types.SimpleNamespace(distance_metric="ip") + + class _Wrapper: + def __init__(self, i): + self._i = i + + def __getattr__(self, name): + return getattr(self._i, name) + + assert _metric_for_collection(_Wrapper(inner)) == "ip" + + +def test_metric_resolver_survives_raising_attribute(): + class _Boom: + @property + def distance_metric(self): + raise RuntimeError("backend down") + + assert _metric_for_collection(_Boom()) == "cosine" + + +def test_real_embeddingcollection_delegates_metric_not_shadowed(): + # Regression: BaseCollection defines distance_metric as a property, so on + # the real EmbeddingCollection subclass it resolves directly and + # __getattr__ never fires. Without an explicit override the wrapper would + # report the base "cosine" default and mask a wrapped non-cosine backend. + from mempalace.backends.embedding_wrapper import EmbeddingCollection + + class _Inner(BaseCollection): + distance_metric = "l2" + + def add(self, **k): ... + def upsert(self, **k): ... + def query(self, **k): ... + def get(self, **k): ... + def delete(self, **k): ... + def count(self): + return 0 + + wrapped = EmbeddingCollection(_Inner()) + assert wrapped.distance_metric == "l2" + assert _metric_for_collection(wrapped) == "l2" + + +# --------------------------------------------------------------------------- +# ChromaCollection — legacy L2 palace reports its real metric +# --------------------------------------------------------------------------- + + +def _chroma_col_with_metadata(meta): + fake_inner = types.SimpleNamespace(metadata=meta) + return ChromaCollection(fake_inner) + + +def test_chroma_reports_cosine_when_set(): + assert _chroma_col_with_metadata({"hnsw:space": "cosine"}).distance_metric == "cosine" + + +def test_chroma_legacy_l2_palace_reports_l2(): + # A pre-cosine palace: the property surfaces the real space so the + # searcher maps distances correctly instead of flooring to 0. + assert _chroma_col_with_metadata({"hnsw:space": "l2"}).distance_metric == "l2" + + +def test_chroma_missing_or_unknown_metadata_reports_l2(): + # Absent/empty/garbage hnsw:space means the collection never had cosine + # set, so it is genuinely using Chroma's HNSW default (L2). Reporting + # cosine here would reintroduce the floor-to-0 bug this fixes. + assert _chroma_col_with_metadata({}).distance_metric == "l2" + assert _chroma_col_with_metadata({"hnsw:space": ""}).distance_metric == "l2" + assert _chroma_col_with_metadata({"hnsw:space": "bogus"}).distance_metric == "l2" + + +# --------------------------------------------------------------------------- +# _hybrid_rank — ranking actually respects the metric +# --------------------------------------------------------------------------- + + +def test_hybrid_rank_l2_keeps_far_candidate_ranked_above_unknown(): + # Two candidates with identical (zero) lexical overlap to the query, so + # only the vector term decides. Under cosine, a d=1.6 hit floors to 0 and + # ties a distance-None hit; under L2 it stays positive and ranks above. + results = [ + {"text": "alpha", "distance": None}, + {"text": "beta", "distance": 1.6}, + ] + ranked = _hybrid_rank(results, "zzzznomatch", metric="l2") + assert ranked[0]["text"] == "beta" # real vector signal beats vector-unknown + + +def test_hybrid_rank_cosine_unchanged_behavior(): + # Cosine path must be byte-for-byte the old behavior (max(0, 1-d)). + results = [ + {"text": "near", "distance": 0.1}, + {"text": "far", "distance": 0.9}, + ] + ranked = _hybrid_rank(results, "zzzznomatch", metric="cosine") + assert ranked[0]["text"] == "near" + assert ranked[1]["text"] == "far" + + +def test_hybrid_rank_empty_is_noop(): + assert _hybrid_rank([], "q", metric="l2") == [] diff --git a/tests/test_embedder_identity.py b/tests/test_embedder_identity.py new file mode 100644 index 0000000..93ddae0 --- /dev/null +++ b/tests/test_embedder_identity.py @@ -0,0 +1,408 @@ +"""Embedder-identity persistence and three-state enforcement (RFC 001). + +A same-dimension model swap (e.g. two 384-d models) silently corrupts +retrieval on the explicit-embedding backends, which have no native model +check. The contract records the model name and refuses a swap on open. These +tests avoid loading any embedding model: the enforcement *check* path needs +only the configured model name (cheap), and persistence is exercised with +``EmbedderIdentity`` objects and explicit vectors. +""" + +import os +import warnings + +import pytest + +from mempalace.backends.base import ( + DimensionMismatchError, + EmbedderIdentity, + EmbedderIdentityMismatchError, + EmbedderIdentityUnknownWarning, + PalaceRef, + check_embedder_identity, +) + + +# --------------------------------------------------------------------------- +# Three-state helper +# --------------------------------------------------------------------------- + + +def test_unknown_when_nothing_stored(): + assert check_embedder_identity(None, EmbedderIdentity("minilm", 384)) == "unknown" + + +def test_unknown_when_current_is_nameless(): + stored = EmbedderIdentity("minilm", 384) + assert check_embedder_identity(stored, EmbedderIdentity("", 384)) == "unknown" + assert check_embedder_identity(stored, None) == "unknown" + + +def test_known_match(): + a = EmbedderIdentity("minilm", 384) + assert check_embedder_identity(a, a) == "known_match" + + +def test_match_skips_unknown_dimension(): + # dimension 0 means "not probed" and must not be treated as a real conflict. + assert ( + check_embedder_identity(EmbedderIdentity("minilm", 384), EmbedderIdentity("minilm", 0)) + == "known_match" + ) + + +def test_model_swap_raises_identity_error(): + with pytest.raises(EmbedderIdentityMismatchError): + check_embedder_identity(EmbedderIdentity("minilm", 384), EmbedderIdentity("gemma", 384)) + + +def test_dimension_change_raises_dimension_error_first(): + # Width change is physically unusable — checked before the name swap. + with pytest.raises(DimensionMismatchError): + check_embedder_identity(EmbedderIdentity("a", 384), EmbedderIdentity("b", 768)) + + +def test_force_returns_mismatch_without_raising(): + assert ( + check_embedder_identity( + EmbedderIdentity("minilm", 384), + EmbedderIdentity("gemma", 384), + force_model_swap=True, + ) + == "known_mismatch" + ) + + +# --------------------------------------------------------------------------- +# Per-backend persistence roundtrip (no model loads) +# --------------------------------------------------------------------------- + + +def _sqlite_collection(tmp_path): + from mempalace.backends.sqlite_exact import SQLiteExactBackend + + backend = SQLiteExactBackend() + ref = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + return backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True) + + +def _chroma_collection(tmp_path): + from mempalace.backends.chroma import ChromaBackend + + backend = ChromaBackend() + ref = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + return backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True) + + +def test_sqlite_identity_roundtrip(tmp_path): + col = _sqlite_collection(tmp_path) + assert col.get_stored_embedder_identity() is None + col.add(documents=["x"], ids=["a"], metadatas=[{}], embeddings=[[0.1, 0.2, 0.3, 0.4]]) + col.set_embedder_identity(EmbedderIdentity("minilm", 4)) + got = col.get_stored_embedder_identity() + assert got is not None and got.model_name == "minilm" and got.dimension == 4 + + +def test_sqlite_set_identity_ignores_nameless(tmp_path): + col = _sqlite_collection(tmp_path) + col.add(documents=["x"], ids=["a"], metadatas=[{}], embeddings=[[0.1, 0.2, 0.3, 0.4]]) + col.set_embedder_identity(EmbedderIdentity("", 4)) + assert col.get_stored_embedder_identity() is None + + +def test_chroma_identity_roundtrip_via_sidecar(tmp_path): + col = _chroma_collection(tmp_path) + assert col.get_stored_embedder_identity() is None + col.set_embedder_identity(EmbedderIdentity("minilm", 384)) + got = col.get_stored_embedder_identity() + assert got is not None and got.model_name == "minilm" and got.dimension == 384 + assert os.path.isfile(os.path.join(str(tmp_path), "mempalace_embedder.json")) + + +def test_pgvector_identity_survives_marker_rewrite(tmp_path): + # Identity lives in a sidecar, separate from the mismatch marker, so a + # marker rebuild (which happens on every write) must not affect it. + from mempalace.backends.pgvector import PgVectorBackend, _PgVectorConfig + + backend = PgVectorBackend() + cfg = _PgVectorConfig(dsn="postgresql://example", namespace=None) + ref = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + # No marker needed to record identity — the sidecar is unguarded. + backend._set_embedder_identity(ref, "mempalace_drawers", EmbedderIdentity("minilm", 384)) + backend._write_marker(ref, cfg) + got = backend._get_embedder_identity(ref, "mempalace_drawers") + assert got is not None and got.model_name == "minilm" and got.dimension == 384 + + +def test_embeddingcollection_delegates_identity_not_shadowed(): + # BaseCollection defines these as concrete methods, so __getattr__ never + # delegates them — the wrapper needs explicit forwarding or it silently + # reports the no-op default and masks the wrapped backend's identity. + from mempalace.backends.base import BaseCollection + from mempalace.backends.embedding_wrapper import EmbeddingCollection + + class _Inner(BaseCollection): + def __init__(self): + self._ident = None + + def add(self, **k): ... + def upsert(self, **k): ... + def query(self, **k): ... + def get(self, **k): ... + def delete(self, **k): ... + def count(self): + return 0 + + def get_stored_embedder_identity(self): + return self._ident + + def set_embedder_identity(self, identity): + self._ident = identity + + inner = _Inner() + wrapped = EmbeddingCollection(inner) + wrapped.set_embedder_identity(EmbedderIdentity("minilm", 384)) + assert inner._ident is not None and inner._ident.model_name == "minilm" + assert wrapped.get_stored_embedder_identity().model_name == "minilm" + + +# --------------------------------------------------------------------------- +# Enforcement via palace.get_collection (sqlite_exact, no model load) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def clear_identity_cache(): + from mempalace import palace + + palace._VALIDATED_IDENTITY.clear() + yield + palace._VALIDATED_IDENTITY.clear() + + +def _seed_sqlite_with_identity(tmp_path, model): + col = _sqlite_collection(tmp_path) + col.add(documents=["x"], ids=["a"], metadatas=[{}], embeddings=[[0.1, 0.2, 0.3, 0.4]]) + if model is not None: + col.set_embedder_identity(EmbedderIdentity(model, 4)) + return col + + +def test_enforcement_match_does_not_raise(tmp_path, monkeypatch, clear_identity_cache): + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm") + monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact") + from mempalace import palace as P + + _seed_sqlite_with_identity(tmp_path, "minilm") + P._VALIDATED_IDENTITY.clear() + # Should not raise. + P.get_collection(str(tmp_path), collection_name="mempalace_drawers", create=False) + + +def test_enforcement_model_swap_raises(tmp_path, monkeypatch, clear_identity_cache): + monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact") + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm") + from mempalace import palace as P + + _seed_sqlite_with_identity(tmp_path, "minilm") + P._VALIDATED_IDENTITY.clear() + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "embeddinggemma") + with pytest.raises(EmbedderIdentityMismatchError): + P.get_collection(str(tmp_path), collection_name="mempalace_drawers", create=False) + + +def test_enforcement_brand_new_records_current_model(tmp_path, monkeypatch, clear_identity_cache): + monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact") + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm") + from mempalace import palace as P + + col = P.get_collection(str(tmp_path), collection_name="mempalace_drawers", create=True) + got = col.get_stored_embedder_identity() + assert got is not None and got.model_name == "minilm" + + +def test_enforcement_legacy_with_data_warns(tmp_path, monkeypatch, clear_identity_cache): + monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact") + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm") + from mempalace import palace as P + + _seed_sqlite_with_identity(tmp_path, None) # data, but no recorded identity + P._VALIDATED_IDENTITY.clear() + with pytest.warns(EmbedderIdentityUnknownWarning): + P.get_collection(str(tmp_path), collection_name="mempalace_drawers", create=False) + + +def test_enforcement_nameless_model_is_a_noop(tmp_path, monkeypatch, clear_identity_cache): + monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact") + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm") + from mempalace import palace as P + + _seed_sqlite_with_identity(tmp_path, None) + P._VALIDATED_IDENTITY.clear() + # A nameless current embedder cannot enforce — no raise, no warning. + monkeypatch.setattr("mempalace.embedding.current_model_name", lambda model=None: "") + with warnings.catch_warnings(): + warnings.simplefilter("error") + P.get_collection(str(tmp_path), collection_name="mempalace_drawers", create=False) + + +# --------------------------------------------------------------------------- +# set_palace_embedder_identity override path +# --------------------------------------------------------------------------- + + +def test_set_palace_identity_override_requires_force(tmp_path, monkeypatch, clear_identity_cache): + monkeypatch.setenv("MEMPALACE_BACKEND", "sqlite_exact") + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm") + from mempalace import palace as P + + _seed_sqlite_with_identity(tmp_path, "minilm") + # Recording a different model without force is refused. + with pytest.raises(EmbedderIdentityMismatchError): + P.set_palace_embedder_identity(str(tmp_path), model="embeddinggemma", force=False) + # With force it goes through, recording the name only (no foreign load). + old, new = P.set_palace_embedder_identity(str(tmp_path), model="embeddinggemma", force=True) + assert old.model_name == "minilm" and new.model_name == "embeddinggemma" + + +def test_set_palace_identity_empty_target_raises(tmp_path, monkeypatch): + # No model given and none configured: recording is a no-op in every backend, + # so refuse rather than claim a phantom success. + monkeypatch.setattr("mempalace.config.MempalaceConfig.embedding_model", property(lambda s: "")) + from mempalace import palace as P + + with pytest.raises(ValueError): + P.set_palace_embedder_identity(str(tmp_path), model=None) + + +def test_enforcement_prefers_effective_identity(monkeypatch, clear_identity_cache): + # A server_embedder collection reports its own effective identity; the + # configured model must be ignored in favor of it. Here effective and + # stored disagree, so enforcement raises even though config says "minilm". + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "minilm") + from mempalace import palace as P + + class _ServerCol: + def effective_embedder_identity(self): + return EmbedderIdentity("server-model", 768) + + def get_stored_embedder_identity(self): + return EmbedderIdentity("other-model", 768) + + def count(self): + return 5 + + def set_embedder_identity(self, identity): + raise AssertionError("must not record on a mismatch") + + with pytest.raises(EmbedderIdentityMismatchError): + P._enforce_embedder_identity(_ServerCol(), "/tmp/x", "c", create=False) + + +def test_chroma_corrupt_sidecar_returns_none(tmp_path): + # A malformed sidecar (non-dict JSON) must not raise — degrade to unknown. + col = _chroma_collection(tmp_path) + path = os.path.join(str(tmp_path), "mempalace_embedder.json") + with open(path, "w", encoding="utf-8") as f: + f.write('["not", "a", "dict"]') + assert col.get_stored_embedder_identity() is None + # And a subsequent set still works (overwrites the junk). + col.set_embedder_identity(EmbedderIdentity("minilm", 384)) + assert col.get_stored_embedder_identity().model_name == "minilm" + + +# --------------------------------------------------------------------------- +# qdrant: identity persisted in the local marker (no live qdrant needed) +# --------------------------------------------------------------------------- + + +def _qdrant_collection(tmp_path, *, write_marker=True): + from mempalace.backends.qdrant import QdrantBackend, QdrantCollection, _QdrantConfig + + backend = QdrantBackend() + config = _QdrantConfig(url="http://localhost:6333", api_key=None, namespace=None) + ref = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + if write_marker: + backend._write_marker(ref, config) + # The identity methods read/write the local marker only; the client is + # never touched, so a placeholder stands in for a live REST connection. + return QdrantCollection( + backend=backend, + client=object(), + config=config, + palace=ref, + collection_name="mempalace_drawers", + remote_collection="mp_drawers_remote", + ) + + +def test_qdrant_identity_survives_marker_rewrite(tmp_path): + from mempalace.backends.qdrant import QdrantBackend, _QdrantConfig + + backend = QdrantBackend() + config = _QdrantConfig(url="http://localhost:6333", api_key=None, namespace=None) + ref = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + backend._write_marker(ref, config) + backend._set_embedder_identity(ref, "mempalace_drawers", EmbedderIdentity("minilm", 384)) + backend._write_marker(ref, config) # rebuild must not wipe embedders + got = backend._get_embedder_identity(ref, "mempalace_drawers") + assert got is not None and got.model_name == "minilm" and got.dimension == 384 + + +def test_qdrant_collection_delegates_identity(tmp_path): + col = _qdrant_collection(tmp_path) + assert col.get_stored_embedder_identity() is None + col.set_embedder_identity(EmbedderIdentity("minilm", 384)) + got = col.get_stored_embedder_identity() + assert got is not None and got.model_name == "minilm" and got.dimension == 384 + + +def test_qdrant_set_identity_creates_sidecar_when_missing(tmp_path): + # Brand-new palace whose first write hasn't created the marker yet: + # recording identity must create it, not silently no-op into permanent + # "unknown" (the marker-on-write vs record-on-open timing gap). + col = _qdrant_collection(tmp_path, write_marker=False) + assert not col._marker_exists() + col.set_embedder_identity(EmbedderIdentity("minilm", 384)) + got = col.get_stored_embedder_identity() + assert got is not None and got.model_name == "minilm" + + +def _pgvector_collection(tmp_path, *, write_marker=True): + from mempalace.backends.pgvector import PgVectorBackend, PgVectorCollection, _PgVectorConfig + + backend = PgVectorBackend() + config = _PgVectorConfig(dsn="postgresql://example", namespace=None) + ref = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + if write_marker: + backend._write_marker(ref, config) + return PgVectorCollection( + backend=backend, + client=object(), + config=config, + palace=ref, + collection_name="mempalace_drawers", + table="mp_drawers_t", + ) + + +def test_pgvector_set_identity_creates_sidecar_when_missing(tmp_path): + # Same brand-new-palace timing gap as qdrant: recording must create the + # marker rather than no-op. + col = _pgvector_collection(tmp_path, write_marker=False) + assert not col._marker_exists() + col.set_embedder_identity(EmbedderIdentity("minilm", 384)) + got = col.get_stored_embedder_identity() + assert got is not None and got.model_name == "minilm" + + +def test_qdrant_enforcement_model_swap_raises(tmp_path, monkeypatch, clear_identity_cache): + # The enforcement check reads the marker (no server) and compares to the + # configured model — a swap raises just like the local backends. + from mempalace import palace as P + + col = _qdrant_collection(tmp_path) + col.set_embedder_identity(EmbedderIdentity("minilm", 384)) + monkeypatch.setenv("MEMPALACE_EMBEDDING_MODEL", "embeddinggemma") + with pytest.raises(EmbedderIdentityMismatchError): + P._enforce_embedder_identity(col, str(tmp_path), "mempalace_drawers", create=False) diff --git a/tests/test_embedding_wrapper.py b/tests/test_embedding_wrapper.py new file mode 100644 index 0000000..5d592d6 --- /dev/null +++ b/tests/test_embedding_wrapper.py @@ -0,0 +1,126 @@ +"""EmbeddingCollection OneOrMany handling (PR #1706 review). + +A bare ``str`` passed as ``documents``/``query_texts`` (ChromaDB's OneOrMany +shape) must be wrapped, not iterated — otherwise ``list("abc")`` embeds per +character and breaks length alignment with ids/metadatas on explicit-vector +backends. +""" + +from mempalace.backends import embedding_wrapper as ew + + +class _FakeInner: + """Captures what the wrapper delegates to the backend.""" + + def __init__(self): + self.calls = {} + + def add(self, *, documents, ids, metadatas=None, embeddings=None): + self.calls["add"] = { + "documents": documents, + "ids": ids, + "metadatas": metadatas, + "embeddings": embeddings, + } + + def upsert(self, *, documents, ids, metadatas=None, embeddings=None): + self.calls["upsert"] = { + "documents": documents, + "ids": ids, + "metadatas": metadatas, + "embeddings": embeddings, + } + + def update(self, *, ids, documents=None, metadatas=None, embeddings=None): + self.calls["update"] = { + "documents": documents, + "ids": ids, + "metadatas": metadatas, + "embeddings": embeddings, + } + + def query(self, *, query_texts=None, query_embeddings=None, **_kw): + self.calls["query"] = {"query_texts": query_texts, "query_embeddings": query_embeddings} + from mempalace.backends.base import QueryResult + + return QueryResult.empty() + + +def _patch_embed(monkeypatch): + """Stub the embedder: one vector per input text, recording the inputs.""" + seen = {} + + def fake(texts): + seen["texts"] = texts + return [[0.0, 0.0] for _ in texts] + + monkeypatch.setattr(ew, "_embed_texts", fake) + return seen + + +def test_as_list_wraps_bare_string(): + assert ew._as_list("hello world") == ["hello world"] + assert ew._as_list({"k": 1}) == [{"k": 1}] # bare dict wrapped, not -> ["k"] + src = ["a", "b"] + assert ew._as_list(src) is src # list returned as-is (no copy) + assert ew._as_list(("a", "b")) == ["a", "b"] # other iterables materialized + + +def test_add_wraps_bare_string_document(monkeypatch): + seen = _patch_embed(monkeypatch) + inner = _FakeInner() + ew.EmbeddingCollection(inner).add(documents="hello world", ids=["d1"]) + # embedded as one whole document, not per character + assert seen["texts"] == ["hello world"] + # and the backend receives a list, length-aligned with ids + assert inner.calls["add"]["documents"] == ["hello world"] + assert len(inner.calls["add"]["embeddings"]) == 1 + + +def test_upsert_wraps_bare_string_document(monkeypatch): + seen = _patch_embed(monkeypatch) + inner = _FakeInner() + ew.EmbeddingCollection(inner).upsert(documents="solo", ids=["d1"]) + assert seen["texts"] == ["solo"] + assert inner.calls["upsert"]["documents"] == ["solo"] + assert len(inner.calls["upsert"]["embeddings"]) == 1 + + +def test_update_wraps_bare_string_document(monkeypatch): + seen = _patch_embed(monkeypatch) + inner = _FakeInner() + ew.EmbeddingCollection(inner).update(ids=["d1"], documents="changed") + assert seen["texts"] == ["changed"] + assert inner.calls["update"]["documents"] == ["changed"] + assert len(inner.calls["update"]["embeddings"]) == 1 + + +def test_query_wraps_bare_string(monkeypatch): + seen = _patch_embed(monkeypatch) + inner = _FakeInner() + ew.EmbeddingCollection(inner).query(query_texts="find me") + assert seen["texts"] == ["find me"] + # query_texts is consumed into a single query embedding + assert len(inner.calls["query"]["query_embeddings"]) == 1 + assert inner.calls["query"]["query_texts"] is None + + +def test_list_inputs_unaffected(monkeypatch): + seen = _patch_embed(monkeypatch) + inner = _FakeInner() + ew.EmbeddingCollection(inner).add(documents=["one", "two"], ids=["a", "b"]) + assert seen["texts"] == ["one", "two"] + assert len(inner.calls["add"]["embeddings"]) == 2 + + +def test_add_wraps_bare_string_ids_and_dict_metadatas(monkeypatch): + _patch_embed(monkeypatch) + inner = _FakeInner() + # a single id (str) and a single metadata (dict) are OneOrMany shapes too + ew.EmbeddingCollection(inner).add(documents="solo", ids="d1", metadatas={"src": "web"}) + call = inner.calls["add"] + assert call["ids"] == ["d1"] # not ['d', '1'] + assert call["metadatas"] == [{"src": "web"}] # not ['src'] + # documents / embeddings / ids / metadatas all length-aligned at 1 + assert call["documents"] == ["solo"] + assert len(call["embeddings"]) == 1 diff --git a/tests/test_embeddinggemma.py b/tests/test_embeddinggemma.py index 3100fce..f108ff9 100644 --- a/tests/test_embeddinggemma.py +++ b/tests/test_embeddinggemma.py @@ -126,7 +126,7 @@ def test_lazy_load_runs_once(patched_lazy_load): ef(["one"]) ef(["two"]) ef(["three"]) - assert patched_lazy_load["hf_hub_download"] == 2 # model + tokenizer, once total + assert patched_lazy_load["hf_hub_download"] == 3 # model + weights + tokenizer, once assert patched_lazy_load["InferenceSession"] == 1 assert patched_lazy_load["Tokenizer.from_file"] == 1 diff --git a/tests/test_entity_detector.py b/tests/test_entity_detector.py index 11278f9..cc74831 100644 --- a/tests/test_entity_detector.py +++ b/tests/test_entity_detector.py @@ -166,6 +166,148 @@ def test_coca_wordlist_contains_all_known_aya_false_positives(): assert not missing, f"COCA wordlist missing known false positives: {missing!r}" +# ── Known-systems lexicon + compound matcher (Tier 3 linguistics cleanup) ─ + + +def test_extract_candidates_detects_claude_code_as_atomic_compound(): + """The flagship Tier 3 case: 'Claude Code' must be detected as a single + atomic compound, NOT decomposed into 'Claude' + 'Code' (where 'Code' + would then get COCA-filtered, leaving 'Claude' alone with wrong count). + """ + text = ( + "Claude Code helped fix the bug. Claude Code refactored the loop. " + "Claude Code wrote the tests. Claude Code shipped clean." + ) + result = extract_candidates(text) + assert "Claude Code" in result, ( + f"'Claude Code' must be detected as a single compound; got: {list(result.keys())!r}" + ) + assert result["Claude Code"] >= 4 + + +def test_extract_candidates_does_not_decompose_known_compound(): + """When 'Claude Code' appears as a compound, the single-word 'Claude' + and 'Code' must NOT appear in results (would be wrong attribution + + COCA-filtered respectively).""" + text = ( + "Claude Code wrote the patch. Claude Code reviewed it. " + "Claude Code ran the tests. Claude Code merged. Claude Code shipped." + ) + result = extract_candidates(text) + assert "Claude Code" in result + assert "Claude" not in result, ( + f"'Claude' alone should NOT appear when only mentioned as part of " + f"'Claude Code'; got: {list(result.keys())!r}" + ) + # 'Code' would be COCA-filtered anyway, but assert defensively. + assert "Code" not in result + + +def test_extract_candidates_compound_matching_is_case_insensitive(): + """'claude code' (lowercase) and 'CLAUDE CODE' (uppercase) and mixed + case all match the canonical 'Claude Code' compound.""" + text = ( + "claude code is great. CLAUDE CODE works. Claude Code rocks. " + "Claude code rules. CLaudE coDe wins." + ) + result = extract_candidates(text) + assert "Claude Code" in result, ( + f"Case-insensitive compound matching must work; got: {list(result.keys())!r}" + ) + + +def test_extract_candidates_real_single_word_name_still_detected(): + """Tier 3 does NOT regress single-word proper-noun detection. + 'Aya' appearing as a standalone name 3+ times still gets detected.""" + text = "Aya wrote the spec. Aya reviewed Cedar's PR. Aya shipped to develop." + result = extract_candidates(text) + assert "Aya" in result, ( + f"Single-word real name 'Aya' should still be detected; got: {list(result.keys())!r}" + ) + + +def test_extract_candidates_single_word_code_still_coca_filtered(): + """Tier 3 must not regress Tier 2. 'Code' alone is still a content + word and filtered by COCA, regardless of the compound pre-pass.""" + text = "Code is fun. Code works. Code helps. Code rules. Code wins." + result = extract_candidates(text) + assert "Code" not in result, ( + f"'Code' standalone should still be COCA-filtered; got: {list(result.keys())!r}" + ) + + +def test_extract_candidates_detects_multiple_distinct_compounds(): + """Multiple known compounds in the same text are each detected + independently.""" + text = ( + "Claude Code wrote the code. Claude Code reviewed.\n" + "GitHub Copilot suggested a fix. GitHub Copilot autocompleted.\n" + "Claude Code merged. GitHub Copilot helped.\n" + ) + result = extract_candidates(text) + assert "Claude Code" in result, f"'Claude Code' missing; got: {list(result.keys())!r}" + assert "GitHub Copilot" in result, f"'GitHub Copilot' missing; got: {list(result.keys())!r}" + + +def test_known_systems_file_loads_with_expected_shape(): + """The data file ships with a stable schema.""" + import json + from pathlib import Path + import mempalace + + pkg_dir = Path(mempalace.__file__).parent + p = pkg_dir / "data" / "known_systems.json" + assert p.exists(), f"known_systems.json must exist at {p}" + d = json.loads(p.read_text(encoding="utf-8")) + assert d.get("schema_version") == 1, ( + f"schema_version must be 1; got {d.get('schema_version')!r}" + ) + compounds = d.get("compounds") + assert isinstance(compounds, list), ( + f"'compounds' must be a list; got {type(compounds).__name__}" + ) + assert len(compounds) >= 20, f"known_systems must have >=20 entries; got {len(compounds)}" + # Every entry must be multi-token (space or hyphen separated) + single_word = [c for c in compounds if " " not in c and "-" not in c] + assert not single_word, ( + f"All entries must be multi-token compounds; single-word entries found: {single_word[:5]!r}" + ) + + +def test_known_systems_file_contains_expected_high_value_entries(): + """The file must contain the most-common AI/dev product compounds + users mention in real palaces.""" + import json + from pathlib import Path + import mempalace + + pkg_dir = Path(mempalace.__file__).parent + p = pkg_dir / "data" / "known_systems.json" + compounds = set(json.loads(p.read_text(encoding="utf-8"))["compounds"]) + must_have = [ + "Claude Code", + "GitHub Copilot", + "Visual Studio Code", + "Gemini Code Assist", + "Docker Desktop", + "GitHub Actions", + ] + missing = [c for c in must_have if c not in compounds] + assert not missing, f"known_systems missing high-value entries: {missing!r}" + + +def test_extract_candidates_unknown_two_word_phrase_still_works(): + """Tier 3 does NOT regress the multi-word regex path. A two-word + proper-noun phrase NOT in known_systems still gets detected via + the existing multi-word pattern.""" + text = "Jane Smith wrote it. Jane Smith reviewed. Jane Smith shipped. Jane Smith merged." + result = extract_candidates(text) + assert "Jane Smith" in result, ( + f"Unknown compound 'Jane Smith' should still be detected via " + f"multi-word regex; got: {list(result.keys())!r}" + ) + + # ── score_entity ──────────────────────────────────────────────────────── diff --git a/tests/test_format_miner.py b/tests/test_format_miner.py index a9ed816..5f998d0 100644 --- a/tests/test_format_miner.py +++ b/tests/test_format_miner.py @@ -801,6 +801,36 @@ def test_mine_formats_respects_limit(_mine_formats_mocks): assert p_md.call_count == 2 +def test_mine_formats_limit_skips_already_mined(_mine_formats_mocks): + """--limit N counts only new work, not already-mined skips (#1535).""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + files = [] + for i in range(6): + p = tmp / f"f{i}.pdf" + p.write_bytes(b"%PDF-1.4 stub") + files.append(p) + + call_idx = 0 + + def fake_already_mined(collection, source_file, **kwargs): + nonlocal call_idx + call_idx += 1 + return call_idx <= 4 + + _mine_formats_mocks["file_already_mined"].side_effect = fake_already_mined + with ( + patch("mempalace.format_miner.scan_formats", return_value=files), + patch( + "mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50 + ) as p_md, + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), limit=1) + assert p_md.call_count == 1 + + def test_mine_formats_wing_defaults_from_directory_name(_mine_formats_mocks): """When wing=None, the directory's basename becomes the wing.""" from unittest.mock import patch diff --git a/tests/test_hallways.py b/tests/test_hallways.py index 94b7ed2..92ba186 100644 --- a/tests/test_hallways.py +++ b/tests/test_hallways.py @@ -27,11 +27,21 @@ def _use_tmp_hallway_file(monkeypatch, tmp_path): def _fake_collection(drawers): - """Build a MagicMock collection whose .get() returns the given drawer set.""" + """Build a MagicMock collection over ``drawers`` that supports the paginated + fetch (``count()`` + ``get(limit=, offset=)``) that compute_hallways_for_wing + uses to stay under SQLite's variable limit (#1619).""" col = MagicMock() - metadatas = [d for d in drawers] - ids = [f"drawer_{i}" for i in range(len(drawers))] - col.get.return_value = {"ids": ids, "metadatas": metadatas} + metas = [d for d in drawers] + col.count.return_value = len(metas) + + def _get(limit=None, offset=0, include=None, where=None, ids=None, **kwargs): + page = metas[offset : offset + limit] if limit is not None else metas + return { + "ids": [f"drawer_{i}" for i in range(offset, offset + len(page))], + "metadatas": page, + } + + col.get.side_effect = _get return col diff --git a/tests/test_hallways_pagination.py b/tests/test_hallways_pagination.py new file mode 100644 index 0000000..8847071 --- /dev/null +++ b/tests/test_hallways_pagination.py @@ -0,0 +1,56 @@ +"""Regression test for #1619. + +``compute_hallways_for_wing`` must fetch drawers by paginating +(``count()`` + ``get(limit=, offset=)``) and filtering the wing client-side, +NOT with a single ``get(where={"wing": wing})`` — the latter binds one SQL +variable per matched id and overflows SQLite's ``SQLITE_MAX_VARIABLE_NUMBER`` +(32766) on wings larger than ~32k drawers, silently leaving the hallway graph +unbuilt on exactly the large wings that benefit most. +""" + +from unittest.mock import MagicMock, patch + +with patch.dict("sys.modules", {"chromadb": MagicMock()}): + from mempalace import hallways as hallways_mod + + +def _use_tmp_hallway_file(monkeypatch, tmp_path): + monkeypatch.setattr(hallways_mod, "_HALLWAY_FILE", str(tmp_path / "hallways.json")) + + +def _collection_that_rejects_where_get(drawers): + """count() + paginated get(limit,offset) work; a where-get raises, exactly + as ChromaDB does when the bound-variable count overflows on a big wing.""" + col = MagicMock() + col.count.return_value = len(drawers) + + def _get(limit=None, offset=0, include=None, where=None, ids=None, **kw): + if where is not None and limit is None: + raise RuntimeError("Error executing plan: too many SQL variables") + filtered_drawers = drawers + if where and "wing" in where: + target_wing = where["wing"] + filtered_drawers = [ + d for d in drawers if isinstance(d, dict) and d.get("wing") == target_wing + ] + page = filtered_drawers[offset : offset + limit] if limit is not None else filtered_drawers + return { + "ids": [f"d{i}" for i in range(offset, offset + len(page))], + "metadatas": page, + } + + col.get.side_effect = _get + return col + + +class TestComputeHallwaysPagination: + def test_large_wing_builds_hallways_via_pagination(self, tmp_path, monkeypatch): + _use_tmp_hallway_file(monkeypatch, tmp_path) + # 3 drawers all co-placing Alice+Bob → one hallway at min_count=2, + # but ONLY if the fetch paginates instead of the variable-bound where-get. + drawers = [{"wing": "wing_alpha", "room": "diary", "entities": "Alice;Bob"}] * 3 + col = _collection_that_rejects_where_get(drawers) + result = hallways_mod.compute_hallways_for_wing("wing_alpha", col=col) + assert any({h["entity_a"], h["entity_b"]} == {"Alice", "Bob"} for h in result), ( + "hallways came back empty — the where-get path crashed; the fetch must paginate (#1619)" + ) diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 723238d..e09e2ba 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -13,6 +13,7 @@ import mempalace.hooks_cli as hooks_cli_mod from mempalace.hooks_cli import ( SAVE_INTERVAL, _count_human_messages, + _diary_agent_for_harness, _extract_recent_messages, _get_mine_targets, _log, @@ -22,6 +23,7 @@ from mempalace.hooks_cli import ( _mine_sync, _parse_harness_input, _sanitize_session_id, + _save_diary_direct, _validate_transcript_path, _wing_from_transcript_path, hook_stop, @@ -336,7 +338,9 @@ def test_stop_hook_saves_silently_at_interval(tmp_path): assert result["systemMessage"].startswith("\u2726 15 memories woven into the palace") assert "hooks" in result["systemMessage"] # tmp_path has no "-Projects-" segment, so _wing_from_transcript_path falls back to "wing_sessions" - mock_save.assert_called_once_with(str(transcript), "test", wing="wing_sessions", toast=False) + mock_save.assert_called_once_with( + str(transcript), "test", wing="wing_sessions", toast=False, agent_name="claude" + ) def test_stop_hook_derives_wing_from_transcript_path(tmp_path): @@ -355,7 +359,9 @@ def test_stop_hook_derives_wing_from_transcript_path(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - mock_save.assert_called_once_with(str(transcript), "test", wing="wing_myproject", toast=False) + mock_save.assert_called_once_with( + str(transcript), "test", wing="wing_myproject", toast=False, agent_name="claude" + ) def test_stop_hook_tracks_save_point(tmp_path): @@ -383,6 +389,84 @@ def test_stop_hook_tracks_save_point(tmp_path): mock_save.assert_not_called() +# --- #1693: hook checkpoints must be discoverable by diary_read --- + + +def test_diary_agent_for_harness_maps_known_harnesses(): + assert _diary_agent_for_harness("claude-code") == "claude" + assert _diary_agent_for_harness("codex") == "codex" + + +def test_diary_agent_for_harness_unknown_falls_back_to_name(): + """A future harness must never collapse to the legacy 'session-hook' + identity, which no diary_read(agent_name=...) call ever matches (#1693).""" + assert _diary_agent_for_harness("cursor") == "cursor" + for harness in ("claude-code", "codex", "cursor", "gemini"): + assert _diary_agent_for_harness(harness) != "session-hook" + + +@pytest.mark.parametrize( + "harness,expected_agent", + [("claude-code", "claude"), ("codex", "codex")], +) +def test_stop_hook_files_checkpoint_under_harness_agent(tmp_path, harness, expected_agent): + """The Stop hook must file checkpoints under the agent identity that the + session's harness reads with, not the legacy hardcoded 'session-hook' + (#1693).""" + # _save_diary_direct is mocked below, so the transcript format is irrelevant + # here: _count_human_messages counts both harness shapes, and we assert only + # the harness -> agent_name routing, not transcript parsing. + transcript = tmp_path / "t.jsonl" + _write_transcript( + transcript, + [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL)], + ) + with patch( + "mempalace.hooks_cli._save_diary_direct", return_value={"count": 5, "themes": []} + ) as mock_save: + _capture_hook_output( + hook_stop, + {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, + harness=harness, + state_dir=tmp_path, + ) + assert mock_save.call_args.kwargs["agent_name"] == expected_agent + + +def test_stop_hook_checkpoint_visible_to_diary_read(monkeypatch, config, palace_path, kg, tmp_path): + """End-to-end regression for #1693: a checkpoint written by the Stop hook + save path is discoverable via diary_read under the harness agent identity, + and is not siloed under the legacy 'session-hook' identity.""" + import chromadb + + from mempalace import mcp_server + from mempalace.mcp_server import tool_diary_read + + monkeypatch.setattr(mcp_server, "_config", config) + monkeypatch.setattr(mcp_server, "_get_kg", lambda *a, **kw: kg) + client = chromadb.PersistentClient(path=palace_path) + client.get_or_create_collection("mempalace_drawers", metadata={"hnsw:space": "cosine"}) + del client + + transcript = tmp_path / "session.jsonl" + _write_transcript( + transcript, + [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(5)], + ) + + agent = _diary_agent_for_harness("claude-code") + res = _save_diary_direct(str(transcript), "sess1", wing="wing_.claude", agent_name=agent) + assert res["count"] > 0 + + visible = tool_diary_read(agent_name="claude") + assert visible.get("total", 0) >= 1 + assert "CHECKPOINT" in visible["entries"][0]["content"] + + # The legacy identity no longer captures hook checkpoints. + legacy = tool_diary_read(agent_name="session-hook") + assert legacy.get("entries") == [] + + # --- hook_session_start --- diff --git a/tests/test_ids.py b/tests/test_ids.py new file mode 100644 index 0000000..1037602 --- /dev/null +++ b/tests/test_ids.py @@ -0,0 +1,186 @@ +"""Tests for mempalace.ids — collision-safe ID construction. + +The RED test that pins this whole PR is +``test_make_drawer_id_from_chunk_does_not_collide_across_boundary`` — it +constructs the classic ``"/path/a1" + "23" == "/path/a" + "123"`` collision +shape and asserts the new delimiter-based recipe produces distinct IDs. +Against the pre-v2 recipe (no delimiter), this test FAILS. Against v2, +it PASSES. +""" + +from __future__ import annotations + +import hashlib + +from mempalace import ids + + +# ── ID_RECIPE constant ───────────────────────────────────────────────── + + +def test_id_recipe_constant_is_v2(): + """Audit code reads ids.ID_RECIPE to tag new drawers. The constant + must be the literal "v2" string; a typo here silently re-introduces + the ambiguity v2 was meant to fix.""" + assert ids.ID_RECIPE == "v2" + + +# ── make_drawer_id_from_chunk ───────────────────────────────────────── + + +def test_make_drawer_id_from_chunk_returns_expected_prefix(): + """Drawer IDs are namespaced by wing and room so cross-wing + collisions are impossible regardless of the hash slice.""" + result = ids.make_drawer_id_from_chunk("proj", "log", "/a", 0) + assert result.startswith("drawer_proj_log_") + + +def test_make_drawer_id_from_chunk_hash_length_is_24_hex(): + """The hash slice must be 24 hex chars to keep drawer IDs storable + in fixed-width metadata columns and to match the historical recipe + length.""" + result = ids.make_drawer_id_from_chunk("w", "r", "/file.md", 5) + hash_part = result.removeprefix("drawer_w_r_") + assert len(hash_part) == 24 + assert all(c in "0123456789abcdef" for c in hash_part) + + +def test_make_drawer_id_from_chunk_does_not_collide_across_boundary(): + """RED test pinning the whole PR. + + Classic collision: source_file="/path/a1" + chunk_index=23 produces + hash input "/path/a123" under the pre-v2 recipe. source_file="/path/a" + + chunk_index=123 produces the SAME "/path/a123" — same hash, same + drawer_id, second ChromaDB upsert overwrites the first. + + Under the v2 recipe (delimiter '|'), the two inputs become + "/path/a1|23" and "/path/a|123" — distinct strings, distinct + hashes, distinct drawer IDs. No collision. + """ + a = ids.make_drawer_id_from_chunk("w", "r", "/path/a1", 23) + b = ids.make_drawer_id_from_chunk("w", "r", "/path/a", 123) + assert a != b, f"Collision survived v2 recipe: {a!r} == {b!r}" + + +def test_make_drawer_id_from_chunk_is_deterministic(): + """Same inputs must always produce the same ID — re-mining a file + that hasn't changed must hit the same drawer slot, or + file_already_mined() loses its idempotency.""" + a = ids.make_drawer_id_from_chunk("w", "r", "/file.md", 7) + b = ids.make_drawer_id_from_chunk("w", "r", "/file.md", 7) + assert a == b + + +def test_make_drawer_id_from_chunk_windows_path_with_colon_does_not_collide(): + """Windows paths contain ':' in drive letters (C:\\Users\\...). The + v2 recipe uses '|' precisely so paths that contain ':' can never + align with a chunk index to collide. This test would FAIL on a + ':'-delimited recipe because Windows paths and URL-like paths + (https://host:8080) commonly end in ':digits'.""" + a = ids.make_drawer_id_from_chunk("w", "r", "C:\\Users\\foo", 5) + b = ids.make_drawer_id_from_chunk("w", "r", "C:\\Users\\foo:", 5) + assert a != b + + +# ── make_drawer_id_from_content ─────────────────────────────────────── + + +def test_make_drawer_id_from_content_does_not_collide_across_boundary(): + """mcp_server.py:1136 hashes wing+room+content with no delimiter. + Architecturally identical defect to the chunk-index sites: + wing="foo"+room="bar" hashes the same as wing="fooba"+room="r". + v2 delimiter breaks this.""" + a = ids.make_drawer_id_from_content("foo", "bar", "x") + b = ids.make_drawer_id_from_content("fooba", "r", "x") + assert a != b + + +def test_make_drawer_id_from_content_returns_expected_prefix(): + """Same namespacing pattern as the chunk-index helper.""" + result = ids.make_drawer_id_from_content("proj", "scratch", "hello") + assert result.startswith("drawer_proj_scratch_") + + +# ── make_convo_drawer_id ────────────────────────────────────────────── + + +def test_make_convo_drawer_id_does_not_collide_across_extract_mode_boundary(): + """convo_miner.py:422 hashes source_file+extract_mode+chunk_index. + Pre-v2 used ':' as delimiter — this test would still PASS on the ':' + recipe for clean inputs, but the migration to '|' is for + consistency with the chunk-index helpers and to remove the + Windows-path / URL-source edge case where ':' can appear in the + source_file itself.""" + a = ids.make_convo_drawer_id("w", "r", "/log.jsonl", "general", 5) + b = ids.make_convo_drawer_id("w", "r", "/log.jsonl", "extract", 5) + assert a != b + + +def test_make_convo_drawer_id_returns_expected_prefix(): + result = ids.make_convo_drawer_id("claude", "diary", "/c.jsonl", "general", 0) + assert result.startswith("drawer_claude_diary_") + + +# ── make_convo_sentinel_id ──────────────────────────────────────────── + + +def test_make_convo_sentinel_id_returns_expected_prefix(): + """Sentinel IDs are namespaced under '_reg_' so they can be + filtered out of normal drawer queries.""" + result = ids.make_convo_sentinel_id("/c.jsonl", "general") + assert result.startswith("_reg_") + + +def test_make_convo_sentinel_id_distinguishes_extract_modes(): + a = ids.make_convo_sentinel_id("/c.jsonl", "general") + b = ids.make_convo_sentinel_id("/c.jsonl", "extract") + assert a != b + + +# ── make_triple_id ──────────────────────────────────────────────────── + + +def test_make_triple_id_returns_expected_prefix(): + """Triple IDs prefix with 't_' and embed the subject/predicate/object + triple in the ID for grep-ability in SQLite.""" + result = ids.make_triple_id("sub1", "loves", "obj1", "2026-01-01", "2026-05-30T10:00:00") + assert result.startswith("t_sub1_loves_obj1_") + + +def test_make_triple_id_hash_length_is_12_hex(): + """Triple IDs historically truncate at 12 hex chars (vs 24 for + drawers) because the subject/predicate/object prefix already + supplies the bulk of the namespace.""" + result = ids.make_triple_id("s", "p", "o", "2026-01-01", "2026-05-30T10:00:00") + hash_part = result.removeprefix("t_s_p_o_") + assert len(hash_part) == 12 + + +def test_make_triple_id_does_not_collide_across_iso_datetime_boundary(): + """Pre-v2 hash input was f'{valid_from}{recorded_at}' with no + delimiter — two ISO datetimes concatenated could in principle + collide (valid_from='2026-01-01' + recorded_at='T12:00:00' == + valid_from='2026-01-01T12' + recorded_at=':00:00' for hash + purposes). v2 delimiter prevents this.""" + a = ids.make_triple_id("s", "p", "o", "2026-01-01", "T12:00:00") + b = ids.make_triple_id("s", "p", "o", "2026-01-01T12", ":00:00") + assert a != b + + +# ── _delimited_sha256 (private helper, smoke test only) ─────────────── + + +def test_private_delimited_sha256_uses_pipe_delimiter(): + """Confirms the implementation actually uses '|' and not ':' — a + subtle copy-paste from the diary_ingest precedent or a stale + ':' precedent from convo_miner could regress the delimiter without + breaking the higher-level tests.""" + result = ids._delimited_sha256(("a", "b"), 64) + expected = hashlib.sha256(b"a|b").hexdigest() + assert result == expected + + +def test_private_delimited_sha256_truncation_honoured(): + """Truncation argument actually shortens the hex output.""" + result = ids._delimited_sha256(("a", "b"), 8) + assert len(result) == 8 diff --git a/tests/test_maintenance_hooks.py b/tests/test_maintenance_hooks.py new file mode 100644 index 0000000..95ac0fd --- /dev/null +++ b/tests/test_maintenance_hooks.py @@ -0,0 +1,275 @@ +"""Backend maintenance hooks (RFC 001). + +Maintenance is observable, not fire-and-forget: ``run_maintenance(kind)`` +returns a ``MaintenanceResult`` and MUST serialize concurrent same-kind runs. +The pgvector ``reindex`` path (the opt-in HNSW build) is exercised here with a +fake client so the advisory-lock flow is tested without a live Postgres. +""" + +import pytest + +from mempalace.backends.base import ( + BaseCollection, + MaintenanceResult, + PalaceRef, + UnsupportedMaintenanceKindError, +) + + +# --------------------------------------------------------------------------- +# Contract surface +# --------------------------------------------------------------------------- + + +def test_maintenance_result_shape(): + r = MaintenanceResult(kind="reindex", status="ran", stats={"ms": 12}) + assert r.kind == "reindex" and r.status == "ran" and r.stats["ms"] == 12 + assert MaintenanceResult(kind="analyze", status="noop").stats == {} + + +def test_default_collection_rejects_all_kinds(): + class _Col(BaseCollection): + def add(self, **k): ... + def upsert(self, **k): ... + def query(self, **k): ... + def get(self, **k): ... + def delete(self, **k): ... + def count(self): + return 0 + + col = _Col() + assert col.maintenance_state() == {} + with pytest.raises(UnsupportedMaintenanceKindError): + col.run_maintenance("analyze") + + +def test_backend_maintenance_kinds_declared(): + from mempalace.backends.chroma import ChromaBackend + from mempalace.backends.pgvector import PgVectorBackend + from mempalace.backends.qdrant import QdrantBackend + from mempalace.backends.sqlite_exact import SQLiteExactBackend + + assert SQLiteExactBackend.maintenance_kinds == frozenset({"analyze", "compact"}) + assert PgVectorBackend.maintenance_kinds == frozenset({"analyze", "reindex"}) + # qdrant self-optimizes; chroma maintenance is the separate repair CLI. + assert QdrantBackend.maintenance_kinds == frozenset() + assert ChromaBackend.maintenance_kinds == frozenset() + + +# --------------------------------------------------------------------------- +# sqlite_exact (CI-runnable, real backend) +# --------------------------------------------------------------------------- + + +def _sqlite_collection(tmp_path, rows=20): + from mempalace.backends.sqlite_exact import SQLiteExactBackend + + col = SQLiteExactBackend().get_collection( + palace=PalaceRef(id=str(tmp_path), local_path=str(tmp_path)), + collection_name="mempalace_drawers", + create=True, + ) + for i in range(rows): + col.add( + documents=[f"doc {i}"], + ids=[f"id{i}"], + metadatas=[{}], + embeddings=[[0.1, 0.2, 0.3, 0.4]], + ) + return col + + +def test_sqlite_maintenance_state(tmp_path): + col = _sqlite_collection(tmp_path, rows=5) + state = col.maintenance_state() + assert state["row_count"] == 5 + assert state["vector_index"] is None # exact scan — no ANN index + assert "page_count" in state and "freelist_pages" in state + + +def test_sqlite_analyze_runs(tmp_path): + col = _sqlite_collection(tmp_path, rows=5) + r = col.run_maintenance("analyze") + assert r.kind == "analyze" and r.status == "ran" + + +def test_sqlite_compact_runs_and_reports_pages(tmp_path): + col = _sqlite_collection(tmp_path, rows=30) + col.delete(ids=[f"id{i}" for i in range(20)]) + r = col.run_maintenance("compact") + assert r.kind == "compact" and r.status == "ran" + assert "pages_reclaimed" in r.stats + + +def test_sqlite_omits_reindex(tmp_path): + # sqlite_exact has no ANN index, so reindex is omitted, not no-op'd. + col = _sqlite_collection(tmp_path, rows=2) + with pytest.raises(UnsupportedMaintenanceKindError): + col.run_maintenance("reindex") + + +def test_sqlite_unknown_kind_raises(tmp_path): + col = _sqlite_collection(tmp_path, rows=2) + with pytest.raises(UnsupportedMaintenanceKindError): + col.run_maintenance("bogus") + + +# --------------------------------------------------------------------------- +# pgvector advisory-lock reindex flow (fake client, no live Postgres) +# --------------------------------------------------------------------------- + + +class _FakeClient: + def __init__(self, has_index=False): + self.has_index = has_index + self.locked = False + self.created = 0 + self.analyzed = 0 + + def table_exists(self, table): + return True + + def count_rows(self, table): + return 7 + + def has_vector_index(self, table): + return self.has_index + + def try_advisory_lock(self, classid, objid): + if self.locked: + return False + self.locked = True + return True + + def advisory_unlock(self, classid, objid): + self.locked = False + + def create_hnsw_index(self, table): + self.has_index = True + self.created += 1 + + def analyze_table(self, table): + self.analyzed += 1 + + +class _FakeBackend: + _closed = False + + +def _pg_collection(client): + from mempalace.backends.pgvector import PgVectorCollection, _PgVectorConfig + + return PgVectorCollection( + backend=_FakeBackend(), + client=client, + config=_PgVectorConfig(dsn="postgresql://example", namespace=None), + palace=PalaceRef(id="/tmp/p", local_path="/tmp/p"), + collection_name="mempalace_drawers", + table="mp_drawers_t", + ) + + +def test_pgvector_reindex_builds_index_under_lock(): + client = _FakeClient(has_index=False) + col = _pg_collection(client) + r = col.run_maintenance("reindex") + assert r.status == "ran" and r.stats.get("vector_index") == "hnsw" + assert client.created == 1 + assert client.locked is False # lock released in finally + + +def test_pgvector_reindex_noop_when_index_exists(): + client = _FakeClient(has_index=True) + col = _pg_collection(client) + r = col.run_maintenance("reindex") + assert r.status == "noop" + assert client.created == 0 # never attempted a build + + +def test_pgvector_reindex_already_running_when_lock_held(): + client = _FakeClient(has_index=False) + client.locked = True # another session is building + col = _pg_collection(client) + r = col.run_maintenance("reindex") + assert r.status == "already_running" + assert client.created == 0 # did not re-trigger the build + + +def test_pgvector_analyze_runs(): + client = _FakeClient() + col = _pg_collection(client) + r = col.run_maintenance("analyze") + assert r.status == "ran" and client.analyzed == 1 + + +def test_pgvector_unknown_kind_raises(): + col = _pg_collection(_FakeClient()) + with pytest.raises(UnsupportedMaintenanceKindError): + col.run_maintenance("compact") # pgvector omits compact (autovacuum) + + +def test_pgvector_maintenance_state_reports_index(): + col = _pg_collection(_FakeClient(has_index=True)) + state = col.maintenance_state() + assert state["row_count"] == 7 + assert state["vector_index"] == "hnsw" and state["index_build_complete"] is True + + +def test_pgvector_maintenance_noop_when_table_missing(): + # Collection opened create=True but never written: no table yet. Maintenance + # must noop, not let a raw "relation does not exist" error escape. + client = _FakeClient() + client.table_exists = lambda table: False + col = _pg_collection(client) + assert col.run_maintenance("reindex").status == "noop" + assert col.run_maintenance("analyze").status == "noop" + assert col.maintenance_state()["row_count"] == 0 + + +def test_hnsw_index_name_never_collides_with_table_name(): + # A naive [:63] truncation would return a 63-char table name verbatim, + # colliding in pg_class. _pg_identifier hashes the overflow instead. + from mempalace.backends.pgvector import _hnsw_index_name + + for table in ("t", "mp_drawers", "x" * 63, "y" * 200): + name = _hnsw_index_name(table) + assert name != table + assert len(name.encode("utf-8")) <= 63 + + +def test_pgvector_advisory_key_is_signed_int4_and_stable(): + from mempalace.backends.pgvector import _MAINTENANCE_LOCK_CLASSID, _advisory_objid + + for table in ("a", "mempalace_drawers_xyz", "x" * 80): + objid = _advisory_objid(table) + assert -(2**31) <= objid < 2**31 + assert _advisory_objid(table) == objid # stable + assert -(2**31) <= _MAINTENANCE_LOCK_CLASSID < 2**31 + + +# --------------------------------------------------------------------------- +# EmbeddingCollection delegation +# --------------------------------------------------------------------------- + + +def test_embeddingcollection_delegates_maintenance(): + from mempalace.backends.embedding_wrapper import EmbeddingCollection + + class _Inner(BaseCollection): + def add(self, **k): ... + def upsert(self, **k): ... + def query(self, **k): ... + def get(self, **k): ... + def delete(self, **k): ... + def count(self): + return 0 + + def maintenance_state(self): + return {"row_count": 3} + + def run_maintenance(self, kind): + return MaintenanceResult(kind=kind, status="ran") + + wrapped = EmbeddingCollection(_Inner()) + assert wrapped.maintenance_state() == {"row_count": 3} + assert wrapped.run_maintenance("analyze").status == "ran" diff --git a/tests/test_mcp_mine.py b/tests/test_mcp_mine.py new file mode 100644 index 0000000..0f73add --- /dev/null +++ b/tests/test_mcp_mine.py @@ -0,0 +1,262 @@ +""" +test_mcp_mine.py — Tests for the ``mempalace_mine`` MCP tool (#1662). + +Mining was previously CLI-only (``mempalace mine``); non-Claude-Code MCP clients +(Desktop Commander, LM Studio, Aionui) had no MCP-callable mine. ``tool_mine`` +wraps the same in-process miners the CLI uses — projects / convos / extract — +synchronously, mirroring the ``tool_sync`` contract. + +The miners print progress + a summary to stdout, which in the MCP server is the +JSON-RPC channel. ``tool_mine`` therefore redirects stdout at the file-descriptor +level around the miner and returns the text as an opaque ``output`` field rather +than letting it corrupt the protocol. These tests assert the dispatch/return +contract, that convos mining actually files drawers (the #1662 gap), and that the +stdout isolation holds. +""" + +import os + +import chromadb + + +def _patch(monkeypatch, config): + from mempalace import mcp_server + + monkeypatch.setattr(mcp_server, "_config", config) + + +def _write(path, text): + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + + +# ── Registration ───────────────────────────────────────────────────────── + + +def test_registered_in_tools(): + from mempalace import mcp_server + + assert "mempalace_mine" in mcp_server.TOOLS + entry = mcp_server.TOOLS["mempalace_mine"] + assert entry["handler"] is mcp_server.tool_mine + assert entry["input_schema"]["required"] == ["source"] + + +# ── Guard rails ────────────────────────────────────────────────────────── + + +def test_no_palace_returns_structured_error(monkeypatch): + from mempalace import mcp_server + + class _EmptyConfig: + palace_path = "" + collection_name = "mempalace_drawers" + + monkeypatch.setattr(mcp_server, "_config", _EmptyConfig()) + result = mcp_server.tool_mine(source="/tmp") + assert result["success"] is False + assert "error" in result + + +def test_invalid_mode_returns_structured_error(monkeypatch, config, tmp_dir): + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "src") + os.makedirs(src) + result = mcp_server.tool_mine(source=src, mode="bogus") + assert result["success"] is False + assert "invalid mode" in result["error"].lower() + + +def test_missing_source_dir_returns_structured_error(monkeypatch, config): + from mempalace import mcp_server + + _patch(monkeypatch, config) + result = mcp_server.tool_mine(source="/nonexistent/path/xyz") + assert result["success"] is False + assert "source" in result["error"].lower() + + +# ── Dispatch + return contract ─────────────────────────────────────────── + + +def test_dry_run_projects_returns_success_and_output(monkeypatch, config, tmp_dir): + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "proj") + os.makedirs(src) + _write(os.path.join(src, "notes.md"), "# Title\n\n" + ("Some real content. " * 40)) + + result = mcp_server.tool_mine(source=src, mode="projects", dry_run=True) + assert result["success"] is True + assert result["mode"] == "projects" + assert result["dry_run"] is True + assert isinstance(result["output"], str) and result["output"] + + +def test_convos_mode_files_drawers(monkeypatch, config, tmp_dir): + """The #1662 core ask: mine conversation transcripts via MCP. + + Proves the tool eliminates the gap rather than masking it — after a real + convos mine the palace collection actually holds the drawers. + """ + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "convos") + os.makedirs(src) + _write( + os.path.join(src, "chat.txt"), + "> What is memory?\nMemory is persistence.\n\n" + "> Why does it matter?\nIt enables continuity across sessions.\n\n" + "> How do we build it?\nWith structured verbatim storage.\n", + ) + + result = mcp_server.tool_mine(source=src, mode="convos", wing="test_convos") + assert result["success"] is True + assert result["mode"] == "convos" + assert result["dry_run"] is False + + client = chromadb.PersistentClient(path=config.palace_path) + try: + col = client.get_collection("mempalace_drawers") + assert col.count() >= 2 + finally: + del client + + +def test_stdout_captured_not_leaked_to_fd(monkeypatch, config, tmp_dir, capfd): + """Miner stdout must land in ``output``, never on the real fd-1 JSON-RPC + channel. ``tool_mine`` redirects fd 1 around the in-process miner.""" + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "convos") + os.makedirs(src) + _write( + os.path.join(src, "chat.txt"), + "> Q one?\nAnswer one is reasonably long so it forms a chunk here.\n\n" + "> Q two?\nAnswer two is also long enough to be filed as a drawer here.\n", + ) + + result = mcp_server.tool_mine(source=src, mode="convos", wing="cap", dry_run=True) + captured = capfd.readouterr() + assert "Done." in result["output"] + assert "Done." not in captured.out + + +def test_mine_already_running_surfaces_structured_error(monkeypatch, config, tmp_dir): + """A held palace lock (MineAlreadyRunning) surfaces as a structured + already-running error, mirroring tool_sync.""" + from mempalace import mcp_server + from mempalace.palace import MineAlreadyRunning + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "proj") + os.makedirs(src) + _write(os.path.join(src, "a.md"), "content " * 50) + + def _boom(*args, **kwargs): + raise MineAlreadyRunning("held by pid 999") + + monkeypatch.setattr("mempalace.miner.mine", _boom) + result = mcp_server.tool_mine(source=src, mode="projects") + assert result["success"] is False + assert result.get("error_class") == "LockHeldByOtherProcess" + + +def test_large_output_is_tail_truncated(monkeypatch, config, tmp_dir): + """A very large miner summary is tail-trimmed (and flagged, never silently) + so the MCP response stays bounded.""" + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "proj") + os.makedirs(src) + + def _chatty(*args, **kwargs): + print("X" * 5000) + return None + + monkeypatch.setattr("mempalace.miner.mine", _chatty) + result = mcp_server.tool_mine(source=src, mode="projects") + assert result["success"] is True + assert result["output_truncated"] is True + assert len(result["output"]) == 4000 + + +def test_import_error_outside_extract_is_not_mislabeled(monkeypatch, config, tmp_dir): + """An ImportError outside extract mode is a real bug, not a missing extra — + it must not be labelled MissingDependency.""" + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "proj") + os.makedirs(src) + + def _broken(*args, **kwargs): + raise ImportError("no module named 'totally_internal'") + + monkeypatch.setattr("mempalace.miner.mine", _broken) + result = mcp_server.tool_mine(source=src, mode="projects") + assert result["success"] is False + assert result.get("error_class") == "ImportError" + assert "mine failed" in result["error"] + + +def test_extract_missing_dependency_is_named(monkeypatch, config, tmp_dir): + """extract mode surfaces a MissingDependency error pointing at the extra.""" + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "docs") + os.makedirs(src) + + def _no_extra(*args, **kwargs): + raise ImportError("No module named 'markitdown'") + + monkeypatch.setattr("mempalace.format_miner.mine_formats", _no_extra) + result = mcp_server.tool_mine(source=src, mode="extract") + assert result["success"] is False + assert result.get("error_class") == "MissingDependency" + assert "mempalace[extract]" in result["error"] + + +def test_system_exit_from_miner_does_not_kill_server(monkeypatch, config, tmp_dir): + """miner.mine turns Ctrl-C into sys.exit(130); in-process that SystemExit + would escape the protocol loop (which only catches Exception) and kill the + server. tool_mine converts it to a structured error instead.""" + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "proj") + os.makedirs(src) + + def _exit(*args, **kwargs): + raise SystemExit(130) + + monkeypatch.setattr("mempalace.miner.mine", _exit) + result = mcp_server.tool_mine(source=src, mode="projects") + assert result["success"] is False + assert result.get("error_class") == "Interrupted" + + +def test_generic_exception_carries_error_class(monkeypatch, config, tmp_dir): + """An unexpected miner failure is surfaced with its exception type so the + caller can distinguish error kinds.""" + from mempalace import mcp_server + + _patch(monkeypatch, config) + src = os.path.join(tmp_dir, "proj") + os.makedirs(src) + + def _boom(*args, **kwargs): + raise RuntimeError("disk gone") + + monkeypatch.setattr("mempalace.miner.mine", _boom) + result = mcp_server.tool_mine(source=src, mode="projects") + assert result["success"] is False + assert "mine failed" in result["error"] + assert result.get("error_class") == "RuntimeError" diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 1870de5..a24f496 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -695,6 +695,75 @@ class TestReadTools: assert "project" in result["wings"] assert "notes" in result["wings"] + def test_status_sqlite_exact_backend_has_no_hnsw_fields( + self, monkeypatch, config, palace_path, kg + ): + import mempalace.backends.embedding_wrapper as embedding_wrapper + from mempalace.palace import get_collection + + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact") + monkeypatch.setattr( + embedding_wrapper, + "_embed_texts", + lambda texts: [[float(len(text)), 1.0] for text in texts], + ) + col = get_collection(palace_path, create=True) + col.add( + ids=["drawer_sqlite"], + documents=["verbatim sqlite drawer"], + metadatas=[{"wing": "w", "room": "r"}], + ) + + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + + monkeypatch.setattr(mcp_server, "_collection_cache", None) + result = mcp_server.tool_status() + + assert result["backend"] == "sqlite_exact" + assert result["total_drawers"] == 1 + assert "hnsw_capacity" not in result + assert result.get("vector_disabled") is not True + + def test_status_qdrant_backend_has_no_hnsw_fields(self, monkeypatch, config, palace_path, kg): + from mempalace.backends import GetResult + + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "qdrant") + monkeypatch.setenv("MEMPALACE_BACKEND", "qdrant") + with open(os.path.join(palace_path, "qdrant_backend.json"), "w", encoding="utf-8") as f: + json.dump({"backend": "qdrant"}, f) + + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + + class _FakeQdrantCollection: + def count(self): + return 2 + + def get(self, **_kwargs): + return GetResult( + ids=["q1", "q2"], + documents=[], + metadatas=[ + {"wing": "project", "room": "backend"}, + {"wing": "project", "room": "api"}, + ], + ) + + monkeypatch.setattr(mcp_server, "_collection_cache", None) + monkeypatch.setattr(mcp_server, "_metadata_cache", None) + monkeypatch.setattr( + mcp_server, "_get_collection", lambda create=False: _FakeQdrantCollection() + ) + + result = mcp_server.tool_status() + + assert result["backend"] == "qdrant" + assert result["total_drawers"] == 2 + assert result["wings"] == {"project": 2} + assert "hnsw_capacity" not in result + assert result.get("vector_disabled") is not True + def test_status_handles_none_metadata_without_partial( self, monkeypatch, config, palace_path, kg ): @@ -1025,6 +1094,34 @@ class TestSearchTool: assert "error" in result assert "index_recovered" not in result + def test_search_retries_once_on_stale_index_error(self, monkeypatch, config, kg): + """Stale-index errors should trigger one cache-reset retry.""" + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + + calls = {"n": 0} + reset_calls = {"n": 0} + + def fake_search(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + return {"error": "Search error: stale-index detected; retry recommended"} + return {"results": [{"text": "ok", "wing": "w", "room": "r"}]} + + def fake_reset(): + reset_calls["n"] += 1 + + monkeypatch.setattr(mcp_server, "search_memories", fake_search) + monkeypatch.setattr(mcp_server, "_force_chroma_cache_reset", fake_reset) + monkeypatch.setattr(mcp_server.time, "sleep", lambda _: None) + + result = mcp_server.tool_search(query="anything") + + assert calls["n"] == 2 + assert reset_calls["n"] == 1 + assert "results" in result + assert result.get("index_recovered") is True + def test_list_drawers_rejects_invalid_wing(self, monkeypatch, config, kg): _patch_mcp_server(monkeypatch, config, kg) from mempalace import mcp_server @@ -1415,6 +1512,108 @@ class TestWriteTools: assert result == {"error": msg} + # ── hallway MCP tools (mirror the tunnel pattern) ── + + def _seed_hallways(self, monkeypatch, tmp_path): + """Point hallways._HALLWAY_FILE at a tmp file and seed two records.""" + from mempalace import hallways + + hallway_file = tmp_path / "hallways.json" + monkeypatch.setattr(hallways, "_HALLWAY_FILE", str(hallway_file)) + seeded = [ + { + "id": "hallway_wing_a_X_Y_aaaa", + "wing": "wing_a", + "entity_a": "X", + "entity_b": "Y", + "co_occurrence_count": 3, + "rooms": ["room1"], + }, + { + "id": "hallway_wing_b_X_Z_bbbb", + "wing": "wing_b", + "entity_a": "X", + "entity_b": "Z", + "co_occurrence_count": 1, + "rooms": ["room2"], + }, + ] + hallways._save_hallways(seeded) + return seeded + + def test_tool_list_hallways_returns_all_without_filter(self, monkeypatch, tmp_path): + """tool_list_hallways with no wing returns every record.""" + from mempalace import mcp_server + + seeded = self._seed_hallways(monkeypatch, tmp_path) + result = mcp_server.tool_list_hallways() + assert isinstance(result, list) + assert len(result) == len(seeded) + ids = {h["id"] for h in result} + assert ids == {h["id"] for h in seeded} + + def test_tool_list_hallways_filters_by_wing(self, monkeypatch, tmp_path): + """tool_list_hallways with wing returns only that wing's records.""" + from mempalace import mcp_server + + self._seed_hallways(monkeypatch, tmp_path) + result = mcp_server.tool_list_hallways(wing="wing_a") + assert len(result) == 1 + assert result[0]["wing"] == "wing_a" + + def test_tool_list_hallways_rejects_invalid_wing_name(self, monkeypatch, tmp_path): + """Invalid wing names go through _sanitize_optional_name and return a + structured error rather than crashing — mirrors tool_list_tunnels.""" + from mempalace import mcp_server + + self._seed_hallways(monkeypatch, tmp_path) + # Forward-slash is not a valid name character per sanitize_name. + result = mcp_server.tool_list_hallways(wing="wing/with/slashes") + assert isinstance(result, dict) + assert "error" in result + + def test_tool_delete_hallway_removes_existing_record(self, monkeypatch, tmp_path): + """tool_delete_hallway removes the record and returns {deleted: True}.""" + from mempalace import mcp_server + + seeded = self._seed_hallways(monkeypatch, tmp_path) + target_id = seeded[0]["id"] + result = mcp_server.tool_delete_hallway(hallway_id=target_id) + assert result == {"deleted": True} + remaining = mcp_server.tool_list_hallways() + assert target_id not in {h["id"] for h in remaining} + + def test_tool_delete_hallway_unknown_id_returns_false(self, monkeypatch, tmp_path): + """Deleting an ID that doesn't exist returns {deleted: False} without error.""" + from mempalace import mcp_server + + self._seed_hallways(monkeypatch, tmp_path) + result = mcp_server.tool_delete_hallway(hallway_id="hallway_does_not_exist") + assert result == {"deleted": False} + + def test_tool_delete_hallway_requires_string_id(self): + """Missing or non-string hallway_id surfaces a structured error.""" + from mempalace import mcp_server + + assert mcp_server.tool_delete_hallway(hallway_id="") == {"error": "hallway_id is required"} + assert mcp_server.tool_delete_hallway(hallway_id=None) == { + "error": "hallway_id is required" + } + + def test_hallway_tools_registered_in_tools_registry(self): + """Both new tools must appear in the public TOOLS registry so MCP clients can dispatch them.""" + from mempalace import mcp_server + + assert "mempalace_list_hallways" in mcp_server.TOOLS + assert "mempalace_delete_hallway" in mcp_server.TOOLS + assert ( + mcp_server.TOOLS["mempalace_list_hallways"]["handler"] is mcp_server.tool_list_hallways + ) + assert ( + mcp_server.TOOLS["mempalace_delete_hallway"]["handler"] + is mcp_server.tool_delete_hallway + ) + def test_add_drawer_normal_content_single_drawer(self, monkeypatch, config, palace_path, kg): """Regression catch: content below CHUNK_SIZE produces exactly one drawer with ``chunks == 1``. Pre-#1539 contract preserved.""" @@ -2290,7 +2489,69 @@ class TestCacheInvalidation: result = mcp_server.tool_reconnect() assert result["success"] is True - close_palace.assert_called_once_with(config.palace_path) + closed_ref = close_palace.call_args.args[0] + assert closed_ref.local_path == config.palace_path + + def test_reconnect_closes_selected_non_chroma_backend( + self, monkeypatch, config, palace_path, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact") + from mempalace import mcp_server, palace + + closed = [] + + class _FakeBackend: + def close_palace(self, path): + closed.append(path) + + class _FakeCol: + def count(self): + return 3 + + monkeypatch.setattr(palace, "get_backend_for_palace", lambda _path: _FakeBackend()) + monkeypatch.setattr(mcp_server, "_is_chroma_backend", lambda: False) + monkeypatch.setattr(mcp_server, "_get_collection", lambda create=False: _FakeCol()) + + result = mcp_server.tool_reconnect() + + assert result["success"] is True + assert result["drawers"] == 3 + assert len(closed) == 1 + assert closed[0].local_path == palace_path + + def test_reconnect_closes_previously_cached_backend(self, monkeypatch, config, palace_path, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import backends, mcp_server, palace + + closed = [] + + class _SelectedBackend: + name = "sqlite_exact" + + def close_palace(self, ref): + closed.append(("selected", ref.local_path)) + + class _CachedBackend: + name = "chroma" + + def close_palace(self, ref): + closed.append(("cached", ref.local_path)) + + class _FakeCol: + def count(self): + return 3 + + monkeypatch.setattr(palace, "get_backend_for_palace", lambda _path: _SelectedBackend()) + monkeypatch.setattr(backends, "get_backend", lambda _name: _CachedBackend()) + monkeypatch.setattr(mcp_server, "_collection_cache_backend", "chroma") + monkeypatch.setattr(mcp_server, "_is_chroma_backend", lambda: False) + monkeypatch.setattr(mcp_server, "_get_collection", lambda create=False: _FakeCol()) + + result = mcp_server.tool_reconnect() + + assert result["success"] is True + assert closed == [("selected", palace_path), ("cached", palace_path)] def test_get_collection_create_true_avoids_get_or_create_on_reopen( self, monkeypatch, config, palace_path, kg @@ -2459,6 +2720,66 @@ class TestCacheInvalidation: assert col is None +class TestImportKillSwitchSafety: + """Importing mcp_server must not recreate ~/.mempalace (#1676). + + The module-level WAL setup used to ``mkdir(parents=True)`` at import, + recreating ``~/.mempalace`` even after the user removed it as the + documented kill-switch gesture (``_palace_root_exists()``, #1305), + silently re-arming the autosave/mining hooks. WAL creation is now + deferred to the first actual write. + """ + + def test_import_does_not_recreate_palace_root(self, tmp_path): + """import mempalace.mcp_server must not create ~/.mempalace. + + Runs in a fresh subprocess with HOME pointed at tmp_path so the + assertion targets a clean filesystem, independent of conftest's + session-level HOME patch. + """ + palace_root = tmp_path / ".mempalace" + env = {k: v for k, v in os.environ.items() if not k.startswith("MEMPAL")} + env["HOME"] = str(tmp_path) + env["USERPROFILE"] = str(tmp_path) + result = subprocess.run( + [sys.executable, "-c", "import mempalace.mcp_server"], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, f"import failed: {result.stderr}" + assert not palace_root.exists(), ( + f"importing mcp_server recreated {palace_root} as a side effect, " + "defeating the _palace_root_exists() kill-switch (#1676)" + ) + + def test_wal_log_creates_dir_lazily_on_first_write(self, tmp_path, monkeypatch): + """_wal_log creates its directory on first use. + + Proves the deferred setup still works (defers WAL creation to write + time, does not disable it) and preserves the WAL permission bits. + """ + from mempalace import mcp_server + + wal_file = tmp_path / "fresh" / "wal" / "write_log.jsonl" + assert not wal_file.parent.exists() + monkeypatch.setattr(mcp_server, "_WAL_FILE", wal_file) + + mcp_server._wal_log("test_op", {"safe": "ok"}) + + assert wal_file.exists(), "lazy WAL init did not create the log on first write" + entry = json.loads(wal_file.read_text().strip()) + assert entry["operation"] == "test_op" + assert entry["params"]["safe"] == "ok" + + # Permission bits the refactor must preserve (POSIX only; Windows + # ignores chmod and the code swallows NotImplementedError). + if sys.platform != "win32": + assert wal_file.stat().st_mode & 0o777 == 0o600 + assert wal_file.parent.stat().st_mode & 0o777 == 0o700 + + class TestKGLazyCache: """Lazy per-path KnowledgeGraph cache (issue #1136).""" @@ -2718,6 +3039,57 @@ class TestStructuredErrors: assert mcp_server._kg_by_path == {} + def test_tool_reconnect_rearms_quarantine_gate(self, monkeypatch): + """``tool_reconnect`` must clear the per-process quarantine gate so + HNSW safety checks re-run on the next open (#1573).""" + from mempalace import mcp_server + from mempalace.backends.chroma import ChromaBackend + + palace_path = "/test/palace/quarantine_rearm" + gate = {palace_path} + monkeypatch.setattr(ChromaBackend, "_quarantined_paths", gate) + monkeypatch.setattr(mcp_server, "_config", type("C", (), {"palace_path": palace_path})()) + monkeypatch.setattr(mcp_server, "_get_collection", lambda: None) + + mcp_server.tool_reconnect() + + assert palace_path not in gate, ( + "tool_reconnect should clear quarantine gate for the palace path" + ) + + def test_get_client_rearms_quarantine_on_reconnect(self, monkeypatch, config, palace_path, kg): + """``_get_client`` must clear the quarantine gate before calling + ``make_client`` so HNSW safety checks re-run on reconnect (#1573).""" + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + from mempalace.backends.chroma import ChromaBackend + + _client, _col = _get_collection(palace_path, create=True) + del _client + + mcp_server._get_collection() + + assert config.palace_path in ChromaBackend._quarantined_paths + + old_mtime = mcp_server._palace_db_mtime + monkeypatch.setattr(mcp_server, "_palace_db_mtime", old_mtime - 10.0) + + quarantine_calls: list[str] = [] + original_prepare = ChromaBackend._prepare_palace_for_open + + @staticmethod + def spy_prepare(path): + quarantine_calls.append(path) + original_prepare(path) + + monkeypatch.setattr(ChromaBackend, "_prepare_palace_for_open", spy_prepare) + + mcp_server._get_client() + + assert len(quarantine_calls) == 1, ( + "_get_client should call _prepare_palace_for_open on reconnect" + ) + def test_call_kg_retries_after_concurrent_close(self, monkeypatch): """A KG closed mid-handler must trigger a one-shot retry with a fresh instance — not surface a -32000 to the MCP client.""" @@ -3053,6 +3425,88 @@ class TestParamShapeDiagnostics: assert "'entry'" in message assert " and " not in message.split("for tool")[0] + def test_diary_write_content_aliases_entry(self, monkeypatch): + """A content-only diary_write call is remapped to 'entry' before + dispatch (#1245 alias), so it satisfies the required param and the + alias key is consumed rather than passed through to the handler. + """ + from mempalace import mcp_server + + captured = {} + + def capture(**kwargs): + captured.update(kwargs) + return {"success": True} + + monkeypatch.setitem(mcp_server.TOOLS["mempalace_diary_write"], "handler", capture) + resp = mcp_server.handle_request( + { + "method": "tools/call", + "id": 5, + "params": { + "name": "mempalace_diary_write", + "arguments": {"agent_name": "test", "content": "hello world"}, + }, + } + ) + assert "error" not in resp + assert captured.get("entry") == "hello world" + assert "content" not in captured + + def test_diary_write_entry_wins_over_content(self, monkeypatch): + """When both 'entry' and the 'content' alias are supplied, 'entry' wins + and the alias is dropped. + """ + from mempalace import mcp_server + + captured = {} + + def capture(**kwargs): + captured.update(kwargs) + return {"success": True} + + monkeypatch.setitem(mcp_server.TOOLS["mempalace_diary_write"], "handler", capture) + resp = mcp_server.handle_request( + { + "method": "tools/call", + "id": 6, + "params": { + "name": "mempalace_diary_write", + "arguments": {"agent_name": "t", "entry": "real", "content": "alias"}, + }, + } + ) + assert "error" not in resp + assert captured.get("entry") == "real" + assert "content" not in captured + + def test_diary_write_explicit_empty_entry_not_overridden_by_content(self, monkeypatch): + """An explicitly supplied (even falsy "") 'entry' wins over 'content' — + the alias only fills in when 'entry' is absent or null, not merely falsy. + """ + from mempalace import mcp_server + + captured = {} + + def capture(**kwargs): + captured.update(kwargs) + return {"success": True} + + monkeypatch.setitem(mcp_server.TOOLS["mempalace_diary_write"], "handler", capture) + resp = mcp_server.handle_request( + { + "method": "tools/call", + "id": 7, + "params": { + "name": "mempalace_diary_write", + "arguments": {"agent_name": "t", "entry": "", "content": "alias"}, + }, + } + ) + assert "error" not in resp + assert captured.get("entry") == "" + assert "content" not in captured + def test_handler_internal_signature_shape_stays_generic(self, monkeypatch): """A TypeError whose function name does not match the dispatched handler — e.g. raised by a helper called inside the handler body — diff --git a/tests/test_migrate.py b/tests/test_migrate.py index 5d9255a..1e0259b 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -286,3 +286,52 @@ def test_migrate_cleans_temp_palace_on_chromadb_failure(tmp_path): assert captured_temp_paths, "mkdtemp was never called — flow short-circuited" for p in captured_temp_paths: assert not os.path.exists(p), f"temp palace was not cleaned up: {p}" + + +def test_migrate_prunes_old_pre_migrate_backups(tmp_path, monkeypatch): + """Repeated migrations must not accumulate full-palace copies forever. + + The backup + prune happen right after copytree, before the (mocked) + chromadb step, so even a migration that fails afterward still trims the + backup set. We let copytree run for real so the fresh backup exists on + disk for the prune to evaluate. + """ + palace_dir = tmp_path / "palace" + palace_dir.mkdir() + (palace_dir / "chroma.sqlite3").write_text("db") + + # Pre-seed 3 stale .pre-migrate.* sibling dirs with old mtimes. + for i in range(3): + stale = tmp_path / f"palace.pre-migrate.2026010{i}_000000" + stale.mkdir() + (stale / "chroma.sqlite3").write_text("old") + os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i)) + + monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "2") + + failing_backend = MagicMock() + failing_backend.get_collection.side_effect = Exception("unreadable") + failing_backend.get_or_create_collection.side_effect = RuntimeError("chromadb boom") + + import mempalace.backends.chroma as _chroma_mod + + with ( + patch("mempalace.migrate.detect_chromadb_version", return_value="0.5.x"), + patch( + "mempalace.migrate.extract_drawers_from_sqlite", + return_value=[{"id": "id1", "document": "doc", "metadata": {"wing": "w", "room": "r"}}], + ), + patch("builtins.input", return_value="y"), + patch.object(_chroma_mod, "ChromaBackend", return_value=failing_backend), + ): + try: + migrate(str(palace_dir), confirm=True) + except Exception: + pass + + backups = sorted(p.name for p in tmp_path.glob("palace.pre-migrate.*")) + # 3 stale + 1 fresh = 4 created; retention keeps only the 2 newest. + assert len(backups) == 2 + # The two oldest stale backups must be gone. + assert "palace.pre-migrate.20260100_000000" not in backups + assert "palace.pre-migrate.20260101_000000" not in backups diff --git a/tests/test_migrate_wings.py b/tests/test_migrate_wings.py new file mode 100644 index 0000000..229823a --- /dev/null +++ b/tests/test_migrate_wings.py @@ -0,0 +1,153 @@ +"""Tests for the wing-name normalization migration (migrate-wings). + +normalize_wing_name strips leading/trailing separators (#1675); palaces built +before that rule filed drawers under the old name (e.g. ``_alpha``). +``migrate_wing_names`` re-keys the ``wing`` metadata in place so those memories +stay discoverable under the new name, merging collisions. IDs are left untouched +(they are opaque keys), and the pass is idempotent. +""" + +from mempalace.migrate import migrate_wing_names, plan_wing_renames + + +# --- pure planner --------------------------------------------------------- + + +def test_plan_renames_strips_leading_and_trailing(): + summary, updates = plan_wing_renames( + [ + ("d1", {"wing": "_alpha", "room": "r"}), + ("d2", {"wing": "beta_", "room": "r"}), + ("d3", {"wing": "clean", "room": "r"}), + ] + ) + assert dict(summary) == {("_alpha", "alpha"): 1, ("beta_", "beta"): 1} + assert {u[0] for u in updates} == {"d1", "d2"} + # only 'wing' is rewritten; other metadata is preserved + by_id = {u[0]: u[1] for u in updates} + assert by_id["d1"]["wing"] == "alpha" + assert by_id["d1"]["room"] == "r" + assert by_id["d2"]["wing"] == "beta" + + +def test_plan_renames_noop_for_clean_wings(): + summary, updates = plan_wing_renames([("d", {"wing": "already_clean", "room": "r"})]) + assert not summary + assert not updates + + +def test_plan_renames_ignores_empty_nonstring_and_all_separator(): + _, updates = plan_wing_renames( + [ + ("a", {"wing": "_"}), # normalizes to "" -> skip (never strand a drawer) + ("b", {"wing": ""}), + ("c", {"wing": None}), + ("d", {}), + ] + ) + assert updates == [] + + +def test_plan_renames_collision_maps_both_to_same_target(): + _, updates = plan_wing_renames( + [ + ("d1", {"wing": "_gamma"}), + ("d2", {"wing": "gamma_"}), + ] + ) + assert {u[1]["wing"] for u in updates} == {"gamma"} + + +# --- integration over a real backend collection --------------------------- + + +def _seed(palace, rows): + from mempalace.palace import get_collection + + col = get_collection(str(palace), create=True) + # Explicit embeddings keep the test hermetic (no embedding model needed) — + # the migration only reads/writes metadata. + col.upsert( + ids=[r["id"] for r in rows], + documents=[r["doc"] for r in rows], + metadatas=[r["meta"] for r in rows], + embeddings=[[float(i + 1)] * 8 for i in range(len(rows))], + ) + return col + + +def _wing_ids(palace, wing): + from mempalace.palace import get_collection + + col = get_collection(str(palace), create=False) + res = col.get(where={"wing": wing}, include=["metadatas"]) + return set(res.ids if hasattr(res, "ids") else res["ids"]) + + +def _meta(wing, room, source_file, idx): + return {"wing": wing, "room": room, "source_file": source_file, "chunk_index": idx} + + +def test_migrate_relabels_old_format_wings(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + _seed( + palace, + [ + {"id": "drawer__alpha_r_1", "doc": "auth jwt", "meta": _meta("_alpha", "r", "a.py", 0)}, + {"id": "drawer_beta__r_2", "doc": "db alembic", "meta": _meta("beta_", "r", "b.py", 0)}, + { + "id": "drawer_clean_r_3", + "doc": "react query", + "meta": _meta("clean", "r", "c.py", 0), + }, + ], + ) + + assert migrate_wing_names(str(palace), confirm=True) is True + + assert _wing_ids(palace, "alpha") == {"drawer__alpha_r_1"} + assert _wing_ids(palace, "beta") == {"drawer_beta__r_2"} + assert _wing_ids(palace, "_alpha") == set() + assert _wing_ids(palace, "beta_") == set() + # an already-clean wing is left untouched + assert _wing_ids(palace, "clean") == {"drawer_clean_r_3"} + + +def test_migrate_merges_collision_into_existing_wing(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + _seed( + palace, + [ + {"id": "drawer_gamma_r_1", "doc": "current", "meta": _meta("gamma", "r", "g1.py", 0)}, + {"id": "drawer__gamma_r_2", "doc": "legacy", "meta": _meta("_gamma", "r", "g2.py", 0)}, + ], + ) + + migrate_wing_names(str(palace), confirm=True) + + assert _wing_ids(palace, "gamma") == {"drawer_gamma_r_1", "drawer__gamma_r_2"} + assert _wing_ids(palace, "_gamma") == set() + + +def test_migrate_dry_run_changes_nothing(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + _seed(palace, [{"id": "drawer__x_r_1", "doc": "d", "meta": _meta("_x", "r", "x.py", 0)}]) + + assert migrate_wing_names(str(palace), dry_run=True) is True + # nothing actually moved + assert _wing_ids(palace, "_x") == {"drawer__x_r_1"} + assert _wing_ids(palace, "x") == set() + + +def test_migrate_is_idempotent(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + _seed(palace, [{"id": "drawer__y_r_1", "doc": "d", "meta": _meta("_y", "r", "y.py", 0)}]) + + assert migrate_wing_names(str(palace), confirm=True) is True + # second run finds nothing left to normalize + assert migrate_wing_names(str(palace), confirm=True) is False + assert _wing_ids(palace, "y") == {"drawer__y_r_1"} diff --git a/tests/test_miner.py b/tests/test_miner.py index 1baadd1..34ceff6 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -9,6 +9,7 @@ import chromadb import pytest import yaml +from mempalace.config import normalize_wing_name from mempalace.miner import detect_room, load_config, mine, scan_project, status from mempalace.palace import NORMALIZE_VERSION, file_already_mined, prefetch_mined_set @@ -257,7 +258,12 @@ def test_load_config_uses_defaults_when_yaml_missing(): assert isinstance(config, dict) assert "wing" in config assert "rooms" in config - assert config["wing"] == project_root.name + # The default wing is the normalized dirname, not the raw name: temp + # dir names can contain leading/trailing '_' (tempfile's alphabet + # includes it), which normalize_wing_name strips. Comparing to the raw + # name was flaky across platforms (it only passed when the random name + # had no separators). + assert config["wing"] == normalize_wing_name(project_root.name) finally: shutil.rmtree(tmpdir) @@ -810,6 +816,133 @@ def test_status_handles_none_metadata_without_crash(tmp_path, capsys): assert "WING: proj" in out +def test_status_does_not_cold_load_vector_index(palace_path, seeded_collection, capsys): + """#1681 regression: a healthy ``status`` must NOT open the collection. + + Opening it cold-loads the HNSW vector index, which costs ~60s of CPU per + call on large palaces. The counts come from chroma.sqlite3 instead. If + anyone reroutes the happy path back through the vector index, the sentinel + patched over ``_open_collection_or_explain`` fires. (Revert ``status`` to + its pre-fix body and this test fails loudly — that's the regression it + guards.) + """ + from unittest.mock import MagicMock, patch + + sentinel = MagicMock(side_effect=AssertionError("status cold-loaded the vector index")) + with patch("mempalace.miner._open_collection_or_explain", sentinel): + status(palace_path) + + sentinel.assert_not_called() + out = capsys.readouterr().out + assert "MemPalace Status — 4 drawers" in out + assert "WING: project" in out + assert "WING: notes" in out + + +def test_sqlite_wing_room_counts_exact_tally(palace_path, seeded_collection): + """The sqlite tally must equal the seeded drawers exactly — 2 project/ + backend, 1 project/frontend, 1 notes/planning — with no double counting. + + Guards the double ``LEFT JOIN embedding_metadata`` against fan-out: if the + join multiplied rows (e.g. a drawer carrying several metadata keys), the + total would exceed 4 and the room counts would inflate. + """ + from mempalace.backends.chroma import _sqlite_wing_room_counts + + result = _sqlite_wing_room_counts(palace_path, "mempalace_drawers") + assert result is not None + total, wing_rooms = result + assert total == 4 + assert {w: dict(r) for w, r in wing_rooms.items()} == { + "project": {"backend": 2, "frontend": 1}, + "notes": {"planning": 1}, + } + + +def test_status_falls_back_to_chroma_when_sqlite_unreadable(palace_path, seeded_collection, capsys): + """When the sqlite fast path returns ``None`` (exotic schema / read error), + ``status`` must fall back to the ChromaDB client path and still report the + correct tally — not crash or print nothing.""" + from unittest.mock import patch + + with patch("mempalace.backends.chroma._sqlite_wing_room_counts", return_value=None): + status(palace_path) + + out = capsys.readouterr().out + assert "MemPalace Status — 4 drawers" in out + assert "WING: project" in out + + +def test_sqlite_wing_room_counts_none_when_collection_absent(palace_path): + """DB exists but the drawers collection was never bootstrapped -> ``None``, + so ``status`` routes to the 'initialized but empty' message instead of + printing a misleading ``0 drawers`` tally (State C, #1498).""" + import chromadb + + from mempalace.backends.chroma import _sqlite_wing_room_counts + + chromadb.PersistentClient(path=palace_path) # creates chroma.sqlite3, no collection + assert _sqlite_wing_room_counts(palace_path, "mempalace_drawers") is None + + +def test_sqlite_wing_room_counts_numeric_wing_not_dropped(palace_path, collection): + """A drawer whose wing/room is stored numerically (int_value, not + string_value) must be tallied under its stringified value — matching the + ChromaDB path, which surfaces the native number — not silently bucketed + under '?'. Without the int/float COALESCE this row would vanish into '?' + while returning a non-None result that skips the fallback.""" + from mempalace.backends.chroma import _sqlite_wing_room_counts + + collection.add( + ids=["drawer_numeric_meta"], + documents=["a drawer filed with a numeric wing and room"], + metadatas=[{"wing": 2026, "room": 7}], + ) + result = _sqlite_wing_room_counts(palace_path, "mempalace_drawers") + assert result is not None + _, wing_rooms = result + assert {w: dict(r) for w, r in wing_rooms.items()} == {"2026": {"7": 1}} + + +def test_sqlite_wing_room_counts_partial_metadata_buckets_question_mark(palace_path, collection): + """A drawer with a wing but no room (and vice versa) must be counted under + '?' for the missing axis, never dropped. Guards the LEFT JOINs against + being narrowed to inner joins, which would silently discard partial + drawers and undercount the total.""" + from mempalace.backends.chroma import _sqlite_wing_room_counts + + collection.add( + ids=["drawer_wing_only", "drawer_room_only"], + documents=["has a wing but no room", "has a room but no wing"], + metadatas=[{"wing": "alpha"}, {"room": "beta"}], + ) + result = _sqlite_wing_room_counts(palace_path, "mempalace_drawers") + assert result is not None + total, wing_rooms = result + assert total == 2 # neither drawer dropped + assert {w: dict(r) for w, r in wing_rooms.items()} == { + "alpha": {"?": 1}, + "?": {"beta": 1}, + } + + +def test_sqlite_wing_room_counts_returns_none_on_locked_db(palace_path, seeded_collection): + """A sustained sqlite lock (writer holding the DB) must degrade to ``None`` + so ``status`` falls back to the slow-but-correct ChromaDB path rather than + raising. busy_timeout waits out *transient* locks; a hard lock still ends + here. Simulated by forcing the read to raise OperationalError.""" + import sqlite3 as _sqlite3 + from unittest.mock import patch + + from mempalace.backends.chroma import _sqlite_wing_room_counts + + with patch( + "mempalace.backends.chroma.sqlite3.connect", + side_effect=_sqlite3.OperationalError("database is locked"), + ): + assert _sqlite_wing_room_counts(palace_path, "mempalace_drawers") is None + + def test_process_file_uses_bounded_upsert_batches(tmp_path, monkeypatch): from mempalace import miner @@ -1976,3 +2109,205 @@ class TestExtractContentDate: content = "Y2K reference: 25/01/00.\n" f.write_text(content) assert _extract_content_date(str(f), content) == "2000-01-25" + + +def test_file_already_mined_handles_multiple_groups_under_one_source_file(tmp_path): + """Under the additive-mining model, a single ``source_file`` can have + multiple ``parent_drawer_id`` groups in the palace — one per mining pass + — each with its own stored ``source_mtime``. ``file_already_mined`` must + return True if ANY group's stored mtime matches the file's current + mtime, regardless of which group ChromaDB's ``get(..., limit=1)`` happens + to return first. + + Current code uses ``collection.get(where={"source_file": X}, limit=1)`` + which has undefined ordering across multiple matching rows. When ChromaDB + returns a stale group (older mining pass with a different stored mtime), + the function returns False, the additive miner concludes the file changed, + and writes yet another duplicate group for a file that has not actually + changed. Steady state: duplicate groups accumulate without bound. + + This test pins the failure space deterministically using a MockCollection + that always returns the stale group on limit=1 (worst-case ordering). A + correct implementation iterates all groups via the paginated pattern + already used in the ``extract_mode is not None`` branch. + + Issue: follow-up to PR #1628 (the every-bare-source_file-query audit). + """ + # Create a real file with known mtime. + test_file = tmp_path / "doc.md" + test_file.write_text("content that hasn't changed since the latest mine.") + current_mtime = os.path.getmtime(str(test_file)) + + # Build a MockCollection that simulates two parent_drawer_id groups + # under the same source_file with DIFFERENT stored mtimes: + # group_A: stale, source_mtime = current_mtime - 100 (older pass) + # group_B: current, source_mtime = current_mtime (latest pass) + # The mock returns group_A for limit=1 calls (worst-case ordering) and + # returns BOTH groups for the paginated limit=1000 calls (what the fix + # must use). + stale_meta = { + "source_file": str(test_file), + "chunk_index": 0, + "parent_drawer_id": "drawer_group_A", + "source_mtime": current_mtime - 100.0, + "normalize_version": NORMALIZE_VERSION, + } + current_meta = { + "source_file": str(test_file), + "chunk_index": 0, + "parent_drawer_id": "drawer_group_B", + "source_mtime": current_mtime, + "normalize_version": NORMALIZE_VERSION, + } + + class MockCollection: + """Simulates the ChromaDB get() contract for two parent_drawer_id + groups sharing one source_file. Returns the STALE group when called + with limit=1 (the worst-case-ordering shape that triggers the bug). + Returns BOTH groups (paginated) when called with limit=1000 — what + a correctly-iterating implementation must do.""" + + def get(self, where=None, limit=None, offset=0, include=None): + if limit == 1: + return {"ids": ["a_0"], "metadatas": [stale_meta]} + if offset == 0: + return {"ids": ["a_0", "b_0"], "metadatas": [stale_meta, current_meta]} + return {"ids": [], "metadatas": []} + + col = MockCollection() + + # EXPECTED: file_already_mined returns True because at least one stored + # group's mtime matches the current file mtime. + # CURRENT BUG: returns False because limit=1 grabs the stale group. + assert file_already_mined(col, str(test_file), check_mtime=True) is True, ( + "file_already_mined returned False even though a group with matching " + "mtime exists. The limit=1 query picked the stale group; the function " + "must iterate all groups for the source_file (mirroring the existing " + "paginated pattern in the extract_mode-is-set branch)." + ) + + +# ── --limit skips already-mined files (#1535) ────────────────────────── + + +def test_mine_limit_skips_already_mined_files(tmp_path, capsys): + """--limit N should count only NEW work, not already-mined skips (#1535).""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=10) + palace_path = project_root / "palace" + + call_count = 0 + + def fake_process_file(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count <= 8: + return (0, "general", None) + return (3, "general", None) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path), limit=5) + + out = capsys.readouterr().out + assert "Drawers filed: 6" in out + assert call_count == 10 + + +def test_mine_limit_stops_after_n_new_files(tmp_path, capsys): + """--limit 3 on 5 unmined files mines exactly 3 and stops.""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=5) + palace_path = project_root / "palace" + + call_count = 0 + + def fake_process_file(*args, **kwargs): + nonlocal call_count + call_count += 1 + return (2, "general", None) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path), limit=3) + + assert call_count == 3 + out = capsys.readouterr().out + assert "Drawers filed: 6" in out + + +def test_mine_limit_zero_mines_all(tmp_path, capsys): + """--limit 0 (default) processes every file.""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=4) + palace_path = project_root / "palace" + + call_count = 0 + + def fake_process_file(*args, **kwargs): + nonlocal call_count + call_count += 1 + return (1, "general", None) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path), limit=0) + + assert call_count == 4 + out = capsys.readouterr().out + assert "Drawers filed: 4" in out + + +def test_mine_limit_dry_run(tmp_path, capsys): + """--dry-run --limit N counts new files toward the limit.""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=5) + palace_path = project_root / "palace" + + call_count = 0 + + def fake_process_file(*args, **kwargs): + nonlocal call_count + call_count += 1 + return (2, "general", None) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path), limit=3, dry_run=True) + + assert call_count == 3 + + +def test_mine_limit_summary_counts(tmp_path, capsys): + """Summary arithmetic is correct when limit causes early exit.""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=8) + palace_path = project_root / "palace" + + call_idx = 0 + + def fake_process_file(*args, **kwargs): + nonlocal call_idx + call_idx += 1 + if call_idx % 2 == 0: + return (0, "general", None) + return (3, "general", None) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path), limit=2) + + out = capsys.readouterr().out + assert "Files processed: 2" in out + assert "Drawers filed: 6" in out + assert "(limit: 2 new)" in out diff --git a/tests/test_palace.py b/tests/test_palace.py index 1e2325f..acd1b49 100644 --- a/tests/test_palace.py +++ b/tests/test_palace.py @@ -61,6 +61,21 @@ def test_open_collection_or_explain_state_c_no_collection(tmp_path): assert any("mempalace mine" in line for line in lines) +def test_open_collection_or_explain_unknown_backend(tmp_path, monkeypatch): + """An unknown backend name (typo in MEMPALACE_BACKEND/--backend) must + surface as a CLI state message, not an escaping KeyError stack trace.""" + emit, lines = _capture() + palace = tmp_path / "palace" + palace.mkdir() + monkeypatch.setenv("MEMPALACE_BACKEND", "does_not_exist") + + result = _open_collection_or_explain(str(palace), out=emit) + + assert result is None + assert any("Unknown backend selected" in line for line in lines) + assert any("does_not_exist" in line for line in lines) + + def test_open_collection_or_explain_state_d_healthy(tmp_path): """State D: healthy palace — returns the opened collection silently.""" emit, lines = _capture() diff --git a/tests/test_pgvector_backend.py b/tests/test_pgvector_backend.py new file mode 100644 index 0000000..6c16ce2 --- /dev/null +++ b/tests/test_pgvector_backend.py @@ -0,0 +1,511 @@ +import os + +import pytest + +from _backend_conformance import assert_partition_isolation + +from mempalace.backends import ( + BackendError, + BackendMismatchError, + CollectionNotInitializedError, + DimensionMismatchError, + PalaceRef, + available_backends, +) +from mempalace.backends.pgvector import ( + PgVectorBackend, + _matches_where, + _vector_distance, + _as_vector_array, +) + + +class _FakePgVectorClient: + """In-memory stand-in for the psycopg-backed client. + + Stores rows per table so the same-instance/different-table isolation the + real backend gets from Postgres is exercised deterministically in CI. The + real client pushes filters/ranking to SQL; this fake applies the same + Python filter + cosine ranking the local-fallback path uses. + """ + + instances: list = [] + + def __init__(self, _config): + self.tables: dict = {} + self.query_calls: list = [] + _FakePgVectorClient.instances.append(self) + + def ping(self): + return None + + def ensure_extension(self): + return None + + def table_exists(self, table): + return table in self.tables + + def table_dimension(self, table): + return self.tables.get(table, {}).get("dimension") + + def create_table(self, table, dimension): + self.tables.setdefault(table, {"dimension": dimension, "rows": {}}) + + def upsert_rows(self, table, rows): + store = self.tables.setdefault( + table, + {"dimension": len(rows[0]["embedding"]) if rows else 0, "rows": {}}, + ) + for row in rows: + store["rows"][row["id"]] = dict(row) + + def _filtered(self, table, where): + rows = list(self.tables.get(table, {"rows": {}})["rows"].values()) + return [row for row in rows if _matches_where(row.get("metadata") or {}, where)] + + def query_rows(self, table, *, vector, limit, where, with_embedding): + self.query_calls.append(where) + q = _as_vector_array(vector) + scored = [] + for row in self._filtered(table, where): + distance = _vector_distance(q, row.get("embedding")) + if distance is not None: + scored.append((distance, row)) + scored.sort(key=lambda item: item[0]) + out = [] + for distance, row in scored[:limit]: + item = { + "id": row["id"], + "document": row["document"], + "metadata": row.get("metadata") or {}, + "embedding": row.get("embedding") if with_embedding else None, + "distance": distance, + } + out.append(item) + return out + + def scroll_rows(self, table, *, where=None, with_embedding=False): + out = [] + for row in self._filtered(table, where): + out.append( + { + "id": row["id"], + "document": row["document"], + "metadata": row.get("metadata") or {}, + "embedding": row.get("embedding") if with_embedding else None, + "distance": None, + } + ) + return out + + def delete_rows(self, table, *, ids=None, where=None): + rows = self.tables.get(table, {"rows": {}})["rows"] + if ids is not None: + for doc_id in ids: + rows.pop(doc_id, None) + return + for doc_id, row in list(rows.items()): + if _matches_where(row.get("metadata") or {}, where): + rows.pop(doc_id, None) + + def count_rows(self, table): + return len(self.tables.get(table, {"rows": {}})["rows"]) + + def drop_table(self, table): + self.tables.pop(table, None) + + def close(self): + return None + + +@pytest.fixture +def fake_pgvector(monkeypatch): + import mempalace.backends.pgvector as pgvector + + _FakePgVectorClient.instances.clear() + monkeypatch.setattr(pgvector, "_PgVectorClient", _FakePgVectorClient) + monkeypatch.delenv("MEMPALACE_PGVECTOR_DSN", raising=False) + monkeypatch.delenv("MEMPALACE_PGVECTOR_NAMESPACE", raising=False) + return _FakePgVectorClient + + +def _collection(tmp_path, name="drawers"): + backend = PgVectorBackend() + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + return backend, backend.get_collection(palace=palace, collection_name=name, create=True) + + +def test_registry_exposes_pgvector(): + assert "pgvector" in available_backends() + + +def test_pgvector_add_query_filters_lexical_and_marker(tmp_path, fake_pgvector): + backend, col = _collection(tmp_path) + assert not os.path.isfile(tmp_path / "pgvector_backend.json") + + col.add( + ids=["a", "b", "c"], + documents=[ + "alpha backend note", + "rareterm pgvector backend note", + "frontend design note", + ], + metadatas=[ + {"wing": "project", "room": "backend", "rank": 1}, + {"wing": "project", "room": "backend", "rank": 3}, + {"wing": "project", "room": "frontend", "rank": 2}, + ], + embeddings=[[1, 0], [0.9, 0.1], [0, 1]], + ) + + assert PgVectorBackend.detect(str(tmp_path)) + assert os.path.isfile(tmp_path / "pgvector_backend.json") + assert col.count() == 3 + + # Equality filter is pushed down (no local fallback); $in stays pushdown. + result = col.query( + query_embeddings=[[1, 0]], + n_results=3, + where={"wing": "project"}, + include=["documents", "metadatas", "distances", "embeddings"], + ) + assert result.ids[0][0] == "a" + assert set(result.ids[0]) == {"a", "b", "c"} + assert result.embeddings[0][0] == pytest.approx([1.0, 0.0]) + + hits = col.lexical_search(query="rareterm backend", n_results=2, where={"wing": "project"}).hits + assert [hit.id for hit in hits] == ["b", "a"] + + backend.close_palace(str(tmp_path)) + with pytest.raises(Exception): + col.count() + + +def test_pgvector_requires_explicit_embeddings(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + with pytest.raises(ValueError, match="explicit embeddings"): + col.add(ids=["a"], documents=["no vector"], metadatas=[{}]) + + +def test_pgvector_marker_not_written_when_first_write_fails(tmp_path, fake_pgvector, monkeypatch): + _backend, col = _collection(tmp_path) + fake_client = fake_pgvector.instances[0] + + def fail_upsert(*_args, **_kwargs): + raise RuntimeError("pg unavailable") + + monkeypatch.setattr(fake_client, "upsert_rows", fail_upsert) + + with pytest.raises(RuntimeError): + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + + assert not os.path.isfile(tmp_path / "pgvector_backend.json") + + +def test_pgvector_dimension_mismatch(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + with pytest.raises(DimensionMismatchError): + col.upsert(ids=["b"], documents=["two"], metadatas=[{}], embeddings=[[1, 0, 0]]) + + +def test_pgvector_add_rejects_duplicate_ids_in_same_batch(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + with pytest.raises(ValueError, match="unique"): + col.add( + ids=["a", "a"], documents=["x", "y"], metadatas=[{}, {}], embeddings=[[1, 0], [0, 1]] + ) + + +def test_pgvector_complex_filters_use_local_fallback(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b", "c"], + documents=["alpha", "beta", "gamma"], + metadatas=[ + {"wing": "x", "rank": 1, "tags": "core,vector"}, + {"wing": "y", "rank": 3, "tags": "sqlite,exact"}, + {"wing": "z", "rank": 2, "tags": "old"}, + ], + embeddings=[[1, 0], [0.9, 0.1], [0, 1]], + ) + + # $or, $contains and comparisons must route to the local exact path and + # still return the correct rows. + or_hits = col.get(where={"$or": [{"wing": "x"}, {"wing": "z"}]}) + assert set(or_hits.ids) == {"a", "c"} + + contains = col.get(where={"tags": {"$contains": "sqlite"}}) + assert contains.ids == ["b"] + + ranked = col.query(query_embeddings=[[1, 0]], n_results=3, where={"rank": {"$gte": 2}}) + assert set(ranked.ids[0]) == {"b", "c"} + + +def test_pgvector_marker_participates_in_backend_mismatch(tmp_path, fake_pgvector): + from mempalace.palace import resolve_backend_name + + _backend, col = _collection(tmp_path) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + + assert resolve_backend_name(str(tmp_path)) == "pgvector" + with pytest.raises(BackendMismatchError): + resolve_backend_name(str(tmp_path), explicit="qdrant") + + +def test_pgvector_marker_rejects_target_change(tmp_path, fake_pgvector, monkeypatch): + _backend, col = _collection(tmp_path) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + + backend2 = PgVectorBackend() + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + with pytest.raises(BackendMismatchError): + backend2.get_collection( + palace=palace, + collection_name="drawers", + create=True, + options={"dsn": "postgresql://other-host:5432/other"}, + ) + + +def test_pgvector_rejects_pure_remote_palace(tmp_path, fake_pgvector): + """No local_path means the marker (the only mismatch-protection anchor) + cannot be written or validated, so the backend refuses rather than silently + opening an unprotected table (RFC 001 isolation contract, PR #1679).""" + backend = PgVectorBackend() + palace = PalaceRef(id="tenant-remote", local_path=None, namespace="tenant-remote") + with pytest.raises(BackendError, match="local palace path"): + backend.get_collection(palace=palace, collection_name="drawers", create=True) + + +def test_pgvector_missing_table_after_marker_is_not_initialized(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + fake_pgvector.instances[0].drop_table(col._table) + + assert col.health().ok is False + with pytest.raises(CollectionNotInitializedError): + col.count() + + +def test_pgvector_cross_palace_isolation_conformance(tmp_path, fake_pgvector): + """Shared per-PalaceRef.id isolation conformance (RFC 001 isolation contract).""" + backend = PgVectorBackend() + cols = [] + for label in ("alpha", "beta"): + path = tmp_path / label + ref = PalaceRef(id=str(path), local_path=str(path)) + cols.append(backend.get_collection(palace=ref, collection_name="drawers", create=True)) + # Same backend + same DSN → same client instance, distinct tables. + assert cols[0]._table != cols[1]._table + assert_partition_isolation(backend, cols[0], cols[1], embedding=[1.0, 0.0]) + + +def test_pgvector_namespace_isolation_conformance(tmp_path, fake_pgvector): + """Shared per-PalaceRef.namespace isolation conformance — pgvector advertises + ``supports_namespace_isolation`` (RFC 001 isolation contract).""" + assert "supports_namespace_isolation" in PgVectorBackend.capabilities + backend = PgVectorBackend() + ref_a = PalaceRef( + id=str(tmp_path / "tenant-a"), + local_path=str(tmp_path / "tenant-a"), + namespace="tenant-a", + ) + ref_b = PalaceRef( + id=str(tmp_path / "tenant-b"), + local_path=str(tmp_path / "tenant-b"), + namespace="tenant-b", + ) + col_a = backend.get_collection(palace=ref_a, collection_name="drawers", create=True) + col_b = backend.get_collection(palace=ref_b, collection_name="drawers", create=True) + # Mechanism: the namespace partitions the table name. + assert col_a._table != col_b._table + assert "tenant_a" in col_a._table and "tenant_b" in col_b._table + # Behaviour: a record under one namespace is invisible under the other. + assert_partition_isolation(backend, col_a, col_b, embedding=[1.0, 0.0]) + + +def test_pgvector_update_merges_documents_and_metadata(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b"], + documents=["alpha", "beta"], + metadatas=[{"wing": "x", "rank": 1}, {"wing": "y", "rank": 2}], + embeddings=[[1, 0], [0, 1]], + ) + col.update(ids=["a"], documents=["alpha-2"], metadatas=[{"rank": 9}]) + got = col.get(ids=["a"], include=["documents", "metadatas"]) + assert got.documents == ["alpha-2"] + # merge keeps the untouched key and overrides the updated one. + assert got.metadatas[0] == {"wing": "x", "rank": 9} + # untouched row is unchanged. + assert col.get(ids=["b"]).ids == ["b"] + with pytest.raises(ValueError, match="at least one"): + col.update(ids=["a"]) + + +def test_pgvector_get_limit_offset_and_embeddings(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b", "c"], + documents=["alpha", "beta", "gamma"], + metadatas=[{"wing": "x"}, {"wing": "x"}, {"wing": "x"}], + embeddings=[[1, 0], [0, 1], [0.5, 0.5]], + ) + page = col.get(where={"wing": "x"}, limit=1, offset=1, include=["documents", "embeddings"]) + assert len(page.ids) == 1 + assert page.embeddings is not None and len(page.embeddings[0]) == 2 + + +def test_pgvector_delete_by_where_pushdown_and_local(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b", "c"], + documents=["alpha", "beta", "gamma"], + metadatas=[{"wing": "x"}, {"wing": "y"}, {"wing": "z"}], + embeddings=[[1, 0], [0, 1], [0.5, 0.5]], + ) + # pushdown equality delete + col.delete(where={"wing": "y"}) + assert set(col.get().ids) == {"a", "c"} + # local-fallback delete ($or routes through the exact path) + col.delete(where={"$or": [{"wing": "x"}, {"wing": "z"}]}) + assert col.count() == 0 + + +def test_pgvector_query_dimension_mismatch_against_known_dim(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + col.add(ids=["a"], documents=["alpha"], metadatas=[{}], embeddings=[[1, 0]]) + with pytest.raises(DimensionMismatchError): + col.query(query_embeddings=[[1, 0, 0]], n_results=1) + + +def test_pgvector_get_collection_positional_and_palace_path_forms(tmp_path, fake_pgvector): + backend = PgVectorBackend() + col = backend.get_collection(str(tmp_path / "p1"), "drawers", create=True) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + assert col.count() == 1 + col2 = backend.get_collection( + palace_path=str(tmp_path / "p2"), collection_name="drawers", create=True + ) + col2.upsert(ids=["b"], documents=["two"], metadatas=[{}], embeddings=[[1, 0]]) + assert col2.count() == 1 + assert col._table != col2._table + + +def test_pgvector_health_and_delete_collection(tmp_path, fake_pgvector): + backend = PgVectorBackend() + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + col = backend.get_collection(palace=palace, collection_name="drawers", create=True) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + assert col.health().ok is True + assert backend.health(palace).ok is True + backend.delete_collection(str(tmp_path), "drawers") + assert col.health().ok is False + + +def test_pgvector_close_marks_backend_closed(tmp_path, fake_pgvector): + backend = PgVectorBackend() + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + col = backend.get_collection(palace=palace, collection_name="drawers", create=True) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + backend.close() + with pytest.raises(BackendError): + backend.get_collection(palace=palace, collection_name="drawers", create=True) + + +def test_pgvector_marker_unreadable_raises_mismatch(tmp_path, fake_pgvector): + _backend, col = _collection(tmp_path) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + marker = tmp_path / "pgvector_backend.json" + marker.write_text("{ not json", encoding="utf-8") + backend2 = PgVectorBackend() + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + with pytest.raises(BackendMismatchError): + backend2.get_collection(palace=palace, collection_name="drawers", create=True) + + +def test_pgvector_dsn_resolved_from_env(tmp_path, fake_pgvector, monkeypatch): + from mempalace.backends.pgvector import _PgVectorConfig + + monkeypatch.setenv("MEMPALACE_PGVECTOR_DSN", "postgresql://example:5432/memdb") + monkeypatch.setenv("MEMPALACE_PGVECTOR_NAMESPACE", "team-a") + config = _PgVectorConfig.from_options() + assert config.dsn == "postgresql://example:5432/memdb" + assert config.namespace == "team-a" + + +def test_palace_wrapper_embeds_for_pgvector(tmp_path, monkeypatch, fake_pgvector): + import mempalace.backends.embedding_wrapper as embedding_wrapper + from mempalace import palace + + monkeypatch.setattr( + embedding_wrapper, "_embed_texts", lambda texts: [[1.0, 0.0] for _ in texts] + ) + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "pgvector") + monkeypatch.setenv("MEMPALACE_BACKEND", "pgvector") + + col = palace.get_collection(str(tmp_path), "mempalace_drawers", create=True) + col.add(documents=["wrapped pgvector document"], ids=["wrapped"], metadatas=[{"wing": "w"}]) + result = col.query(query_texts=["wrapped"], n_results=1) + assert result.ids == [["wrapped"]] + + +def test_pgvector_live_roundtrip_when_enabled(tmp_path): + live_url = os.environ.get("MEMPALACE_PGVECTOR_LIVE_URL") + if not live_url: + pytest.skip("set MEMPALACE_PGVECTOR_LIVE_URL to run live Postgres pgvector test") + + backend = PgVectorBackend() + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path), namespace="livetest") + col = backend.get_collection( + palace=palace, + collection_name="drawers", + create=True, + options={"dsn": live_url}, + ) + try: + col.upsert( + ids=["live-a", "live-b"], + documents=["rareterm live pgvector backend", "other live document"], + metadatas=[{"wing": "live", "rank": 2}, {"wing": "other", "rank": 1}], + embeddings=[[1.0, 0.0], [0.0, 1.0]], + ) + assert PgVectorBackend.detect(str(tmp_path)) + assert col.count() == 2 + + result = col.query(query_embeddings=[[1.0, 0.0]], n_results=2, where={"wing": "live"}) + assert result.ids == [["live-a"]] + + hits = col.lexical_search(query="rareterm", n_results=1).hits + assert hits and hits[0].id == "live-a" + + col.delete(ids=["live-a"]) + assert col.get(ids=["live-a"]).ids == [] + + # Reopen the existing table in a fresh backend and write another + # same-dimension vector. This exercises table_dimension() against a + # live vector(n) column — a regression guard for reading the dimension + # off the raw atttypmod (which is not the bare n) and falsely raising + # DimensionMismatchError on reopen. + backend.close() + backend = PgVectorBackend() + reopened = backend.get_collection( + palace=palace, + collection_name="drawers", + create=False, + options={"dsn": live_url}, + ) + reopened.upsert( + ids=["live-c"], + documents=["third live document"], + metadatas=[{"wing": "live", "rank": 3}], + embeddings=[[0.5, 0.5]], + ) + assert reopened.count() == 2 + finally: + try: + backend.delete_collection(str(tmp_path), "drawers") + except Exception: + pass + backend.close() diff --git a/tests/test_qdrant_backend.py b/tests/test_qdrant_backend.py new file mode 100644 index 0000000..07e2113 --- /dev/null +++ b/tests/test_qdrant_backend.py @@ -0,0 +1,529 @@ +import os +import uuid + +import numpy as np +import pytest + +from _backend_conformance import assert_partition_isolation + +from mempalace.backends import ( + BackendError, + BackendMismatchError, + CollectionNotInitializedError, + DimensionMismatchError, + PalaceRef, + available_backends, +) +from mempalace.backends.qdrant import QdrantBackend + + +def _get_payload_value(payload, key): + value = payload + for part in key.split("."): + if not isinstance(value, dict): + return None + value = value.get(part) + return value + + +def _fake_match_condition(point, condition): + if "must" in condition or "must_not" in condition or "should" in condition: + return _fake_match_filter(point, condition) + if "has_id" in condition: + return point["id"] in set(condition["has_id"]) + key = condition.get("key") + actual = _get_payload_value(point.get("payload") or {}, key) + if "match" in condition: + match = condition["match"] + if "value" in match: + return actual == match["value"] + if "any" in match: + return actual in set(match["any"] or []) + if "text_any" in match: + haystack = str(actual or "").lower() + return any(token in haystack for token in str(match["text_any"]).lower().split()) + if "range" in condition: + range_spec = condition["range"] + try: + if "gt" in range_spec and not actual > range_spec["gt"]: + return False + if "gte" in range_spec and not actual >= range_spec["gte"]: + return False + if "lt" in range_spec and not actual < range_spec["lt"]: + return False + if "lte" in range_spec and not actual <= range_spec["lte"]: + return False + except TypeError: + return False + return True + return True + + +def _fake_match_filter(point, qdrant_filter): + if not qdrant_filter: + return True + must = qdrant_filter.get("must") or [] + must_not = qdrant_filter.get("must_not") or [] + should = qdrant_filter.get("should") or [] + if any(not _fake_match_condition(point, condition) for condition in must): + return False + if any(_fake_match_condition(point, condition) for condition in must_not): + return False + if should and not any(_fake_match_condition(point, condition) for condition in should): + return False + return True + + +class _FakeQdrantClient: + instances = [] + + def __init__(self, _config): + self.collections = {} + self.query_calls = [] + self.scroll_calls = [] + self.created_indexes = [] + _FakeQdrantClient.instances.append(self) + + def request(self, *_args, **_kwargs): + return {"result": {}} + + def collection_exists(self, collection): + return collection in self.collections + + def get_collection_info(self, collection): + if collection not in self.collections: + raise AssertionError("collection missing") + return { + "result": { + "config": { + "params": { + "vectors": { + "size": self.collections[collection]["dimension"], + "distance": "Cosine", + } + } + } + } + } + + def create_collection(self, collection, dimension): + self.collections.setdefault(collection, {"dimension": dimension, "points": {}}) + + def create_payload_index(self, collection, field_name, field_schema): + self.created_indexes.append((collection, field_name, field_schema)) + + def upsert_points(self, collection, points): + self.collections.setdefault( + collection, + {"dimension": len(points[0]["vector"]) if points else 0, "points": {}}, + ) + for point in points: + self.collections[collection]["points"][point["id"]] = dict(point) + + def query_points(self, collection, *, vector, limit, qdrant_filter, with_vector): + self.query_calls.append(qdrant_filter) + points = list(self.collections.get(collection, {"points": {}})["points"].values()) + points = [point for point in points if _fake_match_filter(point, qdrant_filter)] + q = np.asarray(vector, dtype=np.float32) + scored = [] + for point in points: + vec = np.asarray(point["vector"], dtype=np.float32) + denom = float(np.linalg.norm(q)) * float(np.linalg.norm(vec)) + score = 0.0 if denom <= 0 else float(np.dot(q, vec) / denom) + out = {"id": point["id"], "payload": point["payload"], "score": score} + if with_vector: + out["vector"] = point["vector"] + scored.append(out) + scored.sort(key=lambda point: point["score"], reverse=True) + return scored[:limit] + + def scroll_points( + self, + collection, + *, + qdrant_filter=None, + limit=256, + offset=None, + with_vector=False, + ): + self.scroll_calls.append(qdrant_filter) + points = list(self.collections.get(collection, {"points": {}})["points"].values()) + points = [point for point in points if _fake_match_filter(point, qdrant_filter)] + start = int(offset or 0) + selected = points[start : start + limit] + next_offset = start + limit if start + limit < len(points) else None + out = [] + for point in selected: + item = {"id": point["id"], "payload": point["payload"]} + if with_vector: + item["vector"] = point["vector"] + out.append(item) + return out, next_offset + + def delete_points(self, collection, *, point_ids=None, qdrant_filter=None): + points = self.collections.get(collection, {"points": {}})["points"] + if point_ids is not None: + for point_id in point_ids: + points.pop(point_id, None) + return + for point_id, point in list(points.items()): + if _fake_match_filter(point, qdrant_filter): + points.pop(point_id, None) + + def count_points(self, collection): + return len(self.collections.get(collection, {"points": {}})["points"]) + + def delete_collection(self, collection): + self.collections.pop(collection, None) + + +@pytest.fixture +def fake_qdrant(monkeypatch): + import mempalace.backends.qdrant as qdrant + + _FakeQdrantClient.instances.clear() + monkeypatch.setattr(qdrant, "_QdrantRESTClient", _FakeQdrantClient) + monkeypatch.delenv("MEMPALACE_QDRANT_URL", raising=False) + monkeypatch.delenv("MEMPALACE_QDRANT_API_KEY", raising=False) + monkeypatch.delenv("MEMPALACE_QDRANT_NAMESPACE", raising=False) + monkeypatch.delenv("MEMPALACE_QDRANT_TIMEOUT", raising=False) + return _FakeQdrantClient + + +def _collection(tmp_path, name="drawers"): + backend = QdrantBackend() + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + return backend, backend.get_collection(palace=palace, collection_name=name, create=True) + + +def test_registry_exposes_qdrant(): + assert "qdrant" in available_backends() + + +def test_qdrant_add_query_filters_lexical_and_marker(tmp_path, fake_qdrant): + backend, col = _collection(tmp_path) + assert not os.path.isfile(tmp_path / "qdrant_backend.json") + + col.add( + ids=["a", "b", "c"], + documents=[ + "alpha backend note", + "rareterm qdrant backend note", + "frontend design note", + ], + metadatas=[ + {"wing": "project", "room": "backend", "rank": 1}, + {"wing": "project", "room": "backend", "rank": 3}, + {"wing": "project", "room": "frontend", "rank": 2}, + ], + embeddings=[[1, 0], [0.9, 0.1], [0, 1]], + ) + + assert QdrantBackend.detect(str(tmp_path)) + assert os.path.isfile(tmp_path / "qdrant_backend.json") + assert col.count() == 3 + + result = col.query( + query_embeddings=[[1, 0]], + n_results=3, + where={"rank": {"$gte": 2}}, + include=["documents", "metadatas", "distances", "embeddings"], + ) + assert result.ids == [["b", "c"]] + assert result.documents[0][0] == "rareterm qdrant backend note" + assert result.embeddings[0][0] == pytest.approx([0.9, 0.1]) + + hits = col.lexical_search(query="rareterm backend", n_results=2, where={"wing": "project"}).hits + assert [hit.id for hit in hits] == ["b", "a"] + assert fake_qdrant.instances[0].created_indexes[0][1:] == ("document", "text") + + backend.close_palace(str(tmp_path)) + with pytest.raises(Exception): + col.count() + + +def test_qdrant_marker_not_written_when_first_write_fails(tmp_path, fake_qdrant, monkeypatch): + _backend, col = _collection(tmp_path) + fake_client = fake_qdrant.instances[0] + + def fail_upsert(*_args, **_kwargs): + raise RuntimeError("qdrant unavailable") + + monkeypatch.setattr(fake_client, "upsert_points", fail_upsert) + + with pytest.raises(RuntimeError): + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + + assert not os.path.isfile(tmp_path / "qdrant_backend.json") + + +def test_qdrant_upsert_update_delete_get_order_and_multi_collection(tmp_path, fake_qdrant): + backend, drawers = _collection(tmp_path, "drawers") + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + closets = backend.get_collection(palace=palace, collection_name="closets", create=True) + + drawers.upsert( + ids=["one", "two"], + documents=["first document", "second document"], + metadatas=[{"wing": "a"}, {"wing": "b"}], + embeddings=[[1, 0], [0, 1]], + ) + closets.upsert( + ids=["one"], + documents=["closet document"], + metadatas=[{"wing": "closet"}], + embeddings=[[0.5, 0.5]], + ) + + got = drawers.get(ids=["two", "one", "two"], include=["documents", "metadatas"]) + assert got.ids == ["two", "one", "two"] + assert got.documents == ["second document", "first document", "second document"] + + drawers.update(ids=["one"], metadatas=[{"room": "updated"}]) + assert drawers.get(ids=["one"]).metadatas == [{"wing": "a", "room": "updated"}] + + drawers.delete(where={"wing": "b"}) + assert drawers.get().ids == ["one"] + assert closets.get().ids == ["one"] + + +def test_qdrant_complex_filters_use_exact_local_fallback(tmp_path, fake_qdrant): + _backend, col = _collection(tmp_path) + col.upsert( + ids=["a", "b", "c"], + documents=[ + "needle exact substring", + "needle other wing", + "boring filler", + ], + metadatas=[ + {"wing": "target", "room": "backend", "tag": "alpha-beta"}, + {"wing": "other", "room": "backend", "tag": "beta"}, + {"wing": "target", "room": "front", "tag": "gamma"}, + ], + embeddings=[[1, 0], [0.8, 0.2], [0, 1]], + ) + fake_client = fake_qdrant.instances[0] + + result = col.query( + query_embeddings=[[1, 0]], + n_results=5, + where={"$or": [{"wing": "target"}, {"tag": {"$contains": "alpha"}}]}, + where_document={"$contains": "needle"}, + ) + + assert result.ids == [["a"]] + assert fake_client.query_calls == [] + + +def test_qdrant_lexical_empty_text_filter_does_not_full_scan(tmp_path, fake_qdrant): + _backend, col = _collection(tmp_path) + col.upsert( + ids=["a", "b"], + documents=["alpha backend note", "beta frontend note"], + metadatas=[{"wing": "project"}, {"wing": "project"}], + embeddings=[[1, 0], [0, 1]], + ) + fake_client = fake_qdrant.instances[0] + fake_client.scroll_calls.clear() + + hits = col.lexical_search(query="missingterm", n_results=2).hits + + assert hits == [] + assert len(fake_client.scroll_calls) == 1 + assert "text_any" in str(fake_client.scroll_calls[0]) + + +def test_qdrant_dimension_mismatch(tmp_path, fake_qdrant): + _backend, col = _collection(tmp_path) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + + with pytest.raises(DimensionMismatchError): + col.upsert(ids=["b"], documents=["two"], metadatas=[{}], embeddings=[[1, 0, 0]]) + + +def test_qdrant_add_rejects_duplicate_ids_in_same_batch(tmp_path, fake_qdrant): + _backend, col = _collection(tmp_path) + + with pytest.raises(ValueError, match="unique"): + col.add( + ids=["dup", "dup"], + documents=["first", "second"], + metadatas=[{}, {}], + embeddings=[[1, 0], [0, 1]], + ) + + assert not os.path.isfile(tmp_path / "qdrant_backend.json") + + +def test_qdrant_marker_participates_in_backend_mismatch(tmp_path, monkeypatch, fake_qdrant): + from mempalace.palace import resolve_backend_name + + backend, col = _collection(tmp_path) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + backend.close() + (tmp_path / "chroma.sqlite3").write_bytes(b"") + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "chroma") + + with pytest.raises(BackendMismatchError): + resolve_backend_name(str(tmp_path)) + + +def test_qdrant_marker_rejects_remote_target_change(tmp_path, monkeypatch, fake_qdrant): + backend, col = _collection(tmp_path) + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + + monkeypatch.setenv("MEMPALACE_QDRANT_URL", "http://other-qdrant.example:6333") + + with pytest.raises(BackendMismatchError, match="remote target"): + backend.get_collection(palace=palace, collection_name="drawers", create=False) + + +def test_qdrant_namespace_does_not_mix_palaces(tmp_path, fake_qdrant): + backend = QdrantBackend() + palace_a_path = tmp_path / "a" + palace_b_path = tmp_path / "b" + palace_a = PalaceRef(id=str(palace_a_path), local_path=str(palace_a_path), namespace="shared") + palace_b = PalaceRef(id=str(palace_b_path), local_path=str(palace_b_path), namespace="shared") + + col_a = backend.get_collection(palace=palace_a, collection_name="drawers", create=True) + col_b = backend.get_collection(palace=palace_b, collection_name="drawers", create=True) + col_a.upsert(ids=["same"], documents=["palace a"], metadatas=[{}], embeddings=[[1, 0]]) + col_b.upsert(ids=["same"], documents=["palace b"], metadatas=[{}], embeddings=[[1, 0]]) + + assert col_a.get(ids=["same"]).documents == ["palace a"] + assert col_b.get(ids=["same"]).documents == ["palace b"] + assert col_a._remote_collection != col_b._remote_collection + + +def test_qdrant_missing_remote_after_marker_is_unhealthy(tmp_path, fake_qdrant): + _backend, col = _collection(tmp_path) + col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) + fake_client = fake_qdrant.instances[0] + fake_client.delete_collection(col._remote_collection) + + assert col.health().ok is False + with pytest.raises(CollectionNotInitializedError): + col.count() + + +def test_search_reports_backend_error_distinct_from_missing_palace(tmp_path, monkeypatch): + from mempalace import searcher + + def fail_open(*_args, **_kwargs): + raise BackendError("qdrant unavailable") + + monkeypatch.setattr(searcher, "get_collection", fail_open) + + result = searcher.search_memories("needle", str(tmp_path)) + + assert result["error"] == "Backend error" + assert "qdrant unavailable" in result["details"] + + +def test_palace_wrapper_embeds_for_qdrant(tmp_path, monkeypatch, fake_qdrant): + import mempalace.backends.embedding_wrapper as embedding_wrapper + from mempalace import palace + + monkeypatch.setattr( + embedding_wrapper, "_embed_texts", lambda texts: [[1.0, 0.0] for _ in texts] + ) + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "qdrant") + monkeypatch.setenv("MEMPALACE_BACKEND", "qdrant") + + col = palace.get_collection(str(tmp_path), "mempalace_drawers", create=True) + col.add(documents=["wrapped qdrant document"], ids=["wrapped"], metadatas=[{"wing": "w"}]) + result = col.query(query_texts=["wrapped"], n_results=1) + assert result.ids == [["wrapped"]] + + +def test_qdrant_rejects_pure_remote_palace(tmp_path, fake_qdrant): + """No local_path means the marker (the only mismatch-protection anchor) + cannot be written or validated, so the backend must refuse rather than + silently open an unprotected remote collection (RFC 001 isolation contract, PR #1679).""" + backend = QdrantBackend() + palace = PalaceRef(id="tenant-remote", local_path=None, namespace="tenant-remote") + with pytest.raises(BackendError, match="local palace path"): + backend.get_collection(palace=palace, collection_name="drawers", create=True) + + +def test_qdrant_cross_palace_isolation_conformance(tmp_path, fake_qdrant): + """Shared per-PalaceRef.id isolation conformance (RFC 001 isolation contract).""" + backend = QdrantBackend() + cols = [] + for label in ("alpha", "beta"): + path = tmp_path / label + ref = PalaceRef(id=str(path), local_path=str(path)) + cols.append(backend.get_collection(palace=ref, collection_name="drawers", create=True)) + assert_partition_isolation(backend, cols[0], cols[1], embedding=[1.0, 0.0]) + + +def test_qdrant_namespace_isolation_conformance(tmp_path, fake_qdrant): + """Shared per-PalaceRef.namespace isolation conformance — qdrant advertises + ``supports_namespace_isolation`` so it must satisfy the cross-namespace MUST + (RFC 001 isolation contract).""" + assert "supports_namespace_isolation" in QdrantBackend.capabilities + backend = QdrantBackend() + ref_a = PalaceRef( + id=str(tmp_path / "tenant-a"), + local_path=str(tmp_path / "tenant-a"), + namespace="tenant-a", + ) + ref_b = PalaceRef( + id=str(tmp_path / "tenant-b"), + local_path=str(tmp_path / "tenant-b"), + namespace="tenant-b", + ) + col_a = backend.get_collection(palace=ref_a, collection_name="drawers", create=True) + col_b = backend.get_collection(palace=ref_b, collection_name="drawers", create=True) + # Mechanism: the namespace partitions the remote collection name. + assert col_a._remote_collection != col_b._remote_collection + # Behaviour: a record under one namespace is invisible under the other. + assert_partition_isolation(backend, col_a, col_b, embedding=[1.0, 0.0]) + + +def test_qdrant_live_rest_roundtrip_when_enabled(tmp_path): + live_url = os.environ.get("MEMPALACE_QDRANT_LIVE_URL") + if not live_url: + pytest.skip("set MEMPALACE_QDRANT_LIVE_URL to run live Qdrant REST test") + + backend = QdrantBackend() + namespace = f"live_{uuid.uuid4().hex}" + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path), namespace=namespace) + col = backend.get_collection( + palace=palace, + collection_name="drawers", + create=True, + options={ + "url": live_url, + "api_key": os.environ.get("MEMPALACE_QDRANT_LIVE_API_KEY"), + }, + ) + try: + col.upsert( + ids=["live-a", "live-b"], + documents=["rareterm live qdrant backend", "other live document"], + metadatas=[{"wing": "live", "rank": 2}, {"wing": "other", "rank": 1}], + embeddings=[[1.0, 0.0], [0.0, 1.0]], + ) + assert QdrantBackend.detect(str(tmp_path)) + + result = col.query( + query_embeddings=[[1.0, 0.0]], + n_results=2, + where={"wing": "live"}, + ) + assert result.ids == [["live-a"]] + + hits = col.lexical_search(query="rareterm", n_results=1).hits + assert hits and hits[0].id == "live-a" + + col.delete(ids=["live-a"]) + assert col.get(ids=["live-a"]).ids == [] + finally: + try: + col._client.delete_collection(col._remote_collection) + except Exception: + pass + backend.close() diff --git a/tests/test_repair.py b/tests/test_repair.py index 981351e..8824dce 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -1240,6 +1240,57 @@ def test_max_seq_id_backup_created(tmp_path): assert rows[seg["drawers_meta"]] == seg["poisoned_values"][seg["drawers_meta"]] +def test_max_seq_id_backup_pruned_to_max_backups(tmp_path, monkeypatch): + """Old max-seq-id backups beyond MEMPALACE_MAX_BACKUPS are pruned after a repair. + + Without retention, every repair left a full chroma.sqlite3 copy behind + that was never cleaned up — the unbounded disk-growth bug this guards. + """ + palace = str(tmp_path / "palace") + _seed_poisoned_max_seq_id(palace) + + # Pre-seed 4 stale backups with old mtimes so the just-created one is + # unambiguously the newest. + for i in range(4): + stale = os.path.join(palace, f"chroma.sqlite3.max-seq-id-backup-2026010{i}-000000") + with open(stale, "w") as f: + f.write("old") + os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i)) + + monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "2") + + result = repair.repair_max_seq_id(palace, assume_yes=True) + + backups = sorted( + fn for fn in os.listdir(palace) if fn.startswith("chroma.sqlite3.max-seq-id-backup-") + ) + # 4 stale + 1 fresh = 5 written; retention keeps only the 2 newest. + assert len(backups) == 2 + # The backup created by this repair must be one of the survivors. + assert os.path.basename(result["backup"]) in backups + + +def test_max_seq_id_backup_retained_when_pruning_disabled(tmp_path, monkeypatch): + """max_backups=0 keeps every backup (opt-out for external retention).""" + palace = str(tmp_path / "palace") + _seed_poisoned_max_seq_id(palace) + + for i in range(3): + stale = os.path.join(palace, f"chroma.sqlite3.max-seq-id-backup-2026010{i}-000000") + with open(stale, "w") as f: + f.write("old") + os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i)) + + monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "0") + + repair.repair_max_seq_id(palace, assume_yes=True) + + backups = [ + fn for fn in os.listdir(palace) if fn.startswith("chroma.sqlite3.max-seq-id-backup-") + ] + assert len(backups) == 4 + + def test_max_seq_id_rollback_on_verification_failure(tmp_path, monkeypatch): """If the post-update detector still sees poison, raise and leave a backup.""" palace = str(tmp_path / "palace") diff --git a/tests/test_searcher.py b/tests/test_searcher.py index 721bb11..236d5b9 100644 --- a/tests/test_searcher.py +++ b/tests/test_searcher.py @@ -343,8 +343,10 @@ class TestSearchCLI: # Non-zero bm25 reported assert "bm25=" in first_block assert "bm25=0.0" not in first_block - # Cosine still reported for transparency - assert "cosine=" in first_block + # Metric-labeled vector similarity still reported for transparency. + # Label is now "_sim=" (honest about the backend's metric) + # rather than a hard-coded "cosine=". + assert "cosine_sim=" in first_block def test_search_warns_when_palace_uses_wrong_distance_metric(self, fake_palace_path, capsys): """Legacy palaces created without `hnsw:space=cosine` silently diff --git a/tests/test_sqlite_exact_backend.py b/tests/test_sqlite_exact_backend.py new file mode 100644 index 0000000..b5b953e --- /dev/null +++ b/tests/test_sqlite_exact_backend.py @@ -0,0 +1,377 @@ +import math + +import pytest + +from mempalace.backends import ( + BackendMismatchError, + CollectionNotInitializedError, + DimensionMismatchError, + PalaceRef, + QueryResult, + UnsupportedCapabilityError, + available_backends, +) +from mempalace.backends.sqlite_exact import SQLiteExactBackend + + +def _collection(tmp_path, name="mempalace_drawers", create=True): + backend = SQLiteExactBackend() + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + return backend, backend.get_collection(palace=palace, collection_name=name, create=create) + + +def test_sqlite_exact_missing_collection_error_names_collection(tmp_path): + """CollectionNotInitializedError must identify the missing collection, not + the palace path — consistent with line 287 and the other backends.""" + backend, _ = _collection(tmp_path, name="mempalace_drawers") + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + with pytest.raises(CollectionNotInitializedError) as exc: + backend.get_collection(palace=palace, collection_name="does_not_exist", create=False) + assert "does_not_exist" in str(exc.value) + assert str(tmp_path) not in str(exc.value) + + with pytest.raises(CollectionNotInitializedError) as exc2: + backend.delete_collection(str(tmp_path), "also_missing") + assert "also_missing" in str(exc2.value) + assert str(tmp_path) not in str(exc2.value) + + +def test_registry_exposes_sqlite_exact(): + assert "sqlite_exact" in available_backends() + + +def test_sqlite_exact_add_query_filters_and_persistence(tmp_path): + backend, col = _collection(tmp_path) + col.add( + ids=["a", "b", "c"], + documents=[ + "alpha vector memory", + "beta sqlite exact memory", + "gamma filtered memory", + ], + metadatas=[ + {"wing": "alpha", "room": "notes", "chunk_index": 0, "tags": "core,vector"}, + {"wing": "alpha", "room": "notes", "chunk_index": 1, "tags": "sqlite,exact"}, + {"wing": "gamma", "room": "archive", "chunk_index": 2, "tags": "old"}, + ], + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.2, 0.8]], + ) + + ranked = col.query(query_embeddings=[[1.0, 0.0]], n_results=3) + assert ranked.ids[0] == ["a", "c", "b"] + assert ranked.distances[0][0] == pytest.approx(0.0) + + filtered = col.get( + where={ + "$and": [ + {"wing": "alpha"}, + {"chunk_index": {"$gte": 1}}, + {"tags": {"$contains": "sqlite"}}, + ] + }, + include=["documents", "metadatas", "embeddings"], + ) + assert filtered.ids == ["b"] + assert filtered.documents == ["beta sqlite exact memory"] + assert filtered.embeddings == [[0.0, 1.0]] + + col.update(ids=["b"], metadatas=[{"room": "lab"}]) + assert col.get(ids=["b"]).metadatas[0]["room"] == "lab" + + backend.close_palace(str(tmp_path)) + reopened = backend.get_collection( + palace=PalaceRef(id=str(tmp_path), local_path=str(tmp_path)), + collection_name="mempalace_drawers", + create=False, + ) + assert reopened.count() == 3 + assert reopened.get(ids=["a"]).documents == ["alpha vector memory"] + + +def test_sqlite_exact_write_failure_rolls_back_whole_batch(tmp_path): + _backend, col = _collection(tmp_path) + + with pytest.raises(Exception): + col.add( + ids=["dup", "dup"], + documents=["first write", "duplicate write"], + metadatas=[{}, {}], + embeddings=[[1.0, 0.0], [0.0, 1.0]], + ) + + assert col.count() == 0 + + +def test_sqlite_exact_enforces_collection_dimension(tmp_path): + _backend, col = _collection(tmp_path) + col.add(ids=["a"], documents=["two dims"], metadatas=[{}], embeddings=[[1.0, 0.0]]) + + with pytest.raises(DimensionMismatchError): + col.add(ids=["b"], documents=["three dims"], metadatas=[{}], embeddings=[[1.0, 0.0, 0.0]]) + with pytest.raises(DimensionMismatchError): + col.upsert( + ids=["b"], documents=["three dims"], metadatas=[{}], embeddings=[[1.0, 0.0, 0.0]] + ) + with pytest.raises(DimensionMismatchError): + col.update(ids=["a"], embeddings=[[1.0, 0.0, 0.0]]) + with pytest.raises(DimensionMismatchError): + col.query(query_embeddings=[[1.0, 0.0, 0.0]], n_results=1) + + assert col.count() == 1 + assert col.get(ids=["a"]).documents == ["two dims"] + + +def test_sqlite_exact_get_preserves_requested_id_order_and_duplicates(tmp_path): + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b"], + documents=["doc a", "doc b"], + metadatas=[{}, {}], + embeddings=[[1, 0], [0, 1]], + ) + + result = col.get(ids=["b", "a", "b"], include=["documents"]) + + assert result.ids == ["b", "a", "b"] + assert result.documents == ["doc b", "doc a", "doc b"] + + +def test_sqlite_exact_upsert_delete_and_multi_collection_isolation(tmp_path): + backend, drawers = _collection(tmp_path, "drawers") + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + closets = backend.get_collection(palace=palace, collection_name="closets", create=True) + + drawers.upsert( + ids=["same"], documents=["drawer one"], metadatas=[{"kind": "drawer"}], embeddings=[[1, 0]] + ) + closets.upsert( + ids=["same"], documents=["closet one"], metadatas=[{"kind": "closet"}], embeddings=[[0, 1]] + ) + drawers.upsert( + ids=["same"], + documents=["drawer replaced"], + metadatas=[{"kind": "drawer", "version": 2}], + embeddings=[[1, 0]], + ) + + assert drawers.count() == 1 + assert closets.count() == 1 + assert drawers.get(ids=["same"]).documents == ["drawer replaced"] + assert closets.get(ids=["same"]).documents == ["closet one"] + + drawers.delete(where={"version": {"$in": [2, 3]}}) + assert drawers.count() == 0 + assert closets.count() == 1 + + +def test_sqlite_exact_lexical_search_and_python_fallback(tmp_path, monkeypatch): + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b", "c"], + documents=[ + "ordinary project note", + "rareterm rareterm sqlite exact note", + "rareterm unrelated archive", + ], + metadatas=[ + {"wing": "w", "room": "a"}, + {"wing": "w", "room": "b"}, + {"wing": "old", "room": "b"}, + ], + embeddings=[[1, 0], [0, 1], [0.5, 0.5]], + ) + + hits = col.lexical_search(query="rareterm sqlite", n_results=2, where={"wing": "w"}).hits + assert [hit.id for hit in hits] == ["b"] + + monkeypatch.setattr(col, "_fts_available", lambda _cur: False) + fallback_hits = col.lexical_search(query="rareterm sqlite", n_results=2).hits + assert fallback_hits[0].id == "b" + + +def test_sqlite_exact_lexical_search_filters_after_full_fts_window(tmp_path): + _backend, col = _collection(tmp_path) + ids = [f"old-{i}" for i in range(12)] + ["target"] + col.add( + ids=ids, + documents=["needle shared lexical note" for _ in ids], + metadatas=[{"wing": "old"} for _ in range(12)] + [{"wing": "target"}], + embeddings=[[1.0, 0.0] for _ in ids], + ) + + hits = col.lexical_search(query="needle", n_results=1, where={"wing": "target"}).hits + + assert [hit.id for hit in hits] == ["target"] + + +def test_sqlite_exact_logical_filters_evaluate_sibling_predicates(tmp_path): + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b"], + documents=["alpha document", "beta document"], + metadatas=[ + {"wing": "w", "room": "wrong", "kind": "note"}, + {"wing": "w", "room": "right", "kind": "note"}, + ], + embeddings=[[1, 0], [0, 1]], + ) + + result = col.get(where={"$and": [{"wing": "w"}], "room": "right"}) + + assert result.ids == ["b"] + + +def test_sqlite_exact_close_palace_marks_existing_collections_closed(tmp_path): + backend, col = _collection(tmp_path) + palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path)) + col.add(ids=["a"], documents=["doc"], metadatas=[{}], embeddings=[[1, 0]]) + + backend.close_palace(palace) + + assert not col.health().ok + with pytest.raises(Exception): + col.count() + + +def test_palace_wrapper_embeds_for_sqlite_exact(tmp_path, monkeypatch): + import mempalace.backends.embedding_wrapper as embedding_wrapper + from mempalace.palace import get_collection + + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact") + monkeypatch.setattr( + embedding_wrapper, + "_embed_texts", + lambda texts: [[float(len(text)), 1.0] for text in texts], + ) + + col = get_collection(str(tmp_path), create=True) + col.add(ids=["a"], documents=["abcd"], metadatas=[{"wing": "w"}]) + + result = col.query(query_texts=["abcd"], n_results=1) + assert result.ids == [["a"]] + + +def test_backend_mismatch_protection(tmp_path, monkeypatch): + from mempalace.palace import get_collection + + (tmp_path / "chroma.sqlite3").write_bytes(b"") + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact") + + with pytest.raises(BackendMismatchError): + get_collection(str(tmp_path), create=True) + + +def test_mixed_backend_artifacts_are_rejected_even_when_chroma_selected(tmp_path, monkeypatch): + from mempalace.palace import resolve_backend_name + + (tmp_path / "chroma.sqlite3").write_bytes(b"") + (tmp_path / "sqlite_exact.sqlite3").write_bytes(b"") + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "chroma") + + with pytest.raises(BackendMismatchError): + resolve_backend_name(str(tmp_path)) + + +def test_sqlite_exact_exact_ranking_uses_cosine(tmp_path): + _backend, col = _collection(tmp_path) + halfway = [0.5, math.sqrt(0.75)] + col.add( + ids=["half", "orthogonal", "same"], + documents=["half", "orthogonal", "same"], + metadatas=[{}, {}, {}], + embeddings=[halfway, [0.0, 1.0], [1.0, 0.0]], + ) + + result = col.query(query_embeddings=[[1.0, 0.0]], n_results=3) + assert result.ids[0] == ["same", "half", "orthogonal"] + assert result.distances[0] == pytest.approx([0.0, 0.5, 1.0]) + + +def test_search_union_uses_sqlite_exact_lexical_search(tmp_path, monkeypatch): + import mempalace.backends.embedding_wrapper as embedding_wrapper + from mempalace.palace import get_collection + from mempalace.searcher import search_memories + + def fake_embed(texts): + vectors = [] + for text in texts: + if text == "rareterm": + vectors.append([1.0, 0.0]) + elif "rareterm" in text: + vectors.append([0.0, 1.0]) + else: + vectors.append([0.5, math.sqrt(0.75)]) + return vectors + + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact") + monkeypatch.setattr(embedding_wrapper, "_embed_texts", fake_embed) + + col = get_collection(str(tmp_path), create=True) + col.add( + ids=["d1", "d2", "d3", "rare"], + documents=[ + "ordinary support note", + "ordinary billing note", + "ordinary project note", + "rareterm rareterm rareterm policy note", + ], + metadatas=[ + {"wing": "w", "room": "r", "source_file": "/tmp/d1.md", "chunk_index": 0}, + {"wing": "w", "room": "r", "source_file": "/tmp/d2.md", "chunk_index": 0}, + {"wing": "w", "room": "r", "source_file": "/tmp/d3.md", "chunk_index": 0}, + {"wing": "w", "room": "r", "source_file": "/tmp/rare.md", "chunk_index": 0}, + ], + ) + + result = search_memories( + "rareterm", + str(tmp_path), + n_results=1, + candidate_strategy="union", + ) + + assert result["results"][0]["source_file"] == "rare.md" + assert result["results"][0]["matched_via"] == "bm25_backend" + + +def test_search_union_reports_unsupported_lexical_capability(monkeypatch, tmp_path): + import mempalace.searcher as searcher + + class NoLexicalCollection: + def query(self, **_kwargs): + return QueryResult( + ids=[["a"]], + documents=[["ordinary note"]], + metadatas=[[{"source_file": "/tmp/a.md", "chunk_index": 0}]], + distances=[[0.5]], + ) + + def lexical_search(self, **_kwargs): + raise UnsupportedCapabilityError("no lexical support") + + monkeypatch.setattr(searcher, "get_collection", lambda *_args, **_kwargs: NoLexicalCollection()) + monkeypatch.setattr( + searcher, + "get_closets_collection", + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("no closets")), + ) + + result = searcher.search_memories( + "anything", + str(tmp_path), + n_results=1, + candidate_strategy="union", + ) + + assert result["unsupported_capability"] == "supports_lexical_search" + + +def test_search_vector_disabled_fallback_is_chroma_only(tmp_path, monkeypatch): + from mempalace.searcher import search_memories + + monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact") + + result = search_memories("anything", str(tmp_path), vector_disabled=True) + + assert result["unsupported_capability"] == "chroma_hnsw_fallback" + assert result["backend"] == "sqlite_exact" diff --git a/uv.lock b/uv.lock index bd24fa9..2a5ce27 100644 --- a/uv.lock +++ b/uv.lock @@ -1999,6 +1999,10 @@ gpu = [ { name = "onnxruntime-gpu", version = "1.24.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "onnxruntime-gpu", version = "1.25.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] +pgvector = [ + { name = "psycopg", version = "3.2.13", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version < '3.10'" }, + { name = "psycopg", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version >= '3.10'" }, +] spellcheck = [ { name = "autocorrect" }, ] @@ -2032,16 +2036,17 @@ requires-dist = [ { name = "onnxruntime-gpu", marker = "extra == 'gpu'", specifier = ">=1.16" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.0" }, { name = "psutil", marker = "extra == 'dev'", specifier = ">=5.9" }, + { name = "psycopg", extras = ["binary"], marker = "extra == 'pgvector'", specifier = ">=3.1" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0" }, { name = "python-dateutil", specifier = ">=2.8" }, { name = "pyyaml", specifier = ">=6.0,<7" }, - { name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.14" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.15" }, { name = "striprtf", marker = "extra == 'extract'", specifier = ">=0.0.27" }, { name = "tokenizers", specifier = ">=0.15" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" }, ] -provides-extras = ["dev", "spellcheck", "gpu", "dml", "coreml", "multilingual", "extract"] +provides-extras = ["dev", "spellcheck", "pgvector", "gpu", "dml", "coreml", "multilingual", "extract"] [package.metadata.requires-dev] dev = [ @@ -2051,7 +2056,7 @@ dev = [ { name = "psutil", specifier = ">=5.9" }, { name = "pytest", specifier = ">=7.0" }, { name = "pytest-cov", specifier = ">=4.0" }, - { name = "ruff", specifier = "==0.15.14" }, + { name = "ruff", specifier = "==0.15.15" }, ] [[package]] @@ -3704,6 +3709,199 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] +[[package]] +name = "psycopg" +version = "3.2.13" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "tzdata", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/05/d4a05988f15fcf90e0088c735b1f2fc04a30b7fc65461d6ec278f5f2f17a/psycopg-3.2.13.tar.gz", hash = "sha256:309adaeda61d44556046ec9a83a93f42bbe5310120b1995f3af49ab6d9f13c1d", size = 160626, upload-time = "2025-11-21T22:34:32.328Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/14/f2724bd1986158a348316e86fdd0837a838b14a711df3f00e47fba597447/psycopg-3.2.13-py3-none-any.whl", hash = "sha256:a481374514f2da627157f767a9336705ebefe93ea7a0522a6cbacba165da179a", size = 206797, upload-time = "2025-11-21T22:29:39.733Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", version = "3.2.13", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'win32'", + "python_full_version == '3.10.*' and sys_platform == 'win32'", + "python_full_version == '3.10.*' and sys_platform != 'win32'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "tzdata", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.2.13" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/16/325f72b7ebdb906bd6cca6c0caea5b8fd7092c4686237c5669fe3f3cc7f2/psycopg_binary-3.2.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9e25eb65494955c0dabdcd7097b004cbd70b982cf3cbc7186c2e854f788677a9", size = 4013642, upload-time = "2025-11-21T22:29:43.39Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a6/f7616dfcab942d5ad6fb5ce8364148e22a4cd817340ac368b6a6bd17559d/psycopg_binary-3.2.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:732b25c2d932ca0655ea2588563eae831dc0842c93c69be4754a5b0e9760b38d", size = 4076666, upload-time = "2025-11-21T22:29:51.33Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f7/cddf75c43c967c9262afe6863275fdd2e5f877d98c379f5c3a21b6fa419d/psycopg_binary-3.2.13-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7350d9cc4e35529c4548ddda34a1c17f28d3f3a8f792c25cd67e8a04952ed415", size = 4639390, upload-time = "2025-11-21T22:29:57.614Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b9/f86f2e6413ac024b3a759fd446cc90c325a0d7403dce533bd419e1c41164/psycopg_binary-3.2.13-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:090c22795969ee1ace17322b1718769694607d942cef084c6fb4493adfa57da0", size = 4737745, upload-time = "2025-11-21T22:30:01.814Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/1a17c7176875d7e0a848710d87f13fdd3cc08724fa6bfcc43c72846f22b9/psycopg_binary-3.2.13-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ac329532f36342ff99fc1aefdbb531563bec03c7bc3ae934c8347a7a61339df", size = 4419762, upload-time = "2025-11-21T22:30:05.401Z" }, + { url = "https://files.pythonhosted.org/packages/a3/9b/5c7f8c90a3504c45ceadffa1f1f4b2fc8ce9e04494cf67d27dfa265e5681/psycopg_binary-3.2.13-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1db11a7e618d58cfb937c409c7d279a84cbb31d32a7efc63f1e5f426f3613793", size = 3878529, upload-time = "2025-11-21T22:30:09.493Z" }, + { url = "https://files.pythonhosted.org/packages/ea/37/37e7152e6b0813e68361768d1baf0e40d8ed0ac8091471641c2c88e0cec6/psycopg_binary-3.2.13-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:5f5081b2cbb0358bb3625109d41b57411bf9d9c29762a867e38c06d974b245ee", size = 3560767, upload-time = "2025-11-21T22:30:13.88Z" }, + { url = "https://files.pythonhosted.org/packages/f7/b2/929d8e15b8797486d160b797ce84a4d0251a9361f7f31e9b01b439608e3b/psycopg_binary-3.2.13-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5d466ac3a3738647ff2405397946870dc363e33282ced151e7ea74f622947c06", size = 3604456, upload-time = "2025-11-21T22:30:18.392Z" }, + { url = "https://files.pythonhosted.org/packages/c7/74/4d4e7481bc717bbe3de689c4d40439d4e1be07df989da2c38140298cbae5/psycopg_binary-3.2.13-cp310-cp310-win_amd64.whl", hash = "sha256:087acf2b24787ae206718136c1f51bc90cda68b02c3819b0556f418e3565f2c3", size = 2910871, upload-time = "2025-11-21T22:30:22.24Z" }, + { url = "https://files.pythonhosted.org/packages/06/f5/fc70804a999167daf5b876107b99e8fe91c3f785a31753c0e3e7b93446ba/psycopg_binary-3.2.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9cfe87749d010dfd34534ba8c71aa0674db9a3fce65232c98989f77c742c9ce7", size = 4013844, upload-time = "2025-11-21T22:30:25.985Z" }, + { url = "https://files.pythonhosted.org/packages/07/87/857639681f5dfcd567aaf199fe4e5b026a105b0462a604f4fb7eda0735d8/psycopg_binary-3.2.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8db77fac1dfe3f69c982db92a51fd78e1354fa8f523a6781a636123e5c7ffcde", size = 4077002, upload-time = "2025-11-21T22:30:29.539Z" }, + { url = "https://files.pythonhosted.org/packages/7c/1d/2cb7af6a31429b9022455c966d8408a2b5a19acd3de7610402381518e8f7/psycopg_binary-3.2.13-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cbbac4cd5b0e14b91ad8244268ca3fc2f527d1a337b489af57d7669c9d2e1a24", size = 4637181, upload-time = "2025-11-21T22:30:34.126Z" }, + { url = "https://files.pythonhosted.org/packages/28/bd/ffde1ac7e6ab75646c253fbe0378772fb6f0229af8a05cd9862ee8aad0f0/psycopg_binary-3.2.13-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a146f0a59a7e3ca92996f8133b1d5e5922e668f7c656b4a9201e702f4cf25896", size = 4737775, upload-time = "2025-11-21T22:30:38.408Z" }, + { url = "https://files.pythonhosted.org/packages/c2/74/3702732d01639c97943d56ec26860357dfacda0b5a708e82e794d07f499c/psycopg_binary-3.2.13-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:27150515de5f709e4142429db6fd36a1d01f0b8b17d915b5f7bb095364465398", size = 4421537, upload-time = "2025-11-21T22:30:42.696Z" }, + { url = "https://files.pythonhosted.org/packages/f2/8c/915a899857c2211196aa7f1749ba85bed421afaf72f185a0eb91e64ba550/psycopg_binary-3.2.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9942255705255367d94368941e3a913b0daf74b47d191471dbe4dc0de9fbc769", size = 3877500, upload-time = "2025-11-21T22:30:47.064Z" }, + { url = "https://files.pythonhosted.org/packages/36/d9/46060c183413bf62d47df98d7e3b30ab561639bcb583c3796cca30dafa43/psycopg_binary-3.2.13-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:75ebc8335f48c339ec24f4c371595f6b7043147fe6d18e619c8564428ab8adaf", size = 3560186, upload-time = "2025-11-21T22:30:54.522Z" }, + { url = "https://files.pythonhosted.org/packages/56/cf/2987689614632898e4861e4122cd41937ea9b5afcbe3c3061c7265bfa6de/psycopg_binary-3.2.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6fe2982a73b2ea473c9e2b91a35a21af3b03313bed188eccbcde4972483ac60a", size = 3601117, upload-time = "2025-11-21T22:31:01.218Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ef/df7fa8a47ef47d08af8a792343811a98bc7ab48f763560fc1d5acc1f28af/psycopg_binary-3.2.13-cp311-cp311-win_amd64.whl", hash = "sha256:6a50db4661fae78779d3cc38a0a68cabc997ca9d485ec27443b109ef8ac1672a", size = 2912873, upload-time = "2025-11-21T22:31:05.473Z" }, + { url = "https://files.pythonhosted.org/packages/49/9e/f90243b3d0d007a89989b013b0eb3e78ac929fed4eb40a2b317452abafe1/psycopg_binary-3.2.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:223fc610a80bbc4355ad3c9952d468a18bb5cd7065846a8c275f100d80cd4004", size = 3996285, upload-time = "2025-11-21T22:31:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/12/42/7d55f515ee3e2ced5ff9bc493fb2308f5187686b6d9583cd6a9c880d2053/psycopg_binary-3.2.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b67f06a68d68b4621b6a411f9e583df876977afa06b1ba270b1b347d40aa93fc", size = 4070567, upload-time = "2025-11-21T22:31:12.31Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a8/ead4de04d8cf5f35119a75a8dd92fa4a2ec8a309b1aa58855f64616c03d7/psycopg_binary-3.2.13-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:082579f2ae41bdabe20c82810810f3e290ac2206cccf0cb41cf36b3218f53b3c", size = 4616833, upload-time = "2025-11-21T22:31:16.614Z" }, + { url = "https://files.pythonhosted.org/packages/26/2e/4af6ab69ade7d67d31296f88c79c322a3522564e30b3f1458f19e74d67c3/psycopg_binary-3.2.13-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:ff7df7bd8ec2c805f3a4896b8ade971139af0f9f8cf45d05014ac71fe54887be", size = 4711710, upload-time = "2025-11-21T22:31:22.007Z" }, + { url = "https://files.pythonhosted.org/packages/9a/31/bdbd6b2264bb7ae5fe8b775c5524da73329d8888c6137fd8b050ff9cabbc/psycopg_binary-3.2.13-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8f1189dc78553ef4b2e55d9e116fc74870191bc6a9a5f4442412a703c4cc6c3b", size = 4401656, upload-time = "2025-11-21T22:31:26.842Z" }, + { url = "https://files.pythonhosted.org/packages/33/c5/8fd8f96450e4ef242022c9a588305e3dc7309c34bc392a9b4c2da60854b1/psycopg_binary-3.2.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0ef8ed4a4e0f7bf5e941782478a43c14b2b585b031e2266dd3afb87be2775d95", size = 3851747, upload-time = "2025-11-21T22:31:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/4a/47/406d102ae49d253f124644530f1e5b3fd2f92aea59d4f9b8dd1c71cf8e0f/psycopg_binary-3.2.13-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:de06fc9707a49f7c081b5c950974dd6de3dc33d681f7524f0b396471f5a4a480", size = 3524796, upload-time = "2025-11-21T22:31:34.377Z" }, + { url = "https://files.pythonhosted.org/packages/45/6f/a89be8aee27a5522e97dbcb225fe429c489acdf0bb25fc0fadb329dfb39f/psycopg_binary-3.2.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:917ad1cd6e6ef8a9df2f28d7b29c7148f089be46ac56fe838f986c0227652d14", size = 3576536, upload-time = "2025-11-21T22:31:38.06Z" }, + { url = "https://files.pythonhosted.org/packages/ef/f8/c924c7dc792c81bf6181d7d4eeb613c8b2151b3a208f95cedec3c1a25ba3/psycopg_binary-3.2.13-cp312-cp312-win_amd64.whl", hash = "sha256:b53b0d9499805b307017070492189e349256e0946f62c815e442baa01f2ea6c5", size = 2902172, upload-time = "2025-11-21T22:31:41.256Z" }, + { url = "https://files.pythonhosted.org/packages/28/ec/ef37bb44dc02fcc6c0a3eeb93f4baaac13bcb228633fe38ad3fb5a3f6449/psycopg_binary-3.2.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dbae6ab1966e2b61d97e47220556c330c4608bb4cfb3a124aa0595c39995c068", size = 3995628, upload-time = "2025-11-21T22:31:45.921Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ad/4748f5f1a40248af16dba087dbec50bd335ee025cc1fb9bf64773378ceff/psycopg_binary-3.2.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fae933e4564386199fc54845d85413eedb49760e0bcd2b621fde2dd1825b99b3", size = 4069024, upload-time = "2025-11-21T22:31:50.202Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c2/f02ec6bbc30c7fcd3b39823d2d624b42fae480edeb6e50eb3276281d5635/psycopg_binary-3.2.13-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:13e2f8894d410678529ff9f1211f96c5a93ff142f992b302682b42d924428b61", size = 4615127, upload-time = "2025-11-21T22:31:56.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/0d/a54fc2cdd672c84175d6869cc823d6ec2a8909318d491f3c24e6077983f2/psycopg_binary-3.2.13-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f26f7009375cf1e92180e5c517c52da1054f7e690dde90e0ed00fa8b5736bcd4", size = 4710267, upload-time = "2025-11-21T22:32:04.585Z" }, + { url = "https://files.pythonhosted.org/packages/9d/b7/067de1acaf3d312253351f3af4121f972584bd36cada6378d4b0cdcebd38/psycopg_binary-3.2.13-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea2fdbcc9142933a47c66970e0df8b363e3bd1ea4c5ce376f2f3d94a9aeec847", size = 4400795, upload-time = "2025-11-21T22:32:08.883Z" }, + { url = "https://files.pythonhosted.org/packages/64/b5/030e6b1ebfc4d3a8fca03adc5fc827982643bad0b01a1268538d17c08ed3/psycopg_binary-3.2.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac92d6bc1d4a41c7459953a9aa727b9966e937e94c9e072527317fd2a67d488b", size = 3851239, upload-time = "2025-11-21T22:32:12.333Z" }, + { url = "https://files.pythonhosted.org/packages/79/6f/0541845364a7de9eae6807060da6a04b22a8eb2e803606d285d9250fbe93/psycopg_binary-3.2.13-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:8b843c00478739e95c46d6d3472b13123b634685f107831a9bfc41503a06ecbd", size = 3525084, upload-time = "2025-11-21T22:32:15.946Z" }, + { url = "https://files.pythonhosted.org/packages/83/ae/6507890dc30a4bbd9d938d4ff3a4079d009a5ad8170af51c7f762438fdbf/psycopg_binary-3.2.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2f63868cc96bc18486cebec24445affbdd7f7debf28fac466ea935a8b5a4753b", size = 3576787, upload-time = "2025-11-21T22:32:19.922Z" }, + { url = "https://files.pythonhosted.org/packages/9d/64/3d1c2f1fd09b60cdfbe68b9a810b357ba505eff6e4bdb1a2d9f6729da64c/psycopg_binary-3.2.13-cp313-cp313-win_amd64.whl", hash = "sha256:594dfbca3326e997ae738d3d339004e8416b1f7390f52ce8dc2d692393e8fa96", size = 2905584, upload-time = "2025-11-21T22:32:23.399Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b4/7656b3d67bedff2b900c8c4671cb6eb5fb99c2fc36da33579cac89779c25/psycopg_binary-3.2.13-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:502a778c3e07c6b3aabfa56ee230e8c264d2debfab42d11535513a01bdfff0d6", size = 3997201, upload-time = "2025-11-21T22:32:28.185Z" }, + { url = "https://files.pythonhosted.org/packages/e0/2e/3b4afbd94d48df19c3931cedba464b109f89d81ac43178e6a3d654b4e8d5/psycopg_binary-3.2.13-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7561a71d764d6f74d66e8b7d844b0f27fa33de508f65c17b1d56a94c73644776", size = 4071631, upload-time = "2025-11-21T22:32:32.594Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8b/107d06d55992e2f13157eb705ba5a47d06c4cf1bed077dff0c567b10c187/psycopg_binary-3.2.13-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9caf14745a1930b4e03fe4072cd7154eaf6e1241d20c42130ed784408a26b24b", size = 4620918, upload-time = "2025-11-21T22:32:37.357Z" }, + { url = "https://files.pythonhosted.org/packages/e1/47/a925620f261b115f31e813a5bfe640f316413b1864094a60162f4a6e4d67/psycopg_binary-3.2.13-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a6cafabdc0bfa37e11c6f365020fd5916b62d6296df581f4dceaa43a2ce680c", size = 4714494, upload-time = "2025-11-21T22:32:42.138Z" }, + { url = "https://files.pythonhosted.org/packages/46/33/bed384665356bb9ba17dd8e104884d87cc2343d16dffdfd9aaa9a159bd4d/psycopg_binary-3.2.13-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96cb5a27e68acac6d74b64fca38592a692de9c4b7827339190698d58027aa45", size = 4403046, upload-time = "2025-11-21T22:32:47.241Z" }, + { url = "https://files.pythonhosted.org/packages/41/88/749d8e8102fb5df502e2ecb053b79e78e3358af01af652b5dbeb96ab7905/psycopg_binary-3.2.13-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:596176ae3dfbf56fc61108870bfe17c7205d33ac28d524909feb5335201daa0a", size = 3859046, upload-time = "2025-11-21T22:32:51.481Z" }, + { url = "https://files.pythonhosted.org/packages/38/7c/f492e63b517d6dcd564e8c43bc15e11a4c712a848adf8938ce33bfd4c867/psycopg_binary-3.2.13-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:cc3a0408435dfbb77eeca5e8050df4b19a6e9b7e5e5583edf524c4a83d6293b2", size = 3531351, upload-time = "2025-11-21T22:32:55.571Z" }, + { url = "https://files.pythonhosted.org/packages/07/5a/d8743eb23944e5cf2a0bbfa92935c140b5beaacdb872be641065ed70ab2c/psycopg_binary-3.2.13-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:65df0d459ffba14082d8ca4bb2f6ffbb2f8d02968f7d34a747e1031934b76b23", size = 3581034, upload-time = "2025-11-21T22:33:01.648Z" }, + { url = "https://files.pythonhosted.org/packages/46/b2/411d4180252144f7eff024894d2d2ebb98c012c944a282fc20250870e461/psycopg_binary-3.2.13-cp314-cp314-win_amd64.whl", hash = "sha256:5c77f156c7316529ed371b5f95a51139e531328ee39c37493a2afcbc1f79d5de", size = 3000162, upload-time = "2025-11-21T22:33:07.378Z" }, + { url = "https://files.pythonhosted.org/packages/80/dc/3ea3fe5df19af323b4b78e0e98e073f8117b1336e5b6dc6978c067485019/psycopg_binary-3.2.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d8d1b709509d0f8cb857acf740b5eccd5bd2fb208a5b20e895f250519a32459", size = 4015148, upload-time = "2025-11-21T22:33:47.539Z" }, + { url = "https://files.pythonhosted.org/packages/e1/28/a832b014974e7bda61b3c684afe5e47f70d5dc4471cbab90a41a7c2bdf6a/psycopg_binary-3.2.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d45bc5f4335498d32a26c8f8c0bf9ce8c973c19e78a9ee77c031300fb361300", size = 4078197, upload-time = "2025-11-21T22:33:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/5c/8c/5962c876a8bba4a6f8ff941998577e8359c928c700d092893e10f97aa94e/psycopg_binary-3.2.13-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f062d725898bf6fc5cfc6349a0d08ee09f129deb14d7fcd5c30f9f1b349f39dc", size = 4638520, upload-time = "2025-11-21T22:33:57.568Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b2/b557ac96752da8fd4b0ff7a128d148e6809ce576a2add6156c91d55abe0a/psycopg_binary-3.2.13-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:915647b5bbbcde2bd464dc293eec4f74710fa71edc4f85aa6f6c8494a179dc9e", size = 4737730, upload-time = "2025-11-21T22:34:02.969Z" }, + { url = "https://files.pythonhosted.org/packages/b5/9a/af2d96c0e711e90cf340a5f607911cd6df593fe1aec9c46644162161af18/psycopg_binary-3.2.13-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d3aec6e2f1cf4deb1b9a3ac287c0591479f3bd851d0a911d628f8c2c71c14f4a", size = 4421382, upload-time = "2025-11-21T22:34:11.501Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f6/f8135198a2c70ca663b55d44c6fc3beb4e36025679b541a9d489814f2ddc/psycopg_binary-3.2.13-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a56a8b1794cbf27ca04012ac2890d58cfc82b3b310c1dac4fa78fbf6f57e7440", size = 3879259, upload-time = "2025-11-21T22:34:17.706Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8c/3f778fc954f0b691941073a1d8b78c07219594135831cad32a739e4eee97/psycopg_binary-3.2.13-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:4150a5e72f863be442d153829724109d83a76871d9bc801d6bb5b9c84b5b19b9", size = 3560475, upload-time = "2025-11-21T22:34:21.329Z" }, + { url = "https://files.pythonhosted.org/packages/21/d2/731d56c636155f210fbb00cdbb7498c0e04a21052415520da54ac96eca63/psycopg_binary-3.2.13-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:028b49eb465f5d263d250cfd4f168fdabb306d0bbd97fd66a8a1fd7b696a953c", size = 3605616, upload-time = "2025-11-21T22:34:25.229Z" }, + { url = "https://files.pythonhosted.org/packages/8f/22/2619870c9ed44b5eaeae4f7706126754ccadde6319483cd4c490f5d13fbb/psycopg_binary-3.2.13-cp39-cp39-win_amd64.whl", hash = "sha256:532ea34f673148d637be65a96251832252e278540b39fbd683ef37e58ec361c1", size = 2912739, upload-time = "2025-11-21T22:34:29.069Z" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'win32'", + "python_full_version == '3.10.*' and sys_platform == 'win32'", + "python_full_version == '3.10.*' and sys_platform != 'win32'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/bf/70d8a60488f9955cbbcd538beae44d56bb2f1d19e673b72788f2d343ff55/psycopg_binary-3.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a", size = 4609750, upload-time = "2026-05-01T23:24:20.06Z" }, + { url = "https://files.pythonhosted.org/packages/db/b0/29e98ba210c9dbc75a6dc91e3f99b9e06ea901a62ca95804e02a1ae13e6b/psycopg_binary-3.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a", size = 4676700, upload-time = "2026-05-01T23:25:21.727Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ab/3df087b3c12bf74e47c08204172b2fabb5a144679110d5c7ad12d9201323/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429", size = 5496319, upload-time = "2026-05-01T23:25:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/87/9a/f088207b4cd6772f9e0d8a91807e79fa2458d4eb9eb1ae406c68415f2bec/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765", size = 5171906, upload-time = "2026-05-01T23:25:34.004Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/4523a857f253871d75c22e1c2e79fd47e599e736bcba1bad58d83e24be02/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13", size = 6762621, upload-time = "2026-05-01T23:25:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d1/925bf776503345bef428e6c45fb017d0139ddbe0e211814b585c4253dca8/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc", size = 5006319, upload-time = "2026-05-01T23:25:51.419Z" }, + { url = "https://files.pythonhosted.org/packages/6f/aa/99727337206fbba357ca084bf4ea8b29dc986f61842a2685859af61416db/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28", size = 4535388, upload-time = "2026-05-01T23:25:57.957Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a4/567ba2c37d19d8c2f63d836385dfd2495aa5897bbee6cfab104d9ee58624/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e", size = 4224544, upload-time = "2026-05-01T23:26:03.832Z" }, + { url = "https://files.pythonhosted.org/packages/b7/23/86457f5a82731685d7701de7bfaa5eb783dd1fecbf875321897d9d9ce33a/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d", size = 3956282, upload-time = "2026-05-01T23:26:09.983Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d8/249456df16d47de082abd9b73bce8ccdeb0293eb12e590f9150c7cbdb788/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744", size = 4261736, upload-time = "2026-05-01T23:26:16.798Z" }, + { url = "https://files.pythonhosted.org/packages/15/6b/c4abe228acafd8a385c1fb615d4f1e3c9b8ad7a4e4f0e84118ba3ffeed9c/psycopg_binary-3.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949", size = 3570620, upload-time = "2026-05-01T23:26:22.655Z" }, + { url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" }, + { url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" }, + { url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" }, + { url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" }, + { url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" }, + { url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" }, + { url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" }, + { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" }, + { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" }, + { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" }, + { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" }, + { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" }, + { url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" }, + { url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" }, + { url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" }, + { url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" }, + { url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" }, + { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" }, + { url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" }, + { url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, +] + [[package]] name = "pybase64" version = "1.4.3" @@ -4821,27 +5019,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.14" +version = "0.15.15" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/8a/8bce2894573e9dae6ff4d77fe34ad727d79b9e6238ad288c5638990d90f6/ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f", size = 4700910, upload-time = "2026-05-21T14:34:55.177Z" } +sdist = { url = "https://files.pythonhosted.org/packages/84/6f/a76f7d96e5c962f5b69cee865e49c15c1116897c01990faa8a57edb62e7f/ruff-0.15.15.tar.gz", hash = "sha256:b8dff018130b46d8e5bf0f926ef6b60cf871d6d5ae45fc9334e09632daa741d6", size = 4706985, upload-time = "2026-05-28T14:16:57.784Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/c8/74a92c6ff9fcfb4f1f947126d3ebee8389276e161ecc85de5bda7cda51bd/ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108", size = 10739177, upload-time = "2026-05-21T14:34:37.332Z" }, - { url = "https://files.pythonhosted.org/packages/45/91/254a35c20acc38a7223c9d2d594af12e794432464f2cdeb52af1dc4a892d/ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b", size = 11144969, upload-time = "2026-05-21T14:34:43.978Z" }, - { url = "https://files.pythonhosted.org/packages/56/9e/d13e40f83b8d0a94430e6778ce1d94a43b38cf2efe63278bdd2b4c65abbf/ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f", size = 10478207, upload-time = "2026-05-21T14:34:48.378Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f1/b15a7839fa4f332f8acec78e20564f26bb2d866e3d21710b877fd0263000/ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d", size = 10818459, upload-time = "2026-05-21T14:34:22.318Z" }, - { url = "https://files.pythonhosted.org/packages/45/33/53d651177f84f94b400a0e27f8824eeada3dddc9d5ee8aeb048f4352a520/ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4", size = 10541800, upload-time = "2026-05-21T14:34:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a6/868f87e0bf9786ed24b5d0d0ad8676b8a94fd1912f42cddf9cfc7857818a/ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542", size = 11342149, upload-time = "2026-05-21T14:34:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/a7/8b/38cd5c19faffdcc05a408d2b78edccc69492ab9720eadb49ea15ef80d768/ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f", size = 12212563, upload-time = "2026-05-21T14:34:28.579Z" }, - { url = "https://files.pythonhosted.org/packages/3e/4d/a3c5b874a556d5731e3e657aaf04311bb76f0a5c3ec220ed43051be6b64b/ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf", size = 11493299, upload-time = "2026-05-21T14:34:41.836Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c0/56472c251d09858a53e51efbd485b09e1995d8731668b76d52e5dd6ee0f1/ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba", size = 11455931, upload-time = "2026-05-21T14:34:57.276Z" }, - { url = "https://files.pythonhosted.org/packages/2c/4a/e2e7b4d8dbf233d4eace59c75bc3435fa6d8bd3bae82d351d4e4300c0fd1/ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f", size = 11400794, upload-time = "2026-05-21T14:34:39.773Z" }, - { url = "https://files.pythonhosted.org/packages/97/c7/83c0539fe34c3e09136204d1e75d6052492364e0b3cb05e9465423f567d7/ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581", size = 10804759, upload-time = "2026-05-21T14:34:31.045Z" }, - { url = "https://files.pythonhosted.org/packages/86/a6/18f2bfc095a2ab4a78745644e428205532ce6653a5d0fa8501572891534d/ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93", size = 10539517, upload-time = "2026-05-21T14:34:53.064Z" }, - { url = "https://files.pythonhosted.org/packages/54/3a/5a8b3b69c654d4e4bf1d246ac5b49cbcdac6eaab6905925f8915f31e3b80/ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61", size = 11065169, upload-time = "2026-05-21T14:34:24.484Z" }, - { url = "https://files.pythonhosted.org/packages/ed/c5/8864e4e7925b836ea354b31d57641ec03830564e281a8b6f061f8c3e0ec1/ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553", size = 11560214, upload-time = "2026-05-21T14:34:50.975Z" }, - { url = "https://files.pythonhosted.org/packages/36/38/012bf76752e1f89ed50b77b99532d90f3a3e287bc7918e1fc0948ac866ac/ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6", size = 10805548, upload-time = "2026-05-21T14:34:33.453Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b7/4ea2c170f10ad760fff2a5250beb18897719dc8b52b53a24cddbb9dd3f19/ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902", size = 11939523, upload-time = "2026-05-21T14:34:18.077Z" }, - { url = "https://files.pythonhosted.org/packages/62/d5/bc97ff895ec35cf3925d4bd60f3b39d822f377a446906ec9bcc87405e59b/ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826", size = 11208607, upload-time = "2026-05-21T14:34:26.525Z" }, + { url = "https://files.pythonhosted.org/packages/fa/9d/3a45c05b8ab04b4705989de70a79008e27c8003296a0feaee9edc18dd7e9/ruff-0.15.15-py3-none-linux_armv6l.whl", hash = "sha256:cf93e5388f412e1b108b1f8b34a6e036b70fe8aff89393befad96fe48670311b", size = 10710652, upload-time = "2026-05-28T14:16:06.701Z" }, + { url = "https://files.pythonhosted.org/packages/05/66/da974431624bf3b49f6ee1f9543c02d929ff1cba78b0d5a79c38cf21f744/ruff-0.15.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ac5a646d1f6a7dadd5d50842dae2c1f9862ac887ef5d1b1375e02def791fde6e", size = 11096615, upload-time = "2026-05-28T14:16:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/8c/09/7443452e5d290230a712103f2fdceeef7184f3ec99a2bd01c8be78aaceb5/ruff-0.15.15-py3-none-macosx_11_0_arm64.whl", hash = "sha256:77d955a431430c66f72dd94e379ad38a16daea3d25094872ac4edf9e797be530", size = 10436683, upload-time = "2026-05-28T14:16:40.974Z" }, + { url = "https://files.pythonhosted.org/packages/53/01/d330c26a57fa4f3943a14424904027428315b700fe4d14a84bb123a649e5/ruff-0.15.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7614ee79c69788cf6cedd568069ade9cecc22a1ad20494efe8d0c9ebb4b622d4", size = 10769064, upload-time = "2026-05-28T14:16:28.905Z" }, + { url = "https://files.pythonhosted.org/packages/1d/85/cc8770f8bdff541b1da8392d1634141fe4a0e3f4ee596605959b7906c27f/ruff-0.15.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3cdb1679e06a1f6b47bc384714ae96f6e2fb65ca441eb78c43d2ca554176ce1f", size = 10511987, upload-time = "2026-05-28T14:16:43.732Z" }, + { url = "https://files.pythonhosted.org/packages/7c/29/8c190c1472b63013583ba391f3342036e02010544c1270455ed8e519bdf3/ruff-0.15.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2728b93d7b23a603ea2c0ac6eb73d760bd38ec9de35f35fb41e18f7a3fee7622", size = 11275100, upload-time = "2026-05-28T14:16:55.244Z" }, + { url = "https://files.pythonhosted.org/packages/9f/6b/7e145ce2cc8e63d6834eca03d83a0e18d121def5c69f91b4cf4011ed4879/ruff-0.15.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be582fcc0db438902c7792b08d6ddf6c9b9e21addaa10092c2c741cfb09e5a45", size = 12176903, upload-time = "2026-05-28T14:16:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/80/a3/d5974637f68e451f7fadf015cf3101d1cd7d8ba5027cffe0b9e3826ebe6b/ruff-0.15.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7aa77465b8ecaf1a27bea098d696f7fed5e1eccbd10b321b682d6de586ae5627", size = 11404550, upload-time = "2026-05-28T14:16:20.138Z" }, + { url = "https://files.pythonhosted.org/packages/fe/1c/e6e5e568f22be4fb05d6244234aba384c06b451252453b821e1a529263cf/ruff-0.15.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48decfa11d740de4889de623be1463308346312f2409a56e24aa280c86162dc4", size = 11382027, upload-time = "2026-05-28T14:16:46.615Z" }, + { url = "https://files.pythonhosted.org/packages/1d/01/170921b49fcd2e8858825593f91cf7146c3e40a5c3e6df763e4bb0484dde/ruff-0.15.15-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a5015088452ca0081387063649ec67f06d3d1d6b8b936a1f836b5e9657ecd48c", size = 11366041, upload-time = "2026-05-28T14:16:26.247Z" }, + { url = "https://files.pythonhosted.org/packages/87/54/a7bad711d7de93254e15e06a4c375b89a03d18de45d3e5dcc86a4472fb1a/ruff-0.15.15-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f5294aab6356c81600fcdea3a62bb1b924dfd5e91767c12318d3f68f86af57cd", size = 10741795, upload-time = "2026-05-28T14:16:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/c9/31/38c075963668f8b41c6914ee0f6f318727fbe30ab9145cb29e6df464c5fa/ruff-0.15.15-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:db5bd4d802415cca656dc1616070b725952d6ae95eb5d4831e49fbd94a38f75f", size = 10511117, upload-time = "2026-05-28T14:16:31.767Z" }, + { url = "https://files.pythonhosted.org/packages/9d/96/6ff689e1f7e375d1d97075eca022f74c2bab59554a432fe4d2e6f091986a/ruff-0.15.15-py3-none-musllinux_1_2_i686.whl", hash = "sha256:587a6278ed42059191c1a466e490bd7930fb50bd2e255398bc29616c895a61cb", size = 10994867, upload-time = "2026-05-28T14:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c2/5dce0ab9f92a8d534fa62b9bf9caca3eddb8c1a81b616f5e195ada4f0d6e/ruff-0.15.15-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:df0c1c084f5f4be9812f61518a45c440d3c30d69ce4bf6c5270e66d38338f02a", size = 11482101, upload-time = "2026-05-28T14:16:49.598Z" }, + { url = "https://files.pythonhosted.org/packages/b1/c0/1003b60edd697c649faf61f1a34094b1abb38fb3d1181e3f895781250a08/ruff-0.15.15-py3-none-win32.whl", hash = "sha256:29428ea79694afbe756d45fd59b36f22b6b020dc0443cf7de0173046236964b9", size = 10716774, upload-time = "2026-05-28T14:16:52.337Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/1269eddd6945a06c23f055ef7848886e37cf9d6a8bebb386a3115f01470c/ruff-0.15.15-py3-none-win_amd64.whl", hash = "sha256:8df0323902e15e24bc4bf246da830573d3cf3352bd0b9a164eab335d111ff4a4", size = 11868463, upload-time = "2026-05-28T14:16:11.333Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b2/920464c907b191e37469d477a1aa8bc048b8f36c4c1610dfa4ab87b39e18/ruff-0.15.15-py3-none-win_arm64.whl", hash = "sha256:3c8ceca6792f38196b8f589bc92eccd03eef286602da92e5dc05cc42ef6441b7", size = 11138498, upload-time = "2026-05-28T14:16:38.425Z" }, ] [[package]] diff --git a/website/guide/configuration.md b/website/guide/configuration.md index f2efd3f..05b6da2 100644 --- a/website/guide/configuration.md +++ b/website/guide/configuration.md @@ -8,7 +8,8 @@ Located at `~/.mempalace/config.json`: { "palace_path": "/custom/path/to/palace", "collection_name": "mempalace_drawers", - "people_map": {"Kai": "KAI", "Priya": "PRI"} + "people_map": {"Kai": "KAI", "Priya": "PRI"}, + "max_backups": 10 } ``` @@ -17,6 +18,7 @@ Located at `~/.mempalace/config.json`: | `palace_path` | `~/.mempalace/palace` | Where ChromaDB stores your drawers | | `collection_name` | `mempalace_drawers` | ChromaDB collection name | | `people_map` | `{}` | Entity name → AAAK code mappings | +| `max_backups` | `10` | How many timestamped palace backups to keep before the oldest are pruned. Applies to `mempalace migrate` (`.pre-migrate.*`) and `mempalace repair max-seq-id` (`chroma.sqlite3.max-seq-id-backup-*`), which each write a full copy every run. Set to `0` to keep every backup (e.g. when an external retention policy manages cleanup). | ## Project Config @@ -83,3 +85,4 @@ python -m mempalace.mcp_server --palace /custom/palace |----------|-------------| | `MEMPALACE_PALACE_PATH` | Override palace path (same as `--palace`) | | `MEMPAL_DIR` | Directory for auto-mining in hooks | +| `MEMPALACE_MAX_BACKUPS` | Override `max_backups` retention count (`0` disables pruning) | diff --git a/website/reference/mcp-tools.md b/website/reference/mcp-tools.md index 220b15e..b1c2c97 100644 --- a/website/reference/mcp-tools.md +++ b/website/reference/mcp-tools.md @@ -1,6 +1,6 @@ # MCP Tools Reference -Detailed parameter schemas for all 30 MCP tools. +Detailed parameter schemas for all 31 MCP tools. ## Palace — Read Tools @@ -114,6 +114,24 @@ Delete a drawer by ID. Irreversible. --- +### `mempalace_mine` + +Mine a directory into the palace — the MCP equivalent of `mempalace mine`. Wraps the same in-process miners the CLI uses; runs synchronously and returns the miner's summary as `output`. The palace write lock is automatic — a concurrent mine returns a structured already-running error. Orphan cleanup is separate (see `mempalace_sync`). + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `source` | string | **Yes** | Directory to mine | +| `mode` | string | No | `projects` (code/docs, default), `convos` (chat transcripts), or `extract` (office docs; needs the `mempalace[extract]` extra) | +| `wing` | string | No | Target wing (default: source directory name) | +| `agent` | string | No | Recorded on every drawer (default: `mempalace`) | +| `limit` | integer | No | Max files to process (0 = all; default 0) | +| `dry_run` | boolean | No | Report what would be filed without writing (default false) | +| `extract` | string | No | Convos extraction strategy: `exchange` (default) or `general`; ignored by other modes | + +**Returns:** `{ success, mode, dry_run, output }` on success (`output` is the miner's human-readable summary; `output_truncated: true` is added when a very large summary is tail-trimmed), or `{ success: false, error, error_class? }` on failure. + +--- + ### `mempalace_sync` Prune drawers whose source files are gitignored, deleted, or moved. Returns a dry-run report by default; pass `apply=true` to commit deletions. @@ -319,6 +337,30 @@ Delete an explicit tunnel by its ID. --- +### `mempalace_list_hallways` + +List within-wing hallway records (entity-to-entity co-occurrence links built at mine time). Optionally filter by wing. + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `wing` | string | No | Filter hallways by wing | + +**Returns:** `[ { id, wing, entity_a, entity_b, co_occurrence_count, rooms, ... }, ... ]` + +--- + +### `mempalace_delete_hallway` + +Delete a hallway record by its ID. + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `hallway_id` | string | **Yes** | Hallway ID to delete | + +**Returns:** `{ deleted: bool }` + +--- + ### `mempalace_follow_tunnels` Follow tunnels from a room to see what it connects to in other wings. Returns connected rooms with drawer previews.