Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be954671ca | ||
|
|
939a076baf | ||
|
|
19d2f9216d | ||
|
|
72a69e2476 | ||
|
|
051f0cc1ec | ||
|
|
d464c28e0a | ||
|
|
d51e9434f9 | ||
|
|
15833ab01a | ||
|
|
ce929a8241 | ||
|
|
9694e2cb57 | ||
|
|
73d4e43b22 | ||
|
|
e66cfff0c0 | ||
|
|
ab716d02a5 | ||
|
|
ed772a01f9 | ||
|
|
708ef4a9fd | ||
|
|
8ec438ef57 | ||
|
|
e46374d24b | ||
|
|
6ac2de8e04 | ||
|
|
ab669fd3b1 | ||
|
|
b130301bd5 | ||
|
|
ad9874fcd5 | ||
|
|
334fc6d0be | ||
|
|
1ed7f49f13 | ||
|
|
097687cf63 | ||
|
|
46e3b22316 | ||
|
|
bbe6ff9477 | ||
|
|
b1df41b943 | ||
|
|
e1fce46006 | ||
|
|
cace2230d8 | ||
|
|
04af6a436d | ||
|
|
e00bc666e7 | ||
|
|
593cfebd79 | ||
|
|
6434370034 | ||
|
|
82af1fff27 | ||
|
|
5289bdfedb | ||
|
|
694289c423 | ||
|
|
723e45bf2b | ||
|
|
37e34f6cb5 | ||
|
|
f88951e8d4 | ||
|
|
4bc1ea8078 | ||
|
|
dc7d0be10c | ||
|
|
15852edbaf | ||
|
|
eb3ff2e61a | ||
|
|
0225eb4a08 | ||
|
|
1dadd33f35 | ||
|
|
ec5d1eb1c7 | ||
|
|
8757ba2650 | ||
|
|
717a57ff43 | ||
|
|
4e0fce010e | ||
|
|
5ba8399bb2 | ||
|
|
ce46747282 | ||
|
|
02b8753d97 | ||
|
|
300e824e94 | ||
|
|
af2bd0e10a | ||
|
|
526d58ae93 | ||
|
|
4d3daf96d3 | ||
|
|
30b82ba95f | ||
|
|
6aa8e93bc9 | ||
|
|
0dd0ade3b9 | ||
|
|
b5f3915d5b | ||
|
|
e54896699b | ||
|
|
6dd99ec4b0 | ||
|
|
206cfbb6ca | ||
|
|
22804b8e08 | ||
|
|
c19d894f68 | ||
|
|
7b1f6247d0 | ||
|
|
b11f41484c | ||
|
|
10d1ce6a94 | ||
|
|
038a7e0e15 | ||
|
|
29db125b9b | ||
|
|
818e4f3b31 | ||
|
|
48f4eb6dfe | ||
|
|
9b7cfc9940 | ||
|
|
387d50df27 | ||
|
|
55d4e379a7 | ||
|
|
13c38ac588 | ||
|
|
a0508cb065 | ||
|
|
2d4fae1ac3 | ||
|
|
15ac4303f5 | ||
|
|
6957c7e599 | ||
|
|
c4d8804b44 | ||
|
|
d88597cd72 | ||
|
|
753df201ac | ||
|
|
d0d011eb33 | ||
|
|
dbbf4c5af5 | ||
|
|
c9d8a40a37 | ||
|
|
be819da146 | ||
|
|
cbd1567b6a | ||
|
|
653e9774e8 | ||
|
|
5fe54df4c8 | ||
|
|
f51ecc1621 | ||
|
|
d12c96e43e |
@@ -9,7 +9,7 @@
|
||||
"name": "mempalace",
|
||||
"source": "./.claude-plugin",
|
||||
"description": "AI memory system — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, guided setup.",
|
||||
"version": "3.3.6",
|
||||
"version": "3.4.0",
|
||||
"author": {
|
||||
"name": "milla-jovovich"
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mempalace",
|
||||
"version": "3.3.6",
|
||||
"version": "3.4.0",
|
||||
"description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.",
|
||||
"author": {
|
||||
"name": "milla-jovovich"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mempalace",
|
||||
"version": "3.3.6",
|
||||
"version": "3.4.0",
|
||||
"description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.",
|
||||
"author": {
|
||||
"name": "milla-jovovich"
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# Keep the build context lean — only what `uv sync` + the package need.
|
||||
|
||||
# VCS
|
||||
.git/
|
||||
.gitignore
|
||||
|
||||
# Python build artifacts / caches
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
htmlcov/
|
||||
.coverage
|
||||
coverage.xml
|
||||
|
||||
# Virtual environments (rebuilt inside the image)
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
# Local config / secrets
|
||||
.env
|
||||
.env.*
|
||||
.envrc
|
||||
mempal.yaml
|
||||
|
||||
# Editor / OS cruft
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Repo material not needed at runtime
|
||||
.devcontainer/
|
||||
.github/
|
||||
.agents/
|
||||
.claude/
|
||||
.codex/
|
||||
.codex-plugin/
|
||||
.claude-plugin/
|
||||
benchmarks/
|
||||
docs/
|
||||
website/
|
||||
landing/
|
||||
assets/
|
||||
examples/
|
||||
tests/
|
||||
*.md
|
||||
!README.md
|
||||
@@ -0,0 +1,95 @@
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
# `main` is the release branch: pushes here publish the released image and
|
||||
# update `latest`; `v*` tags publish versioned images. develop does not
|
||||
# publish — it is validated via the pull_request trigger below.
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main, develop]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
# Needed for the emulated linux/arm64 build on real pushes.
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# Only authenticate + push for in-repo events. Fork PRs lack the
|
||||
# packages:write token, so they build (to validate the Dockerfile) but
|
||||
# do not push.
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
# latest -> main (the latest release); semver tags -> released versions.
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
# Publish multi-arch (amd64 + arm64 for Apple Silicon / ARM hosts) on
|
||||
# real pushes; keep PRs amd64-only so the emulated arm64 build does not
|
||||
# slow the PR check.
|
||||
platforms: ${{ github.event_name != 'pull_request' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
# Fork PRs get a read-only Actions cache, so writing it just emits 403
|
||||
# noise — only export cache on in-repo events.
|
||||
cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max' || '' }}
|
||||
|
||||
# Build-only validation for the CUDA image so it cannot silently rot (CUDA base
|
||||
# tag drift, cross-stage interpreter/venv copy paths, the `gpu` extra). The
|
||||
# runner has no GPU, so this only proves the image *compiles*; it is never
|
||||
# published (users build it themselves, per the README).
|
||||
build-gpu:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build GPU image (validation only — not published)
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.gpu
|
||||
platforms: linux/amd64
|
||||
push: false
|
||||
cache-from: type=gha,scope=gpu
|
||||
cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=gpu' || '' }}
|
||||
@@ -0,0 +1,124 @@
|
||||
name: Publish to PyPI
|
||||
|
||||
# Publishes mempalace to PyPI via Trusted Publishing (OIDC — no stored token).
|
||||
#
|
||||
# Triggers:
|
||||
# * release: published — the normal path (drafting + publishing a GitHub
|
||||
# Release fires this).
|
||||
# * workflow_dispatch — a manual escape hatch. GitHub does not reliably emit
|
||||
# a `release` event when a *draft* tied to a pre-existing tag is published,
|
||||
# so this lets a maintainer run the publish for any existing tag from the
|
||||
# Actions tab ("Run workflow" → enter the tag, e.g. v3.4.0).
|
||||
#
|
||||
# Build + publish run in ONE job on purpose: a two-job split hands the wheel
|
||||
# between jobs via upload/download-artifact, which failed repeatedly with
|
||||
# `BlobNotFound` on the same-run download. Building and publishing in the same
|
||||
# job removes that handoff entirely. The job is gated by the `pypi` environment
|
||||
# (manual approval), and the checks below keep it self-contained — it never
|
||||
# uploads a tag that isn't on main or doesn't match the version manifest.
|
||||
#
|
||||
# One-time setup required before the first run (see docs/RELEASING.md):
|
||||
# 1. PyPI → Manage project `mempalace` → Publishing → Add a trusted publisher:
|
||||
# Owner: MemPalace Repository: mempalace
|
||||
# Workflow: publish.yml Environment: pypi
|
||||
# 2. GitHub → repo Settings → Environments → New environment `pypi`,
|
||||
# add yourself as a Required reviewer (this is the manual approval gate).
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Existing tag to build and publish (e.g. v3.4.0). Used when the release event didn't fire."
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Build + publish to PyPI
|
||||
runs-on: ubuntu-latest
|
||||
# The `pypi` environment gate (required reviewer) pauses the whole job until
|
||||
# approved, and scopes the OIDC trust on the PyPI side.
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/p/mempalace
|
||||
permissions:
|
||||
contents: read # checkout
|
||||
id-token: write # mint the short-lived OIDC token for Trusted Publishing
|
||||
steps:
|
||||
- name: Resolve the release tag
|
||||
id: tag
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
||||
INPUT_TAG: ${{ github.event.inputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="${RELEASE_TAG:-${INPUT_TAG:-}}"
|
||||
if [[ -z "$tag" ]]; then
|
||||
echo "::error::no tag to publish — neither a release tag nor a workflow_dispatch input was provided"
|
||||
exit 1
|
||||
fi
|
||||
# Validate the shape before it is ever used as a git ref (ref-injection
|
||||
# guard): require vMAJOR.MINOR.PATCH with an optional -/+ suffix, so
|
||||
# loose values like 'v3' or 'v3foo' are rejected.
|
||||
re='^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$'
|
||||
if [[ ! "$tag" =~ $re ]]; then
|
||||
echo "::error::tag '$tag' is not a valid release tag (expected vMAJOR.MINOR.PATCH[-suffix])"
|
||||
exit 1
|
||||
fi
|
||||
echo "tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved tag: $tag"
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
# Fully-qualified refs/tags/ so an unqualified name can't resolve to a
|
||||
# same-named *branch* instead of the tag (checkout prefers branches).
|
||||
# steps.tag.outputs.tag is format-validated above (ref-injection guard).
|
||||
ref: refs/tags/${{ steps.tag.outputs.tag }}
|
||||
fetch-depth: 0 # full history for the ancestry check
|
||||
|
||||
- name: Verify the tag is on main
|
||||
env:
|
||||
TAG: ${{ steps.tag.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin main
|
||||
tag_sha=$(git rev-parse HEAD) # the checked-out tag commit
|
||||
main_sha=$(git rev-parse FETCH_HEAD)
|
||||
if git merge-base --is-ancestor "$tag_sha" "$main_sha"; then
|
||||
echo "Tag $TAG ($tag_sha) is reachable from main — OK."
|
||||
else
|
||||
echo "::error::tag $TAG is not an ancestor of main. Releases must be cut from main."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify the tag matches the version manifest
|
||||
env:
|
||||
TAG: ${{ steps.tag.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# version.py is the source of truth (per CLAUDE.md); version-guard.yml
|
||||
# separately enforces that pyproject + the three plugin manifests agree.
|
||||
py_version=$(grep -E '^__version__' mempalace/version.py | cut -d'"' -f2)
|
||||
tag_version="${TAG#v}"
|
||||
if [[ "$tag_version" != "$py_version" ]]; then
|
||||
echo "::error::tag $TAG does not match mempalace/version.py ($py_version). Bump the version files before releasing."
|
||||
exit 1
|
||||
fi
|
||||
echo "Tag $TAG matches manifest version $py_version — OK."
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Build sdist + wheel
|
||||
run: |
|
||||
python -m pip install --upgrade build
|
||||
python -m build
|
||||
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# MemPalace — CPU image.
|
||||
#
|
||||
# Multi-stage build using uv (the project ships a uv.lock, so we install from
|
||||
# the frozen lockfile for reproducible images). The default runtime is the MCP
|
||||
# server over stdio; the CLI is reachable through the same entrypoint.
|
||||
#
|
||||
# Build:
|
||||
# docker build -t mempalace .
|
||||
# docker build -t mempalace --build-arg EXTRAS="extract,spellcheck" .
|
||||
#
|
||||
# Run (MCP server over stdio, palace persisted on the host):
|
||||
# docker run -i --rm -v mempalace-data:/data mempalace
|
||||
#
|
||||
# Run (CLI):
|
||||
# docker run --rm -v mempalace-data:/data mempalace search "why GraphQL"
|
||||
#
|
||||
# GPU acceleration lives in Dockerfile.gpu (it needs a CUDA base image).
|
||||
|
||||
ARG PYTHON_VERSION=3.12
|
||||
|
||||
# --- builder ----------------------------------------------------------------
|
||||
FROM python:${PYTHON_VERSION}-slim AS builder
|
||||
|
||||
# uv: fast, lockfile-driven installer. Pinned by digest-less tag for clarity;
|
||||
# bump deliberately.
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/
|
||||
|
||||
# Some transitive deps (grpcio, onnxruntime, tokenizers) ship manylinux wheels
|
||||
# for cp312, but keep a compiler around so a missing wheel degrades to a source
|
||||
# build instead of failing the image. Dropped from the final stage.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PYTHON_DOWNLOADS=never
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Optional extras baked into the image. CPU-safe by default; GPU is a separate
|
||||
# Dockerfile. Pass a comma-separated list, e.g. EXTRAS="extract,spellcheck".
|
||||
ARG EXTRAS="extract,spellcheck"
|
||||
|
||||
# Layer 1: dependencies only (no project) — cached across source changes.
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
|
||||
--mount=type=bind,source=uv.lock,target=uv.lock \
|
||||
--mount=type=bind,source=README.md,target=README.md \
|
||||
set -e; \
|
||||
flags=""; \
|
||||
for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \
|
||||
uv sync --frozen --no-install-project --no-dev ${flags}
|
||||
|
||||
# Layer 2: the project itself. --no-editable installs mempalace into the venv's
|
||||
# site-packages (instead of an .pth pointing at /app), so the runtime stage can
|
||||
# copy only /app/.venv and drop the source tree.
|
||||
COPY . /app
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
set -e; \
|
||||
flags=""; \
|
||||
for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \
|
||||
uv sync --frozen --no-dev --no-editable ${flags}
|
||||
|
||||
# --- runtime ----------------------------------------------------------------
|
||||
FROM python:${PYTHON_VERSION}-slim AS runtime
|
||||
|
||||
LABEL org.opencontainers.image.title="MemPalace" \
|
||||
org.opencontainers.image.description="Local-first AI memory — verbatim storage, MCP server + CLI." \
|
||||
org.opencontainers.image.source="https://github.com/MemPalace/mempalace" \
|
||||
org.opencontainers.image.licenses="MIT"
|
||||
|
||||
# /data is the single persistence root: HOME points here, so the palace
|
||||
# (~/.mempalace/palace), config (~/.mempalace), and the embedding-model cache
|
||||
# all land under one mountable volume. The default `minilm` model caches under
|
||||
# ~/.cache/chroma (~80 MB, from ChromaDB's S3); the optional `embeddinggemma`
|
||||
# model caches under ~/.cache/huggingface (~300 MB). Both lazy-download on
|
||||
# first use.
|
||||
ENV HOME=/data \
|
||||
PATH="/app/.venv/bin:${PATH}" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
# Non-root user owning the data volume.
|
||||
RUN groupadd --gid 1000 mempalace \
|
||||
&& useradd --uid 1000 --gid 1000 --home-dir /data --create-home mempalace
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# The resolved virtualenv from the builder — no build toolchain in this layer.
|
||||
COPY --from=builder --chown=mempalace:mempalace /app/.venv /app/.venv
|
||||
COPY --chown=mempalace:mempalace docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
USER mempalace
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Default to the MCP server; `docker run` it with `-i` for stdio JSON-RPC.
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["mcp"]
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# MemPalace — GPU (NVIDIA CUDA) image.
|
||||
#
|
||||
# Multi-stage build using uv (the project ships a uv.lock, so we install from
|
||||
# the frozen lockfile for reproducible images). The `gpu` extra pulls
|
||||
# onnxruntime-gpu, which needs CUDA + cuDNN shared libraries at runtime, so
|
||||
# this variant builds on an nvidia/cuda base instead of python:slim.
|
||||
#
|
||||
# The builder stage is dropped from the final image, so build tools
|
||||
# (compilers, apt cache) never reach production — only the uv-managed
|
||||
# interpreter and the resolved virtualenv do.
|
||||
#
|
||||
# Build:
|
||||
# docker build -f Dockerfile.gpu -t mempalace:gpu .
|
||||
#
|
||||
# Run (requires the NVIDIA Container Toolkit on the host):
|
||||
# docker run -i --rm --gpus all \
|
||||
# -e MEMPALACE_EMBEDDING_DEVICE=cuda \
|
||||
# -v mempalace-data:/data mempalace:gpu
|
||||
#
|
||||
# NOTE: onnxruntime-gpu ties itself to a CUDA major version. If embeddings
|
||||
# fail to load on the GPU, align CUDA_IMAGE below with the CUDA release that
|
||||
# the resolved onnxruntime-gpu wheel targets (see its release notes), then
|
||||
# rebuild.
|
||||
|
||||
ARG CUDA_IMAGE=nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04
|
||||
|
||||
# --- builder ----------------------------------------------------------------
|
||||
FROM ${CUDA_IMAGE} AS builder
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/
|
||||
|
||||
# Minimal toolchain for any source-built wheels; dropped before the runtime
|
||||
# stage so compilers never ship in the production image.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ARG PYTHON_VERSION=3.12
|
||||
ARG EXTRAS="extract,spellcheck,gpu"
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
|
||||
UV_PYTHON_PREFERENCE=only-managed
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Layer 1: dependencies only (no project source). Bind-mounted files keep the
|
||||
# project tree out of this layer, so changing source code does not bust the
|
||||
# deps cache. The uv-managed interpreter is also installed here, into a
|
||||
# stable path that the runtime stage can copy verbatim.
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
|
||||
--mount=type=bind,source=uv.lock,target=uv.lock \
|
||||
--mount=type=bind,source=README.md,target=README.md \
|
||||
set -e; \
|
||||
uv python install ${PYTHON_VERSION}; \
|
||||
flags=""; \
|
||||
for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \
|
||||
uv sync --frozen --no-install-project --no-dev --python ${PYTHON_VERSION} ${flags}
|
||||
|
||||
# Layer 2: the project itself. --no-editable installs mempalace into the
|
||||
# venv's site-packages (instead of an .pth pointing at /app), so the runtime
|
||||
# stage can copy only /app/.venv and drop the source tree.
|
||||
COPY . /app
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
set -e; \
|
||||
flags=""; \
|
||||
for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \
|
||||
uv sync --frozen --no-dev --no-editable ${flags}
|
||||
|
||||
# --- runtime ----------------------------------------------------------------
|
||||
FROM ${CUDA_IMAGE} AS runtime
|
||||
|
||||
LABEL org.opencontainers.image.title="MemPalace (GPU)" \
|
||||
org.opencontainers.image.description="Local-first AI memory with CUDA-accelerated embeddings." \
|
||||
org.opencontainers.image.source="https://github.com/MemPalace/mempalace" \
|
||||
org.opencontainers.image.licenses="MIT"
|
||||
|
||||
# ca-certificates only — needed for the lazy HuggingFace model download on
|
||||
# first use. No build toolchain in this stage.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV HOME=/data \
|
||||
PATH="/app/.venv/bin:${PATH}" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
MEMPALACE_EMBEDDING_DEVICE=cuda
|
||||
|
||||
# Non-root user owning the data volume.
|
||||
RUN groupadd --gid 1000 mempalace \
|
||||
&& useradd --uid 1000 --gid 1000 --home-dir /data --create-home mempalace
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Bring the uv-managed interpreter and the resolved venv across the stage
|
||||
# boundary. /opt/uv/python must be copied alongside .venv: the venv's
|
||||
# shebangs and binary launcher reference it.
|
||||
COPY --from=builder /opt/uv/python /opt/uv/python
|
||||
COPY --from=builder --chown=mempalace:mempalace /app/.venv /app/.venv
|
||||
COPY --chown=mempalace:mempalace docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
USER mempalace
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["mcp"]
|
||||
@@ -1,21 +1,3 @@
|
||||
> [!CAUTION]
|
||||
> # 🚨 CRITICAL SECURITY WARNING: BEWARE OF SCAMS
|
||||
> **MemPalace has NO other official websites.**
|
||||
>
|
||||
> The **ONLY** official sources are:
|
||||
> 1. This **[GitHub repository](https://github.com/MemPalace/mempalace)**
|
||||
> 2. The **[PyPI package](https://pypi.org/project/mempalace/)**
|
||||
> 3. The docs at **[mempalaceofficial.com](https://mempalaceofficial.com)**
|
||||
>
|
||||
> **ANY other domain** (including `.tech`, `.net`, or other `.com` variants) is an **impostor** and may distribute **malware**. Do not download executables from untrusted sites. Details and timeline: [docs/HISTORY.md](docs/HISTORY.md).
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **🚨 Claude Code sessions expire in 30 days w/out auto-save hooks wired!** **[Read this →](https://github.com/MemPalace/mempalace/discussions/1388)**
|
||||
>
|
||||
> Need the shortest recovery/setup path? Use the
|
||||
> [Claude Code retention setup checklist](https://mempalaceofficial.com/guide/claude-code-retention.html).
|
||||
|
||||
|
||||
<div align="center">
|
||||
|
||||
<img src="assets/mempalace_logo.png" alt="MemPalace" width="240">
|
||||
@@ -31,6 +13,14 @@ Local-first AI memory. Verbatim storage, pluggable backend, 96.6% R@5 raw on Lon
|
||||
|
||||
</div>
|
||||
|
||||
> [!CAUTION]
|
||||
> **Beware of impostor sites.** MemPalace has no other official websites. The **only** official sources are this **[GitHub repository](https://github.com/MemPalace/mempalace)**, the **[PyPI package](https://pypi.org/project/mempalace/)**, and the docs at **[mempalaceofficial.com](https://mempalaceofficial.com)**. Any other domain (including `.tech`, `.net`, or other `.com` variants) is an impostor and may distribute malware. Details and timeline: [docs/HISTORY.md](docs/HISTORY.md).
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Claude Code sessions expire in 30 days without auto-save hooks wired.** [Read this →](https://github.com/MemPalace/mempalace/discussions/1388)
|
||||
>
|
||||
> Need the shortest recovery/setup path? Use the [Claude Code retention setup checklist](https://mempalaceofficial.com/guide/claude-code-retention.html).
|
||||
|
||||
---
|
||||
|
||||
## What it is
|
||||
@@ -79,6 +69,79 @@ python -m venv .venv && source .venv/bin/activate
|
||||
pip install mempalace
|
||||
```
|
||||
|
||||
### Docker
|
||||
|
||||
A container image is also available for running the MCP server or the CLI
|
||||
without a local Python toolchain. Everything persists under `/data` (palace,
|
||||
config, and the cached embedding model), so mount a volume there.
|
||||
|
||||
```bash
|
||||
# Build the image (CPU; bundles the `extract` + `spellcheck` extras)
|
||||
docker build -t mempalace .
|
||||
|
||||
# MCP server over stdio — note the `-i` flag (JSON-RPC needs stdin)
|
||||
docker run -i --rm -v mempalace-data:/data mempalace
|
||||
|
||||
# Run any CLI command instead (mount the host directory you want to mine)
|
||||
docker run --rm -v mempalace-data:/data -v /path/to/project:/work mempalace mine /work
|
||||
docker run --rm -v mempalace-data:/data mempalace search "why GraphQL"
|
||||
```
|
||||
|
||||
Wire it into an MCP client (e.g. Claude Code) as a stdio server:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"mempalace": {
|
||||
"command": "docker",
|
||||
"args": ["run", "-i", "--rm", "-v", "mempalace-data:/data", "mempalace"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`docker compose run --rm mcp` works too (see `docker-compose.yml`). For
|
||||
CUDA-accelerated embeddings, build the GPU variant with
|
||||
`docker build -f Dockerfile.gpu -t mempalace:gpu .` and run it with
|
||||
`--gpus all`. Customise the bundled extras at build time, e.g.
|
||||
`docker build --build-arg EXTRAS="extract,spellcheck" -t mempalace .`.
|
||||
|
||||
## Storage backends
|
||||
|
||||
ChromaDB is the default. For the pluggable-backend preview, MemPalace also
|
||||
ships `sqlite_exact` for local exact-vector correctness checks, and two opt-in
|
||||
external service backends — `qdrant` (REST) and `pgvector` (Postgres). The two
|
||||
external backends exercise the storage contract on different substrates (a
|
||||
REST/dict store and a SQL/JSONB store), so it is not accidentally shaped around
|
||||
one vendor.
|
||||
|
||||
```bash
|
||||
# local no-service backend
|
||||
mempalace mine ~/projects/myapp --backend sqlite_exact
|
||||
|
||||
# Qdrant backend, defaulting to http://localhost:6333
|
||||
MEMPALACE_QDRANT_URL=http://localhost:6333 \
|
||||
mempalace mine ~/projects/myapp --backend qdrant
|
||||
|
||||
# Postgres + pgvector backend, defaulting to postgresql://localhost:5432/mempalace
|
||||
# needs the optional driver: pip install mempalace[pgvector]
|
||||
# and the `vector` extension available on the server
|
||||
MEMPALACE_PGVECTOR_DSN=postgresql://localhost:5432/mempalace \
|
||||
mempalace mine ~/projects/myapp --backend pgvector
|
||||
```
|
||||
|
||||
Qdrant can also be configured with `MEMPALACE_QDRANT_API_KEY`,
|
||||
`MEMPALACE_QDRANT_NAMESPACE`, and `MEMPALACE_QDRANT_TIMEOUT`; pgvector with
|
||||
`MEMPALACE_PGVECTOR_NAMESPACE`. Both external backends isolate tenants by
|
||||
namespace (advertised via the `supports_namespace_isolation` capability) and
|
||||
write a local marker (`qdrant_backend.json` / `pgvector_backend.json`) to guard
|
||||
against silently opening a palace against the wrong server.
|
||||
|
||||
When `MEMPALACE_QDRANT_URL` or `MEMPALACE_PGVECTOR_DSN` points anywhere other
|
||||
than your own local or trusted self-hosted service, MemPalace will send and
|
||||
store verbatim drawer text and metadata there. That is an explicit opt-in
|
||||
backend choice, never the default.
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
@@ -215,7 +278,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||
MIT — see [LICENSE](LICENSE).
|
||||
|
||||
<!-- Link Definitions -->
|
||||
[version-shield]: https://img.shields.io/badge/version-3.3.6-4dc9f6?style=flat-square&labelColor=0a0e14
|
||||
[version-shield]: https://img.shields.io/badge/version-3.4.0-4dc9f6?style=flat-square&labelColor=0a0e14
|
||||
[release-link]: https://github.com/MemPalace/mempalace/releases
|
||||
[python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8
|
||||
[python-link]: https://www.python.org/
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# MemPalace via Docker Compose.
|
||||
#
|
||||
# The MCP server speaks JSON-RPC over stdio, so the `mcp` service is meant to
|
||||
# be driven interactively (`docker compose run`), not left running detached:
|
||||
#
|
||||
# docker compose build
|
||||
# docker compose run --rm mcp # MCP server over stdio
|
||||
# docker compose run --rm mcp cli search "GraphQL" # one-off CLI command
|
||||
#
|
||||
# The named volume `mempalace-data` is mounted at /data and holds the palace,
|
||||
# config, and the cached embedding model across runs.
|
||||
|
||||
services:
|
||||
mcp:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mempalace:local
|
||||
# stdio transport: keep STDIN open and allocate no TTY (raw JSON-RPC).
|
||||
stdin_open: true
|
||||
tty: false
|
||||
volumes:
|
||||
- mempalace-data:/data
|
||||
environment:
|
||||
# Everything defaults correctly from HOME=/data (palace ->
|
||||
# /data/.mempalace/palace, config -> /data/.mempalace, model cache ->
|
||||
# /data/.cache). Override here only for non-default locations, e.g.:
|
||||
# - MEMPALACE_EMBEDDING_MODEL=embeddinggemma # multilingual (default: minilm)
|
||||
# - MEMPALACE_PALACE_PATH=/data/custom/palace
|
||||
|
||||
volumes:
|
||||
mempalace-data:
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env sh
|
||||
# Flexible entrypoint: pick the MCP server or the CLI from the first argument.
|
||||
#
|
||||
# docker run -i mempalace -> MCP server over stdio (default)
|
||||
# docker run -i mempalace mcp -> MCP server over stdio (explicit)
|
||||
# docker run mempalace cli search "query" -> CLI passthrough (explicit)
|
||||
# docker run mempalace search "query" -> CLI passthrough (implicit)
|
||||
#
|
||||
# `mcp` and `cli` are dispatch keywords; anything else is forwarded to the
|
||||
# `mempalace` CLI verbatim so subcommands like `mine`, `search`, `wake-up`
|
||||
# work without ceremony.
|
||||
set -e
|
||||
|
||||
case "${1:-mcp}" in
|
||||
mcp)
|
||||
if [ "$#" -gt 0 ]; then
|
||||
shift
|
||||
fi
|
||||
exec mempalace-mcp "$@"
|
||||
;;
|
||||
cli)
|
||||
if [ "$#" -gt 0 ]; then
|
||||
shift
|
||||
fi
|
||||
exec mempalace "$@"
|
||||
;;
|
||||
*)
|
||||
exec mempalace "$@"
|
||||
;;
|
||||
esac
|
||||
@@ -29,3 +29,59 @@ If `pyproject.toml` has no match, **stop** — the entry point is missing and
|
||||
any fresh `pip install` will ship a broken plugin config. Investigate whether
|
||||
the release branch was cut before
|
||||
[#340](https://github.com/MemPalace/mempalace/pull/340) landed on `develop`.
|
||||
|
||||
## Publishing to PyPI
|
||||
|
||||
Releases publish automatically via the
|
||||
[`publish.yml`](../.github/workflows/publish.yml) workflow, using PyPI
|
||||
[Trusted Publishing](https://docs.pypi.org/trusted-publishers/) (OIDC). There
|
||||
is **no API token** stored anywhere — GitHub mints a short-lived identity at
|
||||
upload time. The workflow fires when a **GitHub Release is published**, builds
|
||||
the sdist + wheel, and pauses for manual approval on the `pypi` environment
|
||||
before uploading.
|
||||
|
||||
### One-time setup (owners only)
|
||||
|
||||
Done once per project; both steps require PyPI owner / GitHub admin rights.
|
||||
|
||||
1. **PyPI trusted publisher** — on PyPI, go to **Manage project `mempalace`
|
||||
→ Publishing → Add a trusted publisher** and enter exactly:
|
||||
|
||||
| Field | Value |
|
||||
| --- | --- |
|
||||
| Owner | `MemPalace` |
|
||||
| Repository name | `mempalace` |
|
||||
| Workflow filename | `publish.yml` |
|
||||
| Environment name | `pypi` |
|
||||
|
||||
2. **GitHub environment** — in the repo, **Settings → Environments → New
|
||||
environment** named `pypi`. Add yourself (and any other release approvers)
|
||||
under **Required reviewers**. This is the manual gate the workflow waits on
|
||||
before the upload step runs.
|
||||
|
||||
### Cutting a release
|
||||
|
||||
1. Bump the version in **all five** sources on `develop` so `version-guard.yml`
|
||||
stays green (it is the single source of truth at `mempalace/version.py`,
|
||||
mirrored in `pyproject.toml`, `.claude-plugin/marketplace.json`,
|
||||
`.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json`).
|
||||
2. Land everything for the release on `develop`, then merge `develop → main`.
|
||||
Releases publish **only from `main`** — the workflow refuses any tag whose
|
||||
commit is not an ancestor of `main`. Don't commit the bump directly to
|
||||
`main`: it bypasses branch protection and leaves `develop` behind.
|
||||
3. Run the **entry-point alignment check** above.
|
||||
4. On GitHub, **Releases → Draft a new release**:
|
||||
- **Target:** `main`
|
||||
- **Tag:** `vX.Y.Z` (must equal `mempalace/version.py`; the workflow and
|
||||
`version-guard.yml` both reject a mismatch)
|
||||
- Write the release notes, then **Publish release**.
|
||||
5. The `publish.yml` run validates the tag (on `main`, matches the manifest),
|
||||
builds, and then waits for approval on the `pypi` environment. Approve it to
|
||||
upload to PyPI. Watch the run land the new version on
|
||||
<https://pypi.org/project/mempalace/>.
|
||||
|
||||
To stage a release candidate without shipping to end users, tag a semver
|
||||
pre-release (`vX.Y.Z-rc1`) — `version-guard.yml` skips the strict manifest
|
||||
match for pre-release tags. (Note: a published GitHub Release still triggers
|
||||
`publish.yml`; use a **draft** release, or a plain pushed tag, for dry runs you
|
||||
don't want uploaded.)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# Recovery: legacy wing names split after the normalization change
|
||||
|
||||
**Companion to #1675.** `normalize_wing_name` now strips leading and trailing
|
||||
separators, so a path-encoded project dir like `-home-user-proj` derives the
|
||||
wing `home_user_proj` instead of `_home_user_proj`. Palaces mined before that
|
||||
change filed drawers under the old, separator-padded name. New mining and diary
|
||||
writes land on the new name, so the two no longer meet — the history is
|
||||
**split**, not lost. `mempalace migrate-wings` re-unites them.
|
||||
|
||||
## Symptom
|
||||
|
||||
After upgrading, a project that used to surface its memories returns less than
|
||||
expected, and `mempalace status` shows two wings for one project — e.g. both
|
||||
`_home_user_proj` (old drawers) and `home_user_proj` (newly mined). MCP writes
|
||||
to the padded wing may also have been rejected, since `sanitize_name` does not
|
||||
accept a leading underscore.
|
||||
|
||||
## Recovery
|
||||
|
||||
Preview first — this never modifies anything:
|
||||
|
||||
```bash
|
||||
mempalace migrate-wings --dry-run
|
||||
mempalace migrate-wings --dry-run --palace /path/to/palace
|
||||
```
|
||||
|
||||
The plan lists each rename and flags collisions that will **merge** into an
|
||||
existing wing:
|
||||
|
||||
```
|
||||
Wing-name migration plan:
|
||||
'_home_user_proj' -> 'home_user_proj': 1284 drawer(s), 96 closet(s) (MERGE into existing wing)
|
||||
```
|
||||
|
||||
Apply it:
|
||||
|
||||
```bash
|
||||
mempalace migrate-wings # prompts for confirmation
|
||||
mempalace migrate-wings --yes # no prompt
|
||||
```
|
||||
|
||||
## What it does
|
||||
|
||||
- Re-keys the `wing` **metadata field** on drawers and closets to the normalized
|
||||
form, merging collisions into the existing wing.
|
||||
- Re-keys the `topics_by_wing` registry (merging topic lists on collision).
|
||||
|
||||
## What it leaves alone
|
||||
|
||||
- **Drawer/closet IDs** are untouched. The wing in an ID (`drawer_<wing>_…`) is
|
||||
an opaque prefix that is never decoded back into a wing, so leaving it keeps
|
||||
closet `→drawer_id` pointers valid and lets future mining still skip
|
||||
already-mined files (no duplicates). The verbatim drawer content is never
|
||||
read or rewritten.
|
||||
- **Tunnels** already normalize wing names at read time, so they resolve under
|
||||
the new name without a rewrite.
|
||||
|
||||
## Notes
|
||||
|
||||
- **Idempotent.** A second run reports "nothing to migrate" and changes nothing.
|
||||
- **Backend-agnostic.** Works on any configured storage backend.
|
||||
- Run it once per palace after upgrading. New palaces are born with normalized
|
||||
wing names and never need it.
|
||||
@@ -17,6 +17,7 @@ Public surface:
|
||||
from .base import (
|
||||
BackendClosedError,
|
||||
BackendError,
|
||||
BackendMismatchError,
|
||||
BaseBackend,
|
||||
BaseCollection,
|
||||
CollectionNotInitializedError,
|
||||
@@ -24,14 +25,22 @@ from .base import (
|
||||
EmbedderIdentityMismatchError,
|
||||
GetResult,
|
||||
HealthStatus,
|
||||
LexicalHit,
|
||||
LexicalResult,
|
||||
PalaceNotFoundError,
|
||||
PalaceRef,
|
||||
QueryResult,
|
||||
UnsupportedCapabilityError,
|
||||
UnsupportedFilterError,
|
||||
)
|
||||
from .chroma import ChromaBackend, ChromaCollection
|
||||
from .pgvector import PgVectorBackend, PgVectorCollection
|
||||
from .qdrant import QdrantBackend, QdrantCollection
|
||||
from .sqlite_exact import SQLiteExactBackend, SQLiteExactCollection
|
||||
from .registry import (
|
||||
available_backends,
|
||||
detect_backend_for_path,
|
||||
detect_backends_for_path,
|
||||
get_backend,
|
||||
get_backend_class,
|
||||
register,
|
||||
@@ -43,6 +52,7 @@ from .registry import (
|
||||
__all__ = [
|
||||
"BackendClosedError",
|
||||
"BackendError",
|
||||
"BackendMismatchError",
|
||||
"BaseBackend",
|
||||
"BaseCollection",
|
||||
"ChromaBackend",
|
||||
@@ -52,11 +62,22 @@ __all__ = [
|
||||
"EmbedderIdentityMismatchError",
|
||||
"GetResult",
|
||||
"HealthStatus",
|
||||
"LexicalHit",
|
||||
"LexicalResult",
|
||||
"PalaceNotFoundError",
|
||||
"PalaceRef",
|
||||
"PgVectorBackend",
|
||||
"PgVectorCollection",
|
||||
"QdrantBackend",
|
||||
"QdrantCollection",
|
||||
"QueryResult",
|
||||
"SQLiteExactBackend",
|
||||
"SQLiteExactCollection",
|
||||
"UnsupportedCapabilityError",
|
||||
"UnsupportedFilterError",
|
||||
"available_backends",
|
||||
"detect_backend_for_path",
|
||||
"detect_backends_for_path",
|
||||
"get_backend",
|
||||
"get_backend_class",
|
||||
"register",
|
||||
|
||||
@@ -58,6 +58,14 @@ class UnsupportedFilterError(BackendError):
|
||||
"""
|
||||
|
||||
|
||||
class UnsupportedCapabilityError(BackendError):
|
||||
"""Raised when a backend does not implement an optional capability."""
|
||||
|
||||
|
||||
class BackendMismatchError(BackendError):
|
||||
"""Raised when a selected backend does not match existing palace artifacts."""
|
||||
|
||||
|
||||
class DimensionMismatchError(BackendError):
|
||||
"""Raised when the embedding dimension on write does not match the collection."""
|
||||
|
||||
@@ -78,6 +86,29 @@ class PalaceRef:
|
||||
``id`` is always present and is the key backends use to cache handles.
|
||||
``local_path`` is populated for filesystem-rooted palaces.
|
||||
``namespace`` is used by server-mode backends for tenant / prefix routing.
|
||||
|
||||
Isolation contract (RFC 001 §2.1, conformance: ``tests/test_backend_conformance.py``)
|
||||
-----------------------------------------------------------------------------------
|
||||
``id`` is the *required* isolation key. Within a single backend instance:
|
||||
|
||||
A record written for one ``PalaceRef.id`` MUST NOT be returned,
|
||||
modified, or deleted by an operation issued for a different
|
||||
``PalaceRef.id``. Cross-palace access is a spec violation.
|
||||
|
||||
``namespace`` is *additional* partitioning, honored only by backends that
|
||||
advertise the ``supports_namespace_isolation`` capability. For those
|
||||
backends the same guarantee extends to namespaces:
|
||||
|
||||
A record written under one ``namespace`` MUST NOT be returned,
|
||||
modified, or deleted by an operation issued under a different
|
||||
``namespace`` within the same backend instance. Cross-namespace
|
||||
access is a spec violation.
|
||||
|
||||
Backends that do not advertise ``supports_namespace_isolation`` (e.g.
|
||||
``sqlite_exact``, whose isolation is the on-disk path alone) MAY ignore
|
||||
``namespace`` entirely; callers MUST NOT rely on it for tenant isolation
|
||||
on such backends. Any conforming backend can self-check both guarantees by
|
||||
running the shared assertions in ``tests/_backend_conformance.py``.
|
||||
"""
|
||||
|
||||
id: str
|
||||
@@ -177,6 +208,23 @@ class GetResult(_DictCompatMixin):
|
||||
return cls(ids=[], documents=[], metadatas=[], embeddings=None)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LexicalHit:
|
||||
"""One hit from backend lexical candidate search."""
|
||||
|
||||
id: str
|
||||
document: str
|
||||
metadata: dict
|
||||
score: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LexicalResult:
|
||||
"""Typed return from ``BaseCollection.lexical_search``."""
|
||||
|
||||
hits: list[LexicalHit]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collection contract
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -253,6 +301,15 @@ class BaseCollection(ABC):
|
||||
def health(self) -> HealthStatus:
|
||||
return HealthStatus.healthy()
|
||||
|
||||
def lexical_search(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
n_results: int = 10,
|
||||
where: Optional[dict] = None,
|
||||
) -> LexicalResult:
|
||||
raise UnsupportedCapabilityError("backend does not support lexical_search")
|
||||
|
||||
def update(
|
||||
self,
|
||||
*,
|
||||
@@ -311,6 +368,14 @@ class BaseBackend(ABC):
|
||||
Instances are lightweight on construction — no I/O, no network. All
|
||||
connection work is deferred to ``get_collection``. Instances are thread-
|
||||
safe for concurrent ``get_collection`` calls across different palaces.
|
||||
|
||||
Every backend MUST satisfy the per-``PalaceRef.id`` isolation guarantee in
|
||||
:class:`PalaceRef`. Backends that additionally isolate by
|
||||
``PalaceRef.namespace`` (multi-tenant / hosted deployments) MUST advertise
|
||||
the ``supports_namespace_isolation`` capability token; doing so is a
|
||||
promise to satisfy the cross-namespace guarantee and to pass the namespace
|
||||
arm of the conformance suite. Backends without the token MAY ignore
|
||||
``namespace``.
|
||||
"""
|
||||
|
||||
name: ClassVar[str]
|
||||
|
||||
+641
-86
@@ -2,10 +2,14 @@
|
||||
|
||||
import contextlib
|
||||
import datetime as _dt
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import pickle
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import defaultdict
|
||||
from numbers import Integral
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
@@ -19,6 +23,8 @@ from .base import (
|
||||
CollectionNotInitializedError,
|
||||
GetResult,
|
||||
HealthStatus,
|
||||
LexicalHit,
|
||||
LexicalResult,
|
||||
PalaceNotFoundError,
|
||||
PalaceRef,
|
||||
QueryResult,
|
||||
@@ -32,6 +38,7 @@ logger = logging.getLogger(__name__)
|
||||
_REQUIRED_OPERATORS = frozenset({"$eq", "$ne", "$in", "$nin", "$and", "$or", "$contains"})
|
||||
_OPTIONAL_OPERATORS = frozenset({"$gt", "$gte", "$lt", "$lte"})
|
||||
_SUPPORTED_OPERATORS = _REQUIRED_OPERATORS | _OPTIONAL_OPERATORS
|
||||
_TOKEN_RE = re.compile(r"\w{2,}", re.UNICODE)
|
||||
|
||||
# A healthy HNSW payload should keep link_lists.bin proportional to
|
||||
# data_level0.bin. When link_lists.bin grows orders of magnitude larger than
|
||||
@@ -104,36 +111,38 @@ def _hnsw_payload_appears_sane(seg_dir: str) -> bool:
|
||||
return ratio is None or ratio <= _HNSW_LINK_TO_DATA_MAX_RATIO
|
||||
|
||||
|
||||
# HNSW tuning to prevent link_lists.bin bloat on large mines (#344).
|
||||
# HNSW batch/sync thresholds applied at collection creation.
|
||||
#
|
||||
# With default params (batch_size=100, sync_threshold=1000, initial capacity
|
||||
# 1000), inserting tens of thousands of drawers triggers ~30 index resizes
|
||||
# and hundreds of persistDirty() calls. persistDirty uses relative seek
|
||||
# positioning in link_lists.bin; accumulated seek drift across resize cycles
|
||||
# causes the OS to extend the sparse file with zero-filled regions, each
|
||||
# cycle compounding the next. Result: link_lists.bin grows into hundreds of
|
||||
# GB sparse, after which `status`/`search`/`repair` segfault.
|
||||
# chromadb's Rust HNSW segment writes index_metadata.pickle and
|
||||
# link_lists.bin only when internal counters cross both thresholds
|
||||
# (batch_size gates _apply_batch; sync_threshold gates _persist).
|
||||
# Records below both thresholds stay in memory and are lost on exit.
|
||||
#
|
||||
# Setting large batch and sync thresholds at collection creation defers
|
||||
# persistence until a single large batch completes, breaking the resize+
|
||||
# persist feedback loop. Empirically validated on a 39,792-drawer rebuild
|
||||
# (palace 376 MB, link_lists.bin 0 bytes, no segfault) in 2026-04.
|
||||
# Previously 50k/50k to work around link_lists.bin sparse-file bloat
|
||||
# in pre-1.5.x Python chromadb (#344). chromadb >=1.5.4 Rust bindings
|
||||
# (the minimum mempalace supports) do not exhibit that bloat; verified
|
||||
# at batch_size=2 with 20k records: link_lists.bin = 171 KB, no
|
||||
# sparse-file inflation.
|
||||
#
|
||||
# Note: chromadb 1.5.x exposes a `collection.modify(configuration={"hnsw":
|
||||
# {"batch_size": ..., "sync_threshold": ...}})` retrofit path for already-
|
||||
# created collections (`UpdateHNSWConfiguration` in chromadb's API), but
|
||||
# this PR doesn't pursue that — once link_lists.bin has bloated, the index
|
||||
# is already corrupt and the only known recovery is a fresh mine.
|
||||
# The 50k guard caused #1579: mines under 50k drawers never triggered
|
||||
# _persist(), leaving index_metadata.pickle absent and link_lists.bin
|
||||
# empty. quarantine_stale_hnsw then renamed the segment on every cold
|
||||
# open after a 300s mtime gap, accumulating .drift-* directories.
|
||||
#
|
||||
# Lowered to 2 (empirical Rust-side minimum for chromadb >=1.5.4; the
|
||||
# Rust bindings reject 1 with InvalidArgumentError) so any mine of 2+
|
||||
# drawers triggers a natural persist. Existing palaces created under
|
||||
# the old 50k guard keep those thresholds in their collection metadata
|
||||
# until the user runs repair --mode from-sqlite --archive-existing.
|
||||
_HNSW_BLOAT_GUARD = {
|
||||
"hnsw:batch_size": 50_000,
|
||||
"hnsw:sync_threshold": 50_000,
|
||||
"hnsw:batch_size": 2,
|
||||
"hnsw:sync_threshold": 2,
|
||||
}
|
||||
|
||||
# Missing index_metadata.pickle is normal only while a segment is still fresh
|
||||
# or effectively empty. Once data_level0.bin has non-trivial payload, a
|
||||
# missing metadata pickle means the segment was interrupted after writing HNSW
|
||||
# data but before writing its metadata. Letting Chroma open that shape can
|
||||
# segfault or hang in native HNSW code.
|
||||
# Below this size, data_level0.bin is too small for a meaningful HNSW graph.
|
||||
# Used by _hnsw_link_lists_is_usable_for_payload (empty link_lists is fine
|
||||
# when data is trivially small) and _missing_dimensionality_appears_recoverable
|
||||
# (don't attempt recovery on segments with negligible data).
|
||||
_HNSW_MISSING_METADATA_DATA_FLOOR = 1024
|
||||
|
||||
|
||||
@@ -158,6 +167,127 @@ def _validate_where(where: Optional[dict]) -> None:
|
||||
stack.extend(x for x in v if isinstance(x, dict))
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list[str]:
|
||||
if not text:
|
||||
return []
|
||||
return _TOKEN_RE.findall(text.lower())
|
||||
|
||||
|
||||
def _bm25_scores(
|
||||
query: str,
|
||||
documents: list[str],
|
||||
k1: float = 1.5,
|
||||
b: float = 0.75,
|
||||
) -> list[float]:
|
||||
query_terms = set(_tokenize(query))
|
||||
n_docs = len(documents)
|
||||
if not query_terms or n_docs == 0:
|
||||
return [0.0] * n_docs
|
||||
|
||||
tokenized = [_tokenize(doc) for doc in documents]
|
||||
doc_lens = [len(toks) for toks in tokenized]
|
||||
if not any(doc_lens):
|
||||
return [0.0] * n_docs
|
||||
avgdl = sum(doc_lens) / n_docs or 1.0
|
||||
|
||||
df = {term: 0 for term in query_terms}
|
||||
for toks in tokenized:
|
||||
for term in set(toks) & query_terms:
|
||||
df[term] += 1
|
||||
|
||||
idf = {
|
||||
term: math.log((n_docs - df[term] + 0.5) / (df[term] + 0.5) + 1.0) for term in query_terms
|
||||
}
|
||||
|
||||
scores = []
|
||||
for toks, dl in zip(tokenized, doc_lens):
|
||||
if dl == 0:
|
||||
scores.append(0.0)
|
||||
continue
|
||||
tf: dict[str, int] = {}
|
||||
for token in toks:
|
||||
if token in query_terms:
|
||||
tf[token] = tf.get(token, 0) + 1
|
||||
score = 0.0
|
||||
for term, freq in tf.items():
|
||||
num = freq * (k1 + 1)
|
||||
den = freq + k1 * (1 - b + b * dl / avgdl)
|
||||
score += idf[term] * num / den
|
||||
scores.append(score)
|
||||
return scores
|
||||
|
||||
|
||||
def _coerce_metadata_value(value: Any) -> Any:
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
return value
|
||||
|
||||
|
||||
def _compare_metadata(actual: Any, op: str, expected: Any) -> bool:
|
||||
actual = _coerce_metadata_value(actual)
|
||||
expected = _coerce_metadata_value(expected)
|
||||
if op == "$eq":
|
||||
return actual == expected
|
||||
if op == "$ne":
|
||||
return actual != expected
|
||||
if op == "$in":
|
||||
return actual in (expected or [])
|
||||
if op == "$nin":
|
||||
return actual not in (expected or [])
|
||||
if op == "$contains":
|
||||
return str(expected) in str(actual or "")
|
||||
try:
|
||||
if op == "$gt":
|
||||
return actual > expected
|
||||
if op == "$gte":
|
||||
return actual >= expected
|
||||
if op == "$lt":
|
||||
return actual < expected
|
||||
if op == "$lte":
|
||||
return actual <= expected
|
||||
except TypeError:
|
||||
return False
|
||||
raise UnsupportedFilterError(f"operator {op!r} not supported by chroma backend")
|
||||
|
||||
|
||||
def _matches_where(meta: dict, where: Optional[dict]) -> bool:
|
||||
if not where:
|
||||
return True
|
||||
if not isinstance(where, dict):
|
||||
return False
|
||||
for key, expected in where.items():
|
||||
if key == "$and":
|
||||
if not all(_matches_where(meta, clause) for clause in expected or []):
|
||||
return False
|
||||
continue
|
||||
if key == "$or":
|
||||
if not any(_matches_where(meta, clause) for clause in expected or []):
|
||||
return False
|
||||
continue
|
||||
if key.startswith("$"):
|
||||
raise UnsupportedFilterError(f"operator {key!r} not supported by chroma backend")
|
||||
actual = meta.get(key)
|
||||
if isinstance(expected, dict):
|
||||
for op, operand in expected.items():
|
||||
if not _compare_metadata(actual, op, operand):
|
||||
return False
|
||||
elif actual != expected:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _metadata_cell_value(sval, ival, fval, bval):
|
||||
if sval is not None:
|
||||
return sval
|
||||
if ival is not None:
|
||||
return ival
|
||||
if fval is not None:
|
||||
return fval
|
||||
if bval is not None:
|
||||
return bool(bval)
|
||||
return None
|
||||
|
||||
|
||||
def _segment_appears_healthy(seg_dir: str) -> bool:
|
||||
"""Return True if a chromadb HNSW segment dir looks intact.
|
||||
|
||||
@@ -168,11 +298,14 @@ def _segment_appears_healthy(seg_dir: str) -> bool:
|
||||
``0x2e`` (the protocol/terminator byte sequence chromadb serializes
|
||||
with).
|
||||
|
||||
Missing metadata is healthy only while the segment still looks fresh or
|
||||
empty. If ``data_level0.bin`` already has non-trivial payload but
|
||||
``index_metadata.pickle`` is missing, the segment is partially flushed:
|
||||
Chroma wrote vector data without the metadata it needs to reopen the
|
||||
HNSW reader safely.
|
||||
When metadata is missing, the segment is either *never-persisted*
|
||||
(sub-threshold: fewer records than ``batch_size``, so chromadb never
|
||||
triggered ``_persist()``) or *partially flushed* (persist started but
|
||||
crashed). The two are distinguished by ``link_lists.bin``: chromadb
|
||||
writes link data during persist, so an empty/absent ``link_lists.bin``
|
||||
together with absent metadata means no persist was ever attempted.
|
||||
Note: ``data_level0.bin`` is pre-allocated at index creation and its
|
||||
size does not indicate actual record count.
|
||||
|
||||
Deliberately format-sniffs only; never deserializes. Deserialization
|
||||
can execute arbitrary code, and the byte-sniff is sufficient to
|
||||
@@ -185,28 +318,23 @@ def _segment_appears_healthy(seg_dir: str) -> bool:
|
||||
files and quarantine_stale_hnsw would conservatively rename them
|
||||
out of the way.
|
||||
"""
|
||||
meta_path = os.path.join(seg_dir, "index_metadata.pickle")
|
||||
|
||||
if not os.path.isfile(meta_path):
|
||||
link_path = os.path.join(seg_dir, "link_lists.bin")
|
||||
try:
|
||||
link_has_data = os.path.isfile(link_path) and os.path.getsize(link_path) > 0
|
||||
except OSError:
|
||||
return False
|
||||
# Both absent → sub-threshold, never persisted.
|
||||
# link_lists written but metadata not → interrupted persist.
|
||||
return not link_has_data
|
||||
|
||||
if not _hnsw_payload_appears_sane(seg_dir):
|
||||
return False
|
||||
|
||||
meta_path = os.path.join(seg_dir, "index_metadata.pickle")
|
||||
if not os.path.isfile(meta_path):
|
||||
data_path = os.path.join(seg_dir, "data_level0.bin")
|
||||
try:
|
||||
if (
|
||||
os.path.isfile(data_path)
|
||||
and os.path.getsize(data_path) > _HNSW_MISSING_METADATA_DATA_FLOOR
|
||||
):
|
||||
return False
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
# No metadata and no meaningful vector payload yet: fresh/empty segment.
|
||||
return True
|
||||
|
||||
try:
|
||||
size = os.path.getsize(meta_path)
|
||||
# A real chromadb metadata file is at least tens of bytes; a
|
||||
# smaller-than-floor file is almost certainly truncated.
|
||||
if size < 16:
|
||||
return False
|
||||
with open(meta_path, "rb") as f:
|
||||
@@ -466,14 +594,15 @@ def _hnsw_element_count(palace_path: str, segment_id: str) -> Optional[int]:
|
||||
# read the collection metadata (older palaces missing the row, sqlite
|
||||
# unreadable). 2000 = 2 × chromadb's default sync_threshold of 1000.
|
||||
#
|
||||
# Why dynamic: PR #1191 set ``hnsw:sync_threshold = 50_000`` to prevent
|
||||
# index bloat, which means flush-lag can grow up to 50K naturally. A
|
||||
# fixed 2000 floor would flag every actively-written palace as DIVERGED
|
||||
# the moment its queue exceeded 10% of sqlite_count, even though chromadb
|
||||
# is behaving correctly. The floor must scale with sync_threshold to
|
||||
# distinguish real corruption (#1222 was 176 613 missing of 192 997 —
|
||||
# orders of magnitude past 2 × any reasonable sync_threshold) from
|
||||
# expected steady-state lag.
|
||||
# Why dynamic: legacy palaces may still carry ``sync_threshold = 50_000``
|
||||
# (the pre-#1579 guard), so flush-lag can grow up to 50K on those palaces.
|
||||
# New palaces use sync_threshold=2 (#1579) and flush almost immediately.
|
||||
# A fixed 2000 floor would flag actively-written legacy palaces as
|
||||
# DIVERGED the moment their queue exceeded 10% of sqlite_count, even
|
||||
# though chromadb is behaving correctly. The floor must scale with the
|
||||
# per-collection sync_threshold to distinguish real corruption (#1222 was
|
||||
# 176 613 missing of 192 997, orders of magnitude past any reasonable
|
||||
# sync_threshold) from expected steady-state lag.
|
||||
_HNSW_DIVERGENCE_FALLBACK_FLOOR = 2000
|
||||
_HNSW_DIVERGENCE_FRACTION = 0.10
|
||||
|
||||
@@ -635,6 +764,93 @@ def _sqlite_embedding_count(palace_path: str, collection_name: str) -> Optional[
|
||||
return None
|
||||
|
||||
|
||||
def _sqlite_wing_room_counts(
|
||||
palace_path: str, collection_name: str
|
||||
) -> Optional[tuple[int, dict[str, dict[str, int]]]]:
|
||||
"""Tally drawers by wing/room straight from ``chroma.sqlite3``.
|
||||
|
||||
Returns ``(total, {wing: {room: count}})`` or ``None`` when the read
|
||||
cannot be trusted — missing DB file, the collection has not been
|
||||
bootstrapped, or any sqlite error (including a sustained writer lock).
|
||||
``None`` signals the caller to fall back to the ChromaDB client path
|
||||
(which also emits the right state-specific guidance for absent/empty
|
||||
palaces).
|
||||
|
||||
The point of reading sqlite directly is to count drawers **without opening
|
||||
the collection**, because opening it cold-loads the HNSW vector index. On
|
||||
large palaces that load costs tens of seconds of CPU per call — a steep,
|
||||
pointless tax for an inspection command that only needs metadata the
|
||||
relational tables already hold (#1681). Wings/rooms live in plain
|
||||
``embedding_metadata`` rows, joined to ``embeddings`` on the
|
||||
``(id, key)`` primary key, so the tally is a bounded scan of the metadata
|
||||
segment: sub-second warm, a few seconds cold on a multi-GB DB — versus the
|
||||
~60s the vector-index load costs.
|
||||
|
||||
Sibling readers that count the same way: :func:`_sqlite_embedding_count`
|
||||
(total only) and ``mcp_server._tool_status_via_sqlite`` (independent
|
||||
wing/room histograms for the #1222 fallback). This one cross-tabulates
|
||||
wing→room to match the ChromaDB ``status()`` output shape.
|
||||
|
||||
Notes:
|
||||
- ``busy_timeout`` lets a transient checkpoint lock resolve instead of
|
||||
instantly demoting to the slow HNSW path; a *sustained* lock still
|
||||
raises and falls back (slow but correct).
|
||||
- ``s.scope = 'METADATA'`` makes the single-segment join explicit so a
|
||||
future ChromaDB that also stored per-vector-segment rows could not
|
||||
silently double every count.
|
||||
- ``COALESCE`` over ``string_value``/``int_value``/``float_value`` matches
|
||||
the ChromaDB path, which surfaces a numeric wing/room natively rather
|
||||
than dropping it to ``"?"``.
|
||||
"""
|
||||
db_path = os.path.join(palace_path, "chroma.sqlite3")
|
||||
if not os.path.isfile(db_path):
|
||||
return None
|
||||
try:
|
||||
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||||
try:
|
||||
# Wait out a transient writer/checkpoint lock rather than falling
|
||||
# straight back to the expensive vector-index path (#1681).
|
||||
conn.execute("PRAGMA busy_timeout = 3000")
|
||||
# Distinguish "collection never bootstrapped" (-> None, so the
|
||||
# caller can show the 'initialized but empty' message) from
|
||||
# "collection exists with zero drawers" (-> a real 0 tally).
|
||||
if (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM collections WHERE name = ?", (collection_name,)
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
return None
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT COALESCE(wm.string_value, CAST(wm.int_value AS TEXT),
|
||||
CAST(wm.float_value AS TEXT), '?') AS wing,
|
||||
COALESCE(rm.string_value, CAST(rm.int_value AS TEXT),
|
||||
CAST(rm.float_value AS TEXT), '?') AS room,
|
||||
COUNT(*) AS n
|
||||
FROM embeddings e
|
||||
JOIN segments s ON e.segment_id = s.id AND s.scope = 'METADATA'
|
||||
JOIN collections c ON s.collection = c.id
|
||||
LEFT JOIN embedding_metadata wm ON wm.id = e.id AND wm.key = 'wing'
|
||||
LEFT JOIN embedding_metadata rm ON rm.id = e.id AND rm.key = 'room'
|
||||
WHERE c.name = ?
|
||||
GROUP BY wing, room
|
||||
""",
|
||||
(collection_name,),
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
return None
|
||||
|
||||
total = 0
|
||||
wing_rooms: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
|
||||
for wing, room, n in rows:
|
||||
wing_rooms[wing][room] += int(n)
|
||||
total += int(n)
|
||||
return total, wing_rooms
|
||||
|
||||
|
||||
def _pin_hnsw_threads(collection) -> None:
|
||||
"""Best-effort retrofit: pin ``hnsw:num_threads=1`` on an existing collection.
|
||||
|
||||
@@ -665,6 +881,7 @@ def _pin_hnsw_threads(collection) -> None:
|
||||
|
||||
|
||||
_BLOB_FIX_MARKER = ".blob_seq_ids_migrated"
|
||||
_COLLECTION_TYPE_MARKER = ".collection_type_fixed"
|
||||
|
||||
|
||||
def _valid_dimensionality(value: object) -> bool:
|
||||
@@ -883,6 +1100,69 @@ def _fix_blob_seq_ids(palace_path: str) -> None:
|
||||
logger.exception("Could not write migration marker %s", marker)
|
||||
|
||||
|
||||
def _fix_missing_collection_type(palace_path: str) -> None:
|
||||
"""Add ``_type`` to ``collections.config_json_str`` where absent.
|
||||
|
||||
chromadb <= 1.5.8 writes ``config_json_str = '{}'`` (empty JSON) when
|
||||
creating collections. chromadb 1.5.9 switched from the permissive
|
||||
``load_collection_configuration_from_json_str`` to
|
||||
``CollectionConfigurationInternal.from_json`` which requires a ``_type``
|
||||
key — its absence raises ``KeyError: '_type'`` on palace open.
|
||||
|
||||
This migration adds the missing marker so both old and new chromadb
|
||||
versions can load the collection. The value
|
||||
``"CollectionConfigurationInternal"`` matches what ``to_json()`` writes
|
||||
for freshly-created collections.
|
||||
|
||||
Same lifecycle constraints as :func:`_fix_blob_seq_ids`: must run
|
||||
BEFORE ``PersistentClient`` is created.
|
||||
"""
|
||||
db_path = os.path.join(palace_path, "chroma.sqlite3")
|
||||
if not os.path.isfile(db_path):
|
||||
return
|
||||
marker = os.path.join(palace_path, _COLLECTION_TYPE_MARKER)
|
||||
if os.path.isfile(marker):
|
||||
return
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
try:
|
||||
rows = conn.execute("SELECT id, config_json_str FROM collections").fetchall()
|
||||
except sqlite3.OperationalError:
|
||||
return
|
||||
updates = []
|
||||
for coll_id, config_str in rows:
|
||||
if not config_str:
|
||||
config_str = "{}"
|
||||
try:
|
||||
config = json.loads(config_str)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
if not isinstance(config, dict):
|
||||
continue
|
||||
if "_type" not in config:
|
||||
config["_type"] = "CollectionConfigurationInternal"
|
||||
updates.append((json.dumps(config), coll_id))
|
||||
if updates:
|
||||
conn.executemany(
|
||||
"UPDATE collections SET config_json_str = ? WHERE id = ?",
|
||||
updates,
|
||||
)
|
||||
conn.commit()
|
||||
logger.info(
|
||||
"Fixed %d collection(s) missing _type in config_json_str",
|
||||
len(updates),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Could not fix collection config_json_str in %s", db_path)
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
Path(marker).touch()
|
||||
except OSError:
|
||||
logger.exception("Could not write migration marker %s", marker)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collection adapter
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -973,8 +1253,40 @@ class ChromaCollection(BaseCollection):
|
||||
for m in metadatas
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_documents_for_chromadb(documents):
|
||||
"""Strip lone UTF-16 surrogates from every document before it reaches
|
||||
the chromadb client.
|
||||
|
||||
A single lone surrogate (U+D800–U+DFFF) raises ``UnicodeEncodeError``
|
||||
inside chromadb's encode path and aborts the *entire* add/upsert batch
|
||||
with a ``-32000`` Internal Error, silently dropping every other row in
|
||||
the same batch (#1235).
|
||||
|
||||
#1235 fixed this for the MCP write tools via ``sanitize_content``, but
|
||||
the bulk ingest paths (miner, convo_miner, sweeper, diary_ingest) build
|
||||
documents without routing through that helper and reach this backend
|
||||
directly. Sanitising here makes the chokepoint catch-all complete: the
|
||||
sibling :meth:`_sanitize_metadatas_for_chromadb` already guarantees this
|
||||
for metadata one method over; documents get the same guarantee.
|
||||
"""
|
||||
if documents is None:
|
||||
return None
|
||||
from ..config import strip_lone_surrogates
|
||||
|
||||
# chromadb accepts OneOrMany[Document]: a bare str is a single document,
|
||||
# not an iterable of characters. Handle it explicitly so we don't split
|
||||
# it into per-character documents — that would be exactly the kind of
|
||||
# silent corruption this method exists to prevent.
|
||||
if isinstance(documents, str):
|
||||
return strip_lone_surrogates(documents)
|
||||
return [strip_lone_surrogates(d) if isinstance(d, str) else d for d in documents]
|
||||
|
||||
def add(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
kwargs: dict[str, Any] = {"documents": documents, "ids": ids}
|
||||
kwargs: dict[str, Any] = {
|
||||
"documents": self._sanitize_documents_for_chromadb(documents),
|
||||
"ids": ids,
|
||||
}
|
||||
sanitized = self._sanitize_metadatas_for_chromadb(metadatas)
|
||||
if sanitized is not None:
|
||||
kwargs["metadatas"] = sanitized
|
||||
@@ -984,7 +1296,10 @@ class ChromaCollection(BaseCollection):
|
||||
self._collection.add(**kwargs)
|
||||
|
||||
def upsert(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
kwargs: dict[str, Any] = {"documents": documents, "ids": ids}
|
||||
kwargs: dict[str, Any] = {
|
||||
"documents": self._sanitize_documents_for_chromadb(documents),
|
||||
"ids": ids,
|
||||
}
|
||||
sanitized = self._sanitize_metadatas_for_chromadb(metadatas)
|
||||
if sanitized is not None:
|
||||
kwargs["metadatas"] = sanitized
|
||||
@@ -1005,7 +1320,7 @@ class ChromaCollection(BaseCollection):
|
||||
raise ValueError("update requires at least one of documents, metadatas, embeddings")
|
||||
kwargs: dict[str, Any] = {"ids": ids}
|
||||
if documents is not None:
|
||||
kwargs["documents"] = documents
|
||||
kwargs["documents"] = self._sanitize_documents_for_chromadb(documents)
|
||||
if metadatas is not None:
|
||||
kwargs["metadatas"] = metadatas
|
||||
if embeddings is not None:
|
||||
@@ -1161,6 +1476,242 @@ class ChromaCollection(BaseCollection):
|
||||
def count(self):
|
||||
return self._collection.count()
|
||||
|
||||
def lexical_search(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
n_results: int = 10,
|
||||
where: Optional[dict] = None,
|
||||
) -> LexicalResult:
|
||||
"""Return lexical BM25 candidates for this collection.
|
||||
|
||||
This is the normal healthy-Chroma implementation behind the optional
|
||||
backend capability. The HNSW-disabled fallback in ``searcher.py`` still
|
||||
reads ``chroma.sqlite3`` directly and remains Chroma-only.
|
||||
"""
|
||||
_validate_where(where)
|
||||
sqlite_hits = self._lexical_search_via_sqlite(query=query, n_results=n_results, where=where)
|
||||
if sqlite_hits is not None:
|
||||
return LexicalResult(hits=sqlite_hits)
|
||||
|
||||
# Directly-constructed ChromaCollection test doubles may not carry a
|
||||
# palace path. Keep lexical_search usable in that shape, but normal
|
||||
# MemPalace paths above use Chroma's FTS table instead of scanning every
|
||||
# drawer through the Python client.
|
||||
total = self.count()
|
||||
docs: list[str] = []
|
||||
metas: list[dict] = []
|
||||
ids: list[str] = []
|
||||
offset = 0
|
||||
batch_size = 1000
|
||||
while offset < total:
|
||||
kwargs: dict[str, Any] = {
|
||||
"include": ["documents", "metadatas"],
|
||||
"limit": batch_size,
|
||||
"offset": offset,
|
||||
}
|
||||
if where:
|
||||
kwargs["where"] = where
|
||||
batch = self.get(**kwargs)
|
||||
if not batch.ids:
|
||||
break
|
||||
ids.extend(batch.ids)
|
||||
docs.extend(doc or "" for doc in batch.documents)
|
||||
metas.extend(meta or {} for meta in batch.metadatas)
|
||||
offset += len(batch.ids)
|
||||
|
||||
scores = _bm25_scores(query, docs)
|
||||
hits = [
|
||||
LexicalHit(id=doc_id, document=doc, metadata=meta, score=float(score))
|
||||
for doc_id, doc, meta, score in zip(ids, docs, metas, scores)
|
||||
if score > 0
|
||||
]
|
||||
hits.sort(key=lambda hit: hit.score, reverse=True)
|
||||
return LexicalResult(hits=hits[:n_results])
|
||||
|
||||
def _collection_name(self) -> Optional[str]:
|
||||
name = getattr(self._collection, "name", None)
|
||||
if callable(name):
|
||||
try:
|
||||
name = name()
|
||||
except TypeError:
|
||||
name = None
|
||||
return str(name) if name else None
|
||||
|
||||
def _lexical_search_via_sqlite(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
n_results: int,
|
||||
where: Optional[dict],
|
||||
max_candidates: int = 500,
|
||||
) -> Optional[list[LexicalHit]]:
|
||||
if not self._palace_path:
|
||||
return None
|
||||
db_path = os.path.join(self._palace_path, "chroma.sqlite3")
|
||||
if not os.path.isfile(db_path):
|
||||
return []
|
||||
collection_name = self._collection_name()
|
||||
if not collection_name:
|
||||
return []
|
||||
|
||||
tokens = [t for t in _tokenize(query) if len(t) >= 3]
|
||||
use_recency_fallback = not tokens
|
||||
candidate_ids: list[int] = []
|
||||
# Map internal embeddings.id (rowid, used to join embedding_metadata)
|
||||
# to the public embeddings.embedding_id so returned LexicalHit.id values
|
||||
# round-trip through get(ids=...). The two differ: id is the integer
|
||||
# rowid, embedding_id is the user-facing drawer id.
|
||||
public_ids: dict[int, str] = {}
|
||||
try:
|
||||
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
except sqlite3.Error:
|
||||
logger.debug("Chroma lexical sqlite open failed", exc_info=True)
|
||||
return []
|
||||
|
||||
try:
|
||||
if tokens:
|
||||
fts_query = " OR ".join(tokens)
|
||||
# If a metadata filter is present, do not cap before filtering:
|
||||
# otherwise a common term can fill the window with wrong-scope
|
||||
# rows and hide valid scoped hits later in the FTS result set.
|
||||
limit_sql = "" if where else "LIMIT ?"
|
||||
params = [fts_query, collection_name]
|
||||
if not where:
|
||||
params.append(max(max_candidates, n_results))
|
||||
try:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT e.id, e.embedding_id
|
||||
FROM embedding_fulltext_search
|
||||
JOIN embeddings e ON e.id = embedding_fulltext_search.rowid
|
||||
JOIN segments s ON e.segment_id = s.id
|
||||
JOIN collections c ON s.collection = c.id
|
||||
WHERE embedding_fulltext_search MATCH ?
|
||||
AND c.name = ?
|
||||
{limit_sql}
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
candidate_ids = [int(row[0]) for row in rows]
|
||||
public_ids.update({int(row[0]): str(row[1]) for row in rows})
|
||||
except sqlite3.Error:
|
||||
logger.debug(
|
||||
"Chroma lexical FTS query failed; using recency fallback", exc_info=True
|
||||
)
|
||||
use_recency_fallback = True
|
||||
|
||||
if not candidate_ids and use_recency_fallback:
|
||||
order_expr = "e.created_at DESC"
|
||||
try:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT e.id, e.embedding_id
|
||||
FROM embeddings e
|
||||
JOIN segments s ON e.segment_id = s.id
|
||||
JOIN collections c ON s.collection = c.id
|
||||
WHERE c.name = ?
|
||||
ORDER BY {order_expr}
|
||||
LIMIT ?
|
||||
""",
|
||||
(collection_name, max(max_candidates, n_results)),
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
logger.debug(
|
||||
"Chroma lexical recency fallback failed; ordering by id", exc_info=True
|
||||
)
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT e.id, e.embedding_id
|
||||
FROM embeddings e
|
||||
JOIN segments s ON e.segment_id = s.id
|
||||
JOIN collections c ON s.collection = c.id
|
||||
WHERE c.name = ?
|
||||
ORDER BY e.id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(collection_name, max(max_candidates, n_results)),
|
||||
).fetchall()
|
||||
candidate_ids = [int(row[0]) for row in rows]
|
||||
public_ids.update({int(row[0]): str(row[1]) for row in rows})
|
||||
|
||||
if not candidate_ids:
|
||||
return []
|
||||
|
||||
meta_columns = {
|
||||
row["name"]
|
||||
for row in conn.execute("PRAGMA table_info(embedding_metadata)").fetchall()
|
||||
}
|
||||
value_columns = [
|
||||
col
|
||||
for col in ("string_value", "int_value", "float_value", "bool_value")
|
||||
if col in meta_columns
|
||||
]
|
||||
if not value_columns:
|
||||
return []
|
||||
meta_rows = []
|
||||
for start in range(0, len(candidate_ids), 900):
|
||||
chunk_ids = candidate_ids[start : start + 900]
|
||||
placeholders = ",".join("?" for _ in chunk_ids)
|
||||
meta_rows.extend(
|
||||
conn.execute(
|
||||
f"""
|
||||
SELECT id, key, {", ".join(value_columns)}
|
||||
FROM embedding_metadata
|
||||
WHERE id IN ({placeholders})
|
||||
""",
|
||||
chunk_ids,
|
||||
).fetchall()
|
||||
)
|
||||
except sqlite3.Error:
|
||||
logger.debug("Chroma lexical sqlite read failed", exc_info=True)
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
drawers: dict[int, dict] = {}
|
||||
for row in meta_rows:
|
||||
emb_id = int(row["id"])
|
||||
key = row["key"]
|
||||
values = {col: row[col] if col in row.keys() else None for col in value_columns}
|
||||
value = _metadata_cell_value(
|
||||
values.get("string_value"),
|
||||
values.get("int_value"),
|
||||
values.get("float_value"),
|
||||
values.get("bool_value"),
|
||||
)
|
||||
drawer = drawers.setdefault(emb_id, {"metadata": {}, "document": ""})
|
||||
if key == "chroma:document":
|
||||
drawer["document"] = str(value or "")
|
||||
else:
|
||||
drawer["metadata"][key] = value
|
||||
|
||||
ordered = []
|
||||
for emb_id in candidate_ids:
|
||||
drawer = drawers.get(emb_id)
|
||||
if drawer is None:
|
||||
continue
|
||||
meta = drawer["metadata"]
|
||||
if not _matches_where(meta, where):
|
||||
continue
|
||||
ordered.append((emb_id, drawer["document"], meta))
|
||||
|
||||
docs = [doc for _, doc, _ in ordered]
|
||||
scores = _bm25_scores(query, docs)
|
||||
hits = [
|
||||
LexicalHit(
|
||||
id=public_ids.get(emb_id, str(emb_id)),
|
||||
document=doc,
|
||||
metadata=meta,
|
||||
score=float(score),
|
||||
)
|
||||
for (emb_id, doc, meta), score in zip(ordered, scores)
|
||||
if score > 0
|
||||
]
|
||||
hits.sort(key=lambda hit: hit.score, reverse=True)
|
||||
return hits[:n_results]
|
||||
|
||||
@property
|
||||
def metadata(self) -> dict:
|
||||
"""Pass-through to the underlying ChromaDB collection's metadata.
|
||||
@@ -1199,6 +1750,7 @@ class ChromaBackend(BaseBackend):
|
||||
"supports_embeddings_out",
|
||||
"supports_metadata_filters",
|
||||
"supports_contains_fast",
|
||||
"supports_lexical_search",
|
||||
"local_mode",
|
||||
}
|
||||
)
|
||||
@@ -1318,12 +1870,17 @@ class ChromaBackend(BaseBackend):
|
||||
)
|
||||
|
||||
if cached is None or inode_changed or mtime_changed or mtime_appeared:
|
||||
# An inode swap means we are reopening a different physical DB
|
||||
# (post-restore, fresh palace at the same path, etc.); drop the
|
||||
# per-process gate so the quarantine pre-checks run again
|
||||
# against the new disk state instead of trusting cached "we
|
||||
# already cleaned this path" credit from the prior inode.
|
||||
if inode_changed:
|
||||
# Drop the per-process quarantine gate so the HNSW pre-checks
|
||||
# run again against the new disk state. An inode swap means a
|
||||
# different physical DB (post-restore, fresh palace at the same
|
||||
# path); an mtime/appearance change means an external in-place
|
||||
# write (closet_llm, mine, compress) that may have drifted the
|
||||
# HNSW index while this process was running.
|
||||
if (
|
||||
inode_changed
|
||||
or mtime_changed
|
||||
or (mtime_appeared and palace_path in self._freshness)
|
||||
):
|
||||
ChromaBackend._quarantined_paths.discard(palace_path)
|
||||
ChromaBackend._prepare_palace_for_open(palace_path)
|
||||
cached = chromadb.PersistentClient(path=palace_path)
|
||||
@@ -1340,17 +1897,12 @@ class ChromaBackend(BaseBackend):
|
||||
|
||||
# Per-process record of palaces that have already had the cold-start
|
||||
# quarantine invoked at least once. The proactive HNSW checks are a
|
||||
# *cold-start* protection — they catch segments that arrive stale relative
|
||||
# *cold-start* protection -- they catch segments that arrive stale relative
|
||||
# to ``chroma.sqlite3`` or invalid on disk (e.g. cross-machine replication,
|
||||
# partial restore, crashed-mid-write). Once a long-running process has
|
||||
# opened the palace cleanly, re-firing the stale check on every reconnect
|
||||
# is a *runtime thrash*: the daemon's own writes bump sqlite mtime but HNSW
|
||||
# flushes batch on chromadb's internal cadence, so the mtime gap naturally
|
||||
# exceeds the threshold under steady write load even though nothing is
|
||||
# corrupt.
|
||||
# Real runtime drift is still handled — palace-daemon's ``_auto_repair``
|
||||
# calls :func:`quarantine_stale_hnsw` directly on observed HNSW errors,
|
||||
# which bypasses this gate.
|
||||
# partial restore, crashed-mid-write). The gate is cleared whenever the
|
||||
# palace changes on disk (inode swap, mtime bump, or file appearance), so
|
||||
# external writes that drift HNSW segments are caught on the next open
|
||||
# without requiring a full process restart.
|
||||
#
|
||||
# Thread-safety: this set is mutated without a lock. Two concurrent
|
||||
# ``make_client()`` calls for the same palace can both pass the
|
||||
@@ -1366,25 +1918,29 @@ class ChromaBackend(BaseBackend):
|
||||
"""Run the pre-open safety pass shared by :meth:`make_client` and
|
||||
:meth:`_client`.
|
||||
|
||||
Three steps, all required before constructing a ``PersistentClient``:
|
||||
Four steps, all required before constructing a ``PersistentClient``:
|
||||
|
||||
1. ``_fix_blob_seq_ids`` — repairs the BLOB seq_id quirk that bites
|
||||
1. ``_fix_missing_collection_type`` — adds the ``_type`` marker to
|
||||
``collections.config_json_str`` that chromadb 1.5.9+ requires
|
||||
but <= 1.5.8 never wrote (#1611).
|
||||
2. ``_fix_blob_seq_ids`` — repairs the BLOB seq_id quirk that bites
|
||||
certain chromadb migrations.
|
||||
2. ``quarantine_invalid_hnsw_metadata`` — renames aside any HNSW
|
||||
3. ``quarantine_invalid_hnsw_metadata`` — renames aside any HNSW
|
||||
``index_metadata.pickle`` that fails to load, so chromadb opens
|
||||
against an empty index instead of crashing on the unloadable
|
||||
pickle (#1266 / PR #1285).
|
||||
3. ``quarantine_stale_hnsw`` — also gated by :attr:`_quarantined_paths`
|
||||
so it fires once per palace per process. This is the SIGSEGV
|
||||
prevention path for stale HNSW segments (see #1121, #1132, #1263);
|
||||
wiring it through this helper means CLI mining, search, repair,
|
||||
and status all benefit, not just the legacy ``make_client``
|
||||
callers.
|
||||
4. ``quarantine_stale_hnsw`` -- gated by :attr:`_quarantined_paths`
|
||||
so it fires once per palace until the gate is re-armed by a
|
||||
disk change. This is the SIGSEGV prevention path for stale
|
||||
HNSW segments (see #1121, #1132, #1263); wiring it through
|
||||
this helper means CLI mining, search, repair, and status all
|
||||
benefit, not just the legacy ``make_client`` callers.
|
||||
|
||||
Idempotent: safe to call from any code path that is about to open or
|
||||
re-open a palace. The ``_quarantined_paths`` gate prevents thrash on
|
||||
hot paths (e.g. ``_client()`` is called on every backend operation).
|
||||
"""
|
||||
_fix_missing_collection_type(palace_path)
|
||||
_fix_blob_seq_ids(palace_path)
|
||||
if palace_path not in ChromaBackend._quarantined_paths:
|
||||
quarantine_invalid_hnsw_metadata(palace_path)
|
||||
@@ -1393,15 +1949,14 @@ class ChromaBackend(BaseBackend):
|
||||
|
||||
@staticmethod
|
||||
def make_client(palace_path: str):
|
||||
"""Create a fresh ``PersistentClient`` (fixes BLOB seq_ids first).
|
||||
"""Create a fresh ``PersistentClient`` (runs pre-open safety pass first).
|
||||
|
||||
Deprecated-ish: exposed for legacy long-lived callers that manage their
|
||||
own client cache. New code should obtain a collection through
|
||||
:meth:`get_collection` which manages caching internally.
|
||||
|
||||
Quarantines HNSW segments **once per palace per process**. See
|
||||
:attr:`_quarantined_paths` for the rationale (cold-start protection
|
||||
vs. runtime thrash on steady-write daemons).
|
||||
Quarantines HNSW segments on first open and after any detected
|
||||
disk change. See :attr:`_quarantined_paths` for the gate logic.
|
||||
"""
|
||||
ChromaBackend._prepare_palace_for_open(palace_path)
|
||||
return chromadb.PersistentClient(path=palace_path)
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Core-side embedding adapter for explicit-vector backends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from .base import BaseCollection
|
||||
|
||||
|
||||
def _embed_texts(texts: list[str]) -> list[list[float]]:
|
||||
"""Embed ``texts`` with the configured local embedding function."""
|
||||
if not texts:
|
||||
return []
|
||||
from ..embedding import get_embedding_function
|
||||
|
||||
ef = get_embedding_function()
|
||||
vectors = ef(input=texts)
|
||||
return [list(v) for v in vectors]
|
||||
|
||||
|
||||
def _as_list(value):
|
||||
"""Normalize ChromaDB's ``OneOrMany`` shape (``str`` | ``dict`` | sequence) to a list.
|
||||
|
||||
A bare ``str`` (a document/id) or ``dict`` (a single metadata) must be
|
||||
*wrapped*, not iterated: ``list("abc")`` yields ``['a', 'b', 'c']`` and
|
||||
``list({"k": 1})`` yields ``['k']`` — either desyncs embeddings/metadatas
|
||||
from ``ids`` on explicit-vector backends (pgvector, sqlite_exact). A list is
|
||||
returned unchanged (no copy); any other iterable is materialized once.
|
||||
See PR #1706/#1707 review.
|
||||
"""
|
||||
if isinstance(value, (str, dict)):
|
||||
return [value]
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return list(value)
|
||||
|
||||
|
||||
class EmbeddingCollection(BaseCollection):
|
||||
"""Wrap a collection that requires explicit vectors.
|
||||
|
||||
Backends opt in with the ``requires_explicit_embeddings`` capability.
|
||||
Core callers can keep using ``documents=`` and ``query_texts=``; this
|
||||
wrapper computes vectors locally before delegating to the backend.
|
||||
"""
|
||||
|
||||
def __init__(self, inner: BaseCollection):
|
||||
self._inner = inner
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._inner, name)
|
||||
|
||||
def add(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
documents = _as_list(documents)
|
||||
ids = _as_list(ids)
|
||||
if metadatas is not None:
|
||||
metadatas = _as_list(metadatas)
|
||||
if embeddings is None:
|
||||
embeddings = _embed_texts(documents)
|
||||
return self._inner.add(
|
||||
documents=documents,
|
||||
ids=ids,
|
||||
metadatas=metadatas,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
|
||||
def upsert(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
documents = _as_list(documents)
|
||||
ids = _as_list(ids)
|
||||
if metadatas is not None:
|
||||
metadatas = _as_list(metadatas)
|
||||
if embeddings is None:
|
||||
embeddings = _embed_texts(documents)
|
||||
return self._inner.upsert(
|
||||
documents=documents,
|
||||
ids=ids,
|
||||
metadatas=metadatas,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
|
||||
def query(
|
||||
self,
|
||||
*,
|
||||
query_texts: Optional[list[str] | str] = None,
|
||||
query_embeddings: Optional[list[list[float]]] = None,
|
||||
n_results: int = 10,
|
||||
where: Optional[dict] = None,
|
||||
where_document: Optional[dict] = None,
|
||||
include: Optional[list[str]] = None,
|
||||
):
|
||||
if query_texts is not None and query_embeddings is None:
|
||||
query_embeddings = _embed_texts(_as_list(query_texts))
|
||||
query_texts = None
|
||||
return self._inner.query(
|
||||
query_texts=query_texts,
|
||||
query_embeddings=query_embeddings,
|
||||
n_results=n_results,
|
||||
where=where,
|
||||
where_document=where_document,
|
||||
include=include,
|
||||
)
|
||||
|
||||
def get(
|
||||
self, *, ids=None, where=None, where_document=None, limit=None, offset=None, include=None
|
||||
):
|
||||
return self._inner.get(
|
||||
ids=ids,
|
||||
where=where,
|
||||
where_document=where_document,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
include=include,
|
||||
)
|
||||
|
||||
def delete(self, *, ids=None, where=None):
|
||||
return self._inner.delete(ids=ids, where=where)
|
||||
|
||||
def count(self) -> int:
|
||||
return self._inner.count()
|
||||
|
||||
def estimated_count(self) -> int:
|
||||
return self._inner.estimated_count()
|
||||
|
||||
def close(self) -> None:
|
||||
return self._inner.close()
|
||||
|
||||
def health(self):
|
||||
return self._inner.health()
|
||||
|
||||
def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None):
|
||||
return self._inner.lexical_search(query=query, n_results=n_results, where=where)
|
||||
|
||||
def update(self, *, ids, documents=None, metadatas=None, embeddings=None):
|
||||
ids = _as_list(ids)
|
||||
if documents is not None:
|
||||
documents = _as_list(documents)
|
||||
if embeddings is None:
|
||||
embeddings = _embed_texts(documents)
|
||||
if metadatas is not None:
|
||||
metadatas = _as_list(metadatas)
|
||||
return self._inner.update(
|
||||
ids=ids,
|
||||
documents=documents,
|
||||
metadatas=metadatas,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -125,6 +125,38 @@ def get_backend(name: str) -> BaseBackend:
|
||||
return inst
|
||||
|
||||
|
||||
def detect_backends_for_path(path: str) -> list[str]:
|
||||
"""Return all registered backend names whose artifacts are present at ``path``.
|
||||
|
||||
Detection is a migration/protection aid for local palaces. Backends are
|
||||
checked in registry-name order so callers get deterministic diagnostics if
|
||||
a broken directory contains artifacts from more than one backend.
|
||||
"""
|
||||
_discover_entry_points()
|
||||
detected = []
|
||||
for name in sorted(_registry):
|
||||
cls = _registry[name]
|
||||
try:
|
||||
if cls.detect(path):
|
||||
detected.append(name)
|
||||
except Exception:
|
||||
logger.exception("detect() raised on backend %r", name)
|
||||
return detected
|
||||
|
||||
|
||||
def detect_backend_for_path(path: str) -> Optional[str]:
|
||||
"""Return the single detected backend at ``path``, or ``None``.
|
||||
|
||||
If multiple backend artifacts are present, the first name in registry order
|
||||
is returned for backward compatibility. Callers that enforce mismatch
|
||||
protection should use :func:`detect_backends_for_path`.
|
||||
"""
|
||||
detected = detect_backends_for_path(path)
|
||||
if detected:
|
||||
return detected[0]
|
||||
return None
|
||||
|
||||
|
||||
def reset_backends() -> None:
|
||||
"""Close and drop all cached backend instances (primarily for tests)."""
|
||||
with _lock:
|
||||
@@ -161,14 +193,9 @@ def resolve_backend_for_palace(
|
||||
return candidate
|
||||
|
||||
_discover_entry_points()
|
||||
if palace_path:
|
||||
for name, cls in _registry.items():
|
||||
try:
|
||||
if cls.detect(palace_path):
|
||||
return name
|
||||
except Exception:
|
||||
logger.exception("detect() raised on backend %r", name)
|
||||
continue
|
||||
detected = detect_backend_for_path(palace_path) if palace_path else None
|
||||
if detected:
|
||||
return detected
|
||||
return default
|
||||
|
||||
|
||||
@@ -180,10 +207,19 @@ def resolve_backend_for_palace(
|
||||
def _register_builtins() -> None:
|
||||
"""Register chroma as the in-tree default."""
|
||||
from .chroma import ChromaBackend
|
||||
from .pgvector import PgVectorBackend
|
||||
from .qdrant import QdrantBackend
|
||||
from .sqlite_exact import SQLiteExactBackend
|
||||
|
||||
# Use setdefault semantics so a caller that pre-registered for tests wins.
|
||||
if "chroma" not in _registry:
|
||||
_registry["chroma"] = ChromaBackend
|
||||
if "qdrant" not in _registry:
|
||||
_registry["qdrant"] = QdrantBackend
|
||||
if "sqlite_exact" not in _registry:
|
||||
_registry["sqlite_exact"] = SQLiteExactBackend
|
||||
if "pgvector" not in _registry:
|
||||
_registry["pgvector"] = PgVectorBackend
|
||||
|
||||
|
||||
_register_builtins()
|
||||
|
||||
@@ -0,0 +1,940 @@
|
||||
"""SQLite exact-vector backend for MemPalace.
|
||||
|
||||
This backend is intentionally simple and local-first. It is a correctness
|
||||
backend, not a high-throughput ANN backend: vectors are stored as float32
|
||||
blobs and query uses exact cosine distance over the matching collection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .base import (
|
||||
BackendClosedError,
|
||||
BaseBackend,
|
||||
BaseCollection,
|
||||
CollectionNotInitializedError,
|
||||
DimensionMismatchError,
|
||||
GetResult,
|
||||
HealthStatus,
|
||||
LexicalHit,
|
||||
LexicalResult,
|
||||
PalaceNotFoundError,
|
||||
PalaceRef,
|
||||
QueryResult,
|
||||
UnsupportedFilterError,
|
||||
_IncludeSpec,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DB_FILENAME = "sqlite_exact.sqlite3"
|
||||
_TOKEN_RE = re.compile(r"\w{2,}", re.UNICODE)
|
||||
_SUPPORTED_OPERATORS = frozenset(
|
||||
{"$eq", "$ne", "$in", "$nin", "$and", "$or", "$contains", "$gt", "$gte", "$lt", "$lte"}
|
||||
)
|
||||
|
||||
|
||||
def _utcnow() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _json_dumps(obj: Any) -> str:
|
||||
return json.dumps(obj or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def _json_loads(text: str | None) -> dict:
|
||||
if not text:
|
||||
return {}
|
||||
try:
|
||||
value = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _encode_vector(vector: list[float]) -> bytes:
|
||||
return _as_vector_array(vector).tobytes()
|
||||
|
||||
|
||||
def _as_vector_array(vector: list[float]) -> np.ndarray:
|
||||
arr = np.asarray(vector, dtype=np.float32)
|
||||
if arr.ndim != 1 or arr.size == 0:
|
||||
raise ValueError("embedding must be a non-empty 1D vector")
|
||||
return arr
|
||||
|
||||
|
||||
def _decode_vector(blob: bytes | None) -> list[float]:
|
||||
if not blob:
|
||||
return []
|
||||
return np.frombuffer(blob, dtype=np.float32).astype(float).tolist()
|
||||
|
||||
|
||||
def _decode_array(blob: bytes | None) -> Optional[np.ndarray]:
|
||||
if not blob:
|
||||
return None
|
||||
arr = np.frombuffer(blob, dtype=np.float32)
|
||||
if arr.size == 0:
|
||||
return None
|
||||
return arr
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list[str]:
|
||||
if not text:
|
||||
return []
|
||||
return _TOKEN_RE.findall(text.lower())
|
||||
|
||||
|
||||
def _bm25_scores(query: str, documents: list[str], k1: float = 1.5, b: float = 0.75) -> list[float]:
|
||||
query_terms = set(_tokenize(query))
|
||||
n_docs = len(documents)
|
||||
if not query_terms or n_docs == 0:
|
||||
return [0.0] * n_docs
|
||||
|
||||
tokenized = [_tokenize(d) for d in documents]
|
||||
doc_lens = [len(toks) for toks in tokenized]
|
||||
if not any(doc_lens):
|
||||
return [0.0] * n_docs
|
||||
avgdl = sum(doc_lens) / n_docs or 1.0
|
||||
|
||||
df = {term: 0 for term in query_terms}
|
||||
for toks in tokenized:
|
||||
for term in set(toks) & query_terms:
|
||||
df[term] += 1
|
||||
|
||||
idf = {term: np.log((n_docs - df[term] + 0.5) / (df[term] + 0.5) + 1.0) for term in query_terms}
|
||||
|
||||
scores = []
|
||||
for toks, dl in zip(tokenized, doc_lens):
|
||||
if dl == 0:
|
||||
scores.append(0.0)
|
||||
continue
|
||||
tf: dict[str, int] = {}
|
||||
for token in toks:
|
||||
if token in query_terms:
|
||||
tf[token] = tf.get(token, 0) + 1
|
||||
score = 0.0
|
||||
for term, freq in tf.items():
|
||||
num = freq * (k1 + 1)
|
||||
den = freq + k1 * (1 - b + b * dl / avgdl)
|
||||
score += float(idf[term]) * num / den
|
||||
scores.append(score)
|
||||
return scores
|
||||
|
||||
|
||||
def _validate_where(where: Optional[dict]) -> None:
|
||||
if not where:
|
||||
return
|
||||
stack = [where]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
for key, value in node.items():
|
||||
if key.startswith("$") and key not in _SUPPORTED_OPERATORS:
|
||||
raise UnsupportedFilterError(f"operator {key!r} not supported by sqlite_exact")
|
||||
if isinstance(value, dict):
|
||||
stack.append(value)
|
||||
elif isinstance(value, list):
|
||||
stack.extend(item for item in value if isinstance(item, dict))
|
||||
|
||||
|
||||
def _coerce_comparable(value: Any):
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
return value
|
||||
|
||||
|
||||
def _compare(actual: Any, op: str, expected: Any) -> bool:
|
||||
actual = _coerce_comparable(actual)
|
||||
expected = _coerce_comparable(expected)
|
||||
if op == "$eq":
|
||||
return actual == expected
|
||||
if op == "$ne":
|
||||
return actual != expected
|
||||
if op == "$in":
|
||||
return actual in (expected or [])
|
||||
if op == "$nin":
|
||||
return actual not in (expected or [])
|
||||
if op == "$contains":
|
||||
return str(expected) in str(actual or "")
|
||||
try:
|
||||
if op == "$gt":
|
||||
return actual > expected
|
||||
if op == "$gte":
|
||||
return actual >= expected
|
||||
if op == "$lt":
|
||||
return actual < expected
|
||||
if op == "$lte":
|
||||
return actual <= expected
|
||||
except TypeError:
|
||||
return False
|
||||
raise UnsupportedFilterError(f"operator {op!r} not supported by sqlite_exact")
|
||||
|
||||
|
||||
def _matches_where(meta: dict, where: Optional[dict]) -> bool:
|
||||
if not where:
|
||||
return True
|
||||
if not isinstance(where, dict):
|
||||
return False
|
||||
for key, expected in where.items():
|
||||
if key == "$and":
|
||||
if not all(_matches_where(meta, clause) for clause in expected or []):
|
||||
return False
|
||||
continue
|
||||
if key == "$or":
|
||||
if not any(_matches_where(meta, clause) for clause in expected or []):
|
||||
return False
|
||||
continue
|
||||
if key.startswith("$"):
|
||||
raise UnsupportedFilterError(f"operator {key!r} not supported by sqlite_exact")
|
||||
actual = meta.get(key)
|
||||
if isinstance(expected, dict):
|
||||
for op, operand in expected.items():
|
||||
if not _compare(actual, op, operand):
|
||||
return False
|
||||
elif actual != expected:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _matches_where_document(document: str, where_document: Optional[dict]) -> bool:
|
||||
if not where_document:
|
||||
return True
|
||||
if not isinstance(where_document, dict):
|
||||
return False
|
||||
for key, value in where_document.items():
|
||||
if key == "$contains":
|
||||
if str(value) not in document:
|
||||
return False
|
||||
continue
|
||||
if key == "$and":
|
||||
if not all(_matches_where_document(document, clause) for clause in value or []):
|
||||
return False
|
||||
continue
|
||||
if key == "$or":
|
||||
if not any(_matches_where_document(document, clause) for clause in value or []):
|
||||
return False
|
||||
continue
|
||||
raise UnsupportedFilterError(f"where_document operator {key!r} not supported")
|
||||
return True
|
||||
|
||||
|
||||
def _validate_write_batch(
|
||||
*,
|
||||
documents: list[str],
|
||||
ids: list[str],
|
||||
metadatas: Optional[list[dict]],
|
||||
embeddings: Optional[list[list[float]]],
|
||||
) -> None:
|
||||
n = len(ids)
|
||||
if len(documents) != n:
|
||||
raise ValueError(f"documents length {len(documents)} does not match ids length {n}")
|
||||
if metadatas is not None and len(metadatas) != n:
|
||||
raise ValueError(f"metadatas length {len(metadatas)} does not match ids length {n}")
|
||||
if embeddings is not None and len(embeddings) != n:
|
||||
raise ValueError(f"embeddings length {len(embeddings)} does not match ids length {n}")
|
||||
|
||||
|
||||
class _SQLiteExactHandle:
|
||||
def __init__(self, conn: sqlite3.Connection, lock: threading.RLock):
|
||||
self.conn = conn
|
||||
self.lock = lock
|
||||
self.closed = False
|
||||
|
||||
|
||||
class SQLiteExactCollection(BaseCollection):
|
||||
def __init__(self, handle: _SQLiteExactHandle, collection_name: str):
|
||||
self._handle = handle
|
||||
self._collection_name = collection_name
|
||||
self._closed = False
|
||||
|
||||
def _ensure_open(self) -> None:
|
||||
if self._closed or self._handle.closed:
|
||||
raise BackendClosedError("SQLiteExactCollection has been closed")
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _cursor(self):
|
||||
with self._handle.lock:
|
||||
self._ensure_open()
|
||||
cur = self._handle.conn.cursor()
|
||||
try:
|
||||
yield cur
|
||||
except Exception:
|
||||
self._handle.conn.rollback()
|
||||
raise
|
||||
else:
|
||||
self._handle.conn.commit()
|
||||
finally:
|
||||
cur.close()
|
||||
|
||||
def _collection_id(self, cur) -> int:
|
||||
row = cur.execute(
|
||||
"SELECT id FROM collections WHERE name = ?",
|
||||
(self._collection_name,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise CollectionNotInitializedError(self._collection_name)
|
||||
return int(row[0])
|
||||
|
||||
def _collection_dimension(self, cur, collection_id: int) -> Optional[int]:
|
||||
row = cur.execute(
|
||||
"SELECT dimension FROM collections WHERE id = ?",
|
||||
(collection_id,),
|
||||
).fetchone()
|
||||
if row is None or row[0] is None:
|
||||
return None
|
||||
return int(row[0])
|
||||
|
||||
def _ensure_collection_dimension(self, cur, collection_id: int, dims: list[int]) -> None:
|
||||
distinct = {int(dim) for dim in dims}
|
||||
if not distinct:
|
||||
return
|
||||
if len(distinct) > 1:
|
||||
raise DimensionMismatchError(
|
||||
f"sqlite_exact collection {self._collection_name!r} cannot mix "
|
||||
f"embedding dimensions {sorted(distinct)}"
|
||||
)
|
||||
dim = distinct.pop()
|
||||
stored = self._collection_dimension(cur, collection_id)
|
||||
if stored is None:
|
||||
cur.execute(
|
||||
"UPDATE collections SET dimension = ? WHERE id = ?",
|
||||
(dim, collection_id),
|
||||
)
|
||||
elif stored != dim:
|
||||
raise DimensionMismatchError(
|
||||
f"sqlite_exact collection {self._collection_name!r} expects "
|
||||
f"embedding dimension {stored}, got {dim}"
|
||||
)
|
||||
|
||||
def _fts_available(self, cur) -> bool:
|
||||
row = cur.execute("SELECT value FROM meta WHERE key = 'fts5_available'").fetchone()
|
||||
return bool(row and row[0] == "1")
|
||||
|
||||
def _replace_fts(self, cur, collection_id: int, doc_id: str, document: str) -> None:
|
||||
if not self._fts_available(cur):
|
||||
return
|
||||
cur.execute(
|
||||
"DELETE FROM docs_fts WHERE collection_id = ? AND doc_id = ?",
|
||||
(collection_id, doc_id),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO docs_fts(collection_id, doc_id, document) VALUES (?, ?, ?)",
|
||||
(collection_id, doc_id, document),
|
||||
)
|
||||
|
||||
def add(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
_validate_write_batch(
|
||||
documents=documents,
|
||||
ids=ids,
|
||||
metadatas=metadatas,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
if embeddings is None:
|
||||
raise ValueError("sqlite_exact requires explicit embeddings")
|
||||
metadatas = metadatas or [{} for _ in ids]
|
||||
now = _utcnow()
|
||||
with self._cursor() as cur:
|
||||
collection_id = self._collection_id(cur)
|
||||
prepared = []
|
||||
for doc_id, doc, meta, emb in zip(ids, documents, metadatas, embeddings):
|
||||
arr = _as_vector_array(emb)
|
||||
prepared.append((doc_id, doc, meta, arr.tobytes(), int(arr.size)))
|
||||
self._ensure_collection_dimension(cur, collection_id, [item[4] for item in prepared])
|
||||
for doc_id, doc, meta, emb_blob, dim in prepared:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO documents
|
||||
(collection_id, id, document, metadata_json, embedding, dim, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
collection_id,
|
||||
doc_id,
|
||||
doc,
|
||||
_json_dumps(meta),
|
||||
emb_blob,
|
||||
dim,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
self._replace_fts(cur, collection_id, doc_id, doc)
|
||||
|
||||
def upsert(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
_validate_write_batch(
|
||||
documents=documents,
|
||||
ids=ids,
|
||||
metadatas=metadatas,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
if embeddings is None:
|
||||
raise ValueError("sqlite_exact requires explicit embeddings")
|
||||
metadatas = metadatas or [{} for _ in ids]
|
||||
now = _utcnow()
|
||||
with self._cursor() as cur:
|
||||
collection_id = self._collection_id(cur)
|
||||
prepared = []
|
||||
for doc_id, doc, meta, emb in zip(ids, documents, metadatas, embeddings):
|
||||
arr = _as_vector_array(emb)
|
||||
prepared.append((doc_id, doc, meta, arr.tobytes(), int(arr.size)))
|
||||
self._ensure_collection_dimension(cur, collection_id, [item[4] for item in prepared])
|
||||
for doc_id, doc, meta, emb_blob, dim in prepared:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO documents
|
||||
(collection_id, id, document, metadata_json, embedding, dim, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(collection_id, id) DO UPDATE SET
|
||||
document = excluded.document,
|
||||
metadata_json = excluded.metadata_json,
|
||||
embedding = excluded.embedding,
|
||||
dim = excluded.dim,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(
|
||||
collection_id,
|
||||
doc_id,
|
||||
doc,
|
||||
_json_dumps(meta),
|
||||
emb_blob,
|
||||
dim,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
self._replace_fts(cur, collection_id, doc_id, doc)
|
||||
|
||||
def update(self, *, ids, documents=None, metadatas=None, embeddings=None):
|
||||
if documents is None and metadatas is None and embeddings is None:
|
||||
raise ValueError("update requires at least one of documents, metadatas, embeddings")
|
||||
n = len(ids)
|
||||
for label, value in (
|
||||
("documents", documents),
|
||||
("metadatas", metadatas),
|
||||
("embeddings", embeddings),
|
||||
):
|
||||
if value is not None and len(value) != n:
|
||||
raise ValueError(f"{label} length {len(value)} does not match ids length {n}")
|
||||
with self._cursor() as cur:
|
||||
collection_id = self._collection_id(cur)
|
||||
updates = []
|
||||
for idx, doc_id in enumerate(ids):
|
||||
row = cur.execute(
|
||||
"""
|
||||
SELECT document, metadata_json, embedding, dim
|
||||
FROM documents
|
||||
WHERE collection_id = ? AND id = ?
|
||||
""",
|
||||
(collection_id, doc_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
doc = documents[idx] if documents is not None else row[0]
|
||||
meta = _json_loads(row[1])
|
||||
if metadatas is not None:
|
||||
meta.update(metadatas[idx] or {})
|
||||
if embeddings is not None:
|
||||
arr = _as_vector_array(embeddings[idx])
|
||||
emb_blob = arr.tobytes()
|
||||
dim = int(arr.size)
|
||||
else:
|
||||
emb_blob = row[2]
|
||||
dim = row[3]
|
||||
updates.append((doc_id, doc, meta, emb_blob, dim))
|
||||
if embeddings is not None:
|
||||
self._ensure_collection_dimension(cur, collection_id, [item[4] for item in updates])
|
||||
for doc_id, doc, meta, emb_blob, dim in updates:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE documents
|
||||
SET document = ?, metadata_json = ?, embedding = ?, dim = ?, updated_at = ?
|
||||
WHERE collection_id = ? AND id = ?
|
||||
""",
|
||||
(doc, _json_dumps(meta), emb_blob, dim, _utcnow(), collection_id, doc_id),
|
||||
)
|
||||
self._replace_fts(cur, collection_id, doc_id, doc)
|
||||
|
||||
def _rows(self, cur, *, where=None, where_document=None) -> list[dict]:
|
||||
_validate_where(where)
|
||||
_validate_where(where_document)
|
||||
collection_id = self._collection_id(cur)
|
||||
rows = cur.execute(
|
||||
"""
|
||||
SELECT id, document, metadata_json, embedding
|
||||
FROM documents
|
||||
WHERE collection_id = ?
|
||||
ORDER BY rowid
|
||||
""",
|
||||
(collection_id,),
|
||||
).fetchall()
|
||||
out = []
|
||||
for doc_id, doc, meta_json, emb_blob in rows:
|
||||
meta = _json_loads(meta_json)
|
||||
if not _matches_where(meta, where):
|
||||
continue
|
||||
if not _matches_where_document(doc or "", where_document):
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"id": doc_id,
|
||||
"document": doc or "",
|
||||
"metadata": meta,
|
||||
"embedding": emb_blob,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
def query(
|
||||
self,
|
||||
*,
|
||||
query_texts=None,
|
||||
query_embeddings=None,
|
||||
n_results=10,
|
||||
where=None,
|
||||
where_document=None,
|
||||
include=None,
|
||||
) -> QueryResult:
|
||||
if query_texts is not None:
|
||||
raise ValueError(
|
||||
"sqlite_exact requires query_embeddings; use palace.get_collection wrapper"
|
||||
)
|
||||
if query_embeddings is None:
|
||||
raise ValueError("query requires query_embeddings")
|
||||
if not query_embeddings:
|
||||
raise ValueError("query input must be a non-empty list")
|
||||
|
||||
spec = _IncludeSpec.resolve(include, default_distances=True)
|
||||
outer_ids: list[list[str]] = []
|
||||
outer_docs: list[list[str]] = []
|
||||
outer_metas: list[list[dict]] = []
|
||||
outer_dists: list[list[float]] = []
|
||||
outer_embeds: list[list[list[float]]] = []
|
||||
|
||||
with self._cursor() as cur:
|
||||
collection_id = self._collection_id(cur)
|
||||
expected_dim = self._collection_dimension(cur, collection_id)
|
||||
rows = self._rows(cur, where=where, where_document=where_document)
|
||||
row_vectors = [(row, _decode_array(row["embedding"])) for row in rows]
|
||||
|
||||
for query_vector in query_embeddings:
|
||||
q = _as_vector_array(query_vector)
|
||||
if expected_dim is not None and int(q.size) != expected_dim:
|
||||
raise DimensionMismatchError(
|
||||
f"sqlite_exact collection {self._collection_name!r} expects "
|
||||
f"embedding dimension {expected_dim}, got {int(q.size)}"
|
||||
)
|
||||
q_norm = float(np.linalg.norm(q))
|
||||
scored = []
|
||||
for row, vec in row_vectors:
|
||||
if vec is None or vec.size != q.size:
|
||||
continue
|
||||
denom = q_norm * float(np.linalg.norm(vec))
|
||||
cos = 0.0 if denom <= 0 else float(np.dot(q, vec) / denom)
|
||||
distance = 1.0 - max(-1.0, min(1.0, cos))
|
||||
scored.append((distance, row, vec))
|
||||
scored.sort(key=lambda item: item[0])
|
||||
top = scored[:n_results]
|
||||
|
||||
outer_ids.append([row["id"] for _, row, _ in top])
|
||||
outer_docs.append([row["document"] for _, row, _ in top] if spec.documents else [])
|
||||
outer_metas.append([row["metadata"] for _, row, _ in top] if spec.metadatas else [])
|
||||
outer_dists.append([float(dist) for dist, _, _ in top] if spec.distances else [])
|
||||
if spec.embeddings:
|
||||
outer_embeds.append([vec.astype(float).tolist() for _, _, vec in top])
|
||||
|
||||
return QueryResult(
|
||||
ids=outer_ids,
|
||||
documents=outer_docs,
|
||||
metadatas=outer_metas,
|
||||
distances=outer_dists,
|
||||
embeddings=outer_embeds if spec.embeddings else None,
|
||||
)
|
||||
|
||||
def get(
|
||||
self,
|
||||
*,
|
||||
ids=None,
|
||||
where=None,
|
||||
where_document=None,
|
||||
limit=None,
|
||||
offset=None,
|
||||
include=None,
|
||||
) -> GetResult:
|
||||
spec = _IncludeSpec.resolve(include, default_distances=False)
|
||||
with self._cursor() as cur:
|
||||
rows = self._rows(cur, where=where, where_document=where_document)
|
||||
if ids is not None:
|
||||
by_id = {row["id"]: row for row in rows}
|
||||
rows = [by_id[doc_id] for doc_id in ids if doc_id in by_id]
|
||||
if offset:
|
||||
rows = rows[offset:]
|
||||
if limit is not None:
|
||||
rows = rows[:limit]
|
||||
return GetResult(
|
||||
ids=[row["id"] for row in rows],
|
||||
documents=[row["document"] for row in rows] if spec.documents else [],
|
||||
metadatas=[row["metadata"] for row in rows] if spec.metadatas else [],
|
||||
embeddings=(
|
||||
[_decode_vector(row["embedding"]) for row in rows] if spec.embeddings else None
|
||||
),
|
||||
)
|
||||
|
||||
def delete(self, *, ids=None, where=None):
|
||||
with self._cursor() as cur:
|
||||
collection_id = self._collection_id(cur)
|
||||
if ids is None:
|
||||
rows = self._rows(cur, where=where)
|
||||
ids = [row["id"] for row in rows]
|
||||
for doc_id in ids or []:
|
||||
cur.execute(
|
||||
"DELETE FROM documents WHERE collection_id = ? AND id = ?",
|
||||
(collection_id, doc_id),
|
||||
)
|
||||
if self._fts_available(cur):
|
||||
cur.execute(
|
||||
"DELETE FROM docs_fts WHERE collection_id = ? AND doc_id = ?",
|
||||
(collection_id, doc_id),
|
||||
)
|
||||
|
||||
def count(self) -> int:
|
||||
with self._cursor() as cur:
|
||||
collection_id = self._collection_id(cur)
|
||||
row = cur.execute(
|
||||
"SELECT COUNT(*) FROM documents WHERE collection_id = ?",
|
||||
(collection_id,),
|
||||
).fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None):
|
||||
_validate_where(where)
|
||||
with self._cursor() as cur:
|
||||
hits = self._lexical_search_fts(cur, query=query, n_results=n_results, where=where)
|
||||
if hits is not None:
|
||||
return LexicalResult(hits=hits)
|
||||
rows = self._rows(cur, where=where)
|
||||
scores = _bm25_scores(query, [row["document"] for row in rows])
|
||||
scored = [
|
||||
LexicalHit(
|
||||
id=row["id"],
|
||||
document=row["document"],
|
||||
metadata=row["metadata"],
|
||||
score=score,
|
||||
)
|
||||
for row, score in zip(rows, scores)
|
||||
if score > 0
|
||||
]
|
||||
scored.sort(key=lambda hit: hit.score, reverse=True)
|
||||
return LexicalResult(hits=scored[:n_results])
|
||||
|
||||
def _lexical_search_fts(self, cur, *, query: str, n_results: int, where: Optional[dict]):
|
||||
if not self._fts_available(cur):
|
||||
return None
|
||||
tokens = [t for t in _tokenize(query) if len(t) >= 2]
|
||||
if not tokens:
|
||||
return None
|
||||
fts_query = " OR ".join(tokens)
|
||||
collection_id = self._collection_id(cur)
|
||||
try:
|
||||
limit_sql = "" if where else "LIMIT ?"
|
||||
params = (fts_query, collection_id)
|
||||
if not where:
|
||||
params = (*params, max(n_results * 5, n_results))
|
||||
rows = cur.execute(
|
||||
f"""
|
||||
SELECT doc_id, bm25(docs_fts) AS rank
|
||||
FROM docs_fts
|
||||
WHERE docs_fts MATCH ? AND collection_id = ?
|
||||
ORDER BY rank
|
||||
{limit_sql}
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
logger.debug("sqlite_exact FTS query failed; using Python lexical scan", exc_info=True)
|
||||
return None
|
||||
if not rows:
|
||||
return []
|
||||
ids = [row[0] for row in rows]
|
||||
docs = []
|
||||
for start in range(0, len(ids), 900):
|
||||
chunk_ids = ids[start : start + 900]
|
||||
placeholders = ",".join("?" for _ in chunk_ids)
|
||||
docs.extend(
|
||||
cur.execute(
|
||||
f"""
|
||||
SELECT id, document, metadata_json
|
||||
FROM documents
|
||||
WHERE collection_id = ? AND id IN ({placeholders})
|
||||
""",
|
||||
(collection_id, *chunk_ids),
|
||||
).fetchall()
|
||||
)
|
||||
by_id = {doc_id: (doc or "", _json_loads(meta_json)) for doc_id, doc, meta_json in docs}
|
||||
hits = []
|
||||
for doc_id, rank in rows:
|
||||
doc_meta = by_id.get(doc_id)
|
||||
if doc_meta is None:
|
||||
continue
|
||||
doc, meta = doc_meta
|
||||
if not _matches_where(meta, where):
|
||||
continue
|
||||
hits.append(
|
||||
LexicalHit(
|
||||
id=doc_id,
|
||||
document=doc,
|
||||
metadata=meta,
|
||||
score=-float(rank),
|
||||
)
|
||||
)
|
||||
if len(hits) >= n_results:
|
||||
break
|
||||
return hits
|
||||
|
||||
def close(self) -> None:
|
||||
self._closed = True
|
||||
|
||||
def health(self) -> HealthStatus:
|
||||
if self._closed or self._handle.closed:
|
||||
return HealthStatus.unhealthy("collection closed")
|
||||
return HealthStatus.healthy()
|
||||
|
||||
|
||||
class SQLiteExactBackend(BaseBackend):
|
||||
name = "sqlite_exact"
|
||||
capabilities = frozenset(
|
||||
{
|
||||
"requires_explicit_embeddings",
|
||||
"supports_embeddings_in",
|
||||
"supports_embeddings_passthrough",
|
||||
"supports_embeddings_out",
|
||||
"supports_metadata_filters",
|
||||
"supports_lexical_search",
|
||||
"local_mode",
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
self._clients: dict[str, _SQLiteExactHandle] = {}
|
||||
self._clients_lock = threading.RLock()
|
||||
self._closed = False
|
||||
|
||||
@staticmethod
|
||||
def _db_path(palace_path: str) -> str:
|
||||
return os.path.join(palace_path, _DB_FILENAME)
|
||||
|
||||
def _connect(self, palace_path: str, create: bool):
|
||||
if self._closed:
|
||||
raise BackendClosedError("SQLiteExactBackend has been closed")
|
||||
db_path = self._db_path(palace_path)
|
||||
if not create and not os.path.isfile(db_path):
|
||||
raise PalaceNotFoundError(db_path)
|
||||
if create:
|
||||
os.makedirs(palace_path, exist_ok=True)
|
||||
try:
|
||||
os.chmod(palace_path, 0o700)
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
with self._clients_lock:
|
||||
cached = self._clients.get(palace_path)
|
||||
if cached is not None and not cached.closed:
|
||||
return cached
|
||||
conn = sqlite3.connect(db_path, check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
lock = threading.RLock()
|
||||
handle = _SQLiteExactHandle(conn, lock)
|
||||
with handle.lock:
|
||||
self._init_schema(conn)
|
||||
with self._clients_lock:
|
||||
self._clients[palace_path] = handle
|
||||
return handle
|
||||
|
||||
def _init_schema(self, conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(
|
||||
"""
|
||||
PRAGMA journal_mode=WAL;
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS collections (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
dimension INTEGER,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS documents (
|
||||
collection_id INTEGER NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
document TEXT NOT NULL,
|
||||
metadata_json TEXT NOT NULL,
|
||||
embedding BLOB NOT NULL,
|
||||
dim INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (collection_id, id),
|
||||
FOREIGN KEY(collection_id) REFERENCES collections(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_documents_collection
|
||||
ON documents(collection_id);
|
||||
"""
|
||||
)
|
||||
columns = {row[1] for row in conn.execute("PRAGMA table_info(collections)").fetchall()}
|
||||
if "dimension" not in columns:
|
||||
conn.execute("ALTER TABLE collections ADD COLUMN dimension INTEGER")
|
||||
try:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts
|
||||
USING fts5(collection_id UNINDEXED, doc_id UNINDEXED, document)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO meta(key, value)
|
||||
VALUES ('fts5_available', '1')
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||
"""
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO meta(key, value)
|
||||
VALUES ('fts5_available', '0')
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def get_collection(
|
||||
self,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> SQLiteExactCollection:
|
||||
palace, collection_name, create = self._normalize_args(args, kwargs)
|
||||
palace_path = palace.local_path
|
||||
if palace_path is None:
|
||||
raise PalaceNotFoundError("SQLiteExactBackend requires PalaceRef.local_path")
|
||||
if not create and not os.path.isdir(palace_path):
|
||||
raise PalaceNotFoundError(palace_path)
|
||||
handle = self._connect(palace_path, create=create)
|
||||
with handle.lock:
|
||||
row = handle.conn.execute(
|
||||
"SELECT id FROM collections WHERE name = ?",
|
||||
(collection_name,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
if not create:
|
||||
raise CollectionNotInitializedError(collection_name)
|
||||
handle.conn.execute(
|
||||
"INSERT INTO collections(name, created_at) VALUES (?, ?)",
|
||||
(collection_name, _utcnow()),
|
||||
)
|
||||
handle.conn.commit()
|
||||
return SQLiteExactCollection(handle, collection_name)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_args(args, kwargs):
|
||||
if "palace" in kwargs:
|
||||
palace = kwargs.pop("palace")
|
||||
if not isinstance(palace, PalaceRef):
|
||||
raise TypeError("palace= must be a PalaceRef instance")
|
||||
collection_name = kwargs.pop("collection_name")
|
||||
create = bool(kwargs.pop("create", False))
|
||||
kwargs.pop("options", None)
|
||||
if args or kwargs:
|
||||
raise TypeError("unexpected arguments to get_collection")
|
||||
return palace, collection_name, create
|
||||
if args:
|
||||
palace_path = args[0]
|
||||
rest = list(args[1:])
|
||||
collection_name = kwargs.pop("collection_name", None) or (rest.pop(0) if rest else None)
|
||||
if collection_name is None:
|
||||
raise TypeError("collection_name is required")
|
||||
create = kwargs.pop("create", False)
|
||||
if rest:
|
||||
create = rest.pop(0)
|
||||
if rest or kwargs:
|
||||
raise TypeError("unexpected arguments to get_collection")
|
||||
return PalaceRef(id=palace_path, local_path=palace_path), collection_name, bool(create)
|
||||
if "palace_path" in kwargs:
|
||||
palace_path = kwargs.pop("palace_path")
|
||||
collection_name = kwargs.pop("collection_name")
|
||||
create = bool(kwargs.pop("create", False))
|
||||
if kwargs:
|
||||
raise TypeError("unexpected arguments to get_collection")
|
||||
return PalaceRef(id=palace_path, local_path=palace_path), collection_name, create
|
||||
raise TypeError("get_collection requires palace= or a positional palace_path")
|
||||
|
||||
def close_palace(self, palace: PalaceRef | str) -> None:
|
||||
path = palace.local_path if isinstance(palace, PalaceRef) else palace
|
||||
if path is None:
|
||||
return
|
||||
with self._clients_lock:
|
||||
cached = self._clients.pop(path, None)
|
||||
if cached is not None:
|
||||
with cached.lock:
|
||||
cached.closed = True
|
||||
cached.conn.close()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._clients_lock:
|
||||
handles = list(self._clients.values())
|
||||
self._clients.clear()
|
||||
for handle in handles:
|
||||
with handle.lock:
|
||||
handle.closed = True
|
||||
handle.conn.close()
|
||||
self._closed = True
|
||||
|
||||
def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus:
|
||||
if self._closed:
|
||||
return HealthStatus.unhealthy("backend closed")
|
||||
if palace and palace.local_path and not os.path.isfile(self._db_path(palace.local_path)):
|
||||
return HealthStatus.unhealthy("sqlite_exact database not found")
|
||||
return HealthStatus.healthy()
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path: str) -> bool:
|
||||
return os.path.isfile(os.path.join(path, _DB_FILENAME))
|
||||
|
||||
def create_collection(self, palace_path: str, collection_name: str) -> SQLiteExactCollection:
|
||||
return self.get_collection(palace_path, collection_name, create=True)
|
||||
|
||||
def get_or_create_collection(self, palace_path: str, collection_name: str):
|
||||
return self.get_collection(palace_path, collection_name, create=True)
|
||||
|
||||
def delete_collection(self, palace_path: str, collection_name: str) -> None:
|
||||
handle = self._connect(palace_path, create=False)
|
||||
with handle.lock:
|
||||
row = handle.conn.execute(
|
||||
"SELECT id FROM collections WHERE name = ?",
|
||||
(collection_name,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise CollectionNotInitializedError(collection_name)
|
||||
collection_id = int(row[0])
|
||||
handle.conn.execute("DELETE FROM documents WHERE collection_id = ?", (collection_id,))
|
||||
try:
|
||||
handle.conn.execute(
|
||||
"DELETE FROM docs_fts WHERE collection_id = ?",
|
||||
(collection_id,),
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
handle.conn.execute("DELETE FROM collections WHERE id = ?", (collection_id,))
|
||||
handle.conn.commit()
|
||||
|
||||
|
||||
__all__ = ["SQLiteExactBackend", "SQLiteExactCollection"]
|
||||
+133
-15
@@ -51,6 +51,45 @@ _PASS_ZERO_PER_FILE_CAP = 100_000 # 100KB per file is generous for prose
|
||||
_PASS_ZERO_TOTAL_CAP = 5_000_000 # 5MB total ceiling — bounds memory
|
||||
_PASS_ZERO_LLM_PER_SAMPLE = 2_000 # for Tier 2 LLM call only
|
||||
_PASS_ZERO_LLM_MAX_SAMPLES = 20 # caps the LLM-tier sample count
|
||||
_EXPLICIT_BACKEND_ENV = "MEMPALACE_BACKEND_EXPLICIT"
|
||||
|
||||
|
||||
def _backend_arg(args):
|
||||
"""Return a CLI-selected backend from subcommand or global flags."""
|
||||
return getattr(args, "backend", None) or getattr(args, "global_backend", None)
|
||||
|
||||
|
||||
def _apply_backend_arg(args) -> None:
|
||||
backend = _backend_arg(args)
|
||||
if not backend:
|
||||
return
|
||||
backend = str(backend).strip().lower()
|
||||
from .backends import get_backend_class
|
||||
|
||||
get_backend_class(backend)
|
||||
os.environ[_EXPLICIT_BACKEND_ENV] = backend
|
||||
os.environ["MEMPALACE_BACKEND"] = backend
|
||||
|
||||
|
||||
def _selected_backend_for_palace(palace_path: str) -> str:
|
||||
from .palace import resolve_backend_name
|
||||
|
||||
return resolve_backend_name(palace_path, explicit=os.environ.get(_EXPLICIT_BACKEND_ENV))
|
||||
|
||||
|
||||
def _maintenance_requires_chroma(palace_path: str, command_name: str) -> bool:
|
||||
try:
|
||||
backend_name = _selected_backend_for_palace(palace_path)
|
||||
except Exception as exc: # noqa: BLE001 - user-facing guard before maintenance imports
|
||||
print(f"\n {command_name} cannot resolve the palace backend: {exc}", file=sys.stderr)
|
||||
return False
|
||||
if backend_name == "chroma":
|
||||
return True
|
||||
print(
|
||||
f"\n {command_name} is Chroma-only in this release (selected backend: {backend_name}).",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _gather_origin_samples(project_dir) -> list:
|
||||
@@ -380,6 +419,9 @@ def cmd_init(args):
|
||||
# Pass 2: detect rooms from folder structure
|
||||
detect_rooms_local(project_dir=args.dir, yes=getattr(args, "yes", False))
|
||||
cfg.init()
|
||||
backend = _backend_arg(args)
|
||||
if backend:
|
||||
cfg.set_backend(backend)
|
||||
|
||||
# Pass 3: protect git repos from accidentally committing per-project files
|
||||
_ensure_mempalace_files_gitignored(args.dir)
|
||||
@@ -615,6 +657,8 @@ def cmd_sync(args):
|
||||
"""Prune drawers whose source files are gitignored, deleted, or moved (#1252)."""
|
||||
from .mcp_server import _wal_log
|
||||
from .palace import MineAlreadyRunning
|
||||
from .backends import detect_backend_for_path
|
||||
from .palace import _backend_artifact_label, resolve_backend_name
|
||||
from .sync import sync_palace
|
||||
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
@@ -622,8 +666,16 @@ def cmd_sync(args):
|
||||
if not os.path.isdir(palace_path):
|
||||
print(f"\n No palace found at {palace_path}")
|
||||
return
|
||||
if not os.path.isfile(os.path.join(palace_path, "chroma.sqlite3")):
|
||||
print(f"\n Palace dir at {palace_path} exists but has no chroma.sqlite3 yet.")
|
||||
try:
|
||||
backend_name = resolve_backend_name(palace_path)
|
||||
except Exception as exc: # noqa: BLE001 - user-facing CLI guard
|
||||
print(f"\n Could not resolve palace backend: {exc}", file=sys.stderr)
|
||||
return
|
||||
if detect_backend_for_path(palace_path) is None:
|
||||
print(
|
||||
f"\n Palace dir at {palace_path} exists but has no "
|
||||
f"{_backend_artifact_label(backend_name)} yet."
|
||||
)
|
||||
print(" Run: mempalace mine <dir>")
|
||||
return
|
||||
|
||||
@@ -748,9 +800,11 @@ def cmd_split(args):
|
||||
|
||||
def cmd_migrate(args):
|
||||
"""Migrate palace from a different ChromaDB version."""
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
if not _maintenance_requires_chroma(palace_path, "migrate"):
|
||||
raise SystemExit(2)
|
||||
from .migrate import migrate
|
||||
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
migrate(
|
||||
palace_path=palace_path,
|
||||
dry_run=args.dry_run,
|
||||
@@ -758,6 +812,18 @@ def cmd_migrate(args):
|
||||
)
|
||||
|
||||
|
||||
def cmd_migrate_wings(args):
|
||||
"""Normalize legacy wing names (strip leading/trailing separators)."""
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
from .migrate import migrate_wing_names
|
||||
|
||||
migrate_wing_names(
|
||||
palace_path=palace_path,
|
||||
dry_run=args.dry_run,
|
||||
confirm=getattr(args, "yes", False),
|
||||
)
|
||||
|
||||
|
||||
def cmd_status(args):
|
||||
from .miner import status
|
||||
|
||||
@@ -767,14 +833,24 @@ def cmd_status(args):
|
||||
|
||||
def cmd_repair_status(args):
|
||||
"""Read-only HNSW capacity health check (#1222)."""
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
if not _maintenance_requires_chroma(palace_path, "repair-status"):
|
||||
raise SystemExit(2)
|
||||
from .repair import status as repair_status
|
||||
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
repair_status(palace_path=palace_path)
|
||||
|
||||
|
||||
def cmd_repair(args):
|
||||
"""Rebuild palace vector index from SQLite metadata."""
|
||||
config = MempalaceConfig()
|
||||
collection_name = config.collection_name
|
||||
palace_path = os.path.abspath(
|
||||
os.path.expanduser(args.palace) if args.palace else config.palace_path
|
||||
)
|
||||
if not _maintenance_requires_chroma(palace_path, "repair"):
|
||||
raise SystemExit(2)
|
||||
|
||||
import shutil
|
||||
from .backends.chroma import ChromaBackend
|
||||
from .migrate import confirm_destructive_action, contains_palace_database
|
||||
@@ -790,12 +866,6 @@ def cmd_repair(args):
|
||||
sqlite_integrity_errors,
|
||||
)
|
||||
|
||||
config = MempalaceConfig()
|
||||
collection_name = config.collection_name
|
||||
palace_path = os.path.abspath(
|
||||
os.path.expanduser(args.palace) if args.palace else config.palace_path
|
||||
)
|
||||
|
||||
if getattr(args, "mode", "legacy") == "max-seq-id":
|
||||
from .repair import repair_max_seq_id
|
||||
|
||||
@@ -1004,12 +1074,15 @@ def cmd_instructions(args):
|
||||
def cmd_mcp(args):
|
||||
"""Show how to wire MemPalace into MCP-capable hosts."""
|
||||
base_server_cmd = "mempalace-mcp"
|
||||
cmd_parts = [base_server_cmd]
|
||||
|
||||
if args.palace:
|
||||
resolved_palace = str(Path(args.palace).expanduser())
|
||||
server_cmd = f"{base_server_cmd} --palace {shlex.quote(resolved_palace)}"
|
||||
else:
|
||||
server_cmd = base_server_cmd
|
||||
cmd_parts.extend(["--palace", shlex.quote(resolved_palace)])
|
||||
backend = _backend_arg(args)
|
||||
if backend:
|
||||
cmd_parts.extend(["--backend", shlex.quote(str(backend).strip().lower())])
|
||||
server_cmd = " ".join(cmd_parts)
|
||||
|
||||
print("MemPalace MCP quick setup:")
|
||||
print(f" claude mcp add mempalace -- {server_cmd}")
|
||||
@@ -1204,12 +1277,23 @@ def main():
|
||||
default=None,
|
||||
help="Where the palace lives (default: from ~/.mempalace/config.json or ~/.mempalace/palace)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
dest="global_backend",
|
||||
default=None,
|
||||
help="Storage backend to use for this command (default: config/env/detected/chroma)",
|
||||
)
|
||||
|
||||
sub = parser.add_subparsers(dest="command")
|
||||
|
||||
# init
|
||||
p_init = sub.add_parser("init", help="Detect rooms from your folder structure")
|
||||
p_init.add_argument("dir", help="Project directory to set up")
|
||||
p_init.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to persist for this palace (default: chroma)",
|
||||
)
|
||||
p_init.add_argument(
|
||||
"--yes",
|
||||
action="store_true",
|
||||
@@ -1292,6 +1376,11 @@ def main():
|
||||
# mine
|
||||
p_mine = sub.add_parser("mine", help="Mine files into the palace")
|
||||
p_mine.add_argument("dir", help="Directory to mine")
|
||||
p_mine.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to use for this mine (default: config/env/detected/chroma)",
|
||||
)
|
||||
p_mine.add_argument(
|
||||
"--mode",
|
||||
choices=["projects", "convos", "extract"],
|
||||
@@ -1401,6 +1490,11 @@ def main():
|
||||
# search
|
||||
p_search = sub.add_parser("search", help="Find anything, exact words")
|
||||
p_search.add_argument("query", help="What to search for")
|
||||
p_search.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to use for this search (default: config/env/detected/chroma)",
|
||||
)
|
||||
p_search.add_argument("--wing", default=None, help="Limit to one project")
|
||||
p_search.add_argument("--room", default=None, help="Limit to one room")
|
||||
p_search.add_argument("--results", type=int, default=5, help="Number of results")
|
||||
@@ -1555,10 +1649,15 @@ def main():
|
||||
)
|
||||
|
||||
# mcp
|
||||
sub.add_parser(
|
||||
p_mcp = sub.add_parser(
|
||||
"mcp",
|
||||
help="Show MCP setup command for connecting MemPalace to your AI client",
|
||||
)
|
||||
p_mcp.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to include in the MCP startup command",
|
||||
)
|
||||
|
||||
# status
|
||||
# migrate
|
||||
@@ -1575,9 +1674,27 @@ def main():
|
||||
"--yes", action="store_true", help="Skip confirmation for destructive changes"
|
||||
)
|
||||
|
||||
sub.add_parser("status", help="Show what's been filed")
|
||||
# migrate-wings
|
||||
p_migrate_wings = sub.add_parser(
|
||||
"migrate-wings",
|
||||
help="Normalize legacy wing names (strip leading/trailing separators) so pre-#1675 palaces stay discoverable",
|
||||
)
|
||||
p_migrate_wings.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would change without modifying the palace",
|
||||
)
|
||||
p_migrate_wings.add_argument("--yes", action="store_true", help="Skip the confirmation prompt")
|
||||
|
||||
p_status = sub.add_parser("status", help="Show what's been filed")
|
||||
p_status.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to use for status (default: config/env/detected/chroma)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
_apply_backend_arg(args)
|
||||
|
||||
if not args.command:
|
||||
parser.print_help()
|
||||
@@ -1613,6 +1730,7 @@ def main():
|
||||
"repair": cmd_repair,
|
||||
"repair-status": cmd_repair_status,
|
||||
"migrate": cmd_migrate,
|
||||
"migrate-wings": cmd_migrate_wings,
|
||||
"status": cmd_status,
|
||||
}
|
||||
dispatch[args.command](args)
|
||||
|
||||
@@ -264,7 +264,7 @@ def regenerate_closets(
|
||||
f"Regenerating closets for {len(sources)} source files via {cfg.endpoint} ({cfg.model})..."
|
||||
)
|
||||
if dry_run:
|
||||
print("DRY RUN — no changes will be written")
|
||||
print("DRY RUN - no changes will be written")
|
||||
|
||||
processed = 0
|
||||
failed = 0
|
||||
@@ -286,7 +286,7 @@ def regenerate_closets(
|
||||
parsed, usage = _call_llm(cfg, source, w, r, content)
|
||||
if not parsed:
|
||||
failed += 1
|
||||
print(f" [{i}/{len(sources)}] ✗ {os.path.basename(source)} — LLM failed")
|
||||
print(f" [{i}/{len(sources)}] [FAIL] {os.path.basename(source)} - LLM failed")
|
||||
continue
|
||||
|
||||
if usage:
|
||||
@@ -323,7 +323,7 @@ def regenerate_closets(
|
||||
|
||||
processed += 1
|
||||
n_topics = len(parsed.get("topics", []))
|
||||
print(f" [{i}/{len(sources)}] ✓ {os.path.basename(source)} — {n_topics} topics")
|
||||
print(f" [{i}/{len(sources)}] [OK] {os.path.basename(source)} - {n_topics} topics")
|
||||
|
||||
print(f"\nDone. {processed} regenerated, {failed} failed.")
|
||||
if total_input or total_output:
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Pre-mining defense against drawer_id collisions.
|
||||
|
||||
Runs immediately before a batched chromadb upsert. Computes the union of
|
||||
incoming drawer_ids and existing drawer_ids that share a key with the
|
||||
batch; raises ``CollisionError`` if any drawer_id appears more than once
|
||||
in that union with conflicting ``(source_file, chunk_index)`` metadata.
|
||||
|
||||
Under the v2 hash recipe (see :mod:`mempalace.ids`) accidental collisions
|
||||
are vanishingly rare — SHA-256 truncated to 24 hex chars makes random
|
||||
collision ~2^-96. The scan exists for two reasons:
|
||||
|
||||
1. Catch upstream bugs that emit duplicate ``(source_file, chunk_index)``
|
||||
pairs in the same batch with conflicting content. ChromaDB would
|
||||
silently let the last-write win; the scan surfaces it as an
|
||||
actionable error naming both call sites.
|
||||
2. Catch the astronomical-but-possible SHA-256 hash collision with a
|
||||
clear message instead of a silent overwrite at upsert time.
|
||||
|
||||
The scan does NOT fire on idempotent re-mines — when an incoming drawer
|
||||
matches an existing one with the SAME ``(source_file, chunk_index)``
|
||||
metadata, that is normal re-write behavior, not collision.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
class CollisionError(Exception):
|
||||
"""Raised by :func:`assert_no_collisions` when the pre-mining scan
|
||||
detects a drawer_id that would silently overwrite existing content
|
||||
or duplicate within a batch with conflicting metadata.
|
||||
|
||||
The exception message names every colliding ``drawer_id`` and the
|
||||
full set of ``(source_file, chunk_index)`` pairs producing each one,
|
||||
so a user fixing one collision does not have to rediscover the next
|
||||
by re-running the mine.
|
||||
"""
|
||||
|
||||
|
||||
def _metadata_key(meta: dict) -> tuple:
|
||||
"""Reduce a drawer metadata dict to the tuple used for collision
|
||||
discrimination. Two metadata dicts are 'the same chunk' iff their
|
||||
key tuples match. Falls back to ``(source_file,)`` when
|
||||
``chunk_index`` is absent (diary entries, sentinels)."""
|
||||
source_file = meta.get("source_file")
|
||||
chunk_index = meta.get("chunk_index")
|
||||
if chunk_index is None:
|
||||
return (source_file,)
|
||||
return (source_file, chunk_index)
|
||||
|
||||
|
||||
def assert_no_collisions(
|
||||
proposed: list[tuple[str, dict]],
|
||||
collection,
|
||||
) -> None:
|
||||
"""Abort the mine via ``CollisionError`` if any proposed drawer_id
|
||||
collides with itself or with an existing drawer in ``collection``.
|
||||
|
||||
Args:
|
||||
proposed: list of ``(drawer_id, metadata)`` tuples for the
|
||||
chunks about to be upserted. ``metadata`` must carry at
|
||||
least ``source_file``; ``chunk_index`` is used when
|
||||
present.
|
||||
collection: a ChromaDB-shaped collection with ``get(ids=...)``
|
||||
returning a dict with ``ids`` and ``metadatas`` keys.
|
||||
|
||||
Raises:
|
||||
CollisionError: when a drawer_id maps to two or more distinct
|
||||
``(source_file, chunk_index)`` tuples in the union of
|
||||
incoming and existing rows.
|
||||
"""
|
||||
if not proposed:
|
||||
return
|
||||
|
||||
# Build incoming map: drawer_id -> set of metadata key tuples.
|
||||
# Using a set collapses duplicate-metadata cases (same chunk twice
|
||||
# in the batch) without flagging them as collisions.
|
||||
incoming: dict[str, set[tuple]] = defaultdict(set)
|
||||
for drawer_id, meta in proposed:
|
||||
incoming[drawer_id].add(_metadata_key(meta))
|
||||
|
||||
# Query existing rows for any incoming id. ChromaDB's get(ids=...)
|
||||
# returns only the rows whose ids are present; missing ids are
|
||||
# silently absent from the result, which is what we want.
|
||||
incoming_ids = list(incoming.keys())
|
||||
result = collection.get(ids=incoming_ids, include=["metadatas"])
|
||||
existing_ids: list = result["ids"] if hasattr(result, "__getitem__") else []
|
||||
existing_metas: list = result["metadatas"] if existing_ids else []
|
||||
|
||||
# Merge existing metadata into the incoming map. A real collision is
|
||||
# a drawer_id whose incoming + existing metadata key tuples are not
|
||||
# all the same.
|
||||
for drawer_id, meta in zip(existing_ids, existing_metas):
|
||||
incoming[drawer_id].add(_metadata_key(meta or {}))
|
||||
|
||||
collisions = {did: keys for did, keys in incoming.items() if len(keys) > 1}
|
||||
if collisions:
|
||||
raise CollisionError(_format_collisions(collisions))
|
||||
|
||||
|
||||
def _format_collisions(collisions: dict[str, set[tuple]]) -> str:
|
||||
"""Render a CollisionError message that enumerates every colliding
|
||||
drawer_id and the metadata tuples producing it."""
|
||||
lines = [
|
||||
f"Pre-mining collision scan detected {len(collisions)} "
|
||||
f"colliding drawer_id{'s' if len(collisions) != 1 else ''}:",
|
||||
]
|
||||
for drawer_id, keys in sorted(collisions.items()):
|
||||
lines.append(f" {drawer_id}:")
|
||||
for key in sorted(keys, key=lambda k: tuple(str(part) for part in k)):
|
||||
if len(key) == 1:
|
||||
lines.append(f" source_file={key[0]!r}")
|
||||
else:
|
||||
lines.append(f" source_file={key[0]!r}, chunk_index={key[1]!r}")
|
||||
lines.append(
|
||||
"Each colliding drawer_id would cause the second ChromaDB upsert "
|
||||
"to silently overwrite the first. Fix the upstream chunker / "
|
||||
"miner to emit distinct keys, or investigate the SHA-256 hash "
|
||||
"collision."
|
||||
)
|
||||
return "\n".join(lines)
|
||||
+106
-1
@@ -37,8 +37,13 @@ def normalize_wing_name(name: str) -> str:
|
||||
The same rule is applied by ``init`` when persisting `topics_by_wing`
|
||||
and when writing `mempalace.yaml`, so the miner's lookup matches at
|
||||
mine time regardless of the source dirname.
|
||||
|
||||
Leading/trailing separators are stripped so a path-encoded dirname like
|
||||
``-home-user-proj`` yields ``home_user_proj`` rather than a leading-
|
||||
underscore slug that ``sanitize_name`` (and thus the MCP write tools)
|
||||
would reject.
|
||||
"""
|
||||
return name.lower().replace(" ", "_").replace("-", "_")
|
||||
return name.lower().replace(" ", "_").replace("-", "_").strip("_")
|
||||
|
||||
|
||||
def sanitize_name(value: str, field_name: str = "name") -> str:
|
||||
@@ -191,6 +196,7 @@ def sanitize_content(value: str, max_length: int = 100_000) -> str:
|
||||
|
||||
DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
|
||||
DEFAULT_COLLECTION_NAME = "mempalace_drawers"
|
||||
DEFAULT_BACKEND = "chroma"
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
@@ -325,6 +331,87 @@ class MempalaceConfig:
|
||||
"""ChromaDB collection name."""
|
||||
return self._file_config.get("collection_name", DEFAULT_COLLECTION_NAME)
|
||||
|
||||
@property
|
||||
def backend(self):
|
||||
"""Storage backend name.
|
||||
|
||||
Read from ``config.json`` first, then ``MEMPALACE_BACKEND``, then
|
||||
``"chroma"`` for backwards compatibility with existing palaces.
|
||||
"""
|
||||
cfg_val = self._file_config.get("backend")
|
||||
if cfg_val:
|
||||
return str(cfg_val).strip().lower()
|
||||
env_val = os.environ.get("MEMPALACE_BACKEND")
|
||||
if env_val:
|
||||
return env_val.strip().lower()
|
||||
return DEFAULT_BACKEND
|
||||
|
||||
@property
|
||||
def qdrant_url(self):
|
||||
"""Qdrant endpoint for the opt-in ``qdrant`` backend.
|
||||
|
||||
Defaults to localhost so selecting Qdrant never silently sends memory
|
||||
to a remote service. Users can point at a LAN or cloud endpoint via
|
||||
config or ``MEMPALACE_QDRANT_URL`` when they deliberately choose that.
|
||||
"""
|
||||
env_val = os.environ.get("MEMPALACE_QDRANT_URL")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
return str(self._file_config.get("qdrant_url", "http://localhost:6333")).strip()
|
||||
|
||||
@property
|
||||
def qdrant_api_key(self):
|
||||
"""API key for the opt-in ``qdrant`` backend, if configured."""
|
||||
env_val = os.environ.get("MEMPALACE_QDRANT_API_KEY")
|
||||
if env_val:
|
||||
return env_val
|
||||
value = self._file_config.get("qdrant_api_key")
|
||||
return str(value) if value else None
|
||||
|
||||
@property
|
||||
def qdrant_namespace(self):
|
||||
"""Optional Qdrant collection namespace/prefix."""
|
||||
env_val = os.environ.get("MEMPALACE_QDRANT_NAMESPACE")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
value = self._file_config.get("qdrant_namespace")
|
||||
return str(value).strip() if value else None
|
||||
|
||||
@property
|
||||
def qdrant_timeout(self):
|
||||
"""Qdrant HTTP timeout in seconds."""
|
||||
env_val = os.environ.get("MEMPALACE_QDRANT_TIMEOUT")
|
||||
raw = env_val if env_val is not None else self._file_config.get("qdrant_timeout", 10.0)
|
||||
try:
|
||||
timeout = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
timeout = 10.0
|
||||
return timeout if timeout > 0 else 10.0
|
||||
|
||||
@property
|
||||
def pgvector_dsn(self):
|
||||
"""Postgres DSN for the opt-in ``pgvector`` backend.
|
||||
|
||||
Defaults to a localhost DSN so selecting pgvector never silently sends
|
||||
memory to a remote database. Point at a LAN or cloud Postgres via config
|
||||
or ``MEMPALACE_PGVECTOR_DSN`` only when deliberately chosen.
|
||||
"""
|
||||
env_val = os.environ.get("MEMPALACE_PGVECTOR_DSN")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
return str(
|
||||
self._file_config.get("pgvector_dsn", "postgresql://localhost:5432/mempalace")
|
||||
).strip()
|
||||
|
||||
@property
|
||||
def pgvector_namespace(self):
|
||||
"""Optional pgvector table namespace/prefix for multi-tenant isolation."""
|
||||
env_val = os.environ.get("MEMPALACE_PGVECTOR_NAMESPACE")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
value = self._file_config.get("pgvector_namespace")
|
||||
return str(value).strip() if value else None
|
||||
|
||||
@property
|
||||
def people_map(self):
|
||||
"""Mapping of name variants to canonical names."""
|
||||
@@ -560,6 +647,24 @@ class MempalaceConfig:
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
|
||||
def set_backend(self, backend: str) -> None:
|
||||
"""Persist the storage backend choice to ``config.json``."""
|
||||
backend = str(backend).strip().lower()
|
||||
from .backends import get_backend_class
|
||||
|
||||
get_backend_class(backend)
|
||||
self._file_config["backend"] = backend
|
||||
self._config_dir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
with open(self._config_file, "w", encoding="utf-8") as f:
|
||||
json.dump(self._file_config, f, indent=2, ensure_ascii=False)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
self._config_file.chmod(0o600)
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
|
||||
@property
|
||||
def topic_tunnel_min_count(self):
|
||||
"""Minimum number of overlapping confirmed topics required to create
|
||||
|
||||
+15
-10
@@ -10,13 +10,14 @@ Same palace as project mining. Different ingest strategy.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import hashlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
from typing import Optional
|
||||
|
||||
from .collision_scan import assert_no_collisions
|
||||
from .ids import ID_RECIPE, make_convo_drawer_id, make_convo_sentinel_id
|
||||
from .normalize import normalize
|
||||
from .palace import (
|
||||
NORMALIZE_VERSION,
|
||||
@@ -83,8 +84,7 @@ def _register_file(collection, source_file: str, wing: str, agent: str, extract_
|
||||
re-read and re-processed on every mine run because nothing was written to
|
||||
ChromaDB on the first pass.
|
||||
"""
|
||||
sentinel_key = f"{source_file}:{extract_mode}"
|
||||
sentinel_id = f"_reg_{hashlib.sha256(sentinel_key.encode()).hexdigest()[:24]}"
|
||||
sentinel_id = make_convo_sentinel_id(source_file, extract_mode)
|
||||
collection.upsert(
|
||||
documents=[f"[registry] {source_file}"],
|
||||
ids=[sentinel_id],
|
||||
@@ -98,6 +98,7 @@ def _register_file(collection, source_file: str, wing: str, agent: str, extract_
|
||||
"ingest_mode": "registry",
|
||||
"extract_mode": extract_mode,
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
"id_recipe": ID_RECIPE,
|
||||
}
|
||||
],
|
||||
)
|
||||
@@ -192,11 +193,15 @@ def _chunk_by_exchange(lines: list, chunk_size: int, min_chunk_size: int) -> lis
|
||||
next_line = lines[i]
|
||||
if next_line.strip().startswith(">") or next_line.strip().startswith("---"):
|
||||
break
|
||||
if next_line.strip():
|
||||
ai_lines.append(next_line.strip())
|
||||
# Preserve the line as-is — blank lines and indentation carry meaning
|
||||
# (paragraph breaks, list/code structure) and must survive verbatim.
|
||||
ai_lines.append(next_line)
|
||||
i += 1
|
||||
|
||||
ai_response = " ".join(ai_lines)
|
||||
# Join on newline (not space) so line structure, blank lines, and
|
||||
# indentation reach the drawer unchanged. Trim only trailing blank
|
||||
# lines produced by the loop stopping at the next `>` turn.
|
||||
ai_response = "\n".join(ai_lines).rstrip("\n")
|
||||
content = f"{user_turn}\n{ai_response}" if ai_response else user_turn
|
||||
|
||||
_emit_bounded(chunks, content, chunk_size, min_chunk_size)
|
||||
@@ -419,10 +424,8 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
|
||||
chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room
|
||||
if extract_mode == "general":
|
||||
room_counts_delta[chunk_room] += 1
|
||||
drawer_key = f"{source_file}:{extract_mode}:{chunk['chunk_index']}"
|
||||
drawer_id = (
|
||||
f"drawer_{wing}_{chunk_room}_"
|
||||
f"{hashlib.sha256(drawer_key.encode()).hexdigest()[:24]}"
|
||||
drawer_id = make_convo_drawer_id(
|
||||
wing, chunk_room, source_file, extract_mode, chunk["chunk_index"]
|
||||
)
|
||||
batch_docs.append(chunk["content"])
|
||||
batch_ids.append(drawer_id)
|
||||
@@ -438,8 +441,10 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
|
||||
"ingest_mode": "convos",
|
||||
"extract_mode": extract_mode,
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
"id_recipe": ID_RECIPE,
|
||||
}
|
||||
)
|
||||
assert_no_collisions(list(zip(batch_ids, batch_metas)), collection)
|
||||
try:
|
||||
collection.upsert(
|
||||
documents=batch_docs,
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"description": "Multi-word product/system names that must be detected atomically — not decomposed into their constituent words. When any of these compounds appears in mined content, the entity detector counts the COMPOUND, not the parts. The single-word detection pass that runs after this compound pre-pass would otherwise decompose 'Claude Code' into 'Claude' + 'Code', and the COCA filter (Tier 2) would then drop 'Code' as a content word while attributing usage incorrectly to 'Claude' alone.",
|
||||
"source": "Curated 2026-05-24 from common AI/development product names that mempalace users frequently mention. Each entry is a multi-word name (two-or-more whitespace-separated tokens, or hyphenated). Single-word product names like 'ChatGPT', 'Cursor', 'OrbStack' are NOT included — they have no decomposition risk and are handled by the existing single-word regex.",
|
||||
"case_handling": "Matching is case-insensitive. Canonical form (the entry below) is what gets counted. Caller must use re.IGNORECASE when matching.",
|
||||
"tier_2_interaction": "This file complements coca_content_words.json. Tier 2 filters single-word content nouns. Tier 3 protects multi-word product names. Run THIS pre-pass FIRST so the COCA filter doesn't see the decomposed words.",
|
||||
"compounds": [
|
||||
"Claude Code",
|
||||
"Claude Desktop",
|
||||
"Claude Sonnet",
|
||||
"Claude Opus",
|
||||
"Claude Haiku",
|
||||
"Claude Sonnet 4.5",
|
||||
"Claude Sonnet 4.6",
|
||||
"Claude Opus 4.5",
|
||||
"Claude Opus 4.6",
|
||||
"Claude Opus 4.7",
|
||||
"Gemini Code Assist",
|
||||
"Gemini 1.5 Pro",
|
||||
"Gemini 2.5 Pro",
|
||||
"Gemini 2.5 Flash",
|
||||
"Gemini Pro",
|
||||
"Gemini Flash",
|
||||
"GitHub Copilot",
|
||||
"GitHub Copilot CLI",
|
||||
"Microsoft Copilot",
|
||||
"Visual Studio Code",
|
||||
"Visual Studio",
|
||||
"IntelliJ IDEA",
|
||||
"Android Studio",
|
||||
"Sublime Text",
|
||||
"Replit Agent",
|
||||
"Docker Desktop",
|
||||
"Docker Compose",
|
||||
"GitHub Actions",
|
||||
"GitHub Pages",
|
||||
"GitHub Codespaces",
|
||||
"GitHub Pull Request",
|
||||
"Hugging Face",
|
||||
"Stack Overflow",
|
||||
"Stack Exchange",
|
||||
"Hacker News",
|
||||
"Y Combinator",
|
||||
"Google Cloud Platform",
|
||||
"Amazon Web Services",
|
||||
"Microsoft Azure",
|
||||
"Azure DevOps",
|
||||
"Notion AI",
|
||||
"OpenAI API",
|
||||
"Anthropic Console",
|
||||
"GPT-4",
|
||||
"GPT-4o",
|
||||
"GPT-5",
|
||||
"Llama 3",
|
||||
"Llama 3.1",
|
||||
"Mistral Small",
|
||||
"Mistral Large",
|
||||
"Microsoft Word",
|
||||
"Microsoft Excel",
|
||||
"Microsoft PowerPoint",
|
||||
"Google Docs",
|
||||
"Google Sheets",
|
||||
"Google Drive",
|
||||
"Google Cloud",
|
||||
"Apple Watch",
|
||||
"Apple TV"
|
||||
]
|
||||
}
|
||||
+6
-4
@@ -7,7 +7,9 @@ accumulate. This module finds drawers from the same source_file that
|
||||
are too similar (cosine distance < threshold), keeps the longest/richest
|
||||
version, and deletes the rest.
|
||||
|
||||
No API calls — uses ChromaDB's built-in embedding similarity.
|
||||
Uses the configured storage backend's similarity search. With the default
|
||||
local backends (Chroma, sqlite_exact) this stays on-machine with no external
|
||||
calls; with a remote backend (e.g. Qdrant) it issues queries to that backend.
|
||||
|
||||
Usage (standalone):
|
||||
python -m mempalace.dedup # dedup all
|
||||
@@ -27,7 +29,7 @@ import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
from .backends.chroma import ChromaBackend
|
||||
from .palace import get_collection
|
||||
|
||||
|
||||
COLLECTION_NAME = "mempalace_drawers"
|
||||
@@ -130,7 +132,7 @@ def dedup_source_group(col, drawer_ids, threshold=DEFAULT_THRESHOLD, dry_run=Tru
|
||||
def show_stats(palace_path=None):
|
||||
"""Show duplication statistics without making changes."""
|
||||
palace_path = palace_path or _get_palace_path()
|
||||
col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME)
|
||||
col = get_collection(palace_path, COLLECTION_NAME)
|
||||
|
||||
groups = get_source_groups(col)
|
||||
|
||||
@@ -162,7 +164,7 @@ def dedup_palace(
|
||||
print(" MemPalace Deduplicator")
|
||||
print(f"{'=' * 55}")
|
||||
|
||||
col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME)
|
||||
col = get_collection(palace_path, COLLECTION_NAME)
|
||||
|
||||
print(f" Palace: {palace_path}")
|
||||
print(f" Drawers: {col.count():,}")
|
||||
|
||||
@@ -191,6 +191,9 @@ class EmbeddinggemmaONNX:
|
||||
model_path = hf_hub_download(
|
||||
_EMBEDDINGGEMMA_REPO, subfolder="onnx", filename=_EMBEDDINGGEMMA_ONNX
|
||||
)
|
||||
hf_hub_download(
|
||||
_EMBEDDINGGEMMA_REPO, subfolder="onnx", filename=_EMBEDDINGGEMMA_ONNX + "_data"
|
||||
)
|
||||
tok_path = hf_hub_download(_EMBEDDINGGEMMA_REPO, filename="tokenizer.json")
|
||||
|
||||
self._session = ort.InferenceSession(model_path, providers=self._providers)
|
||||
@@ -223,6 +226,14 @@ class EmbeddinggemmaONNX:
|
||||
norms = np.linalg.norm(sent_emb, axis=1, keepdims=True) + 1e-12
|
||||
return (sent_emb / norms).tolist()
|
||||
|
||||
def embed_query(self, input: list[str]) -> list[list[float]]: # noqa: A002 — ChromaDB EF protocol
|
||||
"""Embed query documents (ChromaDB EF protocol)."""
|
||||
return self(input)
|
||||
|
||||
def embed_documents(self, input: list[str]) -> list[list[float]]: # noqa: A002
|
||||
"""Embed a batch of documents (ChromaDB EF protocol)."""
|
||||
return self(input)
|
||||
|
||||
|
||||
def get_embedding_function(device: Optional[str] = None, model: Optional[str] = None):
|
||||
"""Return a cached embedding function for the requested device + model.
|
||||
|
||||
@@ -78,6 +78,98 @@ def _get_coca_filter() -> frozenset[str]:
|
||||
return frozenset()
|
||||
|
||||
|
||||
# ==================== KNOWN-SYSTEMS COMPOUND LEXICON (Tier 3 linguistics cleanup) ====================
|
||||
#
|
||||
# Multi-word product / system names that must be detected atomically — NOT
|
||||
# decomposed into their constituent words. When "Claude Code" appears in
|
||||
# content, the entity detector counts the compound, not the parts. Without
|
||||
# this pre-pass, the single-word loop would split "Claude Code" into
|
||||
# "Claude" + "Code", and the COCA filter (Tier 2) would drop "Code" as a
|
||||
# content word — leaving "Claude" alone with the wrong attribution.
|
||||
#
|
||||
# Data file: ``mempalace/data/known_systems.json``. Loaded once on first
|
||||
# call via ``_get_known_systems``. Matching is case-insensitive with word
|
||||
# boundaries.
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _get_known_systems() -> tuple[tuple[str, "re.Pattern[str]"], ...]:
|
||||
"""Return the known-systems compound tuple — pairs of (canonical name,
|
||||
pre-compiled case-insensitive word-bounded regex).
|
||||
|
||||
Loads ``mempalace/data/known_systems.json`` on first call, compiles a
|
||||
regex for each valid compound, and caches the resulting tuple of
|
||||
pairs. Subsequent calls are O(1) and skip both the disk read AND the
|
||||
regex compilation. Returns an empty tuple if the data file is missing
|
||||
or malformed — extraction behavior then degrades gracefully
|
||||
(compounds detected only by the existing multi-word regex) rather
|
||||
than crashing.
|
||||
|
||||
Entries are sorted by length descending so the compound matcher
|
||||
prefers longer matches first (e.g. "Visual Studio Code" wins over
|
||||
a hypothetical "Visual Studio" if both were in the lexicon).
|
||||
"""
|
||||
data_path = Path(__file__).parent / "data" / "known_systems.json"
|
||||
try:
|
||||
raw = json.loads(data_path.read_text(encoding="utf-8"))
|
||||
compounds = raw.get("compounds", [])
|
||||
valid = [c for c in compounds if isinstance(c, str) and c.strip()]
|
||||
# Sort by length descending so longest-match-wins during the
|
||||
# pre-pass scan (longer compounds get masked first, so a shorter
|
||||
# compound contained within a longer one doesn't double-count).
|
||||
sorted_compounds = sorted(valid, key=len, reverse=True)
|
||||
|
||||
compiled: list[tuple[str, re.Pattern[str]]] = []
|
||||
for c in sorted_compounds:
|
||||
# Word-boundary, case-insensitive. Compound may contain
|
||||
# hyphens or spaces — re.escape handles special chars; word
|
||||
# boundaries on each side prevent partial-word matches
|
||||
# (e.g. "GPT-4" must not match "GPT-40").
|
||||
pattern = r"(?<!\w)" + re.escape(c) + r"(?!\w)"
|
||||
try:
|
||||
compiled.append((c, re.compile(pattern, re.IGNORECASE)))
|
||||
except re.error:
|
||||
continue
|
||||
return tuple(compiled)
|
||||
except (OSError, json.JSONDecodeError, AttributeError, TypeError):
|
||||
return ()
|
||||
|
||||
|
||||
def _apply_known_systems_prepass(text: str) -> tuple[str, dict[str, int]]:
|
||||
"""Scan ``text`` for known-systems compounds, return a working copy
|
||||
with matched spans masked to whitespace plus a dict of detected
|
||||
compound counts.
|
||||
|
||||
Returning the counts (instead of mutating a caller-supplied container)
|
||||
lets the three call sites (``extract_candidates`` at init-time,
|
||||
``palace.build_closet_lines`` at closet construction, and
|
||||
``miner._extract_entities_for_metadata`` at per-drawer tagging) use
|
||||
whichever container shape they already maintain.
|
||||
|
||||
Compounds are matched case-insensitively with word boundaries; the
|
||||
canonical (lexicon) casing is what gets counted, regardless of how
|
||||
the compound appears in the source text. Regexes are pre-compiled
|
||||
once in ``_get_known_systems`` so this function does no compilation.
|
||||
"""
|
||||
compounds = _get_known_systems()
|
||||
if not compounds:
|
||||
return text, {}
|
||||
working = text
|
||||
compound_counts: dict[str, int] = {}
|
||||
for compound, rx in compounds:
|
||||
matches = list(rx.finditer(working))
|
||||
if not matches:
|
||||
continue
|
||||
compound_counts[compound] = compound_counts.get(compound, 0) + len(matches)
|
||||
# Mask matched spans with spaces so the subsequent regex passes
|
||||
# don't re-decompose. Replacing right-to-left keeps earlier
|
||||
# indices stable.
|
||||
for m in reversed(matches):
|
||||
start, end = m.span()
|
||||
working = working[:start] + (" " * (end - start)) + working[end:]
|
||||
return working, compound_counts
|
||||
|
||||
|
||||
# ==================== LANGUAGE-AWARE PATTERN LOADING ====================
|
||||
|
||||
|
||||
@@ -194,13 +286,21 @@ def extract_candidates(text: str, languages=("en",)) -> dict:
|
||||
|
||||
counts: defaultdict = defaultdict(int)
|
||||
|
||||
# Tier 3 — known-systems compound pre-pass. Find compound product names
|
||||
# ("Claude Code", "GitHub Copilot", ...) FIRST and mask them out of the
|
||||
# working text so the subsequent single-word + multi-word loops don't
|
||||
# re-decompose them into their constituent tokens.
|
||||
working_text, compound_counts = _apply_known_systems_prepass(text)
|
||||
for compound, n in compound_counts.items():
|
||||
counts[compound] += n
|
||||
|
||||
# Single-word candidates — one pre-wrapped pattern per language
|
||||
for wrapped_pat in patterns["candidate_patterns"]:
|
||||
try:
|
||||
rx = re.compile(wrapped_pat)
|
||||
except re.error:
|
||||
continue
|
||||
for word in rx.findall(text):
|
||||
for word in rx.findall(working_text):
|
||||
wl = word.lower()
|
||||
if wl in stopwords:
|
||||
continue
|
||||
@@ -214,13 +314,16 @@ def extract_candidates(text: str, languages=("en",)) -> dict:
|
||||
continue
|
||||
counts[word] += 1
|
||||
|
||||
# Multi-word candidates — one pre-wrapped pattern per language
|
||||
# Multi-word candidates — one pre-wrapped pattern per language.
|
||||
# Runs against the working_text (compounds already masked) so an
|
||||
# unknown two-word phrase like "Jane Smith" still gets caught by
|
||||
# the regex without competing with known compounds.
|
||||
for wrapped_pat in patterns["multi_word_patterns"]:
|
||||
try:
|
||||
rx = re.compile(wrapped_pat)
|
||||
except re.error:
|
||||
continue
|
||||
for phrase in rx.findall(text):
|
||||
for phrase in rx.findall(working_text):
|
||||
if any(w.lower() in stopwords for w in phrase.split()):
|
||||
continue
|
||||
counts[phrase] += 1
|
||||
|
||||
@@ -82,6 +82,8 @@ from .palace import (
|
||||
# mempalace.format_miner.<name>. Lazy imports inside functions would not
|
||||
# expose these as attributes of this module, breaking the test seams.
|
||||
from .config import MempalaceConfig, normalize_wing_name
|
||||
from .collision_scan import assert_no_collisions
|
||||
from .ids import ID_RECIPE, make_drawer_id_from_chunk
|
||||
from .miner import (
|
||||
_compute_topic_tunnels_for_wing,
|
||||
chunk_text,
|
||||
@@ -640,8 +642,7 @@ def _file_chunks_locked(
|
||||
batch_ids: list = []
|
||||
batch_metas: list = []
|
||||
for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]:
|
||||
key = (source_file + str(chunk["chunk_index"])).encode()
|
||||
drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256(key).hexdigest()[:24]}"
|
||||
drawer_id = make_drawer_id_from_chunk(wing, room, source_file, chunk["chunk_index"])
|
||||
content = chunk["content"]
|
||||
meta: dict = {
|
||||
"wing": wing,
|
||||
@@ -654,6 +655,7 @@ def _file_chunks_locked(
|
||||
"extract_mode": "format",
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
"hall": detect_hall(content),
|
||||
"id_recipe": ID_RECIPE,
|
||||
}
|
||||
if source_mtime is not None:
|
||||
meta["source_mtime"] = source_mtime
|
||||
@@ -674,6 +676,7 @@ def _file_chunks_locked(
|
||||
batch_docs.append(content)
|
||||
batch_ids.append(drawer_id)
|
||||
batch_metas.append(meta)
|
||||
assert_no_collisions(list(zip(batch_ids, batch_metas)), collection)
|
||||
try:
|
||||
collection.upsert(
|
||||
documents=batch_docs,
|
||||
|
||||
+26
-5
@@ -179,7 +179,12 @@ def compute_hallways_for_wing(
|
||||
|
||||
Args:
|
||||
wing: wing name to scan.
|
||||
col: ChromaDB collection — must support ``.get(where=..., include=...)``.
|
||||
col: ChromaDB collection — must support ``.count()`` and paginated
|
||||
``.get(limit=..., offset=..., include=...)``. The fetch is filtered
|
||||
to ``wing`` client-side rather than via ``.get(where={"wing": ...})``,
|
||||
which binds one SQL variable per matched id and overflows SQLite's
|
||||
``SQLITE_MAX_VARIABLE_NUMBER`` on large wings (#1619). Fake
|
||||
collections and alternate backends must implement this shape.
|
||||
If ``None``, returns ``[]`` (caller didn't supply a backing
|
||||
store, so nothing to compute against). Tests pass a controlled
|
||||
MagicMock.
|
||||
@@ -198,16 +203,32 @@ def compute_hallways_for_wing(
|
||||
|
||||
min_count = max(1, int(min_count))
|
||||
|
||||
# 1. Query drawers for this wing.
|
||||
# 1. Query drawers for this wing. Paginate the fetch and filter to the
|
||||
# wing client-side: a single get(where={"wing": wing}) binds one SQL
|
||||
# variable per matched id and overflows SQLite's
|
||||
# SQLITE_MAX_VARIABLE_NUMBER (32766) on wings > ~32k drawers (#1619).
|
||||
# Mirrors the established pagination in miner.status / palace /
|
||||
# palace_graph.
|
||||
metadatas: list = []
|
||||
try:
|
||||
results = col.get(where={"wing": wing}, include=["metadatas"])
|
||||
total = col.count()
|
||||
batch_size = 5000
|
||||
offset = 0
|
||||
while offset < total:
|
||||
batch = col.get(limit=batch_size, offset=offset, include=["metadatas"])
|
||||
batch_metas = (batch or {}).get("metadatas") or []
|
||||
if not batch_metas:
|
||||
break
|
||||
metadatas.extend(
|
||||
m for m in batch_metas if isinstance(m, dict) and m.get("wing") == wing
|
||||
)
|
||||
offset += len(batch_metas)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"compute_hallways_for_wing: collection.get failed for %s", wing, exc_info=True
|
||||
"compute_hallways_for_wing: collection fetch failed for %s", wing, exc_info=True
|
||||
)
|
||||
return []
|
||||
|
||||
metadatas = (results or {}).get("metadatas") or []
|
||||
if not metadatas:
|
||||
return []
|
||||
|
||||
|
||||
+27
-5
@@ -651,12 +651,16 @@ def _save_diary_direct(
|
||||
session_id: str,
|
||||
wing: str = "",
|
||||
toast: bool = False,
|
||||
*,
|
||||
agent_name: str,
|
||||
) -> dict:
|
||||
"""Write a diary checkpoint by calling the tool function directly (no MCP roundtrip).
|
||||
|
||||
If `wing` is set, the entry lands in that wing (typically the project wing
|
||||
derived from the transcript path). Otherwise falls back to `tool_diary_write`'s
|
||||
default of `wing_session-hook`.
|
||||
The entry is filed under `agent_name` so the agent that later calls
|
||||
`mempalace_diary_read(agent_name=...)` discovers it (#1693). If `wing` is
|
||||
set, the entry lands in that wing (typically the project wing derived from
|
||||
the transcript path); a `diary_read` with an empty wing spans every wing
|
||||
the agent wrote to, so project-derived wings stay discoverable.
|
||||
|
||||
Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure.
|
||||
"""
|
||||
@@ -679,7 +683,7 @@ def _save_diary_direct(
|
||||
from .mcp_server import tool_diary_write
|
||||
|
||||
result = tool_diary_write(
|
||||
agent_name="session-hook",
|
||||
agent_name=agent_name,
|
||||
entry=entry,
|
||||
topic="checkpoint",
|
||||
wing=wing,
|
||||
@@ -741,6 +745,20 @@ def _ingest_transcript(transcript_path: str):
|
||||
SUPPORTED_HARNESSES = {"claude-code", "codex"}
|
||||
|
||||
|
||||
def _diary_agent_for_harness(harness: str) -> str:
|
||||
"""Return the diary ``agent_name`` a session in ``harness`` reads under.
|
||||
|
||||
Stop-hook checkpoints must be filed beside the agent's own entries so
|
||||
``mempalace_diary_read(agent_name=...)`` surfaces them. The old code filed
|
||||
them under a fixed ``"session-hook"`` identity that no reader ever queried,
|
||||
hiding every checkpoint (#1693). A ``claude-code`` session reads its diary
|
||||
as ``"claude"``; every other harness already reads under its own name, so
|
||||
returning the harness name keeps a newly supported harness discoverable
|
||||
instead of silently invisible again.
|
||||
"""
|
||||
return "claude" if harness == "claude-code" else harness
|
||||
|
||||
|
||||
def _parse_harness_input(data: dict, harness: str) -> dict:
|
||||
"""Parse stdin JSON according to the harness type."""
|
||||
if harness not in SUPPORTED_HARNESSES:
|
||||
@@ -942,7 +960,11 @@ def hook_stop(data: dict, harness: str):
|
||||
result = {"count": 0}
|
||||
if transcript_path:
|
||||
result = _save_diary_direct(
|
||||
transcript_path, session_id, wing=project_wing, toast=toast
|
||||
transcript_path,
|
||||
session_id,
|
||||
wing=project_wing,
|
||||
toast=toast,
|
||||
agent_name=_diary_agent_for_harness(harness),
|
||||
)
|
||||
_ingest_transcript(transcript_path)
|
||||
_maybe_auto_ingest()
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Centralized drawer/triple ID construction with collision-safe delimiter.
|
||||
|
||||
Drawer IDs and content-addressed identifiers built by concatenating strings
|
||||
without a delimiter before hashing form a defect class that allows
|
||||
``hash(s1 + str(i1)) == hash(s2 + str(i2))`` whenever
|
||||
``s1 + str(i1) == s2 + str(i2)``. Under ChromaDB's primary-key constraint
|
||||
the second upsert silently overwrites the first, losing content with no
|
||||
error raised. The styleguide's partial-scope-key-migration rule names this
|
||||
shape — every concat-into-hash site is a candidate that must be triaged.
|
||||
|
||||
This module is the single source of truth for ID construction in mempalace.
|
||||
All call sites use the named helpers below; no module should inline
|
||||
``hashlib.sha256(a + b)`` patterns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
# Recipe tag written to every drawer's metadata under this module's helpers.
|
||||
# Audits compare like-for-like: drawers without ``id_recipe`` are treated
|
||||
# as legacy ``v1`` (pre-delimiter recipe), drawers with ``id_recipe="v2"``
|
||||
# are guaranteed collision-safe within the v2 generation. The constant is
|
||||
# exported so call sites use ``ids.ID_RECIPE`` rather than a magic string.
|
||||
ID_RECIPE: str = "v2"
|
||||
|
||||
# '|' is reserved in Windows filenames and cannot appear in source paths
|
||||
# on any supported platform, making it strictly safer than ':' (which
|
||||
# appears in Windows drive letters and URL ports). Matches the existing
|
||||
# diary_ingest precedent at diary_ingest.py:52,76,91,98.
|
||||
_DELIM: str = "|"
|
||||
|
||||
# SHA-256 hex truncation lengths. Drawer IDs historically truncate at 24
|
||||
# chars; knowledge-graph triple IDs at 12. Preserved per-recipe so existing
|
||||
# fixture comparisons that hard-code truncation length still parse.
|
||||
_HASH_TRUNC_DRAWER: int = 24
|
||||
_HASH_TRUNC_TRIPLE: int = 12
|
||||
|
||||
|
||||
def _delimited_sha256(parts: tuple[object, ...], truncate: int) -> str:
|
||||
"""Hash parts joined by the unambiguous delimiter, truncate to N hex chars.
|
||||
|
||||
Internal helper. Call sites should use the named ``make_*`` wrappers
|
||||
below so the per-site contract is documented in code, not derived
|
||||
from caller arguments.
|
||||
|
||||
Each part is coerced to ``str`` before joining so the helper mirrors
|
||||
the pre-v2 behavior of ``f"{a}{b}"`` for ``None`` and numeric inputs —
|
||||
e.g. ``valid_from=None`` joins as the literal string ``"None"`` rather
|
||||
than crashing.
|
||||
"""
|
||||
key = _DELIM.join(str(p) for p in parts).encode()
|
||||
return hashlib.sha256(key).hexdigest()[:truncate]
|
||||
|
||||
|
||||
def make_drawer_id_from_chunk(wing: str, room: str, source_file: str, chunk_index: int) -> str:
|
||||
"""Drawer ID for the project / format miner paths.
|
||||
|
||||
Hash input is ``f"{source_file}|{chunk_index}"`` — the '|' separator
|
||||
prevents the classic ``"/a1" + "23" == "/a" + "123"`` collision.
|
||||
|
||||
Returns ``drawer_{wing}_{room}_{hash24}`` where hash24 is the first
|
||||
24 hex chars of SHA-256 over the delimited input.
|
||||
"""
|
||||
return (
|
||||
f"drawer_{wing}_{room}_"
|
||||
f"{_delimited_sha256((source_file, str(chunk_index)), _HASH_TRUNC_DRAWER)}"
|
||||
)
|
||||
|
||||
|
||||
def make_drawer_id_from_content(wing: str, room: str, content: str) -> str:
|
||||
"""Drawer ID for the MCP ``add_drawer`` tool path.
|
||||
|
||||
Hash input is ``f"{wing}|{room}|{content}"`` — the delimiters prevent
|
||||
``wing="foo" + room="bar"`` colliding with ``wing="fooba" + room="r"``
|
||||
(architecturally identical defect class to the chunk-index sites,
|
||||
even though astronomically rare in practice since content is large
|
||||
freeform text).
|
||||
"""
|
||||
return f"drawer_{wing}_{room}_{_delimited_sha256((wing, room, content), _HASH_TRUNC_DRAWER)}"
|
||||
|
||||
|
||||
def make_convo_drawer_id(
|
||||
wing: str, room: str, source_file: str, extract_mode: str, chunk_index: int
|
||||
) -> str:
|
||||
"""Drawer ID for the conversation miner path.
|
||||
|
||||
Pre-v2 the convo miner used ':' as delimiter; this helper migrates
|
||||
to '|' for codebase-wide consistency and to remove the Windows-path
|
||||
/ URL-source edge case that ':' carried.
|
||||
|
||||
Hash input is ``f"{source_file}|{extract_mode}|{chunk_index}"``.
|
||||
"""
|
||||
return (
|
||||
f"drawer_{wing}_{room}_"
|
||||
f"{_delimited_sha256((source_file, extract_mode, str(chunk_index)), _HASH_TRUNC_DRAWER)}"
|
||||
)
|
||||
|
||||
|
||||
def make_convo_sentinel_id(source_file: str, extract_mode: str) -> str:
|
||||
"""Sentinel registry ID for the conversation miner zero-chunk-file path.
|
||||
|
||||
Pre-v2 the sentinel used ':' as delimiter; this helper migrates to
|
||||
'|' for the same reasons as ``make_convo_drawer_id``.
|
||||
|
||||
Hash input is ``f"{source_file}|{extract_mode}"``.
|
||||
"""
|
||||
return f"_reg_{_delimited_sha256((source_file, extract_mode), _HASH_TRUNC_DRAWER)}"
|
||||
|
||||
|
||||
def make_triple_id(
|
||||
sub_id: str, predicate: str, obj_id: str, valid_from: str, recorded_at: str
|
||||
) -> str:
|
||||
"""Triple ID for knowledge-graph insertion.
|
||||
|
||||
Pre-v2 the recorded_at hash input was
|
||||
``f"{valid_from}{datetime.now().isoformat()}"`` with no delimiter —
|
||||
two ISO datetimes concatenated could collide in principle (e.g.
|
||||
``valid_from="2026-01-01" + isoformat "T12:..."`` vs
|
||||
``valid_from="2026-01-01T12" + isoformat ":..."``).
|
||||
|
||||
Returns ``t_{sub_id}_{predicate}_{obj_id}_{hash12}`` where hash12 is
|
||||
the first 12 hex chars of SHA-256 over ``f"{valid_from}|{recorded_at}"``.
|
||||
"""
|
||||
return (
|
||||
f"t_{sub_id}_{predicate}_{obj_id}_"
|
||||
f"{_delimited_sha256((valid_from, recorded_at), _HASH_TRUNC_TRIPLE)}"
|
||||
)
|
||||
@@ -35,7 +35,6 @@ Usage:
|
||||
kg.invalidate("Max", "has_issue", "sports_injury", ended="2026-02-15")
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
@@ -44,6 +43,7 @@ from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from .config import sanitize_iso_temporal
|
||||
from .ids import make_triple_id
|
||||
|
||||
|
||||
DEFAULT_KG_PATH = os.path.expanduser("~/.mempalace/knowledge_graph.sqlite3")
|
||||
@@ -302,7 +302,9 @@ class KnowledgeGraph:
|
||||
if existing:
|
||||
return existing["id"] # Already exists and still valid
|
||||
|
||||
triple_id = f"t_{sub_id}_{pred}_{obj_id}_{hashlib.sha256(f'{valid_from}{datetime.now().isoformat()}'.encode()).hexdigest()[:12]}"
|
||||
triple_id = make_triple_id(
|
||||
sub_id, pred, obj_id, valid_from, datetime.now().isoformat()
|
||||
)
|
||||
conn.execute(
|
||||
"""INSERT INTO triples (
|
||||
id, subject, predicate, object, valid_from, valid_to,
|
||||
|
||||
+402
-67
@@ -72,6 +72,7 @@ from .backends.chroma import ( # noqa: E402
|
||||
_pin_hnsw_threads,
|
||||
hnsw_capacity_status,
|
||||
)
|
||||
from .backends import BackendMismatchError, PalaceRef, detect_backend_for_path # noqa: E402
|
||||
from .query_sanitizer import sanitize_query # noqa: E402
|
||||
from .searcher import search_memories # noqa: E402
|
||||
from .palace_graph import ( # noqa: E402
|
||||
@@ -85,6 +86,8 @@ from .palace_graph import ( # noqa: E402
|
||||
)
|
||||
|
||||
from .knowledge_graph import KnowledgeGraph, DEFAULT_KG_PATH # noqa: E402
|
||||
from .collision_scan import assert_no_collisions # noqa: E402
|
||||
from .ids import ID_RECIPE, make_drawer_id_from_content # noqa: E402
|
||||
|
||||
|
||||
def _init_logging() -> None:
|
||||
@@ -160,6 +163,11 @@ def _parse_args():
|
||||
metavar="PATH",
|
||||
help="Path to the palace directory (overrides config file and env var)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
metavar="NAME",
|
||||
help="Storage backend to use (default: config/env/detected/chroma)",
|
||||
)
|
||||
args, unknown = parser.parse_known_args()
|
||||
if unknown:
|
||||
logger.debug("Ignoring unknown args: %s", unknown)
|
||||
@@ -170,6 +178,13 @@ _args = _parse_args()
|
||||
|
||||
if _args.palace:
|
||||
os.environ["MEMPALACE_PALACE_PATH"] = os.path.abspath(_args.palace)
|
||||
if _args.backend:
|
||||
backend_name = str(_args.backend).strip().lower()
|
||||
from .backends import get_backend_class # noqa: E402
|
||||
|
||||
get_backend_class(backend_name)
|
||||
os.environ["MEMPALACE_BACKEND_EXPLICIT"] = backend_name
|
||||
os.environ["MEMPALACE_BACKEND"] = backend_name
|
||||
|
||||
_config = MempalaceConfig()
|
||||
|
||||
@@ -289,6 +304,9 @@ def _call_kg(op):
|
||||
|
||||
_client_cache = None
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_collection_open_error = None
|
||||
_palace_db_inode = 0 # inode of chroma.sqlite3 at cache time
|
||||
_palace_db_mtime = 0.0 # mtime of chroma.sqlite3 at cache time
|
||||
|
||||
@@ -301,7 +319,15 @@ def _is_transient_index_error(result) -> bool:
|
||||
if not isinstance(result, dict):
|
||||
return False
|
||||
err = result.get("error", "")
|
||||
return isinstance(err, str) and ("Error finding id" in err or "Internal error" in err)
|
||||
if not isinstance(err, str):
|
||||
return False
|
||||
err_l = err.lower()
|
||||
return (
|
||||
"error finding id" in err_l
|
||||
or "internal error" in err_l
|
||||
or "stale-index" in err_l
|
||||
or "stale index" in err_l
|
||||
)
|
||||
|
||||
|
||||
def _force_chroma_cache_reset() -> None:
|
||||
@@ -313,21 +339,27 @@ def _force_chroma_cache_reset() -> None:
|
||||
global \
|
||||
_client_cache, \
|
||||
_collection_cache, \
|
||||
_collection_cache_backend, \
|
||||
_collection_cache_palace, \
|
||||
_collection_open_error, \
|
||||
_palace_db_inode, \
|
||||
_palace_db_mtime, \
|
||||
_metadata_cache, \
|
||||
_metadata_cache_time
|
||||
_client_cache = None
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_collection_open_error = None
|
||||
_palace_db_inode = 0
|
||||
_palace_db_mtime = 0.0
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
try:
|
||||
from .palace import _DEFAULT_BACKEND
|
||||
from .palace import get_backend_for_palace
|
||||
|
||||
_DEFAULT_BACKEND._clients.pop(_config.palace_path, None)
|
||||
_DEFAULT_BACKEND._freshness.pop(_config.palace_path, None)
|
||||
backend = get_backend_for_palace(_config.palace_path)
|
||||
backend.close_palace(PalaceRef(id=_config.palace_path, local_path=_config.palace_path))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -356,6 +388,11 @@ def _refresh_vector_disabled_flag() -> None:
|
||||
would defeat the point.
|
||||
"""
|
||||
global _vector_disabled, _vector_disabled_reason, _vector_capacity_status
|
||||
if not _is_chroma_backend():
|
||||
_vector_disabled = False
|
||||
_vector_disabled_reason = ""
|
||||
_vector_capacity_status = None
|
||||
return
|
||||
try:
|
||||
info = hnsw_capacity_status(_config.palace_path, _config.collection_name)
|
||||
except Exception:
|
||||
@@ -387,21 +424,51 @@ def _refresh_vector_disabled_flag() -> None:
|
||||
# This provides an audit trail for detecting memory poisoning and
|
||||
# enables review/rollback of writes from external or untrusted sources.
|
||||
|
||||
_WAL_DIR = Path(os.path.expanduser("~/.mempalace/wal"))
|
||||
_WAL_DIR.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
_WAL_DIR.chmod(0o700)
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
_WAL_FILE = _WAL_DIR / "write_log.jsonl"
|
||||
# Atomically create WAL file with restricted permissions (no TOCTOU race).
|
||||
# os.open with O_CREAT|O_WRONLY and mode 0o600 creates the file if absent
|
||||
# or opens it if present, both in a single syscall.
|
||||
try:
|
||||
_fd = os.open(str(_WAL_FILE), os.O_CREAT | os.O_WRONLY, 0o600)
|
||||
os.close(_fd)
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
_WAL_FILE = Path(os.path.expanduser("~/.mempalace/wal")) / "write_log.jsonl"
|
||||
_WAL_INITIALIZED_DIR = None
|
||||
|
||||
|
||||
def _ensure_wal() -> None:
|
||||
"""Create (and re-harden) the WAL directory lazily, on the first write.
|
||||
|
||||
This must NOT run at import time: a user who removed ``~/.mempalace`` has
|
||||
engaged the documented kill-switch (``hooks_cli._palace_root_exists()``,
|
||||
#1305), and recreating the directory just by importing this module would
|
||||
silently re-arm the autosave/mining hooks they disabled (#1676). Creating
|
||||
it on the first real write keeps the kill-switch contract intact.
|
||||
|
||||
It is deliberately not gated on ``_palace_root_exists()``: by the time a
|
||||
write reaches here the palace is already being recreated by the ChromaDB/KG
|
||||
layer regardless, so gating would only drop audit records, not prevent
|
||||
recreation. Runtime kill-switch enforcement for MCP writes is the broader
|
||||
question tracked in #504.
|
||||
|
||||
Hardening is attempted once per directory and the path cached in
|
||||
``_WAL_INITIALIZED_DIR`` regardless of outcome (keyed on the path, so a
|
||||
test repointing ``_WAL_FILE`` re-initialises), so a persistent failure on a
|
||||
restricted filesystem does not retry on every write. ``mkdir`` runs only
|
||||
when the initial ``chmod`` raises ``FileNotFoundError`` (EAFP). The parent
|
||||
``~/.mempalace`` keeps its umask mode, like the other palace directories;
|
||||
the WAL file is created atomically with mode 0o600 by ``_wal_log``.
|
||||
"""
|
||||
global _WAL_INITIALIZED_DIR
|
||||
wal_dir = _WAL_FILE.parent
|
||||
if _WAL_INITIALIZED_DIR == wal_dir:
|
||||
return
|
||||
try:
|
||||
wal_dir.chmod(0o700)
|
||||
except FileNotFoundError:
|
||||
try:
|
||||
wal_dir.mkdir(parents=True, exist_ok=True)
|
||||
wal_dir.chmod(0o700)
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
# Cache regardless of outcome: one attempt per directory, so a persistent
|
||||
# chmod/mkdir failure (restricted FS) is not retried on every write.
|
||||
_WAL_INITIALIZED_DIR = wal_dir
|
||||
|
||||
|
||||
# Keys whose values should be redacted in WAL entries to avoid logging sensitive content
|
||||
_WAL_REDACT_KEYS = frozenset(
|
||||
@@ -425,6 +492,9 @@ def _wal_log(operation: str, params: dict, result: dict = None):
|
||||
"result": result,
|
||||
}
|
||||
try:
|
||||
# Dir setup shares the append's exception handler below: any WAL
|
||||
# failure is logged and non-fatal, never crashing the tool call.
|
||||
_ensure_wal()
|
||||
fd = os.open(str(_WAL_FILE), os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
|
||||
with os.fdopen(fd, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(entry, default=str) + "\n")
|
||||
@@ -447,10 +517,15 @@ def _get_client():
|
||||
global \
|
||||
_client_cache, \
|
||||
_collection_cache, \
|
||||
_collection_cache_backend, \
|
||||
_collection_cache_palace, \
|
||||
_collection_open_error, \
|
||||
_palace_db_inode, \
|
||||
_palace_db_mtime, \
|
||||
_metadata_cache, \
|
||||
_metadata_cache_time
|
||||
if not _is_chroma_backend():
|
||||
raise RuntimeError("_get_client is only available for the Chroma backend")
|
||||
db_path = os.path.join(_config.palace_path, "chroma.sqlite3")
|
||||
try:
|
||||
st = os.stat(db_path)
|
||||
@@ -467,6 +542,9 @@ def _get_client():
|
||||
if not os.path.isfile(db_path) and _collection_cache is not None:
|
||||
_client_cache = None
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_collection_open_error = None
|
||||
_palace_db_inode = 0
|
||||
_palace_db_mtime = 0.0
|
||||
# Fall through to normal reconnect which will handle missing DB
|
||||
@@ -475,13 +553,18 @@ def _get_client():
|
||||
mtime_changed = current_mtime != 0.0 and abs(current_mtime - _palace_db_mtime) > 0.01
|
||||
|
||||
if _client_cache is None or inode_changed or mtime_changed:
|
||||
# Run the HNSW capacity probe BEFORE chromadb opens the segment —
|
||||
# Run the HNSW capacity probe BEFORE chromadb opens the segment --
|
||||
# if the index is severely undersized, segment load can segfault
|
||||
# the whole MCP server (#1222). The probe is pure sqlite +
|
||||
# metadata-pickle read; never touches the HNSW binary files.
|
||||
# metadata read; never touches the HNSW binary files.
|
||||
_refresh_vector_disabled_flag()
|
||||
if inode_changed or mtime_changed:
|
||||
ChromaBackend._quarantined_paths.discard(_config.palace_path)
|
||||
_client_cache = ChromaBackend.make_client(_config.palace_path)
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_collection_open_error = None
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
_palace_db_inode = current_inode
|
||||
@@ -490,20 +573,114 @@ def _get_client():
|
||||
|
||||
|
||||
def _get_collection(create=False):
|
||||
"""Return the ChromaDB collection, caching the client between calls.
|
||||
"""Return the configured backend collection, caching handles between calls.
|
||||
|
||||
On failure, log the exception and retry once after clearing the client
|
||||
and collection caches. Tools were silently returning ``None`` when a
|
||||
cached client/collection went stale — typically after the chromadb
|
||||
rust bindings invalidated a handle following an out-of-band write —
|
||||
leaving the LLM with no diagnostic and no recovery path. The retry
|
||||
forces ``_get_client()`` to rebuild from scratch (which re-runs
|
||||
``quarantine_stale_hnsw`` per #1322), so the second attempt heals the
|
||||
common stale-handle / stale-HNSW case automatically.
|
||||
forces ``_get_client()`` to rebuild the chromadb client from
|
||||
scratch, so the second attempt heals the common stale-handle case
|
||||
automatically.
|
||||
"""
|
||||
global _client_cache, _collection_cache, _metadata_cache, _metadata_cache_time
|
||||
global \
|
||||
_client_cache, \
|
||||
_collection_cache, \
|
||||
_collection_cache_backend, \
|
||||
_collection_cache_palace, \
|
||||
_collection_open_error, \
|
||||
_palace_db_inode, \
|
||||
_palace_db_mtime, \
|
||||
_metadata_cache, \
|
||||
_metadata_cache_time
|
||||
try:
|
||||
backend_name = _selected_backend_name()
|
||||
except (BackendMismatchError, KeyError) as exc:
|
||||
logger.warning("backend resolution failed for %s: %s", _config.palace_path, exc)
|
||||
_collection_open_error = {
|
||||
"error": "Backend mismatch"
|
||||
if isinstance(exc, BackendMismatchError)
|
||||
else "Unknown backend",
|
||||
"details": str(exc),
|
||||
"hint": "Select the matching backend or use a fresh palace directory.",
|
||||
}
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
return None
|
||||
|
||||
if backend_name != "chroma":
|
||||
for attempt in range(2):
|
||||
try:
|
||||
if (
|
||||
_collection_cache is not None
|
||||
and _collection_cache_backend == backend_name
|
||||
and _collection_cache_palace == _config.palace_path
|
||||
):
|
||||
_collection_open_error = None
|
||||
return _collection_cache
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
if _collection_cache is None:
|
||||
from .palace import get_collection as palace_get_collection
|
||||
|
||||
_collection_cache = palace_get_collection(
|
||||
_config.palace_path,
|
||||
collection_name=_config.collection_name,
|
||||
create=create,
|
||||
backend=backend_name,
|
||||
)
|
||||
_collection_cache_backend = backend_name
|
||||
_collection_cache_palace = _config.palace_path
|
||||
_collection_open_error = None
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
return _collection_cache
|
||||
except (BackendMismatchError, KeyError) as exc:
|
||||
logger.warning("backend open failed for %s: %s", _config.palace_path, exc)
|
||||
_collection_open_error = {
|
||||
"error": "Backend mismatch"
|
||||
if isinstance(exc, BackendMismatchError)
|
||||
else "Unknown backend",
|
||||
"details": str(exc),
|
||||
"hint": "Select the matching backend or use a fresh palace directory.",
|
||||
}
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
return None
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"_get_collection generic attempt %d/2 failed (palace=%s, create=%s)",
|
||||
attempt + 1,
|
||||
_config.palace_path,
|
||||
create,
|
||||
)
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
_collection_open_error = {
|
||||
"error": "Backend open failed",
|
||||
"details": "Could not open the selected backend collection.",
|
||||
"hint": "Run: mempalace status or mempalace repair-status for diagnostics.",
|
||||
}
|
||||
return None
|
||||
|
||||
for attempt in range(2):
|
||||
try:
|
||||
if _collection_cache is not None and (
|
||||
_collection_cache_backend not in (None, "chroma")
|
||||
or _collection_cache_palace not in (None, _config.palace_path)
|
||||
):
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
client = _get_client()
|
||||
# ChromaDB 1.x persists the EF *identity* (its ``name()``) with the
|
||||
# collection but not the EF *instance/configuration*. So a reader or
|
||||
@@ -550,6 +727,9 @@ def _get_collection(create=False):
|
||||
)
|
||||
_pin_hnsw_threads(raw)
|
||||
_collection_cache = ChromaCollection(raw, palace_path=_config.palace_path)
|
||||
_collection_cache_backend = "chroma"
|
||||
_collection_cache_palace = _config.palace_path
|
||||
_collection_open_error = None
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
elif _collection_cache is None:
|
||||
@@ -558,9 +738,29 @@ def _get_collection(create=False):
|
||||
raw = client.get_collection(_config.collection_name, **ef_kwargs)
|
||||
_pin_hnsw_threads(raw)
|
||||
_collection_cache = ChromaCollection(raw, palace_path=_config.palace_path)
|
||||
_collection_cache_backend = "chroma"
|
||||
_collection_cache_palace = _config.palace_path
|
||||
_collection_open_error = None
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
return _collection_cache
|
||||
except (BackendMismatchError, KeyError) as exc:
|
||||
_collection_open_error = {
|
||||
"error": "Backend mismatch"
|
||||
if isinstance(exc, BackendMismatchError)
|
||||
else "Unknown backend",
|
||||
"details": str(exc),
|
||||
"hint": "Select the matching backend or use a fresh palace directory.",
|
||||
}
|
||||
_client_cache = None
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_palace_db_inode = 0
|
||||
_palace_db_mtime = 0.0
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
return None
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"_get_collection attempt %d/2 failed (palace=%s, create=%s)",
|
||||
@@ -570,13 +770,35 @@ def _get_collection(create=False):
|
||||
)
|
||||
if attempt == 0:
|
||||
# Reset all caches so the next attempt forces _get_client()
|
||||
# to rebuild the chromadb client from scratch — that path
|
||||
# re-runs quarantine_stale_hnsw (#1322) and reopens the
|
||||
# collection cleanly, healing the common stale-handle case.
|
||||
# to rebuild the chromadb client from scratch, reopening
|
||||
# the collection cleanly and healing the common
|
||||
# stale-handle case.
|
||||
_client_cache = None
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_palace_db_inode = 0
|
||||
_palace_db_mtime = 0.0
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
_collection_open_error = {
|
||||
"error": "Backend open failed",
|
||||
"details": "Could not open the Chroma collection.",
|
||||
"hint": "Run: mempalace repair-status for diagnostics.",
|
||||
}
|
||||
_client_cache = None
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_palace_db_inode = 0
|
||||
_palace_db_mtime = 0.0
|
||||
_metadata_cache = None
|
||||
_metadata_cache_time = 0
|
||||
_collection_open_error = _collection_open_error or {
|
||||
"error": "Backend open failed",
|
||||
"details": "Could not open the selected backend collection.",
|
||||
"hint": "Run: mempalace status or mempalace repair-status for diagnostics.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
@@ -587,6 +809,42 @@ def _no_palace():
|
||||
}
|
||||
|
||||
|
||||
def _collection_error_or_no_palace():
|
||||
if not _collection_open_error:
|
||||
return _no_palace()
|
||||
result = dict(_collection_open_error)
|
||||
try:
|
||||
result["backend"] = _selected_backend_name()
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def _selected_backend_name() -> str:
|
||||
from .palace import resolve_backend_name
|
||||
|
||||
return resolve_backend_name(
|
||||
_config.palace_path,
|
||||
explicit=os.environ.get("MEMPALACE_BACKEND_EXPLICIT"),
|
||||
)
|
||||
|
||||
|
||||
def _is_chroma_backend() -> bool:
|
||||
try:
|
||||
return _selected_backend_name() == "chroma"
|
||||
except Exception:
|
||||
logger.debug("backend resolution failed", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def _backend_db_exists() -> bool:
|
||||
try:
|
||||
return detect_backend_for_path(_config.palace_path) is not None
|
||||
except Exception:
|
||||
logger.debug("backend artifact detection failed", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
# ==================== HELPERS ====================
|
||||
|
||||
|
||||
@@ -722,6 +980,7 @@ def _tool_status_via_sqlite() -> dict:
|
||||
"rooms": rooms,
|
||||
"protocol": PALACE_PROTOCOL,
|
||||
"aaak_dialect": AAAK_SPEC,
|
||||
"backend": "chroma",
|
||||
"vector_disabled": True,
|
||||
"vector_disabled_reason": _vector_disabled_reason,
|
||||
}
|
||||
@@ -739,7 +998,7 @@ def tool_status():
|
||||
# #1222 failure mode, opening the persistent client to call .count()
|
||||
# can segfault — short-circuit to a pure-sqlite path when divergence
|
||||
# is detected so status stays reachable.
|
||||
db_exists = os.path.isfile(os.path.join(_config.palace_path, "chroma.sqlite3"))
|
||||
db_exists = _backend_db_exists()
|
||||
_refresh_vector_disabled_flag()
|
||||
|
||||
if _vector_disabled:
|
||||
@@ -750,7 +1009,7 @@ def tool_status():
|
||||
# accidentally creating a palace in a non-existent directory (#830).
|
||||
col = _get_collection(create=db_exists)
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
count = col.count()
|
||||
wings = {}
|
||||
rooms = {}
|
||||
@@ -760,6 +1019,7 @@ def tool_status():
|
||||
"rooms": rooms,
|
||||
"protocol": PALACE_PROTOCOL,
|
||||
"aaak_dialect": AAAK_SPEC,
|
||||
"backend": _selected_backend_name(),
|
||||
}
|
||||
try:
|
||||
all_meta = _get_cached_metadata(col)
|
||||
@@ -812,7 +1072,7 @@ When WRITING AAAK: use entity codes, mark emotions, keep structure tight."""
|
||||
def tool_list_wings():
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
wings = {}
|
||||
result = {"wings": wings}
|
||||
try:
|
||||
@@ -835,7 +1095,7 @@ def tool_list_rooms(wing: str = None):
|
||||
return {"error": str(e)}
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
rooms = {}
|
||||
result = {"wing": wing or "all", "rooms": rooms}
|
||||
try:
|
||||
@@ -855,7 +1115,7 @@ def tool_list_rooms(wing: str = None):
|
||||
def tool_get_taxonomy():
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
taxonomy = {}
|
||||
result = {"taxonomy": taxonomy}
|
||||
try:
|
||||
@@ -925,6 +1185,7 @@ def tool_search(
|
||||
n_results=limit,
|
||||
max_distance=dist,
|
||||
vector_disabled=_vector_disabled,
|
||||
collection_name=_config.collection_name,
|
||||
)
|
||||
if not _is_transient_index_error(result):
|
||||
result["index_recovered"] = True
|
||||
@@ -963,7 +1224,7 @@ def tool_check_duplicate(content: str, threshold: float = 0.9):
|
||||
}
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
try:
|
||||
content = strip_lone_surrogates(content)
|
||||
results = col.query(
|
||||
@@ -1009,7 +1270,7 @@ def tool_traverse_graph(start_room: str, max_hops: int = 2):
|
||||
max_hops = max(1, min(max_hops, 10))
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
return traverse(start_room, col=col, max_hops=max_hops)
|
||||
|
||||
|
||||
@@ -1022,7 +1283,7 @@ def tool_find_tunnels(wing_a: str = None, wing_b: str = None):
|
||||
return {"error": str(e)}
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
return find_tunnels(wing_a, wing_b, col=col)
|
||||
|
||||
|
||||
@@ -1030,7 +1291,7 @@ def tool_graph_stats():
|
||||
"""Palace graph overview: nodes, tunnels, edges, connectivity."""
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
return graph_stats(col=col)
|
||||
|
||||
|
||||
@@ -1096,6 +1357,8 @@ def tool_follow_tunnels(wing: str, room: str):
|
||||
except ValueError as e:
|
||||
return {"error": str(e)}
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _collection_error_or_no_palace()
|
||||
return follow_tunnels(wing, room, col=col)
|
||||
|
||||
|
||||
@@ -1130,11 +1393,9 @@ def tool_add_drawer(
|
||||
|
||||
col = _get_collection(create=True)
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
|
||||
drawer_id = (
|
||||
f"drawer_{wing}_{room}_{hashlib.sha256((wing + room + content).encode()).hexdigest()[:24]}"
|
||||
)
|
||||
drawer_id = make_drawer_id_from_content(wing, room, content)
|
||||
|
||||
_wal_log(
|
||||
"add_drawer",
|
||||
@@ -1155,6 +1416,7 @@ def tool_add_drawer(
|
||||
"source_file": source_file or "",
|
||||
"added_by": added_by,
|
||||
"filed_at": datetime.now().isoformat(),
|
||||
"id_recipe": ID_RECIPE,
|
||||
}
|
||||
|
||||
# Idempotency. Three cases to detect a prior committed write:
|
||||
@@ -1216,6 +1478,7 @@ def tool_add_drawer(
|
||||
chunk_metas.append(
|
||||
{**base_meta, "chunk_index": chunk_idx, "parent_drawer_id": drawer_id}
|
||||
)
|
||||
assert_no_collisions(list(zip(chunk_ids, chunk_metas)), col)
|
||||
col.upsert(ids=chunk_ids, documents=chunk_docs, metadatas=chunk_metas)
|
||||
# Probe the LAST chunk id, not the first — its presence confirms
|
||||
# the whole batch landed, not just the leading row.
|
||||
@@ -1244,7 +1507,7 @@ def tool_delete_drawer(drawer_id: str):
|
||||
global _metadata_cache
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
existing = col.get(ids=[drawer_id])
|
||||
if not existing["ids"]:
|
||||
return {"success": False, "error": f"Drawer not found: {drawer_id}"}
|
||||
@@ -1314,7 +1577,7 @@ def tool_get_drawer(drawer_id: str):
|
||||
"""Fetch a single drawer by ID. Returns full content and metadata."""
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
try:
|
||||
result = col.get(ids=[drawer_id], include=["documents", "metadatas"])
|
||||
if not result["ids"]:
|
||||
@@ -1352,7 +1615,7 @@ def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offse
|
||||
return {"error": str(e)}
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
try:
|
||||
where = None
|
||||
conditions = []
|
||||
@@ -1409,7 +1672,7 @@ def tool_update_drawer(drawer_id: str, content: str = None, wing: str = None, ro
|
||||
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
try:
|
||||
existing = col.get(ids=[drawer_id], include=["documents", "metadatas"])
|
||||
if not existing["ids"]:
|
||||
@@ -1627,7 +1890,7 @@ def tool_diary_write(agent_name: str, entry: str, topic: str = "general", wing:
|
||||
room = "diary"
|
||||
col = _get_collection(create=True)
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
|
||||
now = datetime.now()
|
||||
entry_id = (
|
||||
@@ -1750,7 +2013,7 @@ def tool_diary_read(agent_name: str, last_n: int = 10, wing: str = ""):
|
||||
last_n = max(1, min(last_n, 100))
|
||||
col = _get_collection()
|
||||
if not col:
|
||||
return _no_palace()
|
||||
return _collection_error_or_no_palace()
|
||||
|
||||
# Build filter: always scope by agent + room=diary. Wing is optional —
|
||||
# when empty, return entries across all wings for this agent (matches
|
||||
@@ -1884,6 +2147,9 @@ def tool_reconnect():
|
||||
global \
|
||||
_client_cache, \
|
||||
_collection_cache, \
|
||||
_collection_cache_backend, \
|
||||
_collection_cache_palace, \
|
||||
_collection_open_error, \
|
||||
_palace_db_inode, \
|
||||
_palace_db_mtime, \
|
||||
_vector_disabled, \
|
||||
@@ -1891,31 +2157,63 @@ def tool_reconnect():
|
||||
from . import palace as palace_module
|
||||
|
||||
close_errors = []
|
||||
palace_ref = PalaceRef(id=_config.palace_path, local_path=_config.palace_path)
|
||||
closed_backend_names = set()
|
||||
cached_backend_name = _collection_cache_backend
|
||||
try:
|
||||
palace_module._DEFAULT_BACKEND.close_palace(_config.palace_path)
|
||||
backend = palace_module.get_backend_for_palace(_config.palace_path)
|
||||
backend.close_palace(palace_ref)
|
||||
if getattr(backend, "name", None):
|
||||
closed_backend_names.add(backend.name)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to close shared palace backend during reconnect", exc_info=True)
|
||||
close_errors.append(f"backend close_palace failed: {exc}")
|
||||
try:
|
||||
from chromadb.api.client import SharedSystemClient
|
||||
if cached_backend_name and cached_backend_name not in closed_backend_names:
|
||||
try:
|
||||
from .backends import get_backend
|
||||
|
||||
clear_system_cache = getattr(SharedSystemClient, "clear_system_cache", None)
|
||||
if callable(clear_system_cache):
|
||||
clear_system_cache()
|
||||
else:
|
||||
get_backend(cached_backend_name).close_palace(palace_ref)
|
||||
closed_backend_names.add(cached_backend_name)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"SharedSystemClient.clear_system_cache is unavailable; skipping shared Chroma cache clear during reconnect"
|
||||
"Failed to close previously cached %s backend during reconnect",
|
||||
cached_backend_name,
|
||||
exc_info=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Failed to clear Chroma shared system cache during reconnect",
|
||||
exc_info=True,
|
||||
)
|
||||
close_errors.append(f"shared Chroma cache clear failed: {exc}")
|
||||
close_errors.append(f"cached {cached_backend_name} close_palace failed: {exc}")
|
||||
if _client_cache is not None:
|
||||
try:
|
||||
close = getattr(_client_cache, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to close MCP-local Chroma client during reconnect", exc_info=True)
|
||||
close_errors.append(f"local Chroma client close failed: {exc}")
|
||||
if _is_chroma_backend():
|
||||
try:
|
||||
from chromadb.api.client import SharedSystemClient
|
||||
|
||||
clear_system_cache = getattr(SharedSystemClient, "clear_system_cache", None)
|
||||
if callable(clear_system_cache):
|
||||
clear_system_cache()
|
||||
else:
|
||||
logger.debug(
|
||||
"SharedSystemClient.clear_system_cache is unavailable; skipping shared Chroma cache clear during reconnect"
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Failed to clear Chroma shared system cache during reconnect",
|
||||
exc_info=True,
|
||||
)
|
||||
close_errors.append(f"shared Chroma cache clear failed: {exc}")
|
||||
_client_cache = None
|
||||
_collection_cache = None
|
||||
_collection_cache_backend = None
|
||||
_collection_cache_palace = None
|
||||
_collection_open_error = None
|
||||
_palace_db_inode = 0
|
||||
_palace_db_mtime = 0.0
|
||||
ChromaBackend._quarantined_paths.discard(_config.palace_path)
|
||||
# Force probe re-run on next _get_client by clearing the flag now;
|
||||
# _refresh_vector_disabled_flag will re-set it if the divergence
|
||||
# still applies after the reconnect.
|
||||
@@ -1933,12 +2231,17 @@ def tool_reconnect():
|
||||
try:
|
||||
col = _get_collection()
|
||||
if col is None:
|
||||
open_error = _collection_error_or_no_palace()
|
||||
result = {
|
||||
"success": False,
|
||||
"message": "No palace found after reconnect",
|
||||
"message": open_error.get("error", "No palace found after reconnect"),
|
||||
"drawers": 0,
|
||||
"vector_disabled": _vector_disabled,
|
||||
}
|
||||
if "details" in open_error:
|
||||
result["details"] = open_error["details"]
|
||||
if "hint" in open_error:
|
||||
result["hint"] = open_error["hint"]
|
||||
if close_errors:
|
||||
result["error"] = "; ".join(close_errors)
|
||||
return result
|
||||
@@ -2354,8 +2657,18 @@ TOOLS = {
|
||||
"type": "string",
|
||||
"description": "Target wing for this diary entry (optional). If omitted, uses wing_{agent_name}. Use this to write diary entries to a project wing instead of an agent-specific wing.",
|
||||
},
|
||||
"content": {
|
||||
"type": "string",
|
||||
"description": "Alias for 'entry' — accepted because add_drawer uses 'content'. Provide either 'entry' or 'content'; 'entry' wins if both are given.",
|
||||
},
|
||||
},
|
||||
"required": ["agent_name", "entry"],
|
||||
# agent_name is always required; 'entry' or its alias 'content' must
|
||||
# be present (the server remaps content->entry at dispatch).
|
||||
"required": ["agent_name"],
|
||||
"anyOf": [
|
||||
{"required": ["entry"]},
|
||||
{"required": ["content"]},
|
||||
],
|
||||
},
|
||||
"handler": tool_diary_write,
|
||||
},
|
||||
@@ -2562,6 +2875,17 @@ def handle_request(request):
|
||||
"error": {"code": -32602, "message": f"Invalid value for parameter '{key}'"},
|
||||
}
|
||||
tool_args.pop("wait_for_previous", None)
|
||||
# 'content' is an accepted alias for diary_write's 'entry' (callers often
|
||||
# reuse add_drawer's 'content' name). Map it in here, before dispatch, so a
|
||||
# content-only call still satisfies the required 'entry' param while the
|
||||
# signature-based missing-parameter diagnostic (-32602) keeps working.
|
||||
# 'entry' wins if both are supplied.
|
||||
if tool_name == "mempalace_diary_write" and "content" in tool_args:
|
||||
content_val = tool_args.pop("content")
|
||||
# Only fill from the alias when the caller did not supply 'entry' at
|
||||
# all (or passed it as null). An explicit entry — even "" — wins.
|
||||
if "entry" not in tool_args or tool_args["entry"] is None:
|
||||
tool_args["entry"] = content_val
|
||||
try:
|
||||
result = TOOLS[tool_name]["handler"](**tool_args)
|
||||
return {
|
||||
@@ -2724,8 +3048,17 @@ def _maybe_eager_warmup_embedder() -> None:
|
||||
)
|
||||
return
|
||||
palace_path = _config.palace_path
|
||||
db_path = os.path.join(palace_path, "chroma.sqlite3")
|
||||
if not os.path.isfile(db_path):
|
||||
try:
|
||||
backend_name = _selected_backend_name()
|
||||
except Exception as exc: # fail-soft per docstring
|
||||
logger.warning(
|
||||
"MEMPALACE_EAGER_WARMUP=%s: backend resolution failed for %s (%s)",
|
||||
raw,
|
||||
palace_path,
|
||||
exc,
|
||||
)
|
||||
return
|
||||
if not _backend_db_exists():
|
||||
# Pre-check (NOT a try/except on _ChromaNotFoundError, which never
|
||||
# propagates out of _get_collection — see docstring). No palace
|
||||
# file means nothing to warm AND avoids the chromadb-client
|
||||
@@ -2769,9 +3102,11 @@ def _maybe_eager_warmup_embedder() -> None:
|
||||
type(exc).__name__,
|
||||
)
|
||||
else:
|
||||
warmed = "embedder + HNSW ready" if backend_name == "chroma" else "embedder + backend ready"
|
||||
logger.info(
|
||||
"MEMPALACE_EAGER_WARMUP=%s: embedder + HNSW ready (palace=%s, device=%s)",
|
||||
"MEMPALACE_EAGER_WARMUP=%s: %s (palace=%s, device=%s)",
|
||||
raw,
|
||||
warmed,
|
||||
palace_path,
|
||||
device,
|
||||
)
|
||||
|
||||
@@ -360,3 +360,218 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
|
||||
|
||||
print(f"\n{'=' * 60}\n")
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wing-name normalization migration (#1675 follow-up)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# normalize_wing_name now strips leading/trailing separators, so a path-encoded
|
||||
# dirname like ``-home-user-proj`` derives ``home_user_proj`` instead of
|
||||
# ``_home_user_proj``. Palaces built before that rule filed drawers under the
|
||||
# old, leading-underscore wing, which the new derivation no longer matches —
|
||||
# searches and diary reads under the new name miss the old memories.
|
||||
#
|
||||
# This migration re-keys the ``wing`` metadata field on drawers and closets to
|
||||
# the normalized form, merging collisions. Drawer/closet IDs embed the wing as
|
||||
# an opaque prefix that is never decoded back into a wing (verified: nothing
|
||||
# splits a wing out of an ID; mining idempotency keys on ``source_file``), so
|
||||
# the IDs are left untouched — closet ``→drawer_id`` pointers stay valid and
|
||||
# future mining still skips already-mined files. Tunnels resolve via existing
|
||||
# read-time normalization and need no rewrite. The pass is idempotent.
|
||||
|
||||
|
||||
def _normalized_wing_target(wing):
|
||||
"""Return the normalized wing if it differs from ``wing``, else ``None``.
|
||||
|
||||
``None`` means "no migration needed" — either the value is not a non-empty
|
||||
string, normalization is a no-op, or it would normalize to empty.
|
||||
"""
|
||||
from .config import normalize_wing_name
|
||||
|
||||
if not isinstance(wing, str) or not wing:
|
||||
return None
|
||||
# Apply the full normalization and explicitly strip leading/trailing
|
||||
# separators. The strip is this migration's whole purpose (#1675); doing it
|
||||
# here rather than relying on normalize_wing_name keeps the migration correct
|
||||
# even when run against a build whose normalize_wing_name predates #1675, and
|
||||
# matches the post-#1675 derivation exactly.
|
||||
target = normalize_wing_name(wing).strip("_")
|
||||
if not target or target == wing:
|
||||
return None
|
||||
return target
|
||||
|
||||
|
||||
def plan_wing_renames(items):
|
||||
"""Pure planner over ``(id, metadata)`` pairs.
|
||||
|
||||
Returns ``(summary, updates)`` where ``summary`` is ``{(old, new): count}``
|
||||
and ``updates`` is ``[(id, new_metadata), ...]`` for only the records whose
|
||||
wing changes. Metadata is copied; only the ``wing`` key is rewritten.
|
||||
"""
|
||||
summary = defaultdict(int)
|
||||
updates = []
|
||||
for rec_id, meta in items:
|
||||
meta = dict(meta or {})
|
||||
target = _normalized_wing_target(meta.get("wing"))
|
||||
if target is None:
|
||||
continue
|
||||
summary[(meta["wing"], target)] += 1
|
||||
meta["wing"] = target
|
||||
updates.append((rec_id, meta))
|
||||
return summary, updates
|
||||
|
||||
|
||||
def _iter_collection_items(col, batch_size=1000):
|
||||
"""Yield ``(id, metadata)`` for every record in a backend collection."""
|
||||
total = col.count()
|
||||
offset = 0
|
||||
while offset < total:
|
||||
batch = col.get(limit=batch_size, offset=offset, include=["metadatas"])
|
||||
ids = batch.ids if hasattr(batch, "ids") else batch["ids"]
|
||||
metas = batch.metadatas if hasattr(batch, "metadatas") else batch["metadatas"]
|
||||
if not ids:
|
||||
break
|
||||
for rec_id, meta in zip(ids, metas):
|
||||
yield rec_id, meta
|
||||
offset += len(ids)
|
||||
|
||||
|
||||
def _apply_wing_updates(col, updates, batch_size=500):
|
||||
"""Re-label the ``wing`` metadata field in place for the planned updates."""
|
||||
for i in range(0, len(updates), batch_size):
|
||||
chunk = updates[i : i + batch_size]
|
||||
col.update(ids=[u[0] for u in chunk], metadatas=[u[1] for u in chunk])
|
||||
|
||||
|
||||
def _plan_topics_by_wing_renames():
|
||||
"""Return ``{old_wing: new_wing}`` for ``topics_by_wing`` keys to normalize."""
|
||||
try:
|
||||
from .miner import _load_known_entities_raw
|
||||
|
||||
reg = _load_known_entities_raw()
|
||||
except Exception:
|
||||
return {}
|
||||
tbw = reg.get("topics_by_wing")
|
||||
if not isinstance(tbw, dict):
|
||||
return {}
|
||||
renames = {}
|
||||
for wing in list(tbw.keys()):
|
||||
target = _normalized_wing_target(wing)
|
||||
if target is not None:
|
||||
renames[wing] = target
|
||||
return renames
|
||||
|
||||
|
||||
def _apply_topics_by_wing_renames(renames):
|
||||
"""Re-key ``topics_by_wing`` in known_entities.json, merging on collision."""
|
||||
if not renames:
|
||||
return
|
||||
import json
|
||||
|
||||
from .miner import _ENTITY_REGISTRY_PATH, _load_known_entities_raw
|
||||
|
||||
try:
|
||||
reg = _load_known_entities_raw()
|
||||
except Exception:
|
||||
return
|
||||
tbw = reg.get("topics_by_wing")
|
||||
if not isinstance(tbw, dict):
|
||||
return
|
||||
for old, new in renames.items():
|
||||
if old not in tbw:
|
||||
continue
|
||||
old_topics = tbw.pop(old) or []
|
||||
if new in tbw:
|
||||
merged = list(tbw[new])
|
||||
for topic in old_topics:
|
||||
if topic not in merged:
|
||||
merged.append(topic)
|
||||
tbw[new] = merged
|
||||
else:
|
||||
tbw[new] = old_topics
|
||||
reg["topics_by_wing"] = tbw
|
||||
os.makedirs(os.path.dirname(_ENTITY_REGISTRY_PATH), exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(_ENTITY_REGISTRY_PATH), suffix=".tmp")
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(reg, f, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp, _ENTITY_REGISTRY_PATH)
|
||||
except Exception:
|
||||
if os.path.exists(tmp):
|
||||
os.remove(tmp)
|
||||
raise
|
||||
|
||||
|
||||
def migrate_wing_names(palace_path: str, dry_run: bool = False, confirm: bool = False) -> bool:
|
||||
"""Normalize legacy wing names in ``palace_path`` (strip leading/trailing
|
||||
separators), so palaces built before #1675 keep their memories discoverable.
|
||||
|
||||
Re-keys the ``wing`` metadata on drawers and closets in place (IDs untouched)
|
||||
and the ``topics_by_wing`` registry, merging collisions. Idempotent.
|
||||
|
||||
Returns True if anything was (or, in dry-run, would be) migrated.
|
||||
"""
|
||||
from .palace import get_closets_collection, get_collection
|
||||
|
||||
try:
|
||||
drawers = get_collection(palace_path, create=False)
|
||||
except Exception as exc:
|
||||
print(f" No drawer collection found at {palace_path} ({exc}).")
|
||||
return False
|
||||
|
||||
d_items = list(_iter_collection_items(drawers))
|
||||
all_wings = {(m or {}).get("wing") for _, m in d_items if (m or {}).get("wing")}
|
||||
d_summary, d_updates = plan_wing_renames(d_items)
|
||||
|
||||
closets = None
|
||||
c_summary, c_updates = defaultdict(int), []
|
||||
try:
|
||||
closets = get_closets_collection(palace_path, create=False)
|
||||
c_summary, c_updates = plan_wing_renames(_iter_collection_items(closets))
|
||||
except Exception:
|
||||
closets = None
|
||||
|
||||
topic_renames = _plan_topics_by_wing_renames()
|
||||
|
||||
if not d_updates and not c_updates and not topic_renames:
|
||||
print(" All wing names are already normalized — nothing to migrate.")
|
||||
return False
|
||||
|
||||
print("\n Wing-name migration plan:")
|
||||
merged = defaultdict(lambda: [0, 0])
|
||||
for key, count in d_summary.items():
|
||||
merged[key][0] = count
|
||||
for key, count in c_summary.items():
|
||||
merged[key][1] = count
|
||||
for (old, new), (d_count, c_count) in sorted(merged.items()):
|
||||
note = " (MERGE into existing wing)" if new in all_wings else ""
|
||||
print(f" {old!r} -> {new!r}: {d_count} drawer(s), {c_count} closet(s){note}")
|
||||
if topic_renames:
|
||||
print(f" topics_by_wing: {len(topic_renames)} key(s) re-keyed")
|
||||
|
||||
if dry_run:
|
||||
print("\n DRY RUN — no changes made.\n")
|
||||
return True
|
||||
|
||||
if not confirm:
|
||||
try:
|
||||
resp = input(" Apply this wing-name migration? [y/N] ").strip().lower()
|
||||
except EOFError:
|
||||
resp = ""
|
||||
if resp not in ("y", "yes"):
|
||||
print(" Aborted.")
|
||||
return False
|
||||
|
||||
_apply_wing_updates(drawers, d_updates)
|
||||
if closets is not None and c_updates:
|
||||
_apply_wing_updates(closets, c_updates)
|
||||
_apply_topics_by_wing_renames(topic_renames)
|
||||
|
||||
parts = [f"{len(d_updates)} drawer(s)"]
|
||||
if c_updates:
|
||||
parts.append(f"{len(c_updates)} closet(s)")
|
||||
if topic_renames:
|
||||
parts.append(f"{len(topic_renames)} topic key(s)")
|
||||
print(f"\n Migrated {', '.join(parts)}.\n")
|
||||
return True
|
||||
|
||||
+38
-8
@@ -19,7 +19,7 @@ from datetime import datetime
|
||||
from collections import defaultdict
|
||||
from typing import Optional
|
||||
|
||||
from .entity_detector import _get_coca_filter
|
||||
from .entity_detector import _apply_known_systems_prepass, _get_coca_filter
|
||||
from .palace import (
|
||||
NORMALIZE_VERSION,
|
||||
SKIP_DIRS,
|
||||
@@ -41,7 +41,9 @@ from .palace import (
|
||||
# ``mempalace.miner.compute_hallways_for_wing``. The integration call
|
||||
# lives at the end of _mine_impl, alongside the existing
|
||||
# ``_compute_topic_tunnels_for_wing`` post-mine block.
|
||||
from .collision_scan import assert_no_collisions
|
||||
from .hallways import compute_hallways_for_wing
|
||||
from .ids import ID_RECIPE, make_drawer_id_from_chunk
|
||||
|
||||
logger = logging.getLogger("mempalace_mcp")
|
||||
|
||||
@@ -871,8 +873,14 @@ def _extract_entities_for_metadata(content: str) -> str:
|
||||
|
||||
coca_filter = _get_coca_filter()
|
||||
window = content[:_ENTITY_EXTRACT_WINDOW]
|
||||
words = _candidate_entity_words(window)
|
||||
freq: dict = {}
|
||||
# Tier 3 linguistics cleanup — known-systems compound pre-pass. Detects
|
||||
# multi-word product names atomically and masks them from the window so
|
||||
# the single-word extraction below doesn't decompose them into their
|
||||
# constituent tokens (which would then either get COCA-filtered or
|
||||
# appear as wrongly-attributed standalone entities).
|
||||
working_window, compound_counts = _apply_known_systems_prepass(window)
|
||||
words = _candidate_entity_words(working_window)
|
||||
freq: dict = dict(compound_counts)
|
||||
for w in words:
|
||||
if w in _ENTITY_STOPLIST:
|
||||
continue
|
||||
@@ -1219,6 +1227,7 @@ def _build_drawer_metadata(
|
||||
"added_by": agent,
|
||||
"filed_at": datetime.now().isoformat(),
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
"id_recipe": ID_RECIPE,
|
||||
}
|
||||
if source_mtime is not None:
|
||||
metadata["source_mtime"] = source_mtime
|
||||
@@ -1244,7 +1253,7 @@ def add_drawer(
|
||||
miner uses ``_build_drawer_metadata`` + a batched ``collection.upsert``
|
||||
to amortize the embedding model's forward-pass cost across chunks.
|
||||
"""
|
||||
drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(chunk_index)).encode()).hexdigest()[:24]}"
|
||||
drawer_id = make_drawer_id_from_chunk(wing, room, source_file, chunk_index)
|
||||
try:
|
||||
source_mtime = os.path.getmtime(source_file)
|
||||
except OSError:
|
||||
@@ -1377,7 +1386,7 @@ def process_file(
|
||||
batch_ids: list = []
|
||||
batch_metas: list = []
|
||||
for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]:
|
||||
drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}"
|
||||
drawer_id = make_drawer_id_from_chunk(wing, room, source_file, chunk["chunk_index"])
|
||||
batch_docs.append(chunk["content"])
|
||||
batch_ids.append(drawer_id)
|
||||
batch_metas.append(
|
||||
@@ -1394,6 +1403,7 @@ def process_file(
|
||||
content_date=file_content_date,
|
||||
)
|
||||
)
|
||||
assert_no_collisions(list(zip(batch_ids, batch_metas)), collection)
|
||||
collection.upsert(
|
||||
documents=batch_docs,
|
||||
ids=batch_ids,
|
||||
@@ -1407,8 +1417,7 @@ def process_file(
|
||||
# fully replace the prior closets, not append to them.
|
||||
if closets_col and drawers_added > 0:
|
||||
drawer_ids = [
|
||||
f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(c['chunk_index'])).encode()).hexdigest()[:24]}"
|
||||
for c in chunks
|
||||
make_drawer_id_from_chunk(wing, room, source_file, c["chunk_index"]) for c in chunks
|
||||
]
|
||||
# Pass drawer_metas so build_closet_lines can emit the Tier 6a
|
||||
# 4-segment pointer (``topic|entities|YYYY-MM-DD:Lstart-Lend|→ids``)
|
||||
@@ -1905,7 +1914,23 @@ def _compute_entity_tunnels_for_wing(wing: str) -> int:
|
||||
|
||||
|
||||
def status(palace_path: str):
|
||||
"""Show what's been filed in the palace."""
|
||||
"""Show what's been filed in the palace.
|
||||
|
||||
Tallies drawers by wing/room directly from ``chroma.sqlite3`` so a routine
|
||||
status check never cold-loads the HNSW vector index — a load that costs
|
||||
tens of seconds of CPU per call on large palaces (#1681). Falls back to the
|
||||
ChromaDB client path when the sqlite read is unavailable (missing DB,
|
||||
un-bootstrapped collection, or an unexpected schema); the fallback also
|
||||
emits the state-specific guidance for absent/empty palaces.
|
||||
"""
|
||||
from .backends.chroma import _sqlite_wing_room_counts
|
||||
|
||||
counts = _sqlite_wing_room_counts(palace_path, "mempalace_drawers")
|
||||
if counts is not None:
|
||||
total, wing_rooms = counts
|
||||
_print_status(total, wing_rooms)
|
||||
return
|
||||
|
||||
col = _open_collection_or_explain(palace_path)
|
||||
if col is None:
|
||||
return
|
||||
@@ -1926,6 +1951,11 @@ def status(palace_path: str):
|
||||
wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1
|
||||
offset += len(batch)
|
||||
|
||||
_print_status(total, wing_rooms)
|
||||
|
||||
|
||||
def _print_status(total: int, wing_rooms: dict[str, dict[str, int]]) -> None:
|
||||
"""Render the wing/room histogram shared by both status code paths."""
|
||||
print(f"\n{'=' * 55}")
|
||||
print(f" MemPalace Status — {total} drawers")
|
||||
print(f"{'=' * 55}\n")
|
||||
|
||||
@@ -122,7 +122,7 @@ def normalize(filepath: str) -> str:
|
||||
if file_size > 500 * 1024 * 1024: # 500 MB safety limit
|
||||
raise IOError(f"File too large ({file_size // (1024 * 1024)} MB): {filepath}")
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
|
||||
with open(filepath, "r", encoding="utf-8-sig", errors="replace") as f:
|
||||
content = f.read()
|
||||
except OSError as e:
|
||||
raise IOError(f"Could not read {filepath}: {e}") from e
|
||||
@@ -523,6 +523,8 @@ def _format_tool_use(block: dict) -> str:
|
||||
"""Format a tool_use block into a human-readable one-liner."""
|
||||
name = block.get("name", "Unknown")
|
||||
inp = block.get("input", {})
|
||||
if isinstance(inp, list):
|
||||
inp = {}
|
||||
|
||||
if name == "Bash":
|
||||
cmd = inp.get("command", "")
|
||||
|
||||
+214
-60
@@ -13,9 +13,20 @@ import sys
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
from .backends import BackendClosedError, CollectionNotInitializedError, PalaceNotFoundError
|
||||
from .backends.chroma import ChromaBackend
|
||||
from .entity_detector import _get_coca_filter
|
||||
from .backends import (
|
||||
BackendClosedError,
|
||||
BackendMismatchError,
|
||||
CollectionNotInitializedError,
|
||||
PalaceNotFoundError,
|
||||
PalaceRef,
|
||||
detect_backend_for_path,
|
||||
detect_backends_for_path,
|
||||
get_backend,
|
||||
get_backend_class,
|
||||
resolve_backend_for_palace,
|
||||
)
|
||||
from .backends.embedding_wrapper import EmbeddingCollection
|
||||
from .entity_detector import _apply_known_systems_prepass, _get_coca_filter
|
||||
|
||||
logger = logging.getLogger("mempalace_mcp")
|
||||
|
||||
@@ -45,7 +56,8 @@ SKIP_DIRS = {
|
||||
"target",
|
||||
}
|
||||
|
||||
_DEFAULT_BACKEND = ChromaBackend()
|
||||
_DEFAULT_BACKEND = get_backend("chroma")
|
||||
_EXPLICIT_BACKEND_ENV = "MEMPALACE_BACKEND_EXPLICIT"
|
||||
|
||||
# Schema version for drawer normalization. Bump when the normalization
|
||||
# pipeline changes in a way that existing drawers should be rebuilt to pick up
|
||||
@@ -62,22 +74,122 @@ def get_collection(
|
||||
palace_path: str,
|
||||
collection_name: Optional[str] = None,
|
||||
create: bool = True,
|
||||
backend: Optional[str] = None,
|
||||
):
|
||||
"""Get the palace collection through the backend layer."""
|
||||
if collection_name is None:
|
||||
from .config import get_configured_collection_name
|
||||
|
||||
collection_name = get_configured_collection_name()
|
||||
return _DEFAULT_BACKEND.get_collection(
|
||||
backend_obj = get_backend_for_palace(palace_path, explicit=backend)
|
||||
palace_ref = PalaceRef(id=palace_path, local_path=palace_path)
|
||||
try:
|
||||
collection = backend_obj.get_collection(
|
||||
palace=palace_ref,
|
||||
collection_name=collection_name,
|
||||
create=create,
|
||||
)
|
||||
except TypeError as exc:
|
||||
if "unexpected keyword argument 'palace'" not in str(exc):
|
||||
raise
|
||||
collection = backend_obj.get_collection(
|
||||
palace_path,
|
||||
collection_name=collection_name,
|
||||
create=create,
|
||||
)
|
||||
if "requires_explicit_embeddings" in getattr(backend_obj, "capabilities", frozenset()):
|
||||
return EmbeddingCollection(collection)
|
||||
return collection
|
||||
|
||||
|
||||
def get_closets_collection(
|
||||
palace_path: str,
|
||||
create: bool = True,
|
||||
backend: Optional[str] = None,
|
||||
):
|
||||
"""Get the closets collection — the searchable index layer."""
|
||||
return get_collection(
|
||||
palace_path,
|
||||
collection_name=collection_name,
|
||||
collection_name="mempalace_closets",
|
||||
create=create,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
|
||||
def get_closets_collection(palace_path: str, create: bool = True):
|
||||
"""Get the closets collection — the searchable index layer."""
|
||||
return get_collection(palace_path, collection_name="mempalace_closets", create=create)
|
||||
def _config_backend_value(palace_path: str) -> Optional[str]:
|
||||
try:
|
||||
from .config import MempalaceConfig
|
||||
|
||||
cfg = MempalaceConfig()
|
||||
cfg_palace = os.path.abspath(os.path.expanduser(cfg.palace_path))
|
||||
target_palace = os.path.abspath(os.path.expanduser(palace_path))
|
||||
if cfg_palace != target_palace:
|
||||
return None
|
||||
value = cfg._file_config.get("backend")
|
||||
return str(value).strip().lower() if value else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _env_backend_value() -> Optional[str]:
|
||||
value = os.environ.get("MEMPALACE_BACKEND")
|
||||
return value.strip().lower() if value else None
|
||||
|
||||
|
||||
def resolve_backend_name(palace_path: str, explicit: Optional[str] = None) -> str:
|
||||
"""Resolve and validate the selected backend for ``palace_path``.
|
||||
|
||||
Public resolution order:
|
||||
|
||||
1. Explicit CLI/MCP flag or direct ``get_collection(..., backend=...)``.
|
||||
2. ``backend`` in ``~/.mempalace/config.json``.
|
||||
3. ``MEMPALACE_BACKEND``.
|
||||
4. Detected existing palace artifacts.
|
||||
5. ``chroma``.
|
||||
|
||||
If artifacts for a different backend are already present, raise
|
||||
``BackendMismatchError`` so normal write paths cannot silently mix storage
|
||||
formats in one palace directory.
|
||||
"""
|
||||
explicit = explicit or os.environ.get(_EXPLICIT_BACKEND_ENV)
|
||||
selected = resolve_backend_for_palace(
|
||||
explicit=explicit.strip().lower() if explicit else None,
|
||||
config_value=_config_backend_value(palace_path),
|
||||
env_value=_env_backend_value(),
|
||||
palace_path=palace_path,
|
||||
default="chroma",
|
||||
)
|
||||
get_backend_class(selected)
|
||||
detected_backends = detect_backends_for_path(palace_path)
|
||||
if len(detected_backends) > 1:
|
||||
raise BackendMismatchError(
|
||||
f"palace at {palace_path!r} contains multiple backend artifacts: "
|
||||
f"{', '.join(detected_backends)}"
|
||||
)
|
||||
detected = detected_backends[0] if detected_backends else None
|
||||
if detected and detected != selected:
|
||||
raise BackendMismatchError(
|
||||
f"palace at {palace_path!r} contains {detected!r} backend artifacts, "
|
||||
f"but {selected!r} was selected"
|
||||
)
|
||||
return selected
|
||||
|
||||
|
||||
def get_backend_for_palace(palace_path: str, explicit: Optional[str] = None):
|
||||
"""Return the resolved backend instance for ``palace_path``."""
|
||||
return get_backend(resolve_backend_name(palace_path, explicit=explicit))
|
||||
|
||||
|
||||
def _backend_artifact_label(backend_name: Optional[str]) -> str:
|
||||
if backend_name == "chroma":
|
||||
return "chroma.sqlite3"
|
||||
if backend_name == "qdrant":
|
||||
return "qdrant_backend.json"
|
||||
if backend_name == "pgvector":
|
||||
return "pgvector_backend.json"
|
||||
if backend_name == "sqlite_exact":
|
||||
return "sqlite_exact.sqlite3"
|
||||
return "backend database"
|
||||
|
||||
|
||||
def _open_collection_or_explain(
|
||||
@@ -85,6 +197,7 @@ def _open_collection_or_explain(
|
||||
*,
|
||||
collection_name: Optional[str] = None,
|
||||
out=None,
|
||||
opener=None,
|
||||
):
|
||||
"""Open the palace collection or print a state-specific message and return ``None``.
|
||||
|
||||
@@ -101,11 +214,11 @@ def _open_collection_or_explain(
|
||||
first when the vector path is disabled (see PR #831 / issue #830).
|
||||
|
||||
State A: palace dir is absent.
|
||||
State B: dir is present but ``chroma.sqlite3`` is absent. The helper
|
||||
short-circuits to a message before reaching the backend, because
|
||||
``chromadb.PersistentClient`` lazily creates the DB file on first
|
||||
open — calling the backend on this state would silently mutate
|
||||
the filesystem for what should be a read-only inspection.
|
||||
State B: dir is present but no backend database artifact is present.
|
||||
The helper short-circuits to a message before reaching the backend,
|
||||
because some backends lazily create their DB file on first open —
|
||||
calling the backend on this state would silently mutate the filesystem
|
||||
for what should be a read-only inspection.
|
||||
State C: DB is present but the ``mempalace_drawers`` collection has
|
||||
never been bootstrapped (``init`` ran, ``mine`` has not).
|
||||
State D: healthy — returns the opened collection.
|
||||
@@ -116,17 +229,41 @@ def _open_collection_or_explain(
|
||||
callable (e.g. a repair progress emitter) to route messages through it.
|
||||
"""
|
||||
emit = out if out is not None else print
|
||||
open_collection = opener or get_collection
|
||||
|
||||
if not os.path.isdir(palace_path):
|
||||
emit(f"\n No palace found at {palace_path}")
|
||||
emit(" Run: mempalace init <dir> then mempalace mine <dir>")
|
||||
return None
|
||||
if not os.path.isfile(os.path.join(palace_path, "chroma.sqlite3")):
|
||||
emit(f"\n Palace dir at {palace_path} exists but has no chroma.sqlite3 yet.")
|
||||
try:
|
||||
backend_name = resolve_backend_name(palace_path)
|
||||
except BackendMismatchError as e:
|
||||
emit(f"\n Backend mismatch at {palace_path}: {e}")
|
||||
emit(" Select the matching backend or use a fresh palace directory.")
|
||||
return None
|
||||
except KeyError as e:
|
||||
# Unknown backend name (e.g. a typo in MEMPALACE_BACKEND/--backend):
|
||||
# resolve_backend_name -> get_backend_class raises KeyError carrying the
|
||||
# available-backend list. Surface it as a CLI state message rather than
|
||||
# letting it escape as a stack trace.
|
||||
emit(f"\n Unknown backend selected for {palace_path}: {e.args[0] if e.args else e}")
|
||||
emit(" Set --backend or MEMPALACE_BACKEND to a registered backend.")
|
||||
return None
|
||||
detected = detect_backend_for_path(palace_path)
|
||||
if detected is None:
|
||||
emit(
|
||||
f"\n Palace dir at {palace_path} exists but has no "
|
||||
f"{_backend_artifact_label(backend_name)} yet."
|
||||
)
|
||||
emit(" Run: mempalace mine <dir>")
|
||||
return None
|
||||
try:
|
||||
return get_collection(palace_path, collection_name=collection_name, create=False)
|
||||
return open_collection(
|
||||
palace_path,
|
||||
collection_name=collection_name,
|
||||
create=False,
|
||||
backend=backend_name,
|
||||
)
|
||||
except CollectionNotInitializedError:
|
||||
emit(f"\n Palace at {palace_path} is initialized but empty (no drawers yet).")
|
||||
emit(" Run: mempalace mine <dir>")
|
||||
@@ -135,6 +272,10 @@ def _open_collection_or_explain(
|
||||
emit(f"\n No palace found at {palace_path}")
|
||||
emit(" Run: mempalace init <dir> then mempalace mine <dir>")
|
||||
return None
|
||||
except BackendMismatchError as e:
|
||||
emit(f"\n Backend mismatch at {palace_path}: {e}")
|
||||
emit(" Select the matching backend or use a fresh palace directory.")
|
||||
return None
|
||||
except BackendClosedError:
|
||||
# Surface this as a programmer error, not a palace-state UX message:
|
||||
# a closed backend means the caller violated the backend lifecycle,
|
||||
@@ -265,9 +406,15 @@ def build_closet_lines(source_file, drawer_ids, content, wing, room, drawer_meta
|
||||
|
||||
# Extract proper nouns (2+ occurrences). Uses i18n-aware patterns so
|
||||
# non-Latin names (Cyrillic, accented Latin, etc.) are also detected.
|
||||
# Tier 3 linguistics cleanup — known-systems compound pre-pass. Detects
|
||||
# multi-word product names ("Claude Code", "GitHub Copilot", …) atomically
|
||||
# and masks them out of the working window so the single-word extraction
|
||||
# below doesn't decompose them.
|
||||
working_window, compound_counts = _apply_known_systems_prepass(window)
|
||||
|
||||
coca_filter = _get_coca_filter()
|
||||
words = _candidate_entity_words(window)
|
||||
word_freq = {}
|
||||
words = _candidate_entity_words(working_window)
|
||||
word_freq: dict = dict(compound_counts)
|
||||
for w in words:
|
||||
if w in _ENTITY_STOPLIST:
|
||||
continue
|
||||
@@ -478,6 +625,9 @@ def _validate_palace_fts5_after_mine(palace_path: str) -> None:
|
||||
operator sees the same recovery banner regardless of which command surfaces
|
||||
the bug.
|
||||
"""
|
||||
if resolve_backend_name(palace_path) != "chroma":
|
||||
return
|
||||
|
||||
# Defer-import: keeps the repair module graph out of mine's hot import path.
|
||||
from .repair import _close_chroma_handles, sqlite_integrity_errors
|
||||
|
||||
@@ -735,47 +885,51 @@ def file_already_mined(
|
||||
treated as exchange-mode drawers.
|
||||
"""
|
||||
try:
|
||||
stored_meta = None
|
||||
if extract_mode is None:
|
||||
results = collection.get(where={"source_file": source_file}, limit=1)
|
||||
if not results.get("ids"):
|
||||
return False
|
||||
stored_meta = results.get("metadatas", [{}])[0] or {}
|
||||
else:
|
||||
offset = 0
|
||||
while True:
|
||||
results = collection.get(
|
||||
where={"source_file": source_file},
|
||||
limit=1000,
|
||||
offset=offset,
|
||||
include=["metadatas"],
|
||||
)
|
||||
ids = results.get("ids") or []
|
||||
metadatas = results.get("metadatas") or []
|
||||
stored_meta = next(
|
||||
(
|
||||
meta or {}
|
||||
for meta in metadatas
|
||||
if _metadata_matches_extract_mode(meta or {}, extract_mode)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if stored_meta is not None or not ids:
|
||||
break
|
||||
offset += len(ids)
|
||||
if stored_meta is None:
|
||||
return False
|
||||
# Pre-v2 drawers have no version field — treat them as stale.
|
||||
stored_version = stored_meta.get("normalize_version", 1)
|
||||
if stored_version < NORMALIZE_VERSION:
|
||||
return False
|
||||
if check_mtime:
|
||||
stored_mtime = stored_meta.get("source_mtime")
|
||||
if stored_mtime is None:
|
||||
return False
|
||||
current_mtime = os.path.getmtime(source_file)
|
||||
return abs(float(stored_mtime) - current_mtime) < 0.001
|
||||
return True
|
||||
# Under the additive-mining model, a single ``source_file`` can have
|
||||
# multiple ``parent_drawer_id`` groups in the palace — one per
|
||||
# mining pass — each with its own stored ``source_mtime`` and
|
||||
# ``normalize_version``. The function must return True if ANY stored
|
||||
# group is current (matching version + matching mtime when checked),
|
||||
# because ChromaDB's ``get(..., limit=1)`` has undefined ordering
|
||||
# across multiple matching rows: a ``limit=1`` shortcut picks
|
||||
# whichever row ChromaDB orders first and only checks that one,
|
||||
# causing spurious re-mines whenever the stale group is returned.
|
||||
# Iterating via the same paginated pattern used in the
|
||||
# extract_mode-is-set branch lets the function short-circuit on the
|
||||
# first matching group regardless of ordering.
|
||||
current_mtime = os.path.getmtime(source_file) if check_mtime else None
|
||||
offset = 0
|
||||
while True:
|
||||
results = collection.get(
|
||||
where={"source_file": source_file},
|
||||
limit=1000,
|
||||
offset=offset,
|
||||
include=["metadatas"],
|
||||
)
|
||||
ids = results.get("ids") or []
|
||||
metadatas = results.get("metadatas") or []
|
||||
for meta in metadatas:
|
||||
meta = meta or {}
|
||||
# extract_mode scoping (was the existing ``else`` branch):
|
||||
if extract_mode is not None and not _metadata_matches_extract_mode(
|
||||
meta, extract_mode
|
||||
):
|
||||
continue
|
||||
# Pre-v2 drawers have no version field — treat them as stale.
|
||||
stored_version = meta.get("normalize_version", 1)
|
||||
if stored_version < NORMALIZE_VERSION:
|
||||
continue
|
||||
if not check_mtime:
|
||||
return True
|
||||
stored_mtime = meta.get("source_mtime")
|
||||
if stored_mtime is None:
|
||||
continue
|
||||
if abs(float(stored_mtime) - current_mtime) < 0.001:
|
||||
return True
|
||||
if not ids:
|
||||
break
|
||||
offset += len(ids)
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
+235
-79
@@ -16,8 +16,19 @@ import re
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from .backends import CollectionNotInitializedError, PalaceNotFoundError
|
||||
from .palace import get_closets_collection, get_collection
|
||||
from .backends import (
|
||||
BackendError,
|
||||
BackendMismatchError,
|
||||
CollectionNotInitializedError,
|
||||
PalaceNotFoundError,
|
||||
UnsupportedCapabilityError,
|
||||
)
|
||||
from .palace import (
|
||||
_open_collection_or_explain,
|
||||
get_closets_collection,
|
||||
get_collection,
|
||||
resolve_backend_name,
|
||||
)
|
||||
|
||||
# Closet pointer line format: "topic|entities|→drawer_id_a,drawer_id_b"
|
||||
# Multiple lines may join with newlines inside one closet document.
|
||||
@@ -296,32 +307,11 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
|
||||
Search the palace. Returns verbatim drawer content.
|
||||
Optionally filter by wing (project) or room (aspect).
|
||||
"""
|
||||
# Filesystem-first checks distinguish State A / State B before reaching
|
||||
# chromadb. PersistentClient lazily creates chroma.sqlite3 on first open
|
||||
# of an empty palace dir, so without these checks State B collapses into
|
||||
# the "initialized but empty" State C message and mutates the dir as a
|
||||
# side effect of a read-only search call (#1498).
|
||||
if not os.path.isdir(palace_path):
|
||||
print(f"\n No palace found at {palace_path}")
|
||||
print(" Run: mempalace init <dir> then mempalace mine <dir>")
|
||||
raise SearchError(f"No palace found at {palace_path}")
|
||||
if not os.path.isfile(os.path.join(palace_path, "chroma.sqlite3")):
|
||||
print(f"\n Palace dir at {palace_path} exists but has no chroma.sqlite3 yet.")
|
||||
print(" Run: mempalace mine <dir>")
|
||||
col = _open_collection_or_explain(palace_path, opener=get_collection)
|
||||
if col is None:
|
||||
if not os.path.isdir(palace_path):
|
||||
raise SearchError(f"No palace found at {palace_path}")
|
||||
raise SearchError(f"No palace database at {palace_path}")
|
||||
try:
|
||||
col = get_collection(palace_path, create=False)
|
||||
except CollectionNotInitializedError as e:
|
||||
# State C from #1498: palace initialized but never mined.
|
||||
print(f"\n Palace at {palace_path} is initialized but empty (no drawers yet).")
|
||||
print(" Run: mempalace mine <dir>")
|
||||
raise SearchError(f"Palace at {palace_path} is initialized but empty") from e
|
||||
except PalaceNotFoundError as e:
|
||||
# Backend filesystem-race fallback: dir was deleted between our
|
||||
# check above and the backend call. Same message as State A.
|
||||
print(f"\n No palace found at {palace_path}")
|
||||
print(" Run: mempalace init <dir> then mempalace mine <dir>")
|
||||
raise SearchError(f"No palace found at {palace_path}") from e
|
||||
|
||||
# Alert the user if this palace predates hnsw:space=cosine being set on
|
||||
# creation — their similarity scores will be junk until they run repair.
|
||||
@@ -636,14 +626,14 @@ def _bm25_only_via_sqlite(
|
||||
|
||||
def _merge_bm25_union_candidates(
|
||||
hits: list,
|
||||
drawers_col,
|
||||
query: str,
|
||||
palace_path: str,
|
||||
wing: str,
|
||||
room: str,
|
||||
n_results: int,
|
||||
max_distance: float = 0.0,
|
||||
) -> None:
|
||||
"""Append top-K BM25-only candidates from sqlite into ``hits`` in place.
|
||||
"""Append top-K backend lexical candidates into ``hits`` in place.
|
||||
|
||||
Used by ``search_memories(..., candidate_strategy="union")`` to widen
|
||||
the rerank pool's *source* (not just its size) — vector-only candidate
|
||||
@@ -668,19 +658,41 @@ def _merge_bm25_union_candidates(
|
||||
if max_distance > 0.0:
|
||||
return
|
||||
|
||||
where = build_where_filter(wing, room)
|
||||
try:
|
||||
bm25_extra = _bm25_only_via_sqlite(
|
||||
query,
|
||||
palace_path,
|
||||
wing=wing,
|
||||
room=room,
|
||||
lexical = drawers_col.lexical_search(
|
||||
query=query,
|
||||
n_results=n_results * 3,
|
||||
_include_internal=True,
|
||||
).get("results", [])
|
||||
where=where or None,
|
||||
)
|
||||
except UnsupportedCapabilityError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.debug("candidate_strategy=union: BM25 fetch failed", exc_info=True)
|
||||
logger.debug("candidate_strategy=union: lexical fetch failed", exc_info=True)
|
||||
return
|
||||
|
||||
bm25_extra = []
|
||||
for hit in lexical.hits:
|
||||
meta = hit.metadata or {}
|
||||
full_source = meta.get("source_file", "") or ""
|
||||
bm25_extra.append(
|
||||
{
|
||||
"text": hit.document or "",
|
||||
"wing": meta.get("wing", "unknown"),
|
||||
"room": meta.get("room", "unknown"),
|
||||
"source_file": Path(full_source).name if full_source else "?",
|
||||
"created_at": meta.get("filed_at", "unknown"),
|
||||
"similarity": None,
|
||||
"distance": None,
|
||||
"effective_distance": None,
|
||||
"closet_boost": 0.0,
|
||||
"matched_via": "bm25_backend",
|
||||
"bm25_score": round(float(hit.score), 3),
|
||||
"_source_file_full": full_source,
|
||||
"_chunk_index": meta.get("chunk_index"),
|
||||
}
|
||||
)
|
||||
|
||||
def _dedup_key(entry: dict):
|
||||
full = entry.get("_source_file_full")
|
||||
ci = entry.get("_chunk_index")
|
||||
@@ -728,8 +740,8 @@ def _validate_candidate_strategy(strategy: str) -> None:
|
||||
def _apply_candidate_strategy(
|
||||
strategy: str,
|
||||
hits: list,
|
||||
drawers_col,
|
||||
query: str,
|
||||
palace_path: str,
|
||||
wing: str,
|
||||
room: str,
|
||||
n_results: int,
|
||||
@@ -742,7 +754,163 @@ def _apply_candidate_strategy(
|
||||
"""
|
||||
merger = _CANDIDATE_MERGERS[strategy]
|
||||
if merger is not None:
|
||||
merger(hits, query, palace_path, wing, room, n_results, max_distance=max_distance)
|
||||
merger(hits, drawers_col, query, wing, room, n_results, max_distance=max_distance)
|
||||
|
||||
|
||||
def _finalize_candidate_hits(
|
||||
*,
|
||||
candidate_strategy: str,
|
||||
hits: list,
|
||||
drawers_col,
|
||||
query: str,
|
||||
wing: str,
|
||||
room: str,
|
||||
n_results: int,
|
||||
max_distance: float,
|
||||
) -> tuple:
|
||||
try:
|
||||
_apply_candidate_strategy(
|
||||
candidate_strategy,
|
||||
hits,
|
||||
drawers_col,
|
||||
query,
|
||||
wing,
|
||||
room,
|
||||
n_results,
|
||||
max_distance=max_distance,
|
||||
)
|
||||
except UnsupportedCapabilityError:
|
||||
return [], {
|
||||
"error": "candidate_strategy='union' requires a backend with lexical_search support",
|
||||
"unsupported_capability": "supports_lexical_search",
|
||||
"hint": "Use candidate_strategy='vector' or select a backend that supports lexical search.",
|
||||
}
|
||||
|
||||
hits = _hybrid_rank(hits, query)[:n_results]
|
||||
for h in hits:
|
||||
h.pop("_sort_key", None)
|
||||
h.pop("_source_file_full", None)
|
||||
h.pop("_chunk_index", None)
|
||||
return hits, None
|
||||
|
||||
|
||||
def _backend_mismatch_result(error: BackendMismatchError) -> dict:
|
||||
return {
|
||||
"error": "Backend mismatch",
|
||||
"details": str(error),
|
||||
"hint": "Select the matching backend or use a fresh palace directory.",
|
||||
}
|
||||
|
||||
|
||||
def _unknown_backend_result(error: KeyError) -> dict:
|
||||
return {
|
||||
"error": "Unknown backend",
|
||||
"details": str(error),
|
||||
"hint": "Check MEMPALACE_BACKEND or the configured backend name.",
|
||||
}
|
||||
|
||||
|
||||
def _vector_disabled_search(
|
||||
*,
|
||||
query: str,
|
||||
palace_path: str,
|
||||
wing: str,
|
||||
room: str,
|
||||
n_results: int,
|
||||
collection_name: str,
|
||||
) -> dict:
|
||||
try:
|
||||
backend_name = resolve_backend_name(palace_path)
|
||||
except BackendMismatchError as e:
|
||||
return _backend_mismatch_result(e)
|
||||
except KeyError as e:
|
||||
return _unknown_backend_result(e)
|
||||
if backend_name != "chroma":
|
||||
return {
|
||||
"error": "vector_disabled fallback is Chroma-only",
|
||||
"unsupported_capability": "chroma_hnsw_fallback",
|
||||
"backend": backend_name,
|
||||
"hint": "Disable vector_disabled for non-Chroma backends.",
|
||||
}
|
||||
return _bm25_only_via_sqlite(
|
||||
query,
|
||||
palace_path,
|
||||
wing=wing,
|
||||
room=room,
|
||||
n_results=n_results,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
|
||||
def _open_search_collection(palace_path: str, collection_name: str):
|
||||
try:
|
||||
return get_collection(palace_path, collection_name=collection_name, create=False), None
|
||||
except BackendMismatchError as e:
|
||||
return None, _backend_mismatch_result(e)
|
||||
except KeyError as e:
|
||||
return None, _unknown_backend_result(e)
|
||||
except (CollectionNotInitializedError, PalaceNotFoundError) as e:
|
||||
logger.error("No palace found at %s: %s", palace_path, e)
|
||||
return None, {
|
||||
"error": "No palace found",
|
||||
"hint": "Run: mempalace init <dir> && mempalace mine <dir>",
|
||||
}
|
||||
except BackendError as e:
|
||||
logger.error("Backend error opening palace at %s: %s", palace_path, e)
|
||||
return None, {
|
||||
"error": "Backend error",
|
||||
"details": str(e),
|
||||
"hint": "Check the selected backend configuration and availability.",
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error("No palace found at %s: %s", palace_path, e)
|
||||
return None, {
|
||||
"error": "No palace found",
|
||||
"hint": "Run: mempalace init <dir> && mempalace mine <dir>",
|
||||
}
|
||||
|
||||
|
||||
def _query_drawers_with_filter_fallback(drawers_col, dkwargs, query, n_results, wing, room):
|
||||
"""Run the filtered drawer query, falling back to an unfiltered query plus a
|
||||
Python-side post-filter when ChromaDB raises on the filtered query.
|
||||
|
||||
A ChromaDB HNSW/SQLite index mismatch makes filtered queries fail with
|
||||
"Error finding id" even when unfiltered search works fine — it happens when
|
||||
drawers are ingested via two different paths (e.g. bulk import vs MCP tool
|
||||
calls), leaving the vector index inconsistent with the metadata store. We
|
||||
retry unfiltered (over-fetching) and re-apply the wing/room filter in Python.
|
||||
See #1245 / #1035.
|
||||
"""
|
||||
where = dkwargs.get("where")
|
||||
try:
|
||||
return drawers_col.query(**dkwargs)
|
||||
except Exception as filter_err:
|
||||
if not where:
|
||||
raise
|
||||
logger.warning(
|
||||
"Filtered search failed (%s); falling back to unfiltered + post-filter",
|
||||
filter_err,
|
||||
)
|
||||
raw = drawers_col.query(
|
||||
query_texts=[query],
|
||||
n_results=min(n_results * 15, 500),
|
||||
include=["documents", "metadatas", "distances"],
|
||||
)
|
||||
fdocs, fmetas, fdists = [], [], []
|
||||
for doc, meta, dist in zip(
|
||||
_first_or_empty(raw, "documents"),
|
||||
_first_or_empty(raw, "metadatas"),
|
||||
_first_or_empty(raw, "distances"),
|
||||
):
|
||||
meta = meta or {}
|
||||
if wing and meta.get("wing") != wing:
|
||||
continue
|
||||
if room and meta.get("room") != room:
|
||||
continue
|
||||
fdocs.append(doc)
|
||||
fmetas.append(meta)
|
||||
fdists.append(dist)
|
||||
return {"documents": [fdocs], "metadatas": [fmetas], "distances": [fdists]}
|
||||
|
||||
|
||||
def search_memories(
|
||||
@@ -780,14 +948,12 @@ def search_memories(
|
||||
``n_results * 3`` rows from the vector index are the rerank pool.
|
||||
Cheap; works well when query and target docs agree in the
|
||||
embedding space.
|
||||
* ``"union"`` — also pull top ``n_results * 3`` BM25 candidates
|
||||
from the sqlite FTS5 index and merge them into the rerank pool
|
||||
(deduped by source_file). Catches docs with strong BM25 signal
|
||||
that are vector-distant from the query (e.g. terminology guides
|
||||
looked up by narrative-shaped queries; policy clauses surfaced
|
||||
by scenario descriptions). Adds one sqlite open + FTS5 MATCH
|
||||
per query; perf cost is small but unmeasured at corpus scale.
|
||||
Opt in until the cost is characterized.
|
||||
* ``"union"`` — also pull top ``n_results * 3`` lexical candidates
|
||||
through the backend's ``lexical_search`` capability and merge
|
||||
them into the rerank pool (deduped by source_file). Catches docs
|
||||
with strong BM25 signal that are vector-distant from the query.
|
||||
Perf depends on the selected backend; opt in until the cost is
|
||||
characterized.
|
||||
|
||||
When ``max_distance > 0.0`` is also set, BM25-only candidates
|
||||
are skipped — they have no vector distance and would silently
|
||||
@@ -799,23 +965,18 @@ def search_memories(
|
||||
_validate_candidate_strategy(candidate_strategy)
|
||||
|
||||
if vector_disabled:
|
||||
return _bm25_only_via_sqlite(
|
||||
query,
|
||||
palace_path,
|
||||
return _vector_disabled_search(
|
||||
query=query,
|
||||
palace_path=palace_path,
|
||||
wing=wing,
|
||||
room=room,
|
||||
n_results=n_results,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
try:
|
||||
drawers_col = get_collection(palace_path, collection_name=collection_name, create=False)
|
||||
except Exception as e:
|
||||
logger.error("No palace found at %s: %s", palace_path, e)
|
||||
return {
|
||||
"error": "No palace found",
|
||||
"hint": "Run: mempalace init <dir> && mempalace mine <dir>",
|
||||
}
|
||||
drawers_col, open_error = _open_search_collection(palace_path, collection_name)
|
||||
if open_error:
|
||||
return open_error
|
||||
|
||||
where = build_where_filter(wing, room)
|
||||
|
||||
@@ -834,7 +995,9 @@ def search_memories(
|
||||
}
|
||||
if where:
|
||||
dkwargs["where"] = where
|
||||
drawer_results = drawers_col.query(**dkwargs)
|
||||
drawer_results = _query_drawers_with_filter_fallback(
|
||||
drawers_col, dkwargs, query, n_results, wing, room
|
||||
)
|
||||
except Exception as e:
|
||||
return {"error": f"Search error: {e}"}
|
||||
|
||||
@@ -985,31 +1148,24 @@ def search_memories(
|
||||
|
||||
# Candidate strategy hook: optionally widen the rerank pool's *source*
|
||||
# before ranking. Default ("vector") is a no-op; "union" merges top-K
|
||||
# BM25 candidates from sqlite. See `_apply_candidate_strategy`.
|
||||
# backend lexical candidates. See `_apply_candidate_strategy`.
|
||||
# ``max_distance`` is forwarded so union mode can refuse to inject
|
||||
# BM25-only (distance=None) candidates that would silently bypass the
|
||||
# caller's strict distance threshold.
|
||||
_apply_candidate_strategy(
|
||||
candidate_strategy,
|
||||
hits,
|
||||
query,
|
||||
palace_path,
|
||||
wing,
|
||||
room,
|
||||
n_results,
|
||||
# The helper also runs the final BM25 hybrid re-rank and strips internal
|
||||
# dedup fields before returning.
|
||||
hits, strategy_error = _finalize_candidate_hits(
|
||||
candidate_strategy=candidate_strategy,
|
||||
hits=hits,
|
||||
drawers_col=drawers_col,
|
||||
query=query,
|
||||
wing=wing,
|
||||
room=room,
|
||||
n_results=n_results,
|
||||
max_distance=max_distance,
|
||||
)
|
||||
|
||||
# BM25 hybrid re-rank within the final candidate set, then trim back
|
||||
# to the requested size. Without the trim, ``candidate_strategy="union"``
|
||||
# would return up to 4× ``n_results`` (vector hits + BM25 union pool),
|
||||
# breaking the existing ``search_memories`` size contract that the MCP
|
||||
# ``limit`` parameter is built on.
|
||||
hits = _hybrid_rank(hits, query)[:n_results]
|
||||
for h in hits:
|
||||
h.pop("_sort_key", None)
|
||||
h.pop("_source_file_full", None)
|
||||
h.pop("_chunk_index", None)
|
||||
if strategy_error:
|
||||
return strategy_error
|
||||
|
||||
return {
|
||||
"query": query,
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""Single source of truth for the MemPalace package version."""
|
||||
|
||||
__version__ = "3.3.6"
|
||||
__version__ = "3.4.0"
|
||||
|
||||
+11
-3
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "mempalace"
|
||||
version = "3.3.6"
|
||||
version = "3.4.0"
|
||||
description = "Give your AI a memory — mine projects and conversations into a searchable palace. No API key required."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
@@ -60,6 +60,9 @@ mempalace-mcp = "mempalace.mcp_server:main"
|
||||
|
||||
[project.entry-points."mempalace.backends"]
|
||||
chroma = "mempalace.backends.chroma:ChromaBackend"
|
||||
pgvector = "mempalace.backends.pgvector:PgVectorBackend"
|
||||
qdrant = "mempalace.backends.qdrant:QdrantBackend"
|
||||
sqlite_exact = "mempalace.backends.sqlite_exact:SQLiteExactBackend"
|
||||
|
||||
# RFC 002 source-adapter entry-point group. Core publishes no first-party
|
||||
# adapters under this group yet; ``miner.py`` and ``convo_miner.py`` migrate
|
||||
@@ -71,7 +74,7 @@ chroma = "mempalace.backends.chroma:ChromaBackend"
|
||||
dev = [
|
||||
"pytest>=7.0",
|
||||
"pytest-cov>=4.0",
|
||||
"ruff==0.15.14",
|
||||
"ruff==0.15.15",
|
||||
"psutil>=5.9",
|
||||
# Property-based testing — generates hundreds of random inputs per
|
||||
# test to find counterexamples the hand-written positive tests miss.
|
||||
@@ -92,6 +95,11 @@ dev = [
|
||||
"mypy>=1.0",
|
||||
]
|
||||
spellcheck = ["autocorrect>=2.0"]
|
||||
# Opt-in Postgres + pgvector backend. Only the psycopg driver is needed on the
|
||||
# client; the server must have the `vector` extension available. Selected via
|
||||
# MEMPALACE_BACKEND=pgvector / --backend pgvector; never required for the
|
||||
# default (Chroma) install.
|
||||
pgvector = ["psycopg[binary]>=3.1"]
|
||||
# Hardware acceleration for the ONNX embedding model. Install exactly one:
|
||||
# pip install mempalace[gpu] — NVIDIA CUDA
|
||||
# pip install mempalace[dml] — DirectML (Windows AMD/Intel/NVIDIA)
|
||||
@@ -123,7 +131,7 @@ extract = [
|
||||
dev = [
|
||||
"pytest>=7.0",
|
||||
"pytest-cov>=4.0",
|
||||
"ruff==0.15.14",
|
||||
"ruff==0.15.15",
|
||||
"psutil>=5.9",
|
||||
"hypothesis>=6.0",
|
||||
"pre-commit>=3.0",
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Shared backend isolation conformance assertions (RFC 001 isolation contract).
|
||||
|
||||
Any backend's test module can import these to prove the isolation guarantees
|
||||
declared on :class:`mempalace.backends.PalaceRef`:
|
||||
|
||||
* per-``PalaceRef.id`` isolation — required of every backend;
|
||||
* per-``PalaceRef.namespace`` isolation — required of backends advertising the
|
||||
``supports_namespace_isolation`` capability.
|
||||
|
||||
This module is intentionally not a ``test_*`` file: it ships assertions, not
|
||||
test cases, so pytest does not collect it directly.
|
||||
"""
|
||||
|
||||
_PROBE_ID = "conformance-isolation-probe"
|
||||
_PROBE_DOC = "partition isolation probe document"
|
||||
_PROBE_EMBEDDING = [1.0, 0.0, 0.0, 0.0]
|
||||
|
||||
|
||||
def assert_partition_isolation(backend, writer, other, *, embedding=None):
|
||||
"""Assert ``writer`` and ``other`` are isolated partitions of ``backend``.
|
||||
|
||||
A record written to ``writer`` MUST NOT be returned, modified, or deleted
|
||||
through ``other`` (query / get / count / delete), and ``writer`` MUST still
|
||||
hold it afterwards. ``writer`` and ``other`` are two collections that the
|
||||
isolation contract says must not see each other — distinct palace ids (the
|
||||
universal guarantee) or distinct namespaces (the namespace guarantee).
|
||||
|
||||
Embeddings are supplied only for backends that require explicit vectors, so
|
||||
the same assertion works for text-embedding backends (Chroma) and
|
||||
explicit-vector backends (qdrant, sqlite_exact) alike.
|
||||
"""
|
||||
explicit = "requires_explicit_embeddings" in backend.capabilities
|
||||
vector = list(embedding if embedding is not None else _PROBE_EMBEDDING)
|
||||
|
||||
baseline_other = other.count()
|
||||
|
||||
add_kwargs = {
|
||||
"ids": [_PROBE_ID],
|
||||
"documents": [_PROBE_DOC],
|
||||
"metadatas": [{"wing": "conformance"}],
|
||||
}
|
||||
if explicit:
|
||||
add_kwargs["embeddings"] = [vector]
|
||||
writer.add(**add_kwargs)
|
||||
|
||||
if explicit:
|
||||
leaked = other.query(query_embeddings=[vector], n_results=10)
|
||||
else:
|
||||
leaked = other.query(query_texts=[_PROBE_DOC], n_results=10)
|
||||
hit_ids = leaked.ids[0] if leaked.ids else []
|
||||
assert _PROBE_ID not in hit_ids, "query() leaked a record across the isolation boundary"
|
||||
|
||||
assert other.get(ids=[_PROBE_ID]).ids == [], (
|
||||
"get() leaked a record across the isolation boundary"
|
||||
)
|
||||
assert other.count() == baseline_other, "count() leaked a record across the isolation boundary"
|
||||
|
||||
# A delete issued against the other partition MUST NOT touch writer's record.
|
||||
other.delete(ids=[_PROBE_ID])
|
||||
survivor = writer.get(ids=[_PROBE_ID])
|
||||
assert survivor.ids == [_PROBE_ID], "delete() crossed the isolation boundary"
|
||||
@@ -79,6 +79,12 @@ def _reset_mcp_cache():
|
||||
|
||||
mcp_server._client_cache = None
|
||||
mcp_server._collection_cache = None
|
||||
if hasattr(mcp_server, "_collection_cache_backend"):
|
||||
mcp_server._collection_cache_backend = None
|
||||
if hasattr(mcp_server, "_collection_cache_palace"):
|
||||
mcp_server._collection_cache_palace = None
|
||||
if hasattr(mcp_server, "_collection_open_error"):
|
||||
mcp_server._collection_open_error = None
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Backend isolation conformance suite (RFC 001 isolation contract).
|
||||
|
||||
Runs the shared isolation assertions from ``_backend_conformance`` against the
|
||||
built-in local backends. Server-mode backends (qdrant) run the same assertions
|
||||
under their own fake/live client in ``test_qdrant_backend.py``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from _backend_conformance import assert_partition_isolation
|
||||
|
||||
from mempalace.backends import PalaceRef
|
||||
from mempalace.backends.chroma import ChromaBackend
|
||||
from mempalace.backends.sqlite_exact import SQLiteExactBackend
|
||||
|
||||
_LOCAL_BACKENDS = [
|
||||
pytest.param(ChromaBackend, id="chroma"),
|
||||
pytest.param(SQLiteExactBackend, id="sqlite_exact"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend_cls", _LOCAL_BACKENDS)
|
||||
def test_cross_palace_isolation(backend_cls, tmp_path):
|
||||
"""One backend instance must isolate two distinct palaces (PalaceRef.id)."""
|
||||
backend = backend_cls()
|
||||
try:
|
||||
cols = []
|
||||
for label in ("alpha", "beta"):
|
||||
path = tmp_path / label
|
||||
ref = PalaceRef(id=str(path), local_path=str(path))
|
||||
cols.append(
|
||||
backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True)
|
||||
)
|
||||
assert_partition_isolation(backend, cols[0], cols[1])
|
||||
finally:
|
||||
backend.close()
|
||||
|
||||
|
||||
def test_local_backends_do_not_claim_namespace_isolation():
|
||||
"""Local backends isolate by on-disk path, not namespace; they must not
|
||||
advertise the namespace-isolation capability (RFC 001 isolation contract)."""
|
||||
assert "supports_namespace_isolation" not in ChromaBackend.capabilities
|
||||
assert "supports_namespace_isolation" not in SQLiteExactBackend.capabilities
|
||||
+429
-17
@@ -23,12 +23,16 @@ from mempalace.backends.chroma import (
|
||||
ChromaCollection,
|
||||
_HNSW_MISSING_METADATA_DATA_FLOOR,
|
||||
_fix_blob_seq_ids,
|
||||
_fix_missing_collection_type,
|
||||
_pin_hnsw_threads,
|
||||
_segment_appears_healthy,
|
||||
quarantine_invalid_hnsw_metadata,
|
||||
quarantine_stale_hnsw,
|
||||
)
|
||||
|
||||
# embeddinggemma-300m Matryoshka truncation (first 384 of 768 dims).
|
||||
_TEST_EMBED_DIM = 384
|
||||
|
||||
|
||||
class _FakeCollection:
|
||||
"""Stand-in for a chromadb.Collection returning raw chroma-shaped dicts."""
|
||||
@@ -180,6 +184,103 @@ def test_chroma_detect_matches_palace_with_chroma_sqlite(tmp_path):
|
||||
assert ChromaBackend.detect(str(tmp_path.parent)) is False
|
||||
|
||||
|
||||
def test_chroma_lexical_search_uses_sqlite_fts_not_full_collection_scan(tmp_path):
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE collections (id INTEGER PRIMARY KEY, name TEXT NOT NULL);
|
||||
CREATE TABLE segments (id INTEGER PRIMARY KEY, collection INTEGER NOT NULL);
|
||||
CREATE TABLE embeddings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
segment_id INTEGER NOT NULL,
|
||||
embedding_id TEXT,
|
||||
created_at TEXT
|
||||
);
|
||||
CREATE TABLE embedding_metadata (
|
||||
id INTEGER,
|
||||
key TEXT,
|
||||
string_value TEXT,
|
||||
int_value INTEGER,
|
||||
float_value REAL,
|
||||
bool_value INTEGER
|
||||
);
|
||||
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value);
|
||||
"""
|
||||
)
|
||||
conn.execute("INSERT INTO collections(id, name) VALUES (1, 'mempalace_drawers')")
|
||||
conn.execute("INSERT INTO segments(id, collection) VALUES (1, 1)")
|
||||
ids = list(range(1, 14))
|
||||
for emb_id in ids:
|
||||
wing = "target" if emb_id == 13 else "old"
|
||||
doc = "needle shared lexical note"
|
||||
conn.execute(
|
||||
"INSERT INTO embeddings(id, segment_id, embedding_id, created_at) VALUES (?, 1, ?, ?)",
|
||||
(emb_id, f"public-{emb_id}", f"2026-01-01T00:00:{emb_id:02d}"),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO embedding_fulltext_search(rowid, string_value) VALUES (?, ?)",
|
||||
(emb_id, doc),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO embedding_metadata(id, key, string_value) VALUES (?, 'chroma:document', ?)",
|
||||
(emb_id, doc),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO embedding_metadata(id, key, string_value) VALUES (?, 'wing', ?)",
|
||||
(emb_id, wing),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
class _NoScanCollection:
|
||||
name = "mempalace_drawers"
|
||||
|
||||
def count(self):
|
||||
raise AssertionError("lexical_search should use Chroma sqlite FTS")
|
||||
|
||||
def get(self, **_kwargs):
|
||||
raise AssertionError("lexical_search should use Chroma sqlite FTS")
|
||||
|
||||
collection = ChromaCollection(_NoScanCollection(), palace_path=str(tmp_path))
|
||||
|
||||
hits = collection.lexical_search(query="needle", n_results=1, where={"wing": "target"}).hits
|
||||
|
||||
assert [hit.metadata["wing"] for hit in hits] == ["target"]
|
||||
# Hit ids must be the public embedding_id (so lexical_search -> get(ids=...)
|
||||
# round-trips), not the internal embeddings.id rowid.
|
||||
assert [hit.id for hit in hits] == ["public-13"]
|
||||
|
||||
|
||||
def test_chroma_lexical_search_ids_roundtrip_through_get(tmp_path):
|
||||
"""lexical_search must return public drawer ids that get(ids=...) accepts.
|
||||
|
||||
Regression for the sqlite FTS path returning the internal rowid instead of
|
||||
embeddings.embedding_id, which silently broke hybrid search id round-trips.
|
||||
"""
|
||||
backend = ChromaBackend()
|
||||
palace = tmp_path / "palace"
|
||||
ref = PalaceRef(id=str(palace), local_path=str(palace))
|
||||
col = backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True)
|
||||
col.add(
|
||||
ids=["drawer-alpha", "drawer-bravo", "drawer-charlie"],
|
||||
documents=[
|
||||
"rareterm needle lexical note",
|
||||
"unrelated content here",
|
||||
"another rareterm needle entry",
|
||||
],
|
||||
metadatas=[{"wing": "w"}, {"wing": "w"}, {"wing": "w"}],
|
||||
)
|
||||
hits = col.lexical_search(query="rareterm needle", n_results=5).hits
|
||||
hit_ids = [hit.id for hit in hits]
|
||||
assert hit_ids, "expected lexical hits"
|
||||
assert set(hit_ids) <= {"drawer-alpha", "drawer-bravo", "drawer-charlie"}
|
||||
# Every returned id must resolve back through get() — proving it is a public id.
|
||||
fetched = col.get(ids=hit_ids)
|
||||
assert set(fetched.ids) == set(hit_ids)
|
||||
backend.close()
|
||||
|
||||
|
||||
def test_query_rejects_missing_input():
|
||||
fake = _FakeCollection()
|
||||
collection = ChromaCollection(fake)
|
||||
@@ -391,14 +492,12 @@ def test_chroma_backend_creates_collection_with_cosine_distance(tmp_path):
|
||||
|
||||
|
||||
def test_chroma_backend_sets_hnsw_bloat_guard_on_creation(tmp_path):
|
||||
"""The HNSW guard from #344 must land on freshly-created collection metadata.
|
||||
"""HNSW batch/sync thresholds must land on freshly-created collection metadata.
|
||||
|
||||
Without batch_size + sync_threshold, mining ~10K+ drawers triggers the
|
||||
resize+persist drift that bloats link_lists.bin into hundreds of GB sparse
|
||||
and segfaults `status` / `search` / `repair`. The guard belongs at
|
||||
collection-creation time so every fresh palace gets it without needing
|
||||
a runtime retrofit. Asserting both keys land on the persisted metadata
|
||||
also covers the #1161 "config silently dropped" concern at CI time.
|
||||
Low thresholds (2/2 per #1579) make chromadb's Rust HNSW segment
|
||||
persist index_metadata and link_lists after any mine of 2+ drawers.
|
||||
Asserting both keys land on the persisted metadata also covers the
|
||||
#1161 "config silently dropped" concern at CI time.
|
||||
"""
|
||||
palace_path = tmp_path / "palace"
|
||||
|
||||
@@ -410,8 +509,8 @@ def test_chroma_backend_sets_hnsw_bloat_guard_on_creation(tmp_path):
|
||||
|
||||
client = chromadb.PersistentClient(path=str(palace_path))
|
||||
col = client.get_collection("mempalace_drawers")
|
||||
assert col.metadata.get("hnsw:batch_size") == 50_000
|
||||
assert col.metadata.get("hnsw:sync_threshold") == 50_000
|
||||
assert col.metadata.get("hnsw:batch_size") == 2
|
||||
assert col.metadata.get("hnsw:sync_threshold") == 2
|
||||
|
||||
|
||||
def test_chroma_backend_create_collection_sets_hnsw_bloat_guard(tmp_path):
|
||||
@@ -422,8 +521,85 @@ def test_chroma_backend_create_collection_sets_hnsw_bloat_guard(tmp_path):
|
||||
|
||||
client = chromadb.PersistentClient(path=str(palace_path))
|
||||
col = client.get_collection("mempalace_drawers")
|
||||
assert col.metadata.get("hnsw:batch_size") == 50_000
|
||||
assert col.metadata.get("hnsw:sync_threshold") == 50_000
|
||||
assert col.metadata.get("hnsw:batch_size") == 2
|
||||
assert col.metadata.get("hnsw:sync_threshold") == 2
|
||||
|
||||
|
||||
def test_sub_threshold_mine_persists_hnsw_metadata(tmp_path):
|
||||
"""Regression for #1579: small mines must persist HNSW metadata.
|
||||
|
||||
_HNSW_BLOAT_GUARD sets batch_size=2 and sync_threshold=2 so that any
|
||||
upsert of 2+ records crosses both thresholds, triggering chromadb's
|
||||
_apply_batch and _persist. Without this, index_metadata and link_lists
|
||||
stay empty and quarantine_stale_hnsw renames the segment on cold open.
|
||||
"""
|
||||
palace_path = str(tmp_path / "palace")
|
||||
backend = ChromaBackend()
|
||||
try:
|
||||
col = backend.get_collection(palace_path, "mempalace_drawers", create=True)
|
||||
|
||||
col.upsert(
|
||||
ids=["a", "b", "c"],
|
||||
documents=["doc a", "doc b", "doc c"],
|
||||
embeddings=[[0.1] * _TEST_EMBED_DIM, [0.2] * _TEST_EMBED_DIM, [0.3] * _TEST_EMBED_DIM],
|
||||
metadatas=[{"wing": "t"}, {"wing": "t"}, {"wing": "t"}],
|
||||
)
|
||||
finally:
|
||||
backend.close()
|
||||
|
||||
found_healthy_segment = False
|
||||
for entry in (tmp_path / "palace").iterdir():
|
||||
if not entry.is_dir() or entry.name.startswith("."):
|
||||
continue
|
||||
meta = entry / "index_metadata.pickle"
|
||||
link = entry / "link_lists.bin"
|
||||
data = entry / "data_level0.bin"
|
||||
if data.exists() and data.stat().st_size > _HNSW_MISSING_METADATA_DATA_FLOOR:
|
||||
assert meta.exists(), "index_metadata missing after sub-threshold upsert"
|
||||
assert link.exists() and link.stat().st_size > 0, "link_lists empty"
|
||||
assert _segment_appears_healthy(str(entry))
|
||||
found_healthy_segment = True
|
||||
|
||||
assert found_healthy_segment, "no VECTOR segment with data found"
|
||||
|
||||
# stale_seconds=0.0 forces the stage-2 integrity gate (_segment_appears_healthy)
|
||||
# to run on every segment regardless of mtime delta, proving the fix directly.
|
||||
moved = quarantine_stale_hnsw(palace_path, stale_seconds=0.0)
|
||||
assert moved == [], f"quarantine fired on freshly-persisted segment: {moved}"
|
||||
|
||||
|
||||
def test_single_record_upsert_not_quarantined(tmp_path):
|
||||
"""A single-record upsert must not trigger quarantine.
|
||||
|
||||
With batch_size=2 chromadb only persists HNSW metadata after the second
|
||||
record. A one-record segment has no index_metadata.pickle and no
|
||||
link_lists.bin data; _segment_appears_healthy must treat that combination
|
||||
as sub-threshold (never persisted), not as corruption.
|
||||
"""
|
||||
palace_path = str(tmp_path / "palace")
|
||||
backend = ChromaBackend()
|
||||
try:
|
||||
col = backend.get_collection(palace_path, "mempalace_drawers", create=True)
|
||||
col.upsert(
|
||||
ids=["solo"],
|
||||
documents=["only one drawer"],
|
||||
embeddings=[[0.5] * _TEST_EMBED_DIM],
|
||||
metadatas=[{"wing": "t"}],
|
||||
)
|
||||
finally:
|
||||
backend.close()
|
||||
|
||||
for entry in (tmp_path / "palace").iterdir():
|
||||
if not entry.is_dir() or entry.name.startswith("."):
|
||||
continue
|
||||
data = entry / "data_level0.bin"
|
||||
if data.exists() and data.stat().st_size > 0:
|
||||
assert _segment_appears_healthy(str(entry)), (
|
||||
f"single-record segment flagged unhealthy: data={data.stat().st_size}B"
|
||||
)
|
||||
|
||||
moved = quarantine_stale_hnsw(palace_path, stale_seconds=0.0)
|
||||
assert moved == [], f"quarantine fired on single-record segment: {moved}"
|
||||
|
||||
|
||||
def test_get_collection_create_true_is_idempotent(tmp_path):
|
||||
@@ -449,7 +625,7 @@ def test_get_collection_create_true_preserves_existing_metadata(tmp_path):
|
||||
backend.get_collection(palace, collection_name="mempalace_drawers", create=True)
|
||||
col = backend.get_collection(palace, collection_name="mempalace_drawers", create=True)
|
||||
assert col._collection.metadata["hnsw:space"] == "cosine"
|
||||
assert col._collection.metadata.get("hnsw:batch_size") == 50_000
|
||||
assert col._collection.metadata.get("hnsw:batch_size") == 2
|
||||
|
||||
|
||||
def test_fix_blob_seq_ids_converts_blobs_to_integers(tmp_path):
|
||||
@@ -614,6 +790,186 @@ def test_fix_blob_seq_ids_skips_sqlite_when_marker_present(tmp_path):
|
||||
mock_connect.assert_not_called()
|
||||
|
||||
|
||||
# ── _fix_missing_collection_type ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_fix_collection_type_adds_type(tmp_path):
|
||||
"""Legacy config_json_str '{}' gets _type added."""
|
||||
import json
|
||||
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
|
||||
conn.execute(
|
||||
"INSERT INTO collections (id, config_json_str) VALUES (?, ?)",
|
||||
("col-1", "{}"),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
row = conn.execute("SELECT config_json_str FROM collections WHERE id = 'col-1'").fetchone()
|
||||
config = json.loads(row[0])
|
||||
assert config["_type"] == "CollectionConfigurationInternal"
|
||||
|
||||
|
||||
def test_fix_collection_type_preserves_existing(tmp_path):
|
||||
"""Config that already has _type is left unchanged."""
|
||||
import json
|
||||
|
||||
original = json.dumps({"_type": "CollectionConfigurationInternal", "extra": 1})
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
|
||||
conn.execute(
|
||||
"INSERT INTO collections (id, config_json_str) VALUES (?, ?)",
|
||||
("col-1", original),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
row = conn.execute("SELECT config_json_str FROM collections WHERE id = 'col-1'").fetchone()
|
||||
assert row[0] == original
|
||||
|
||||
|
||||
def test_fix_collection_type_noop_without_db(tmp_path):
|
||||
"""No error when palace has no chroma.sqlite3, no marker written."""
|
||||
from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER
|
||||
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
assert not (tmp_path / _COLLECTION_TYPE_MARKER).exists()
|
||||
|
||||
|
||||
def test_fix_collection_type_writes_marker(tmp_path):
|
||||
"""Marker is written after a successful migration."""
|
||||
from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER
|
||||
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
|
||||
conn.execute(
|
||||
"INSERT INTO collections (id, config_json_str) VALUES (?, ?)",
|
||||
("col-1", "{}"),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
marker = tmp_path / _COLLECTION_TYPE_MARKER
|
||||
assert not marker.exists()
|
||||
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
|
||||
assert marker.is_file()
|
||||
|
||||
|
||||
def test_fix_collection_type_skips_with_marker(tmp_path):
|
||||
"""When the marker exists, sqlite3 is not opened."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER
|
||||
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
db_path.write_bytes(b"sentinel")
|
||||
(tmp_path / _COLLECTION_TYPE_MARKER).touch()
|
||||
|
||||
with patch("mempalace.backends.chroma.sqlite3.connect") as mock_connect:
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
|
||||
mock_connect.assert_not_called()
|
||||
|
||||
|
||||
def test_fix_collection_type_writes_marker_when_already_has_type(tmp_path):
|
||||
"""Marker written even when all collections already have _type (noop case)."""
|
||||
import json
|
||||
|
||||
from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER
|
||||
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
|
||||
conn.execute(
|
||||
"INSERT INTO collections (id, config_json_str) VALUES (?, ?)",
|
||||
("col-1", json.dumps({"_type": "CollectionConfigurationInternal"})),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
marker = tmp_path / _COLLECTION_TYPE_MARKER
|
||||
assert not marker.exists()
|
||||
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
|
||||
assert marker.is_file(), "marker must be written even when no collections needed fixing"
|
||||
|
||||
|
||||
def test_fix_collection_type_multi_collection_mixed(tmp_path):
|
||||
"""Multiple collections: NULL, empty, and already-valid configs."""
|
||||
import json
|
||||
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
|
||||
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-null", None))
|
||||
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-empty", "{}"))
|
||||
conn.execute(
|
||||
"INSERT INTO collections VALUES (?, ?)",
|
||||
("col-ok", json.dumps({"_type": "CollectionConfigurationInternal"})),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
rows = {
|
||||
r[0]: json.loads(r[1]) if r[1] else None
|
||||
for r in conn.execute("SELECT id, config_json_str FROM collections")
|
||||
}
|
||||
assert rows["col-null"]["_type"] == "CollectionConfigurationInternal"
|
||||
assert rows["col-empty"]["_type"] == "CollectionConfigurationInternal"
|
||||
assert rows["col-ok"] == {"_type": "CollectionConfigurationInternal"}
|
||||
|
||||
|
||||
def test_fix_collection_type_skips_non_dict_json(tmp_path):
|
||||
"""Non-dict JSON (array, null literal) is skipped without error."""
|
||||
import json
|
||||
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
|
||||
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-arr", "[]"))
|
||||
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-null", "null"))
|
||||
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-ok", "{}"))
|
||||
conn.commit()
|
||||
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
rows = dict(conn.execute("SELECT id, config_json_str FROM collections").fetchall())
|
||||
assert rows["col-arr"] == "[]"
|
||||
assert rows["col-null"] == "null"
|
||||
assert json.loads(rows["col-ok"])["_type"] == "CollectionConfigurationInternal"
|
||||
|
||||
|
||||
def test_fix_collection_type_skips_malformed_json(tmp_path):
|
||||
"""Malformed JSON in one row does not prevent fixing other rows."""
|
||||
import json
|
||||
|
||||
db_path = tmp_path / "chroma.sqlite3"
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
|
||||
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-bad", "{corrupt"))
|
||||
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-ok", "{}"))
|
||||
conn.commit()
|
||||
|
||||
_fix_missing_collection_type(str(tmp_path))
|
||||
|
||||
with closing(sqlite3.connect(str(db_path))) as conn:
|
||||
rows = dict(conn.execute("SELECT id, config_json_str FROM collections").fetchall())
|
||||
assert rows["col-bad"] == "{corrupt"
|
||||
assert json.loads(rows["col-ok"])["_type"] == "CollectionConfigurationInternal"
|
||||
|
||||
|
||||
# ── quarantine_stale_hnsw ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -698,17 +1054,18 @@ def test_quarantine_stale_hnsw_leaves_empty_segment_without_metadata_alone(tmp_p
|
||||
|
||||
|
||||
def test_segment_without_metadata_but_with_nontrivial_data_is_unhealthy(tmp_path):
|
||||
"""Data without index_metadata.pickle is a partial flush, not a fresh segment."""
|
||||
"""Interrupted persist: link_lists written but metadata absent is unhealthy."""
|
||||
|
||||
seg = tmp_path / "abcd-1234-5678"
|
||||
seg.mkdir()
|
||||
(seg / "data_level0.bin").write_bytes(b"\0" * (_HNSW_MISSING_METADATA_DATA_FLOOR + 1))
|
||||
(seg / "link_lists.bin").write_bytes(b"\x01" * 128)
|
||||
|
||||
assert not _segment_appears_healthy(str(seg))
|
||||
|
||||
|
||||
def test_segment_without_metadata_and_tiny_data_is_still_treated_as_fresh(tmp_path):
|
||||
"""Tiny data payloads can occur before metadata has flushed; leave them alone."""
|
||||
"""No metadata and no link_lists means no persist was attempted; treat as fresh."""
|
||||
|
||||
seg = tmp_path / "abcd-1234-5678"
|
||||
seg.mkdir()
|
||||
@@ -718,7 +1075,7 @@ def test_segment_without_metadata_and_tiny_data_is_still_treated_as_fresh(tmp_pa
|
||||
|
||||
|
||||
def test_quarantine_stale_hnsw_renames_missing_metadata_with_nontrivial_data(tmp_path):
|
||||
"""Regression for #1274: missing pickle + non-trivial data must quarantine."""
|
||||
"""Regression for #1274: missing pickle + link data must quarantine."""
|
||||
|
||||
now = 1_700_000_000.0
|
||||
palace, seg = _make_palace_with_segment(
|
||||
@@ -728,6 +1085,7 @@ def test_quarantine_stale_hnsw_renames_missing_metadata_with_nontrivial_data(tmp
|
||||
meta_bytes=None,
|
||||
)
|
||||
(seg / "data_level0.bin").write_bytes(b"\0" * (_HNSW_MISSING_METADATA_DATA_FLOOR + 1))
|
||||
(seg / "link_lists.bin").write_bytes(b"\x01" * 128)
|
||||
os.utime(seg / "data_level0.bin", (now - 7200, now - 7200))
|
||||
|
||||
moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0)
|
||||
@@ -946,6 +1304,42 @@ def test_client_quarantines_only_on_first_call_per_palace(tmp_path, monkeypatch)
|
||||
)
|
||||
|
||||
|
||||
def test_client_rearms_quarantine_on_mtime_change(tmp_path, monkeypatch):
|
||||
"""When the DB file's mtime changes between ``_client()`` calls (external
|
||||
in-place write), the quarantine gate re-arms so HNSW checks run again.
|
||||
|
||||
Before #1573, the gate was only cleared on *inode* change (full palace
|
||||
replacement); mtime-only changes left the gate armed, so long-running
|
||||
processes were blind to external drift."""
|
||||
palace_path = str(tmp_path / "palace")
|
||||
os.makedirs(palace_path, exist_ok=True)
|
||||
db_file = Path(palace_path) / "chroma.sqlite3"
|
||||
db_file.write_text("")
|
||||
|
||||
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def _spy(path, stale_seconds=300.0):
|
||||
calls.append(path)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _spy)
|
||||
|
||||
backend = ChromaBackend()
|
||||
try:
|
||||
backend._client(palace_path)
|
||||
assert len(calls) == 1, "quarantine should fire on first open"
|
||||
|
||||
_, cached_mtime = backend._freshness[palace_path]
|
||||
os.utime(str(db_file), (cached_mtime + 1.0, cached_mtime + 1.0))
|
||||
|
||||
backend._client(palace_path)
|
||||
assert len(calls) == 2, "quarantine should re-fire after mtime change (gate re-armed)"
|
||||
finally:
|
||||
backend.close()
|
||||
|
||||
|
||||
# ── _pin_hnsw_threads (per-process retrofit, separate from this PR's gate) ──
|
||||
|
||||
|
||||
@@ -1218,6 +1612,9 @@ def test_chroma_backend_preflights_metadata_before_persistent_client(tmp_path, m
|
||||
|
||||
return inner
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type")
|
||||
)
|
||||
monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob"))
|
||||
monkeypatch.setattr(
|
||||
"mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid")
|
||||
@@ -1235,13 +1632,16 @@ def test_chroma_backend_preflights_metadata_before_persistent_client(tmp_path, m
|
||||
backend._client(str(palace))
|
||||
|
||||
assert calls == [
|
||||
("collection_type", str(palace)),
|
||||
("blob", str(palace)),
|
||||
("invalid", str(palace)),
|
||||
("stale", str(palace)),
|
||||
]
|
||||
|
||||
|
||||
def test_chroma_backend_stale_quarantine_is_cold_start_only_on_refresh(tmp_path, monkeypatch):
|
||||
def test_chroma_backend_quarantine_rearms_on_mtime_refresh(tmp_path, monkeypatch):
|
||||
"""When the DB mtime changes between ``_client()`` calls, the quarantine
|
||||
gate re-arms and the HNSW safety checks run again (#1573)."""
|
||||
palace = tmp_path / "palace"
|
||||
palace.mkdir()
|
||||
(palace / "chroma.sqlite3").write_text("")
|
||||
@@ -1255,6 +1655,9 @@ def test_chroma_backend_stale_quarantine_is_cold_start_only_on_refresh(tmp_path,
|
||||
return inner
|
||||
|
||||
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
|
||||
monkeypatch.setattr(
|
||||
"mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type")
|
||||
)
|
||||
monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob"))
|
||||
monkeypatch.setattr(
|
||||
"mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid")
|
||||
@@ -1276,10 +1679,14 @@ def test_chroma_backend_stale_quarantine_is_cold_start_only_on_refresh(tmp_path,
|
||||
backend._client(str(palace))
|
||||
|
||||
assert calls == [
|
||||
("collection_type", str(palace)),
|
||||
("blob", str(palace)),
|
||||
("invalid", str(palace)),
|
||||
("stale", str(palace)),
|
||||
("collection_type", str(palace)),
|
||||
("blob", str(palace)),
|
||||
("invalid", str(palace)),
|
||||
("stale", str(palace)),
|
||||
]
|
||||
|
||||
|
||||
@@ -1297,6 +1704,9 @@ def test_chroma_backend_requarantines_after_inode_replacement(tmp_path, monkeypa
|
||||
return inner
|
||||
|
||||
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
|
||||
monkeypatch.setattr(
|
||||
"mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type")
|
||||
)
|
||||
monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob"))
|
||||
monkeypatch.setattr(
|
||||
"mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid")
|
||||
@@ -1318,9 +1728,11 @@ def test_chroma_backend_requarantines_after_inode_replacement(tmp_path, monkeypa
|
||||
backend._client(str(palace))
|
||||
|
||||
assert calls == [
|
||||
("collection_type", str(palace)),
|
||||
("blob", str(palace)),
|
||||
("invalid", str(palace)),
|
||||
("stale", str(palace)),
|
||||
("collection_type", str(palace)),
|
||||
("blob", str(palace)),
|
||||
("invalid", str(palace)),
|
||||
("stale", str(palace)),
|
||||
@@ -1371,7 +1783,7 @@ def test_get_collection_translates_ef_mismatch_to_helpful_error(tmp_path):
|
||||
return "embeddinggemma_300m"
|
||||
|
||||
def __call__(self, input):
|
||||
return [[0.0] * 384 for _ in input]
|
||||
return [[0.0] * _TEST_EMBED_DIM for _ in input]
|
||||
|
||||
original_resolver = backend._resolve_embedding_function
|
||||
backend._resolve_embedding_function = lambda: _ConflictingEF()
|
||||
|
||||
@@ -177,3 +177,88 @@ class TestToolsAcceptSurrogates:
|
||||
topic="log",
|
||||
)
|
||||
assert result["success"] is True
|
||||
|
||||
|
||||
# ── Backend chokepoint (bulk ingest paths) ──────────────────────────────────
|
||||
|
||||
|
||||
class _CapturingCollection:
|
||||
"""Minimal chromadb.Collection stand-in that records the kwargs it receives,
|
||||
so we can assert what actually reaches the chromadb client."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def add(self, **kwargs):
|
||||
self.calls.append(("add", kwargs))
|
||||
|
||||
def upsert(self, **kwargs):
|
||||
self.calls.append(("upsert", kwargs))
|
||||
|
||||
def update(self, **kwargs):
|
||||
self.calls.append(("update", kwargs))
|
||||
|
||||
|
||||
class TestBackendChokepointStripsDocuments:
|
||||
"""#1235 sanitised the MCP write tools, but the bulk ingest paths
|
||||
(miner, convo_miner, sweeper, diary_ingest) build documents without routing
|
||||
through ``sanitize_content`` and reach ``ChromaCollection`` directly. A lone
|
||||
surrogate in *document* text crashes the whole add/upsert batch (-32000),
|
||||
silently dropping the other rows. The backend chokepoint must strip
|
||||
documents, mirroring ``_sanitize_metadatas_for_chromadb``."""
|
||||
|
||||
@staticmethod
|
||||
def _collection():
|
||||
from mempalace.backends.chroma import ChromaCollection
|
||||
|
||||
fake = _CapturingCollection()
|
||||
return fake, ChromaCollection(fake)
|
||||
|
||||
def test_add_strips_lone_surrogate_in_document(self):
|
||||
fake, col = self._collection()
|
||||
col.add(documents=["clean\udc95doc"], ids=["1"])
|
||||
_, kwargs = fake.calls[0]
|
||||
assert kwargs["documents"] == ["clean�doc"]
|
||||
kwargs["documents"][0].encode("utf-8") # the crash path must not raise
|
||||
|
||||
def test_upsert_strips_lone_surrogate_in_document(self):
|
||||
fake, col = self._collection()
|
||||
col.upsert(documents=["a\ud800b"], ids=["1"], metadatas=[{"wing": "w"}])
|
||||
_, kwargs = fake.calls[0]
|
||||
assert kwargs["documents"] == ["a�b"]
|
||||
|
||||
def test_update_strips_lone_surrogate_in_document(self):
|
||||
fake, col = self._collection()
|
||||
col.update(ids=["1"], documents=["x\udcffy"])
|
||||
_, kwargs = fake.calls[0]
|
||||
assert kwargs["documents"] == ["x�y"]
|
||||
|
||||
def test_one_poison_message_does_not_drop_the_batch(self):
|
||||
"""A single bad row used to abort the entire batch, silently dropping
|
||||
the others. After sanitising, every row survives."""
|
||||
fake, col = self._collection()
|
||||
col.upsert(
|
||||
documents=["ok one", "poison\udc95row", "ok three"],
|
||||
ids=["1", "2", "3"],
|
||||
)
|
||||
_, kwargs = fake.calls[0]
|
||||
assert kwargs["documents"] == ["ok one", "poison�row", "ok three"]
|
||||
assert len(kwargs["ids"]) == 3
|
||||
for d in kwargs["documents"]:
|
||||
d.encode("utf-8") # must not raise
|
||||
|
||||
def test_real_emoji_and_none_metadata_preserved(self):
|
||||
fake, col = self._collection()
|
||||
col.add(documents=["ship \U0001f680"], ids=["1"])
|
||||
_, kwargs = fake.calls[0]
|
||||
assert kwargs["documents"] == ["ship \U0001f680"]
|
||||
|
||||
def test_single_string_document_is_not_split_into_chars(self):
|
||||
"""chromadb accepts a bare str as one document (OneOrMany[Document]).
|
||||
The sanitiser must keep it whole and clean, not split it into
|
||||
per-character documents."""
|
||||
fake, col = self._collection()
|
||||
col.upsert(documents="one\udc95document", ids=["1"])
|
||||
_, kwargs = fake.calls[0]
|
||||
assert kwargs["documents"] == "one�document"
|
||||
assert isinstance(kwargs["documents"], str)
|
||||
|
||||
@@ -713,6 +713,40 @@ def test_main_status_dispatches():
|
||||
mock_cmd.assert_called_once()
|
||||
|
||||
|
||||
def test_main_backend_flag_sets_explicit_backend(monkeypatch):
|
||||
monkeypatch.delenv("MEMPALACE_BACKEND_EXPLICIT", raising=False)
|
||||
monkeypatch.delenv("MEMPALACE_BACKEND", raising=False)
|
||||
with (
|
||||
patch("sys.argv", ["mempalace", "status", "--backend", "sqlite_exact"]),
|
||||
patch("mempalace.cli.cmd_status") as mock_cmd,
|
||||
):
|
||||
main()
|
||||
|
||||
mock_cmd.assert_called_once()
|
||||
args = mock_cmd.call_args.args[0]
|
||||
assert args.backend == "sqlite_exact"
|
||||
assert os.environ["MEMPALACE_BACKEND_EXPLICIT"] == "sqlite_exact"
|
||||
os.environ.pop("MEMPALACE_BACKEND_EXPLICIT", None)
|
||||
os.environ.pop("MEMPALACE_BACKEND", None)
|
||||
|
||||
|
||||
def test_main_backend_flag_accepts_qdrant(monkeypatch):
|
||||
monkeypatch.delenv("MEMPALACE_BACKEND_EXPLICIT", raising=False)
|
||||
monkeypatch.delenv("MEMPALACE_BACKEND", raising=False)
|
||||
with (
|
||||
patch("sys.argv", ["mempalace", "status", "--backend", "qdrant"]),
|
||||
patch("mempalace.cli.cmd_status") as mock_cmd,
|
||||
):
|
||||
main()
|
||||
|
||||
mock_cmd.assert_called_once()
|
||||
args = mock_cmd.call_args.args[0]
|
||||
assert args.backend == "qdrant"
|
||||
assert os.environ["MEMPALACE_BACKEND_EXPLICIT"] == "qdrant"
|
||||
os.environ.pop("MEMPALACE_BACKEND_EXPLICIT", None)
|
||||
os.environ.pop("MEMPALACE_BACKEND", None)
|
||||
|
||||
|
||||
def test_main_search_dispatches():
|
||||
with (
|
||||
patch("sys.argv", ["mempalace", "search", "my query"]),
|
||||
@@ -790,6 +824,34 @@ def test_mcp_command_uses_custom_palace_path_when_provided(monkeypatch, capsys):
|
||||
assert captured.err == ""
|
||||
|
||||
|
||||
def test_mcp_command_includes_backend_when_provided(monkeypatch, capsys):
|
||||
monkeypatch.delenv("MEMPALACE_BACKEND_EXPLICIT", raising=False)
|
||||
monkeypatch.delenv("MEMPALACE_BACKEND", raising=False)
|
||||
monkeypatch.setattr(sys, "argv", ["mempalace", "mcp", "--backend", "sqlite_exact"])
|
||||
|
||||
main()
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "mempalace-mcp --backend sqlite_exact" in captured.out
|
||||
assert captured.err == ""
|
||||
os.environ.pop("MEMPALACE_BACKEND_EXPLICIT", None)
|
||||
os.environ.pop("MEMPALACE_BACKEND", None)
|
||||
|
||||
|
||||
def test_mcp_command_includes_qdrant_backend(monkeypatch, capsys):
|
||||
monkeypatch.delenv("MEMPALACE_BACKEND_EXPLICIT", raising=False)
|
||||
monkeypatch.delenv("MEMPALACE_BACKEND", raising=False)
|
||||
monkeypatch.setattr(sys, "argv", ["mempalace", "mcp", "--backend", "qdrant"])
|
||||
|
||||
main()
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "mempalace-mcp --backend qdrant" in captured.out
|
||||
assert captured.err == ""
|
||||
os.environ.pop("MEMPALACE_BACKEND_EXPLICIT", None)
|
||||
os.environ.pop("MEMPALACE_BACKEND", None)
|
||||
|
||||
|
||||
def test_main_hook_no_subcommand_prints_help(capsys):
|
||||
with patch("sys.argv", ["mempalace", "hook"]):
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Tests for mempalace.collision_scan — pre-mining defense against
|
||||
drawer_id collisions.
|
||||
|
||||
The scan runs immediately before batched chromadb upserts and aborts the
|
||||
mine with an actionable error if any proposed drawer_id appears more than
|
||||
once in the union of (incoming-vs-incoming) and (incoming-vs-existing).
|
||||
|
||||
Under the v2 hash recipe these collisions are vanishingly rare in practice
|
||||
— SHA-256 truncated to 24 hex chars makes accidental collision ~2^-96.
|
||||
The scan's real value is (a) catching upstream bugs that emit duplicate
|
||||
(source_file, chunk_index) pairs in the same batch, and (b) surfacing the
|
||||
astronomical-but-possible SHA-256 collision with a clear error instead of
|
||||
a silent overwrite at the ChromaDB upsert.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from mempalace.collision_scan import CollisionError, assert_no_collisions
|
||||
|
||||
|
||||
class _MockGet:
|
||||
"""Stand-in for ChromaDB's get() result. Real ChromaDB returns a
|
||||
dict-like with ``ids`` and ``metadatas`` keys; we mirror that shape."""
|
||||
|
||||
def __init__(self, ids: list[str], metadatas: list[dict]):
|
||||
self._payload = {"ids": ids, "metadatas": metadatas}
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self._payload[key]
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self._payload.get(key, default)
|
||||
|
||||
|
||||
class _MockCollection:
|
||||
"""Stand-in for a ChromaDB collection. Stores a fixed mapping of
|
||||
drawer_id → metadata for the test to model 'existing palace state'.
|
||||
``get(ids=[...])`` returns only the rows whose ids are in storage.
|
||||
"""
|
||||
|
||||
def __init__(self, existing: Optional[dict[str, dict]] = None):
|
||||
self._existing = existing or {}
|
||||
|
||||
def get(self, ids=None, include=None, **kwargs):
|
||||
ids = ids or []
|
||||
rows = [(did, self._existing[did]) for did in ids if did in self._existing]
|
||||
return _MockGet(
|
||||
ids=[did for did, _ in rows],
|
||||
metadatas=[meta for _, meta in rows],
|
||||
)
|
||||
|
||||
|
||||
# ── Happy path: no collisions ────────────────────────────────────────
|
||||
|
||||
|
||||
def test_assert_no_collisions_passes_for_clean_batch():
|
||||
"""Distinct incoming ids + no overlap with existing = no error."""
|
||||
proposed = [
|
||||
("drawer_a", {"source_file": "/file_a.md", "chunk_index": 0}),
|
||||
("drawer_b", {"source_file": "/file_a.md", "chunk_index": 1}),
|
||||
("drawer_c", {"source_file": "/file_b.md", "chunk_index": 0}),
|
||||
]
|
||||
col = _MockCollection()
|
||||
assert_no_collisions(proposed, col) is None
|
||||
|
||||
|
||||
def test_assert_no_collisions_passes_for_clean_batch_with_existing_drawers():
|
||||
"""Existing drawers with DIFFERENT ids than incoming = no error.
|
||||
The scan only fires when an id appears more than once across the
|
||||
union of incoming + existing."""
|
||||
proposed = [
|
||||
("drawer_new_1", {"source_file": "/new.md", "chunk_index": 0}),
|
||||
]
|
||||
col = _MockCollection(
|
||||
existing={
|
||||
"drawer_old_1": {"source_file": "/old.md", "chunk_index": 0},
|
||||
"drawer_old_2": {"source_file": "/old.md", "chunk_index": 1},
|
||||
}
|
||||
)
|
||||
assert_no_collisions(proposed, col) is None
|
||||
|
||||
|
||||
def test_assert_no_collisions_treats_idempotent_re_mine_as_clean():
|
||||
"""If incoming drawer_id matches an existing id AND the
|
||||
(source_file, chunk_index) metadata also matches, that's a normal
|
||||
re-mine of the same chunk — NOT a collision. The scan must let it
|
||||
pass; otherwise re-mining a clean palace would always raise."""
|
||||
proposed = [
|
||||
("drawer_same", {"source_file": "/file.md", "chunk_index": 5}),
|
||||
]
|
||||
col = _MockCollection(
|
||||
existing={
|
||||
"drawer_same": {"source_file": "/file.md", "chunk_index": 5},
|
||||
}
|
||||
)
|
||||
assert_no_collisions(proposed, col) is None
|
||||
|
||||
|
||||
# ── Incoming-vs-incoming collisions ──────────────────────────────────
|
||||
|
||||
|
||||
def test_assert_no_collisions_raises_on_incoming_duplicate_with_different_metadata():
|
||||
"""Two incoming chunks producing the same drawer_id with DIFFERENT
|
||||
(source_file, chunk_index) pairs = an upstream bug or astronomical
|
||||
SHA-256 hash collision. Either way, abort the mine."""
|
||||
proposed = [
|
||||
("drawer_X", {"source_file": "/file_a.md", "chunk_index": 0}),
|
||||
("drawer_X", {"source_file": "/file_b.md", "chunk_index": 1}),
|
||||
]
|
||||
col = _MockCollection()
|
||||
with pytest.raises(CollisionError) as exc_info:
|
||||
assert_no_collisions(proposed, col)
|
||||
# Error message names the colliding (source_file, chunk_index) pairs
|
||||
msg = str(exc_info.value)
|
||||
assert "drawer_X" in msg
|
||||
assert "/file_a.md" in msg
|
||||
assert "/file_b.md" in msg
|
||||
|
||||
|
||||
def test_assert_no_collisions_passes_on_incoming_duplicate_with_same_metadata():
|
||||
"""Two incoming chunks with the SAME (source_file, chunk_index)
|
||||
producing the same drawer_id = a duplicate chunk in the batch (still
|
||||
an upstream bug, but the downstream collision damage is zero since
|
||||
they'd write identical content). The scan does not fire on this
|
||||
case because it's not the collision shape v2 was designed to
|
||||
catch."""
|
||||
proposed = [
|
||||
("drawer_X", {"source_file": "/file.md", "chunk_index": 5}),
|
||||
("drawer_X", {"source_file": "/file.md", "chunk_index": 5}),
|
||||
]
|
||||
col = _MockCollection()
|
||||
assert_no_collisions(proposed, col) is None
|
||||
|
||||
|
||||
# ── Incoming-vs-existing collisions ──────────────────────────────────
|
||||
|
||||
|
||||
def test_assert_no_collisions_raises_on_incoming_matching_existing_with_different_metadata():
|
||||
"""Incoming chunk produces the same drawer_id as an existing drawer
|
||||
whose stored (source_file, chunk_index) DIFFERS = SHA-256 collision
|
||||
or recipe-version skew. Abort the mine so the upsert doesn't
|
||||
silently overwrite the existing row."""
|
||||
proposed = [
|
||||
("drawer_Y", {"source_file": "/incoming.md", "chunk_index": 3}),
|
||||
]
|
||||
col = _MockCollection(
|
||||
existing={
|
||||
"drawer_Y": {"source_file": "/existing.md", "chunk_index": 7},
|
||||
}
|
||||
)
|
||||
with pytest.raises(CollisionError) as exc_info:
|
||||
assert_no_collisions(proposed, col)
|
||||
msg = str(exc_info.value)
|
||||
assert "drawer_Y" in msg
|
||||
assert "/incoming.md" in msg
|
||||
assert "/existing.md" in msg
|
||||
|
||||
|
||||
# ── Error message quality ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_collision_error_lists_all_collisions_not_just_first():
|
||||
"""If a batch contains multiple distinct collisions, the error
|
||||
surfaces ALL of them — a user fixing one and re-running shouldn't
|
||||
rediscover the next one from scratch."""
|
||||
proposed = [
|
||||
("drawer_A", {"source_file": "/f1.md", "chunk_index": 0}),
|
||||
("drawer_A", {"source_file": "/f2.md", "chunk_index": 0}), # collision 1
|
||||
("drawer_B", {"source_file": "/f3.md", "chunk_index": 0}),
|
||||
("drawer_B", {"source_file": "/f4.md", "chunk_index": 0}), # collision 2
|
||||
]
|
||||
col = _MockCollection()
|
||||
with pytest.raises(CollisionError) as exc_info:
|
||||
assert_no_collisions(proposed, col)
|
||||
msg = str(exc_info.value)
|
||||
assert "drawer_A" in msg
|
||||
assert "drawer_B" in msg
|
||||
assert "/f1.md" in msg
|
||||
assert "/f2.md" in msg
|
||||
assert "/f3.md" in msg
|
||||
assert "/f4.md" in msg
|
||||
|
||||
|
||||
# ── Edge cases ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_assert_no_collisions_passes_for_empty_batch():
|
||||
"""An empty mining batch is trivially collision-free — the scan
|
||||
must not raise on len(proposed) == 0 so callers don't have to guard
|
||||
the call site."""
|
||||
assert_no_collisions([], _MockCollection()) is None
|
||||
|
||||
|
||||
def test_assert_no_collisions_handles_metadata_without_chunk_index():
|
||||
"""Some drawer types (e.g. diary entries, sentinels) don't carry
|
||||
chunk_index. The scan should compare whatever metadata is present
|
||||
without crashing on missing keys."""
|
||||
proposed = [
|
||||
("drawer_diary_1", {"source_file": "/diary.md"}),
|
||||
]
|
||||
col = _MockCollection(
|
||||
existing={
|
||||
"drawer_diary_1": {"source_file": "/diary_other.md"},
|
||||
}
|
||||
)
|
||||
with pytest.raises(CollisionError):
|
||||
assert_no_collisions(proposed, col)
|
||||
|
||||
|
||||
def test_assert_no_collisions_tolerates_chromadb_get_failure():
|
||||
"""ChromaDB's get() can raise on transient backend errors. The
|
||||
scan should NOT swallow those — the caller's broad-except can decide
|
||||
whether to abort the mine or proceed. The scan's contract is
|
||||
'either confirms no collisions, or raises'. Hiding backend errors
|
||||
behind a False is the silent-failure shape this PR was meant to
|
||||
eliminate."""
|
||||
|
||||
class _RaisingCollection:
|
||||
def get(self, **kwargs):
|
||||
raise RuntimeError("chromadb is sad")
|
||||
|
||||
proposed = [("drawer_X", {"source_file": "/f.md", "chunk_index": 0})]
|
||||
with pytest.raises(RuntimeError, match="chromadb is sad"):
|
||||
assert_no_collisions(proposed, _RaisingCollection())
|
||||
@@ -17,6 +17,7 @@ def test_default_config():
|
||||
cfg = MempalaceConfig(config_dir=tempfile.mkdtemp())
|
||||
assert "palace" in cfg.palace_path
|
||||
assert cfg.collection_name == "mempalace_drawers"
|
||||
assert cfg.backend == "chroma"
|
||||
|
||||
|
||||
def test_config_from_file():
|
||||
@@ -27,6 +28,54 @@ def test_config_from_file():
|
||||
assert cfg.palace_path == "/custom/palace"
|
||||
|
||||
|
||||
def test_backend_from_config_wins_over_env(tmp_path, monkeypatch):
|
||||
with open(tmp_path / "config.json", "w") as f:
|
||||
json.dump({"backend": "sqlite_exact"}, f)
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND", "chroma")
|
||||
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert cfg.backend == "sqlite_exact"
|
||||
|
||||
|
||||
def test_backend_from_env_when_config_absent(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND", "SQLite_Exact")
|
||||
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert cfg.backend == "sqlite_exact"
|
||||
|
||||
|
||||
def test_qdrant_config_from_env_and_file(tmp_path, monkeypatch):
|
||||
with open(tmp_path / "config.json", "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
"qdrant_url": "http://config.example:6333",
|
||||
"qdrant_api_key": "config-key",
|
||||
"qdrant_namespace": "config-ns",
|
||||
"qdrant_timeout": 2,
|
||||
},
|
||||
f,
|
||||
)
|
||||
monkeypatch.setenv("MEMPALACE_QDRANT_URL", "http://env.example:6333")
|
||||
monkeypatch.setenv("MEMPALACE_QDRANT_API_KEY", "env-key")
|
||||
monkeypatch.setenv("MEMPALACE_QDRANT_NAMESPACE", "env-ns")
|
||||
monkeypatch.setenv("MEMPALACE_QDRANT_TIMEOUT", "3.5")
|
||||
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
|
||||
assert cfg.qdrant_url == "http://env.example:6333"
|
||||
assert cfg.qdrant_api_key == "env-key"
|
||||
assert cfg.qdrant_namespace == "env-ns"
|
||||
assert cfg.qdrant_timeout == 3.5
|
||||
|
||||
|
||||
def test_set_backend_persists_choice(tmp_path):
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
cfg.set_backend("sqlite_exact")
|
||||
|
||||
reloaded = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert reloaded.backend == "sqlite_exact"
|
||||
|
||||
|
||||
def test_embedding_device_defaults_to_auto(monkeypatch):
|
||||
monkeypatch.delenv("MEMPALACE_EMBEDDING_DEVICE", raising=False)
|
||||
cfg = MempalaceConfig(config_dir=tempfile.mkdtemp())
|
||||
@@ -115,6 +164,17 @@ def test_init():
|
||||
cfg = MempalaceConfig(config_dir=tmpdir)
|
||||
cfg.init()
|
||||
assert os.path.exists(os.path.join(tmpdir, "config.json"))
|
||||
with open(os.path.join(tmpdir, "config.json")) as f:
|
||||
saved = json.load(f)
|
||||
assert "backend" not in saved
|
||||
assert MempalaceConfig(config_dir=tmpdir).backend == "chroma"
|
||||
|
||||
|
||||
def test_set_backend_rejects_unknown_backend(tmp_path):
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
|
||||
with pytest.raises(KeyError):
|
||||
cfg.set_backend("does_not_exist")
|
||||
|
||||
|
||||
# --- normalize_wing_name ---
|
||||
@@ -136,6 +196,16 @@ def test_normalize_wing_name_mixed():
|
||||
assert normalize_wing_name("My-Cool App") == "my_cool_app"
|
||||
|
||||
|
||||
def test_normalize_wing_name_strips_leading_separator():
|
||||
# Claude Code path-encoded project dirs begin with a separator; the slug
|
||||
# must not start with "_" or sanitize_name / MCP writes would reject it.
|
||||
assert normalize_wing_name("-home-user-linux-book") == "home_user_linux_book"
|
||||
|
||||
|
||||
def test_normalize_wing_name_strips_trailing_separator():
|
||||
assert normalize_wing_name("project-") == "project"
|
||||
|
||||
|
||||
# --- sanitize_name ---
|
||||
|
||||
|
||||
|
||||
@@ -186,6 +186,64 @@ class TestChunkExchanges:
|
||||
assert chunks[1]["content"] == "b" * CHUNK_SIZE
|
||||
assert chunks[2]["content"] == "c" * CHUNK_SIZE
|
||||
|
||||
def test_ai_response_preserves_blank_lines(self):
|
||||
"""Blank lines inside an AI response must survive ingestion (verbatim principle).
|
||||
|
||||
A response with paragraph breaks separates distinct ideas; collapsing the
|
||||
blank lines loses that boundary and fuses unrelated content.
|
||||
"""
|
||||
# Three `>` turns route through _chunk_by_exchange (the exchange-pair path).
|
||||
content = (
|
||||
"> explain the architecture\n"
|
||||
"First paragraph introducing the system.\n"
|
||||
"\n"
|
||||
"Second paragraph about the data layer.\n"
|
||||
"\n"
|
||||
"Third paragraph about retrieval.\n"
|
||||
"\n"
|
||||
"> what about caching?\n"
|
||||
"Cache lives in memory and is invalidated on write.\n"
|
||||
"\n"
|
||||
"> and persistence?\n"
|
||||
"Persistence lives on disk via SQLite and Chroma.\n"
|
||||
)
|
||||
chunks = chunk_exchanges(content)
|
||||
assert len(chunks) >= 1
|
||||
stored = "\n".join(c["content"] for c in chunks)
|
||||
# Paragraph break between the three bodies must survive as `\n\n`.
|
||||
assert "First paragraph introducing the system.\n\nSecond paragraph" in stored
|
||||
assert "Second paragraph about the data layer.\n\nThird paragraph" in stored
|
||||
|
||||
def test_ai_response_preserves_line_structure(self):
|
||||
"""Line-oriented content (lists, code fences, tables) must keep newlines.
|
||||
|
||||
Joining lines with a single space fuses structurally distinct tokens,
|
||||
breaks downstream search, and destroys code blocks.
|
||||
"""
|
||||
content = (
|
||||
"> show me the steps\n"
|
||||
"1. First step\n"
|
||||
"2. Second step\n"
|
||||
"3. Third step\n"
|
||||
"```python\n"
|
||||
"def hello():\n"
|
||||
" return 'world'\n"
|
||||
"```\n"
|
||||
"\n"
|
||||
"> what next?\n"
|
||||
"Run the test suite.\n"
|
||||
"\n"
|
||||
"> anything else?\n"
|
||||
"Ship the feature.\n"
|
||||
)
|
||||
chunks = chunk_exchanges(content)
|
||||
assert len(chunks) >= 1
|
||||
stored = "\n".join(c["content"] for c in chunks)
|
||||
# Each list item keeps its own line (not "1. First step 2. Second step").
|
||||
assert "1. First step\n2. Second step\n3. Third step" in stored
|
||||
# Code fence survives intact, with indentation preserved.
|
||||
assert "```python\ndef hello():\n return 'world'\n```" in stored
|
||||
|
||||
|
||||
class TestEmitBounded:
|
||||
"""Direct unit tests for the chunk-size-enforcement helper."""
|
||||
@@ -385,6 +443,11 @@ class TestFileChunksLocked:
|
||||
def delete(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def get(self, ids=None, include=None, **kwargs):
|
||||
# Pre-mining collision scan probes the collection; empty
|
||||
# palace under test, so nothing matches.
|
||||
return {"ids": [], "metadatas": []}
|
||||
|
||||
def upsert(self, documents, ids, metadatas):
|
||||
self.batch_sizes.append(len(documents))
|
||||
|
||||
|
||||
+15
-17
@@ -198,15 +198,13 @@ def test_dedup_source_group_query_failure_keeps():
|
||||
# ── show_stats ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _install_mock_backend(mock_backend_cls, collection):
|
||||
mock_backend = MagicMock()
|
||||
mock_backend.get_collection.return_value = collection
|
||||
mock_backend_cls.return_value = mock_backend
|
||||
return mock_backend
|
||||
def _install_mock_collection(mock_get_collection, collection):
|
||||
mock_get_collection.return_value = collection
|
||||
return collection
|
||||
|
||||
|
||||
@patch("mempalace.dedup.ChromaBackend")
|
||||
def test_show_stats(mock_backend_cls, tmp_path):
|
||||
@patch("mempalace.dedup.get_collection")
|
||||
def test_show_stats(mock_get_collection, tmp_path):
|
||||
mock_col = MagicMock()
|
||||
mock_col.count.return_value = 5
|
||||
mock_col.get.side_effect = [
|
||||
@@ -222,7 +220,7 @@ def test_show_stats(mock_backend_cls, tmp_path):
|
||||
},
|
||||
{"ids": []},
|
||||
]
|
||||
_install_mock_backend(mock_backend_cls, mock_col)
|
||||
_install_mock_collection(mock_get_collection, mock_col)
|
||||
|
||||
dedup.show_stats(palace_path=str(tmp_path)) # should not raise
|
||||
|
||||
@@ -232,11 +230,11 @@ def test_show_stats(mock_backend_cls, tmp_path):
|
||||
|
||||
@patch("mempalace.dedup.dedup_source_group")
|
||||
@patch("mempalace.dedup.get_source_groups")
|
||||
@patch("mempalace.dedup.ChromaBackend")
|
||||
def test_dedup_palace_dry_run(mock_backend_cls, mock_groups, mock_dedup_group, tmp_path):
|
||||
@patch("mempalace.dedup.get_collection")
|
||||
def test_dedup_palace_dry_run(mock_get_collection, mock_groups, mock_dedup_group, tmp_path):
|
||||
mock_col = MagicMock()
|
||||
mock_col.count.return_value = 10
|
||||
_install_mock_backend(mock_backend_cls, mock_col)
|
||||
_install_mock_collection(mock_get_collection, mock_col)
|
||||
|
||||
mock_groups.return_value = {"a.txt": ["d1", "d2", "d3", "d4", "d5"]}
|
||||
mock_dedup_group.return_value = (["d1", "d2", "d3"], ["d4", "d5"])
|
||||
@@ -247,11 +245,11 @@ def test_dedup_palace_dry_run(mock_backend_cls, mock_groups, mock_dedup_group, t
|
||||
|
||||
@patch("mempalace.dedup.dedup_source_group")
|
||||
@patch("mempalace.dedup.get_source_groups")
|
||||
@patch("mempalace.dedup.ChromaBackend")
|
||||
def test_dedup_palace_with_wing(mock_backend_cls, mock_groups, mock_dedup_group, tmp_path):
|
||||
@patch("mempalace.dedup.get_collection")
|
||||
def test_dedup_palace_with_wing(mock_get_collection, mock_groups, mock_dedup_group, tmp_path):
|
||||
mock_col = MagicMock()
|
||||
mock_col.count.return_value = 10
|
||||
_install_mock_backend(mock_backend_cls, mock_col)
|
||||
_install_mock_collection(mock_get_collection, mock_col)
|
||||
|
||||
mock_groups.return_value = {}
|
||||
dedup.dedup_palace(palace_path=str(tmp_path), wing="test_wing", dry_run=True)
|
||||
@@ -260,11 +258,11 @@ def test_dedup_palace_with_wing(mock_backend_cls, mock_groups, mock_dedup_group,
|
||||
|
||||
@patch("mempalace.dedup.dedup_source_group")
|
||||
@patch("mempalace.dedup.get_source_groups")
|
||||
@patch("mempalace.dedup.ChromaBackend")
|
||||
def test_dedup_palace_no_groups(mock_backend_cls, mock_groups, mock_dedup_group, tmp_path):
|
||||
@patch("mempalace.dedup.get_collection")
|
||||
def test_dedup_palace_no_groups(mock_get_collection, mock_groups, mock_dedup_group, tmp_path):
|
||||
mock_col = MagicMock()
|
||||
mock_col.count.return_value = 3
|
||||
_install_mock_backend(mock_backend_cls, mock_col)
|
||||
_install_mock_collection(mock_get_collection, mock_col)
|
||||
|
||||
mock_groups.return_value = {}
|
||||
dedup.dedup_palace(palace_path=str(tmp_path), dry_run=True)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""EmbeddingCollection OneOrMany handling (PR #1706 review).
|
||||
|
||||
A bare ``str`` passed as ``documents``/``query_texts`` (ChromaDB's OneOrMany
|
||||
shape) must be wrapped, not iterated — otherwise ``list("abc")`` embeds per
|
||||
character and breaks length alignment with ids/metadatas on explicit-vector
|
||||
backends.
|
||||
"""
|
||||
|
||||
from mempalace.backends import embedding_wrapper as ew
|
||||
|
||||
|
||||
class _FakeInner:
|
||||
"""Captures what the wrapper delegates to the backend."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = {}
|
||||
|
||||
def add(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
self.calls["add"] = {
|
||||
"documents": documents,
|
||||
"ids": ids,
|
||||
"metadatas": metadatas,
|
||||
"embeddings": embeddings,
|
||||
}
|
||||
|
||||
def upsert(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
self.calls["upsert"] = {
|
||||
"documents": documents,
|
||||
"ids": ids,
|
||||
"metadatas": metadatas,
|
||||
"embeddings": embeddings,
|
||||
}
|
||||
|
||||
def update(self, *, ids, documents=None, metadatas=None, embeddings=None):
|
||||
self.calls["update"] = {
|
||||
"documents": documents,
|
||||
"ids": ids,
|
||||
"metadatas": metadatas,
|
||||
"embeddings": embeddings,
|
||||
}
|
||||
|
||||
def query(self, *, query_texts=None, query_embeddings=None, **_kw):
|
||||
self.calls["query"] = {"query_texts": query_texts, "query_embeddings": query_embeddings}
|
||||
from mempalace.backends.base import QueryResult
|
||||
|
||||
return QueryResult.empty()
|
||||
|
||||
|
||||
def _patch_embed(monkeypatch):
|
||||
"""Stub the embedder: one vector per input text, recording the inputs."""
|
||||
seen = {}
|
||||
|
||||
def fake(texts):
|
||||
seen["texts"] = texts
|
||||
return [[0.0, 0.0] for _ in texts]
|
||||
|
||||
monkeypatch.setattr(ew, "_embed_texts", fake)
|
||||
return seen
|
||||
|
||||
|
||||
def test_as_list_wraps_bare_string():
|
||||
assert ew._as_list("hello world") == ["hello world"]
|
||||
assert ew._as_list({"k": 1}) == [{"k": 1}] # bare dict wrapped, not -> ["k"]
|
||||
src = ["a", "b"]
|
||||
assert ew._as_list(src) is src # list returned as-is (no copy)
|
||||
assert ew._as_list(("a", "b")) == ["a", "b"] # other iterables materialized
|
||||
|
||||
|
||||
def test_add_wraps_bare_string_document(monkeypatch):
|
||||
seen = _patch_embed(monkeypatch)
|
||||
inner = _FakeInner()
|
||||
ew.EmbeddingCollection(inner).add(documents="hello world", ids=["d1"])
|
||||
# embedded as one whole document, not per character
|
||||
assert seen["texts"] == ["hello world"]
|
||||
# and the backend receives a list, length-aligned with ids
|
||||
assert inner.calls["add"]["documents"] == ["hello world"]
|
||||
assert len(inner.calls["add"]["embeddings"]) == 1
|
||||
|
||||
|
||||
def test_upsert_wraps_bare_string_document(monkeypatch):
|
||||
seen = _patch_embed(monkeypatch)
|
||||
inner = _FakeInner()
|
||||
ew.EmbeddingCollection(inner).upsert(documents="solo", ids=["d1"])
|
||||
assert seen["texts"] == ["solo"]
|
||||
assert inner.calls["upsert"]["documents"] == ["solo"]
|
||||
assert len(inner.calls["upsert"]["embeddings"]) == 1
|
||||
|
||||
|
||||
def test_update_wraps_bare_string_document(monkeypatch):
|
||||
seen = _patch_embed(monkeypatch)
|
||||
inner = _FakeInner()
|
||||
ew.EmbeddingCollection(inner).update(ids=["d1"], documents="changed")
|
||||
assert seen["texts"] == ["changed"]
|
||||
assert inner.calls["update"]["documents"] == ["changed"]
|
||||
assert len(inner.calls["update"]["embeddings"]) == 1
|
||||
|
||||
|
||||
def test_query_wraps_bare_string(monkeypatch):
|
||||
seen = _patch_embed(monkeypatch)
|
||||
inner = _FakeInner()
|
||||
ew.EmbeddingCollection(inner).query(query_texts="find me")
|
||||
assert seen["texts"] == ["find me"]
|
||||
# query_texts is consumed into a single query embedding
|
||||
assert len(inner.calls["query"]["query_embeddings"]) == 1
|
||||
assert inner.calls["query"]["query_texts"] is None
|
||||
|
||||
|
||||
def test_list_inputs_unaffected(monkeypatch):
|
||||
seen = _patch_embed(monkeypatch)
|
||||
inner = _FakeInner()
|
||||
ew.EmbeddingCollection(inner).add(documents=["one", "two"], ids=["a", "b"])
|
||||
assert seen["texts"] == ["one", "two"]
|
||||
assert len(inner.calls["add"]["embeddings"]) == 2
|
||||
|
||||
|
||||
def test_add_wraps_bare_string_ids_and_dict_metadatas(monkeypatch):
|
||||
_patch_embed(monkeypatch)
|
||||
inner = _FakeInner()
|
||||
# a single id (str) and a single metadata (dict) are OneOrMany shapes too
|
||||
ew.EmbeddingCollection(inner).add(documents="solo", ids="d1", metadatas={"src": "web"})
|
||||
call = inner.calls["add"]
|
||||
assert call["ids"] == ["d1"] # not ['d', '1']
|
||||
assert call["metadatas"] == [{"src": "web"}] # not ['src']
|
||||
# documents / embeddings / ids / metadatas all length-aligned at 1
|
||||
assert call["documents"] == ["solo"]
|
||||
assert len(call["embeddings"]) == 1
|
||||
@@ -126,7 +126,7 @@ def test_lazy_load_runs_once(patched_lazy_load):
|
||||
ef(["one"])
|
||||
ef(["two"])
|
||||
ef(["three"])
|
||||
assert patched_lazy_load["hf_hub_download"] == 2 # model + tokenizer, once total
|
||||
assert patched_lazy_load["hf_hub_download"] == 3 # model + weights + tokenizer, once
|
||||
assert patched_lazy_load["InferenceSession"] == 1
|
||||
assert patched_lazy_load["Tokenizer.from_file"] == 1
|
||||
|
||||
|
||||
@@ -166,6 +166,148 @@ def test_coca_wordlist_contains_all_known_aya_false_positives():
|
||||
assert not missing, f"COCA wordlist missing known false positives: {missing!r}"
|
||||
|
||||
|
||||
# ── Known-systems lexicon + compound matcher (Tier 3 linguistics cleanup) ─
|
||||
|
||||
|
||||
def test_extract_candidates_detects_claude_code_as_atomic_compound():
|
||||
"""The flagship Tier 3 case: 'Claude Code' must be detected as a single
|
||||
atomic compound, NOT decomposed into 'Claude' + 'Code' (where 'Code'
|
||||
would then get COCA-filtered, leaving 'Claude' alone with wrong count).
|
||||
"""
|
||||
text = (
|
||||
"Claude Code helped fix the bug. Claude Code refactored the loop. "
|
||||
"Claude Code wrote the tests. Claude Code shipped clean."
|
||||
)
|
||||
result = extract_candidates(text)
|
||||
assert "Claude Code" in result, (
|
||||
f"'Claude Code' must be detected as a single compound; got: {list(result.keys())!r}"
|
||||
)
|
||||
assert result["Claude Code"] >= 4
|
||||
|
||||
|
||||
def test_extract_candidates_does_not_decompose_known_compound():
|
||||
"""When 'Claude Code' appears as a compound, the single-word 'Claude'
|
||||
and 'Code' must NOT appear in results (would be wrong attribution +
|
||||
COCA-filtered respectively)."""
|
||||
text = (
|
||||
"Claude Code wrote the patch. Claude Code reviewed it. "
|
||||
"Claude Code ran the tests. Claude Code merged. Claude Code shipped."
|
||||
)
|
||||
result = extract_candidates(text)
|
||||
assert "Claude Code" in result
|
||||
assert "Claude" not in result, (
|
||||
f"'Claude' alone should NOT appear when only mentioned as part of "
|
||||
f"'Claude Code'; got: {list(result.keys())!r}"
|
||||
)
|
||||
# 'Code' would be COCA-filtered anyway, but assert defensively.
|
||||
assert "Code" not in result
|
||||
|
||||
|
||||
def test_extract_candidates_compound_matching_is_case_insensitive():
|
||||
"""'claude code' (lowercase) and 'CLAUDE CODE' (uppercase) and mixed
|
||||
case all match the canonical 'Claude Code' compound."""
|
||||
text = (
|
||||
"claude code is great. CLAUDE CODE works. Claude Code rocks. "
|
||||
"Claude code rules. CLaudE coDe wins."
|
||||
)
|
||||
result = extract_candidates(text)
|
||||
assert "Claude Code" in result, (
|
||||
f"Case-insensitive compound matching must work; got: {list(result.keys())!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_extract_candidates_real_single_word_name_still_detected():
|
||||
"""Tier 3 does NOT regress single-word proper-noun detection.
|
||||
'Aya' appearing as a standalone name 3+ times still gets detected."""
|
||||
text = "Aya wrote the spec. Aya reviewed Cedar's PR. Aya shipped to develop."
|
||||
result = extract_candidates(text)
|
||||
assert "Aya" in result, (
|
||||
f"Single-word real name 'Aya' should still be detected; got: {list(result.keys())!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_extract_candidates_single_word_code_still_coca_filtered():
|
||||
"""Tier 3 must not regress Tier 2. 'Code' alone is still a content
|
||||
word and filtered by COCA, regardless of the compound pre-pass."""
|
||||
text = "Code is fun. Code works. Code helps. Code rules. Code wins."
|
||||
result = extract_candidates(text)
|
||||
assert "Code" not in result, (
|
||||
f"'Code' standalone should still be COCA-filtered; got: {list(result.keys())!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_extract_candidates_detects_multiple_distinct_compounds():
|
||||
"""Multiple known compounds in the same text are each detected
|
||||
independently."""
|
||||
text = (
|
||||
"Claude Code wrote the code. Claude Code reviewed.\n"
|
||||
"GitHub Copilot suggested a fix. GitHub Copilot autocompleted.\n"
|
||||
"Claude Code merged. GitHub Copilot helped.\n"
|
||||
)
|
||||
result = extract_candidates(text)
|
||||
assert "Claude Code" in result, f"'Claude Code' missing; got: {list(result.keys())!r}"
|
||||
assert "GitHub Copilot" in result, f"'GitHub Copilot' missing; got: {list(result.keys())!r}"
|
||||
|
||||
|
||||
def test_known_systems_file_loads_with_expected_shape():
|
||||
"""The data file ships with a stable schema."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import mempalace
|
||||
|
||||
pkg_dir = Path(mempalace.__file__).parent
|
||||
p = pkg_dir / "data" / "known_systems.json"
|
||||
assert p.exists(), f"known_systems.json must exist at {p}"
|
||||
d = json.loads(p.read_text(encoding="utf-8"))
|
||||
assert d.get("schema_version") == 1, (
|
||||
f"schema_version must be 1; got {d.get('schema_version')!r}"
|
||||
)
|
||||
compounds = d.get("compounds")
|
||||
assert isinstance(compounds, list), (
|
||||
f"'compounds' must be a list; got {type(compounds).__name__}"
|
||||
)
|
||||
assert len(compounds) >= 20, f"known_systems must have >=20 entries; got {len(compounds)}"
|
||||
# Every entry must be multi-token (space or hyphen separated)
|
||||
single_word = [c for c in compounds if " " not in c and "-" not in c]
|
||||
assert not single_word, (
|
||||
f"All entries must be multi-token compounds; single-word entries found: {single_word[:5]!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_known_systems_file_contains_expected_high_value_entries():
|
||||
"""The file must contain the most-common AI/dev product compounds
|
||||
users mention in real palaces."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import mempalace
|
||||
|
||||
pkg_dir = Path(mempalace.__file__).parent
|
||||
p = pkg_dir / "data" / "known_systems.json"
|
||||
compounds = set(json.loads(p.read_text(encoding="utf-8"))["compounds"])
|
||||
must_have = [
|
||||
"Claude Code",
|
||||
"GitHub Copilot",
|
||||
"Visual Studio Code",
|
||||
"Gemini Code Assist",
|
||||
"Docker Desktop",
|
||||
"GitHub Actions",
|
||||
]
|
||||
missing = [c for c in must_have if c not in compounds]
|
||||
assert not missing, f"known_systems missing high-value entries: {missing!r}"
|
||||
|
||||
|
||||
def test_extract_candidates_unknown_two_word_phrase_still_works():
|
||||
"""Tier 3 does NOT regress the multi-word regex path. A two-word
|
||||
proper-noun phrase NOT in known_systems still gets detected via
|
||||
the existing multi-word pattern."""
|
||||
text = "Jane Smith wrote it. Jane Smith reviewed. Jane Smith shipped. Jane Smith merged."
|
||||
result = extract_candidates(text)
|
||||
assert "Jane Smith" in result, (
|
||||
f"Unknown compound 'Jane Smith' should still be detected via "
|
||||
f"multi-word regex; got: {list(result.keys())!r}"
|
||||
)
|
||||
|
||||
|
||||
# ── score_entity ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
+14
-4
@@ -27,11 +27,21 @@ def _use_tmp_hallway_file(monkeypatch, tmp_path):
|
||||
|
||||
|
||||
def _fake_collection(drawers):
|
||||
"""Build a MagicMock collection whose .get() returns the given drawer set."""
|
||||
"""Build a MagicMock collection over ``drawers`` that supports the paginated
|
||||
fetch (``count()`` + ``get(limit=, offset=)``) that compute_hallways_for_wing
|
||||
uses to stay under SQLite's variable limit (#1619)."""
|
||||
col = MagicMock()
|
||||
metadatas = [d for d in drawers]
|
||||
ids = [f"drawer_{i}" for i in range(len(drawers))]
|
||||
col.get.return_value = {"ids": ids, "metadatas": metadatas}
|
||||
metas = [d for d in drawers]
|
||||
col.count.return_value = len(metas)
|
||||
|
||||
def _get(limit=None, offset=0, include=None, where=None, ids=None, **kwargs):
|
||||
page = metas[offset : offset + limit] if limit is not None else metas
|
||||
return {
|
||||
"ids": [f"drawer_{i}" for i in range(offset, offset + len(page))],
|
||||
"metadatas": page,
|
||||
}
|
||||
|
||||
col.get.side_effect = _get
|
||||
return col
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Regression test for #1619.
|
||||
|
||||
``compute_hallways_for_wing`` must fetch drawers by paginating
|
||||
(``count()`` + ``get(limit=, offset=)``) and filtering the wing client-side,
|
||||
NOT with a single ``get(where={"wing": wing})`` — the latter binds one SQL
|
||||
variable per matched id and overflows SQLite's ``SQLITE_MAX_VARIABLE_NUMBER``
|
||||
(32766) on wings larger than ~32k drawers, silently leaving the hallway graph
|
||||
unbuilt on exactly the large wings that benefit most.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
with patch.dict("sys.modules", {"chromadb": MagicMock()}):
|
||||
from mempalace import hallways as hallways_mod
|
||||
|
||||
|
||||
def _use_tmp_hallway_file(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(hallways_mod, "_HALLWAY_FILE", str(tmp_path / "hallways.json"))
|
||||
|
||||
|
||||
def _collection_that_rejects_where_get(drawers):
|
||||
"""count() + paginated get(limit,offset) work; a where-get raises, exactly
|
||||
as ChromaDB does when the bound-variable count overflows on a big wing."""
|
||||
col = MagicMock()
|
||||
col.count.return_value = len(drawers)
|
||||
|
||||
def _get(limit=None, offset=0, include=None, where=None, ids=None, **kw):
|
||||
if where is not None and limit is None:
|
||||
raise RuntimeError("Error executing plan: too many SQL variables")
|
||||
filtered_drawers = drawers
|
||||
if where and "wing" in where:
|
||||
target_wing = where["wing"]
|
||||
filtered_drawers = [
|
||||
d for d in drawers if isinstance(d, dict) and d.get("wing") == target_wing
|
||||
]
|
||||
page = filtered_drawers[offset : offset + limit] if limit is not None else filtered_drawers
|
||||
return {
|
||||
"ids": [f"d{i}" for i in range(offset, offset + len(page))],
|
||||
"metadatas": page,
|
||||
}
|
||||
|
||||
col.get.side_effect = _get
|
||||
return col
|
||||
|
||||
|
||||
class TestComputeHallwaysPagination:
|
||||
def test_large_wing_builds_hallways_via_pagination(self, tmp_path, monkeypatch):
|
||||
_use_tmp_hallway_file(monkeypatch, tmp_path)
|
||||
# 3 drawers all co-placing Alice+Bob → one hallway at min_count=2,
|
||||
# but ONLY if the fetch paginates instead of the variable-bound where-get.
|
||||
drawers = [{"wing": "wing_alpha", "room": "diary", "entities": "Alice;Bob"}] * 3
|
||||
col = _collection_that_rejects_where_get(drawers)
|
||||
result = hallways_mod.compute_hallways_for_wing("wing_alpha", col=col)
|
||||
assert any({h["entity_a"], h["entity_b"]} == {"Alice", "Bob"} for h in result), (
|
||||
"hallways came back empty — the where-get path crashed; the fetch must paginate (#1619)"
|
||||
)
|
||||
+86
-2
@@ -13,6 +13,7 @@ import mempalace.hooks_cli as hooks_cli_mod
|
||||
from mempalace.hooks_cli import (
|
||||
SAVE_INTERVAL,
|
||||
_count_human_messages,
|
||||
_diary_agent_for_harness,
|
||||
_extract_recent_messages,
|
||||
_get_mine_targets,
|
||||
_log,
|
||||
@@ -22,6 +23,7 @@ from mempalace.hooks_cli import (
|
||||
_mine_sync,
|
||||
_parse_harness_input,
|
||||
_sanitize_session_id,
|
||||
_save_diary_direct,
|
||||
_validate_transcript_path,
|
||||
_wing_from_transcript_path,
|
||||
hook_stop,
|
||||
@@ -336,7 +338,9 @@ def test_stop_hook_saves_silently_at_interval(tmp_path):
|
||||
assert result["systemMessage"].startswith("\u2726 15 memories woven into the palace")
|
||||
assert "hooks" in result["systemMessage"]
|
||||
# tmp_path has no "-Projects-" segment, so _wing_from_transcript_path falls back to "wing_sessions"
|
||||
mock_save.assert_called_once_with(str(transcript), "test", wing="wing_sessions", toast=False)
|
||||
mock_save.assert_called_once_with(
|
||||
str(transcript), "test", wing="wing_sessions", toast=False, agent_name="claude"
|
||||
)
|
||||
|
||||
|
||||
def test_stop_hook_derives_wing_from_transcript_path(tmp_path):
|
||||
@@ -355,7 +359,9 @@ def test_stop_hook_derives_wing_from_transcript_path(tmp_path):
|
||||
{"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)},
|
||||
state_dir=tmp_path,
|
||||
)
|
||||
mock_save.assert_called_once_with(str(transcript), "test", wing="wing_myproject", toast=False)
|
||||
mock_save.assert_called_once_with(
|
||||
str(transcript), "test", wing="wing_myproject", toast=False, agent_name="claude"
|
||||
)
|
||||
|
||||
|
||||
def test_stop_hook_tracks_save_point(tmp_path):
|
||||
@@ -383,6 +389,84 @@ def test_stop_hook_tracks_save_point(tmp_path):
|
||||
mock_save.assert_not_called()
|
||||
|
||||
|
||||
# --- #1693: hook checkpoints must be discoverable by diary_read ---
|
||||
|
||||
|
||||
def test_diary_agent_for_harness_maps_known_harnesses():
|
||||
assert _diary_agent_for_harness("claude-code") == "claude"
|
||||
assert _diary_agent_for_harness("codex") == "codex"
|
||||
|
||||
|
||||
def test_diary_agent_for_harness_unknown_falls_back_to_name():
|
||||
"""A future harness must never collapse to the legacy 'session-hook'
|
||||
identity, which no diary_read(agent_name=...) call ever matches (#1693)."""
|
||||
assert _diary_agent_for_harness("cursor") == "cursor"
|
||||
for harness in ("claude-code", "codex", "cursor", "gemini"):
|
||||
assert _diary_agent_for_harness(harness) != "session-hook"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"harness,expected_agent",
|
||||
[("claude-code", "claude"), ("codex", "codex")],
|
||||
)
|
||||
def test_stop_hook_files_checkpoint_under_harness_agent(tmp_path, harness, expected_agent):
|
||||
"""The Stop hook must file checkpoints under the agent identity that the
|
||||
session's harness reads with, not the legacy hardcoded 'session-hook'
|
||||
(#1693)."""
|
||||
# _save_diary_direct is mocked below, so the transcript format is irrelevant
|
||||
# here: _count_human_messages counts both harness shapes, and we assert only
|
||||
# the harness -> agent_name routing, not transcript parsing.
|
||||
transcript = tmp_path / "t.jsonl"
|
||||
_write_transcript(
|
||||
transcript,
|
||||
[{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL)],
|
||||
)
|
||||
with patch(
|
||||
"mempalace.hooks_cli._save_diary_direct", return_value={"count": 5, "themes": []}
|
||||
) as mock_save:
|
||||
_capture_hook_output(
|
||||
hook_stop,
|
||||
{"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)},
|
||||
harness=harness,
|
||||
state_dir=tmp_path,
|
||||
)
|
||||
assert mock_save.call_args.kwargs["agent_name"] == expected_agent
|
||||
|
||||
|
||||
def test_stop_hook_checkpoint_visible_to_diary_read(monkeypatch, config, palace_path, kg, tmp_path):
|
||||
"""End-to-end regression for #1693: a checkpoint written by the Stop hook
|
||||
save path is discoverable via diary_read under the harness agent identity,
|
||||
and is not siloed under the legacy 'session-hook' identity."""
|
||||
import chromadb
|
||||
|
||||
from mempalace import mcp_server
|
||||
from mempalace.mcp_server import tool_diary_read
|
||||
|
||||
monkeypatch.setattr(mcp_server, "_config", config)
|
||||
monkeypatch.setattr(mcp_server, "_get_kg", lambda *a, **kw: kg)
|
||||
client = chromadb.PersistentClient(path=palace_path)
|
||||
client.get_or_create_collection("mempalace_drawers", metadata={"hnsw:space": "cosine"})
|
||||
del client
|
||||
|
||||
transcript = tmp_path / "session.jsonl"
|
||||
_write_transcript(
|
||||
transcript,
|
||||
[{"message": {"role": "user", "content": f"msg {i}"}} for i in range(5)],
|
||||
)
|
||||
|
||||
agent = _diary_agent_for_harness("claude-code")
|
||||
res = _save_diary_direct(str(transcript), "sess1", wing="wing_.claude", agent_name=agent)
|
||||
assert res["count"] > 0
|
||||
|
||||
visible = tool_diary_read(agent_name="claude")
|
||||
assert visible.get("total", 0) >= 1
|
||||
assert "CHECKPOINT" in visible["entries"][0]["content"]
|
||||
|
||||
# The legacy identity no longer captures hook checkpoints.
|
||||
legacy = tool_diary_read(agent_name="session-hook")
|
||||
assert legacy.get("entries") == []
|
||||
|
||||
|
||||
# --- hook_session_start ---
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Tests for mempalace.ids — collision-safe ID construction.
|
||||
|
||||
The RED test that pins this whole PR is
|
||||
``test_make_drawer_id_from_chunk_does_not_collide_across_boundary`` — it
|
||||
constructs the classic ``"/path/a1" + "23" == "/path/a" + "123"`` collision
|
||||
shape and asserts the new delimiter-based recipe produces distinct IDs.
|
||||
Against the pre-v2 recipe (no delimiter), this test FAILS. Against v2,
|
||||
it PASSES.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
from mempalace import ids
|
||||
|
||||
|
||||
# ── ID_RECIPE constant ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_id_recipe_constant_is_v2():
|
||||
"""Audit code reads ids.ID_RECIPE to tag new drawers. The constant
|
||||
must be the literal "v2" string; a typo here silently re-introduces
|
||||
the ambiguity v2 was meant to fix."""
|
||||
assert ids.ID_RECIPE == "v2"
|
||||
|
||||
|
||||
# ── make_drawer_id_from_chunk ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_make_drawer_id_from_chunk_returns_expected_prefix():
|
||||
"""Drawer IDs are namespaced by wing and room so cross-wing
|
||||
collisions are impossible regardless of the hash slice."""
|
||||
result = ids.make_drawer_id_from_chunk("proj", "log", "/a", 0)
|
||||
assert result.startswith("drawer_proj_log_")
|
||||
|
||||
|
||||
def test_make_drawer_id_from_chunk_hash_length_is_24_hex():
|
||||
"""The hash slice must be 24 hex chars to keep drawer IDs storable
|
||||
in fixed-width metadata columns and to match the historical recipe
|
||||
length."""
|
||||
result = ids.make_drawer_id_from_chunk("w", "r", "/file.md", 5)
|
||||
hash_part = result.removeprefix("drawer_w_r_")
|
||||
assert len(hash_part) == 24
|
||||
assert all(c in "0123456789abcdef" for c in hash_part)
|
||||
|
||||
|
||||
def test_make_drawer_id_from_chunk_does_not_collide_across_boundary():
|
||||
"""RED test pinning the whole PR.
|
||||
|
||||
Classic collision: source_file="/path/a1" + chunk_index=23 produces
|
||||
hash input "/path/a123" under the pre-v2 recipe. source_file="/path/a"
|
||||
+ chunk_index=123 produces the SAME "/path/a123" — same hash, same
|
||||
drawer_id, second ChromaDB upsert overwrites the first.
|
||||
|
||||
Under the v2 recipe (delimiter '|'), the two inputs become
|
||||
"/path/a1|23" and "/path/a|123" — distinct strings, distinct
|
||||
hashes, distinct drawer IDs. No collision.
|
||||
"""
|
||||
a = ids.make_drawer_id_from_chunk("w", "r", "/path/a1", 23)
|
||||
b = ids.make_drawer_id_from_chunk("w", "r", "/path/a", 123)
|
||||
assert a != b, f"Collision survived v2 recipe: {a!r} == {b!r}"
|
||||
|
||||
|
||||
def test_make_drawer_id_from_chunk_is_deterministic():
|
||||
"""Same inputs must always produce the same ID — re-mining a file
|
||||
that hasn't changed must hit the same drawer slot, or
|
||||
file_already_mined() loses its idempotency."""
|
||||
a = ids.make_drawer_id_from_chunk("w", "r", "/file.md", 7)
|
||||
b = ids.make_drawer_id_from_chunk("w", "r", "/file.md", 7)
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_make_drawer_id_from_chunk_windows_path_with_colon_does_not_collide():
|
||||
"""Windows paths contain ':' in drive letters (C:\\Users\\...). The
|
||||
v2 recipe uses '|' precisely so paths that contain ':' can never
|
||||
align with a chunk index to collide. This test would FAIL on a
|
||||
':'-delimited recipe because Windows paths and URL-like paths
|
||||
(https://host:8080) commonly end in ':digits'."""
|
||||
a = ids.make_drawer_id_from_chunk("w", "r", "C:\\Users\\foo", 5)
|
||||
b = ids.make_drawer_id_from_chunk("w", "r", "C:\\Users\\foo:", 5)
|
||||
assert a != b
|
||||
|
||||
|
||||
# ── make_drawer_id_from_content ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_make_drawer_id_from_content_does_not_collide_across_boundary():
|
||||
"""mcp_server.py:1136 hashes wing+room+content with no delimiter.
|
||||
Architecturally identical defect to the chunk-index sites:
|
||||
wing="foo"+room="bar" hashes the same as wing="fooba"+room="r".
|
||||
v2 delimiter breaks this."""
|
||||
a = ids.make_drawer_id_from_content("foo", "bar", "x")
|
||||
b = ids.make_drawer_id_from_content("fooba", "r", "x")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_make_drawer_id_from_content_returns_expected_prefix():
|
||||
"""Same namespacing pattern as the chunk-index helper."""
|
||||
result = ids.make_drawer_id_from_content("proj", "scratch", "hello")
|
||||
assert result.startswith("drawer_proj_scratch_")
|
||||
|
||||
|
||||
# ── make_convo_drawer_id ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_make_convo_drawer_id_does_not_collide_across_extract_mode_boundary():
|
||||
"""convo_miner.py:422 hashes source_file+extract_mode+chunk_index.
|
||||
Pre-v2 used ':' as delimiter — this test would still PASS on the ':'
|
||||
recipe for clean inputs, but the migration to '|' is for
|
||||
consistency with the chunk-index helpers and to remove the
|
||||
Windows-path / URL-source edge case where ':' can appear in the
|
||||
source_file itself."""
|
||||
a = ids.make_convo_drawer_id("w", "r", "/log.jsonl", "general", 5)
|
||||
b = ids.make_convo_drawer_id("w", "r", "/log.jsonl", "extract", 5)
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_make_convo_drawer_id_returns_expected_prefix():
|
||||
result = ids.make_convo_drawer_id("claude", "diary", "/c.jsonl", "general", 0)
|
||||
assert result.startswith("drawer_claude_diary_")
|
||||
|
||||
|
||||
# ── make_convo_sentinel_id ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_make_convo_sentinel_id_returns_expected_prefix():
|
||||
"""Sentinel IDs are namespaced under '_reg_' so they can be
|
||||
filtered out of normal drawer queries."""
|
||||
result = ids.make_convo_sentinel_id("/c.jsonl", "general")
|
||||
assert result.startswith("_reg_")
|
||||
|
||||
|
||||
def test_make_convo_sentinel_id_distinguishes_extract_modes():
|
||||
a = ids.make_convo_sentinel_id("/c.jsonl", "general")
|
||||
b = ids.make_convo_sentinel_id("/c.jsonl", "extract")
|
||||
assert a != b
|
||||
|
||||
|
||||
# ── make_triple_id ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_make_triple_id_returns_expected_prefix():
|
||||
"""Triple IDs prefix with 't_' and embed the subject/predicate/object
|
||||
triple in the ID for grep-ability in SQLite."""
|
||||
result = ids.make_triple_id("sub1", "loves", "obj1", "2026-01-01", "2026-05-30T10:00:00")
|
||||
assert result.startswith("t_sub1_loves_obj1_")
|
||||
|
||||
|
||||
def test_make_triple_id_hash_length_is_12_hex():
|
||||
"""Triple IDs historically truncate at 12 hex chars (vs 24 for
|
||||
drawers) because the subject/predicate/object prefix already
|
||||
supplies the bulk of the namespace."""
|
||||
result = ids.make_triple_id("s", "p", "o", "2026-01-01", "2026-05-30T10:00:00")
|
||||
hash_part = result.removeprefix("t_s_p_o_")
|
||||
assert len(hash_part) == 12
|
||||
|
||||
|
||||
def test_make_triple_id_does_not_collide_across_iso_datetime_boundary():
|
||||
"""Pre-v2 hash input was f'{valid_from}{recorded_at}' with no
|
||||
delimiter — two ISO datetimes concatenated could in principle
|
||||
collide (valid_from='2026-01-01' + recorded_at='T12:00:00' ==
|
||||
valid_from='2026-01-01T12' + recorded_at=':00:00' for hash
|
||||
purposes). v2 delimiter prevents this."""
|
||||
a = ids.make_triple_id("s", "p", "o", "2026-01-01", "T12:00:00")
|
||||
b = ids.make_triple_id("s", "p", "o", "2026-01-01T12", ":00:00")
|
||||
assert a != b
|
||||
|
||||
|
||||
# ── _delimited_sha256 (private helper, smoke test only) ───────────────
|
||||
|
||||
|
||||
def test_private_delimited_sha256_uses_pipe_delimiter():
|
||||
"""Confirms the implementation actually uses '|' and not ':' — a
|
||||
subtle copy-paste from the diary_ingest precedent or a stale
|
||||
':' precedent from convo_miner could regress the delimiter without
|
||||
breaking the higher-level tests."""
|
||||
result = ids._delimited_sha256(("a", "b"), 64)
|
||||
expected = hashlib.sha256(b"a|b").hexdigest()
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_private_delimited_sha256_truncation_honoured():
|
||||
"""Truncation argument actually shortens the hex output."""
|
||||
result = ids._delimited_sha256(("a", "b"), 8)
|
||||
assert len(result) == 8
|
||||
+353
-1
@@ -695,6 +695,75 @@ class TestReadTools:
|
||||
assert "project" in result["wings"]
|
||||
assert "notes" in result["wings"]
|
||||
|
||||
def test_status_sqlite_exact_backend_has_no_hnsw_fields(
|
||||
self, monkeypatch, config, palace_path, kg
|
||||
):
|
||||
import mempalace.backends.embedding_wrapper as embedding_wrapper
|
||||
from mempalace.palace import get_collection
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact")
|
||||
monkeypatch.setattr(
|
||||
embedding_wrapper,
|
||||
"_embed_texts",
|
||||
lambda texts: [[float(len(text)), 1.0] for text in texts],
|
||||
)
|
||||
col = get_collection(palace_path, create=True)
|
||||
col.add(
|
||||
ids=["drawer_sqlite"],
|
||||
documents=["verbatim sqlite drawer"],
|
||||
metadatas=[{"wing": "w", "room": "r"}],
|
||||
)
|
||||
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace import mcp_server
|
||||
|
||||
monkeypatch.setattr(mcp_server, "_collection_cache", None)
|
||||
result = mcp_server.tool_status()
|
||||
|
||||
assert result["backend"] == "sqlite_exact"
|
||||
assert result["total_drawers"] == 1
|
||||
assert "hnsw_capacity" not in result
|
||||
assert result.get("vector_disabled") is not True
|
||||
|
||||
def test_status_qdrant_backend_has_no_hnsw_fields(self, monkeypatch, config, palace_path, kg):
|
||||
from mempalace.backends import GetResult
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "qdrant")
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND", "qdrant")
|
||||
with open(os.path.join(palace_path, "qdrant_backend.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({"backend": "qdrant"}, f)
|
||||
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace import mcp_server
|
||||
|
||||
class _FakeQdrantCollection:
|
||||
def count(self):
|
||||
return 2
|
||||
|
||||
def get(self, **_kwargs):
|
||||
return GetResult(
|
||||
ids=["q1", "q2"],
|
||||
documents=[],
|
||||
metadatas=[
|
||||
{"wing": "project", "room": "backend"},
|
||||
{"wing": "project", "room": "api"},
|
||||
],
|
||||
)
|
||||
|
||||
monkeypatch.setattr(mcp_server, "_collection_cache", None)
|
||||
monkeypatch.setattr(mcp_server, "_metadata_cache", None)
|
||||
monkeypatch.setattr(
|
||||
mcp_server, "_get_collection", lambda create=False: _FakeQdrantCollection()
|
||||
)
|
||||
|
||||
result = mcp_server.tool_status()
|
||||
|
||||
assert result["backend"] == "qdrant"
|
||||
assert result["total_drawers"] == 2
|
||||
assert result["wings"] == {"project": 2}
|
||||
assert "hnsw_capacity" not in result
|
||||
assert result.get("vector_disabled") is not True
|
||||
|
||||
def test_status_handles_none_metadata_without_partial(
|
||||
self, monkeypatch, config, palace_path, kg
|
||||
):
|
||||
@@ -1025,6 +1094,34 @@ class TestSearchTool:
|
||||
assert "error" in result
|
||||
assert "index_recovered" not in result
|
||||
|
||||
def test_search_retries_once_on_stale_index_error(self, monkeypatch, config, kg):
|
||||
"""Stale-index errors should trigger one cache-reset retry."""
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace import mcp_server
|
||||
|
||||
calls = {"n": 0}
|
||||
reset_calls = {"n": 0}
|
||||
|
||||
def fake_search(*args, **kwargs):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
return {"error": "Search error: stale-index detected; retry recommended"}
|
||||
return {"results": [{"text": "ok", "wing": "w", "room": "r"}]}
|
||||
|
||||
def fake_reset():
|
||||
reset_calls["n"] += 1
|
||||
|
||||
monkeypatch.setattr(mcp_server, "search_memories", fake_search)
|
||||
monkeypatch.setattr(mcp_server, "_force_chroma_cache_reset", fake_reset)
|
||||
monkeypatch.setattr(mcp_server.time, "sleep", lambda _: None)
|
||||
|
||||
result = mcp_server.tool_search(query="anything")
|
||||
|
||||
assert calls["n"] == 2
|
||||
assert reset_calls["n"] == 1
|
||||
assert "results" in result
|
||||
assert result.get("index_recovered") is True
|
||||
|
||||
def test_list_drawers_rejects_invalid_wing(self, monkeypatch, config, kg):
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace import mcp_server
|
||||
@@ -2290,7 +2387,69 @@ class TestCacheInvalidation:
|
||||
|
||||
result = mcp_server.tool_reconnect()
|
||||
assert result["success"] is True
|
||||
close_palace.assert_called_once_with(config.palace_path)
|
||||
closed_ref = close_palace.call_args.args[0]
|
||||
assert closed_ref.local_path == config.palace_path
|
||||
|
||||
def test_reconnect_closes_selected_non_chroma_backend(
|
||||
self, monkeypatch, config, palace_path, kg
|
||||
):
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact")
|
||||
from mempalace import mcp_server, palace
|
||||
|
||||
closed = []
|
||||
|
||||
class _FakeBackend:
|
||||
def close_palace(self, path):
|
||||
closed.append(path)
|
||||
|
||||
class _FakeCol:
|
||||
def count(self):
|
||||
return 3
|
||||
|
||||
monkeypatch.setattr(palace, "get_backend_for_palace", lambda _path: _FakeBackend())
|
||||
monkeypatch.setattr(mcp_server, "_is_chroma_backend", lambda: False)
|
||||
monkeypatch.setattr(mcp_server, "_get_collection", lambda create=False: _FakeCol())
|
||||
|
||||
result = mcp_server.tool_reconnect()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["drawers"] == 3
|
||||
assert len(closed) == 1
|
||||
assert closed[0].local_path == palace_path
|
||||
|
||||
def test_reconnect_closes_previously_cached_backend(self, monkeypatch, config, palace_path, kg):
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace import backends, mcp_server, palace
|
||||
|
||||
closed = []
|
||||
|
||||
class _SelectedBackend:
|
||||
name = "sqlite_exact"
|
||||
|
||||
def close_palace(self, ref):
|
||||
closed.append(("selected", ref.local_path))
|
||||
|
||||
class _CachedBackend:
|
||||
name = "chroma"
|
||||
|
||||
def close_palace(self, ref):
|
||||
closed.append(("cached", ref.local_path))
|
||||
|
||||
class _FakeCol:
|
||||
def count(self):
|
||||
return 3
|
||||
|
||||
monkeypatch.setattr(palace, "get_backend_for_palace", lambda _path: _SelectedBackend())
|
||||
monkeypatch.setattr(backends, "get_backend", lambda _name: _CachedBackend())
|
||||
monkeypatch.setattr(mcp_server, "_collection_cache_backend", "chroma")
|
||||
monkeypatch.setattr(mcp_server, "_is_chroma_backend", lambda: False)
|
||||
monkeypatch.setattr(mcp_server, "_get_collection", lambda create=False: _FakeCol())
|
||||
|
||||
result = mcp_server.tool_reconnect()
|
||||
|
||||
assert result["success"] is True
|
||||
assert closed == [("selected", palace_path), ("cached", palace_path)]
|
||||
|
||||
def test_get_collection_create_true_avoids_get_or_create_on_reopen(
|
||||
self, monkeypatch, config, palace_path, kg
|
||||
@@ -2459,6 +2618,66 @@ class TestCacheInvalidation:
|
||||
assert col is None
|
||||
|
||||
|
||||
class TestImportKillSwitchSafety:
|
||||
"""Importing mcp_server must not recreate ~/.mempalace (#1676).
|
||||
|
||||
The module-level WAL setup used to ``mkdir(parents=True)`` at import,
|
||||
recreating ``~/.mempalace`` even after the user removed it as the
|
||||
documented kill-switch gesture (``_palace_root_exists()``, #1305),
|
||||
silently re-arming the autosave/mining hooks. WAL creation is now
|
||||
deferred to the first actual write.
|
||||
"""
|
||||
|
||||
def test_import_does_not_recreate_palace_root(self, tmp_path):
|
||||
"""import mempalace.mcp_server must not create ~/.mempalace.
|
||||
|
||||
Runs in a fresh subprocess with HOME pointed at tmp_path so the
|
||||
assertion targets a clean filesystem, independent of conftest's
|
||||
session-level HOME patch.
|
||||
"""
|
||||
palace_root = tmp_path / ".mempalace"
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith("MEMPAL")}
|
||||
env["HOME"] = str(tmp_path)
|
||||
env["USERPROFILE"] = str(tmp_path)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", "import mempalace.mcp_server"],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert result.returncode == 0, f"import failed: {result.stderr}"
|
||||
assert not palace_root.exists(), (
|
||||
f"importing mcp_server recreated {palace_root} as a side effect, "
|
||||
"defeating the _palace_root_exists() kill-switch (#1676)"
|
||||
)
|
||||
|
||||
def test_wal_log_creates_dir_lazily_on_first_write(self, tmp_path, monkeypatch):
|
||||
"""_wal_log creates its directory on first use.
|
||||
|
||||
Proves the deferred setup still works (defers WAL creation to write
|
||||
time, does not disable it) and preserves the WAL permission bits.
|
||||
"""
|
||||
from mempalace import mcp_server
|
||||
|
||||
wal_file = tmp_path / "fresh" / "wal" / "write_log.jsonl"
|
||||
assert not wal_file.parent.exists()
|
||||
monkeypatch.setattr(mcp_server, "_WAL_FILE", wal_file)
|
||||
|
||||
mcp_server._wal_log("test_op", {"safe": "ok"})
|
||||
|
||||
assert wal_file.exists(), "lazy WAL init did not create the log on first write"
|
||||
entry = json.loads(wal_file.read_text().strip())
|
||||
assert entry["operation"] == "test_op"
|
||||
assert entry["params"]["safe"] == "ok"
|
||||
|
||||
# Permission bits the refactor must preserve (POSIX only; Windows
|
||||
# ignores chmod and the code swallows NotImplementedError).
|
||||
if sys.platform != "win32":
|
||||
assert wal_file.stat().st_mode & 0o777 == 0o600
|
||||
assert wal_file.parent.stat().st_mode & 0o777 == 0o700
|
||||
|
||||
|
||||
class TestKGLazyCache:
|
||||
"""Lazy per-path KnowledgeGraph cache (issue #1136)."""
|
||||
|
||||
@@ -2718,6 +2937,57 @@ class TestStructuredErrors:
|
||||
|
||||
assert mcp_server._kg_by_path == {}
|
||||
|
||||
def test_tool_reconnect_rearms_quarantine_gate(self, monkeypatch):
|
||||
"""``tool_reconnect`` must clear the per-process quarantine gate so
|
||||
HNSW safety checks re-run on the next open (#1573)."""
|
||||
from mempalace import mcp_server
|
||||
from mempalace.backends.chroma import ChromaBackend
|
||||
|
||||
palace_path = "/test/palace/quarantine_rearm"
|
||||
gate = {palace_path}
|
||||
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", gate)
|
||||
monkeypatch.setattr(mcp_server, "_config", type("C", (), {"palace_path": palace_path})())
|
||||
monkeypatch.setattr(mcp_server, "_get_collection", lambda: None)
|
||||
|
||||
mcp_server.tool_reconnect()
|
||||
|
||||
assert palace_path not in gate, (
|
||||
"tool_reconnect should clear quarantine gate for the palace path"
|
||||
)
|
||||
|
||||
def test_get_client_rearms_quarantine_on_reconnect(self, monkeypatch, config, palace_path, kg):
|
||||
"""``_get_client`` must clear the quarantine gate before calling
|
||||
``make_client`` so HNSW safety checks re-run on reconnect (#1573)."""
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace import mcp_server
|
||||
from mempalace.backends.chroma import ChromaBackend
|
||||
|
||||
_client, _col = _get_collection(palace_path, create=True)
|
||||
del _client
|
||||
|
||||
mcp_server._get_collection()
|
||||
|
||||
assert config.palace_path in ChromaBackend._quarantined_paths
|
||||
|
||||
old_mtime = mcp_server._palace_db_mtime
|
||||
monkeypatch.setattr(mcp_server, "_palace_db_mtime", old_mtime - 10.0)
|
||||
|
||||
quarantine_calls: list[str] = []
|
||||
original_prepare = ChromaBackend._prepare_palace_for_open
|
||||
|
||||
@staticmethod
|
||||
def spy_prepare(path):
|
||||
quarantine_calls.append(path)
|
||||
original_prepare(path)
|
||||
|
||||
monkeypatch.setattr(ChromaBackend, "_prepare_palace_for_open", spy_prepare)
|
||||
|
||||
mcp_server._get_client()
|
||||
|
||||
assert len(quarantine_calls) == 1, (
|
||||
"_get_client should call _prepare_palace_for_open on reconnect"
|
||||
)
|
||||
|
||||
def test_call_kg_retries_after_concurrent_close(self, monkeypatch):
|
||||
"""A KG closed mid-handler must trigger a one-shot retry with a fresh
|
||||
instance — not surface a -32000 to the MCP client."""
|
||||
@@ -3053,6 +3323,88 @@ class TestParamShapeDiagnostics:
|
||||
assert "'entry'" in message
|
||||
assert " and " not in message.split("for tool")[0]
|
||||
|
||||
def test_diary_write_content_aliases_entry(self, monkeypatch):
|
||||
"""A content-only diary_write call is remapped to 'entry' before
|
||||
dispatch (#1245 alias), so it satisfies the required param and the
|
||||
alias key is consumed rather than passed through to the handler.
|
||||
"""
|
||||
from mempalace import mcp_server
|
||||
|
||||
captured = {}
|
||||
|
||||
def capture(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"success": True}
|
||||
|
||||
monkeypatch.setitem(mcp_server.TOOLS["mempalace_diary_write"], "handler", capture)
|
||||
resp = mcp_server.handle_request(
|
||||
{
|
||||
"method": "tools/call",
|
||||
"id": 5,
|
||||
"params": {
|
||||
"name": "mempalace_diary_write",
|
||||
"arguments": {"agent_name": "test", "content": "hello world"},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert "error" not in resp
|
||||
assert captured.get("entry") == "hello world"
|
||||
assert "content" not in captured
|
||||
|
||||
def test_diary_write_entry_wins_over_content(self, monkeypatch):
|
||||
"""When both 'entry' and the 'content' alias are supplied, 'entry' wins
|
||||
and the alias is dropped.
|
||||
"""
|
||||
from mempalace import mcp_server
|
||||
|
||||
captured = {}
|
||||
|
||||
def capture(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"success": True}
|
||||
|
||||
monkeypatch.setitem(mcp_server.TOOLS["mempalace_diary_write"], "handler", capture)
|
||||
resp = mcp_server.handle_request(
|
||||
{
|
||||
"method": "tools/call",
|
||||
"id": 6,
|
||||
"params": {
|
||||
"name": "mempalace_diary_write",
|
||||
"arguments": {"agent_name": "t", "entry": "real", "content": "alias"},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert "error" not in resp
|
||||
assert captured.get("entry") == "real"
|
||||
assert "content" not in captured
|
||||
|
||||
def test_diary_write_explicit_empty_entry_not_overridden_by_content(self, monkeypatch):
|
||||
"""An explicitly supplied (even falsy "") 'entry' wins over 'content' —
|
||||
the alias only fills in when 'entry' is absent or null, not merely falsy.
|
||||
"""
|
||||
from mempalace import mcp_server
|
||||
|
||||
captured = {}
|
||||
|
||||
def capture(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"success": True}
|
||||
|
||||
monkeypatch.setitem(mcp_server.TOOLS["mempalace_diary_write"], "handler", capture)
|
||||
resp = mcp_server.handle_request(
|
||||
{
|
||||
"method": "tools/call",
|
||||
"id": 7,
|
||||
"params": {
|
||||
"name": "mempalace_diary_write",
|
||||
"arguments": {"agent_name": "t", "entry": "", "content": "alias"},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert "error" not in resp
|
||||
assert captured.get("entry") == ""
|
||||
assert "content" not in captured
|
||||
|
||||
def test_handler_internal_signature_shape_stays_generic(self, monkeypatch):
|
||||
"""A TypeError whose function name does not match the dispatched
|
||||
handler — e.g. raised by a helper called inside the handler body —
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Tests for the wing-name normalization migration (migrate-wings).
|
||||
|
||||
normalize_wing_name strips leading/trailing separators (#1675); palaces built
|
||||
before that rule filed drawers under the old name (e.g. ``_alpha``).
|
||||
``migrate_wing_names`` re-keys the ``wing`` metadata in place so those memories
|
||||
stay discoverable under the new name, merging collisions. IDs are left untouched
|
||||
(they are opaque keys), and the pass is idempotent.
|
||||
"""
|
||||
|
||||
from mempalace.migrate import migrate_wing_names, plan_wing_renames
|
||||
|
||||
|
||||
# --- pure planner ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_plan_renames_strips_leading_and_trailing():
|
||||
summary, updates = plan_wing_renames(
|
||||
[
|
||||
("d1", {"wing": "_alpha", "room": "r"}),
|
||||
("d2", {"wing": "beta_", "room": "r"}),
|
||||
("d3", {"wing": "clean", "room": "r"}),
|
||||
]
|
||||
)
|
||||
assert dict(summary) == {("_alpha", "alpha"): 1, ("beta_", "beta"): 1}
|
||||
assert {u[0] for u in updates} == {"d1", "d2"}
|
||||
# only 'wing' is rewritten; other metadata is preserved
|
||||
by_id = {u[0]: u[1] for u in updates}
|
||||
assert by_id["d1"]["wing"] == "alpha"
|
||||
assert by_id["d1"]["room"] == "r"
|
||||
assert by_id["d2"]["wing"] == "beta"
|
||||
|
||||
|
||||
def test_plan_renames_noop_for_clean_wings():
|
||||
summary, updates = plan_wing_renames([("d", {"wing": "already_clean", "room": "r"})])
|
||||
assert not summary
|
||||
assert not updates
|
||||
|
||||
|
||||
def test_plan_renames_ignores_empty_nonstring_and_all_separator():
|
||||
_, updates = plan_wing_renames(
|
||||
[
|
||||
("a", {"wing": "_"}), # normalizes to "" -> skip (never strand a drawer)
|
||||
("b", {"wing": ""}),
|
||||
("c", {"wing": None}),
|
||||
("d", {}),
|
||||
]
|
||||
)
|
||||
assert updates == []
|
||||
|
||||
|
||||
def test_plan_renames_collision_maps_both_to_same_target():
|
||||
_, updates = plan_wing_renames(
|
||||
[
|
||||
("d1", {"wing": "_gamma"}),
|
||||
("d2", {"wing": "gamma_"}),
|
||||
]
|
||||
)
|
||||
assert {u[1]["wing"] for u in updates} == {"gamma"}
|
||||
|
||||
|
||||
# --- integration over a real backend collection ---------------------------
|
||||
|
||||
|
||||
def _seed(palace, rows):
|
||||
from mempalace.palace import get_collection
|
||||
|
||||
col = get_collection(str(palace), create=True)
|
||||
# Explicit embeddings keep the test hermetic (no embedding model needed) —
|
||||
# the migration only reads/writes metadata.
|
||||
col.upsert(
|
||||
ids=[r["id"] for r in rows],
|
||||
documents=[r["doc"] for r in rows],
|
||||
metadatas=[r["meta"] for r in rows],
|
||||
embeddings=[[float(i + 1)] * 8 for i in range(len(rows))],
|
||||
)
|
||||
return col
|
||||
|
||||
|
||||
def _wing_ids(palace, wing):
|
||||
from mempalace.palace import get_collection
|
||||
|
||||
col = get_collection(str(palace), create=False)
|
||||
res = col.get(where={"wing": wing}, include=["metadatas"])
|
||||
return set(res.ids if hasattr(res, "ids") else res["ids"])
|
||||
|
||||
|
||||
def _meta(wing, room, source_file, idx):
|
||||
return {"wing": wing, "room": room, "source_file": source_file, "chunk_index": idx}
|
||||
|
||||
|
||||
def test_migrate_relabels_old_format_wings(tmp_path):
|
||||
palace = tmp_path / "palace"
|
||||
palace.mkdir()
|
||||
_seed(
|
||||
palace,
|
||||
[
|
||||
{"id": "drawer__alpha_r_1", "doc": "auth jwt", "meta": _meta("_alpha", "r", "a.py", 0)},
|
||||
{"id": "drawer_beta__r_2", "doc": "db alembic", "meta": _meta("beta_", "r", "b.py", 0)},
|
||||
{
|
||||
"id": "drawer_clean_r_3",
|
||||
"doc": "react query",
|
||||
"meta": _meta("clean", "r", "c.py", 0),
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
assert migrate_wing_names(str(palace), confirm=True) is True
|
||||
|
||||
assert _wing_ids(palace, "alpha") == {"drawer__alpha_r_1"}
|
||||
assert _wing_ids(palace, "beta") == {"drawer_beta__r_2"}
|
||||
assert _wing_ids(palace, "_alpha") == set()
|
||||
assert _wing_ids(palace, "beta_") == set()
|
||||
# an already-clean wing is left untouched
|
||||
assert _wing_ids(palace, "clean") == {"drawer_clean_r_3"}
|
||||
|
||||
|
||||
def test_migrate_merges_collision_into_existing_wing(tmp_path):
|
||||
palace = tmp_path / "palace"
|
||||
palace.mkdir()
|
||||
_seed(
|
||||
palace,
|
||||
[
|
||||
{"id": "drawer_gamma_r_1", "doc": "current", "meta": _meta("gamma", "r", "g1.py", 0)},
|
||||
{"id": "drawer__gamma_r_2", "doc": "legacy", "meta": _meta("_gamma", "r", "g2.py", 0)},
|
||||
],
|
||||
)
|
||||
|
||||
migrate_wing_names(str(palace), confirm=True)
|
||||
|
||||
assert _wing_ids(palace, "gamma") == {"drawer_gamma_r_1", "drawer__gamma_r_2"}
|
||||
assert _wing_ids(palace, "_gamma") == set()
|
||||
|
||||
|
||||
def test_migrate_dry_run_changes_nothing(tmp_path):
|
||||
palace = tmp_path / "palace"
|
||||
palace.mkdir()
|
||||
_seed(palace, [{"id": "drawer__x_r_1", "doc": "d", "meta": _meta("_x", "r", "x.py", 0)}])
|
||||
|
||||
assert migrate_wing_names(str(palace), dry_run=True) is True
|
||||
# nothing actually moved
|
||||
assert _wing_ids(palace, "_x") == {"drawer__x_r_1"}
|
||||
assert _wing_ids(palace, "x") == set()
|
||||
|
||||
|
||||
def test_migrate_is_idempotent(tmp_path):
|
||||
palace = tmp_path / "palace"
|
||||
palace.mkdir()
|
||||
_seed(palace, [{"id": "drawer__y_r_1", "doc": "d", "meta": _meta("_y", "r", "y.py", 0)}])
|
||||
|
||||
assert migrate_wing_names(str(palace), confirm=True) is True
|
||||
# second run finds nothing left to normalize
|
||||
assert migrate_wing_names(str(palace), confirm=True) is False
|
||||
assert _wing_ids(palace, "y") == {"drawer__y_r_1"}
|
||||
+210
-1
@@ -9,6 +9,7 @@ import chromadb
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from mempalace.config import normalize_wing_name
|
||||
from mempalace.miner import detect_room, load_config, mine, scan_project, status
|
||||
from mempalace.palace import NORMALIZE_VERSION, file_already_mined, prefetch_mined_set
|
||||
|
||||
@@ -257,7 +258,12 @@ def test_load_config_uses_defaults_when_yaml_missing():
|
||||
assert isinstance(config, dict)
|
||||
assert "wing" in config
|
||||
assert "rooms" in config
|
||||
assert config["wing"] == project_root.name
|
||||
# The default wing is the normalized dirname, not the raw name: temp
|
||||
# dir names can contain leading/trailing '_' (tempfile's alphabet
|
||||
# includes it), which normalize_wing_name strips. Comparing to the raw
|
||||
# name was flaky across platforms (it only passed when the random name
|
||||
# had no separators).
|
||||
assert config["wing"] == normalize_wing_name(project_root.name)
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
@@ -810,6 +816,133 @@ def test_status_handles_none_metadata_without_crash(tmp_path, capsys):
|
||||
assert "WING: proj" in out
|
||||
|
||||
|
||||
def test_status_does_not_cold_load_vector_index(palace_path, seeded_collection, capsys):
|
||||
"""#1681 regression: a healthy ``status`` must NOT open the collection.
|
||||
|
||||
Opening it cold-loads the HNSW vector index, which costs ~60s of CPU per
|
||||
call on large palaces. The counts come from chroma.sqlite3 instead. If
|
||||
anyone reroutes the happy path back through the vector index, the sentinel
|
||||
patched over ``_open_collection_or_explain`` fires. (Revert ``status`` to
|
||||
its pre-fix body and this test fails loudly — that's the regression it
|
||||
guards.)
|
||||
"""
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sentinel = MagicMock(side_effect=AssertionError("status cold-loaded the vector index"))
|
||||
with patch("mempalace.miner._open_collection_or_explain", sentinel):
|
||||
status(palace_path)
|
||||
|
||||
sentinel.assert_not_called()
|
||||
out = capsys.readouterr().out
|
||||
assert "MemPalace Status — 4 drawers" in out
|
||||
assert "WING: project" in out
|
||||
assert "WING: notes" in out
|
||||
|
||||
|
||||
def test_sqlite_wing_room_counts_exact_tally(palace_path, seeded_collection):
|
||||
"""The sqlite tally must equal the seeded drawers exactly — 2 project/
|
||||
backend, 1 project/frontend, 1 notes/planning — with no double counting.
|
||||
|
||||
Guards the double ``LEFT JOIN embedding_metadata`` against fan-out: if the
|
||||
join multiplied rows (e.g. a drawer carrying several metadata keys), the
|
||||
total would exceed 4 and the room counts would inflate.
|
||||
"""
|
||||
from mempalace.backends.chroma import _sqlite_wing_room_counts
|
||||
|
||||
result = _sqlite_wing_room_counts(palace_path, "mempalace_drawers")
|
||||
assert result is not None
|
||||
total, wing_rooms = result
|
||||
assert total == 4
|
||||
assert {w: dict(r) for w, r in wing_rooms.items()} == {
|
||||
"project": {"backend": 2, "frontend": 1},
|
||||
"notes": {"planning": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_status_falls_back_to_chroma_when_sqlite_unreadable(palace_path, seeded_collection, capsys):
|
||||
"""When the sqlite fast path returns ``None`` (exotic schema / read error),
|
||||
``status`` must fall back to the ChromaDB client path and still report the
|
||||
correct tally — not crash or print nothing."""
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch("mempalace.backends.chroma._sqlite_wing_room_counts", return_value=None):
|
||||
status(palace_path)
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "MemPalace Status — 4 drawers" in out
|
||||
assert "WING: project" in out
|
||||
|
||||
|
||||
def test_sqlite_wing_room_counts_none_when_collection_absent(palace_path):
|
||||
"""DB exists but the drawers collection was never bootstrapped -> ``None``,
|
||||
so ``status`` routes to the 'initialized but empty' message instead of
|
||||
printing a misleading ``0 drawers`` tally (State C, #1498)."""
|
||||
import chromadb
|
||||
|
||||
from mempalace.backends.chroma import _sqlite_wing_room_counts
|
||||
|
||||
chromadb.PersistentClient(path=palace_path) # creates chroma.sqlite3, no collection
|
||||
assert _sqlite_wing_room_counts(palace_path, "mempalace_drawers") is None
|
||||
|
||||
|
||||
def test_sqlite_wing_room_counts_numeric_wing_not_dropped(palace_path, collection):
|
||||
"""A drawer whose wing/room is stored numerically (int_value, not
|
||||
string_value) must be tallied under its stringified value — matching the
|
||||
ChromaDB path, which surfaces the native number — not silently bucketed
|
||||
under '?'. Without the int/float COALESCE this row would vanish into '?'
|
||||
while returning a non-None result that skips the fallback."""
|
||||
from mempalace.backends.chroma import _sqlite_wing_room_counts
|
||||
|
||||
collection.add(
|
||||
ids=["drawer_numeric_meta"],
|
||||
documents=["a drawer filed with a numeric wing and room"],
|
||||
metadatas=[{"wing": 2026, "room": 7}],
|
||||
)
|
||||
result = _sqlite_wing_room_counts(palace_path, "mempalace_drawers")
|
||||
assert result is not None
|
||||
_, wing_rooms = result
|
||||
assert {w: dict(r) for w, r in wing_rooms.items()} == {"2026": {"7": 1}}
|
||||
|
||||
|
||||
def test_sqlite_wing_room_counts_partial_metadata_buckets_question_mark(palace_path, collection):
|
||||
"""A drawer with a wing but no room (and vice versa) must be counted under
|
||||
'?' for the missing axis, never dropped. Guards the LEFT JOINs against
|
||||
being narrowed to inner joins, which would silently discard partial
|
||||
drawers and undercount the total."""
|
||||
from mempalace.backends.chroma import _sqlite_wing_room_counts
|
||||
|
||||
collection.add(
|
||||
ids=["drawer_wing_only", "drawer_room_only"],
|
||||
documents=["has a wing but no room", "has a room but no wing"],
|
||||
metadatas=[{"wing": "alpha"}, {"room": "beta"}],
|
||||
)
|
||||
result = _sqlite_wing_room_counts(palace_path, "mempalace_drawers")
|
||||
assert result is not None
|
||||
total, wing_rooms = result
|
||||
assert total == 2 # neither drawer dropped
|
||||
assert {w: dict(r) for w, r in wing_rooms.items()} == {
|
||||
"alpha": {"?": 1},
|
||||
"?": {"beta": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_sqlite_wing_room_counts_returns_none_on_locked_db(palace_path, seeded_collection):
|
||||
"""A sustained sqlite lock (writer holding the DB) must degrade to ``None``
|
||||
so ``status`` falls back to the slow-but-correct ChromaDB path rather than
|
||||
raising. busy_timeout waits out *transient* locks; a hard lock still ends
|
||||
here. Simulated by forcing the read to raise OperationalError."""
|
||||
import sqlite3 as _sqlite3
|
||||
from unittest.mock import patch
|
||||
|
||||
from mempalace.backends.chroma import _sqlite_wing_room_counts
|
||||
|
||||
with patch(
|
||||
"mempalace.backends.chroma.sqlite3.connect",
|
||||
side_effect=_sqlite3.OperationalError("database is locked"),
|
||||
):
|
||||
assert _sqlite_wing_room_counts(palace_path, "mempalace_drawers") is None
|
||||
|
||||
|
||||
def test_process_file_uses_bounded_upsert_batches(tmp_path, monkeypatch):
|
||||
from mempalace import miner
|
||||
|
||||
@@ -1976,3 +2109,79 @@ class TestExtractContentDate:
|
||||
content = "Y2K reference: 25/01/00.\n"
|
||||
f.write_text(content)
|
||||
assert _extract_content_date(str(f), content) == "2000-01-25"
|
||||
|
||||
|
||||
def test_file_already_mined_handles_multiple_groups_under_one_source_file(tmp_path):
|
||||
"""Under the additive-mining model, a single ``source_file`` can have
|
||||
multiple ``parent_drawer_id`` groups in the palace — one per mining pass
|
||||
— each with its own stored ``source_mtime``. ``file_already_mined`` must
|
||||
return True if ANY group's stored mtime matches the file's current
|
||||
mtime, regardless of which group ChromaDB's ``get(..., limit=1)`` happens
|
||||
to return first.
|
||||
|
||||
Current code uses ``collection.get(where={"source_file": X}, limit=1)``
|
||||
which has undefined ordering across multiple matching rows. When ChromaDB
|
||||
returns a stale group (older mining pass with a different stored mtime),
|
||||
the function returns False, the additive miner concludes the file changed,
|
||||
and writes yet another duplicate group for a file that has not actually
|
||||
changed. Steady state: duplicate groups accumulate without bound.
|
||||
|
||||
This test pins the failure space deterministically using a MockCollection
|
||||
that always returns the stale group on limit=1 (worst-case ordering). A
|
||||
correct implementation iterates all groups via the paginated pattern
|
||||
already used in the ``extract_mode is not None`` branch.
|
||||
|
||||
Issue: follow-up to PR #1628 (the every-bare-source_file-query audit).
|
||||
"""
|
||||
# Create a real file with known mtime.
|
||||
test_file = tmp_path / "doc.md"
|
||||
test_file.write_text("content that hasn't changed since the latest mine.")
|
||||
current_mtime = os.path.getmtime(str(test_file))
|
||||
|
||||
# Build a MockCollection that simulates two parent_drawer_id groups
|
||||
# under the same source_file with DIFFERENT stored mtimes:
|
||||
# group_A: stale, source_mtime = current_mtime - 100 (older pass)
|
||||
# group_B: current, source_mtime = current_mtime (latest pass)
|
||||
# The mock returns group_A for limit=1 calls (worst-case ordering) and
|
||||
# returns BOTH groups for the paginated limit=1000 calls (what the fix
|
||||
# must use).
|
||||
stale_meta = {
|
||||
"source_file": str(test_file),
|
||||
"chunk_index": 0,
|
||||
"parent_drawer_id": "drawer_group_A",
|
||||
"source_mtime": current_mtime - 100.0,
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
}
|
||||
current_meta = {
|
||||
"source_file": str(test_file),
|
||||
"chunk_index": 0,
|
||||
"parent_drawer_id": "drawer_group_B",
|
||||
"source_mtime": current_mtime,
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
}
|
||||
|
||||
class MockCollection:
|
||||
"""Simulates the ChromaDB get() contract for two parent_drawer_id
|
||||
groups sharing one source_file. Returns the STALE group when called
|
||||
with limit=1 (the worst-case-ordering shape that triggers the bug).
|
||||
Returns BOTH groups (paginated) when called with limit=1000 — what
|
||||
a correctly-iterating implementation must do."""
|
||||
|
||||
def get(self, where=None, limit=None, offset=0, include=None):
|
||||
if limit == 1:
|
||||
return {"ids": ["a_0"], "metadatas": [stale_meta]}
|
||||
if offset == 0:
|
||||
return {"ids": ["a_0", "b_0"], "metadatas": [stale_meta, current_meta]}
|
||||
return {"ids": [], "metadatas": []}
|
||||
|
||||
col = MockCollection()
|
||||
|
||||
# EXPECTED: file_already_mined returns True because at least one stored
|
||||
# group's mtime matches the current file mtime.
|
||||
# CURRENT BUG: returns False because limit=1 grabs the stale group.
|
||||
assert file_already_mined(col, str(test_file), check_mtime=True) is True, (
|
||||
"file_already_mined returned False even though a group with matching "
|
||||
"mtime exists. The limit=1 query picked the stale group; the function "
|
||||
"must iterate all groups for the source_file (mirroring the existing "
|
||||
"paginated pattern in the extract_mode-is-set branch)."
|
||||
)
|
||||
|
||||
@@ -61,6 +61,21 @@ def test_open_collection_or_explain_state_c_no_collection(tmp_path):
|
||||
assert any("mempalace mine" in line for line in lines)
|
||||
|
||||
|
||||
def test_open_collection_or_explain_unknown_backend(tmp_path, monkeypatch):
|
||||
"""An unknown backend name (typo in MEMPALACE_BACKEND/--backend) must
|
||||
surface as a CLI state message, not an escaping KeyError stack trace."""
|
||||
emit, lines = _capture()
|
||||
palace = tmp_path / "palace"
|
||||
palace.mkdir()
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND", "does_not_exist")
|
||||
|
||||
result = _open_collection_or_explain(str(palace), out=emit)
|
||||
|
||||
assert result is None
|
||||
assert any("Unknown backend selected" in line for line in lines)
|
||||
assert any("does_not_exist" in line for line in lines)
|
||||
|
||||
|
||||
def test_open_collection_or_explain_state_d_healthy(tmp_path):
|
||||
"""State D: healthy palace — returns the opened collection silently."""
|
||||
emit, lines = _capture()
|
||||
|
||||
@@ -0,0 +1,511 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from _backend_conformance import assert_partition_isolation
|
||||
|
||||
from mempalace.backends import (
|
||||
BackendError,
|
||||
BackendMismatchError,
|
||||
CollectionNotInitializedError,
|
||||
DimensionMismatchError,
|
||||
PalaceRef,
|
||||
available_backends,
|
||||
)
|
||||
from mempalace.backends.pgvector import (
|
||||
PgVectorBackend,
|
||||
_matches_where,
|
||||
_vector_distance,
|
||||
_as_vector_array,
|
||||
)
|
||||
|
||||
|
||||
class _FakePgVectorClient:
|
||||
"""In-memory stand-in for the psycopg-backed client.
|
||||
|
||||
Stores rows per table so the same-instance/different-table isolation the
|
||||
real backend gets from Postgres is exercised deterministically in CI. The
|
||||
real client pushes filters/ranking to SQL; this fake applies the same
|
||||
Python filter + cosine ranking the local-fallback path uses.
|
||||
"""
|
||||
|
||||
instances: list = []
|
||||
|
||||
def __init__(self, _config):
|
||||
self.tables: dict = {}
|
||||
self.query_calls: list = []
|
||||
_FakePgVectorClient.instances.append(self)
|
||||
|
||||
def ping(self):
|
||||
return None
|
||||
|
||||
def ensure_extension(self):
|
||||
return None
|
||||
|
||||
def table_exists(self, table):
|
||||
return table in self.tables
|
||||
|
||||
def table_dimension(self, table):
|
||||
return self.tables.get(table, {}).get("dimension")
|
||||
|
||||
def create_table(self, table, dimension):
|
||||
self.tables.setdefault(table, {"dimension": dimension, "rows": {}})
|
||||
|
||||
def upsert_rows(self, table, rows):
|
||||
store = self.tables.setdefault(
|
||||
table,
|
||||
{"dimension": len(rows[0]["embedding"]) if rows else 0, "rows": {}},
|
||||
)
|
||||
for row in rows:
|
||||
store["rows"][row["id"]] = dict(row)
|
||||
|
||||
def _filtered(self, table, where):
|
||||
rows = list(self.tables.get(table, {"rows": {}})["rows"].values())
|
||||
return [row for row in rows if _matches_where(row.get("metadata") or {}, where)]
|
||||
|
||||
def query_rows(self, table, *, vector, limit, where, with_embedding):
|
||||
self.query_calls.append(where)
|
||||
q = _as_vector_array(vector)
|
||||
scored = []
|
||||
for row in self._filtered(table, where):
|
||||
distance = _vector_distance(q, row.get("embedding"))
|
||||
if distance is not None:
|
||||
scored.append((distance, row))
|
||||
scored.sort(key=lambda item: item[0])
|
||||
out = []
|
||||
for distance, row in scored[:limit]:
|
||||
item = {
|
||||
"id": row["id"],
|
||||
"document": row["document"],
|
||||
"metadata": row.get("metadata") or {},
|
||||
"embedding": row.get("embedding") if with_embedding else None,
|
||||
"distance": distance,
|
||||
}
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
def scroll_rows(self, table, *, where=None, with_embedding=False):
|
||||
out = []
|
||||
for row in self._filtered(table, where):
|
||||
out.append(
|
||||
{
|
||||
"id": row["id"],
|
||||
"document": row["document"],
|
||||
"metadata": row.get("metadata") or {},
|
||||
"embedding": row.get("embedding") if with_embedding else None,
|
||||
"distance": None,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
def delete_rows(self, table, *, ids=None, where=None):
|
||||
rows = self.tables.get(table, {"rows": {}})["rows"]
|
||||
if ids is not None:
|
||||
for doc_id in ids:
|
||||
rows.pop(doc_id, None)
|
||||
return
|
||||
for doc_id, row in list(rows.items()):
|
||||
if _matches_where(row.get("metadata") or {}, where):
|
||||
rows.pop(doc_id, None)
|
||||
|
||||
def count_rows(self, table):
|
||||
return len(self.tables.get(table, {"rows": {}})["rows"])
|
||||
|
||||
def drop_table(self, table):
|
||||
self.tables.pop(table, None)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_pgvector(monkeypatch):
|
||||
import mempalace.backends.pgvector as pgvector
|
||||
|
||||
_FakePgVectorClient.instances.clear()
|
||||
monkeypatch.setattr(pgvector, "_PgVectorClient", _FakePgVectorClient)
|
||||
monkeypatch.delenv("MEMPALACE_PGVECTOR_DSN", raising=False)
|
||||
monkeypatch.delenv("MEMPALACE_PGVECTOR_NAMESPACE", raising=False)
|
||||
return _FakePgVectorClient
|
||||
|
||||
|
||||
def _collection(tmp_path, name="drawers"):
|
||||
backend = PgVectorBackend()
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
return backend, backend.get_collection(palace=palace, collection_name=name, create=True)
|
||||
|
||||
|
||||
def test_registry_exposes_pgvector():
|
||||
assert "pgvector" in available_backends()
|
||||
|
||||
|
||||
def test_pgvector_add_query_filters_lexical_and_marker(tmp_path, fake_pgvector):
|
||||
backend, col = _collection(tmp_path)
|
||||
assert not os.path.isfile(tmp_path / "pgvector_backend.json")
|
||||
|
||||
col.add(
|
||||
ids=["a", "b", "c"],
|
||||
documents=[
|
||||
"alpha backend note",
|
||||
"rareterm pgvector backend note",
|
||||
"frontend design note",
|
||||
],
|
||||
metadatas=[
|
||||
{"wing": "project", "room": "backend", "rank": 1},
|
||||
{"wing": "project", "room": "backend", "rank": 3},
|
||||
{"wing": "project", "room": "frontend", "rank": 2},
|
||||
],
|
||||
embeddings=[[1, 0], [0.9, 0.1], [0, 1]],
|
||||
)
|
||||
|
||||
assert PgVectorBackend.detect(str(tmp_path))
|
||||
assert os.path.isfile(tmp_path / "pgvector_backend.json")
|
||||
assert col.count() == 3
|
||||
|
||||
# Equality filter is pushed down (no local fallback); $in stays pushdown.
|
||||
result = col.query(
|
||||
query_embeddings=[[1, 0]],
|
||||
n_results=3,
|
||||
where={"wing": "project"},
|
||||
include=["documents", "metadatas", "distances", "embeddings"],
|
||||
)
|
||||
assert result.ids[0][0] == "a"
|
||||
assert set(result.ids[0]) == {"a", "b", "c"}
|
||||
assert result.embeddings[0][0] == pytest.approx([1.0, 0.0])
|
||||
|
||||
hits = col.lexical_search(query="rareterm backend", n_results=2, where={"wing": "project"}).hits
|
||||
assert [hit.id for hit in hits] == ["b", "a"]
|
||||
|
||||
backend.close_palace(str(tmp_path))
|
||||
with pytest.raises(Exception):
|
||||
col.count()
|
||||
|
||||
|
||||
def test_pgvector_requires_explicit_embeddings(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
with pytest.raises(ValueError, match="explicit embeddings"):
|
||||
col.add(ids=["a"], documents=["no vector"], metadatas=[{}])
|
||||
|
||||
|
||||
def test_pgvector_marker_not_written_when_first_write_fails(tmp_path, fake_pgvector, monkeypatch):
|
||||
_backend, col = _collection(tmp_path)
|
||||
fake_client = fake_pgvector.instances[0]
|
||||
|
||||
def fail_upsert(*_args, **_kwargs):
|
||||
raise RuntimeError("pg unavailable")
|
||||
|
||||
monkeypatch.setattr(fake_client, "upsert_rows", fail_upsert)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
|
||||
assert not os.path.isfile(tmp_path / "pgvector_backend.json")
|
||||
|
||||
|
||||
def test_pgvector_dimension_mismatch(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
with pytest.raises(DimensionMismatchError):
|
||||
col.upsert(ids=["b"], documents=["two"], metadatas=[{}], embeddings=[[1, 0, 0]])
|
||||
|
||||
|
||||
def test_pgvector_add_rejects_duplicate_ids_in_same_batch(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
with pytest.raises(ValueError, match="unique"):
|
||||
col.add(
|
||||
ids=["a", "a"], documents=["x", "y"], metadatas=[{}, {}], embeddings=[[1, 0], [0, 1]]
|
||||
)
|
||||
|
||||
|
||||
def test_pgvector_complex_filters_use_local_fallback(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(
|
||||
ids=["a", "b", "c"],
|
||||
documents=["alpha", "beta", "gamma"],
|
||||
metadatas=[
|
||||
{"wing": "x", "rank": 1, "tags": "core,vector"},
|
||||
{"wing": "y", "rank": 3, "tags": "sqlite,exact"},
|
||||
{"wing": "z", "rank": 2, "tags": "old"},
|
||||
],
|
||||
embeddings=[[1, 0], [0.9, 0.1], [0, 1]],
|
||||
)
|
||||
|
||||
# $or, $contains and comparisons must route to the local exact path and
|
||||
# still return the correct rows.
|
||||
or_hits = col.get(where={"$or": [{"wing": "x"}, {"wing": "z"}]})
|
||||
assert set(or_hits.ids) == {"a", "c"}
|
||||
|
||||
contains = col.get(where={"tags": {"$contains": "sqlite"}})
|
||||
assert contains.ids == ["b"]
|
||||
|
||||
ranked = col.query(query_embeddings=[[1, 0]], n_results=3, where={"rank": {"$gte": 2}})
|
||||
assert set(ranked.ids[0]) == {"b", "c"}
|
||||
|
||||
|
||||
def test_pgvector_marker_participates_in_backend_mismatch(tmp_path, fake_pgvector):
|
||||
from mempalace.palace import resolve_backend_name
|
||||
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
|
||||
assert resolve_backend_name(str(tmp_path)) == "pgvector"
|
||||
with pytest.raises(BackendMismatchError):
|
||||
resolve_backend_name(str(tmp_path), explicit="qdrant")
|
||||
|
||||
|
||||
def test_pgvector_marker_rejects_target_change(tmp_path, fake_pgvector, monkeypatch):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
|
||||
backend2 = PgVectorBackend()
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
with pytest.raises(BackendMismatchError):
|
||||
backend2.get_collection(
|
||||
palace=palace,
|
||||
collection_name="drawers",
|
||||
create=True,
|
||||
options={"dsn": "postgresql://other-host:5432/other"},
|
||||
)
|
||||
|
||||
|
||||
def test_pgvector_rejects_pure_remote_palace(tmp_path, fake_pgvector):
|
||||
"""No local_path means the marker (the only mismatch-protection anchor)
|
||||
cannot be written or validated, so the backend refuses rather than silently
|
||||
opening an unprotected table (RFC 001 isolation contract, PR #1679)."""
|
||||
backend = PgVectorBackend()
|
||||
palace = PalaceRef(id="tenant-remote", local_path=None, namespace="tenant-remote")
|
||||
with pytest.raises(BackendError, match="local palace path"):
|
||||
backend.get_collection(palace=palace, collection_name="drawers", create=True)
|
||||
|
||||
|
||||
def test_pgvector_missing_table_after_marker_is_not_initialized(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
fake_pgvector.instances[0].drop_table(col._table)
|
||||
|
||||
assert col.health().ok is False
|
||||
with pytest.raises(CollectionNotInitializedError):
|
||||
col.count()
|
||||
|
||||
|
||||
def test_pgvector_cross_palace_isolation_conformance(tmp_path, fake_pgvector):
|
||||
"""Shared per-PalaceRef.id isolation conformance (RFC 001 isolation contract)."""
|
||||
backend = PgVectorBackend()
|
||||
cols = []
|
||||
for label in ("alpha", "beta"):
|
||||
path = tmp_path / label
|
||||
ref = PalaceRef(id=str(path), local_path=str(path))
|
||||
cols.append(backend.get_collection(palace=ref, collection_name="drawers", create=True))
|
||||
# Same backend + same DSN → same client instance, distinct tables.
|
||||
assert cols[0]._table != cols[1]._table
|
||||
assert_partition_isolation(backend, cols[0], cols[1], embedding=[1.0, 0.0])
|
||||
|
||||
|
||||
def test_pgvector_namespace_isolation_conformance(tmp_path, fake_pgvector):
|
||||
"""Shared per-PalaceRef.namespace isolation conformance — pgvector advertises
|
||||
``supports_namespace_isolation`` (RFC 001 isolation contract)."""
|
||||
assert "supports_namespace_isolation" in PgVectorBackend.capabilities
|
||||
backend = PgVectorBackend()
|
||||
ref_a = PalaceRef(
|
||||
id=str(tmp_path / "tenant-a"),
|
||||
local_path=str(tmp_path / "tenant-a"),
|
||||
namespace="tenant-a",
|
||||
)
|
||||
ref_b = PalaceRef(
|
||||
id=str(tmp_path / "tenant-b"),
|
||||
local_path=str(tmp_path / "tenant-b"),
|
||||
namespace="tenant-b",
|
||||
)
|
||||
col_a = backend.get_collection(palace=ref_a, collection_name="drawers", create=True)
|
||||
col_b = backend.get_collection(palace=ref_b, collection_name="drawers", create=True)
|
||||
# Mechanism: the namespace partitions the table name.
|
||||
assert col_a._table != col_b._table
|
||||
assert "tenant_a" in col_a._table and "tenant_b" in col_b._table
|
||||
# Behaviour: a record under one namespace is invisible under the other.
|
||||
assert_partition_isolation(backend, col_a, col_b, embedding=[1.0, 0.0])
|
||||
|
||||
|
||||
def test_pgvector_update_merges_documents_and_metadata(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(
|
||||
ids=["a", "b"],
|
||||
documents=["alpha", "beta"],
|
||||
metadatas=[{"wing": "x", "rank": 1}, {"wing": "y", "rank": 2}],
|
||||
embeddings=[[1, 0], [0, 1]],
|
||||
)
|
||||
col.update(ids=["a"], documents=["alpha-2"], metadatas=[{"rank": 9}])
|
||||
got = col.get(ids=["a"], include=["documents", "metadatas"])
|
||||
assert got.documents == ["alpha-2"]
|
||||
# merge keeps the untouched key and overrides the updated one.
|
||||
assert got.metadatas[0] == {"wing": "x", "rank": 9}
|
||||
# untouched row is unchanged.
|
||||
assert col.get(ids=["b"]).ids == ["b"]
|
||||
with pytest.raises(ValueError, match="at least one"):
|
||||
col.update(ids=["a"])
|
||||
|
||||
|
||||
def test_pgvector_get_limit_offset_and_embeddings(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(
|
||||
ids=["a", "b", "c"],
|
||||
documents=["alpha", "beta", "gamma"],
|
||||
metadatas=[{"wing": "x"}, {"wing": "x"}, {"wing": "x"}],
|
||||
embeddings=[[1, 0], [0, 1], [0.5, 0.5]],
|
||||
)
|
||||
page = col.get(where={"wing": "x"}, limit=1, offset=1, include=["documents", "embeddings"])
|
||||
assert len(page.ids) == 1
|
||||
assert page.embeddings is not None and len(page.embeddings[0]) == 2
|
||||
|
||||
|
||||
def test_pgvector_delete_by_where_pushdown_and_local(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(
|
||||
ids=["a", "b", "c"],
|
||||
documents=["alpha", "beta", "gamma"],
|
||||
metadatas=[{"wing": "x"}, {"wing": "y"}, {"wing": "z"}],
|
||||
embeddings=[[1, 0], [0, 1], [0.5, 0.5]],
|
||||
)
|
||||
# pushdown equality delete
|
||||
col.delete(where={"wing": "y"})
|
||||
assert set(col.get().ids) == {"a", "c"}
|
||||
# local-fallback delete ($or routes through the exact path)
|
||||
col.delete(where={"$or": [{"wing": "x"}, {"wing": "z"}]})
|
||||
assert col.count() == 0
|
||||
|
||||
|
||||
def test_pgvector_query_dimension_mismatch_against_known_dim(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(ids=["a"], documents=["alpha"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
with pytest.raises(DimensionMismatchError):
|
||||
col.query(query_embeddings=[[1, 0, 0]], n_results=1)
|
||||
|
||||
|
||||
def test_pgvector_get_collection_positional_and_palace_path_forms(tmp_path, fake_pgvector):
|
||||
backend = PgVectorBackend()
|
||||
col = backend.get_collection(str(tmp_path / "p1"), "drawers", create=True)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
assert col.count() == 1
|
||||
col2 = backend.get_collection(
|
||||
palace_path=str(tmp_path / "p2"), collection_name="drawers", create=True
|
||||
)
|
||||
col2.upsert(ids=["b"], documents=["two"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
assert col2.count() == 1
|
||||
assert col._table != col2._table
|
||||
|
||||
|
||||
def test_pgvector_health_and_delete_collection(tmp_path, fake_pgvector):
|
||||
backend = PgVectorBackend()
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
col = backend.get_collection(palace=palace, collection_name="drawers", create=True)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
assert col.health().ok is True
|
||||
assert backend.health(palace).ok is True
|
||||
backend.delete_collection(str(tmp_path), "drawers")
|
||||
assert col.health().ok is False
|
||||
|
||||
|
||||
def test_pgvector_close_marks_backend_closed(tmp_path, fake_pgvector):
|
||||
backend = PgVectorBackend()
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
col = backend.get_collection(palace=palace, collection_name="drawers", create=True)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
backend.close()
|
||||
with pytest.raises(BackendError):
|
||||
backend.get_collection(palace=palace, collection_name="drawers", create=True)
|
||||
|
||||
|
||||
def test_pgvector_marker_unreadable_raises_mismatch(tmp_path, fake_pgvector):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
marker = tmp_path / "pgvector_backend.json"
|
||||
marker.write_text("{ not json", encoding="utf-8")
|
||||
backend2 = PgVectorBackend()
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
with pytest.raises(BackendMismatchError):
|
||||
backend2.get_collection(palace=palace, collection_name="drawers", create=True)
|
||||
|
||||
|
||||
def test_pgvector_dsn_resolved_from_env(tmp_path, fake_pgvector, monkeypatch):
|
||||
from mempalace.backends.pgvector import _PgVectorConfig
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_PGVECTOR_DSN", "postgresql://example:5432/memdb")
|
||||
monkeypatch.setenv("MEMPALACE_PGVECTOR_NAMESPACE", "team-a")
|
||||
config = _PgVectorConfig.from_options()
|
||||
assert config.dsn == "postgresql://example:5432/memdb"
|
||||
assert config.namespace == "team-a"
|
||||
|
||||
|
||||
def test_palace_wrapper_embeds_for_pgvector(tmp_path, monkeypatch, fake_pgvector):
|
||||
import mempalace.backends.embedding_wrapper as embedding_wrapper
|
||||
from mempalace import palace
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding_wrapper, "_embed_texts", lambda texts: [[1.0, 0.0] for _ in texts]
|
||||
)
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "pgvector")
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND", "pgvector")
|
||||
|
||||
col = palace.get_collection(str(tmp_path), "mempalace_drawers", create=True)
|
||||
col.add(documents=["wrapped pgvector document"], ids=["wrapped"], metadatas=[{"wing": "w"}])
|
||||
result = col.query(query_texts=["wrapped"], n_results=1)
|
||||
assert result.ids == [["wrapped"]]
|
||||
|
||||
|
||||
def test_pgvector_live_roundtrip_when_enabled(tmp_path):
|
||||
live_url = os.environ.get("MEMPALACE_PGVECTOR_LIVE_URL")
|
||||
if not live_url:
|
||||
pytest.skip("set MEMPALACE_PGVECTOR_LIVE_URL to run live Postgres pgvector test")
|
||||
|
||||
backend = PgVectorBackend()
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path), namespace="livetest")
|
||||
col = backend.get_collection(
|
||||
palace=palace,
|
||||
collection_name="drawers",
|
||||
create=True,
|
||||
options={"dsn": live_url},
|
||||
)
|
||||
try:
|
||||
col.upsert(
|
||||
ids=["live-a", "live-b"],
|
||||
documents=["rareterm live pgvector backend", "other live document"],
|
||||
metadatas=[{"wing": "live", "rank": 2}, {"wing": "other", "rank": 1}],
|
||||
embeddings=[[1.0, 0.0], [0.0, 1.0]],
|
||||
)
|
||||
assert PgVectorBackend.detect(str(tmp_path))
|
||||
assert col.count() == 2
|
||||
|
||||
result = col.query(query_embeddings=[[1.0, 0.0]], n_results=2, where={"wing": "live"})
|
||||
assert result.ids == [["live-a"]]
|
||||
|
||||
hits = col.lexical_search(query="rareterm", n_results=1).hits
|
||||
assert hits and hits[0].id == "live-a"
|
||||
|
||||
col.delete(ids=["live-a"])
|
||||
assert col.get(ids=["live-a"]).ids == []
|
||||
|
||||
# Reopen the existing table in a fresh backend and write another
|
||||
# same-dimension vector. This exercises table_dimension() against a
|
||||
# live vector(n) column — a regression guard for reading the dimension
|
||||
# off the raw atttypmod (which is not the bare n) and falsely raising
|
||||
# DimensionMismatchError on reopen.
|
||||
backend.close()
|
||||
backend = PgVectorBackend()
|
||||
reopened = backend.get_collection(
|
||||
palace=palace,
|
||||
collection_name="drawers",
|
||||
create=False,
|
||||
options={"dsn": live_url},
|
||||
)
|
||||
reopened.upsert(
|
||||
ids=["live-c"],
|
||||
documents=["third live document"],
|
||||
metadatas=[{"wing": "live", "rank": 3}],
|
||||
embeddings=[[0.5, 0.5]],
|
||||
)
|
||||
assert reopened.count() == 2
|
||||
finally:
|
||||
try:
|
||||
backend.delete_collection(str(tmp_path), "drawers")
|
||||
except Exception:
|
||||
pass
|
||||
backend.close()
|
||||
@@ -0,0 +1,529 @@
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from _backend_conformance import assert_partition_isolation
|
||||
|
||||
from mempalace.backends import (
|
||||
BackendError,
|
||||
BackendMismatchError,
|
||||
CollectionNotInitializedError,
|
||||
DimensionMismatchError,
|
||||
PalaceRef,
|
||||
available_backends,
|
||||
)
|
||||
from mempalace.backends.qdrant import QdrantBackend
|
||||
|
||||
|
||||
def _get_payload_value(payload, key):
|
||||
value = payload
|
||||
for part in key.split("."):
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
value = value.get(part)
|
||||
return value
|
||||
|
||||
|
||||
def _fake_match_condition(point, condition):
|
||||
if "must" in condition or "must_not" in condition or "should" in condition:
|
||||
return _fake_match_filter(point, condition)
|
||||
if "has_id" in condition:
|
||||
return point["id"] in set(condition["has_id"])
|
||||
key = condition.get("key")
|
||||
actual = _get_payload_value(point.get("payload") or {}, key)
|
||||
if "match" in condition:
|
||||
match = condition["match"]
|
||||
if "value" in match:
|
||||
return actual == match["value"]
|
||||
if "any" in match:
|
||||
return actual in set(match["any"] or [])
|
||||
if "text_any" in match:
|
||||
haystack = str(actual or "").lower()
|
||||
return any(token in haystack for token in str(match["text_any"]).lower().split())
|
||||
if "range" in condition:
|
||||
range_spec = condition["range"]
|
||||
try:
|
||||
if "gt" in range_spec and not actual > range_spec["gt"]:
|
||||
return False
|
||||
if "gte" in range_spec and not actual >= range_spec["gte"]:
|
||||
return False
|
||||
if "lt" in range_spec and not actual < range_spec["lt"]:
|
||||
return False
|
||||
if "lte" in range_spec and not actual <= range_spec["lte"]:
|
||||
return False
|
||||
except TypeError:
|
||||
return False
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def _fake_match_filter(point, qdrant_filter):
|
||||
if not qdrant_filter:
|
||||
return True
|
||||
must = qdrant_filter.get("must") or []
|
||||
must_not = qdrant_filter.get("must_not") or []
|
||||
should = qdrant_filter.get("should") or []
|
||||
if any(not _fake_match_condition(point, condition) for condition in must):
|
||||
return False
|
||||
if any(_fake_match_condition(point, condition) for condition in must_not):
|
||||
return False
|
||||
if should and not any(_fake_match_condition(point, condition) for condition in should):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class _FakeQdrantClient:
|
||||
instances = []
|
||||
|
||||
def __init__(self, _config):
|
||||
self.collections = {}
|
||||
self.query_calls = []
|
||||
self.scroll_calls = []
|
||||
self.created_indexes = []
|
||||
_FakeQdrantClient.instances.append(self)
|
||||
|
||||
def request(self, *_args, **_kwargs):
|
||||
return {"result": {}}
|
||||
|
||||
def collection_exists(self, collection):
|
||||
return collection in self.collections
|
||||
|
||||
def get_collection_info(self, collection):
|
||||
if collection not in self.collections:
|
||||
raise AssertionError("collection missing")
|
||||
return {
|
||||
"result": {
|
||||
"config": {
|
||||
"params": {
|
||||
"vectors": {
|
||||
"size": self.collections[collection]["dimension"],
|
||||
"distance": "Cosine",
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def create_collection(self, collection, dimension):
|
||||
self.collections.setdefault(collection, {"dimension": dimension, "points": {}})
|
||||
|
||||
def create_payload_index(self, collection, field_name, field_schema):
|
||||
self.created_indexes.append((collection, field_name, field_schema))
|
||||
|
||||
def upsert_points(self, collection, points):
|
||||
self.collections.setdefault(
|
||||
collection,
|
||||
{"dimension": len(points[0]["vector"]) if points else 0, "points": {}},
|
||||
)
|
||||
for point in points:
|
||||
self.collections[collection]["points"][point["id"]] = dict(point)
|
||||
|
||||
def query_points(self, collection, *, vector, limit, qdrant_filter, with_vector):
|
||||
self.query_calls.append(qdrant_filter)
|
||||
points = list(self.collections.get(collection, {"points": {}})["points"].values())
|
||||
points = [point for point in points if _fake_match_filter(point, qdrant_filter)]
|
||||
q = np.asarray(vector, dtype=np.float32)
|
||||
scored = []
|
||||
for point in points:
|
||||
vec = np.asarray(point["vector"], dtype=np.float32)
|
||||
denom = float(np.linalg.norm(q)) * float(np.linalg.norm(vec))
|
||||
score = 0.0 if denom <= 0 else float(np.dot(q, vec) / denom)
|
||||
out = {"id": point["id"], "payload": point["payload"], "score": score}
|
||||
if with_vector:
|
||||
out["vector"] = point["vector"]
|
||||
scored.append(out)
|
||||
scored.sort(key=lambda point: point["score"], reverse=True)
|
||||
return scored[:limit]
|
||||
|
||||
def scroll_points(
|
||||
self,
|
||||
collection,
|
||||
*,
|
||||
qdrant_filter=None,
|
||||
limit=256,
|
||||
offset=None,
|
||||
with_vector=False,
|
||||
):
|
||||
self.scroll_calls.append(qdrant_filter)
|
||||
points = list(self.collections.get(collection, {"points": {}})["points"].values())
|
||||
points = [point for point in points if _fake_match_filter(point, qdrant_filter)]
|
||||
start = int(offset or 0)
|
||||
selected = points[start : start + limit]
|
||||
next_offset = start + limit if start + limit < len(points) else None
|
||||
out = []
|
||||
for point in selected:
|
||||
item = {"id": point["id"], "payload": point["payload"]}
|
||||
if with_vector:
|
||||
item["vector"] = point["vector"]
|
||||
out.append(item)
|
||||
return out, next_offset
|
||||
|
||||
def delete_points(self, collection, *, point_ids=None, qdrant_filter=None):
|
||||
points = self.collections.get(collection, {"points": {}})["points"]
|
||||
if point_ids is not None:
|
||||
for point_id in point_ids:
|
||||
points.pop(point_id, None)
|
||||
return
|
||||
for point_id, point in list(points.items()):
|
||||
if _fake_match_filter(point, qdrant_filter):
|
||||
points.pop(point_id, None)
|
||||
|
||||
def count_points(self, collection):
|
||||
return len(self.collections.get(collection, {"points": {}})["points"])
|
||||
|
||||
def delete_collection(self, collection):
|
||||
self.collections.pop(collection, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_qdrant(monkeypatch):
|
||||
import mempalace.backends.qdrant as qdrant
|
||||
|
||||
_FakeQdrantClient.instances.clear()
|
||||
monkeypatch.setattr(qdrant, "_QdrantRESTClient", _FakeQdrantClient)
|
||||
monkeypatch.delenv("MEMPALACE_QDRANT_URL", raising=False)
|
||||
monkeypatch.delenv("MEMPALACE_QDRANT_API_KEY", raising=False)
|
||||
monkeypatch.delenv("MEMPALACE_QDRANT_NAMESPACE", raising=False)
|
||||
monkeypatch.delenv("MEMPALACE_QDRANT_TIMEOUT", raising=False)
|
||||
return _FakeQdrantClient
|
||||
|
||||
|
||||
def _collection(tmp_path, name="drawers"):
|
||||
backend = QdrantBackend()
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
return backend, backend.get_collection(palace=palace, collection_name=name, create=True)
|
||||
|
||||
|
||||
def test_registry_exposes_qdrant():
|
||||
assert "qdrant" in available_backends()
|
||||
|
||||
|
||||
def test_qdrant_add_query_filters_lexical_and_marker(tmp_path, fake_qdrant):
|
||||
backend, col = _collection(tmp_path)
|
||||
assert not os.path.isfile(tmp_path / "qdrant_backend.json")
|
||||
|
||||
col.add(
|
||||
ids=["a", "b", "c"],
|
||||
documents=[
|
||||
"alpha backend note",
|
||||
"rareterm qdrant backend note",
|
||||
"frontend design note",
|
||||
],
|
||||
metadatas=[
|
||||
{"wing": "project", "room": "backend", "rank": 1},
|
||||
{"wing": "project", "room": "backend", "rank": 3},
|
||||
{"wing": "project", "room": "frontend", "rank": 2},
|
||||
],
|
||||
embeddings=[[1, 0], [0.9, 0.1], [0, 1]],
|
||||
)
|
||||
|
||||
assert QdrantBackend.detect(str(tmp_path))
|
||||
assert os.path.isfile(tmp_path / "qdrant_backend.json")
|
||||
assert col.count() == 3
|
||||
|
||||
result = col.query(
|
||||
query_embeddings=[[1, 0]],
|
||||
n_results=3,
|
||||
where={"rank": {"$gte": 2}},
|
||||
include=["documents", "metadatas", "distances", "embeddings"],
|
||||
)
|
||||
assert result.ids == [["b", "c"]]
|
||||
assert result.documents[0][0] == "rareterm qdrant backend note"
|
||||
assert result.embeddings[0][0] == pytest.approx([0.9, 0.1])
|
||||
|
||||
hits = col.lexical_search(query="rareterm backend", n_results=2, where={"wing": "project"}).hits
|
||||
assert [hit.id for hit in hits] == ["b", "a"]
|
||||
assert fake_qdrant.instances[0].created_indexes[0][1:] == ("document", "text")
|
||||
|
||||
backend.close_palace(str(tmp_path))
|
||||
with pytest.raises(Exception):
|
||||
col.count()
|
||||
|
||||
|
||||
def test_qdrant_marker_not_written_when_first_write_fails(tmp_path, fake_qdrant, monkeypatch):
|
||||
_backend, col = _collection(tmp_path)
|
||||
fake_client = fake_qdrant.instances[0]
|
||||
|
||||
def fail_upsert(*_args, **_kwargs):
|
||||
raise RuntimeError("qdrant unavailable")
|
||||
|
||||
monkeypatch.setattr(fake_client, "upsert_points", fail_upsert)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
|
||||
assert not os.path.isfile(tmp_path / "qdrant_backend.json")
|
||||
|
||||
|
||||
def test_qdrant_upsert_update_delete_get_order_and_multi_collection(tmp_path, fake_qdrant):
|
||||
backend, drawers = _collection(tmp_path, "drawers")
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
closets = backend.get_collection(palace=palace, collection_name="closets", create=True)
|
||||
|
||||
drawers.upsert(
|
||||
ids=["one", "two"],
|
||||
documents=["first document", "second document"],
|
||||
metadatas=[{"wing": "a"}, {"wing": "b"}],
|
||||
embeddings=[[1, 0], [0, 1]],
|
||||
)
|
||||
closets.upsert(
|
||||
ids=["one"],
|
||||
documents=["closet document"],
|
||||
metadatas=[{"wing": "closet"}],
|
||||
embeddings=[[0.5, 0.5]],
|
||||
)
|
||||
|
||||
got = drawers.get(ids=["two", "one", "two"], include=["documents", "metadatas"])
|
||||
assert got.ids == ["two", "one", "two"]
|
||||
assert got.documents == ["second document", "first document", "second document"]
|
||||
|
||||
drawers.update(ids=["one"], metadatas=[{"room": "updated"}])
|
||||
assert drawers.get(ids=["one"]).metadatas == [{"wing": "a", "room": "updated"}]
|
||||
|
||||
drawers.delete(where={"wing": "b"})
|
||||
assert drawers.get().ids == ["one"]
|
||||
assert closets.get().ids == ["one"]
|
||||
|
||||
|
||||
def test_qdrant_complex_filters_use_exact_local_fallback(tmp_path, fake_qdrant):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(
|
||||
ids=["a", "b", "c"],
|
||||
documents=[
|
||||
"needle exact substring",
|
||||
"needle other wing",
|
||||
"boring filler",
|
||||
],
|
||||
metadatas=[
|
||||
{"wing": "target", "room": "backend", "tag": "alpha-beta"},
|
||||
{"wing": "other", "room": "backend", "tag": "beta"},
|
||||
{"wing": "target", "room": "front", "tag": "gamma"},
|
||||
],
|
||||
embeddings=[[1, 0], [0.8, 0.2], [0, 1]],
|
||||
)
|
||||
fake_client = fake_qdrant.instances[0]
|
||||
|
||||
result = col.query(
|
||||
query_embeddings=[[1, 0]],
|
||||
n_results=5,
|
||||
where={"$or": [{"wing": "target"}, {"tag": {"$contains": "alpha"}}]},
|
||||
where_document={"$contains": "needle"},
|
||||
)
|
||||
|
||||
assert result.ids == [["a"]]
|
||||
assert fake_client.query_calls == []
|
||||
|
||||
|
||||
def test_qdrant_lexical_empty_text_filter_does_not_full_scan(tmp_path, fake_qdrant):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(
|
||||
ids=["a", "b"],
|
||||
documents=["alpha backend note", "beta frontend note"],
|
||||
metadatas=[{"wing": "project"}, {"wing": "project"}],
|
||||
embeddings=[[1, 0], [0, 1]],
|
||||
)
|
||||
fake_client = fake_qdrant.instances[0]
|
||||
fake_client.scroll_calls.clear()
|
||||
|
||||
hits = col.lexical_search(query="missingterm", n_results=2).hits
|
||||
|
||||
assert hits == []
|
||||
assert len(fake_client.scroll_calls) == 1
|
||||
assert "text_any" in str(fake_client.scroll_calls[0])
|
||||
|
||||
|
||||
def test_qdrant_dimension_mismatch(tmp_path, fake_qdrant):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
|
||||
with pytest.raises(DimensionMismatchError):
|
||||
col.upsert(ids=["b"], documents=["two"], metadatas=[{}], embeddings=[[1, 0, 0]])
|
||||
|
||||
|
||||
def test_qdrant_add_rejects_duplicate_ids_in_same_batch(tmp_path, fake_qdrant):
|
||||
_backend, col = _collection(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="unique"):
|
||||
col.add(
|
||||
ids=["dup", "dup"],
|
||||
documents=["first", "second"],
|
||||
metadatas=[{}, {}],
|
||||
embeddings=[[1, 0], [0, 1]],
|
||||
)
|
||||
|
||||
assert not os.path.isfile(tmp_path / "qdrant_backend.json")
|
||||
|
||||
|
||||
def test_qdrant_marker_participates_in_backend_mismatch(tmp_path, monkeypatch, fake_qdrant):
|
||||
from mempalace.palace import resolve_backend_name
|
||||
|
||||
backend, col = _collection(tmp_path)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
backend.close()
|
||||
(tmp_path / "chroma.sqlite3").write_bytes(b"")
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "chroma")
|
||||
|
||||
with pytest.raises(BackendMismatchError):
|
||||
resolve_backend_name(str(tmp_path))
|
||||
|
||||
|
||||
def test_qdrant_marker_rejects_remote_target_change(tmp_path, monkeypatch, fake_qdrant):
|
||||
backend, col = _collection(tmp_path)
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_QDRANT_URL", "http://other-qdrant.example:6333")
|
||||
|
||||
with pytest.raises(BackendMismatchError, match="remote target"):
|
||||
backend.get_collection(palace=palace, collection_name="drawers", create=False)
|
||||
|
||||
|
||||
def test_qdrant_namespace_does_not_mix_palaces(tmp_path, fake_qdrant):
|
||||
backend = QdrantBackend()
|
||||
palace_a_path = tmp_path / "a"
|
||||
palace_b_path = tmp_path / "b"
|
||||
palace_a = PalaceRef(id=str(palace_a_path), local_path=str(palace_a_path), namespace="shared")
|
||||
palace_b = PalaceRef(id=str(palace_b_path), local_path=str(palace_b_path), namespace="shared")
|
||||
|
||||
col_a = backend.get_collection(palace=palace_a, collection_name="drawers", create=True)
|
||||
col_b = backend.get_collection(palace=palace_b, collection_name="drawers", create=True)
|
||||
col_a.upsert(ids=["same"], documents=["palace a"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
col_b.upsert(ids=["same"], documents=["palace b"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
|
||||
assert col_a.get(ids=["same"]).documents == ["palace a"]
|
||||
assert col_b.get(ids=["same"]).documents == ["palace b"]
|
||||
assert col_a._remote_collection != col_b._remote_collection
|
||||
|
||||
|
||||
def test_qdrant_missing_remote_after_marker_is_unhealthy(tmp_path, fake_qdrant):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
fake_client = fake_qdrant.instances[0]
|
||||
fake_client.delete_collection(col._remote_collection)
|
||||
|
||||
assert col.health().ok is False
|
||||
with pytest.raises(CollectionNotInitializedError):
|
||||
col.count()
|
||||
|
||||
|
||||
def test_search_reports_backend_error_distinct_from_missing_palace(tmp_path, monkeypatch):
|
||||
from mempalace import searcher
|
||||
|
||||
def fail_open(*_args, **_kwargs):
|
||||
raise BackendError("qdrant unavailable")
|
||||
|
||||
monkeypatch.setattr(searcher, "get_collection", fail_open)
|
||||
|
||||
result = searcher.search_memories("needle", str(tmp_path))
|
||||
|
||||
assert result["error"] == "Backend error"
|
||||
assert "qdrant unavailable" in result["details"]
|
||||
|
||||
|
||||
def test_palace_wrapper_embeds_for_qdrant(tmp_path, monkeypatch, fake_qdrant):
|
||||
import mempalace.backends.embedding_wrapper as embedding_wrapper
|
||||
from mempalace import palace
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding_wrapper, "_embed_texts", lambda texts: [[1.0, 0.0] for _ in texts]
|
||||
)
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "qdrant")
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND", "qdrant")
|
||||
|
||||
col = palace.get_collection(str(tmp_path), "mempalace_drawers", create=True)
|
||||
col.add(documents=["wrapped qdrant document"], ids=["wrapped"], metadatas=[{"wing": "w"}])
|
||||
result = col.query(query_texts=["wrapped"], n_results=1)
|
||||
assert result.ids == [["wrapped"]]
|
||||
|
||||
|
||||
def test_qdrant_rejects_pure_remote_palace(tmp_path, fake_qdrant):
|
||||
"""No local_path means the marker (the only mismatch-protection anchor)
|
||||
cannot be written or validated, so the backend must refuse rather than
|
||||
silently open an unprotected remote collection (RFC 001 isolation contract, PR #1679)."""
|
||||
backend = QdrantBackend()
|
||||
palace = PalaceRef(id="tenant-remote", local_path=None, namespace="tenant-remote")
|
||||
with pytest.raises(BackendError, match="local palace path"):
|
||||
backend.get_collection(palace=palace, collection_name="drawers", create=True)
|
||||
|
||||
|
||||
def test_qdrant_cross_palace_isolation_conformance(tmp_path, fake_qdrant):
|
||||
"""Shared per-PalaceRef.id isolation conformance (RFC 001 isolation contract)."""
|
||||
backend = QdrantBackend()
|
||||
cols = []
|
||||
for label in ("alpha", "beta"):
|
||||
path = tmp_path / label
|
||||
ref = PalaceRef(id=str(path), local_path=str(path))
|
||||
cols.append(backend.get_collection(palace=ref, collection_name="drawers", create=True))
|
||||
assert_partition_isolation(backend, cols[0], cols[1], embedding=[1.0, 0.0])
|
||||
|
||||
|
||||
def test_qdrant_namespace_isolation_conformance(tmp_path, fake_qdrant):
|
||||
"""Shared per-PalaceRef.namespace isolation conformance — qdrant advertises
|
||||
``supports_namespace_isolation`` so it must satisfy the cross-namespace MUST
|
||||
(RFC 001 isolation contract)."""
|
||||
assert "supports_namespace_isolation" in QdrantBackend.capabilities
|
||||
backend = QdrantBackend()
|
||||
ref_a = PalaceRef(
|
||||
id=str(tmp_path / "tenant-a"),
|
||||
local_path=str(tmp_path / "tenant-a"),
|
||||
namespace="tenant-a",
|
||||
)
|
||||
ref_b = PalaceRef(
|
||||
id=str(tmp_path / "tenant-b"),
|
||||
local_path=str(tmp_path / "tenant-b"),
|
||||
namespace="tenant-b",
|
||||
)
|
||||
col_a = backend.get_collection(palace=ref_a, collection_name="drawers", create=True)
|
||||
col_b = backend.get_collection(palace=ref_b, collection_name="drawers", create=True)
|
||||
# Mechanism: the namespace partitions the remote collection name.
|
||||
assert col_a._remote_collection != col_b._remote_collection
|
||||
# Behaviour: a record under one namespace is invisible under the other.
|
||||
assert_partition_isolation(backend, col_a, col_b, embedding=[1.0, 0.0])
|
||||
|
||||
|
||||
def test_qdrant_live_rest_roundtrip_when_enabled(tmp_path):
|
||||
live_url = os.environ.get("MEMPALACE_QDRANT_LIVE_URL")
|
||||
if not live_url:
|
||||
pytest.skip("set MEMPALACE_QDRANT_LIVE_URL to run live Qdrant REST test")
|
||||
|
||||
backend = QdrantBackend()
|
||||
namespace = f"live_{uuid.uuid4().hex}"
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path), namespace=namespace)
|
||||
col = backend.get_collection(
|
||||
palace=palace,
|
||||
collection_name="drawers",
|
||||
create=True,
|
||||
options={
|
||||
"url": live_url,
|
||||
"api_key": os.environ.get("MEMPALACE_QDRANT_LIVE_API_KEY"),
|
||||
},
|
||||
)
|
||||
try:
|
||||
col.upsert(
|
||||
ids=["live-a", "live-b"],
|
||||
documents=["rareterm live qdrant backend", "other live document"],
|
||||
metadatas=[{"wing": "live", "rank": 2}, {"wing": "other", "rank": 1}],
|
||||
embeddings=[[1.0, 0.0], [0.0, 1.0]],
|
||||
)
|
||||
assert QdrantBackend.detect(str(tmp_path))
|
||||
|
||||
result = col.query(
|
||||
query_embeddings=[[1.0, 0.0]],
|
||||
n_results=2,
|
||||
where={"wing": "live"},
|
||||
)
|
||||
assert result.ids == [["live-a"]]
|
||||
|
||||
hits = col.lexical_search(query="rareterm", n_results=1).hits
|
||||
assert hits and hits[0].id == "live-a"
|
||||
|
||||
col.delete(ids=["live-a"])
|
||||
assert col.get(ids=["live-a"]).ids == []
|
||||
finally:
|
||||
try:
|
||||
col._client.delete_collection(col._remote_collection)
|
||||
except Exception:
|
||||
pass
|
||||
backend.close()
|
||||
@@ -0,0 +1,377 @@
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from mempalace.backends import (
|
||||
BackendMismatchError,
|
||||
CollectionNotInitializedError,
|
||||
DimensionMismatchError,
|
||||
PalaceRef,
|
||||
QueryResult,
|
||||
UnsupportedCapabilityError,
|
||||
available_backends,
|
||||
)
|
||||
from mempalace.backends.sqlite_exact import SQLiteExactBackend
|
||||
|
||||
|
||||
def _collection(tmp_path, name="mempalace_drawers", create=True):
|
||||
backend = SQLiteExactBackend()
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
return backend, backend.get_collection(palace=palace, collection_name=name, create=create)
|
||||
|
||||
|
||||
def test_sqlite_exact_missing_collection_error_names_collection(tmp_path):
|
||||
"""CollectionNotInitializedError must identify the missing collection, not
|
||||
the palace path — consistent with line 287 and the other backends."""
|
||||
backend, _ = _collection(tmp_path, name="mempalace_drawers")
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
with pytest.raises(CollectionNotInitializedError) as exc:
|
||||
backend.get_collection(palace=palace, collection_name="does_not_exist", create=False)
|
||||
assert "does_not_exist" in str(exc.value)
|
||||
assert str(tmp_path) not in str(exc.value)
|
||||
|
||||
with pytest.raises(CollectionNotInitializedError) as exc2:
|
||||
backend.delete_collection(str(tmp_path), "also_missing")
|
||||
assert "also_missing" in str(exc2.value)
|
||||
assert str(tmp_path) not in str(exc2.value)
|
||||
|
||||
|
||||
def test_registry_exposes_sqlite_exact():
|
||||
assert "sqlite_exact" in available_backends()
|
||||
|
||||
|
||||
def test_sqlite_exact_add_query_filters_and_persistence(tmp_path):
|
||||
backend, col = _collection(tmp_path)
|
||||
col.add(
|
||||
ids=["a", "b", "c"],
|
||||
documents=[
|
||||
"alpha vector memory",
|
||||
"beta sqlite exact memory",
|
||||
"gamma filtered memory",
|
||||
],
|
||||
metadatas=[
|
||||
{"wing": "alpha", "room": "notes", "chunk_index": 0, "tags": "core,vector"},
|
||||
{"wing": "alpha", "room": "notes", "chunk_index": 1, "tags": "sqlite,exact"},
|
||||
{"wing": "gamma", "room": "archive", "chunk_index": 2, "tags": "old"},
|
||||
],
|
||||
embeddings=[[1.0, 0.0], [0.0, 1.0], [0.2, 0.8]],
|
||||
)
|
||||
|
||||
ranked = col.query(query_embeddings=[[1.0, 0.0]], n_results=3)
|
||||
assert ranked.ids[0] == ["a", "c", "b"]
|
||||
assert ranked.distances[0][0] == pytest.approx(0.0)
|
||||
|
||||
filtered = col.get(
|
||||
where={
|
||||
"$and": [
|
||||
{"wing": "alpha"},
|
||||
{"chunk_index": {"$gte": 1}},
|
||||
{"tags": {"$contains": "sqlite"}},
|
||||
]
|
||||
},
|
||||
include=["documents", "metadatas", "embeddings"],
|
||||
)
|
||||
assert filtered.ids == ["b"]
|
||||
assert filtered.documents == ["beta sqlite exact memory"]
|
||||
assert filtered.embeddings == [[0.0, 1.0]]
|
||||
|
||||
col.update(ids=["b"], metadatas=[{"room": "lab"}])
|
||||
assert col.get(ids=["b"]).metadatas[0]["room"] == "lab"
|
||||
|
||||
backend.close_palace(str(tmp_path))
|
||||
reopened = backend.get_collection(
|
||||
palace=PalaceRef(id=str(tmp_path), local_path=str(tmp_path)),
|
||||
collection_name="mempalace_drawers",
|
||||
create=False,
|
||||
)
|
||||
assert reopened.count() == 3
|
||||
assert reopened.get(ids=["a"]).documents == ["alpha vector memory"]
|
||||
|
||||
|
||||
def test_sqlite_exact_write_failure_rolls_back_whole_batch(tmp_path):
|
||||
_backend, col = _collection(tmp_path)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
col.add(
|
||||
ids=["dup", "dup"],
|
||||
documents=["first write", "duplicate write"],
|
||||
metadatas=[{}, {}],
|
||||
embeddings=[[1.0, 0.0], [0.0, 1.0]],
|
||||
)
|
||||
|
||||
assert col.count() == 0
|
||||
|
||||
|
||||
def test_sqlite_exact_enforces_collection_dimension(tmp_path):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(ids=["a"], documents=["two dims"], metadatas=[{}], embeddings=[[1.0, 0.0]])
|
||||
|
||||
with pytest.raises(DimensionMismatchError):
|
||||
col.add(ids=["b"], documents=["three dims"], metadatas=[{}], embeddings=[[1.0, 0.0, 0.0]])
|
||||
with pytest.raises(DimensionMismatchError):
|
||||
col.upsert(
|
||||
ids=["b"], documents=["three dims"], metadatas=[{}], embeddings=[[1.0, 0.0, 0.0]]
|
||||
)
|
||||
with pytest.raises(DimensionMismatchError):
|
||||
col.update(ids=["a"], embeddings=[[1.0, 0.0, 0.0]])
|
||||
with pytest.raises(DimensionMismatchError):
|
||||
col.query(query_embeddings=[[1.0, 0.0, 0.0]], n_results=1)
|
||||
|
||||
assert col.count() == 1
|
||||
assert col.get(ids=["a"]).documents == ["two dims"]
|
||||
|
||||
|
||||
def test_sqlite_exact_get_preserves_requested_id_order_and_duplicates(tmp_path):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(
|
||||
ids=["a", "b"],
|
||||
documents=["doc a", "doc b"],
|
||||
metadatas=[{}, {}],
|
||||
embeddings=[[1, 0], [0, 1]],
|
||||
)
|
||||
|
||||
result = col.get(ids=["b", "a", "b"], include=["documents"])
|
||||
|
||||
assert result.ids == ["b", "a", "b"]
|
||||
assert result.documents == ["doc b", "doc a", "doc b"]
|
||||
|
||||
|
||||
def test_sqlite_exact_upsert_delete_and_multi_collection_isolation(tmp_path):
|
||||
backend, drawers = _collection(tmp_path, "drawers")
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
closets = backend.get_collection(palace=palace, collection_name="closets", create=True)
|
||||
|
||||
drawers.upsert(
|
||||
ids=["same"], documents=["drawer one"], metadatas=[{"kind": "drawer"}], embeddings=[[1, 0]]
|
||||
)
|
||||
closets.upsert(
|
||||
ids=["same"], documents=["closet one"], metadatas=[{"kind": "closet"}], embeddings=[[0, 1]]
|
||||
)
|
||||
drawers.upsert(
|
||||
ids=["same"],
|
||||
documents=["drawer replaced"],
|
||||
metadatas=[{"kind": "drawer", "version": 2}],
|
||||
embeddings=[[1, 0]],
|
||||
)
|
||||
|
||||
assert drawers.count() == 1
|
||||
assert closets.count() == 1
|
||||
assert drawers.get(ids=["same"]).documents == ["drawer replaced"]
|
||||
assert closets.get(ids=["same"]).documents == ["closet one"]
|
||||
|
||||
drawers.delete(where={"version": {"$in": [2, 3]}})
|
||||
assert drawers.count() == 0
|
||||
assert closets.count() == 1
|
||||
|
||||
|
||||
def test_sqlite_exact_lexical_search_and_python_fallback(tmp_path, monkeypatch):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(
|
||||
ids=["a", "b", "c"],
|
||||
documents=[
|
||||
"ordinary project note",
|
||||
"rareterm rareterm sqlite exact note",
|
||||
"rareterm unrelated archive",
|
||||
],
|
||||
metadatas=[
|
||||
{"wing": "w", "room": "a"},
|
||||
{"wing": "w", "room": "b"},
|
||||
{"wing": "old", "room": "b"},
|
||||
],
|
||||
embeddings=[[1, 0], [0, 1], [0.5, 0.5]],
|
||||
)
|
||||
|
||||
hits = col.lexical_search(query="rareterm sqlite", n_results=2, where={"wing": "w"}).hits
|
||||
assert [hit.id for hit in hits] == ["b"]
|
||||
|
||||
monkeypatch.setattr(col, "_fts_available", lambda _cur: False)
|
||||
fallback_hits = col.lexical_search(query="rareterm sqlite", n_results=2).hits
|
||||
assert fallback_hits[0].id == "b"
|
||||
|
||||
|
||||
def test_sqlite_exact_lexical_search_filters_after_full_fts_window(tmp_path):
|
||||
_backend, col = _collection(tmp_path)
|
||||
ids = [f"old-{i}" for i in range(12)] + ["target"]
|
||||
col.add(
|
||||
ids=ids,
|
||||
documents=["needle shared lexical note" for _ in ids],
|
||||
metadatas=[{"wing": "old"} for _ in range(12)] + [{"wing": "target"}],
|
||||
embeddings=[[1.0, 0.0] for _ in ids],
|
||||
)
|
||||
|
||||
hits = col.lexical_search(query="needle", n_results=1, where={"wing": "target"}).hits
|
||||
|
||||
assert [hit.id for hit in hits] == ["target"]
|
||||
|
||||
|
||||
def test_sqlite_exact_logical_filters_evaluate_sibling_predicates(tmp_path):
|
||||
_backend, col = _collection(tmp_path)
|
||||
col.add(
|
||||
ids=["a", "b"],
|
||||
documents=["alpha document", "beta document"],
|
||||
metadatas=[
|
||||
{"wing": "w", "room": "wrong", "kind": "note"},
|
||||
{"wing": "w", "room": "right", "kind": "note"},
|
||||
],
|
||||
embeddings=[[1, 0], [0, 1]],
|
||||
)
|
||||
|
||||
result = col.get(where={"$and": [{"wing": "w"}], "room": "right"})
|
||||
|
||||
assert result.ids == ["b"]
|
||||
|
||||
|
||||
def test_sqlite_exact_close_palace_marks_existing_collections_closed(tmp_path):
|
||||
backend, col = _collection(tmp_path)
|
||||
palace = PalaceRef(id=str(tmp_path), local_path=str(tmp_path))
|
||||
col.add(ids=["a"], documents=["doc"], metadatas=[{}], embeddings=[[1, 0]])
|
||||
|
||||
backend.close_palace(palace)
|
||||
|
||||
assert not col.health().ok
|
||||
with pytest.raises(Exception):
|
||||
col.count()
|
||||
|
||||
|
||||
def test_palace_wrapper_embeds_for_sqlite_exact(tmp_path, monkeypatch):
|
||||
import mempalace.backends.embedding_wrapper as embedding_wrapper
|
||||
from mempalace.palace import get_collection
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact")
|
||||
monkeypatch.setattr(
|
||||
embedding_wrapper,
|
||||
"_embed_texts",
|
||||
lambda texts: [[float(len(text)), 1.0] for text in texts],
|
||||
)
|
||||
|
||||
col = get_collection(str(tmp_path), create=True)
|
||||
col.add(ids=["a"], documents=["abcd"], metadatas=[{"wing": "w"}])
|
||||
|
||||
result = col.query(query_texts=["abcd"], n_results=1)
|
||||
assert result.ids == [["a"]]
|
||||
|
||||
|
||||
def test_backend_mismatch_protection(tmp_path, monkeypatch):
|
||||
from mempalace.palace import get_collection
|
||||
|
||||
(tmp_path / "chroma.sqlite3").write_bytes(b"")
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact")
|
||||
|
||||
with pytest.raises(BackendMismatchError):
|
||||
get_collection(str(tmp_path), create=True)
|
||||
|
||||
|
||||
def test_mixed_backend_artifacts_are_rejected_even_when_chroma_selected(tmp_path, monkeypatch):
|
||||
from mempalace.palace import resolve_backend_name
|
||||
|
||||
(tmp_path / "chroma.sqlite3").write_bytes(b"")
|
||||
(tmp_path / "sqlite_exact.sqlite3").write_bytes(b"")
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "chroma")
|
||||
|
||||
with pytest.raises(BackendMismatchError):
|
||||
resolve_backend_name(str(tmp_path))
|
||||
|
||||
|
||||
def test_sqlite_exact_exact_ranking_uses_cosine(tmp_path):
|
||||
_backend, col = _collection(tmp_path)
|
||||
halfway = [0.5, math.sqrt(0.75)]
|
||||
col.add(
|
||||
ids=["half", "orthogonal", "same"],
|
||||
documents=["half", "orthogonal", "same"],
|
||||
metadatas=[{}, {}, {}],
|
||||
embeddings=[halfway, [0.0, 1.0], [1.0, 0.0]],
|
||||
)
|
||||
|
||||
result = col.query(query_embeddings=[[1.0, 0.0]], n_results=3)
|
||||
assert result.ids[0] == ["same", "half", "orthogonal"]
|
||||
assert result.distances[0] == pytest.approx([0.0, 0.5, 1.0])
|
||||
|
||||
|
||||
def test_search_union_uses_sqlite_exact_lexical_search(tmp_path, monkeypatch):
|
||||
import mempalace.backends.embedding_wrapper as embedding_wrapper
|
||||
from mempalace.palace import get_collection
|
||||
from mempalace.searcher import search_memories
|
||||
|
||||
def fake_embed(texts):
|
||||
vectors = []
|
||||
for text in texts:
|
||||
if text == "rareterm":
|
||||
vectors.append([1.0, 0.0])
|
||||
elif "rareterm" in text:
|
||||
vectors.append([0.0, 1.0])
|
||||
else:
|
||||
vectors.append([0.5, math.sqrt(0.75)])
|
||||
return vectors
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact")
|
||||
monkeypatch.setattr(embedding_wrapper, "_embed_texts", fake_embed)
|
||||
|
||||
col = get_collection(str(tmp_path), create=True)
|
||||
col.add(
|
||||
ids=["d1", "d2", "d3", "rare"],
|
||||
documents=[
|
||||
"ordinary support note",
|
||||
"ordinary billing note",
|
||||
"ordinary project note",
|
||||
"rareterm rareterm rareterm policy note",
|
||||
],
|
||||
metadatas=[
|
||||
{"wing": "w", "room": "r", "source_file": "/tmp/d1.md", "chunk_index": 0},
|
||||
{"wing": "w", "room": "r", "source_file": "/tmp/d2.md", "chunk_index": 0},
|
||||
{"wing": "w", "room": "r", "source_file": "/tmp/d3.md", "chunk_index": 0},
|
||||
{"wing": "w", "room": "r", "source_file": "/tmp/rare.md", "chunk_index": 0},
|
||||
],
|
||||
)
|
||||
|
||||
result = search_memories(
|
||||
"rareterm",
|
||||
str(tmp_path),
|
||||
n_results=1,
|
||||
candidate_strategy="union",
|
||||
)
|
||||
|
||||
assert result["results"][0]["source_file"] == "rare.md"
|
||||
assert result["results"][0]["matched_via"] == "bm25_backend"
|
||||
|
||||
|
||||
def test_search_union_reports_unsupported_lexical_capability(monkeypatch, tmp_path):
|
||||
import mempalace.searcher as searcher
|
||||
|
||||
class NoLexicalCollection:
|
||||
def query(self, **_kwargs):
|
||||
return QueryResult(
|
||||
ids=[["a"]],
|
||||
documents=[["ordinary note"]],
|
||||
metadatas=[[{"source_file": "/tmp/a.md", "chunk_index": 0}]],
|
||||
distances=[[0.5]],
|
||||
)
|
||||
|
||||
def lexical_search(self, **_kwargs):
|
||||
raise UnsupportedCapabilityError("no lexical support")
|
||||
|
||||
monkeypatch.setattr(searcher, "get_collection", lambda *_args, **_kwargs: NoLexicalCollection())
|
||||
monkeypatch.setattr(
|
||||
searcher,
|
||||
"get_closets_collection",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("no closets")),
|
||||
)
|
||||
|
||||
result = searcher.search_memories(
|
||||
"anything",
|
||||
str(tmp_path),
|
||||
n_results=1,
|
||||
candidate_strategy="union",
|
||||
)
|
||||
|
||||
assert result["unsupported_capability"] == "supports_lexical_search"
|
||||
|
||||
|
||||
def test_search_vector_disabled_fallback_is_chroma_only(tmp_path, monkeypatch):
|
||||
from mempalace.searcher import search_memories
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact")
|
||||
|
||||
result = search_memories("anything", str(tmp_path), vector_disabled=True)
|
||||
|
||||
assert result["unsupported_capability"] == "chroma_hnsw_fallback"
|
||||
assert result["backend"] == "sqlite_exact"
|
||||
@@ -1999,6 +1999,10 @@ gpu = [
|
||||
{ name = "onnxruntime-gpu", version = "1.24.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
|
||||
{ name = "onnxruntime-gpu", version = "1.25.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
||||
]
|
||||
pgvector = [
|
||||
{ name = "psycopg", version = "3.2.13", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version < '3.10'" },
|
||||
{ name = "psycopg", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version >= '3.10'" },
|
||||
]
|
||||
spellcheck = [
|
||||
{ name = "autocorrect" },
|
||||
]
|
||||
@@ -2032,16 +2036,17 @@ requires-dist = [
|
||||
{ name = "onnxruntime-gpu", marker = "extra == 'gpu'", specifier = ">=1.16" },
|
||||
{ name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.0" },
|
||||
{ name = "psutil", marker = "extra == 'dev'", specifier = ">=5.9" },
|
||||
{ name = "psycopg", extras = ["binary"], marker = "extra == 'pgvector'", specifier = ">=3.1" },
|
||||
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0" },
|
||||
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0" },
|
||||
{ name = "python-dateutil", specifier = ">=2.8" },
|
||||
{ name = "pyyaml", specifier = ">=6.0,<7" },
|
||||
{ name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.14" },
|
||||
{ name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.15" },
|
||||
{ name = "striprtf", marker = "extra == 'extract'", specifier = ">=0.0.27" },
|
||||
{ name = "tokenizers", specifier = ">=0.15" },
|
||||
{ name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" },
|
||||
]
|
||||
provides-extras = ["dev", "spellcheck", "gpu", "dml", "coreml", "multilingual", "extract"]
|
||||
provides-extras = ["dev", "spellcheck", "pgvector", "gpu", "dml", "coreml", "multilingual", "extract"]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
@@ -2051,7 +2056,7 @@ dev = [
|
||||
{ name = "psutil", specifier = ">=5.9" },
|
||||
{ name = "pytest", specifier = ">=7.0" },
|
||||
{ name = "pytest-cov", specifier = ">=4.0" },
|
||||
{ name = "ruff", specifier = "==0.15.14" },
|
||||
{ name = "ruff", specifier = "==0.15.15" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3704,6 +3709,199 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg"
|
||||
version = "3.2.13"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.10'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.10'" },
|
||||
{ name = "tzdata", marker = "python_full_version < '3.10' and sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/44/05/d4a05988f15fcf90e0088c735b1f2fc04a30b7fc65461d6ec278f5f2f17a/psycopg-3.2.13.tar.gz", hash = "sha256:309adaeda61d44556046ec9a83a93f42bbe5310120b1995f3af49ab6d9f13c1d", size = 160626, upload-time = "2025-11-21T22:34:32.328Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/14/f2724bd1986158a348316e86fdd0837a838b14a711df3f00e47fba597447/psycopg-3.2.13-py3-none-any.whl", hash = "sha256:a481374514f2da627157f767a9336705ebefe93ea7a0522a6cbacba165da179a", size = 206797, upload-time = "2025-11-21T22:29:39.733Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
binary = [
|
||||
{ name = "psycopg-binary", version = "3.2.13", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and implementation_name != 'pypy'" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg"
|
||||
version = "3.3.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.15' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.15' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.14.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.14.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.10.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.10.*' and sys_platform != 'win32'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" },
|
||||
{ name = "tzdata", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
binary = [
|
||||
{ name = "psycopg-binary", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and implementation_name != 'pypy'" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg-binary"
|
||||
version = "3.2.13"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.10'",
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/16/325f72b7ebdb906bd6cca6c0caea5b8fd7092c4686237c5669fe3f3cc7f2/psycopg_binary-3.2.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9e25eb65494955c0dabdcd7097b004cbd70b982cf3cbc7186c2e854f788677a9", size = 4013642, upload-time = "2025-11-21T22:29:43.39Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/a6/f7616dfcab942d5ad6fb5ce8364148e22a4cd817340ac368b6a6bd17559d/psycopg_binary-3.2.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:732b25c2d932ca0655ea2588563eae831dc0842c93c69be4754a5b0e9760b38d", size = 4076666, upload-time = "2025-11-21T22:29:51.33Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/f7/cddf75c43c967c9262afe6863275fdd2e5f877d98c379f5c3a21b6fa419d/psycopg_binary-3.2.13-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7350d9cc4e35529c4548ddda34a1c17f28d3f3a8f792c25cd67e8a04952ed415", size = 4639390, upload-time = "2025-11-21T22:29:57.614Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/b9/f86f2e6413ac024b3a759fd446cc90c325a0d7403dce533bd419e1c41164/psycopg_binary-3.2.13-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:090c22795969ee1ace17322b1718769694607d942cef084c6fb4493adfa57da0", size = 4737745, upload-time = "2025-11-21T22:30:01.814Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/aa/1a17c7176875d7e0a848710d87f13fdd3cc08724fa6bfcc43c72846f22b9/psycopg_binary-3.2.13-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ac329532f36342ff99fc1aefdbb531563bec03c7bc3ae934c8347a7a61339df", size = 4419762, upload-time = "2025-11-21T22:30:05.401Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/9b/5c7f8c90a3504c45ceadffa1f1f4b2fc8ce9e04494cf67d27dfa265e5681/psycopg_binary-3.2.13-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1db11a7e618d58cfb937c409c7d279a84cbb31d32a7efc63f1e5f426f3613793", size = 3878529, upload-time = "2025-11-21T22:30:09.493Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ea/37/37e7152e6b0813e68361768d1baf0e40d8ed0ac8091471641c2c88e0cec6/psycopg_binary-3.2.13-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:5f5081b2cbb0358bb3625109d41b57411bf9d9c29762a867e38c06d974b245ee", size = 3560767, upload-time = "2025-11-21T22:30:13.88Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f7/b2/929d8e15b8797486d160b797ce84a4d0251a9361f7f31e9b01b439608e3b/psycopg_binary-3.2.13-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5d466ac3a3738647ff2405397946870dc363e33282ced151e7ea74f622947c06", size = 3604456, upload-time = "2025-11-21T22:30:18.392Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/74/4d4e7481bc717bbe3de689c4d40439d4e1be07df989da2c38140298cbae5/psycopg_binary-3.2.13-cp310-cp310-win_amd64.whl", hash = "sha256:087acf2b24787ae206718136c1f51bc90cda68b02c3819b0556f418e3565f2c3", size = 2910871, upload-time = "2025-11-21T22:30:22.24Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/06/f5/fc70804a999167daf5b876107b99e8fe91c3f785a31753c0e3e7b93446ba/psycopg_binary-3.2.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9cfe87749d010dfd34534ba8c71aa0674db9a3fce65232c98989f77c742c9ce7", size = 4013844, upload-time = "2025-11-21T22:30:25.985Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/87/857639681f5dfcd567aaf199fe4e5b026a105b0462a604f4fb7eda0735d8/psycopg_binary-3.2.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8db77fac1dfe3f69c982db92a51fd78e1354fa8f523a6781a636123e5c7ffcde", size = 4077002, upload-time = "2025-11-21T22:30:29.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/1d/2cb7af6a31429b9022455c966d8408a2b5a19acd3de7610402381518e8f7/psycopg_binary-3.2.13-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cbbac4cd5b0e14b91ad8244268ca3fc2f527d1a337b489af57d7669c9d2e1a24", size = 4637181, upload-time = "2025-11-21T22:30:34.126Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/28/bd/ffde1ac7e6ab75646c253fbe0378772fb6f0229af8a05cd9862ee8aad0f0/psycopg_binary-3.2.13-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a146f0a59a7e3ca92996f8133b1d5e5922e668f7c656b4a9201e702f4cf25896", size = 4737775, upload-time = "2025-11-21T22:30:38.408Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/74/3702732d01639c97943d56ec26860357dfacda0b5a708e82e794d07f499c/psycopg_binary-3.2.13-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:27150515de5f709e4142429db6fd36a1d01f0b8b17d915b5f7bb095364465398", size = 4421537, upload-time = "2025-11-21T22:30:42.696Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f2/8c/915a899857c2211196aa7f1749ba85bed421afaf72f185a0eb91e64ba550/psycopg_binary-3.2.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9942255705255367d94368941e3a913b0daf74b47d191471dbe4dc0de9fbc769", size = 3877500, upload-time = "2025-11-21T22:30:47.064Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/d9/46060c183413bf62d47df98d7e3b30ab561639bcb583c3796cca30dafa43/psycopg_binary-3.2.13-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:75ebc8335f48c339ec24f4c371595f6b7043147fe6d18e619c8564428ab8adaf", size = 3560186, upload-time = "2025-11-21T22:30:54.522Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/cf/2987689614632898e4861e4122cd41937ea9b5afcbe3c3061c7265bfa6de/psycopg_binary-3.2.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6fe2982a73b2ea473c9e2b91a35a21af3b03313bed188eccbcde4972483ac60a", size = 3601117, upload-time = "2025-11-21T22:31:01.218Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e2/ef/df7fa8a47ef47d08af8a792343811a98bc7ab48f763560fc1d5acc1f28af/psycopg_binary-3.2.13-cp311-cp311-win_amd64.whl", hash = "sha256:6a50db4661fae78779d3cc38a0a68cabc997ca9d485ec27443b109ef8ac1672a", size = 2912873, upload-time = "2025-11-21T22:31:05.473Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/9e/f90243b3d0d007a89989b013b0eb3e78ac929fed4eb40a2b317452abafe1/psycopg_binary-3.2.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:223fc610a80bbc4355ad3c9952d468a18bb5cd7065846a8c275f100d80cd4004", size = 3996285, upload-time = "2025-11-21T22:31:08.95Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/42/7d55f515ee3e2ced5ff9bc493fb2308f5187686b6d9583cd6a9c880d2053/psycopg_binary-3.2.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b67f06a68d68b4621b6a411f9e583df876977afa06b1ba270b1b347d40aa93fc", size = 4070567, upload-time = "2025-11-21T22:31:12.31Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/a8/ead4de04d8cf5f35119a75a8dd92fa4a2ec8a309b1aa58855f64616c03d7/psycopg_binary-3.2.13-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:082579f2ae41bdabe20c82810810f3e290ac2206cccf0cb41cf36b3218f53b3c", size = 4616833, upload-time = "2025-11-21T22:31:16.614Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/2e/4af6ab69ade7d67d31296f88c79c322a3522564e30b3f1458f19e74d67c3/psycopg_binary-3.2.13-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:ff7df7bd8ec2c805f3a4896b8ade971139af0f9f8cf45d05014ac71fe54887be", size = 4711710, upload-time = "2025-11-21T22:31:22.007Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/31/bdbd6b2264bb7ae5fe8b775c5524da73329d8888c6137fd8b050ff9cabbc/psycopg_binary-3.2.13-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8f1189dc78553ef4b2e55d9e116fc74870191bc6a9a5f4442412a703c4cc6c3b", size = 4401656, upload-time = "2025-11-21T22:31:26.842Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/c5/8fd8f96450e4ef242022c9a588305e3dc7309c34bc392a9b4c2da60854b1/psycopg_binary-3.2.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0ef8ed4a4e0f7bf5e941782478a43c14b2b585b031e2266dd3afb87be2775d95", size = 3851747, upload-time = "2025-11-21T22:31:30.5Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/47/406d102ae49d253f124644530f1e5b3fd2f92aea59d4f9b8dd1c71cf8e0f/psycopg_binary-3.2.13-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:de06fc9707a49f7c081b5c950974dd6de3dc33d681f7524f0b396471f5a4a480", size = 3524796, upload-time = "2025-11-21T22:31:34.377Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/6f/a89be8aee27a5522e97dbcb225fe429c489acdf0bb25fc0fadb329dfb39f/psycopg_binary-3.2.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:917ad1cd6e6ef8a9df2f28d7b29c7148f089be46ac56fe838f986c0227652d14", size = 3576536, upload-time = "2025-11-21T22:31:38.06Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/f8/c924c7dc792c81bf6181d7d4eeb613c8b2151b3a208f95cedec3c1a25ba3/psycopg_binary-3.2.13-cp312-cp312-win_amd64.whl", hash = "sha256:b53b0d9499805b307017070492189e349256e0946f62c815e442baa01f2ea6c5", size = 2902172, upload-time = "2025-11-21T22:31:41.256Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/28/ec/ef37bb44dc02fcc6c0a3eeb93f4baaac13bcb228633fe38ad3fb5a3f6449/psycopg_binary-3.2.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dbae6ab1966e2b61d97e47220556c330c4608bb4cfb3a124aa0595c39995c068", size = 3995628, upload-time = "2025-11-21T22:31:45.921Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/ad/4748f5f1a40248af16dba087dbec50bd335ee025cc1fb9bf64773378ceff/psycopg_binary-3.2.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fae933e4564386199fc54845d85413eedb49760e0bcd2b621fde2dd1825b99b3", size = 4069024, upload-time = "2025-11-21T22:31:50.202Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/c2/f02ec6bbc30c7fcd3b39823d2d624b42fae480edeb6e50eb3276281d5635/psycopg_binary-3.2.13-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:13e2f8894d410678529ff9f1211f96c5a93ff142f992b302682b42d924428b61", size = 4615127, upload-time = "2025-11-21T22:31:56.517Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/0d/a54fc2cdd672c84175d6869cc823d6ec2a8909318d491f3c24e6077983f2/psycopg_binary-3.2.13-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f26f7009375cf1e92180e5c517c52da1054f7e690dde90e0ed00fa8b5736bcd4", size = 4710267, upload-time = "2025-11-21T22:32:04.585Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/b7/067de1acaf3d312253351f3af4121f972584bd36cada6378d4b0cdcebd38/psycopg_binary-3.2.13-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea2fdbcc9142933a47c66970e0df8b363e3bd1ea4c5ce376f2f3d94a9aeec847", size = 4400795, upload-time = "2025-11-21T22:32:08.883Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/b5/030e6b1ebfc4d3a8fca03adc5fc827982643bad0b01a1268538d17c08ed3/psycopg_binary-3.2.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac92d6bc1d4a41c7459953a9aa727b9966e937e94c9e072527317fd2a67d488b", size = 3851239, upload-time = "2025-11-21T22:32:12.333Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/79/6f/0541845364a7de9eae6807060da6a04b22a8eb2e803606d285d9250fbe93/psycopg_binary-3.2.13-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:8b843c00478739e95c46d6d3472b13123b634685f107831a9bfc41503a06ecbd", size = 3525084, upload-time = "2025-11-21T22:32:15.946Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/ae/6507890dc30a4bbd9d938d4ff3a4079d009a5ad8170af51c7f762438fdbf/psycopg_binary-3.2.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2f63868cc96bc18486cebec24445affbdd7f7debf28fac466ea935a8b5a4753b", size = 3576787, upload-time = "2025-11-21T22:32:19.922Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/64/3d1c2f1fd09b60cdfbe68b9a810b357ba505eff6e4bdb1a2d9f6729da64c/psycopg_binary-3.2.13-cp313-cp313-win_amd64.whl", hash = "sha256:594dfbca3326e997ae738d3d339004e8416b1f7390f52ce8dc2d692393e8fa96", size = 2905584, upload-time = "2025-11-21T22:32:23.399Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/b4/7656b3d67bedff2b900c8c4671cb6eb5fb99c2fc36da33579cac89779c25/psycopg_binary-3.2.13-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:502a778c3e07c6b3aabfa56ee230e8c264d2debfab42d11535513a01bdfff0d6", size = 3997201, upload-time = "2025-11-21T22:32:28.185Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/2e/3b4afbd94d48df19c3931cedba464b109f89d81ac43178e6a3d654b4e8d5/psycopg_binary-3.2.13-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7561a71d764d6f74d66e8b7d844b0f27fa33de508f65c17b1d56a94c73644776", size = 4071631, upload-time = "2025-11-21T22:32:32.594Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/8b/107d06d55992e2f13157eb705ba5a47d06c4cf1bed077dff0c567b10c187/psycopg_binary-3.2.13-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9caf14745a1930b4e03fe4072cd7154eaf6e1241d20c42130ed784408a26b24b", size = 4620918, upload-time = "2025-11-21T22:32:37.357Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/47/a925620f261b115f31e813a5bfe640f316413b1864094a60162f4a6e4d67/psycopg_binary-3.2.13-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a6cafabdc0bfa37e11c6f365020fd5916b62d6296df581f4dceaa43a2ce680c", size = 4714494, upload-time = "2025-11-21T22:32:42.138Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/33/bed384665356bb9ba17dd8e104884d87cc2343d16dffdfd9aaa9a159bd4d/psycopg_binary-3.2.13-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96cb5a27e68acac6d74b64fca38592a692de9c4b7827339190698d58027aa45", size = 4403046, upload-time = "2025-11-21T22:32:47.241Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/41/88/749d8e8102fb5df502e2ecb053b79e78e3358af01af652b5dbeb96ab7905/psycopg_binary-3.2.13-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:596176ae3dfbf56fc61108870bfe17c7205d33ac28d524909feb5335201daa0a", size = 3859046, upload-time = "2025-11-21T22:32:51.481Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/38/7c/f492e63b517d6dcd564e8c43bc15e11a4c712a848adf8938ce33bfd4c867/psycopg_binary-3.2.13-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:cc3a0408435dfbb77eeca5e8050df4b19a6e9b7e5e5583edf524c4a83d6293b2", size = 3531351, upload-time = "2025-11-21T22:32:55.571Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/5a/d8743eb23944e5cf2a0bbfa92935c140b5beaacdb872be641065ed70ab2c/psycopg_binary-3.2.13-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:65df0d459ffba14082d8ca4bb2f6ffbb2f8d02968f7d34a747e1031934b76b23", size = 3581034, upload-time = "2025-11-21T22:33:01.648Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/b2/411d4180252144f7eff024894d2d2ebb98c012c944a282fc20250870e461/psycopg_binary-3.2.13-cp314-cp314-win_amd64.whl", hash = "sha256:5c77f156c7316529ed371b5f95a51139e531328ee39c37493a2afcbc1f79d5de", size = 3000162, upload-time = "2025-11-21T22:33:07.378Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/dc/3ea3fe5df19af323b4b78e0e98e073f8117b1336e5b6dc6978c067485019/psycopg_binary-3.2.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d8d1b709509d0f8cb857acf740b5eccd5bd2fb208a5b20e895f250519a32459", size = 4015148, upload-time = "2025-11-21T22:33:47.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/28/a832b014974e7bda61b3c684afe5e47f70d5dc4471cbab90a41a7c2bdf6a/psycopg_binary-3.2.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d45bc5f4335498d32a26c8f8c0bf9ce8c973c19e78a9ee77c031300fb361300", size = 4078197, upload-time = "2025-11-21T22:33:52.494Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/8c/5962c876a8bba4a6f8ff941998577e8359c928c700d092893e10f97aa94e/psycopg_binary-3.2.13-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f062d725898bf6fc5cfc6349a0d08ee09f129deb14d7fcd5c30f9f1b349f39dc", size = 4638520, upload-time = "2025-11-21T22:33:57.568Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/b2/b557ac96752da8fd4b0ff7a128d148e6809ce576a2add6156c91d55abe0a/psycopg_binary-3.2.13-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:915647b5bbbcde2bd464dc293eec4f74710fa71edc4f85aa6f6c8494a179dc9e", size = 4737730, upload-time = "2025-11-21T22:34:02.969Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b5/9a/af2d96c0e711e90cf340a5f607911cd6df593fe1aec9c46644162161af18/psycopg_binary-3.2.13-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d3aec6e2f1cf4deb1b9a3ac287c0591479f3bd851d0a911d628f8c2c71c14f4a", size = 4421382, upload-time = "2025-11-21T22:34:11.501Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/f6/f8135198a2c70ca663b55d44c6fc3beb4e36025679b541a9d489814f2ddc/psycopg_binary-3.2.13-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a56a8b1794cbf27ca04012ac2890d58cfc82b3b310c1dac4fa78fbf6f57e7440", size = 3879259, upload-time = "2025-11-21T22:34:17.706Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/8c/3f778fc954f0b691941073a1d8b78c07219594135831cad32a739e4eee97/psycopg_binary-3.2.13-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:4150a5e72f863be442d153829724109d83a76871d9bc801d6bb5b9c84b5b19b9", size = 3560475, upload-time = "2025-11-21T22:34:21.329Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/d2/731d56c636155f210fbb00cdbb7498c0e04a21052415520da54ac96eca63/psycopg_binary-3.2.13-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:028b49eb465f5d263d250cfd4f168fdabb306d0bbd97fd66a8a1fd7b696a953c", size = 3605616, upload-time = "2025-11-21T22:34:25.229Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/22/2619870c9ed44b5eaeae4f7706126754ccadde6319483cd4c490f5d13fbb/psycopg_binary-3.2.13-cp39-cp39-win_amd64.whl", hash = "sha256:532ea34f673148d637be65a96251832252e278540b39fbd683ef37e58ec361c1", size = 2912739, upload-time = "2025-11-21T22:34:29.069Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg-binary"
|
||||
version = "3.3.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.15' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.15' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.14.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.14.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.10.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.10.*' and sys_platform != 'win32'",
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/bf/70d8a60488f9955cbbcd538beae44d56bb2f1d19e673b72788f2d343ff55/psycopg_binary-3.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a", size = 4609750, upload-time = "2026-05-01T23:24:20.06Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/b0/29e98ba210c9dbc75a6dc91e3f99b9e06ea901a62ca95804e02a1ae13e6b/psycopg_binary-3.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a", size = 4676700, upload-time = "2026-05-01T23:25:21.727Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/ab/3df087b3c12bf74e47c08204172b2fabb5a144679110d5c7ad12d9201323/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429", size = 5496319, upload-time = "2026-05-01T23:25:28.271Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/9a/f088207b4cd6772f9e0d8a91807e79fa2458d4eb9eb1ae406c68415f2bec/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765", size = 5171906, upload-time = "2026-05-01T23:25:34.004Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/45/4523a857f253871d75c22e1c2e79fd47e599e736bcba1bad58d83e24be02/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13", size = 6762621, upload-time = "2026-05-01T23:25:41.392Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/d1/925bf776503345bef428e6c45fb017d0139ddbe0e211814b585c4253dca8/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc", size = 5006319, upload-time = "2026-05-01T23:25:51.419Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/aa/99727337206fbba357ca084bf4ea8b29dc986f61842a2685859af61416db/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28", size = 4535388, upload-time = "2026-05-01T23:25:57.957Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/a4/567ba2c37d19d8c2f63d836385dfd2495aa5897bbee6cfab104d9ee58624/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e", size = 4224544, upload-time = "2026-05-01T23:26:03.832Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/23/86457f5a82731685d7701de7bfaa5eb783dd1fecbf875321897d9d9ce33a/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d", size = 3956282, upload-time = "2026-05-01T23:26:09.983Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/d8/249456df16d47de082abd9b73bce8ccdeb0293eb12e590f9150c7cbdb788/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744", size = 4261736, upload-time = "2026-05-01T23:26:16.798Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/6b/c4abe228acafd8a385c1fb615d4f1e3c9b8ad7a4e4f0e84118ba3ffeed9c/psycopg_binary-3.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949", size = 3570620, upload-time = "2026-05-01T23:26:22.655Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pybase64"
|
||||
version = "1.4.3"
|
||||
@@ -4821,27 +5019,27 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.15.14"
|
||||
version = "0.15.15"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/dc/8a/8bce2894573e9dae6ff4d77fe34ad727d79b9e6238ad288c5638990d90f6/ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f", size = 4700910, upload-time = "2026-05-21T14:34:55.177Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/84/6f/a76f7d96e5c962f5b69cee865e49c15c1116897c01990faa8a57edb62e7f/ruff-0.15.15.tar.gz", hash = "sha256:b8dff018130b46d8e5bf0f926ef6b60cf871d6d5ae45fc9334e09632daa741d6", size = 4706985, upload-time = "2026-05-28T14:16:57.784Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b9/c8/74a92c6ff9fcfb4f1f947126d3ebee8389276e161ecc85de5bda7cda51bd/ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108", size = 10739177, upload-time = "2026-05-21T14:34:37.332Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/91/254a35c20acc38a7223c9d2d594af12e794432464f2cdeb52af1dc4a892d/ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b", size = 11144969, upload-time = "2026-05-21T14:34:43.978Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/9e/d13e40f83b8d0a94430e6778ce1d94a43b38cf2efe63278bdd2b4c65abbf/ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f", size = 10478207, upload-time = "2026-05-21T14:34:48.378Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/f1/b15a7839fa4f332f8acec78e20564f26bb2d866e3d21710b877fd0263000/ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d", size = 10818459, upload-time = "2026-05-21T14:34:22.318Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/33/53d651177f84f94b400a0e27f8824eeada3dddc9d5ee8aeb048f4352a520/ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4", size = 10541800, upload-time = "2026-05-21T14:34:20.209Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/a6/868f87e0bf9786ed24b5d0d0ad8676b8a94fd1912f42cddf9cfc7857818a/ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542", size = 11342149, upload-time = "2026-05-21T14:34:46.365Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/8b/38cd5c19faffdcc05a408d2b78edccc69492ab9720eadb49ea15ef80d768/ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f", size = 12212563, upload-time = "2026-05-21T14:34:28.579Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3e/4d/a3c5b874a556d5731e3e657aaf04311bb76f0a5c3ec220ed43051be6b64b/ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf", size = 11493299, upload-time = "2026-05-21T14:34:41.836Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/c0/56472c251d09858a53e51efbd485b09e1995d8731668b76d52e5dd6ee0f1/ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba", size = 11455931, upload-time = "2026-05-21T14:34:57.276Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/4a/e2e7b4d8dbf233d4eace59c75bc3435fa6d8bd3bae82d351d4e4300c0fd1/ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f", size = 11400794, upload-time = "2026-05-21T14:34:39.773Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/c7/83c0539fe34c3e09136204d1e75d6052492364e0b3cb05e9465423f567d7/ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581", size = 10804759, upload-time = "2026-05-21T14:34:31.045Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/a6/18f2bfc095a2ab4a78745644e428205532ce6653a5d0fa8501572891534d/ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93", size = 10539517, upload-time = "2026-05-21T14:34:53.064Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/3a/5a8b3b69c654d4e4bf1d246ac5b49cbcdac6eaab6905925f8915f31e3b80/ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61", size = 11065169, upload-time = "2026-05-21T14:34:24.484Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/c5/8864e4e7925b836ea354b31d57641ec03830564e281a8b6f061f8c3e0ec1/ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553", size = 11560214, upload-time = "2026-05-21T14:34:50.975Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/38/012bf76752e1f89ed50b77b99532d90f3a3e287bc7918e1fc0948ac866ac/ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6", size = 10805548, upload-time = "2026-05-21T14:34:33.453Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/b7/4ea2c170f10ad760fff2a5250beb18897719dc8b52b53a24cddbb9dd3f19/ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902", size = 11939523, upload-time = "2026-05-21T14:34:18.077Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/d5/bc97ff895ec35cf3925d4bd60f3b39d822f377a446906ec9bcc87405e59b/ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826", size = 11208607, upload-time = "2026-05-21T14:34:26.525Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/9d/3a45c05b8ab04b4705989de70a79008e27c8003296a0feaee9edc18dd7e9/ruff-0.15.15-py3-none-linux_armv6l.whl", hash = "sha256:cf93e5388f412e1b108b1f8b34a6e036b70fe8aff89393befad96fe48670311b", size = 10710652, upload-time = "2026-05-28T14:16:06.701Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/66/da974431624bf3b49f6ee1f9543c02d929ff1cba78b0d5a79c38cf21f744/ruff-0.15.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ac5a646d1f6a7dadd5d50842dae2c1f9862ac887ef5d1b1375e02def791fde6e", size = 11096615, upload-time = "2026-05-28T14:16:23.313Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/09/7443452e5d290230a712103f2fdceeef7184f3ec99a2bd01c8be78aaceb5/ruff-0.15.15-py3-none-macosx_11_0_arm64.whl", hash = "sha256:77d955a431430c66f72dd94e379ad38a16daea3d25094872ac4edf9e797be530", size = 10436683, upload-time = "2026-05-28T14:16:40.974Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/53/01/d330c26a57fa4f3943a14424904027428315b700fe4d14a84bb123a649e5/ruff-0.15.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7614ee79c69788cf6cedd568069ade9cecc22a1ad20494efe8d0c9ebb4b622d4", size = 10769064, upload-time = "2026-05-28T14:16:28.905Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1d/85/cc8770f8bdff541b1da8392d1634141fe4a0e3f4ee596605959b7906c27f/ruff-0.15.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3cdb1679e06a1f6b47bc384714ae96f6e2fb65ca441eb78c43d2ca554176ce1f", size = 10511987, upload-time = "2026-05-28T14:16:43.732Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/29/8c190c1472b63013583ba391f3342036e02010544c1270455ed8e519bdf3/ruff-0.15.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2728b93d7b23a603ea2c0ac6eb73d760bd38ec9de35f35fb41e18f7a3fee7622", size = 11275100, upload-time = "2026-05-28T14:16:55.244Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/6b/7e145ce2cc8e63d6834eca03d83a0e18d121def5c69f91b4cf4011ed4879/ruff-0.15.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be582fcc0db438902c7792b08d6ddf6c9b9e21addaa10092c2c741cfb09e5a45", size = 12176903, upload-time = "2026-05-28T14:16:14.368Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/a3/d5974637f68e451f7fadf015cf3101d1cd7d8ba5027cffe0b9e3826ebe6b/ruff-0.15.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7aa77465b8ecaf1a27bea098d696f7fed5e1eccbd10b321b682d6de586ae5627", size = 11404550, upload-time = "2026-05-28T14:16:20.138Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fe/1c/e6e5e568f22be4fb05d6244234aba384c06b451252453b821e1a529263cf/ruff-0.15.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48decfa11d740de4889de623be1463308346312f2409a56e24aa280c86162dc4", size = 11382027, upload-time = "2026-05-28T14:16:46.615Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1d/01/170921b49fcd2e8858825593f91cf7146c3e40a5c3e6df763e4bb0484dde/ruff-0.15.15-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a5015088452ca0081387063649ec67f06d3d1d6b8b936a1f836b5e9657ecd48c", size = 11366041, upload-time = "2026-05-28T14:16:26.247Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/54/a7bad711d7de93254e15e06a4c375b89a03d18de45d3e5dcc86a4472fb1a/ruff-0.15.15-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f5294aab6356c81600fcdea3a62bb1b924dfd5e91767c12318d3f68f86af57cd", size = 10741795, upload-time = "2026-05-28T14:16:17.11Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c9/31/38c075963668f8b41c6914ee0f6f318727fbe30ab9145cb29e6df464c5fa/ruff-0.15.15-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:db5bd4d802415cca656dc1616070b725952d6ae95eb5d4831e49fbd94a38f75f", size = 10511117, upload-time = "2026-05-28T14:16:31.767Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/96/6ff689e1f7e375d1d97075eca022f74c2bab59554a432fe4d2e6f091986a/ruff-0.15.15-py3-none-musllinux_1_2_i686.whl", hash = "sha256:587a6278ed42059191c1a466e490bd7930fb50bd2e255398bc29616c895a61cb", size = 10994867, upload-time = "2026-05-28T14:16:35.149Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/c2/5dce0ab9f92a8d534fa62b9bf9caca3eddb8c1a81b616f5e195ada4f0d6e/ruff-0.15.15-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:df0c1c084f5f4be9812f61518a45c440d3c30d69ce4bf6c5270e66d38338f02a", size = 11482101, upload-time = "2026-05-28T14:16:49.598Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/c0/1003b60edd697c649faf61f1a34094b1abb38fb3d1181e3f895781250a08/ruff-0.15.15-py3-none-win32.whl", hash = "sha256:29428ea79694afbe756d45fd59b36f22b6b020dc0443cf7de0173046236964b9", size = 10716774, upload-time = "2026-05-28T14:16:52.337Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/a8/1269eddd6945a06c23f055ef7848886e37cf9d6a8bebb386a3115f01470c/ruff-0.15.15-py3-none-win_amd64.whl", hash = "sha256:8df0323902e15e24bc4bf246da830573d3cf3352bd0b9a164eab335d111ff4a4", size = 11868463, upload-time = "2026-05-28T14:16:11.333Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/b2/920464c907b191e37469d477a1aa8bc048b8f36c4c1610dfa4ab87b39e18/ruff-0.15.15-py3-none-win_arm64.whl", hash = "sha256:3c8ceca6792f38196b8f589bc92eccd03eef286602da92e5dc05cc42ef6441b7", size = 11138498, upload-time = "2026-05-28T14:16:38.425Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user