Files
mempalace/tests/test_backends.py
T
Pim MesselinkandClaude Opus 4.7 cd7a8658a2 fix(backends): require SQLite magic header for chroma + sqlite_exact detect() (#1893) (#1896)
* fix(chroma): require SQLite magic header for ChromaBackend.detect() (#1893)

Closes #1893.

ChromaBackend.detect() was returning True for a 0-byte chroma.sqlite3 file
because the check was just os.path.isfile(...). On a palace that has any
other backend marker alongside a stale 0-byte chroma.sqlite3,
resolve_backend_name then raises BackendMismatchError and the palace becomes
unopenable until the user manually rm's the empty file.

The 0-byte file appears as a side effect of any sqlite3.connect() on a
missing path — Python creates the file immediately but writes the SQLite
header only on the first statement. So any code path that touches the
chroma.sqlite3 path with bare sqlite3.connect(), including chromadb's own
PersistentClient lazy-init (see the comment at backends/chroma.py:2052),
can leave a 0-byte artifact behind.

Fix: detect() now reads the first 16 bytes and compares to the SQLite
magic prefix b"SQLite format 3\x00" instead of relying on file presence
alone. One extra open() + 16-byte read; detect() isn't a hot path.

Properties:
- Rejects 0-byte files (the symptom #1893 is about).
- Rejects non-SQLite garbage at the canonical path (partial writes, etc.).
- Doesn't false-negative on real chroma palaces: any chroma palace whose
  PersistentClient has done any work has the magic header on disk
  (verified — CREATE TABLE is enough to land the header).
- Doesn't couple detect() to chroma's specific schema; the magic header
  is stable across chromadb releases.

Test sweep: many test files used (chroma.sqlite3).touch() or
.write_bytes(b"") as a "fake palace" shortcut, exploiting the loose
isfile() check (one such site even had the comment "# pass the isfile
guard"). After this change, those stand-ins no longer register as chroma
palaces. Introduced tests/_chroma_palace_helper.py::make_minimal_chroma_sqlite
following the existing _backend_conformance.py precedent, and updated 15
call sites across 8 test files to use it. The existing
test_chroma_detect_matches_palace_with_chroma_sqlite (which encoded the
buggy semantics with write_bytes(b"")) is renamed to
test_chroma_detect_matches_palace_with_sqlite_header and now writes a
real SQLite database via the helper. Added two new tests for the
rejection paths (empty file, non-SQLite garbage).

Full env-cleared suite: 3137 passed, 20 skipped, 0 failed. ruff check
and ruff format --check both clean.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KC5Qsknh2zFRtRvVyjXiTA

* fix(sqlite_exact): require SQLite magic header for SQLiteExactBackend.detect()

Per gemini-code-assist review on #1892 PR #1896: SQLiteExactBackend has the
same os.path.isfile() detection pattern as ChromaBackend did, with the same
0-byte-file vulnerability. Mirrors the chroma fix for repo-wide consistency.

- SQLiteExactBackend.detect() now does the same 16-byte SQLite magic-prefix
  check as ChromaBackend.detect().
- _chroma_palace_helper.py: factored its body into a private
  _write_minimal_sqlite_file() and gained a sibling
  make_minimal_sqlite_exact_sqlite() for the sqlite_exact filename. No churn
  to any existing chroma call sites.
- test_sqlite_exact_backend.py:426 (the one site that wrote b"" for
  sqlite_exact.sqlite3) updated to use the new helper.
- Three new tests in test_sqlite_exact_backend.py mirror the chroma trio:
  matches with valid header, rejects empty file, rejects non-SQLite garbage.

Full env-cleared suite: 3140 passed, 20 skipped, 0 failed.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KC5Qsknh2zFRtRvVyjXiTA

---------

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
2026-06-28 18:01:59 -03:00

1910 lines
69 KiB
Python

import os
import pickle
import shutil
import sqlite3
from contextlib import closing
from pathlib import Path
import chromadb
import pytest
from mempalace.backends import (
CollectionNotInitializedError,
GetResult,
PalaceNotFoundError,
PalaceRef,
QueryResult,
UnsupportedFilterError,
available_backends,
get_backend,
)
from mempalace.backends.chroma import (
ChromaBackend,
ChromaCollection,
_HNSW_MISSING_METADATA_DATA_FLOOR,
_fix_blob_seq_ids,
_fix_missing_collection_type,
_pin_hnsw_threads,
_segment_appears_healthy,
quarantine_invalid_hnsw_metadata,
quarantine_stale_hnsw,
)
# embeddinggemma-300m Matryoshka truncation (first 384 of 768 dims).
_TEST_EMBED_DIM = 384
class _FakeCollection:
"""Stand-in for a chromadb.Collection returning raw chroma-shaped dicts."""
def __init__(self, query_response=None, get_response=None, count_value=7):
self.calls = []
self._query_response = query_response or {
"ids": [["a", "b"]],
"documents": [["da", "db"]],
"metadatas": [[{"wing": "w1"}, {"wing": "w2"}]],
"distances": [[0.1, 0.2]],
}
self._get_response = get_response or {
"ids": ["a"],
"documents": ["da"],
"metadatas": [{"wing": "w1"}],
}
self._count_value = count_value
def add(self, **kwargs):
self.calls.append(("add", kwargs))
def upsert(self, **kwargs):
self.calls.append(("upsert", kwargs))
def update(self, **kwargs):
self.calls.append(("update", kwargs))
def query(self, **kwargs):
self.calls.append(("query", kwargs))
return self._query_response
def get(self, **kwargs):
self.calls.append(("get", kwargs))
return self._get_response
def delete(self, **kwargs):
self.calls.append(("delete", kwargs))
def count(self):
self.calls.append(("count", {}))
return self._count_value
def test_chroma_collection_returns_typed_query_result():
fake = _FakeCollection()
collection = ChromaCollection(fake)
result = collection.query(query_texts=["q"])
assert isinstance(result, QueryResult)
assert result.ids == [["a", "b"]]
assert result.documents == [["da", "db"]]
assert result.metadatas == [[{"wing": "w1"}, {"wing": "w2"}]]
assert result.distances == [[0.1, 0.2]]
assert result.embeddings is None
def test_chroma_collection_returns_typed_get_result():
fake = _FakeCollection()
collection = ChromaCollection(fake)
result = collection.get(where={"wing": "w1"})
assert isinstance(result, GetResult)
assert result.ids == ["a"]
assert result.documents == ["da"]
assert result.metadatas == [{"wing": "w1"}]
def test_query_result_empty_preserves_outer_dimension():
empty = QueryResult.empty(num_queries=2)
assert empty.ids == [[], []]
assert empty.documents == [[], []]
assert empty.distances == [[], []]
assert empty.embeddings is None
def test_typed_results_support_dict_compat_access():
"""Transitional compat shim per base.py — retained until callers migrate to attrs."""
result = GetResult(ids=["a"], documents=["da"], metadatas=[{"w": 1}])
assert result["ids"] == ["a"]
assert result.get("documents") == ["da"]
assert result.get("missing", "default") == "default"
assert "ids" in result
assert "missing" not in result
def test_chroma_collection_query_empty_result_preserves_outer_shape():
fake = _FakeCollection(
query_response={"ids": [], "documents": [], "metadatas": [], "distances": []}
)
collection = ChromaCollection(fake)
result = collection.query(query_texts=["q1", "q2"])
assert result.ids == [[], []]
assert result.documents == [[], []]
assert result.distances == [[], []]
def test_chroma_collection_rejects_unknown_where_operator():
fake = _FakeCollection()
collection = ChromaCollection(fake)
with pytest.raises(UnsupportedFilterError):
collection.query(query_texts=["q"], where={"$regex": "foo"})
def test_chroma_collection_delegates_writes():
fake = _FakeCollection()
collection = ChromaCollection(fake)
collection.add(documents=["d"], ids=["1"], metadatas=[{"wing": "w"}])
collection.upsert(documents=["u"], ids=["2"], metadatas=[{"room": "r"}])
collection.delete(ids=["1"])
assert collection.count() == 7
kinds = [call[0] for call in fake.calls]
assert kinds == ["add", "upsert", "delete", "count"]
def test_registry_exposes_chroma_by_default():
names = available_backends()
assert "chroma" in names
assert isinstance(get_backend("chroma"), ChromaBackend)
def test_registry_unknown_backend_raises():
with pytest.raises(KeyError):
get_backend("no-such-backend-exists")
def test_resolve_backend_priority_order(tmp_path):
from mempalace.backends import resolve_backend_for_palace
# explicit kwarg wins over everything
assert resolve_backend_for_palace(explicit="pg", config_value="lance") == "pg"
# config value wins over env / default
assert resolve_backend_for_palace(config_value="lance", env_value="qdrant") == "lance"
# env wins over default
assert resolve_backend_for_palace(env_value="qdrant", default="chroma") == "qdrant"
# falls back to default
assert resolve_backend_for_palace() == "chroma"
def test_chroma_detect_matches_palace_with_sqlite_header(tmp_path):
"""A real SQLite database at ``<path>/chroma.sqlite3`` registers as chroma.
Uses ``sqlite3.connect`` + a write so the SQLite magic header is actually
on disk — the only thing detection looks at.
"""
db_path = tmp_path / "chroma.sqlite3"
conn = sqlite3.connect(db_path)
conn.execute("CREATE TABLE detect_smoke(x)")
conn.commit()
conn.close()
assert ChromaBackend.detect(str(tmp_path)) is True
assert ChromaBackend.detect(str(tmp_path.parent)) is False
def test_chroma_detect_rejects_empty_chroma_sqlite(tmp_path):
"""A 0-byte ``chroma.sqlite3`` is not a chroma palace (closes #1893).
Bare ``sqlite3.connect()`` against a missing path leaves a 0-byte file
behind because the SQLite header is written on the first statement, not
on connect. Detection must reject that artifact so it cannot trip
``BackendMismatchError`` against a real non-chroma backend marker in the
same directory.
"""
(tmp_path / "chroma.sqlite3").write_bytes(b"")
assert ChromaBackend.detect(str(tmp_path)) is False
def test_chroma_detect_rejects_non_sqlite_file(tmp_path):
"""A non-SQLite file at the ``chroma.sqlite3`` path is not chroma.
Defends against partial writes / garbage content / anything that lands at
the canonical path but isn't actually a SQLite database.
"""
(tmp_path / "chroma.sqlite3").write_bytes(b"not a sqlite file" * 4)
assert ChromaBackend.detect(str(tmp_path)) is False
def test_chroma_lexical_search_uses_sqlite_fts_not_full_collection_scan(tmp_path):
db_path = tmp_path / "chroma.sqlite3"
conn = sqlite3.connect(db_path)
conn.executescript(
"""
CREATE TABLE collections (id INTEGER PRIMARY KEY, name TEXT NOT NULL);
CREATE TABLE segments (id INTEGER PRIMARY KEY, collection INTEGER NOT NULL);
CREATE TABLE embeddings (
id INTEGER PRIMARY KEY,
segment_id INTEGER NOT NULL,
embedding_id TEXT,
created_at TEXT
);
CREATE TABLE embedding_metadata (
id INTEGER,
key TEXT,
string_value TEXT,
int_value INTEGER,
float_value REAL,
bool_value INTEGER
);
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value);
"""
)
conn.execute("INSERT INTO collections(id, name) VALUES (1, 'mempalace_drawers')")
conn.execute("INSERT INTO segments(id, collection) VALUES (1, 1)")
ids = list(range(1, 14))
for emb_id in ids:
wing = "target" if emb_id == 13 else "old"
doc = "needle shared lexical note"
conn.execute(
"INSERT INTO embeddings(id, segment_id, embedding_id, created_at) VALUES (?, 1, ?, ?)",
(emb_id, f"public-{emb_id}", f"2026-01-01T00:00:{emb_id:02d}"),
)
conn.execute(
"INSERT INTO embedding_fulltext_search(rowid, string_value) VALUES (?, ?)",
(emb_id, doc),
)
conn.execute(
"INSERT INTO embedding_metadata(id, key, string_value) VALUES (?, 'chroma:document', ?)",
(emb_id, doc),
)
conn.execute(
"INSERT INTO embedding_metadata(id, key, string_value) VALUES (?, 'wing', ?)",
(emb_id, wing),
)
conn.commit()
conn.close()
class _NoScanCollection:
name = "mempalace_drawers"
def count(self):
raise AssertionError("lexical_search should use Chroma sqlite FTS")
def get(self, **_kwargs):
raise AssertionError("lexical_search should use Chroma sqlite FTS")
collection = ChromaCollection(_NoScanCollection(), palace_path=str(tmp_path))
hits = collection.lexical_search(query="needle", n_results=1, where={"wing": "target"}).hits
assert [hit.metadata["wing"] for hit in hits] == ["target"]
# Hit ids must be the public embedding_id (so lexical_search -> get(ids=...)
# round-trips), not the internal embeddings.id rowid.
assert [hit.id for hit in hits] == ["public-13"]
def test_chroma_lexical_search_ids_roundtrip_through_get(tmp_path):
"""lexical_search must return public drawer ids that get(ids=...) accepts.
Regression for the sqlite FTS path returning the internal rowid instead of
embeddings.embedding_id, which silently broke hybrid search id round-trips.
"""
backend = ChromaBackend()
palace = tmp_path / "palace"
ref = PalaceRef(id=str(palace), local_path=str(palace))
col = backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True)
col.add(
ids=["drawer-alpha", "drawer-bravo", "drawer-charlie"],
documents=[
"rareterm needle lexical note",
"unrelated content here",
"another rareterm needle entry",
],
metadatas=[{"wing": "w"}, {"wing": "w"}, {"wing": "w"}],
)
hits = col.lexical_search(query="rareterm needle", n_results=5).hits
hit_ids = [hit.id for hit in hits]
assert hit_ids, "expected lexical hits"
assert set(hit_ids) <= {"drawer-alpha", "drawer-bravo", "drawer-charlie"}
# Every returned id must resolve back through get() — proving it is a public id.
fetched = col.get(ids=hit_ids)
assert set(fetched.ids) == set(hit_ids)
backend.close()
def test_query_rejects_missing_input():
fake = _FakeCollection()
collection = ChromaCollection(fake)
with pytest.raises(ValueError):
collection.query()
def test_query_rejects_both_texts_and_embeddings():
fake = _FakeCollection()
collection = ChromaCollection(fake)
with pytest.raises(ValueError):
collection.query(query_texts=["q"], query_embeddings=[[0.1, 0.2]])
def test_query_rejects_empty_input_list():
fake = _FakeCollection()
collection = ChromaCollection(fake)
with pytest.raises(ValueError):
collection.query(query_texts=[])
def test_query_empty_preserves_embeddings_outer_shape_when_requested():
fake = _FakeCollection(
query_response={"ids": [], "documents": [], "metadatas": [], "distances": []}
)
collection = ChromaCollection(fake)
requested = collection.query(query_texts=["q1", "q2"], include=["documents", "embeddings"])
assert requested.embeddings == [[], []]
not_requested = collection.query(query_texts=["q1", "q2"], include=["documents"])
assert not_requested.embeddings is None
def test_chroma_close_palace_releases_sqlite_lock_for_reopen(tmp_path):
"""close_palace must release chromadb's rust-side SQLite file lock so
a fresh PersistentClient on the same path after shutil.rmtree can
write without hitting SQLITE_READONLY_DBMOVED."""
backend = ChromaBackend()
palace_path = tmp_path / "palace-a"
ref = PalaceRef(id=str(palace_path), local_path=str(palace_path))
col = backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True)
col.upsert(documents=["hello"], ids=["a"], metadatas=[{"k": "v"}])
backend.close_palace(ref)
shutil.rmtree(palace_path)
col = backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True)
col.upsert(documents=["world"], ids=["b"], metadatas=[{"k": "v2"}])
assert col.count() == 1
def test_chroma_close_releases_all_cached_clients(tmp_path):
"""close() must release every cached client's SQLite file lock so any
of their palace paths can be reopened by a fresh backend in the same
process."""
backend = ChromaBackend()
palace_a = tmp_path / "palace-a"
palace_b = tmp_path / "palace-b"
ref_a = PalaceRef(id=str(palace_a), local_path=str(palace_a))
ref_b = PalaceRef(id=str(palace_b), local_path=str(palace_b))
for ref in (ref_a, ref_b):
backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True).upsert(
documents=["x"], ids=["x"], metadatas=[{"k": "v"}]
)
backend.close()
for path in (palace_a, palace_b):
shutil.rmtree(path)
ref = PalaceRef(id=str(path), local_path=str(path))
fresh = ChromaBackend()
col = fresh.get_collection(palace=ref, collection_name="mempalace_drawers", create=True)
col.upsert(documents=["y"], ids=["y"], metadatas=[{"k": "v2"}])
assert col.count() == 1
fresh.close()
def test_chroma_cache_invalidates_when_db_file_missing(tmp_path):
"""A palace rebuild that removes chroma.sqlite3 must drop the stale cache.
Primes backend._clients/_freshness directly with a sentinel rather than
opening a real ``PersistentClient``: on Windows the sqlite file handle
would still be live and ``Path.unlink`` would raise ``PermissionError``,
making the test unable to exercise the branch we care about. The decision
logic under test is pure (no chromadb calls before the branch), so a
sentinel is sufficient.
"""
backend = ChromaBackend()
palace_path = tmp_path / "palace"
palace_path.mkdir()
db_file = palace_path / "chroma.sqlite3"
db_file.write_bytes(b"") # any file is enough for _db_stat to see it
st = db_file.stat()
sentinel = object()
backend._clients[str(palace_path)] = sentinel
backend._freshness[str(palace_path)] = (st.st_ino, st.st_mtime)
# Simulate a rebuild mid-flight: chroma.sqlite3 goes away. Safe to unlink
# because nothing in this test is holding an OS handle on the file.
db_file.unlink()
prior_freshness = (st.st_ino, st.st_mtime)
new_client = backend._client(str(palace_path))
# Cache was replaced (not the sentinel) and freshness reflects the post-
# rebuild stat (chromadb re-creates chroma.sqlite3 during PersistentClient
# construction; _client re-stats after the constructor so freshness is
# not frozen at the pre-rebuild value). The stale cached sentinel would
# have served wrong data if returned.
assert new_client is not sentinel
assert backend._freshness[str(palace_path)] != prior_freshness
def test_chroma_cache_picks_up_db_created_after_first_open(tmp_path):
"""The 0 → nonzero stat transition invalidates a cache built before the DB existed."""
backend = ChromaBackend()
palace_path = tmp_path / "palace"
palace_path.mkdir()
# Seed an entry in the caches as if a prior _client() call had opened the
# palace when chroma.sqlite3 did not exist yet. Freshness (0, 0.0) is the
# signal that the DB was absent at cache time.
sentinel = object()
backend._clients[str(palace_path)] = sentinel
backend._freshness[str(palace_path)] = (0, 0.0)
# The DB file now appears (real chromadb would have created it by now).
# Use a real chromadb call so _fix_blob_seq_ids and PersistentClient succeed.
import chromadb as _chromadb
_chromadb.PersistentClient(path=str(palace_path)).get_or_create_collection("seed")
assert (palace_path / "chroma.sqlite3").is_file()
# Next _client() call must detect the 0 → nonzero transition and rebuild.
refreshed = backend._client(str(palace_path))
assert refreshed is not sentinel
assert backend._freshness[str(palace_path)] != (0, 0.0)
def test_base_collection_update_default_rejects_mismatched_lengths():
"""The ABC default update() raises ValueError rather than silently misaligning."""
from mempalace.backends.base import BaseCollection
collection = ChromaCollection(_FakeCollection())
with pytest.raises(ValueError, match="documents length"):
BaseCollection.update(collection, ids=["1", "2"], documents=["only-one"])
with pytest.raises(ValueError, match="metadatas length"):
BaseCollection.update(collection, ids=["1", "2"], metadatas=[{"k": 9}])
def test_chroma_backend_accepts_palace_ref_kwarg(tmp_path):
palace_path = tmp_path / "palace"
backend = ChromaBackend()
collection = backend.get_collection(
palace=PalaceRef(id=str(palace_path), local_path=str(palace_path)),
collection_name="mempalace_drawers",
create=True,
)
assert palace_path.is_dir()
assert isinstance(collection, ChromaCollection)
def test_chroma_backend_create_false_raises_without_creating_directory(tmp_path):
palace_path = tmp_path / "missing-palace"
with pytest.raises(FileNotFoundError):
ChromaBackend().get_collection(
str(palace_path),
collection_name="mempalace_drawers",
create=False,
)
assert not palace_path.exists()
def test_chroma_backend_create_true_creates_directory_and_collection(tmp_path):
palace_path = tmp_path / "palace"
collection = ChromaBackend().get_collection(
str(palace_path),
collection_name="mempalace_drawers",
create=True,
)
assert palace_path.is_dir()
assert isinstance(collection, ChromaCollection)
client = chromadb.PersistentClient(path=str(palace_path))
client.get_collection("mempalace_drawers")
def test_chroma_backend_creates_collection_with_cosine_distance(tmp_path):
palace_path = tmp_path / "palace"
ChromaBackend().get_collection(
str(palace_path),
collection_name="mempalace_drawers",
create=True,
)
client = chromadb.PersistentClient(path=str(palace_path))
col = client.get_collection("mempalace_drawers")
assert col.metadata.get("hnsw:space") == "cosine"
def test_chroma_backend_sets_hnsw_bloat_guard_on_creation(tmp_path):
"""HNSW batch/sync thresholds must land on freshly-created collection metadata.
Low thresholds (2/2 per #1579) make chromadb's Rust HNSW segment
persist index_metadata and link_lists after any mine of 2+ drawers.
Asserting both keys land on the persisted metadata also covers the
#1161 "config silently dropped" concern at CI time.
"""
palace_path = tmp_path / "palace"
ChromaBackend().get_collection(
str(palace_path),
collection_name="mempalace_drawers",
create=True,
)
client = chromadb.PersistentClient(path=str(palace_path))
col = client.get_collection("mempalace_drawers")
assert col.metadata.get("hnsw:batch_size") == 2
assert col.metadata.get("hnsw:sync_threshold") == 2
def test_chroma_backend_create_collection_sets_hnsw_bloat_guard(tmp_path):
"""Same guard must apply via the legacy create_collection() path."""
palace_path = tmp_path / "palace"
ChromaBackend().create_collection(str(palace_path), "mempalace_drawers")
client = chromadb.PersistentClient(path=str(palace_path))
col = client.get_collection("mempalace_drawers")
assert col.metadata.get("hnsw:batch_size") == 2
assert col.metadata.get("hnsw:sync_threshold") == 2
def test_sub_threshold_mine_persists_hnsw_metadata(tmp_path):
"""Regression for #1579: small mines must persist HNSW metadata.
_HNSW_BLOAT_GUARD sets batch_size=2 and sync_threshold=2 so that any
upsert of 2+ records crosses both thresholds, triggering chromadb's
_apply_batch and _persist. Without this, index_metadata and link_lists
stay empty and quarantine_stale_hnsw renames the segment on cold open.
"""
palace_path = str(tmp_path / "palace")
backend = ChromaBackend()
try:
col = backend.get_collection(palace_path, "mempalace_drawers", create=True)
col.upsert(
ids=["a", "b", "c"],
documents=["doc a", "doc b", "doc c"],
embeddings=[[0.1] * _TEST_EMBED_DIM, [0.2] * _TEST_EMBED_DIM, [0.3] * _TEST_EMBED_DIM],
metadatas=[{"wing": "t"}, {"wing": "t"}, {"wing": "t"}],
)
finally:
backend.close()
found_healthy_segment = False
for entry in (tmp_path / "palace").iterdir():
if not entry.is_dir() or entry.name.startswith("."):
continue
meta = entry / "index_metadata.pickle"
link = entry / "link_lists.bin"
data = entry / "data_level0.bin"
if data.exists() and data.stat().st_size > _HNSW_MISSING_METADATA_DATA_FLOOR:
assert meta.exists(), "index_metadata missing after sub-threshold upsert"
assert link.exists() and link.stat().st_size > 0, "link_lists empty"
assert _segment_appears_healthy(str(entry))
found_healthy_segment = True
assert found_healthy_segment, "no VECTOR segment with data found"
# stale_seconds=0.0 forces the stage-2 integrity gate (_segment_appears_healthy)
# to run on every segment regardless of mtime delta, proving the fix directly.
moved = quarantine_stale_hnsw(palace_path, stale_seconds=0.0)
assert moved == [], f"quarantine fired on freshly-persisted segment: {moved}"
def test_single_record_upsert_not_quarantined(tmp_path):
"""A single-record upsert must not trigger quarantine.
With batch_size=2 chromadb only persists HNSW metadata after the second
record. A one-record segment has no index_metadata.pickle and no
link_lists.bin data; _segment_appears_healthy must treat that combination
as sub-threshold (never persisted), not as corruption.
"""
palace_path = str(tmp_path / "palace")
backend = ChromaBackend()
try:
col = backend.get_collection(palace_path, "mempalace_drawers", create=True)
col.upsert(
ids=["solo"],
documents=["only one drawer"],
embeddings=[[0.5] * _TEST_EMBED_DIM],
metadatas=[{"wing": "t"}],
)
finally:
backend.close()
for entry in (tmp_path / "palace").iterdir():
if not entry.is_dir() or entry.name.startswith("."):
continue
data = entry / "data_level0.bin"
if data.exists() and data.stat().st_size > 0:
assert _segment_appears_healthy(str(entry)), (
f"single-record segment flagged unhealthy: data={data.stat().st_size}B"
)
moved = quarantine_stale_hnsw(palace_path, stale_seconds=0.0)
assert moved == [], f"quarantine fired on single-record segment: {moved}"
def test_get_collection_create_true_is_idempotent(tmp_path):
"""Calling get_collection(create=True) twice on the same name must not crash.
ChromaDB 1.5.x's Rust bindings SIGSEGV when get_or_create_collection is
called with metadata that differs from the stored collection metadata. The
fix splits the call into get_collection -> fallback create_collection so the
metadata-comparison codepath in chromadb_rust_bindings is never reached for
existing collections. Regression guard for issue #1089.
"""
palace = str(tmp_path / "palace")
backend = ChromaBackend()
backend.get_collection(palace, collection_name="mempalace_drawers", create=True)
col2 = backend.get_collection(palace, collection_name="mempalace_drawers", create=True)
assert isinstance(col2, ChromaCollection)
def test_get_collection_create_true_preserves_existing_metadata(tmp_path):
"""Existing collection metadata is not overwritten when reopened with create=True."""
palace = str(tmp_path / "palace")
backend = ChromaBackend()
backend.get_collection(palace, collection_name="mempalace_drawers", create=True)
col = backend.get_collection(palace, collection_name="mempalace_drawers", create=True)
assert col._collection.metadata["hnsw:space"] == "cosine"
assert col._collection.metadata.get("hnsw:batch_size") == 2
def test_fix_blob_seq_ids_converts_blobs_to_integers(tmp_path):
"""Simulate a ChromaDB 0.6.x database with BLOB seq_ids and verify repair."""
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE embeddings (rowid INTEGER PRIMARY KEY, seq_id)")
# Insert BLOB seq_id like ChromaDB 0.6.x would
blob_42 = (42).to_bytes(8, byteorder="big")
conn.execute("INSERT INTO embeddings (seq_id) VALUES (?)", (blob_42,))
conn.commit()
_fix_blob_seq_ids(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
row = conn.execute("SELECT seq_id, typeof(seq_id) FROM embeddings").fetchone()
assert row == (42, "integer")
def test_fix_blob_seq_ids_noop_without_blobs(tmp_path):
"""No error when seq_ids are already integers."""
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE embeddings (rowid INTEGER PRIMARY KEY, seq_id INTEGER)")
conn.execute("INSERT INTO embeddings (seq_id) VALUES (42)")
conn.commit()
_fix_blob_seq_ids(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
row = conn.execute("SELECT seq_id, typeof(seq_id) FROM embeddings").fetchone()
assert row == (42, "integer")
def test_fix_blob_seq_ids_noop_without_database(tmp_path):
"""No error when palace has no chroma.sqlite3."""
_fix_blob_seq_ids(str(tmp_path)) # should not raise
def test_fix_blob_seq_ids_does_not_touch_max_seq_id(tmp_path):
"""chromadb 1.5.x owns max_seq_id; the shim must not interpret its BLOBs.
Regression guard for the 2026-04-20 incident: the old shim ran
int.from_bytes(..., 'big') over chromadb 1.5.x's native
b'\\x11\\x11' + ASCII-digit BLOB, producing a ~1.23e18 integer that
silently suppressed every subsequent embeddings_queue write.
"""
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE embeddings (rowid INTEGER PRIMARY KEY, seq_id)")
conn.execute("CREATE TABLE max_seq_id (rowid INTEGER PRIMARY KEY, seq_id)")
sysdb10_blob = b"\x11\x11502607"
conn.execute("INSERT INTO max_seq_id (seq_id) VALUES (?)", (sysdb10_blob,))
conn.commit()
_fix_blob_seq_ids(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
row = conn.execute("SELECT seq_id, typeof(seq_id) FROM max_seq_id").fetchone()
assert row == (sysdb10_blob, "blob")
def test_fix_blob_seq_ids_skips_sysdb10_prefix_in_embeddings(tmp_path):
"""Defense-in-depth: sysdb-10 prefix in embeddings.seq_id is skipped."""
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE embeddings (rowid INTEGER PRIMARY KEY, seq_id)")
sysdb10_blob = b"\x11\x11502607"
conn.execute("INSERT INTO embeddings (seq_id) VALUES (?)", (sysdb10_blob,))
conn.commit()
_fix_blob_seq_ids(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
row = conn.execute("SELECT seq_id, typeof(seq_id) FROM embeddings").fetchone()
# Still a BLOB — not converted to 1.23e18.
assert row == (sysdb10_blob, "blob")
def test_fix_blob_seq_ids_still_converts_legacy_blobs_in_embeddings(tmp_path):
"""Regression guard: pure big-endian u64 BLOBs still convert for genuine 0.6.x."""
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE embeddings (rowid INTEGER PRIMARY KEY, seq_id)")
conn.execute("INSERT INTO embeddings (seq_id) VALUES (?)", ((42).to_bytes(8, "big"),))
conn.execute("INSERT INTO embeddings (seq_id) VALUES (?)", (b"\x11\x11502607",))
conn.execute("INSERT INTO embeddings (seq_id) VALUES (?)", ((7).to_bytes(8, "big"),))
conn.commit()
_fix_blob_seq_ids(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
rows = conn.execute(
"SELECT seq_id, typeof(seq_id) FROM embeddings ORDER BY rowid"
).fetchall()
assert rows[0] == (42, "integer")
assert rows[1] == (b"\x11\x11502607", "blob") # sysdb-10 row left alone
assert rows[2] == (7, "integer")
def test_fix_blob_seq_ids_writes_marker_after_blob_path(tmp_path):
"""The .blob_seq_ids_migrated marker is written after a successful BLOB → INTEGER conversion."""
from mempalace.backends.chroma import _BLOB_FIX_MARKER
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE embeddings (rowid INTEGER PRIMARY KEY, seq_id)")
conn.execute("INSERT INTO embeddings (seq_id) VALUES (?)", ((42).to_bytes(8, "big"),))
conn.commit()
marker = tmp_path / _BLOB_FIX_MARKER
assert not marker.exists()
_fix_blob_seq_ids(str(tmp_path))
assert marker.is_file(), "marker must be written after a successful migration"
def test_fix_blob_seq_ids_writes_marker_when_already_integer(tmp_path):
"""The marker is written even when the migration is a no-op (already INTEGER).
The point of the marker is to skip the sqlite3 open on subsequent calls,
not to record that a conversion happened. So a clean palace gets the
marker on first run too — next ``_fix_blob_seq_ids`` call short-circuits
before touching the sqlite3 file.
"""
from mempalace.backends.chroma import _BLOB_FIX_MARKER
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE embeddings (rowid INTEGER PRIMARY KEY, seq_id INTEGER)")
conn.execute("INSERT INTO embeddings (seq_id) VALUES (42)")
conn.commit()
marker = tmp_path / _BLOB_FIX_MARKER
assert not marker.exists()
_fix_blob_seq_ids(str(tmp_path))
assert marker.is_file(), "marker must be written even when no BLOBs found"
def test_fix_blob_seq_ids_closes_sqlite_connection(tmp_path, monkeypatch):
"""The migration closes sqlite connections after the pre-open probe."""
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE embeddings (rowid INTEGER PRIMARY KEY, seq_id INTEGER)")
conn.execute("INSERT INTO embeddings (seq_id) VALUES (42)")
conn.commit()
closed = []
real_connect = sqlite3.connect
class TrackingConnection(sqlite3.Connection):
def close(self):
closed.append(True)
super().close()
def tracking_connect(*args, **kwargs):
kwargs["factory"] = TrackingConnection
return real_connect(*args, **kwargs)
monkeypatch.setattr("mempalace.backends.chroma.sqlite3.connect", tracking_connect)
_fix_blob_seq_ids(str(tmp_path))
assert closed == [True]
def test_fix_blob_seq_ids_skips_sqlite_when_marker_present(tmp_path):
"""When the marker exists, ``_fix_blob_seq_ids`` does not open sqlite3.
This is the load-bearing property of the marker — opening Python's
sqlite3 against a live ChromaDB 1.5.x WAL DB corrupts the next
PersistentClient call (#1090). Once a palace has been migrated, we
never want to open it again, even read-only.
"""
from unittest.mock import patch
from mempalace.backends.chroma import _BLOB_FIX_MARKER
# Pre-create the marker so the function should short-circuit.
db_path = tmp_path / "chroma.sqlite3"
db_path.write_bytes(b"sentinel") # presence required for the function to proceed
(tmp_path / _BLOB_FIX_MARKER).touch()
with patch("mempalace.backends.chroma.sqlite3.connect") as mock_connect:
_fix_blob_seq_ids(str(tmp_path))
mock_connect.assert_not_called()
# ── _fix_missing_collection_type ─────────────────────────────────────────
def test_fix_collection_type_adds_type(tmp_path):
"""Legacy config_json_str '{}' gets _type added."""
import json
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
conn.execute(
"INSERT INTO collections (id, config_json_str) VALUES (?, ?)",
("col-1", "{}"),
)
conn.commit()
_fix_missing_collection_type(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
row = conn.execute("SELECT config_json_str FROM collections WHERE id = 'col-1'").fetchone()
config = json.loads(row[0])
assert config["_type"] == "CollectionConfigurationInternal"
def test_fix_collection_type_preserves_existing(tmp_path):
"""Config that already has _type is left unchanged."""
import json
original = json.dumps({"_type": "CollectionConfigurationInternal", "extra": 1})
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
conn.execute(
"INSERT INTO collections (id, config_json_str) VALUES (?, ?)",
("col-1", original),
)
conn.commit()
_fix_missing_collection_type(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
row = conn.execute("SELECT config_json_str FROM collections WHERE id = 'col-1'").fetchone()
assert row[0] == original
def test_fix_collection_type_noop_without_db(tmp_path):
"""No error when palace has no chroma.sqlite3, no marker written."""
from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER
_fix_missing_collection_type(str(tmp_path))
assert not (tmp_path / _COLLECTION_TYPE_MARKER).exists()
def test_fix_collection_type_writes_marker(tmp_path):
"""Marker is written after a successful migration."""
from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
conn.execute(
"INSERT INTO collections (id, config_json_str) VALUES (?, ?)",
("col-1", "{}"),
)
conn.commit()
marker = tmp_path / _COLLECTION_TYPE_MARKER
assert not marker.exists()
_fix_missing_collection_type(str(tmp_path))
assert marker.is_file()
def test_fix_collection_type_skips_with_marker(tmp_path):
"""When the marker exists, sqlite3 is not opened."""
from unittest.mock import patch
from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER
db_path = tmp_path / "chroma.sqlite3"
db_path.write_bytes(b"sentinel")
(tmp_path / _COLLECTION_TYPE_MARKER).touch()
with patch("mempalace.backends.chroma.sqlite3.connect") as mock_connect:
_fix_missing_collection_type(str(tmp_path))
mock_connect.assert_not_called()
def test_fix_collection_type_writes_marker_when_already_has_type(tmp_path):
"""Marker written even when all collections already have _type (noop case)."""
import json
from mempalace.backends.chroma import _COLLECTION_TYPE_MARKER
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
conn.execute(
"INSERT INTO collections (id, config_json_str) VALUES (?, ?)",
("col-1", json.dumps({"_type": "CollectionConfigurationInternal"})),
)
conn.commit()
marker = tmp_path / _COLLECTION_TYPE_MARKER
assert not marker.exists()
_fix_missing_collection_type(str(tmp_path))
assert marker.is_file(), "marker must be written even when no collections needed fixing"
def test_fix_collection_type_multi_collection_mixed(tmp_path):
"""Multiple collections: NULL, empty, and already-valid configs."""
import json
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-null", None))
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-empty", "{}"))
conn.execute(
"INSERT INTO collections VALUES (?, ?)",
("col-ok", json.dumps({"_type": "CollectionConfigurationInternal"})),
)
conn.commit()
_fix_missing_collection_type(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
rows = {
r[0]: json.loads(r[1]) if r[1] else None
for r in conn.execute("SELECT id, config_json_str FROM collections")
}
assert rows["col-null"]["_type"] == "CollectionConfigurationInternal"
assert rows["col-empty"]["_type"] == "CollectionConfigurationInternal"
assert rows["col-ok"] == {"_type": "CollectionConfigurationInternal"}
def test_fix_collection_type_skips_non_dict_json(tmp_path):
"""Non-dict JSON (array, null literal) is skipped without error."""
import json
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-arr", "[]"))
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-null", "null"))
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-ok", "{}"))
conn.commit()
_fix_missing_collection_type(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
rows = dict(conn.execute("SELECT id, config_json_str FROM collections").fetchall())
assert rows["col-arr"] == "[]"
assert rows["col-null"] == "null"
assert json.loads(rows["col-ok"])["_type"] == "CollectionConfigurationInternal"
def test_fix_collection_type_skips_malformed_json(tmp_path):
"""Malformed JSON in one row does not prevent fixing other rows."""
import json
db_path = tmp_path / "chroma.sqlite3"
with closing(sqlite3.connect(str(db_path))) as conn:
conn.execute("CREATE TABLE collections (id TEXT PRIMARY KEY, config_json_str TEXT)")
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-bad", "{corrupt"))
conn.execute("INSERT INTO collections VALUES (?, ?)", ("col-ok", "{}"))
conn.commit()
_fix_missing_collection_type(str(tmp_path))
with closing(sqlite3.connect(str(db_path))) as conn:
rows = dict(conn.execute("SELECT id, config_json_str FROM collections").fetchall())
assert rows["col-bad"] == "{corrupt"
assert json.loads(rows["col-ok"])["_type"] == "CollectionConfigurationInternal"
# ── quarantine_stale_hnsw ─────────────────────────────────────────────────
# Marker bytes for the chromadb segment metadata file. A complete
# write begins with PROTO opcode (0x80) and ends with STOP opcode
# (0x2e); _segment_appears_healthy sniffs these bytes without parsing
# the file.
_HEALTHY_META = b"\x80\x04" + b"\x00" * 32 + b"\x2e"
_CORRUPT_META = b"\x00" * 64
def _make_palace_with_segment(tmp_path, hnsw_mtime, sqlite_mtime, meta_bytes=_HEALTHY_META):
"""Helper: build a palace dir with one HNSW segment + sqlite at given
mtimes. ``meta_bytes`` controls whether the segment looks healthy
(default), corrupt (``_CORRUPT_META``), or has no metadata file at
all (``None``)."""
palace = tmp_path / "palace"
palace.mkdir()
(palace / "chroma.sqlite3").write_text("")
seg = palace / "abcd-1234-5678"
seg.mkdir()
(seg / "data_level0.bin").write_text("")
if meta_bytes is not None:
(seg / "index_metadata.pickle").write_bytes(meta_bytes)
os.utime(seg / "data_level0.bin", (hnsw_mtime, hnsw_mtime))
os.utime(palace / "chroma.sqlite3", (sqlite_mtime, sqlite_mtime))
return palace, seg
def test_quarantine_stale_hnsw_renames_corrupt_segment(tmp_path):
"""Segment with stale mtime AND a malformed metadata file gets renamed."""
now = 1_700_000_000.0
palace, seg = _make_palace_with_segment(
tmp_path,
hnsw_mtime=now - 7200,
sqlite_mtime=now,
meta_bytes=_CORRUPT_META,
)
moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0)
assert len(moved) == 1
assert ".drift-" in moved[0]
assert not seg.exists()
renamed = list(palace.iterdir())
drift_dirs = [p for p in renamed if ".drift-" in p.name]
assert len(drift_dirs) == 1
assert (drift_dirs[0] / "data_level0.bin").exists()
def test_quarantine_stale_hnsw_leaves_healthy_segment_with_drift_alone(tmp_path):
"""Segment with stale mtime but a complete metadata file is NOT
renamed — this is the chromadb-1.5.x async-flush steady state, not
corruption. Production case at 06:24 PDT 2026-04-26: cold-start
quarantine renamed three healthy segments after a clean shutdown,
leaving 151K-drawer palace with vector_ranked=0."""
now = 1_700_000_000.0
palace, seg = _make_palace_with_segment(
tmp_path,
hnsw_mtime=now - 7200,
sqlite_mtime=now,
meta_bytes=_HEALTHY_META,
)
moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0)
assert moved == []
assert seg.exists()
def test_quarantine_stale_hnsw_leaves_empty_segment_without_metadata_alone(tmp_path):
"""Missing metadata is okay only when the segment has no meaningful data yet."""
now = 1_700_000_000.0
palace, seg = _make_palace_with_segment(
tmp_path,
hnsw_mtime=now - 7200,
sqlite_mtime=now,
meta_bytes=None,
)
moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0)
assert moved == []
assert seg.exists()
def test_segment_without_metadata_but_with_nontrivial_data_is_unhealthy(tmp_path):
"""Interrupted persist: link_lists written but metadata absent is unhealthy."""
seg = tmp_path / "abcd-1234-5678"
seg.mkdir()
(seg / "data_level0.bin").write_bytes(b"\0" * (_HNSW_MISSING_METADATA_DATA_FLOOR + 1))
(seg / "link_lists.bin").write_bytes(b"\x01" * 128)
assert not _segment_appears_healthy(str(seg))
def test_segment_without_metadata_and_tiny_data_is_still_treated_as_fresh(tmp_path):
"""No metadata and no link_lists means no persist was attempted; treat as fresh."""
seg = tmp_path / "abcd-1234-5678"
seg.mkdir()
(seg / "data_level0.bin").write_bytes(b"\0" * _HNSW_MISSING_METADATA_DATA_FLOOR)
assert _segment_appears_healthy(str(seg))
def test_quarantine_stale_hnsw_renames_missing_metadata_with_nontrivial_data(tmp_path):
"""Regression for #1274: missing pickle + link data must quarantine."""
now = 1_700_000_000.0
palace, seg = _make_palace_with_segment(
tmp_path,
hnsw_mtime=now - 7200,
sqlite_mtime=now,
meta_bytes=None,
)
(seg / "data_level0.bin").write_bytes(b"\0" * (_HNSW_MISSING_METADATA_DATA_FLOOR + 1))
(seg / "link_lists.bin").write_bytes(b"\x01" * 128)
os.utime(seg / "data_level0.bin", (now - 7200, now - 7200))
moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0)
assert len(moved) == 1
assert ".drift-" in moved[0]
assert not seg.exists()
drift_dirs = [p for p in palace.iterdir() if ".drift-" in p.name]
assert len(drift_dirs) == 1
assert (drift_dirs[0] / "data_level0.bin").exists()
def test_quarantine_stale_hnsw_renames_truncated_metadata(tmp_path):
"""Segment with a truncated (under-floor-size) metadata file is
quarantined — shape of a partial-flush during process kill."""
now = 1_700_000_000.0
palace, seg = _make_palace_with_segment(
tmp_path,
hnsw_mtime=now - 7200,
sqlite_mtime=now,
meta_bytes=b"\x80\x04",
)
moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0)
assert len(moved) == 1
assert ".drift-" in moved[0]
def test_quarantine_stale_hnsw_leaves_fresh_segment_alone(tmp_path):
"""Segment with recent mtime vs sqlite is not touched (mtime gate
short-circuits before integrity gate)."""
now = 1_700_000_000.0
palace, seg = _make_palace_with_segment(tmp_path, hnsw_mtime=now - 10, sqlite_mtime=now)
moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0)
assert moved == []
assert seg.exists()
def test_quarantine_stale_hnsw_no_palace(tmp_path):
"""Missing palace path or chroma.sqlite3: return [] without raising."""
assert quarantine_stale_hnsw(str(tmp_path / "missing")) == []
empty = tmp_path / "empty"
empty.mkdir()
assert quarantine_stale_hnsw(str(empty)) == []
def test_quarantine_stale_hnsw_skips_already_quarantined(tmp_path):
"""Directories already named with ``.drift-`` suffix are never re-renamed."""
now = 1_700_000_000.0
palace = tmp_path / "palace"
palace.mkdir()
(palace / "chroma.sqlite3").write_text("")
os.utime(palace / "chroma.sqlite3", (now, now))
drift = palace / "abcd-1234.drift-20260101-000000"
drift.mkdir()
(drift / "data_level0.bin").write_text("")
os.utime(drift / "data_level0.bin", (now - 99999, now - 99999))
moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0)
assert moved == []
assert drift.exists()
# ── make_client cold-start gate ──────────────────────────────────────────
def test_make_client_quarantines_only_on_first_call_per_palace(tmp_path, monkeypatch):
"""Quarantine fires on first ``make_client()`` for a palace, then is
skipped on subsequent calls — prevents runtime thrash where a daemon's
own steady writes bump ``chroma.sqlite3`` faster than HNSW flushes,
making the mtime heuristic falsely trigger every reconnect.
Invalid metadata quarantine shares the same cold-start gate here; the
more aggressive refresh path lives in ``_client()``."""
from mempalace.backends.chroma import ChromaBackend
palace_path = str(tmp_path / "palace")
os.makedirs(palace_path, exist_ok=True)
(Path(palace_path) / "chroma.sqlite3").write_text("")
# Reset the per-process cache so this test is independent of others.
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
calls: list[str] = []
def _spy(path, stale_seconds=300.0):
calls.append(path)
return []
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _spy)
ChromaBackend.make_client(palace_path)
ChromaBackend.make_client(palace_path)
ChromaBackend.make_client(palace_path)
assert calls == [palace_path], (
"quarantine_stale_hnsw should fire once per palace per process, not on every reconnect"
)
def test_make_client_gates_invalid_metadata_on_first_call(tmp_path, monkeypatch):
"""Invalid metadata quarantine is gated on the first make_client() call."""
from mempalace.backends.chroma import ChromaBackend
palace_path = str(tmp_path / "palace")
os.makedirs(palace_path, exist_ok=True)
(Path(palace_path) / "chroma.sqlite3").write_text("")
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
calls: list[str] = []
def _invalid(path, *args, **kwargs):
calls.append(path)
return []
def _stale(path, stale_seconds=300.0):
return []
monkeypatch.setattr("mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _invalid)
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _stale)
ChromaBackend.make_client(palace_path)
ChromaBackend.make_client(palace_path)
assert calls == [palace_path]
def test_make_client_quarantines_each_palace_independently(tmp_path, monkeypatch):
"""Two distinct palaces each get one quarantine attempt — the gate is
keyed by palace path, not global."""
from mempalace.backends.chroma import ChromaBackend
palace_a = str(tmp_path / "palace_a")
palace_b = str(tmp_path / "palace_b")
for p in (palace_a, palace_b):
os.makedirs(p, exist_ok=True)
(Path(p) / "chroma.sqlite3").write_text("")
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
calls: list[str] = []
def _spy(path, stale_seconds=300.0):
calls.append(path)
return []
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _spy)
ChromaBackend.make_client(palace_a)
ChromaBackend.make_client(palace_b)
ChromaBackend.make_client(palace_a) # already gated
ChromaBackend.make_client(palace_b) # already gated
assert calls == [palace_a, palace_b]
# ── _client() cold-start gate (#1121, #1132, #1263) ──────────────────────
def test_client_quarantines_corrupt_segment_on_first_open(tmp_path, monkeypatch):
"""The instance ``_client()`` path must run ``quarantine_stale_hnsw``
on first open, mirroring the ``make_client()`` static helper. Before
PR #1173's wiring was extended here, CLI mining / search / repair /
status all skipped the quarantine pass and would SIGSEGV on a stale
HNSW segment (#1121, #1132, #1263)."""
now = 1_700_000_000.0
palace, seg = _make_palace_with_segment(
tmp_path,
hnsw_mtime=now - 7200,
sqlite_mtime=now,
meta_bytes=_CORRUPT_META,
)
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
backend = ChromaBackend()
try:
backend._client(str(palace))
finally:
backend.close()
assert not seg.exists(), "_client() should have quarantined the corrupt segment"
drift_dirs = [p for p in palace.iterdir() if ".drift-" in p.name]
assert len(drift_dirs) == 1
def test_client_quarantines_only_on_first_call_per_palace(tmp_path, monkeypatch):
"""Repeated ``_client()`` calls for the same palace re-run quarantine
at most once — the ``_quarantined_paths`` gate prevents runtime
thrash on hot paths (``_client()`` is hit on every backend op)."""
palace_path = str(tmp_path / "palace")
os.makedirs(palace_path, exist_ok=True)
(Path(palace_path) / "chroma.sqlite3").write_text("")
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
calls: list[str] = []
def _spy(path, stale_seconds=300.0):
calls.append(path)
return []
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _spy)
backend = ChromaBackend()
try:
backend._client(palace_path)
backend._client(palace_path)
backend._client(palace_path)
finally:
backend.close()
assert calls == [palace_path], (
"quarantine_stale_hnsw should fire once per palace per process from _client(), not on every call"
)
def test_client_rearms_quarantine_on_mtime_change(tmp_path, monkeypatch):
"""When the DB file's mtime changes between ``_client()`` calls (external
in-place write), the quarantine gate re-arms so HNSW checks run again.
Before #1573, the gate was only cleared on *inode* change (full palace
replacement); mtime-only changes left the gate armed, so long-running
processes were blind to external drift."""
palace_path = str(tmp_path / "palace")
os.makedirs(palace_path, exist_ok=True)
db_file = Path(palace_path) / "chroma.sqlite3"
db_file.write_text("")
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
calls: list[str] = []
def _spy(path, stale_seconds=300.0):
calls.append(path)
return []
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _spy)
backend = ChromaBackend()
try:
backend._client(palace_path)
assert len(calls) == 1, "quarantine should fire on first open"
_, cached_mtime = backend._freshness[palace_path]
os.utime(str(db_file), (cached_mtime + 1.0, cached_mtime + 1.0))
backend._client(palace_path)
assert len(calls) == 2, "quarantine should re-fire after mtime change (gate re-armed)"
finally:
backend.close()
# ── _pin_hnsw_threads (per-process retrofit, separate from this PR's gate) ──
def test_pin_hnsw_threads_retrofits_legacy_collection(tmp_path):
"""Legacy collections (created without num_threads) get the retrofit applied."""
palace_path = tmp_path / "legacy-palace"
palace_path.mkdir()
client = chromadb.PersistentClient(path=str(palace_path))
col = client.create_collection(
"mempalace_drawers",
metadata={"hnsw:space": "cosine"}, # no num_threads — legacy
)
assert col.configuration_json.get("hnsw", {}).get("num_threads") is None
_pin_hnsw_threads(col)
assert col.configuration_json["hnsw"]["num_threads"] == 1
def test_pin_hnsw_threads_swallows_all_errors():
"""Retrofit never raises even when collection.modify explodes."""
class _ExplodingCollection:
def modify(self, *args, **kwargs):
raise RuntimeError("boom")
_pin_hnsw_threads(_ExplodingCollection()) # must not raise
def test_get_collection_applies_retrofit_on_existing_palace(tmp_path):
"""ChromaBackend.get_collection(create=False) applies the retrofit."""
palace_path = tmp_path / "palace"
palace_path.mkdir()
# Simulate a legacy palace: create collection without num_threads
bootstrap_client = chromadb.PersistentClient(path=str(palace_path))
bootstrap_client.create_collection("mempalace_drawers", metadata={"hnsw:space": "cosine"})
del bootstrap_client # drop reference so a fresh client reopens cleanly
wrapper = ChromaBackend().get_collection(
str(palace_path),
collection_name="mempalace_drawers",
create=False,
)
assert wrapper._collection.configuration_json["hnsw"]["num_threads"] == 1
def test_get_collection_raises_palace_not_found_when_dir_missing(tmp_path):
"""create=False on a missing dir raises PalaceNotFoundError, not the
new CollectionNotInitializedError. The two states must be distinguishable
so callers can render state-specific messages (#1498)."""
missing = tmp_path / "no-such-dir"
with pytest.raises(PalaceNotFoundError) as excinfo:
ChromaBackend().get_collection(
str(missing),
collection_name="mempalace_drawers",
create=False,
)
# Must be the parent class, not the new subclass: dir is genuinely absent.
assert not isinstance(excinfo.value, CollectionNotInitializedError)
def test_get_collection_raises_collection_not_initialized_on_empty_palace(tmp_path):
"""When the palace dir + DB exist but the collection has never been
created, ChromaBackend.get_collection(create=False) raises the new
CollectionNotInitializedError instead of leaking chromadb.NotFoundError
(#1498)."""
palace_path = tmp_path / "palace"
palace_path.mkdir()
# PersistentClient lazily creates chroma.sqlite3 — no collection yet.
chromadb.PersistentClient(path=str(palace_path))
assert (palace_path / "chroma.sqlite3").is_file()
with pytest.raises(CollectionNotInitializedError) as excinfo:
ChromaBackend().get_collection(
str(palace_path),
collection_name="mempalace_drawers",
create=False,
)
# Backward-compat: subclass of PalaceNotFoundError (and FileNotFoundError).
assert isinstance(excinfo.value, PalaceNotFoundError)
assert isinstance(excinfo.value, FileNotFoundError)
def test_quarantine_invalid_hnsw_metadata_renames_missing_dimensionality(tmp_path):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
with open(seg / "index_metadata.pickle", "wb") as f:
pickle.dump({"dimensionality": None, "id_to_label": {"a": 1}}, f)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert len(moved) == 1
assert ".corrupt-" in moved[0]
assert not seg.exists()
def test_quarantine_invalid_hnsw_metadata_keeps_consistent_missing_dimensionality(tmp_path):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
(seg / "data_level0.bin").write_bytes(b"x" * 2048)
(seg / "link_lists.bin").write_bytes(b"x" * 128)
with open(seg / "index_metadata.pickle", "wb") as f:
pickle.dump(
{
"dimensionality": None,
"total_elements_added": 2,
"max_seq_id": None,
"id_to_label": {"a": 1, "b": 2},
"label_to_id": {1: "a", 2: "b"},
"id_to_seq_id": {},
},
f,
)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert moved == []
assert seg.exists()
def test_quarantine_invalid_hnsw_metadata_keeps_post_deletion_missing_dimensionality(tmp_path):
"""A deleted-from segment has total_elements_added > live label count (the
counter is monotonic); that dim-None shape is recoverable, not corruption (#1710).
"""
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
(seg / "data_level0.bin").write_bytes(b"x" * 2048)
(seg / "link_lists.bin").write_bytes(b"x" * 128)
with open(seg / "index_metadata.pickle", "wb") as f:
pickle.dump(
{
"dimensionality": None,
"total_elements_added": 5,
"max_seq_id": None,
"id_to_label": {"a": 1, "b": 2},
"label_to_id": {1: "a", 2: "b"},
"id_to_seq_id": {},
},
f,
)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert moved == []
assert seg.exists()
def test_quarantine_invalid_hnsw_metadata_renames_mismatched_missing_dimensionality(tmp_path):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
(seg / "data_level0.bin").write_bytes(b"x" * 2048)
(seg / "link_lists.bin").write_bytes(b"x" * 128)
with open(seg / "index_metadata.pickle", "wb") as f:
pickle.dump(
{
"dimensionality": None,
"total_elements_added": 2,
"max_seq_id": None,
"id_to_label": {"a": 1, "b": 2},
"label_to_id": {1: "b", 2: "a"},
"id_to_seq_id": {},
},
f,
)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert len(moved) == 1
assert ".corrupt-" in moved[0]
assert not seg.exists()
def test_quarantine_invalid_hnsw_metadata_allows_uninitialized_segment(tmp_path):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
with open(seg / "index_metadata.pickle", "wb") as f:
pickle.dump({"dimensionality": None, "id_to_label": {}}, f)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert moved == []
assert seg.exists()
def test_quarantine_invalid_hnsw_metadata_rejects_non_dict_id_to_label(tmp_path):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
with open(seg / "index_metadata.pickle", "wb") as f:
pickle.dump({"dimensionality": 8, "id_to_label": ["a", "b"]}, f)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert len(moved) == 1
assert ".corrupt-" in moved[0]
assert not seg.exists()
def test_quarantine_invalid_hnsw_metadata_rejects_non_schema_payload(tmp_path):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
with open(seg / "index_metadata.pickle", "wb") as f:
pickle.dump(["not", "a", "metadata", "object"], f)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert len(moved) == 1
assert ".corrupt-" in moved[0]
assert not seg.exists()
def _dangerous_pickle_payload_executed():
raise AssertionError("unsafe pickle payload executed")
class _DangerousPickle:
def __reduce__(self):
return (_dangerous_pickle_payload_executed, ())
def test_quarantine_invalid_hnsw_metadata_rejects_unsafe_pickle(tmp_path):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
with open(seg / "index_metadata.pickle", "wb") as f:
pickle.dump(_DangerousPickle(), f)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert len(moved) == 1
assert ".corrupt-" in moved[0]
assert not seg.exists()
def test_quarantine_invalid_hnsw_metadata_skips_transient_read_errors(tmp_path, monkeypatch):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
meta = seg / "index_metadata.pickle"
meta.write_bytes(b"partial")
monkeypatch.setattr(
"mempalace.backends.chroma._SafePersistentDataUnpickler.load",
lambda path: (_ for _ in ()).throw(EOFError("flush in progress")),
)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert moved == []
assert seg.exists()
def test_quarantine_invalid_hnsw_metadata_skips_truncated_pickle(tmp_path, monkeypatch):
palace = tmp_path / "palace"
palace.mkdir()
seg = palace / "abcd-1234-5678"
seg.mkdir()
meta = seg / "index_metadata.pickle"
meta.write_bytes(b"partial")
monkeypatch.setattr(
"mempalace.backends.chroma._SafePersistentDataUnpickler.load",
lambda path: (_ for _ in ()).throw(pickle.UnpicklingError("pickle data was truncated")),
)
moved = quarantine_invalid_hnsw_metadata(str(palace))
assert moved == []
assert seg.exists()
def test_chroma_backend_preflights_metadata_before_persistent_client(tmp_path, monkeypatch):
palace = tmp_path / "palace"
palace.mkdir()
calls = []
def _record(name):
def inner(path, *args, **kwargs):
calls.append((name, path))
return [] if name != "blob" else None
return inner
monkeypatch.setattr(
"mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type")
)
monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob"))
monkeypatch.setattr(
"mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid")
)
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _record("stale"))
class DummyClient:
pass
monkeypatch.setattr(
"mempalace.backends.chroma.chromadb.PersistentClient", lambda path: DummyClient()
)
backend = ChromaBackend()
backend._client(str(palace))
assert calls == [
("collection_type", str(palace)),
("blob", str(palace)),
("invalid", str(palace)),
("stale", str(palace)),
]
def test_chroma_backend_quarantine_rearms_on_mtime_refresh(tmp_path, monkeypatch):
"""When the DB mtime changes between ``_client()`` calls, the quarantine
gate re-arms and the HNSW safety checks run again (#1573)."""
palace = tmp_path / "palace"
palace.mkdir()
(palace / "chroma.sqlite3").write_text("")
calls = []
def _record(name):
def inner(path, *args, **kwargs):
calls.append((name, path))
return [] if name != "blob" else None
return inner
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
monkeypatch.setattr(
"mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type")
)
monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob"))
monkeypatch.setattr(
"mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid")
)
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _record("stale"))
class DummyClient:
pass
monkeypatch.setattr(
"mempalace.backends.chroma.chromadb.PersistentClient", lambda path: DummyClient()
)
backend = ChromaBackend()
stats = iter([(1, 1.0), (1, 1.0), (1, 2.0), (1, 2.0)])
monkeypatch.setattr(backend, "_db_stat", lambda path: next(stats))
backend._client(str(palace))
backend._client(str(palace))
assert calls == [
("collection_type", str(palace)),
("blob", str(palace)),
("invalid", str(palace)),
("stale", str(palace)),
("collection_type", str(palace)),
("blob", str(palace)),
("invalid", str(palace)),
("stale", str(palace)),
]
def test_chroma_backend_requarantines_after_inode_replacement(tmp_path, monkeypatch):
palace = tmp_path / "palace"
palace.mkdir()
(palace / "chroma.sqlite3").write_text("")
calls = []
def _record(name):
def inner(path, *args, **kwargs):
calls.append((name, path))
return [] if name != "blob" else None
return inner
monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set())
monkeypatch.setattr(
"mempalace.backends.chroma._fix_missing_collection_type", _record("collection_type")
)
monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob"))
monkeypatch.setattr(
"mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid")
)
monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _record("stale"))
class DummyClient:
pass
monkeypatch.setattr(
"mempalace.backends.chroma.chromadb.PersistentClient", lambda path: DummyClient()
)
backend = ChromaBackend()
stats = iter([(1, 1.0), (1, 1.0), (2, 2.0), (2, 2.0)])
monkeypatch.setattr(backend, "_db_stat", lambda path: next(stats))
backend._client(str(palace))
backend._client(str(palace))
assert calls == [
("collection_type", str(palace)),
("blob", str(palace)),
("invalid", str(palace)),
("stale", str(palace)),
("collection_type", str(palace)),
("blob", str(palace)),
("invalid", str(palace)),
("stale", str(palace)),
]
def test_explain_ef_mismatch_recognizes_chromadb_conflict():
"""When ChromaDB rejects a collection read due to an EF-name mismatch
(user changed MEMPALACE_EMBEDDING_MODEL on an existing palace), the
backend wraps the bare ValueError with a message that tells the user
how to recover. Without this, users hit a stack trace and don't know
rebuild-index exists."""
err = ValueError(
"An embedding function already exists in the collection configuration, "
"and a new one is provided. Embedding function conflict: new: "
"embeddinggemma_300m vs persisted: default"
)
msg = ChromaBackend._explain_ef_mismatch(err, "/tmp/palace.db")
assert msg is not None
assert "/tmp/palace.db" in msg
assert "MEMPALACE_EMBEDDING_MODEL" in msg
assert "mempalace --palace /tmp/palace.db repair rebuild-index" in msg
def test_explain_ef_mismatch_returns_none_for_unrelated_errors():
"""Don't paper over unrelated ValueErrors with the EF-mismatch message —
the caller needs to re-raise unmodified so debugging stays sane."""
err = ValueError("Some other ChromaDB problem")
assert ChromaBackend._explain_ef_mismatch(err, "/tmp/palace.db") is None
def test_get_collection_translates_ef_mismatch_to_helpful_error(tmp_path):
"""End-to-end: create a palace with the default EF, then try to read it
with a different EF name and confirm we surface the rebuild-index hint."""
backend = ChromaBackend()
palace_path = str(tmp_path / "palace")
os.makedirs(palace_path, exist_ok=True)
# Create the collection using the default (minilm-based) EF.
coll = backend.get_collection(palace_path, "drawers", create=True)
coll.add(documents=["seed"], ids=["1"])
# Now swap in an incompatible EF name (simulates the user setting
# MEMPALACE_EMBEDDING_MODEL=embeddinggemma without rebuild-index).
class _ConflictingEF:
@staticmethod
def name() -> str:
return "embeddinggemma_300m"
def __call__(self, input):
return [[0.0] * _TEST_EMBED_DIM for _ in input]
original_resolver = backend._resolve_embedding_function
backend._resolve_embedding_function = lambda: _ConflictingEF()
# Drop the cached client so the next call goes through the open path.
backend.close_palace(palace_path)
try:
with pytest.raises(ValueError, match=r"rebuild-index"):
backend.get_collection(palace_path, "drawers", create=False)
finally:
backend._resolve_embedding_function = original_resolver
backend.close_palace(palace_path)
def test_palace_get_collection_uses_configured_collection_name(monkeypatch):
from mempalace import palace
captured = {}
def fake_get_collection(palace_path, collection_name=None, create=False):
captured["palace_path"] = palace_path
captured["collection_name"] = collection_name
captured["create"] = create
return object()
monkeypatch.setattr(palace._DEFAULT_BACKEND, "get_collection", fake_get_collection)
monkeypatch.setattr("mempalace.config.get_configured_collection_name", lambda: "custom_drawers")
palace.get_collection("/palace", create=False)
assert captured == {
"palace_path": "/palace",
"collection_name": "custom_drawers",
"create": False,
}