From e811b6d10a28fa5bca31e34e35c8c8ce2d9a68f6 Mon Sep 17 00:00:00 2001 From: Ivan Dergachev Date: Mon, 11 May 2026 13:54:57 +0400 Subject: [PATCH] fix: avoid false HNSW metadata quarantine --- mempalace/backends/chroma.py | 70 +++++++++++++++++++++++++++++++----- tests/test_backends.py | 53 +++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 8 deletions(-) diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index 387a2b9..980f2e8 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -671,20 +671,59 @@ def _valid_dimensionality(value: object) -> bool: return isinstance(value, Integral) and not isinstance(value, bool) and int(value) > 0 -def _persisted_metadata_fields(obj: object) -> tuple[object, object]: +def _persisted_metadata_value(obj: object, name: str) -> object: if isinstance(obj, dict): - return obj.get("dimensionality"), obj.get("id_to_label") - return getattr(obj, "dimensionality", None), getattr(obj, "id_to_label", None) + return obj.get(name) + return getattr(obj, name, None) + + +def _persisted_metadata_fields(obj: object) -> tuple[object, object]: + return _persisted_metadata_value(obj, "dimensionality"), _persisted_metadata_value( + obj, "id_to_label" + ) + + +def _missing_dimensionality_appears_recoverable( + persisted: object, id_to_label: dict, seg_dir: str +) -> bool: + total = _persisted_metadata_value(persisted, "total_elements_added") + label_to_id = _persisted_metadata_value(persisted, "label_to_id") + data_path = os.path.join(seg_dir, "data_level0.bin") + link_path = os.path.join(seg_dir, "link_lists.bin") + + if not isinstance(total, Integral) or isinstance(total, bool): + return False + if not isinstance(label_to_id, dict): + return False + try: + if not ( + os.path.isfile(data_path) + and os.path.isfile(link_path) + and os.path.getsize(data_path) > _HNSW_MISSING_METADATA_DATA_FLOOR + ): + return False + except OSError: + return False + if not _hnsw_payload_appears_sane(seg_dir): + return False + + label_count = len(id_to_label) + if int(total) != label_count or len(label_to_id) != label_count: + return False + try: + return all(label_to_id.get(label) == item_id for item_id, label in id_to_label.items()) + except TypeError: + return False def quarantine_invalid_hnsw_metadata(palace_path: str) -> list[str]: """Quarantine segment dirs whose ``index_metadata.pickle`` is unreadable or invalid. Chroma's persisted HNSW metadata is untrusted disk state. If a segment has - labels but no valid positive dimensionality, current Chroma versions can - accept the pickle and crash later in the Rust loader. We rename the entire - segment out of the way before ``PersistentClient`` opens so Chroma can - rebuild cleanly instead of touching known-bad metadata. + labels but invalid or partial metadata, current Chroma versions can accept + the pickle and crash later in the Rust loader. We rename the entire segment + out of the way before ``PersistentClient`` opens so Chroma can rebuild + cleanly instead of touching known-bad metadata. """ try: entries = os.listdir(palace_path) @@ -735,7 +774,22 @@ def quarantine_invalid_hnsw_metadata(palace_path: str) -> list[str]: reason = f"invalid id_to_label type {type(id_to_label).__name__}" else: has_labels = bool(id_to_label) - if has_labels and not _valid_dimensionality(dimensionality): + if ( + has_labels + and dimensionality is None + and not _missing_dimensionality_appears_recoverable( + persisted, id_to_label, seg_dir + ) + ): + reason = ( + "labels present but dimensionality is missing or invalid " + f"({dimensionality!r})" + ) + elif ( + has_labels + and dimensionality is not None + and not _valid_dimensionality(dimensionality) + ): reason = ( "labels present but dimensionality is missing or invalid " f"({dimensionality!r})" diff --git a/tests/test_backends.py b/tests/test_backends.py index 1fc8eab..0ee10d3 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -1047,6 +1047,59 @@ def test_quarantine_invalid_hnsw_metadata_renames_missing_dimensionality(tmp_pat assert not seg.exists() +def test_quarantine_invalid_hnsw_metadata_keeps_consistent_missing_dimensionality(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + (seg / "data_level0.bin").write_bytes(b"x" * 2048) + (seg / "link_lists.bin").write_bytes(b"x" * 128) + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump( + { + "dimensionality": None, + "total_elements_added": 2, + "max_seq_id": None, + "id_to_label": {"a": 1, "b": 2}, + "label_to_id": {1: "a", 2: "b"}, + "id_to_seq_id": {}, + }, + f, + ) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert moved == [] + assert seg.exists() + + +def test_quarantine_invalid_hnsw_metadata_renames_mismatched_missing_dimensionality(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + (seg / "data_level0.bin").write_bytes(b"x" * 2048) + (seg / "link_lists.bin").write_bytes(b"x" * 128) + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump( + { + "dimensionality": None, + "total_elements_added": 2, + "max_seq_id": None, + "id_to_label": {"a": 1, "b": 2}, + "label_to_id": {1: "b", 2: "a"}, + "id_to_seq_id": {}, + }, + f, + ) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert len(moved) == 1 + assert ".corrupt-" in moved[0] + assert not seg.exists() + + def test_quarantine_invalid_hnsw_metadata_allows_uninitialized_segment(tmp_path): palace = tmp_path / "palace" palace.mkdir()