diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index bbb9c93..b01750b 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -168,6 +168,46 @@ _collection_cache = None _palace_db_inode = 0 # inode of chroma.sqlite3 at cache time _palace_db_mtime = 0.0 # mtime of chroma.sqlite3 at cache time + +def _is_transient_index_error(result) -> bool: + # Chroma can return "Internal error: Error finding id" during the + # HNSW flush window after a bulk CLI mine — SQLite rows are + # committed but the binary segment metadata isn't flushed yet. + # Self-heals once the flush completes (~30-60s). See issue #1315. + if not isinstance(result, dict): + return False + err = result.get("error", "") + return isinstance(err, str) and ("Error finding id" in err or "Internal error" in err) + + +def _force_chroma_cache_reset() -> None: + # Drop both the MCP-local client cache and the shared backend's + # per-palace cache so the next call rebuilds against the post-flush + # state. Without clearing _DEFAULT_BACKEND._clients the retry + # would just hit the same stale handle, since tool_search routes + # via search_memories -> palace.get_collection -> backend cache. + global \ + _client_cache, \ + _collection_cache, \ + _palace_db_inode, \ + _palace_db_mtime, \ + _metadata_cache, \ + _metadata_cache_time + _client_cache = None + _collection_cache = None + _palace_db_inode = 0 + _palace_db_mtime = 0.0 + _metadata_cache = None + _metadata_cache_time = 0 + try: + from .palace import _DEFAULT_BACKEND + + _DEFAULT_BACKEND._clients.pop(_config.palace_path, None) + _DEFAULT_BACKEND._freshness.pop(_config.palace_path, None) + except Exception: + pass + + # ── Vector-search disabled flag (#1222) ────────────────────────────────── # Set when ``hnsw_capacity_status`` reports a divergence between sqlite # and the HNSW segment large enough that chromadb would segfault on @@ -721,6 +761,24 @@ def tool_search( max_distance=dist, vector_disabled=_vector_disabled, ) + if _is_transient_index_error(result): + # Post-bulk-write HNSW flush window (#1315): drop caches, give + # the segment a moment to settle, retry once. Caller never sees + # the transient unless the second attempt also fails. + _force_chroma_cache_reset() + time.sleep(2) + _refresh_vector_disabled_flag() + result = search_memories( + sanitized["clean_query"], + palace_path=_config.palace_path, + wing=wing, + room=room, + n_results=limit, + max_distance=dist, + vector_disabled=_vector_disabled, + ) + if not _is_transient_index_error(result): + result["index_recovered"] = True if _vector_disabled: result["vector_disabled"] = True result["vector_disabled_reason"] = _vector_disabled_reason diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index ae20bf3..95b173e 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -414,6 +414,76 @@ class TestSearchTool: result = mcp_server.tool_search(query="JWT", room="../backend") assert "error" in result + def test_search_retries_once_on_hnsw_flush_transient(self, monkeypatch, config, kg): + """Issue #1315: post-bulk-mine 'Error finding id' is retried once.""" + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + + calls = {"n": 0} + reset_calls = {"n": 0} + + def fake_search(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + return { + "error": "Search error: Error executing plan: Internal error: Error finding id" + } + return {"results": [{"text": "ok", "wing": "w", "room": "r"}]} + + def fake_reset(): + reset_calls["n"] += 1 + + monkeypatch.setattr(mcp_server, "search_memories", fake_search) + monkeypatch.setattr(mcp_server, "_force_chroma_cache_reset", fake_reset) + monkeypatch.setattr(mcp_server.time, "sleep", lambda _: None) + + result = mcp_server.tool_search(query="anything") + + assert calls["n"] == 2 + assert reset_calls["n"] == 1 + assert "results" in result + assert result.get("index_recovered") is True + + def test_search_does_not_retry_on_non_transient_error(self, monkeypatch, config, kg): + """Validation / unrelated errors must not trigger the retry path.""" + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + + calls = {"n": 0} + + def fake_search(*args, **kwargs): + calls["n"] += 1 + return {"error": "Search error: invalid query syntax"} + + monkeypatch.setattr(mcp_server, "search_memories", fake_search) + + result = mcp_server.tool_search(query="anything") + + assert calls["n"] == 1 + assert "error" in result + assert "index_recovered" not in result + + def test_search_returns_second_error_if_retry_also_fails(self, monkeypatch, config, kg): + """If the transient persists past the retry, surface the second error.""" + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + + calls = {"n": 0} + + def fake_search(*args, **kwargs): + calls["n"] += 1 + return {"error": "Search error: Error executing plan: Internal error: Error finding id"} + + monkeypatch.setattr(mcp_server, "search_memories", fake_search) + monkeypatch.setattr(mcp_server, "_force_chroma_cache_reset", lambda: None) + monkeypatch.setattr(mcp_server.time, "sleep", lambda _: None) + + result = mcp_server.tool_search(query="anything") + + assert calls["n"] == 2 + assert "error" in result + assert "index_recovered" not in result + def test_list_drawers_rejects_invalid_wing(self, monkeypatch, config, kg): _patch_mcp_server(monkeypatch, config, kg) from mempalace import mcp_server @@ -503,9 +573,9 @@ class TestWriteTools: assert result1["success"] is True assert result2["success"] is True - assert ( - result1["drawer_id"] != result2["drawer_id"] - ), "Documents with shared header but different content must have distinct drawer IDs" + assert result1["drawer_id"] != result2["drawer_id"], ( + "Documents with shared header but different content must have distinct drawer IDs" + ) def test_delete_drawer(self, monkeypatch, config, palace_path, seeded_collection, kg): _patch_mcp_server(monkeypatch, config, kg) @@ -1243,9 +1313,9 @@ class TestCacheInvalidation: all_calls = captured["get"] + captured["create"] assert all_calls, "expected get_collection or create_collection to be called" for kwargs in all_calls: - assert ( - "embedding_function" in kwargs - ), f"missing embedding_function= in chromadb call: {kwargs}" + assert "embedding_function" in kwargs, ( + f"missing embedding_function= in chromadb call: {kwargs}" + ) assert kwargs["embedding_function"] is not None # Same expectation on the create=False (cache-miss) reopen path.