fix(mcp): retry tool_search once on Chroma "Error finding id" transient (#1315)

After a bulk CLI mine, ChromaDB's HNSW segment metadata can be unflushed
  for ~30-60s. Wing-scoped MCP search hits "Internal error: Error finding id"
  during that window, and the existing inode/mtime cache invalidation isn't
  enough — tool_search routes via search_memories -> palace.get_collection
  -> _DEFAULT_BACKEND._client, which has its own per-palace cache.

  This wraps tool_search with a single retry that drops both the MCP-local
  cache and _DEFAULT_BACKEND._clients/_freshness for the palace, sleeps 2s,
  retries once, and tags successful retries with index_recovered=True.

  Does not address tool_check_duplicate or other index-touching tools, nor
  the underlying flush window — options 1-3 from #1315 (auto-flush after
  mine, fail-fast detection, SQLite-only fallback) are still on the table
  for a complete fix.

  Refs #1315
This commit is contained in:
JP de Boer knt
2026-05-07 02:11:07 +02:00
parent 435f0ad348
commit be95ea7f33
2 changed files with 134 additions and 6 deletions
+58
View File
@@ -168,6 +168,46 @@ _collection_cache = None
_palace_db_inode = 0 # inode of chroma.sqlite3 at cache time
_palace_db_mtime = 0.0 # mtime of chroma.sqlite3 at cache time
def _is_transient_index_error(result) -> bool:
# Chroma can return "Internal error: Error finding id" during the
# HNSW flush window after a bulk CLI mine — SQLite rows are
# committed but the binary segment metadata isn't flushed yet.
# Self-heals once the flush completes (~30-60s). See issue #1315.
if not isinstance(result, dict):
return False
err = result.get("error", "")
return isinstance(err, str) and ("Error finding id" in err or "Internal error" in err)
def _force_chroma_cache_reset() -> None:
# Drop both the MCP-local client cache and the shared backend's
# per-palace cache so the next call rebuilds against the post-flush
# state. Without clearing _DEFAULT_BACKEND._clients the retry
# would just hit the same stale handle, since tool_search routes
# via search_memories -> palace.get_collection -> backend cache.
global \
_client_cache, \
_collection_cache, \
_palace_db_inode, \
_palace_db_mtime, \
_metadata_cache, \
_metadata_cache_time
_client_cache = None
_collection_cache = None
_palace_db_inode = 0
_palace_db_mtime = 0.0
_metadata_cache = None
_metadata_cache_time = 0
try:
from .palace import _DEFAULT_BACKEND
_DEFAULT_BACKEND._clients.pop(_config.palace_path, None)
_DEFAULT_BACKEND._freshness.pop(_config.palace_path, None)
except Exception:
pass
# ── Vector-search disabled flag (#1222) ──────────────────────────────────
# Set when ``hnsw_capacity_status`` reports a divergence between sqlite
# and the HNSW segment large enough that chromadb would segfault on
@@ -721,6 +761,24 @@ def tool_search(
max_distance=dist,
vector_disabled=_vector_disabled,
)
if _is_transient_index_error(result):
# Post-bulk-write HNSW flush window (#1315): drop caches, give
# the segment a moment to settle, retry once. Caller never sees
# the transient unless the second attempt also fails.
_force_chroma_cache_reset()
time.sleep(2)
_refresh_vector_disabled_flag()
result = search_memories(
sanitized["clean_query"],
palace_path=_config.palace_path,
wing=wing,
room=room,
n_results=limit,
max_distance=dist,
vector_disabled=_vector_disabled,
)
if not _is_transient_index_error(result):
result["index_recovered"] = True
if _vector_disabled:
result["vector_disabled"] = True
result["vector_disabled_reason"] = _vector_disabled_reason
+76 -6
View File
@@ -414,6 +414,76 @@ class TestSearchTool:
result = mcp_server.tool_search(query="JWT", room="../backend")
assert "error" in result
def test_search_retries_once_on_hnsw_flush_transient(self, monkeypatch, config, kg):
"""Issue #1315: post-bulk-mine 'Error finding id' is retried once."""
_patch_mcp_server(monkeypatch, config, kg)
from mempalace import mcp_server
calls = {"n": 0}
reset_calls = {"n": 0}
def fake_search(*args, **kwargs):
calls["n"] += 1
if calls["n"] == 1:
return {
"error": "Search error: Error executing plan: Internal error: Error finding id"
}
return {"results": [{"text": "ok", "wing": "w", "room": "r"}]}
def fake_reset():
reset_calls["n"] += 1
monkeypatch.setattr(mcp_server, "search_memories", fake_search)
monkeypatch.setattr(mcp_server, "_force_chroma_cache_reset", fake_reset)
monkeypatch.setattr(mcp_server.time, "sleep", lambda _: None)
result = mcp_server.tool_search(query="anything")
assert calls["n"] == 2
assert reset_calls["n"] == 1
assert "results" in result
assert result.get("index_recovered") is True
def test_search_does_not_retry_on_non_transient_error(self, monkeypatch, config, kg):
"""Validation / unrelated errors must not trigger the retry path."""
_patch_mcp_server(monkeypatch, config, kg)
from mempalace import mcp_server
calls = {"n": 0}
def fake_search(*args, **kwargs):
calls["n"] += 1
return {"error": "Search error: invalid query syntax"}
monkeypatch.setattr(mcp_server, "search_memories", fake_search)
result = mcp_server.tool_search(query="anything")
assert calls["n"] == 1
assert "error" in result
assert "index_recovered" not in result
def test_search_returns_second_error_if_retry_also_fails(self, monkeypatch, config, kg):
"""If the transient persists past the retry, surface the second error."""
_patch_mcp_server(monkeypatch, config, kg)
from mempalace import mcp_server
calls = {"n": 0}
def fake_search(*args, **kwargs):
calls["n"] += 1
return {"error": "Search error: Error executing plan: Internal error: Error finding id"}
monkeypatch.setattr(mcp_server, "search_memories", fake_search)
monkeypatch.setattr(mcp_server, "_force_chroma_cache_reset", lambda: None)
monkeypatch.setattr(mcp_server.time, "sleep", lambda _: None)
result = mcp_server.tool_search(query="anything")
assert calls["n"] == 2
assert "error" in result
assert "index_recovered" not in result
def test_list_drawers_rejects_invalid_wing(self, monkeypatch, config, kg):
_patch_mcp_server(monkeypatch, config, kg)
from mempalace import mcp_server
@@ -503,9 +573,9 @@ class TestWriteTools:
assert result1["success"] is True
assert result2["success"] is True
assert (
result1["drawer_id"] != result2["drawer_id"]
), "Documents with shared header but different content must have distinct drawer IDs"
assert result1["drawer_id"] != result2["drawer_id"], (
"Documents with shared header but different content must have distinct drawer IDs"
)
def test_delete_drawer(self, monkeypatch, config, palace_path, seeded_collection, kg):
_patch_mcp_server(monkeypatch, config, kg)
@@ -1243,9 +1313,9 @@ class TestCacheInvalidation:
all_calls = captured["get"] + captured["create"]
assert all_calls, "expected get_collection or create_collection to be called"
for kwargs in all_calls:
assert (
"embedding_function" in kwargs
), f"missing embedding_function= in chromadb call: {kwargs}"
assert "embedding_function" in kwargs, (
f"missing embedding_function= in chromadb call: {kwargs}"
)
assert kwargs["embedding_function"] is not None
# Same expectation on the create=False (cache-miss) reopen path.