fix(lancedb): fix FTS with_position optimize crash + disk bloat (#336)
FTS indexes built with with_position=True crash lance's optimize/compaction on lancedb >= 0.32 when merging an unindexed tail (Max offset exceeds length of values; upstream lance-format/lance#7653). The crash aborts optimize() including version cleanup, so the index dir grows unbounded until the disk fills. everos recall is OR-mode BM25 and never does phrase queries, so positions are never read -- disabling is lossless. - base: default with_position=False - infra: migrate_fts_indexes() rebuilds pre-fix indexes once at startup + reclaims orphans - cascade worker: count consecutive optimize failures, escalate warning->error Fixes #335. Co-authored-by: zhanghui <zhanghui@shanda.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
zhanghui
Claude Opus 4.8
parent
56ee9c8e5c
commit
45656d331e
@@ -647,3 +647,59 @@ async def test_write_lock_is_per_table(tmp_path: Path) -> None:
|
||||
assert repo_a._write_lock(repo_a.table_name) is not repo_b._write_lock(
|
||||
repo_b.table_name
|
||||
)
|
||||
|
||||
|
||||
# ── migrate_fts_indexes (one-time rebuild of pre-fix with_position indexes) ──
|
||||
|
||||
|
||||
async def test_migrate_fts_indexes_runs_once_and_rebuilds(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""migrate_fts_indexes rebuilds existing FTS indexes once, guarded by a
|
||||
version marker in the LanceDB dir (fix for lance-format/lance#7653).
|
||||
|
||||
White-box surfaces: the ``.fts_index_version`` marker file and
|
||||
``AsyncTable.list_indices`` on the global-connection table.
|
||||
"""
|
||||
monkeypatch.setenv("EVEROS_ROOT", str(tmp_path))
|
||||
import everos.infra.persistence.lancedb as lancedb_infra
|
||||
from everos.core.persistence import MemoryRoot
|
||||
from everos.infra.persistence.lancedb import (
|
||||
dispose_connection,
|
||||
get_table,
|
||||
migrate_fts_indexes,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(lancedb_infra, "_BUSINESS_SCHEMAS", (_SearchNote,))
|
||||
await dispose_connection()
|
||||
try:
|
||||
table = await get_table(_SearchNote.TABLE_NAME, _SearchNote)
|
||||
await table.add(
|
||||
[
|
||||
_SearchNote(
|
||||
id="1",
|
||||
text="hello world",
|
||||
tokens="hello world",
|
||||
vector=[1, 0, 0, 0],
|
||||
)
|
||||
]
|
||||
)
|
||||
await _SearchNote.ensure_fts_indexes(table)
|
||||
assert any("tokens" in (i.columns or []) for i in await table.list_indices())
|
||||
|
||||
marker = MemoryRoot.default().lancedb_dir / ".fts_index_version"
|
||||
assert not marker.exists()
|
||||
|
||||
# First run: migrates + writes the marker, index still present.
|
||||
await migrate_fts_indexes()
|
||||
assert marker.read_text().strip() == "2"
|
||||
assert any("tokens" in (i.columns or []) for i in await table.list_indices())
|
||||
|
||||
# Marker present → second run is a no-op. Drop the index, re-run,
|
||||
# and confirm it is NOT rebuilt (migration skipped, not re-executed).
|
||||
for i in await table.list_indices():
|
||||
await table.drop_index(i.name)
|
||||
await migrate_fts_indexes()
|
||||
assert not list(await table.list_indices())
|
||||
finally:
|
||||
await dispose_connection()
|
||||
|
||||
@@ -571,3 +571,65 @@ async def test_rebuild_failure_does_not_crash_daemon(
|
||||
await w.stop()
|
||||
# Worker is still alive (stop() returned cleanly).
|
||||
assert w._task is None
|
||||
|
||||
|
||||
class _OptimizeFailingRepo(_FakeLanceRepo):
|
||||
"""Fake repo whose ``optimize()`` raises until ``fail`` is cleared."""
|
||||
|
||||
def __init__(self, **kw) -> None: # type: ignore[no-untyped-def]
|
||||
super().__init__(**kw)
|
||||
self.fail = True
|
||||
|
||||
async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None:
|
||||
if self.fail:
|
||||
raise RuntimeError("Max offset of 9 exceeds length of values 3")
|
||||
await super().optimize(cleanup_older_than=cleanup_older_than)
|
||||
|
||||
|
||||
async def test_optimize_failures_counted_escalated_and_reset(
|
||||
patched_repo: _FakeRepo,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Layer-2 stop-gap for lance-format/lance#7653.
|
||||
|
||||
Consecutive ``optimize()`` failures are counted, escalate
|
||||
warning→error once the threshold is hit, and reset to 0 on the next
|
||||
success — instead of being swallowed as a silent warning stream that
|
||||
lets the index dir grow until the disk fills.
|
||||
"""
|
||||
from everos.memory.cascade import worker as wmod
|
||||
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
class _SpyLogger:
|
||||
def __getattr__(self, level: str): # type: ignore[no-untyped-def]
|
||||
def rec(event: str, **_kw) -> None: # type: ignore[no-untyped-def]
|
||||
calls.append((level, event))
|
||||
|
||||
return rec
|
||||
|
||||
monkeypatch.setattr(wmod, "logger", _SpyLogger())
|
||||
|
||||
repo = _OptimizeFailingRepo()
|
||||
w = CascadeWorker(
|
||||
{"episode": _OkHandlerWithRepo(repo)},
|
||||
retry_backoff_seconds=0,
|
||||
optimize_min_interval_seconds=0.05,
|
||||
)
|
||||
w._optimizer_states["episode"] = wmod._KindOptimizerState()
|
||||
|
||||
threshold = wmod._OPTIMIZE_FAILURE_ALERT_THRESHOLD
|
||||
for _ in range(threshold):
|
||||
await w._run_optimize_once("episode")
|
||||
|
||||
state = w._optimizer_states["episode"]
|
||||
assert state.optimize_failures == threshold
|
||||
|
||||
fail_logs = [lvl for lvl, ev in calls if ev == "cascade_lancedb_optimize_failed"]
|
||||
assert fail_logs[:-1] == ["warning"] * (threshold - 1)
|
||||
assert fail_logs[-1] == "error"
|
||||
|
||||
# A subsequent success resets the streak.
|
||||
repo.fail = False
|
||||
await w._run_optimize_once("episode")
|
||||
assert state.optimize_failures == 0
|
||||
|
||||
Reference in New Issue
Block a user