fix(lancedb): fix FTS with_position optimize crash + disk bloat (#336)
FTS indexes built with with_position=True crash lance's optimize/compaction on lancedb >= 0.32 when merging an unindexed tail (Max offset exceeds length of values; upstream lance-format/lance#7653). The crash aborts optimize() including version cleanup, so the index dir grows unbounded until the disk fills. everos recall is OR-mode BM25 and never does phrase queries, so positions are never read -- disabling is lossless. - base: default with_position=False - infra: migrate_fts_indexes() rebuilds pre-fix indexes once at startup + reclaims orphans - cascade worker: count consecutive optimize failures, escalate warning->error Fixes #335. Co-authored-by: zhanghui <zhanghui@shanda.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
zhanghui
Claude Opus 4.8
parent
56ee9c8e5c
commit
45656d331e
@@ -571,3 +571,65 @@ async def test_rebuild_failure_does_not_crash_daemon(
|
||||
await w.stop()
|
||||
# Worker is still alive (stop() returned cleanly).
|
||||
assert w._task is None
|
||||
|
||||
|
||||
class _OptimizeFailingRepo(_FakeLanceRepo):
|
||||
"""Fake repo whose ``optimize()`` raises until ``fail`` is cleared."""
|
||||
|
||||
def __init__(self, **kw) -> None: # type: ignore[no-untyped-def]
|
||||
super().__init__(**kw)
|
||||
self.fail = True
|
||||
|
||||
async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None:
|
||||
if self.fail:
|
||||
raise RuntimeError("Max offset of 9 exceeds length of values 3")
|
||||
await super().optimize(cleanup_older_than=cleanup_older_than)
|
||||
|
||||
|
||||
async def test_optimize_failures_counted_escalated_and_reset(
|
||||
patched_repo: _FakeRepo,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Layer-2 stop-gap for lance-format/lance#7653.
|
||||
|
||||
Consecutive ``optimize()`` failures are counted, escalate
|
||||
warning→error once the threshold is hit, and reset to 0 on the next
|
||||
success — instead of being swallowed as a silent warning stream that
|
||||
lets the index dir grow until the disk fills.
|
||||
"""
|
||||
from everos.memory.cascade import worker as wmod
|
||||
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
class _SpyLogger:
|
||||
def __getattr__(self, level: str): # type: ignore[no-untyped-def]
|
||||
def rec(event: str, **_kw) -> None: # type: ignore[no-untyped-def]
|
||||
calls.append((level, event))
|
||||
|
||||
return rec
|
||||
|
||||
monkeypatch.setattr(wmod, "logger", _SpyLogger())
|
||||
|
||||
repo = _OptimizeFailingRepo()
|
||||
w = CascadeWorker(
|
||||
{"episode": _OkHandlerWithRepo(repo)},
|
||||
retry_backoff_seconds=0,
|
||||
optimize_min_interval_seconds=0.05,
|
||||
)
|
||||
w._optimizer_states["episode"] = wmod._KindOptimizerState()
|
||||
|
||||
threshold = wmod._OPTIMIZE_FAILURE_ALERT_THRESHOLD
|
||||
for _ in range(threshold):
|
||||
await w._run_optimize_once("episode")
|
||||
|
||||
state = w._optimizer_states["episode"]
|
||||
assert state.optimize_failures == threshold
|
||||
|
||||
fail_logs = [lvl for lvl, ev in calls if ev == "cascade_lancedb_optimize_failed"]
|
||||
assert fail_logs[:-1] == ["warning"] * (threshold - 1)
|
||||
assert fail_logs[-1] == "error"
|
||||
|
||||
# A subsequent success resets the streak.
|
||||
repo.fail = False
|
||||
await w._run_optimize_once("episode")
|
||||
assert state.optimize_failures == 0
|
||||
|
||||
Reference in New Issue
Block a user