fix(lancedb): fix FTS with_position optimize crash + disk bloat (#336)

FTS indexes built with with_position=True crash lance's optimize/compaction
on lancedb >= 0.32 when merging an unindexed tail (Max offset exceeds length
of values; upstream lance-format/lance#7653). The crash aborts optimize()
including version cleanup, so the index dir grows unbounded until the disk
fills. everos recall is OR-mode BM25 and never does phrase queries, so
positions are never read -- disabling is lossless.

- base: default with_position=False
- infra: migrate_fts_indexes() rebuilds pre-fix indexes once at startup + reclaims orphans
- cascade worker: count consecutive optimize failures, escalate warning->error

Fixes #335.

Co-authored-by: zhanghui <zhanghui@shanda.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
zhanghui
2026-07-10 16:54:58 +08:00
committed by GitHub
co-authored by zhanghui Claude Opus 4.8
parent 56ee9c8e5c
commit 45656d331e
5 changed files with 224 additions and 8 deletions
@@ -647,3 +647,59 @@ async def test_write_lock_is_per_table(tmp_path: Path) -> None:
assert repo_a._write_lock(repo_a.table_name) is not repo_b._write_lock(
repo_b.table_name
)
# ── migrate_fts_indexes (one-time rebuild of pre-fix with_position indexes) ──
async def test_migrate_fts_indexes_runs_once_and_rebuilds(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""migrate_fts_indexes rebuilds existing FTS indexes once, guarded by a
version marker in the LanceDB dir (fix for lance-format/lance#7653).
White-box surfaces: the ``.fts_index_version`` marker file and
``AsyncTable.list_indices`` on the global-connection table.
"""
monkeypatch.setenv("EVEROS_ROOT", str(tmp_path))
import everos.infra.persistence.lancedb as lancedb_infra
from everos.core.persistence import MemoryRoot
from everos.infra.persistence.lancedb import (
dispose_connection,
get_table,
migrate_fts_indexes,
)
monkeypatch.setattr(lancedb_infra, "_BUSINESS_SCHEMAS", (_SearchNote,))
await dispose_connection()
try:
table = await get_table(_SearchNote.TABLE_NAME, _SearchNote)
await table.add(
[
_SearchNote(
id="1",
text="hello world",
tokens="hello world",
vector=[1, 0, 0, 0],
)
]
)
await _SearchNote.ensure_fts_indexes(table)
assert any("tokens" in (i.columns or []) for i in await table.list_indices())
marker = MemoryRoot.default().lancedb_dir / ".fts_index_version"
assert not marker.exists()
# First run: migrates + writes the marker, index still present.
await migrate_fts_indexes()
assert marker.read_text().strip() == "2"
assert any("tokens" in (i.columns or []) for i in await table.list_indices())
# Marker present → second run is a no-op. Drop the index, re-run,
# and confirm it is NOT rebuilt (migration skipped, not re-executed).
for i in await table.list_indices():
await table.drop_index(i.name)
await migrate_fts_indexes()
assert not list(await table.list_indices())
finally:
await dispose_connection()
@@ -571,3 +571,65 @@ async def test_rebuild_failure_does_not_crash_daemon(
await w.stop()
# Worker is still alive (stop() returned cleanly).
assert w._task is None
class _OptimizeFailingRepo(_FakeLanceRepo):
"""Fake repo whose ``optimize()`` raises until ``fail`` is cleared."""
def __init__(self, **kw) -> None: # type: ignore[no-untyped-def]
super().__init__(**kw)
self.fail = True
async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None:
if self.fail:
raise RuntimeError("Max offset of 9 exceeds length of values 3")
await super().optimize(cleanup_older_than=cleanup_older_than)
async def test_optimize_failures_counted_escalated_and_reset(
patched_repo: _FakeRepo,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Layer-2 stop-gap for lance-format/lance#7653.
Consecutive ``optimize()`` failures are counted, escalate
warning→error once the threshold is hit, and reset to 0 on the next
success — instead of being swallowed as a silent warning stream that
lets the index dir grow until the disk fills.
"""
from everos.memory.cascade import worker as wmod
calls: list[tuple[str, str]] = []
class _SpyLogger:
def __getattr__(self, level: str): # type: ignore[no-untyped-def]
def rec(event: str, **_kw) -> None: # type: ignore[no-untyped-def]
calls.append((level, event))
return rec
monkeypatch.setattr(wmod, "logger", _SpyLogger())
repo = _OptimizeFailingRepo()
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(repo)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.05,
)
w._optimizer_states["episode"] = wmod._KindOptimizerState()
threshold = wmod._OPTIMIZE_FAILURE_ALERT_THRESHOLD
for _ in range(threshold):
await w._run_optimize_once("episode")
state = w._optimizer_states["episode"]
assert state.optimize_failures == threshold
fail_logs = [lvl for lvl, ev in calls if ev == "cascade_lancedb_optimize_failed"]
assert fail_logs[:-1] == ["warning"] * (threshold - 1)
assert fail_logs[-1] == "error"
# A subsequent success resets the streak.
repo.fail = False
await w._run_optimize_once("episode")
assert state.optimize_failures == 0