From 72c1facdbfa3c07d97d29db106482cf643ae8231 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Wed, 5 Aug 2026 13:48:15 +0800 Subject: [PATCH] fix(cascade): record which beat lost the optimize commit race (#390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benign-conflict log said a commit race was lost but not by which maintenance beat, and lance labels both beats' commit identically ("This Rewrite transaction was preempted by concurrent transaction ..."), so the message alone cannot separate them. The costs differ sharply: - a lost LIGHT beat is free — compaction retries ~10s later; - a lost HEAVY beat means that table skipped a whole prune cadence, so its superseded files stay on disk until the next one lands. Reading an index-dir growth incident off the logs therefore meant back-inferring which beats were heavy from the 300s cadence. That was done once during the storage soak to explain a 13-minute window where one table's cleanup stalled with no failure logged, and it cost two 35-minute debug-level reruns to confirm — the field makes it a grep. Adds `pruned` to the conflict log, mirroring the sibling failure log. Log level (debug) and the benign-conflict semantics are unchanged: the streak is still not incremented and no fallback rebuild is triggered. Test asserts both beats in one run: the heavy beat's conflict logs pruned=True and the light beat's logs pruned=False. Mutation-verified — dropping the field fails the test with KeyError. Co-authored-by: zhanghui Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Elliot Chen <2340896+cyfyifanchen@users.noreply.github.com> --- CHANGELOG.md | 12 ++++ src/everos/memory/cascade/worker.py | 12 ++++ .../test_memory/test_cascade/test_worker.py | 65 +++++++++++++++++++ 3 files changed, 89 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d4940..97a5dae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **`cascade_lancedb_optimize_conflict` now records `pruned`** — which + maintenance beat lost the commit race. Lance labels both beats' commit the + same way (`This Rewrite transaction was preempted by concurrent transaction + …`), so the message alone cannot separate a free loss from a costly one: a + lost **light** beat retries ~10s later, while a lost **heavy** beat means + that table skipped a whole prune cadence and its superseded files stay on + disk. Attributing index-dir growth previously meant back-inferring which + beats were heavy from the 300s cadence. Log level (`debug`) and the + benign-conflict semantics are unchanged — this adds one field. + ## [1.2.2] - 2026-08-04 ### Added diff --git a/src/everos/memory/cascade/worker.py b/src/everos/memory/cascade/worker.py index 89c6fb5..6117761 100644 --- a/src/everos/memory/cascade/worker.py +++ b/src/everos/memory/cascade/worker.py @@ -827,9 +827,21 @@ class CascadeWorker: # the right detector for it, and unlike a rebuild it does not # destroy indexes to "fix" a lost race. if _is_benign_commit_conflict(exc): + # ``pruned`` is the whole diagnostic value of this line. Lance + # labels both beats' commit the same way ("This Rewrite + # transaction was preempted by ..."), so the message alone + # cannot tell them apart — but the consequences differ: + # a lost LIGHT beat is free (compaction retries ~10s later), + # while a lost HEAVY beat means that table skipped a whole + # prune cadence and its index dir keeps the superseded files. + # Without the flag, reading a disk-growth incident off the + # logs means back-inferring which beats were heavy from the + # 300s cadence (done once during the storage soak — slow and + # fragile). Mirrors ``pruned`` on the sibling failure log. logger.debug( "cascade_lancedb_optimize_conflict", kind=kind, + pruned=should_prune, error=f"{type(exc).__name__}: {exc}", ) return diff --git a/tests/unit/test_memory/test_cascade/test_worker.py b/tests/unit/test_memory/test_cascade/test_worker.py index 2fb1c54..6d588d1 100644 --- a/tests/unit/test_memory/test_cascade/test_worker.py +++ b/tests/unit/test_memory/test_cascade/test_worker.py @@ -1071,6 +1071,71 @@ async def test_heavy_beat_commit_conflict_is_also_benign( ) +async def test_conflict_log_names_the_lost_beat( + patched_repo: _FakeRepo, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The conflict log must say WHICH beat lost the race. + + Lance labels both beats' commit identically ("This Rewrite transaction + was preempted by concurrent transaction ..."), so the error message + cannot tell them apart — yet the cost differs sharply: + + - a lost LIGHT beat is free (compaction retries ~10s later); + - a lost HEAVY beat means that table skipped a whole prune cadence, so + its superseded files stay on disk until the next one lands. + + Without ``pruned`` on this line, attributing index-dir growth means + back-inferring which beats were heavy from the 300s cadence. + """ + from everos.memory.cascade import worker as wmod + + calls: list[tuple[str, dict[str, object]]] = [] + + class _SpyLogger: + def __getattr__(self, level: str): # type: ignore[no-untyped-def] + def rec(event: str, **kw) -> None: # type: ignore[no-untyped-def] + calls.append((event, kw)) + + return rec + + monkeypatch.setattr(wmod, "logger", _SpyLogger()) + + class _ConflictRepo(_FakeLanceRepo): + """Loses the commit race on both beats.""" + + async def optimize(self) -> None: + raise RuntimeError("Retryable commit conflict for version 215") + + async def prune(self, older_than: dt.timedelta) -> None: + raise RuntimeError("Retryable commit conflict for version 215") + + # Freeze the clock: which beat runs must not depend on the runner's uptime + # (`monotonic()` is boot-relative — a fresh CI runner reads ~100s). + monkeypatch.setattr(wmod.time, "monotonic", lambda: 10_000.0) + w = CascadeWorker( + {"episode": _OkHandlerWithRepo(_ConflictRepo())}, + retry_backoff_seconds=0, + optimize_prune_interval_seconds=300.0, + ) + state = wmod._KindOptimizerState() # last_prune_attempt_at=0 → HEAVY first + w._optimizer_states["episode"] = state + + await w._run_optimize_once("episode") # heavy: prune loses the race + # The attempt clock advanced to the (frozen) now, so the cadence has not + # elapsed — the next beat is the light one. + await w._run_optimize_once("episode") + + lost = [kw for ev, kw in calls if ev == "cascade_lancedb_optimize_conflict"] + assert len(lost) == 2, "both beats must have logged a conflict" + assert lost[0]["pruned"] is True, ( + "the heavy beat lost — this is the one that costs a prune cadence" + ) + assert lost[1]["pruned"] is False, ( + "the light beat lost — free, the next beat retries it" + ) + + async def test_non_conflict_failure_counts_even_when_message_says_retryable( patched_repo: _FakeRepo, ) -> None: