fix(cascade): record which beat lost the optimize commit race (#390)

The benign-conflict log said a commit race was lost but not by which
maintenance beat, and lance labels both beats' commit identically
("This Rewrite transaction was preempted by concurrent transaction ..."),
so the message alone cannot separate them. The costs differ sharply:

- a lost LIGHT beat is free — compaction retries ~10s later;
- a lost HEAVY beat means that table skipped a whole prune cadence, so
  its superseded files stay on disk until the next one lands.

Reading an index-dir growth incident off the logs therefore meant
back-inferring which beats were heavy from the 300s cadence. That was
done once during the storage soak to explain a 13-minute window where
one table's cleanup stalled with no failure logged, and it cost two
35-minute debug-level reruns to confirm — the field makes it a grep.

Adds `pruned` to the conflict log, mirroring the sibling failure log.
Log level (debug) and the benign-conflict semantics are unchanged: the
streak is still not incremented and no fallback rebuild is triggered.

Test asserts both beats in one run: the heavy beat's conflict logs
pruned=True and the light beat's logs pruned=False. Mutation-verified —
dropping the field fails the test with KeyError.

Co-authored-by: zhanghui <zhanghui@shanda.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Elliot Chen <2340896+cyfyifanchen@users.noreply.github.com>
This commit is contained in:
zhanghui
2026-08-05 13:48:15 +08:00
committed by GitHub
co-authored by zhanghui Claude Opus 5 Elliot Chen
parent ace0d79486
commit 72c1facdbf
3 changed files with 89 additions and 0 deletions
+12
View File
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- **`cascade_lancedb_optimize_conflict` now records `pruned`** — which
maintenance beat lost the commit race. Lance labels both beats' commit the
same way (`This Rewrite transaction was preempted by concurrent transaction
…`), so the message alone cannot separate a free loss from a costly one: a
lost **light** beat retries ~10s later, while a lost **heavy** beat means
that table skipped a whole prune cadence and its superseded files stay on
disk. Attributing index-dir growth previously meant back-inferring which
beats were heavy from the 300s cadence. Log level (`debug`) and the
benign-conflict semantics are unchanged — this adds one field.
## [1.2.2] - 2026-08-04
### Added
+12
View File
@@ -827,9 +827,21 @@ class CascadeWorker:
# the right detector for it, and unlike a rebuild it does not
# destroy indexes to "fix" a lost race.
if _is_benign_commit_conflict(exc):
# ``pruned`` is the whole diagnostic value of this line. Lance
# labels both beats' commit the same way ("This Rewrite
# transaction was preempted by ..."), so the message alone
# cannot tell them apart — but the consequences differ:
# a lost LIGHT beat is free (compaction retries ~10s later),
# while a lost HEAVY beat means that table skipped a whole
# prune cadence and its index dir keeps the superseded files.
# Without the flag, reading a disk-growth incident off the
# logs means back-inferring which beats were heavy from the
# 300s cadence (done once during the storage soak — slow and
# fragile). Mirrors ``pruned`` on the sibling failure log.
logger.debug(
"cascade_lancedb_optimize_conflict",
kind=kind,
pruned=should_prune,
error=f"{type(exc).__name__}: {exc}",
)
return
@@ -1071,6 +1071,71 @@ async def test_heavy_beat_commit_conflict_is_also_benign(
)
async def test_conflict_log_names_the_lost_beat(
patched_repo: _FakeRepo,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The conflict log must say WHICH beat lost the race.
Lance labels both beats' commit identically ("This Rewrite transaction
was preempted by concurrent transaction ..."), so the error message
cannot tell them apart — yet the cost differs sharply:
- a lost LIGHT beat is free (compaction retries ~10s later);
- a lost HEAVY beat means that table skipped a whole prune cadence, so
its superseded files stay on disk until the next one lands.
Without ``pruned`` on this line, attributing index-dir growth means
back-inferring which beats were heavy from the 300s cadence.
"""
from everos.memory.cascade import worker as wmod
calls: list[tuple[str, dict[str, object]]] = []
class _SpyLogger:
def __getattr__(self, level: str): # type: ignore[no-untyped-def]
def rec(event: str, **kw) -> None: # type: ignore[no-untyped-def]
calls.append((event, kw))
return rec
monkeypatch.setattr(wmod, "logger", _SpyLogger())
class _ConflictRepo(_FakeLanceRepo):
"""Loses the commit race on both beats."""
async def optimize(self) -> None:
raise RuntimeError("Retryable commit conflict for version 215")
async def prune(self, older_than: dt.timedelta) -> None:
raise RuntimeError("Retryable commit conflict for version 215")
# Freeze the clock: which beat runs must not depend on the runner's uptime
# (`monotonic()` is boot-relative — a fresh CI runner reads ~100s).
monkeypatch.setattr(wmod.time, "monotonic", lambda: 10_000.0)
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(_ConflictRepo())},
retry_backoff_seconds=0,
optimize_prune_interval_seconds=300.0,
)
state = wmod._KindOptimizerState() # last_prune_attempt_at=0 → HEAVY first
w._optimizer_states["episode"] = state
await w._run_optimize_once("episode") # heavy: prune loses the race
# The attempt clock advanced to the (frozen) now, so the cadence has not
# elapsed — the next beat is the light one.
await w._run_optimize_once("episode")
lost = [kw for ev, kw in calls if ev == "cascade_lancedb_optimize_conflict"]
assert len(lost) == 2, "both beats must have logged a conflict"
assert lost[0]["pruned"] is True, (
"the heavy beat lost — this is the one that costs a prune cadence"
)
assert lost[1]["pruned"] is False, (
"the light beat lost — free, the next beat retries it"
)
async def test_non_conflict_failure_counts_even_when_message_says_retryable(
patched_repo: _FakeRepo,
) -> None: