fix(backups): add max_backups retention to bound backup disk usage
mempalace migrate (.pre-migrate.* full-palace copies) and mempalace repair max-seq-id (chroma.sqlite3.max-seq-id-backup-* DB copies) each wrote a fresh, full-size, timestamped backup every run and never deleted the old ones. On a machine that mines or repairs on a schedule, those copies could silently accumulate until they filled the disk. Add a configurable max_backups setting (default 10; env MEMPALACE_MAX_BACKUPS or config.json) and a shared prune_backups helper that trims the oldest copies after each new backup is written. Pruning is keyed by filesystem mtime, scoped strictly to each backup's own naming pattern so live data is never touched, and best-effort so a deletion failure can never abort the migrate/repair that just succeeded. Set max_backups to 0 to keep every backup.
This commit is contained in:
parent
02b8753d97
commit
9be2b97b6c
|
|
@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|||
|
||||
---
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **Backup retention to prevent unbounded disk usage.** `mempalace migrate` (full-palace `<palace>.pre-migrate.<timestamp>` copies) and `mempalace repair max-seq-id` (`chroma.sqlite3.max-seq-id-backup-<timestamp>` copies) each wrote a fresh, full-size, timestamped backup every run and never deleted the old ones. On a machine that mines or repairs on a schedule, those copies could silently accumulate until they filled the disk — one palace was found with hundreds of GB of stale backups beside a few hundred MB of live data, hidden from a normal `du` of the home directory. A new `max_backups` setting (default `10`, env `MEMPALACE_MAX_BACKUPS`, or `config.json`) now prunes the oldest backups after each new one is written. Set it to `0` to keep every backup. Pruning is keyed by filesystem mtime, scoped strictly to each backup's own naming pattern (live data is never touched), and best-effort so a deletion failure can never abort a migration or repair that already succeeded.
|
||||
|
||||
---
|
||||
|
||||
## [3.3.6] — 2026-05-24
|
||||
|
||||
### Features
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
"""Retention pruning for timestamped palace backups.
|
||||
|
||||
``mempalace migrate`` and ``mempalace repair max-seq-id`` each write a fresh,
|
||||
timestamped backup every time they run and historically never deleted the old
|
||||
ones. On a machine that mines or repairs on a schedule those full-size copies
|
||||
accumulate silently — a real palace was found with hundreds of gigabytes of
|
||||
backups sitting beside only a few hundred megabytes of live data, nearly
|
||||
filling the disk. This module prunes the backup set down to a bounded count
|
||||
after each new backup is written.
|
||||
|
||||
The retention count comes from ``MempalaceConfig.max_backups`` (default 10).
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
|
||||
|
||||
def prune_backups(pattern, max_backups, *, log=None):
|
||||
"""Delete the oldest backups matching ``pattern`` so at most ``max_backups`` remain.
|
||||
|
||||
Args:
|
||||
pattern: A glob pattern matching the backup paths (files or
|
||||
directories). The caller is responsible for ``glob.escape``-ing
|
||||
any literal, non-wildcard portion that can contain glob
|
||||
metacharacters — palace paths sometimes do (e.g. a ``[``).
|
||||
max_backups: Number of most-recent backups to keep. ``None`` or any
|
||||
value ``<= 0`` disables pruning and returns immediately, so a
|
||||
backup set is never touched when the user has opted out.
|
||||
log: Optional callable (e.g. ``print``) for human-readable progress.
|
||||
|
||||
Returns:
|
||||
The list of paths that were successfully removed.
|
||||
|
||||
Recency is determined by filesystem mtime rather than by parsing the
|
||||
timestamp out of the name, so it stays correct even when two backup
|
||||
producers use different timestamp formats. Deletion failures are logged
|
||||
and skipped: pruning is best-effort cleanup and must never abort the
|
||||
migrate/repair operation that just completed successfully.
|
||||
"""
|
||||
if max_backups is None or max_backups <= 0:
|
||||
return []
|
||||
|
||||
scored = []
|
||||
for path in glob.glob(pattern):
|
||||
try:
|
||||
scored.append((os.path.getmtime(path), path))
|
||||
except OSError:
|
||||
# Vanished between glob and stat (concurrent prune / cleanup);
|
||||
# nothing for us to remove.
|
||||
continue
|
||||
|
||||
if len(scored) <= max_backups:
|
||||
return []
|
||||
|
||||
# Newest first; the path breaks mtime ties so ordering is deterministic.
|
||||
scored.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
||||
|
||||
removed = []
|
||||
for _mtime, path in scored[max_backups:]:
|
||||
try:
|
||||
if os.path.isdir(path) and not os.path.islink(path):
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
os.remove(path)
|
||||
except OSError as exc:
|
||||
if log:
|
||||
log(f" Backup prune: could not remove {path}: {exc}")
|
||||
continue
|
||||
removed.append(path)
|
||||
if log:
|
||||
log(f" Backup prune: removed old backup {path}")
|
||||
|
||||
return removed
|
||||
|
|
@ -192,6 +192,12 @@ def sanitize_content(value: str, max_length: int = 100_000) -> str:
|
|||
DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
|
||||
DEFAULT_COLLECTION_NAME = "mempalace_drawers"
|
||||
|
||||
# How many timestamped palace backups to retain before the oldest are
|
||||
# pruned. Applies to the accumulating backups written by ``mempalace
|
||||
# migrate`` and ``mempalace repair max-seq-id`` — see
|
||||
# ``MempalaceConfig.max_backups``.
|
||||
DEFAULT_MAX_BACKUPS = 10
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_configured_collection_name() -> str:
|
||||
|
|
@ -586,6 +592,36 @@ class MempalaceConfig:
|
|||
parsed = 1
|
||||
return max(1, parsed)
|
||||
|
||||
@property
|
||||
def max_backups(self) -> int:
|
||||
"""Number of timestamped palace backups to retain before pruning.
|
||||
|
||||
Applies to the accumulating, timestamped backups created by
|
||||
``mempalace migrate`` (``<palace>.pre-migrate.<timestamp>``) and
|
||||
``mempalace repair max-seq-id``
|
||||
(``chroma.sqlite3.max-seq-id-backup-<timestamp>``). Each of those
|
||||
commands writes a fresh full-size copy every run and historically
|
||||
never deleted the old ones, so on a machine that mines or repairs on
|
||||
a schedule the backup set could silently grow until it filled the
|
||||
disk. After each backup is written, copies beyond this count (oldest
|
||||
first) are removed.
|
||||
|
||||
Reads ``MEMPALACE_MAX_BACKUPS`` env first, then ``max_backups`` in
|
||||
``config.json``, then the default of ``10``. A value of ``0`` disables
|
||||
pruning and keeps every backup (use when an external retention policy
|
||||
manages cleanup). Negative or non-numeric values fall back to the
|
||||
default rather than crashing migrate/repair.
|
||||
"""
|
||||
env_val = os.environ.get("MEMPALACE_MAX_BACKUPS")
|
||||
if env_val is not None:
|
||||
coerced = self._try_coerce_int(env_val, minimum=0)
|
||||
if coerced is not None:
|
||||
return coerced
|
||||
coerced = self._try_coerce_int(
|
||||
self._file_config.get("max_backups", DEFAULT_MAX_BACKUPS), minimum=0
|
||||
)
|
||||
return DEFAULT_MAX_BACKUPS if coerced is None else coerced
|
||||
|
||||
@property
|
||||
def hook_silent_save(self):
|
||||
"""Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ Usage:
|
|||
"""
|
||||
|
||||
import errno
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
|
|
@ -28,6 +29,9 @@ from collections import defaultdict
|
|||
from contextlib import closing
|
||||
from datetime import datetime
|
||||
|
||||
from .backups import prune_backups
|
||||
from .config import MempalaceConfig
|
||||
|
||||
|
||||
def _restore_stale_palace(palace_path: str, stale_path: str) -> None:
|
||||
"""Roll back a failed swap.
|
||||
|
|
@ -293,6 +297,16 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
|
|||
print(f"\n Backing up to {backup_path}...")
|
||||
shutil.copytree(palace_path, backup_path)
|
||||
|
||||
# Enforce backup retention so repeated migrations cannot fill the disk
|
||||
# with full-palace copies. The backup we just created is the newest, so
|
||||
# it survives; only older ``.pre-migrate.*`` siblings beyond the limit
|
||||
# are removed. Best-effort — never let cleanup fail the migration.
|
||||
prune_backups(
|
||||
glob.escape(palace_path) + ".pre-migrate.*",
|
||||
MempalaceConfig().max_backups,
|
||||
log=print,
|
||||
)
|
||||
|
||||
# Build fresh palace in a temp directory (avoids chromadb reading old state).
|
||||
# Wrap the whole import-and-swap dance in try/finally so the temp dir is
|
||||
# cleaned up if any of the chromadb writes, the verify count, or the
|
||||
|
|
|
|||
|
|
@ -1529,12 +1529,26 @@ def repair_max_seq_id(
|
|||
return result
|
||||
|
||||
if backup:
|
||||
import glob
|
||||
|
||||
from .backups import prune_backups
|
||||
from .config import MempalaceConfig
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
backup_path = os.path.join(palace_path, f"chroma.sqlite3.max-seq-id-backup-{timestamp}")
|
||||
shutil.copy2(db_path, backup_path)
|
||||
result["backup"] = backup_path
|
||||
print(f" Backup: {backup_path}")
|
||||
|
||||
# Retain only the most recent N backups (the copy just written is the
|
||||
# newest and is kept). Without this, every max-seq-id repair leaves a
|
||||
# full chroma.sqlite3 copy behind that is never cleaned up.
|
||||
prune_backups(
|
||||
os.path.join(glob.escape(palace_path), "chroma.sqlite3.max-seq-id-backup-*"),
|
||||
MempalaceConfig().max_backups,
|
||||
log=print,
|
||||
)
|
||||
|
||||
_close_chroma_handles(palace_path)
|
||||
|
||||
with sqlite3.connect(db_path) as conn:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
"""Tests for backup retention pruning (mempalace.backups.prune_backups).
|
||||
|
||||
These guard the fix for unbounded backup growth: ``mempalace migrate`` and
|
||||
``mempalace repair max-seq-id`` each drop a fresh full-size, timestamped copy
|
||||
every run, and used to never delete the old ones — a palace was found with
|
||||
hundreds of GB of stale backups beside a few hundred MB of live data.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from mempalace.backups import prune_backups
|
||||
|
||||
|
||||
def _make_backup_dir(parent, name, mtime):
|
||||
"""Create a directory backup with a fixed mtime."""
|
||||
path = parent / name
|
||||
path.mkdir()
|
||||
(path / "chroma.sqlite3").write_text("db")
|
||||
os.utime(path, (mtime, mtime))
|
||||
return path
|
||||
|
||||
|
||||
def _make_backup_file(parent, name, mtime):
|
||||
"""Create a file backup with a fixed mtime."""
|
||||
path = parent / name
|
||||
path.write_text("db")
|
||||
os.utime(path, (mtime, mtime))
|
||||
return path
|
||||
|
||||
|
||||
def test_prune_keeps_newest_and_removes_oldest(tmp_path):
|
||||
# 5 backups, mtimes 100..500; keep 2 newest (400, 500).
|
||||
paths = [_make_backup_file(tmp_path, f"b.{i}", mtime=i * 100) for i in range(1, 6)]
|
||||
|
||||
removed = prune_backups(str(tmp_path / "b.*"), max_backups=2)
|
||||
|
||||
surviving = {p.name for p in tmp_path.iterdir()}
|
||||
assert surviving == {"b.4", "b.5"}
|
||||
assert set(removed) == {str(paths[0]), str(paths[1]), str(paths[2])}
|
||||
|
||||
|
||||
def test_prune_removes_directory_backups(tmp_path):
|
||||
"""migrate writes directory backups (full copytree) — must rmtree them."""
|
||||
_make_backup_dir(tmp_path, "palace.pre-migrate.1", mtime=100)
|
||||
_make_backup_dir(tmp_path, "palace.pre-migrate.2", mtime=200)
|
||||
keep = _make_backup_dir(tmp_path, "palace.pre-migrate.3", mtime=300)
|
||||
|
||||
removed = prune_backups(str(tmp_path / "palace.pre-migrate.*"), max_backups=1)
|
||||
|
||||
assert keep.is_dir()
|
||||
assert len(removed) == 2
|
||||
assert not (tmp_path / "palace.pre-migrate.1").exists()
|
||||
assert not (tmp_path / "palace.pre-migrate.2").exists()
|
||||
|
||||
|
||||
def test_prune_noop_when_under_limit(tmp_path):
|
||||
_make_backup_file(tmp_path, "b.1", mtime=100)
|
||||
_make_backup_file(tmp_path, "b.2", mtime=200)
|
||||
|
||||
removed = prune_backups(str(tmp_path / "b.*"), max_backups=10)
|
||||
|
||||
assert removed == []
|
||||
assert len(list(tmp_path.iterdir())) == 2
|
||||
|
||||
|
||||
def test_prune_noop_when_exactly_at_limit(tmp_path):
|
||||
_make_backup_file(tmp_path, "b.1", mtime=100)
|
||||
_make_backup_file(tmp_path, "b.2", mtime=200)
|
||||
|
||||
removed = prune_backups(str(tmp_path / "b.*"), max_backups=2)
|
||||
|
||||
assert removed == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled", [0, -1, None])
|
||||
def test_prune_disabled_keeps_everything(tmp_path, disabled):
|
||||
for i in range(1, 6):
|
||||
_make_backup_file(tmp_path, f"b.{i}", mtime=i * 100)
|
||||
|
||||
removed = prune_backups(str(tmp_path / "b.*"), max_backups=disabled)
|
||||
|
||||
assert removed == []
|
||||
assert len(list(tmp_path.iterdir())) == 5
|
||||
|
||||
|
||||
def test_prune_no_matches(tmp_path):
|
||||
assert prune_backups(str(tmp_path / "nope.*"), max_backups=3) == []
|
||||
|
||||
|
||||
def test_prune_only_touches_matching_pattern(tmp_path):
|
||||
"""Live data and unrelated files must never be swept up by a backup glob."""
|
||||
_make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-1", mtime=100)
|
||||
_make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-2", mtime=200)
|
||||
_make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-3", mtime=300)
|
||||
# The live database and an unrelated file — must survive.
|
||||
live = _make_backup_file(tmp_path, "chroma.sqlite3", mtime=400)
|
||||
other = _make_backup_file(tmp_path, "tunnels.json", mtime=400)
|
||||
|
||||
prune_backups(
|
||||
str(tmp_path / "chroma.sqlite3.max-seq-id-backup-*"),
|
||||
max_backups=1,
|
||||
)
|
||||
|
||||
assert live.exists()
|
||||
assert other.exists()
|
||||
assert (tmp_path / "chroma.sqlite3.max-seq-id-backup-3").exists()
|
||||
assert not (tmp_path / "chroma.sqlite3.max-seq-id-backup-1").exists()
|
||||
assert not (tmp_path / "chroma.sqlite3.max-seq-id-backup-2").exists()
|
||||
|
||||
|
||||
def test_prune_respects_glob_escape_for_metacharacter_paths(tmp_path):
|
||||
"""Palace paths can contain glob metacharacters like ``[``.
|
||||
|
||||
Without ``glob.escape`` the pattern would silently match nothing (the
|
||||
bracket is read as a character class), leaving backups unpruned. Callers
|
||||
escape the literal prefix; this confirms the helper prunes correctly once
|
||||
they do.
|
||||
"""
|
||||
import glob
|
||||
|
||||
weird = tmp_path / "weird[name]"
|
||||
weird.mkdir()
|
||||
for i in range(1, 4):
|
||||
_make_backup_file(weird, f"chroma.sqlite3.max-seq-id-backup-{i}", mtime=i * 100)
|
||||
|
||||
pattern = os.path.join(glob.escape(str(weird)), "chroma.sqlite3.max-seq-id-backup-*")
|
||||
removed = prune_backups(pattern, max_backups=1)
|
||||
|
||||
assert len(removed) == 2
|
||||
assert (weird / "chroma.sqlite3.max-seq-id-backup-3").exists()
|
||||
|
||||
|
||||
def test_prune_is_best_effort_on_delete_failure(tmp_path, monkeypatch):
|
||||
"""A failed deletion is logged and skipped, never raised — pruning must
|
||||
not undo a migrate/repair that already succeeded."""
|
||||
for i in range(1, 5):
|
||||
_make_backup_file(tmp_path, f"b.{i}", mtime=i * 100)
|
||||
|
||||
real_remove = os.remove
|
||||
|
||||
def flaky_remove(path):
|
||||
if path.endswith("b.1"):
|
||||
raise OSError("permission denied")
|
||||
return real_remove(path)
|
||||
|
||||
monkeypatch.setattr(os, "remove", flaky_remove)
|
||||
|
||||
logs = []
|
||||
removed = prune_backups(str(tmp_path / "b.*"), max_backups=2, log=logs.append)
|
||||
|
||||
# b.1 and b.2 were over the limit; b.1 failed, b.2 succeeded.
|
||||
assert str(tmp_path / "b.2") in removed
|
||||
assert str(tmp_path / "b.1") not in removed
|
||||
assert (tmp_path / "b.1").exists()
|
||||
assert any("could not remove" in line for line in logs)
|
||||
|
|
@ -618,3 +618,63 @@ def test_hooks_auto_save_env_override_true():
|
|||
assert cfg.hooks_auto_save is True
|
||||
finally:
|
||||
del os.environ["MEMPALACE_HOOKS_AUTO_SAVE"]
|
||||
|
||||
|
||||
# --- max_backups (backup retention) ---
|
||||
|
||||
|
||||
def test_max_backups_default(monkeypatch):
|
||||
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
|
||||
cfg = MempalaceConfig(config_dir=tempfile.mkdtemp())
|
||||
assert cfg.max_backups == 10
|
||||
|
||||
|
||||
def test_max_backups_from_config(monkeypatch, tmp_path):
|
||||
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
|
||||
with open(tmp_path / "config.json", "w") as f:
|
||||
json.dump({"max_backups": 3}, f)
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert cfg.max_backups == 3
|
||||
|
||||
|
||||
def test_max_backups_zero_disables(monkeypatch, tmp_path):
|
||||
"""0 is a valid, explicit "keep everything" — not garbage."""
|
||||
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
|
||||
with open(tmp_path / "config.json", "w") as f:
|
||||
json.dump({"max_backups": 0}, f)
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert cfg.max_backups == 0
|
||||
|
||||
|
||||
def test_max_backups_env_overrides_config(monkeypatch, tmp_path):
|
||||
with open(tmp_path / "config.json", "w") as f:
|
||||
json.dump({"max_backups": 3}, f)
|
||||
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "7")
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert cfg.max_backups == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", ["abc", "", "-5", "1.5", "true"])
|
||||
def test_max_backups_garbage_falls_back_to_default(monkeypatch, tmp_path, bad):
|
||||
"""A hand-edited bad value must never crash migrate/repair."""
|
||||
with open(tmp_path / "config.json", "w") as f:
|
||||
json.dump({"max_backups": bad}, f)
|
||||
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert cfg.max_backups == 10
|
||||
|
||||
|
||||
def test_max_backups_negative_in_config_falls_back(monkeypatch, tmp_path):
|
||||
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
|
||||
with open(tmp_path / "config.json", "w") as f:
|
||||
json.dump({"max_backups": -3}, f)
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert cfg.max_backups == 10
|
||||
|
||||
|
||||
def test_max_backups_bad_env_falls_back_to_config(monkeypatch, tmp_path):
|
||||
with open(tmp_path / "config.json", "w") as f:
|
||||
json.dump({"max_backups": 4}, f)
|
||||
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "garbage")
|
||||
cfg = MempalaceConfig(config_dir=str(tmp_path))
|
||||
assert cfg.max_backups == 4
|
||||
|
|
|
|||
|
|
@ -286,3 +286,52 @@ def test_migrate_cleans_temp_palace_on_chromadb_failure(tmp_path):
|
|||
assert captured_temp_paths, "mkdtemp was never called — flow short-circuited"
|
||||
for p in captured_temp_paths:
|
||||
assert not os.path.exists(p), f"temp palace was not cleaned up: {p}"
|
||||
|
||||
|
||||
def test_migrate_prunes_old_pre_migrate_backups(tmp_path, monkeypatch):
|
||||
"""Repeated migrations must not accumulate full-palace copies forever.
|
||||
|
||||
The backup + prune happen right after copytree, before the (mocked)
|
||||
chromadb step, so even a migration that fails afterward still trims the
|
||||
backup set. We let copytree run for real so the fresh backup exists on
|
||||
disk for the prune to evaluate.
|
||||
"""
|
||||
palace_dir = tmp_path / "palace"
|
||||
palace_dir.mkdir()
|
||||
(palace_dir / "chroma.sqlite3").write_text("db")
|
||||
|
||||
# Pre-seed 3 stale .pre-migrate.* sibling dirs with old mtimes.
|
||||
for i in range(3):
|
||||
stale = tmp_path / f"palace.pre-migrate.2026010{i}_000000"
|
||||
stale.mkdir()
|
||||
(stale / "chroma.sqlite3").write_text("old")
|
||||
os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i))
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "2")
|
||||
|
||||
failing_backend = MagicMock()
|
||||
failing_backend.get_collection.side_effect = Exception("unreadable")
|
||||
failing_backend.get_or_create_collection.side_effect = RuntimeError("chromadb boom")
|
||||
|
||||
import mempalace.backends.chroma as _chroma_mod
|
||||
|
||||
with (
|
||||
patch("mempalace.migrate.detect_chromadb_version", return_value="0.5.x"),
|
||||
patch(
|
||||
"mempalace.migrate.extract_drawers_from_sqlite",
|
||||
return_value=[{"id": "id1", "document": "doc", "metadata": {"wing": "w", "room": "r"}}],
|
||||
),
|
||||
patch("builtins.input", return_value="y"),
|
||||
patch.object(_chroma_mod, "ChromaBackend", return_value=failing_backend),
|
||||
):
|
||||
try:
|
||||
migrate(str(palace_dir), confirm=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
backups = sorted(p.name for p in tmp_path.glob("palace.pre-migrate.*"))
|
||||
# 3 stale + 1 fresh = 4 created; retention keeps only the 2 newest.
|
||||
assert len(backups) == 2
|
||||
# The two oldest stale backups must be gone.
|
||||
assert "palace.pre-migrate.20260100_000000" not in backups
|
||||
assert "palace.pre-migrate.20260101_000000" not in backups
|
||||
|
|
|
|||
|
|
@ -1240,6 +1240,57 @@ def test_max_seq_id_backup_created(tmp_path):
|
|||
assert rows[seg["drawers_meta"]] == seg["poisoned_values"][seg["drawers_meta"]]
|
||||
|
||||
|
||||
def test_max_seq_id_backup_pruned_to_max_backups(tmp_path, monkeypatch):
|
||||
"""Old max-seq-id backups beyond MEMPALACE_MAX_BACKUPS are pruned after a repair.
|
||||
|
||||
Without retention, every repair left a full chroma.sqlite3 copy behind
|
||||
that was never cleaned up — the unbounded disk-growth bug this guards.
|
||||
"""
|
||||
palace = str(tmp_path / "palace")
|
||||
_seed_poisoned_max_seq_id(palace)
|
||||
|
||||
# Pre-seed 4 stale backups with old mtimes so the just-created one is
|
||||
# unambiguously the newest.
|
||||
for i in range(4):
|
||||
stale = os.path.join(palace, f"chroma.sqlite3.max-seq-id-backup-2026010{i}-000000")
|
||||
with open(stale, "w") as f:
|
||||
f.write("old")
|
||||
os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i))
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "2")
|
||||
|
||||
result = repair.repair_max_seq_id(palace, assume_yes=True)
|
||||
|
||||
backups = sorted(
|
||||
fn for fn in os.listdir(palace) if fn.startswith("chroma.sqlite3.max-seq-id-backup-")
|
||||
)
|
||||
# 4 stale + 1 fresh = 5 written; retention keeps only the 2 newest.
|
||||
assert len(backups) == 2
|
||||
# The backup created by this repair must be one of the survivors.
|
||||
assert os.path.basename(result["backup"]) in backups
|
||||
|
||||
|
||||
def test_max_seq_id_backup_retained_when_pruning_disabled(tmp_path, monkeypatch):
|
||||
"""max_backups=0 keeps every backup (opt-out for external retention)."""
|
||||
palace = str(tmp_path / "palace")
|
||||
_seed_poisoned_max_seq_id(palace)
|
||||
|
||||
for i in range(3):
|
||||
stale = os.path.join(palace, f"chroma.sqlite3.max-seq-id-backup-2026010{i}-000000")
|
||||
with open(stale, "w") as f:
|
||||
f.write("old")
|
||||
os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i))
|
||||
|
||||
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "0")
|
||||
|
||||
repair.repair_max_seq_id(palace, assume_yes=True)
|
||||
|
||||
backups = [
|
||||
fn for fn in os.listdir(palace) if fn.startswith("chroma.sqlite3.max-seq-id-backup-")
|
||||
]
|
||||
assert len(backups) == 4
|
||||
|
||||
|
||||
def test_max_seq_id_rollback_on_verification_failure(tmp_path, monkeypatch):
|
||||
"""If the post-update detector still sees poison, raise and leave a backup."""
|
||||
palace = str(tmp_path / "palace")
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ Located at `~/.mempalace/config.json`:
|
|||
{
|
||||
"palace_path": "/custom/path/to/palace",
|
||||
"collection_name": "mempalace_drawers",
|
||||
"people_map": {"Kai": "KAI", "Priya": "PRI"}
|
||||
"people_map": {"Kai": "KAI", "Priya": "PRI"},
|
||||
"max_backups": 10
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -17,6 +18,7 @@ Located at `~/.mempalace/config.json`:
|
|||
| `palace_path` | `~/.mempalace/palace` | Where ChromaDB stores your drawers |
|
||||
| `collection_name` | `mempalace_drawers` | ChromaDB collection name |
|
||||
| `people_map` | `{}` | Entity name → AAAK code mappings |
|
||||
| `max_backups` | `10` | How many timestamped palace backups to keep before the oldest are pruned. Applies to `mempalace migrate` (`<palace>.pre-migrate.*`) and `mempalace repair max-seq-id` (`chroma.sqlite3.max-seq-id-backup-*`), which each write a full copy every run. Set to `0` to keep every backup (e.g. when an external retention policy manages cleanup). |
|
||||
|
||||
## Project Config
|
||||
|
||||
|
|
@ -83,3 +85,4 @@ python -m mempalace.mcp_server --palace /custom/palace
|
|||
|----------|-------------|
|
||||
| `MEMPALACE_PALACE_PATH` | Override palace path (same as `--palace`) |
|
||||
| `MEMPAL_DIR` | Directory for auto-mining in hooks |
|
||||
| `MEMPALACE_MAX_BACKUPS` | Override `max_backups` retention count (`0` disables pruning) |
|
||||
|
|
|
|||
Loading…
Reference in New Issue