fix(backups): add max_backups retention to bound backup disk usage

mempalace migrate (.pre-migrate.* full-palace copies) and mempalace repair
max-seq-id (chroma.sqlite3.max-seq-id-backup-* DB copies) each wrote a fresh,
full-size, timestamped backup every run and never deleted the old ones. On a
machine that mines or repairs on a schedule, those copies could silently
accumulate until they filled the disk.

Add a configurable max_backups setting (default 10; env MEMPALACE_MAX_BACKUPS
or config.json) and a shared prune_backups helper that trims the oldest copies
after each new backup is written. Pruning is keyed by filesystem mtime, scoped
strictly to each backup's own naming pattern so live data is never touched, and
best-effort so a deletion failure can never abort the migrate/repair that just
succeeded. Set max_backups to 0 to keep every backup.
This commit is contained in:
margaretjgu 2026-06-05 14:52:06 -04:00
parent 02b8753d97
commit 9be2b97b6c
10 changed files with 467 additions and 1 deletions

View File

@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
---
## [Unreleased]
### Bug Fixes
- **Backup retention to prevent unbounded disk usage.** `mempalace migrate` (full-palace `<palace>.pre-migrate.<timestamp>` copies) and `mempalace repair max-seq-id` (`chroma.sqlite3.max-seq-id-backup-<timestamp>` copies) each wrote a fresh, full-size, timestamped backup every run and never deleted the old ones. On a machine that mines or repairs on a schedule, those copies could silently accumulate until they filled the disk — one palace was found with hundreds of GB of stale backups beside a few hundred MB of live data, hidden from a normal `du` of the home directory. A new `max_backups` setting (default `10`, env `MEMPALACE_MAX_BACKUPS`, or `config.json`) now prunes the oldest backups after each new one is written. Set it to `0` to keep every backup. Pruning is keyed by filesystem mtime, scoped strictly to each backup's own naming pattern (live data is never touched), and best-effort so a deletion failure can never abort a migration or repair that already succeeded.
---
## [3.3.6] — 2026-05-24
### Features

74
mempalace/backups.py Normal file
View File

@ -0,0 +1,74 @@
"""Retention pruning for timestamped palace backups.
``mempalace migrate`` and ``mempalace repair max-seq-id`` each write a fresh,
timestamped backup every time they run and historically never deleted the old
ones. On a machine that mines or repairs on a schedule those full-size copies
accumulate silently a real palace was found with hundreds of gigabytes of
backups sitting beside only a few hundred megabytes of live data, nearly
filling the disk. This module prunes the backup set down to a bounded count
after each new backup is written.
The retention count comes from ``MempalaceConfig.max_backups`` (default 10).
"""
import glob
import os
import shutil
def prune_backups(pattern, max_backups, *, log=None):
"""Delete the oldest backups matching ``pattern`` so at most ``max_backups`` remain.
Args:
pattern: A glob pattern matching the backup paths (files or
directories). The caller is responsible for ``glob.escape``-ing
any literal, non-wildcard portion that can contain glob
metacharacters palace paths sometimes do (e.g. a ``[``).
max_backups: Number of most-recent backups to keep. ``None`` or any
value ``<= 0`` disables pruning and returns immediately, so a
backup set is never touched when the user has opted out.
log: Optional callable (e.g. ``print``) for human-readable progress.
Returns:
The list of paths that were successfully removed.
Recency is determined by filesystem mtime rather than by parsing the
timestamp out of the name, so it stays correct even when two backup
producers use different timestamp formats. Deletion failures are logged
and skipped: pruning is best-effort cleanup and must never abort the
migrate/repair operation that just completed successfully.
"""
if max_backups is None or max_backups <= 0:
return []
scored = []
for path in glob.glob(pattern):
try:
scored.append((os.path.getmtime(path), path))
except OSError:
# Vanished between glob and stat (concurrent prune / cleanup);
# nothing for us to remove.
continue
if len(scored) <= max_backups:
return []
# Newest first; the path breaks mtime ties so ordering is deterministic.
scored.sort(key=lambda item: (item[0], item[1]), reverse=True)
removed = []
for _mtime, path in scored[max_backups:]:
try:
if os.path.isdir(path) and not os.path.islink(path):
shutil.rmtree(path)
else:
os.remove(path)
except OSError as exc:
if log:
log(f" Backup prune: could not remove {path}: {exc}")
continue
removed.append(path)
if log:
log(f" Backup prune: removed old backup {path}")
return removed

View File

@ -192,6 +192,12 @@ def sanitize_content(value: str, max_length: int = 100_000) -> str:
DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
DEFAULT_COLLECTION_NAME = "mempalace_drawers"
# How many timestamped palace backups to retain before the oldest are
# pruned. Applies to the accumulating backups written by ``mempalace
# migrate`` and ``mempalace repair max-seq-id`` — see
# ``MempalaceConfig.max_backups``.
DEFAULT_MAX_BACKUPS = 10
@lru_cache(maxsize=1)
def get_configured_collection_name() -> str:
@ -586,6 +592,36 @@ class MempalaceConfig:
parsed = 1
return max(1, parsed)
@property
def max_backups(self) -> int:
"""Number of timestamped palace backups to retain before pruning.
Applies to the accumulating, timestamped backups created by
``mempalace migrate`` (``<palace>.pre-migrate.<timestamp>``) and
``mempalace repair max-seq-id``
(``chroma.sqlite3.max-seq-id-backup-<timestamp>``). Each of those
commands writes a fresh full-size copy every run and historically
never deleted the old ones, so on a machine that mines or repairs on
a schedule the backup set could silently grow until it filled the
disk. After each backup is written, copies beyond this count (oldest
first) are removed.
Reads ``MEMPALACE_MAX_BACKUPS`` env first, then ``max_backups`` in
``config.json``, then the default of ``10``. A value of ``0`` disables
pruning and keeps every backup (use when an external retention policy
manages cleanup). Negative or non-numeric values fall back to the
default rather than crashing migrate/repair.
"""
env_val = os.environ.get("MEMPALACE_MAX_BACKUPS")
if env_val is not None:
coerced = self._try_coerce_int(env_val, minimum=0)
if coerced is not None:
return coerced
coerced = self._try_coerce_int(
self._file_config.get("max_backups", DEFAULT_MAX_BACKUPS), minimum=0
)
return DEFAULT_MAX_BACKUPS if coerced is None else coerced
@property
def hook_silent_save(self):
"""Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""

View File

@ -19,6 +19,7 @@ Usage:
"""
import errno
import glob
import os
import shutil
import sqlite3
@ -28,6 +29,9 @@ from collections import defaultdict
from contextlib import closing
from datetime import datetime
from .backups import prune_backups
from .config import MempalaceConfig
def _restore_stale_palace(palace_path: str, stale_path: str) -> None:
"""Roll back a failed swap.
@ -293,6 +297,16 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
print(f"\n Backing up to {backup_path}...")
shutil.copytree(palace_path, backup_path)
# Enforce backup retention so repeated migrations cannot fill the disk
# with full-palace copies. The backup we just created is the newest, so
# it survives; only older ``.pre-migrate.*`` siblings beyond the limit
# are removed. Best-effort — never let cleanup fail the migration.
prune_backups(
glob.escape(palace_path) + ".pre-migrate.*",
MempalaceConfig().max_backups,
log=print,
)
# Build fresh palace in a temp directory (avoids chromadb reading old state).
# Wrap the whole import-and-swap dance in try/finally so the temp dir is
# cleaned up if any of the chromadb writes, the verify count, or the

View File

@ -1529,12 +1529,26 @@ def repair_max_seq_id(
return result
if backup:
import glob
from .backups import prune_backups
from .config import MempalaceConfig
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
backup_path = os.path.join(palace_path, f"chroma.sqlite3.max-seq-id-backup-{timestamp}")
shutil.copy2(db_path, backup_path)
result["backup"] = backup_path
print(f" Backup: {backup_path}")
# Retain only the most recent N backups (the copy just written is the
# newest and is kept). Without this, every max-seq-id repair leaves a
# full chroma.sqlite3 copy behind that is never cleaned up.
prune_backups(
os.path.join(glob.escape(palace_path), "chroma.sqlite3.max-seq-id-backup-*"),
MempalaceConfig().max_backups,
log=print,
)
_close_chroma_handles(palace_path)
with sqlite3.connect(db_path) as conn:

157
tests/test_backups.py Normal file
View File

@ -0,0 +1,157 @@
"""Tests for backup retention pruning (mempalace.backups.prune_backups).
These guard the fix for unbounded backup growth: ``mempalace migrate`` and
``mempalace repair max-seq-id`` each drop a fresh full-size, timestamped copy
every run, and used to never delete the old ones a palace was found with
hundreds of GB of stale backups beside a few hundred MB of live data.
"""
import os
import pytest
from mempalace.backups import prune_backups
def _make_backup_dir(parent, name, mtime):
"""Create a directory backup with a fixed mtime."""
path = parent / name
path.mkdir()
(path / "chroma.sqlite3").write_text("db")
os.utime(path, (mtime, mtime))
return path
def _make_backup_file(parent, name, mtime):
"""Create a file backup with a fixed mtime."""
path = parent / name
path.write_text("db")
os.utime(path, (mtime, mtime))
return path
def test_prune_keeps_newest_and_removes_oldest(tmp_path):
# 5 backups, mtimes 100..500; keep 2 newest (400, 500).
paths = [_make_backup_file(tmp_path, f"b.{i}", mtime=i * 100) for i in range(1, 6)]
removed = prune_backups(str(tmp_path / "b.*"), max_backups=2)
surviving = {p.name for p in tmp_path.iterdir()}
assert surviving == {"b.4", "b.5"}
assert set(removed) == {str(paths[0]), str(paths[1]), str(paths[2])}
def test_prune_removes_directory_backups(tmp_path):
"""migrate writes directory backups (full copytree) — must rmtree them."""
_make_backup_dir(tmp_path, "palace.pre-migrate.1", mtime=100)
_make_backup_dir(tmp_path, "palace.pre-migrate.2", mtime=200)
keep = _make_backup_dir(tmp_path, "palace.pre-migrate.3", mtime=300)
removed = prune_backups(str(tmp_path / "palace.pre-migrate.*"), max_backups=1)
assert keep.is_dir()
assert len(removed) == 2
assert not (tmp_path / "palace.pre-migrate.1").exists()
assert not (tmp_path / "palace.pre-migrate.2").exists()
def test_prune_noop_when_under_limit(tmp_path):
_make_backup_file(tmp_path, "b.1", mtime=100)
_make_backup_file(tmp_path, "b.2", mtime=200)
removed = prune_backups(str(tmp_path / "b.*"), max_backups=10)
assert removed == []
assert len(list(tmp_path.iterdir())) == 2
def test_prune_noop_when_exactly_at_limit(tmp_path):
_make_backup_file(tmp_path, "b.1", mtime=100)
_make_backup_file(tmp_path, "b.2", mtime=200)
removed = prune_backups(str(tmp_path / "b.*"), max_backups=2)
assert removed == []
@pytest.mark.parametrize("disabled", [0, -1, None])
def test_prune_disabled_keeps_everything(tmp_path, disabled):
for i in range(1, 6):
_make_backup_file(tmp_path, f"b.{i}", mtime=i * 100)
removed = prune_backups(str(tmp_path / "b.*"), max_backups=disabled)
assert removed == []
assert len(list(tmp_path.iterdir())) == 5
def test_prune_no_matches(tmp_path):
assert prune_backups(str(tmp_path / "nope.*"), max_backups=3) == []
def test_prune_only_touches_matching_pattern(tmp_path):
"""Live data and unrelated files must never be swept up by a backup glob."""
_make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-1", mtime=100)
_make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-2", mtime=200)
_make_backup_file(tmp_path, "chroma.sqlite3.max-seq-id-backup-3", mtime=300)
# The live database and an unrelated file — must survive.
live = _make_backup_file(tmp_path, "chroma.sqlite3", mtime=400)
other = _make_backup_file(tmp_path, "tunnels.json", mtime=400)
prune_backups(
str(tmp_path / "chroma.sqlite3.max-seq-id-backup-*"),
max_backups=1,
)
assert live.exists()
assert other.exists()
assert (tmp_path / "chroma.sqlite3.max-seq-id-backup-3").exists()
assert not (tmp_path / "chroma.sqlite3.max-seq-id-backup-1").exists()
assert not (tmp_path / "chroma.sqlite3.max-seq-id-backup-2").exists()
def test_prune_respects_glob_escape_for_metacharacter_paths(tmp_path):
"""Palace paths can contain glob metacharacters like ``[``.
Without ``glob.escape`` the pattern would silently match nothing (the
bracket is read as a character class), leaving backups unpruned. Callers
escape the literal prefix; this confirms the helper prunes correctly once
they do.
"""
import glob
weird = tmp_path / "weird[name]"
weird.mkdir()
for i in range(1, 4):
_make_backup_file(weird, f"chroma.sqlite3.max-seq-id-backup-{i}", mtime=i * 100)
pattern = os.path.join(glob.escape(str(weird)), "chroma.sqlite3.max-seq-id-backup-*")
removed = prune_backups(pattern, max_backups=1)
assert len(removed) == 2
assert (weird / "chroma.sqlite3.max-seq-id-backup-3").exists()
def test_prune_is_best_effort_on_delete_failure(tmp_path, monkeypatch):
"""A failed deletion is logged and skipped, never raised — pruning must
not undo a migrate/repair that already succeeded."""
for i in range(1, 5):
_make_backup_file(tmp_path, f"b.{i}", mtime=i * 100)
real_remove = os.remove
def flaky_remove(path):
if path.endswith("b.1"):
raise OSError("permission denied")
return real_remove(path)
monkeypatch.setattr(os, "remove", flaky_remove)
logs = []
removed = prune_backups(str(tmp_path / "b.*"), max_backups=2, log=logs.append)
# b.1 and b.2 were over the limit; b.1 failed, b.2 succeeded.
assert str(tmp_path / "b.2") in removed
assert str(tmp_path / "b.1") not in removed
assert (tmp_path / "b.1").exists()
assert any("could not remove" in line for line in logs)

View File

@ -618,3 +618,63 @@ def test_hooks_auto_save_env_override_true():
assert cfg.hooks_auto_save is True
finally:
del os.environ["MEMPALACE_HOOKS_AUTO_SAVE"]
# --- max_backups (backup retention) ---
def test_max_backups_default(monkeypatch):
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
cfg = MempalaceConfig(config_dir=tempfile.mkdtemp())
assert cfg.max_backups == 10
def test_max_backups_from_config(monkeypatch, tmp_path):
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
with open(tmp_path / "config.json", "w") as f:
json.dump({"max_backups": 3}, f)
cfg = MempalaceConfig(config_dir=str(tmp_path))
assert cfg.max_backups == 3
def test_max_backups_zero_disables(monkeypatch, tmp_path):
"""0 is a valid, explicit "keep everything" — not garbage."""
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
with open(tmp_path / "config.json", "w") as f:
json.dump({"max_backups": 0}, f)
cfg = MempalaceConfig(config_dir=str(tmp_path))
assert cfg.max_backups == 0
def test_max_backups_env_overrides_config(monkeypatch, tmp_path):
with open(tmp_path / "config.json", "w") as f:
json.dump({"max_backups": 3}, f)
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "7")
cfg = MempalaceConfig(config_dir=str(tmp_path))
assert cfg.max_backups == 7
@pytest.mark.parametrize("bad", ["abc", "", "-5", "1.5", "true"])
def test_max_backups_garbage_falls_back_to_default(monkeypatch, tmp_path, bad):
"""A hand-edited bad value must never crash migrate/repair."""
with open(tmp_path / "config.json", "w") as f:
json.dump({"max_backups": bad}, f)
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
cfg = MempalaceConfig(config_dir=str(tmp_path))
assert cfg.max_backups == 10
def test_max_backups_negative_in_config_falls_back(monkeypatch, tmp_path):
monkeypatch.delenv("MEMPALACE_MAX_BACKUPS", raising=False)
with open(tmp_path / "config.json", "w") as f:
json.dump({"max_backups": -3}, f)
cfg = MempalaceConfig(config_dir=str(tmp_path))
assert cfg.max_backups == 10
def test_max_backups_bad_env_falls_back_to_config(monkeypatch, tmp_path):
with open(tmp_path / "config.json", "w") as f:
json.dump({"max_backups": 4}, f)
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "garbage")
cfg = MempalaceConfig(config_dir=str(tmp_path))
assert cfg.max_backups == 4

View File

@ -286,3 +286,52 @@ def test_migrate_cleans_temp_palace_on_chromadb_failure(tmp_path):
assert captured_temp_paths, "mkdtemp was never called — flow short-circuited"
for p in captured_temp_paths:
assert not os.path.exists(p), f"temp palace was not cleaned up: {p}"
def test_migrate_prunes_old_pre_migrate_backups(tmp_path, monkeypatch):
"""Repeated migrations must not accumulate full-palace copies forever.
The backup + prune happen right after copytree, before the (mocked)
chromadb step, so even a migration that fails afterward still trims the
backup set. We let copytree run for real so the fresh backup exists on
disk for the prune to evaluate.
"""
palace_dir = tmp_path / "palace"
palace_dir.mkdir()
(palace_dir / "chroma.sqlite3").write_text("db")
# Pre-seed 3 stale .pre-migrate.* sibling dirs with old mtimes.
for i in range(3):
stale = tmp_path / f"palace.pre-migrate.2026010{i}_000000"
stale.mkdir()
(stale / "chroma.sqlite3").write_text("old")
os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i))
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "2")
failing_backend = MagicMock()
failing_backend.get_collection.side_effect = Exception("unreadable")
failing_backend.get_or_create_collection.side_effect = RuntimeError("chromadb boom")
import mempalace.backends.chroma as _chroma_mod
with (
patch("mempalace.migrate.detect_chromadb_version", return_value="0.5.x"),
patch(
"mempalace.migrate.extract_drawers_from_sqlite",
return_value=[{"id": "id1", "document": "doc", "metadata": {"wing": "w", "room": "r"}}],
),
patch("builtins.input", return_value="y"),
patch.object(_chroma_mod, "ChromaBackend", return_value=failing_backend),
):
try:
migrate(str(palace_dir), confirm=True)
except Exception:
pass
backups = sorted(p.name for p in tmp_path.glob("palace.pre-migrate.*"))
# 3 stale + 1 fresh = 4 created; retention keeps only the 2 newest.
assert len(backups) == 2
# The two oldest stale backups must be gone.
assert "palace.pre-migrate.20260100_000000" not in backups
assert "palace.pre-migrate.20260101_000000" not in backups

View File

@ -1240,6 +1240,57 @@ def test_max_seq_id_backup_created(tmp_path):
assert rows[seg["drawers_meta"]] == seg["poisoned_values"][seg["drawers_meta"]]
def test_max_seq_id_backup_pruned_to_max_backups(tmp_path, monkeypatch):
"""Old max-seq-id backups beyond MEMPALACE_MAX_BACKUPS are pruned after a repair.
Without retention, every repair left a full chroma.sqlite3 copy behind
that was never cleaned up the unbounded disk-growth bug this guards.
"""
palace = str(tmp_path / "palace")
_seed_poisoned_max_seq_id(palace)
# Pre-seed 4 stale backups with old mtimes so the just-created one is
# unambiguously the newest.
for i in range(4):
stale = os.path.join(palace, f"chroma.sqlite3.max-seq-id-backup-2026010{i}-000000")
with open(stale, "w") as f:
f.write("old")
os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i))
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "2")
result = repair.repair_max_seq_id(palace, assume_yes=True)
backups = sorted(
fn for fn in os.listdir(palace) if fn.startswith("chroma.sqlite3.max-seq-id-backup-")
)
# 4 stale + 1 fresh = 5 written; retention keeps only the 2 newest.
assert len(backups) == 2
# The backup created by this repair must be one of the survivors.
assert os.path.basename(result["backup"]) in backups
def test_max_seq_id_backup_retained_when_pruning_disabled(tmp_path, monkeypatch):
"""max_backups=0 keeps every backup (opt-out for external retention)."""
palace = str(tmp_path / "palace")
_seed_poisoned_max_seq_id(palace)
for i in range(3):
stale = os.path.join(palace, f"chroma.sqlite3.max-seq-id-backup-2026010{i}-000000")
with open(stale, "w") as f:
f.write("old")
os.utime(stale, (1_700_000_000 + i, 1_700_000_000 + i))
monkeypatch.setenv("MEMPALACE_MAX_BACKUPS", "0")
repair.repair_max_seq_id(palace, assume_yes=True)
backups = [
fn for fn in os.listdir(palace) if fn.startswith("chroma.sqlite3.max-seq-id-backup-")
]
assert len(backups) == 4
def test_max_seq_id_rollback_on_verification_failure(tmp_path, monkeypatch):
"""If the post-update detector still sees poison, raise and leave a backup."""
palace = str(tmp_path / "palace")

View File

@ -8,7 +8,8 @@ Located at `~/.mempalace/config.json`:
{
"palace_path": "/custom/path/to/palace",
"collection_name": "mempalace_drawers",
"people_map": {"Kai": "KAI", "Priya": "PRI"}
"people_map": {"Kai": "KAI", "Priya": "PRI"},
"max_backups": 10
}
```
@ -17,6 +18,7 @@ Located at `~/.mempalace/config.json`:
| `palace_path` | `~/.mempalace/palace` | Where ChromaDB stores your drawers |
| `collection_name` | `mempalace_drawers` | ChromaDB collection name |
| `people_map` | `{}` | Entity name → AAAK code mappings |
| `max_backups` | `10` | How many timestamped palace backups to keep before the oldest are pruned. Applies to `mempalace migrate` (`<palace>.pre-migrate.*`) and `mempalace repair max-seq-id` (`chroma.sqlite3.max-seq-id-backup-*`), which each write a full copy every run. Set to `0` to keep every backup (e.g. when an external retention policy manages cleanup). |
## Project Config
@ -83,3 +85,4 @@ python -m mempalace.mcp_server --palace /custom/palace
|----------|-------------|
| `MEMPALACE_PALACE_PATH` | Override palace path (same as `--palace`) |
| `MEMPAL_DIR` | Directory for auto-mining in hooks |
| `MEMPALACE_MAX_BACKUPS` | Override `max_backups` retention count (`0` disables pruning) |