diff --git a/docs/authored-at.md b/docs/authored-at.md new file mode 100644 index 0000000..6f4d72f --- /dev/null +++ b/docs/authored-at.md @@ -0,0 +1,45 @@ +# Authored date (`authored_at`) + +Conversation transcripts carry a per-line ISO-8601 `timestamp` (both Claude Code and +Codex JSONL). The miner records the most recent one per file as the drawer's +**`authored_at`** — when the content was actually written. + +This is distinct from the ingest date: + +| Field | Meaning | +|-------|---------| +| `filed_at` / result `created_at` | When the drawer was **mined** (written to the palace). A bulk re-mine collapses these to a single instant. | +| `authored_at` | When the underlying content was **written**, recovered from the transcript timestamps. Survives re-mining. | + +`authored_at` is surfaced in search results (and shown in the CLI `search` output), and is +used as a deterministic tie-break in hybrid ranking: candidates with identical scores order +with the more recently authored drawer first. Drawers without per-line timestamps (e.g. +markdown) fall back to `filed_at`. + +## Backfilling existing memory + +New mines populate `authored_at` automatically. Drawers mined before this feature only have +`filed_at`. Re-mining does **not** fix them — the scanner skips files already mined at the +current `NORMALIZE_VERSION`. Two options: + +1. **In-place backfill (recommended — no re-embedding).** `scripts/backfill_authored_at.py` + reads each convos drawer's source transcript and updates only the `authored_at` metadata. + Idempotent and safe to re-run; embeddings are untouched. + + ```bash + python scripts/backfill_authored_at.py \ + --palace ~/.mempalace/palace \ + --sessions ~/.claude --sessions ~/.codex # dry run + python scripts/backfill_authored_at.py \ + --palace ~/.mempalace/palace \ + --sessions ~/.claude --sessions ~/.codex --apply # write + ``` + + For the Docker MCP image, mount the volume and session dirs read-only — see the header of + `scripts/backfill_authored_at.py` for the exact `docker run` invocation. + + > Back up first: `tar czf palace-backup.tgz -C .` (or snapshot the + > `mempalace-data` volume). + +2. **Drop and recreate.** Delete the affected drawers and re-mine the transcripts; the fresh + mine stamps `authored_at`. Simpler, but re-embeds everything. diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 21f43d4..2da32db 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -10,6 +10,7 @@ Same palace as project mining. Different ingest strategy. import os import sys +import json import logging import stat from pathlib import Path @@ -408,7 +409,42 @@ def scan_convos(convo_dir: str) -> list: # ============================================================================= -def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extract_mode): +def _extract_authored_at(filepath): + """Most-recent message timestamp in a transcript, used as the drawer's authored date. + + Both Claude Code and Codex JSONL transcripts carry a top-level ISO-8601 + ``timestamp`` on each line. We take the max so ``authored_at`` reflects when the + content was actually written, independent of when it was mined (``filed_at``). + This restores chronology: a session from days ago keeps its real date even when + re-mined today, instead of every drawer collapsing to ingest time. Returns None + for formats without per-line timestamps (e.g. plain ``.md``). + """ + path = Path(filepath) + if path.suffix != ".jsonl": + return None + latest = None + try: + with path.open(encoding="utf-8", errors="ignore") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + ts = json.loads(line).get("timestamp") + except (ValueError, TypeError, AttributeError): + continue + # ISO-8601 timestamps are strings; guard against a non-string + # ``timestamp`` so a malformed line can't raise TypeError on compare. + if isinstance(ts, str) and (latest is None or ts > latest): + latest = ts + except OSError: + return None + return latest + + +def _file_chunks_locked( + collection, source_file, chunks, wing, room, agent, extract_mode, authored_at=None +): """Lock the source file, purge stale drawers, and upsert fresh chunks. Combines the per-file serialization that prevents concurrent agents from @@ -463,6 +499,7 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr "chunk_index": chunk["chunk_index"], "added_by": agent, "filed_at": filed_at, + "authored_at": authored_at if authored_at is not None else filed_at, "ingest_mode": "convos", "extract_mode": extract_mode, "normalize_version": NORMALIZE_VERSION, @@ -726,7 +763,14 @@ def _mine_convos_impl( # Lock + purge stale + file fresh chunks. Lock serializes concurrent # agents; purge removes pre-v2 drawers so the schema bump applies. drawers_added, room_delta, skipped = _file_chunks_locked( - collection, source_file, chunks, wing, room, agent, extract_mode + collection, + source_file, + chunks, + wing, + room, + agent, + extract_mode, + authored_at=_extract_authored_at(filepath), ) if skipped: files_skipped += 1 diff --git a/mempalace/layers.py b/mempalace/layers.py index 6acf852..1d6ceeb 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -306,6 +306,9 @@ class Layer3: lines.append(f" {snippet}") if source: lines.append(f" src: {source}") + authored = (meta.get("authored_at") or "")[:10] + if authored: + lines.append(f" authored: {authored}") return "\n".join(lines) diff --git a/mempalace/searcher.py b/mempalace/searcher.py index eac488f..417ec7c 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -221,7 +221,18 @@ def _hybrid_rank( r["bm25_score"] = round(raw, 3) scored.append((vector_weight * vec_sim + bm25_weight * norm, r)) - scored.sort(key=lambda pair: pair[0], reverse=True) + # Break exact score ties toward the more recently authored drawer so equal-score + # candidates rank chronologically instead of in arbitrary backend order. ISO-8601 + # ``authored_at`` strings sort chronologically; missing dates sort oldest. + # authored_at lives at the top level on the search_memories path and nested under + # "metadata" on the candidate-union path; check both so the tie-break works for each. + scored.sort( + key=lambda pair: ( + pair[0], + pair[1].get("authored_at") or pair[1].get("metadata", {}).get("authored_at") or "", + ), + reverse=True, + ) results[:] = [r for _, r in scored] return results @@ -681,6 +692,7 @@ def _bm25_only_via_sqlite( "source_file": Path(full_source).name if full_source else "?", "source_path": full_source, "created_at": meta.get("filed_at", "unknown"), + "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")), # No vector distance available in BM25-only mode. "similarity": None, "distance": None, @@ -783,6 +795,7 @@ def _merge_bm25_union_candidates( "source_file": Path(full_source).name if full_source else "?", "source_path": full_source, "created_at": meta.get("filed_at", "unknown"), + "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")), "similarity": None, "distance": None, "effective_distance": None, @@ -1198,6 +1211,7 @@ def search_memories( "source_file": Path(source).name if source else "?", "source_path": source, "created_at": meta.get("filed_at", "unknown"), + "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")), "similarity": round(_distance_to_similarity(effective_dist, metric), 3), "distance": round(dist, 4), "effective_distance": round(effective_dist, 4), diff --git a/scripts/backfill_authored_at.py b/scripts/backfill_authored_at.py new file mode 100644 index 0000000..e3b1db2 --- /dev/null +++ b/scripts/backfill_authored_at.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Backfill ``authored_at`` onto existing conversation drawers. + +New mines stamp ``authored_at`` automatically (see ``convo_miner._extract_authored_at``), +but drawers mined before that change only have ``filed_at`` (ingest time). Re-mining does +NOT fix them: the scanner skips files already mined at the current ``NORMALIZE_VERSION``. + +This migration updates the affected drawers IN PLACE — metadata only, embeddings are left +untouched, so there is no re-embedding cost. It is idempotent (drawers already correct are +skipped) and safe to re-run. It only touches ``ingest_mode == "convos"`` drawers; markdown +drawers have no per-line timestamps and keep their ``filed_at`` fallback. + +Drawers whose source transcript is no longer on disk are left as-is (they keep falling back +to ``filed_at``), so point ``--sessions`` at the directories that still hold your ``.jsonl`` +transcripts (e.g. ``~/.claude`` and ``~/.codex``). + +Usage (dry-run prints what would change; pass --apply to write): + + python scripts/backfill_authored_at.py \ + --palace ~/.mempalace/palace \ + --sessions ~/.claude --sessions ~/.codex [--apply] + +In Docker (the MCP image), mount the volume and your session dirs read-only: + + docker run --rm \ + -v mempalace-data:/data \ + -v ~/.claude:/sessions/claude:ro -v ~/.codex:/sessions/codex:ro \ + -v "$PWD/scripts/backfill_authored_at.py:/tmp/backfill.py:ro" \ + --entrypoint /app/.venv/bin/python mempalace:local \ + /tmp/backfill.py --palace /data/.mempalace/palace \ + --sessions /sessions/claude --sessions /sessions/codex --apply +""" + +import argparse +import glob +import os + +import chromadb + +from mempalace.convo_miner import _extract_authored_at + +COLLECTION = "mempalace_drawers" +PAGE = 2000 +BATCH = 1000 + + +def _index_sessions(session_dirs): + """Map ``basename.jsonl -> realpath`` for every transcript under the given dirs.""" + index = {} + for root in session_dirs: + for f in glob.glob(os.path.join(os.path.expanduser(root), "**", "*.jsonl"), recursive=True): + index.setdefault(os.path.basename(f), f) + return index + + +def backfill_authored_at(collection, session_dirs, apply=False): + """Stamp ``authored_at`` on convos drawers from their source transcript timestamps. + + Returns a stats dict: ``scanned``, ``updated``, ``resolved_files``, ``unresolved_files``. + """ + index = _index_sessions(session_dirs) + cache = {} + unresolved = set() + pending_ids, pending_metas = [], [] + scanned = updated = 0 + + def flush(): + nonlocal pending_ids, pending_metas, updated + if pending_ids and apply: + collection.update(ids=pending_ids, metadatas=pending_metas) + updated += len(pending_ids) + pending_ids, pending_metas = [], [] + + offset = 0 + while True: + res = collection.get( + where={"ingest_mode": "convos"}, include=["metadatas"], limit=PAGE, offset=offset + ) + ids = res["ids"] + if not ids: + break + for drawer_id, meta in zip(ids, res["metadatas"]): + scanned += 1 + basename = os.path.basename(meta.get("source_file") or "") + if basename in cache: + authored = cache[basename] + else: + path = index.get(basename) + authored = _extract_authored_at(path) if path else None + cache[basename] = authored + if path is None and basename: + unresolved.add(basename) + if authored and meta.get("authored_at") != authored: + new_meta = dict(meta) + new_meta["authored_at"] = authored + pending_ids.append(drawer_id) + pending_metas.append(new_meta) + if len(pending_ids) >= BATCH: + flush() + offset += len(ids) + flush() + return { + "scanned": scanned, + "updated": updated, + "resolved_files": sum(1 for v in cache.values() if v), + "unresolved_files": len(unresolved), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--palace", required=True, help="Path to the ChromaDB palace dir") + parser.add_argument( + "--sessions", + action="append", + default=[], + required=True, + help="Directory holding .jsonl transcripts (repeatable)", + ) + parser.add_argument( + "--apply", + action="store_true", + help="Write changes (default is a dry run that only reports counts)", + ) + args = parser.parse_args() + + client = chromadb.PersistentClient(path=os.path.expanduser(args.palace)) + collection = client.get_collection(COLLECTION) + stats = backfill_authored_at(collection, args.sessions, apply=args.apply) + mode = "APPLIED" if args.apply else "DRY-RUN (use --apply to write)" + print( + f"{mode}: scanned={stats['scanned']} updated={stats['updated']} " + f"resolved_files={stats['resolved_files']} unresolved_files={stats['unresolved_files']}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_backfill_authored_at.py b/tests/test_backfill_authored_at.py new file mode 100644 index 0000000..10d0855 --- /dev/null +++ b/tests/test_backfill_authored_at.py @@ -0,0 +1,86 @@ +"""Integration tests for the authored_at backfill migration (scripts/).""" + +import importlib.util +import uuid +from pathlib import Path + +import chromadb + +# The migration ships as a script, not a package module; load it directly. +_SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "backfill_authored_at.py" +_spec = importlib.util.spec_from_file_location("backfill_authored_at", _SCRIPT) +backfill_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(backfill_mod) + + +def _collection(): + # Unique name per call: EphemeralClient shares one in-memory instance across the + # process, so a fixed collection name would leak drawers between tests. + client = chromadb.EphemeralClient() + return client.create_collection(f"drawers_{uuid.uuid4().hex}") + + +def _add(col, drawer_id, source_file, authored_at=None): + meta = {"ingest_mode": "convos", "source_file": source_file, "filed_at": "2026-06-27T00:00:00"} + if authored_at is not None: + meta["authored_at"] = authored_at + col.add(ids=[drawer_id], documents=["hello"], metadatas=[meta], embeddings=[[0.1, 0.2, 0.3]]) + + +def _transcript(dir_path, name, *timestamps): + dir_path.mkdir(parents=True, exist_ok=True) + f = dir_path / name + f.write_text("".join(f'{{"timestamp": "{ts}"}}\n' for ts in timestamps)) + return f + + +def test_backfill_sets_latest_timestamp(tmp_path): + sessions = tmp_path / "claude" + _transcript(sessions, "abc.jsonl", "2026-06-10T08:00:00.000Z", "2026-06-12T09:00:00.000Z") + col = _collection() + # Stored source_file uses an old mount prefix; resolution is by basename. + _add(col, "d1", "/old/mount/abc.jsonl") + + stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True) + + assert stats["scanned"] == 1 + assert stats["updated"] == 1 + got = col.get(ids=["d1"], include=["metadatas"])["metadatas"][0] + assert got["authored_at"] == "2026-06-12T09:00:00.000Z" + + +def test_dry_run_writes_nothing(tmp_path): + sessions = tmp_path / "claude" + _transcript(sessions, "abc.jsonl", "2026-06-12T09:00:00.000Z") + col = _collection() + _add(col, "d1", "/old/mount/abc.jsonl") + + stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=False) + + assert stats["updated"] == 1 # would update + assert "authored_at" not in col.get(ids=["d1"], include=["metadatas"])["metadatas"][0] + + +def test_idempotent_second_run_updates_nothing(tmp_path): + sessions = tmp_path / "claude" + _transcript(sessions, "abc.jsonl", "2026-06-12T09:00:00.000Z") + col = _collection() + _add(col, "d1", "/old/mount/abc.jsonl") + + backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True) + stats2 = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True) + + assert stats2["updated"] == 0 + + +def test_unresolved_transcript_is_left_alone(tmp_path): + sessions = tmp_path / "claude" + sessions.mkdir() + col = _collection() + _add(col, "d1", "/old/mount/missing.jsonl") # no file on disk + + stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True) + + assert stats["updated"] == 0 + assert stats["unresolved_files"] == 1 + assert "authored_at" not in col.get(ids=["d1"], include=["metadatas"])["metadatas"][0] diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index 2970cb5..d79e859 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -8,6 +8,7 @@ import pytest from mempalace.convo_miner import ( CHUNK_SIZE, _emit_bounded, + _extract_authored_at, _file_chunks_locked, chunk_exchanges, detect_convo_room, @@ -468,3 +469,64 @@ class TestFileChunksLocked: assert dict(room_counts) == {} assert skipped is False assert col.batch_sizes == [2, 2, 1] + + +class TestExtractAuthoredAt: + """authored_at = max per-line ``timestamp`` in a transcript (real authored date, + independent of mine time). Both Claude Code and Codex JSONL carry a top-level + ISO-8601 ``timestamp`` per line.""" + + def test_returns_latest_timestamp(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text( + '{"type": "user", "timestamp": "2026-06-21T10:00:00.000Z"}\n' + '{"type": "assistant", "timestamp": "2026-06-23T14:30:00.000Z"}\n' + '{"type": "user", "timestamp": "2026-06-22T09:00:00.000Z"}\n' + ) + assert _extract_authored_at(f) == "2026-06-23T14:30:00.000Z" + + def test_ignores_lines_without_timestamp(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text( + '{"type": "summary", "summary": "x"}\n' + '{"type": "assistant", "timestamp": "2026-06-23T14:30:00.000Z"}\n' + ) + assert _extract_authored_at(f) == "2026-06-23T14:30:00.000Z" + + def test_tolerates_blank_and_malformed_lines(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text( + "\n" + "not json\n" + "[1, 2, 3]\n" # valid JSON, but no .get() + '{"timestamp": "2026-06-25T00:00:00.000Z"}\n' + ) + assert _extract_authored_at(f) == "2026-06-25T00:00:00.000Z" + + def test_none_for_non_jsonl(self, tmp_path): + f = tmp_path / "notes.md" + f.write_text("# heading\n") + assert _extract_authored_at(f) is None + + def test_none_when_no_timestamps(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text('{"type": "user", "content": "hi"}\n') + assert _extract_authored_at(f) is None + + def test_none_for_missing_file(self, tmp_path): + assert _extract_authored_at(tmp_path / "absent.jsonl") is None + + def test_non_string_timestamp_does_not_crash(self, tmp_path): + # A non-string timestamp must be skipped, not raise TypeError on compare. + f = tmp_path / "session.jsonl" + f.write_text( + '{"type": "user", "timestamp": 1234567890}\n' + '{"type": "assistant", "timestamp": {"nested": true}}\n' + '{"type": "user", "timestamp": "2026-06-24T00:00:00.000Z"}\n' + ) + assert _extract_authored_at(f) == "2026-06-24T00:00:00.000Z" + + def test_only_non_string_timestamps_returns_none(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text('{"timestamp": 1}\n{"timestamp": false}\n') + assert _extract_authored_at(f) is None diff --git a/tests/test_hybrid_search.py b/tests/test_hybrid_search.py index 35aa579..98c11ae 100644 --- a/tests/test_hybrid_search.py +++ b/tests/test_hybrid_search.py @@ -12,7 +12,7 @@ from mempalace.palace import ( get_collection, upsert_closet_lines, ) -from mempalace.searcher import search_memories +from mempalace.searcher import _hybrid_rank, search_memories def _seed_drawers(palace_path): @@ -173,3 +173,34 @@ class TestSourceFileFilter: ids = [h["source_file"] for h in result["results"]] assert "fixture_D1.md" not in ids assert set(ids) <= {"fixture_D4.md"} + + +def test_hybrid_rank_breaks_score_ties_by_authored_at(): + """Identical-content hits get identical vector + BM25 scores; the tie must break + toward the more recently authored drawer, not arbitrary backend order.""" + older = { + "text": "alpha beta gamma", + "distance": 0.2, + "metadata": {"authored_at": "2026-06-21T10:00:00.000Z"}, + } + newer = { + "text": "alpha beta gamma", + "distance": 0.2, + "metadata": {"authored_at": "2026-06-27T10:00:00.000Z"}, + } + # Input order puts the older drawer first; the tiebreak should reorder it. + results = [older, newer] + _hybrid_rank(results, "alpha beta gamma") + assert results[0]["metadata"]["authored_at"] == "2026-06-27T10:00:00.000Z" + assert results[1]["metadata"]["authored_at"] == "2026-06-21T10:00:00.000Z" + + +def test_hybrid_rank_tiebreak_handles_top_level_authored_at(): + """The search_memories path puts authored_at at the top level (no `metadata` + nesting); the tie-break must read it there too.""" + older = {"text": "alpha beta gamma", "distance": 0.2, "authored_at": "2026-06-21T10:00:00.000Z"} + newer = {"text": "alpha beta gamma", "distance": 0.2, "authored_at": "2026-06-27T10:00:00.000Z"} + results = [older, newer] + _hybrid_rank(results, "alpha beta gamma") + assert results[0]["authored_at"] == "2026-06-27T10:00:00.000Z" + assert results[1]["authored_at"] == "2026-06-21T10:00:00.000Z"