Files
mempalace/tests/test_hybrid_search.py
T
cff43adb63 feat(convo): preserve authored timestamp from transcripts (#1890)
* feat(convo): preserve authored timestamp from transcripts

Conversation drawers only carried `filed_at` (ingest time), so a bulk
re-mine collapsed every drawer to a single instant and the chronological
signal was lost — even though each Claude Code / Codex JSONL line already
carries an ISO-8601 `timestamp`. The recency-window fallback and any
date-aware consumer then saw ingest order, not when content was written.

- convo_miner: derive `authored_at` (per-file max line `timestamp`) and
  store it as drawer metadata; falls back to `filed_at` when absent
- searcher: surface `authored_at` in search results, and break exact
  hybrid-score ties toward the more recently authored drawer (ISO strings
  sort chronologically; missing dates sort oldest) — benchmark-neutral as
  it only reorders exact ties
- tests: cover `_extract_authored_at` (latest wins, skips/tolerates lines
  without timestamps, non-jsonl/missing -> None) and the tie-break

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* feat(search): surface authored_at in CLI + backfill for existing data

Completes the authored_at work so the field is visible end-to-end and
existing palaces can adopt it without re-mining.

- layers: CLI `search` output shows an `authored:` date line per result
  (peer of the existing date; markdown drawers fall back to filed_at)
- scripts/backfill_authored_at.py: in-place migration that stamps
  authored_at on convos drawers from their source transcripts — metadata
  only (no re-embedding), idempotent, dry-run by default
- docs/authored-at.md: documents created_at (ingest) vs authored_at
  (written) and both backfill paths (in-place / drop-and-recreate)
- tests: backfill integration tests over an ephemeral ChromaDB collection

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* fix(search): address review — non-string timestamp guard + top-level authored_at tiebreak

Two correctness fixes from the PR review:

- _extract_authored_at: only compare when the parsed `timestamp` is a str.
  A non-string timestamp on a malformed/foreign JSONL line previously raised
  TypeError outside the try and could crash the mine.
- _hybrid_rank: the tie-break read `authored_at` only from nested `metadata`,
  but the search_memories path (MCP / Claude Code) carries it at the top level
  of each hit — so the tie-break silently no-op'd there. Read both shapes.
- tests: non-string timestamp cases, and a top-level-shape tie-break test
  (which fails before this fix).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* style: apply ruff format to authored_at changes

CI ruff format --check flagged 4 files; ruff check already passed.
Formatting only — no behavior change.

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Igor Lins e Silva <4753812+igorls@users.noreply.github.com>
2026-06-28 18:13:06 -03:00

207 lines
8.4 KiB
Python

"""Tests for the hybrid closet+drawer retrieval in search_memories.
The hybrid path queries drawers directly (the floor) AND closets, applying a
rank-based boost to drawers whose source_file appears in top closet hits.
This avoids the "weak-closets regression" where low-signal closets (from
regex extraction on narrative content) could hide drawers that direct
search would have found.
"""
from mempalace.palace import (
get_closets_collection,
get_collection,
upsert_closet_lines,
)
from mempalace.searcher import _hybrid_rank, search_memories
def _seed_drawers(palace_path):
"""Insert 4 short drawers with deterministic content."""
col = get_collection(palace_path, create=True)
col.upsert(
ids=["D1", "D2", "D3", "D4"],
documents=[
"We switched the auth service to use JWT tokens with a 24h expiry.",
"Database migration to PostgreSQL 15 completed last Tuesday.",
"The frontend team is debating whether to adopt TanStack Query.",
"Kafka consumer rebalance timeout set to 45 seconds after incident.",
],
metadatas=[
{"wing": "backend", "room": "auth", "source_file": "fixture_D1.md"},
{"wing": "backend", "room": "db", "source_file": "fixture_D2.md"},
{"wing": "frontend", "room": "state", "source_file": "fixture_D3.md"},
{"wing": "backend", "room": "queue", "source_file": "fixture_D4.md"},
],
)
def _seed_strong_closet_for(palace_path, drawer_id, source_file, topics):
"""Insert a closet whose content strongly overlaps the query keywords."""
col = get_closets_collection(palace_path)
lines = [f"{t}||→{drawer_id}" for t in topics]
upsert_closet_lines(
col,
closet_id_base=f"closet_{drawer_id}",
lines=lines,
metadata={
"wing": "backend",
"room": "auth",
"source_file": source_file,
"generated_by": "test",
},
)
# ── core invariant: closets can only HELP, never HIDE ─────────────────────
class TestHybridInvariant:
def test_no_closets_degrades_to_direct_drawer_search(self, tmp_path):
palace = str(tmp_path / "palace")
_seed_drawers(palace)
# No closets created.
result = search_memories("Kafka rebalance timeout", palace, n_results=3)
ids = [h["source_file"] for h in result["results"]]
assert ids, "should return results"
assert "fixture_D4.md" in ids, "direct drawer search alone should surface the Kafka drawer"
def test_weak_closets_do_not_hide_direct_drawer_hits(self, tmp_path):
"""A closet that points at a wrong drawer must NOT suppress the
drawer that direct search would have ranked first."""
palace = str(tmp_path / "palace")
_seed_drawers(palace)
# Seed a misleading closet: it matches a generic phrase but points at D3.
_seed_strong_closet_for(
palace,
drawer_id="D3",
source_file="fixture_D3.md",
topics=["Kafka queue tuning", "consumer rebalance config"],
)
result = search_memories("Kafka consumer rebalance timeout", palace, n_results=5)
ids = [h["source_file"] for h in result["results"]]
assert "fixture_D4.md" in ids, (
"D4 must appear — direct drawer search alone would rank it first. "
"Closet pointing to D3 should only boost D3, never hide D4."
)
def test_closet_boost_lifts_matching_drawer(self, tmp_path):
"""When a closet agrees with direct search, the matching drawer
should be boosted to rank 1."""
palace = str(tmp_path / "palace")
_seed_drawers(palace)
_seed_strong_closet_for(
palace,
drawer_id="D1",
source_file="fixture_D1.md",
topics=["JWT auth tokens", "session expiry", "authentication service"],
)
result = search_memories("JWT auth tokens expiry", palace, n_results=3)
ids = [h["source_file"] for h in result["results"]]
assert ids[0] == "fixture_D1.md"
top = result["results"][0]
assert top["matched_via"] == "drawer+closet"
assert top["closet_boost"] > 0
# ── closet_boost metadata ────────────────────────────────────────────────
class TestClosetMetadata:
def test_closet_preview_exposed_when_boosted(self, tmp_path):
palace = str(tmp_path / "palace")
_seed_drawers(palace)
_seed_strong_closet_for(
palace,
drawer_id="D1",
source_file="fixture_D1.md",
topics=["JWT auth tokens", "session expiry", "authentication service"],
)
result = search_memories("JWT auth tokens expiry", palace, n_results=2)
top = result["results"][0]
assert top["source_file"] == "fixture_D1.md"
assert top["matched_via"] == "drawer+closet"
assert top["closet_boost"] > 0
assert "closet_preview" in top
def test_drawer_only_hits_have_no_closet_preview(self, tmp_path):
palace = str(tmp_path / "palace")
_seed_drawers(palace)
# No closets
result = search_memories("TanStack Query", palace, n_results=2)
assert result["results"]
for h in result["results"]:
assert h["matched_via"] == "drawer"
assert "closet_preview" not in h
assert h["closet_boost"] == 0.0
# ── source_file filter scopes both drawer and closet queries (#1815) ──────
class TestSourceFileFilter:
def test_source_file_filter_excludes_other_sources(self, tmp_path):
palace = str(tmp_path / "palace")
_seed_drawers(palace)
result = search_memories(
"Kafka consumer rebalance timeout",
palace,
n_results=5,
source_file="fixture_D4.md",
)
ids = [h["source_file"] for h in result["results"]]
assert ids, "the matching source_file drawer should be returned"
assert set(ids) == {"fixture_D4.md"}
def test_source_file_filter_overrides_closet_boost_for_other_source(self, tmp_path):
# A strong closet pointing at D1 must NOT leak D1 in when the search
# is scoped to a different source_file — the where clause is applied
# to the closet query too, not just the drawer query.
palace = str(tmp_path / "palace")
_seed_drawers(palace)
_seed_strong_closet_for(
palace,
drawer_id="D1",
source_file="fixture_D1.md",
topics=["Kafka queue tuning", "consumer rebalance config"],
)
result = search_memories(
"Kafka consumer rebalance",
palace,
n_results=5,
source_file="fixture_D4.md",
)
ids = [h["source_file"] for h in result["results"]]
assert "fixture_D1.md" not in ids
assert set(ids) <= {"fixture_D4.md"}
def test_hybrid_rank_breaks_score_ties_by_authored_at():
"""Identical-content hits get identical vector + BM25 scores; the tie must break
toward the more recently authored drawer, not arbitrary backend order."""
older = {
"text": "alpha beta gamma",
"distance": 0.2,
"metadata": {"authored_at": "2026-06-21T10:00:00.000Z"},
}
newer = {
"text": "alpha beta gamma",
"distance": 0.2,
"metadata": {"authored_at": "2026-06-27T10:00:00.000Z"},
}
# Input order puts the older drawer first; the tiebreak should reorder it.
results = [older, newer]
_hybrid_rank(results, "alpha beta gamma")
assert results[0]["metadata"]["authored_at"] == "2026-06-27T10:00:00.000Z"
assert results[1]["metadata"]["authored_at"] == "2026-06-21T10:00:00.000Z"
def test_hybrid_rank_tiebreak_handles_top_level_authored_at():
"""The search_memories path puts authored_at at the top level (no `metadata`
nesting); the tie-break must read it there too."""
older = {"text": "alpha beta gamma", "distance": 0.2, "authored_at": "2026-06-21T10:00:00.000Z"}
newer = {"text": "alpha beta gamma", "distance": 0.2, "authored_at": "2026-06-27T10:00:00.000Z"}
results = [older, newer]
_hybrid_rank(results, "alpha beta gamma")
assert results[0]["authored_at"] == "2026-06-27T10:00:00.000Z"
assert results[1]["authored_at"] == "2026-06-21T10:00:00.000Z"