feat(search): add since/before date window to search surfaces (#463)
Co-Authored-By: Matthew Clapp <1807922+nautis@users.noreply.github.com>
This commit is contained in:
mvalentsev
Igor Lins e Silva
parent
2174eb047c
commit
5036e3c05e
|
|
@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|||
### Features
|
||||
|
||||
- **Embeddings via any OpenAI-compatible `/v1/embeddings` endpoint.** New `embedding_model: "openai-compat"` option computes embeddings on a server (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or a self-hosted endpoint) instead of a local ONNX model — useful for larger/multilingual embedders such as Qwen3-Embedding, or GPU offload. New `OpenAICompatEmbeddingFunction` in [`mempalace/embedding.py`](mempalace/embedding.py) speaks the standard `/v1/embeddings` protocol over stdlib `urllib` (no new dependency), batches requests, re-sorts the response by `index`, and L2-normalizes for the cosine collection. Endpoint settings are resolved by `MempalaceConfig` as a single source of truth — `embedding_api_url` / `embedding_api_model` / `embedding_api_key` in `config.json`, each overridable via the matching `MEMPALACE_EMBEDDING_API_*` env var. The embedding function's `name()` encodes the model id so changing it forces `mempalace repair rebuild-index` (different vector space). Mirrors the existing `openai-compat` LLM provider naming; stays local when the endpoint is on your machine/LAN. (#1559)
|
||||
- **`mempalace_search` / `mempalace search` — `since`/`before` date window.** Semantic search is poor at temporal queries ("what did we discuss this week?" scores ~0.35 even when matching drawers exist), so the search surfaces now accept the same `[since, before)` window `list_drawers` gained in #1128: inclusive/exclusive ISO bounds compared wall-clock against each drawer's `filed_at`, undated drawers excluded while a bound is active. The window applies on every candidate path (vector, `candidate_strategy="union"`, and the BM25-only fallback); the vector candidate pool widens under an active window (ChromaDB cannot range-compare string metadata server-side), and a full pool is flagged via `date_filter_pool_truncated` instead of passing silently. Shared parsing lives in the new `mempalace.date_window` module, also backing the `list_drawers` filter. (#463)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
|
|
|
|||
|
|
@ -1292,6 +1292,8 @@ def cmd_search(args):
|
|||
wing=args.wing,
|
||||
room=args.room,
|
||||
n_results=args.results,
|
||||
since=args.since,
|
||||
before=args.before,
|
||||
)
|
||||
except SearchError:
|
||||
sys.exit(1)
|
||||
|
|
@ -2650,6 +2652,20 @@ def main():
|
|||
p_search.add_argument("--wing", default=None, help="Limit to one project")
|
||||
p_search.add_argument("--room", default=None, help="Limit to one room")
|
||||
p_search.add_argument("--results", type=int, default=5, help="Number of results")
|
||||
p_search.add_argument(
|
||||
"--since",
|
||||
default=None,
|
||||
help=(
|
||||
"Only drawers filed on/after this ISO date/datetime (inclusive), "
|
||||
"e.g. 2026-04-01. Drawers without a filed_at are excluded while "
|
||||
"a date bound is set"
|
||||
),
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--before",
|
||||
default=None,
|
||||
help="Only drawers filed strictly before this ISO date/datetime (exclusive)",
|
||||
)
|
||||
|
||||
# compress
|
||||
p_compress = sub.add_parser(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
"""Shared date-window parsing for read-path filters.
|
||||
|
||||
``list_drawers`` (#1128) and ``search`` (#463) accept the same optional
|
||||
``since``/``before`` bounds and compare them against drawer ``filed_at``
|
||||
metadata. The parsing and window semantics live here — a side-effect-free
|
||||
module — so ``searcher`` can use them without importing ``mcp_server``
|
||||
(whose import installs MCP stdio protection) and ``mcp_server`` keeps its
|
||||
existing behavior by aliasing these functions.
|
||||
|
||||
Semantics (issue spec, #1128):
|
||||
* ``since`` is inclusive, ``before`` is exclusive: ``[since, before)``.
|
||||
* Comparison is wall-clock and timezone-naive. ``filed_at`` values are
|
||||
written as naive local ISO strings (``datetime.now().isoformat()``)
|
||||
almost everywhere; the one aware writer (``diary_ingest``, UTC) is
|
||||
compared on its wall-clock fields after the offset is dropped.
|
||||
* A drawer whose ``filed_at`` is missing or unparseable is EXCLUDED
|
||||
whenever a bound is active — a date-filtered result must never
|
||||
silently include rows of unknown age.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def parse_date_bound(value: Optional[str] = None, field_name: str = "date") -> Optional[datetime]:
|
||||
"""Parse an optional ISO-8601 date/datetime filter bound.
|
||||
|
||||
Accepts a date (``"2026-04-01"``), a naive timestamp
|
||||
(``"2026-04-01T09:30:00"``), or one carrying a ``Z``/``+HH:MM`` offset.
|
||||
Returns a naive ``datetime`` for wall-clock
|
||||
comparison against drawer ``filed_at`` values, which are stored as naive
|
||||
local ISO strings (``datetime.now().isoformat()``). Any timezone offset on
|
||||
the input is dropped so an aware bound never raises a ``TypeError`` against
|
||||
a naive ``filed_at``. Comparison is therefore wall-clock, which is what the
|
||||
local-first single-machine model wants; an offset bound is matched on its
|
||||
wall-clock fields, not its absolute instant, so a bound whose offset differs
|
||||
from the zone ``filed_at`` was recorded in is matched by clock time.
|
||||
The accepted grammar is a date, an ISO timestamp (optionally fractional),
|
||||
and an optional ``Z``/``±HH:MM`` offset; other ISO 8601 forms (basic format,
|
||||
week dates) are outside the contract and are rejected on the Python 3.9 floor
|
||||
even where a newer ``fromisoformat`` would accept them.
|
||||
Blank / whitespace-only means "no filter" (``None``).
|
||||
Raises ``ValueError`` on an unparseable value so the caller can surface a
|
||||
clear error, mirroring the wing/room sanitizers.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f"{field_name} must be an ISO date string")
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
# datetime.fromisoformat before Python 3.11 rejects a trailing "Z" (Zulu),
|
||||
# and appending "+00:00" would break a date-only value on 3.9/3.10
|
||||
# ("2026-04-01+00:00" is rejected there). Any offset is dropped below for
|
||||
# wall-clock comparison anyway, so just strip a trailing Z/z; both date and
|
||||
# date-time Zulu inputs then parse on the 3.9 floor.
|
||||
iso = value[:-1] if value.endswith(("Z", "z")) else value
|
||||
try:
|
||||
parsed = datetime.fromisoformat(iso)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"{field_name} must be an ISO date string "
|
||||
f"(e.g. '2026-04-01' or '2026-04-01T09:30:00'), got {value!r}"
|
||||
) from exc
|
||||
if parsed.tzinfo is not None:
|
||||
parsed = parsed.replace(tzinfo=None)
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_window(since: Optional[str] = None, before: Optional[str] = None):
|
||||
"""Parse a ``[since, before)`` pair, rejecting an inverted window.
|
||||
|
||||
Returns ``(since_dt, before_dt)`` — either side ``None`` when absent.
|
||||
Raises ``ValueError`` (naming the offending field or the inversion) so
|
||||
callers surface the same message everywhere a window is accepted.
|
||||
"""
|
||||
since_dt = parse_date_bound(since, "since")
|
||||
before_dt = parse_date_bound(before, "before")
|
||||
if since_dt is not None and before_dt is not None and since_dt >= before_dt:
|
||||
raise ValueError(f"since ({since!r}) must be earlier than before ({before!r})")
|
||||
return since_dt, before_dt
|
||||
|
||||
|
||||
def filed_at_in_window(
|
||||
filed_at, since_dt: Optional[datetime], before_dt: Optional[datetime]
|
||||
) -> bool:
|
||||
"""True if a drawer's ``filed_at`` falls in ``[since, before)``.
|
||||
|
||||
``since`` is inclusive and ``before`` is exclusive, matching the issue spec.
|
||||
Parsing (``Z``/offset normalization, tz drop) is delegated to
|
||||
``parse_date_bound`` so a bound and a ``filed_at`` are compared
|
||||
identically. A drawer whose ``filed_at`` is missing or unparseable cannot
|
||||
be confirmed in-window, so it is EXCLUDED whenever a bound is active — a
|
||||
date-filtered listing must never silently include rows of unknown age.
|
||||
"""
|
||||
try:
|
||||
filed_dt = parse_date_bound(filed_at, "filed_at")
|
||||
except ValueError:
|
||||
return False
|
||||
if filed_dt is None:
|
||||
return False
|
||||
if since_dt is not None and filed_dt < since_dt:
|
||||
return False
|
||||
if before_dt is not None and filed_dt >= before_dt:
|
||||
return False
|
||||
return True
|
||||
|
|
@ -79,6 +79,7 @@ from .backends.chroma import ( # noqa: E402
|
|||
reset_hnsw_capacity_cache,
|
||||
)
|
||||
from .backends import BackendMismatchError, PalaceRef, detect_backend_for_path # noqa: E402
|
||||
from .date_window import filed_at_in_window, parse_date_bound # noqa: E402
|
||||
from .query_sanitizer import sanitize_query # noqa: E402
|
||||
from .searcher import ( # noqa: E402
|
||||
_distance_to_similarity,
|
||||
|
|
@ -1816,75 +1817,12 @@ def _sanitize_optional_source_file(value: str = None) -> str:
|
|||
return value
|
||||
|
||||
|
||||
def _parse_date_filter(value: Optional[str] = None, field_name: str = "date") -> Optional[datetime]:
|
||||
"""Parse an optional ISO-8601 date/datetime filter bound (#1128).
|
||||
|
||||
Accepts a date (``"2026-04-01"``), a naive timestamp
|
||||
(``"2026-04-01T09:30:00"``), or one carrying a ``Z``/``+HH:MM`` offset.
|
||||
Returns a naive ``datetime`` for wall-clock
|
||||
comparison against drawer ``filed_at`` values, which are stored as naive
|
||||
local ISO strings (``datetime.now().isoformat()``). Any timezone offset on
|
||||
the input is dropped so an aware bound never raises a ``TypeError`` against
|
||||
a naive ``filed_at``. Comparison is therefore wall-clock, which is what the
|
||||
local-first single-machine model wants; an offset bound is matched on its
|
||||
wall-clock fields, not its absolute instant, so a bound whose offset differs
|
||||
from the zone ``filed_at`` was recorded in is matched by clock time.
|
||||
The accepted grammar is a date, an ISO timestamp (optionally fractional),
|
||||
and an optional ``Z``/``±HH:MM`` offset; other ISO 8601 forms (basic format,
|
||||
week dates) are outside the contract and are rejected on the Python 3.9 floor
|
||||
even where a newer ``fromisoformat`` would accept them.
|
||||
Blank / whitespace-only means "no filter" (``None``).
|
||||
Raises ``ValueError`` on an unparseable value so the caller can surface a
|
||||
clear error, mirroring the wing/room sanitizers.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f"{field_name} must be an ISO date string")
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
# datetime.fromisoformat before Python 3.11 rejects a trailing "Z" (Zulu),
|
||||
# and appending "+00:00" would break a date-only value on 3.9/3.10
|
||||
# ("2026-04-01+00:00" is rejected there). Any offset is dropped below for
|
||||
# wall-clock comparison anyway, so just strip a trailing Z/z; both date and
|
||||
# date-time Zulu inputs then parse on the 3.9 floor.
|
||||
iso = value[:-1] if value.endswith(("Z", "z")) else value
|
||||
try:
|
||||
parsed = datetime.fromisoformat(iso)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"{field_name} must be an ISO date string "
|
||||
f"(e.g. '2026-04-01' or '2026-04-01T09:30:00'), got {value!r}"
|
||||
) from exc
|
||||
if parsed.tzinfo is not None:
|
||||
parsed = parsed.replace(tzinfo=None)
|
||||
return parsed
|
||||
|
||||
|
||||
def _filed_at_in_window(
|
||||
filed_at, since_dt: Optional[datetime], before_dt: Optional[datetime]
|
||||
) -> bool:
|
||||
"""True if a drawer's ``filed_at`` falls in ``[since, before)`` (#1128).
|
||||
|
||||
``since`` is inclusive and ``before`` is exclusive, matching the issue spec.
|
||||
Parsing (``Z``/offset normalization, tz drop) is delegated to
|
||||
``_parse_date_filter`` so a bound and a ``filed_at`` are compared
|
||||
identically. A drawer whose ``filed_at`` is missing or unparseable cannot
|
||||
be confirmed in-window, so it is EXCLUDED whenever a bound is active — a
|
||||
date-filtered listing must never silently include rows of unknown age.
|
||||
"""
|
||||
try:
|
||||
filed_dt = _parse_date_filter(filed_at, "filed_at")
|
||||
except ValueError:
|
||||
return False
|
||||
if filed_dt is None:
|
||||
return False
|
||||
if since_dt is not None and filed_dt < since_dt:
|
||||
return False
|
||||
if before_dt is not None and filed_dt >= before_dt:
|
||||
return False
|
||||
return True
|
||||
# The #1128 date-filter helpers moved to ``mempalace.date_window`` so the
|
||||
# search-side window (#463) can share them without importing this module
|
||||
# (whose import installs MCP stdio protection). Aliased under their
|
||||
# historical private names — every call site and test keeps working.
|
||||
_parse_date_filter = parse_date_bound
|
||||
_filed_at_in_window = filed_at_in_window
|
||||
|
||||
|
||||
# ==================== READ TOOLS ====================
|
||||
|
|
@ -2423,6 +2361,8 @@ def tool_search(
|
|||
wing: str = None,
|
||||
room: str = None,
|
||||
source_file: str = None,
|
||||
since: str = None,
|
||||
before: str = None,
|
||||
max_distance: float = 1.5,
|
||||
min_similarity: float = None,
|
||||
context: str = None,
|
||||
|
|
@ -2434,6 +2374,8 @@ def tool_search(
|
|||
source_file = _sanitize_optional_source_file(source_file)
|
||||
except ValueError as e:
|
||||
return {"error": str(e)}
|
||||
# since/before are validated inside search_memories (shared
|
||||
# parse_window), which returns the same {"error": ...} shape.
|
||||
# Backwards compat: accept old name
|
||||
# Backwards compat: convert old similarity scale (higher=stricter) to
|
||||
# distance scale (lower=stricter). Similarity 0.8 → distance 0.2.
|
||||
|
|
@ -2451,6 +2393,8 @@ def tool_search(
|
|||
wing=wing,
|
||||
room=room,
|
||||
source_file=source_file,
|
||||
since=since,
|
||||
before=before,
|
||||
n_results=limit,
|
||||
max_distance=dist,
|
||||
vector_disabled=_vector_disabled,
|
||||
|
|
@ -2469,6 +2413,8 @@ def tool_search(
|
|||
wing=wing,
|
||||
room=room,
|
||||
source_file=source_file,
|
||||
since=since,
|
||||
before=before,
|
||||
n_results=limit,
|
||||
max_distance=dist,
|
||||
vector_disabled=_vector_disabled,
|
||||
|
|
@ -5011,6 +4957,22 @@ TOOLS = {
|
|||
"'source_path' field; the displayed 'source_file' is only a basename."
|
||||
),
|
||||
},
|
||||
"since": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Only drawers filed on/after this ISO date or datetime "
|
||||
"(inclusive), e.g. '2026-04-01' or '2026-04-01T09:30:00'. "
|
||||
"Compares the drawer's created_at (filed_at) wall-clock; "
|
||||
"drawers without a filed_at are excluded while set."
|
||||
),
|
||||
},
|
||||
"before": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Only drawers filed strictly before this ISO date or "
|
||||
"datetime (exclusive). Same comparison rules as 'since'."
|
||||
),
|
||||
},
|
||||
"max_distance": {
|
||||
"type": "number",
|
||||
"description": "Max cosine distance threshold (0=identical, 2=opposite). Results further than this are dropped. Lower = stricter. Default 1.5. Set to 0 to disable.",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import math
|
|||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
|
@ -26,6 +27,7 @@ from .backends import (
|
|||
UnsupportedCapabilityError,
|
||||
)
|
||||
from .config import MempalaceConfig, sqlite_read_uri
|
||||
from .date_window import filed_at_in_window, parse_window
|
||||
from .i18n import _canonical_lang, get_stopwords
|
||||
from .palace import (
|
||||
_open_collection_or_explain,
|
||||
|
|
@ -532,6 +534,8 @@ def _print_search_results_bm25_only(
|
|||
room: str,
|
||||
n_results: int,
|
||||
stop_words: frozenset = frozenset(),
|
||||
since_dt=None,
|
||||
before_dt=None,
|
||||
) -> None:
|
||||
"""CLI fallback printer for when HNSW divergence fences off vector search.
|
||||
|
||||
|
|
@ -544,6 +548,11 @@ def _print_search_results_bm25_only(
|
|||
:func:`_vector_disabled_search` forwards it on the MCP side: this path
|
||||
still ranks by BM25, so dropping the filter would rank a diverged
|
||||
palace by different rules than a healthy one.
|
||||
|
||||
An active ``[since_dt, before_dt)`` window is forwarded to the BM25
|
||||
reader, which post-filters on it. A diverged index degrades the
|
||||
ranking; it must never widen the result set past the window the
|
||||
caller asked for.
|
||||
"""
|
||||
result = _bm25_only_via_sqlite(
|
||||
query=query,
|
||||
|
|
@ -552,6 +561,8 @@ def _print_search_results_bm25_only(
|
|||
room=room,
|
||||
n_results=n_results,
|
||||
stop_words=stop_words,
|
||||
since_dt=since_dt,
|
||||
before_dt=before_dt,
|
||||
)
|
||||
hits = result.get("results", [])
|
||||
|
||||
|
|
@ -590,15 +601,35 @@ def _print_search_results_bm25_only(
|
|||
print()
|
||||
|
||||
|
||||
def search(query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5):
|
||||
def search(
|
||||
query: str,
|
||||
palace_path: str,
|
||||
wing: str = None,
|
||||
room: str = None,
|
||||
n_results: int = 5,
|
||||
since: str = None,
|
||||
before: str = None,
|
||||
):
|
||||
"""
|
||||
Search the palace. Returns verbatim drawer content.
|
||||
Optionally filter by wing (project) or room (aspect).
|
||||
Optionally filter by wing (project) or room (aspect), and/or narrow to
|
||||
drawers whose ``filed_at`` falls in the ``[since, before)`` window —
|
||||
same semantics as ``search_memories``/``list_drawers`` (#1128/#463).
|
||||
"""
|
||||
# Resolved before the fence below: both exits from this function rank by
|
||||
# BM25, so the filter has to be in hand on either branch.
|
||||
stop_words = _resolve_stop_words(None)
|
||||
|
||||
# Parse the window before probing the palace: an inverted or malformed
|
||||
# bound is a caller error and must raise identically whether or not the
|
||||
# index turns out to be diverged.
|
||||
try:
|
||||
since_dt, before_dt = parse_window(since, before)
|
||||
except ValueError as e:
|
||||
print(f"\n {e}")
|
||||
raise SearchError(str(e)) from e
|
||||
date_window_active = since_dt is not None or before_dt is not None
|
||||
|
||||
# Probe a Chroma palace before get_collection(). Opening the client can
|
||||
# load native index state, and embedder-identity enforcement may call
|
||||
# collection.count(); both happen before the old query-only guard and can
|
||||
|
|
@ -615,7 +646,14 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
|
|||
|
||||
if backend_name == "chroma" and _hnsw_capacity_diverged(palace_path):
|
||||
return _print_search_results_bm25_only(
|
||||
query, palace_path, wing, room, n_results, stop_words=stop_words
|
||||
query,
|
||||
palace_path,
|
||||
wing,
|
||||
room,
|
||||
n_results,
|
||||
stop_words=stop_words,
|
||||
since_dt=since_dt,
|
||||
before_dt=before_dt,
|
||||
)
|
||||
|
||||
col = _open_collection_or_explain(palace_path, opener=get_collection)
|
||||
|
|
@ -633,7 +671,12 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
|
|||
try:
|
||||
kwargs = {
|
||||
"query_texts": [query],
|
||||
"n_results": n_results,
|
||||
# The window is a post-filter (ChromaDB can't range-compare
|
||||
# string metadata), so widen the fetch the same way the
|
||||
# programmatic path does and trim back after filtering.
|
||||
"n_results": _candidate_pool_size(n_results, date_window_active)
|
||||
if date_window_active
|
||||
else n_results,
|
||||
"include": ["documents", "metadatas", "distances"],
|
||||
}
|
||||
if where:
|
||||
|
|
@ -649,6 +692,19 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
|
|||
metas = _first_or_empty(results, "metadatas")
|
||||
dists = _first_or_empty(results, "distances")
|
||||
|
||||
if date_window_active:
|
||||
kept = [
|
||||
(doc, meta, dist)
|
||||
for doc, meta, dist in zip(docs, metas, dists)
|
||||
if filed_at_in_window((meta or {}).get("filed_at"), since_dt, before_dt)
|
||||
]
|
||||
# Keep the whole in-window pool here; the hybrid re-rank below must
|
||||
# see every survivor before the display cut to n_results, or a
|
||||
# BM25-strong drawer deep in the pool could never surface.
|
||||
docs = [k[0] for k in kept]
|
||||
metas = [k[1] for k in kept]
|
||||
dists = [k[2] for k in kept]
|
||||
|
||||
if not docs:
|
||||
print(f'\n No results found for: "{query}"')
|
||||
return
|
||||
|
|
@ -667,6 +723,10 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
|
|||
for doc, meta, dist in zip(docs, metas, dists)
|
||||
]
|
||||
hits = _hybrid_rank(hits, query, metric=metric, stop_words=stop_words)
|
||||
if date_window_active:
|
||||
# The widened fetch exists only to survive the window filter; the
|
||||
# display contract stays "top n_results", now cut AFTER the re-rank.
|
||||
hits = hits[:n_results]
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f' Results for: "{query}"')
|
||||
|
|
@ -674,6 +734,10 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
|
|||
print(f" Wing: {wing}")
|
||||
if room:
|
||||
print(f" Room: {room}")
|
||||
if since:
|
||||
print(f" Since: {since}")
|
||||
if before:
|
||||
print(f" Before: {before}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
for i, hit in enumerate(hits, 1):
|
||||
|
|
@ -697,6 +761,37 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
|
|||
print()
|
||||
|
||||
|
||||
def _window_sql_prefilters(since_dt, before_dt) -> list:
|
||||
"""(operator, bound-string) pairs for the SQL date-window narrowing.
|
||||
|
||||
A SQL-side *narrowing* on the ISO ``filed_at`` string, kept at
|
||||
whole-DAY granularity so it is provably wider than the window for
|
||||
every ISO-8601 spelling that shares the YYYY-MM-DD prefix (bare date,
|
||||
space separator, minute precision, Z/offset suffixes) — a
|
||||
full-isoformat bound would sort after some of those on the boundary
|
||||
day and drop an in-window row at the SQL layer, where the
|
||||
authoritative Python re-filter (offset drop, unparseable exclusion —
|
||||
mirroring the wing/room double-check) can't recover it. Day
|
||||
granularity costs at most one extra day of candidates per bound;
|
||||
Python decides the exact window.
|
||||
"""
|
||||
prefilters = []
|
||||
if since_dt is not None:
|
||||
prefilters.append((">=", since_dt.date().isoformat()))
|
||||
if before_dt is not None:
|
||||
try:
|
||||
upper = (before_dt + timedelta(days=1)).date().isoformat()
|
||||
except OverflowError:
|
||||
# before at the calendar ceiling ("9999-12-31" as an open-ended
|
||||
# sentinel): there is no next day to bound by, so skip the SQL
|
||||
# narrowing entirely — the Python re-filter stays authoritative
|
||||
# and such a window is effectively unbounded above anyway.
|
||||
upper = None
|
||||
if upper is not None:
|
||||
prefilters.append(("<", upper))
|
||||
return prefilters
|
||||
|
||||
|
||||
def _bm25_only_via_sqlite(
|
||||
query: str,
|
||||
palace_path: str,
|
||||
|
|
@ -708,6 +803,8 @@ def _bm25_only_via_sqlite(
|
|||
_include_internal: bool = False,
|
||||
collection_name: str = None,
|
||||
stop_words: frozenset = frozenset(),
|
||||
since_dt=None,
|
||||
before_dt=None,
|
||||
) -> dict:
|
||||
"""BM25-only search reading drawers directly from chroma.sqlite3.
|
||||
|
||||
|
|
@ -759,6 +856,19 @@ def _bm25_only_via_sqlite(
|
|||
"""
|
||||
)
|
||||
params.extend([key, value])
|
||||
for op, sql_bound in _window_sql_prefilters(since_dt, before_dt):
|
||||
clauses.append(
|
||||
f"""
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM embedding_metadata mf
|
||||
WHERE mf.id = {row_id_expr}
|
||||
AND mf.key = 'filed_at'
|
||||
AND mf.string_value {op} ?
|
||||
)
|
||||
"""
|
||||
)
|
||||
params.append(sql_bound)
|
||||
return "".join(clauses), params
|
||||
|
||||
try:
|
||||
|
|
@ -766,6 +876,7 @@ def _bm25_only_via_sqlite(
|
|||
except sqlite3.Error as e:
|
||||
return _search_error_result(f"sqlite open failed: {e}")
|
||||
|
||||
window_active = since_dt is not None or before_dt is not None
|
||||
try:
|
||||
# FTS5 MATCH expects whitespace-separated tokens. Drop tokens
|
||||
# shorter than 3 chars (trigram tokenizer can't match them).
|
||||
|
|
@ -849,6 +960,11 @@ def _bm25_only_via_sqlite(
|
|||
logger.debug("id-ordered fallback also failed", exc_info=True)
|
||||
candidate_ids = []
|
||||
|
||||
# A full candidate page means rows beyond it never got a chance to
|
||||
# match the window — mirror the vector path's truncation honesty
|
||||
# (``date_filter_pool_truncated``) instead of a silently thin result.
|
||||
window_pool_truncated = window_active and len(candidate_ids) >= max_candidates
|
||||
|
||||
if not candidate_ids:
|
||||
return {
|
||||
"query": query,
|
||||
|
|
@ -899,6 +1015,8 @@ def _bm25_only_via_sqlite(
|
|||
continue
|
||||
if source_file and meta.get("source_file") != source_file:
|
||||
continue
|
||||
if window_active and not filed_at_in_window(meta.get("filed_at"), since_dt, before_dt):
|
||||
continue
|
||||
full_source = meta.get("source_file", "") or ""
|
||||
candidates.append(
|
||||
{
|
||||
|
|
@ -942,7 +1060,7 @@ def _bm25_only_via_sqlite(
|
|||
h.pop("_source_file_full", None)
|
||||
h.pop("_chunk_index", None)
|
||||
|
||||
return {
|
||||
result = {
|
||||
"query": query,
|
||||
"filters": {"wing": wing, "room": room, "source_file": source_file},
|
||||
"total_before_filter": len(candidates),
|
||||
|
|
@ -950,6 +1068,9 @@ def _bm25_only_via_sqlite(
|
|||
"fallback": "bm25_only_via_sqlite",
|
||||
"fallback_reason": "vector_search_disabled",
|
||||
}
|
||||
if window_pool_truncated:
|
||||
result["date_filter_pool_truncated"] = True
|
||||
return result
|
||||
|
||||
|
||||
def _merge_bm25_union_candidates(
|
||||
|
|
@ -962,6 +1083,8 @@ def _merge_bm25_union_candidates(
|
|||
max_distance: float = 0.0,
|
||||
source_file: str = None,
|
||||
stop_words: frozenset = frozenset(),
|
||||
since_dt=None,
|
||||
before_dt=None,
|
||||
) -> None:
|
||||
"""Append top-K backend lexical candidates into ``hits`` in place.
|
||||
|
||||
|
|
@ -1004,6 +1127,13 @@ def _merge_bm25_union_candidates(
|
|||
bm25_extra = []
|
||||
for hit in lexical.hits:
|
||||
meta = hit.metadata or {}
|
||||
# The window applies to every candidate source; a lexically strong
|
||||
# drawer outside [since, before) must not enter through this side
|
||||
# door (the vector-path candidates are filtered upstream).
|
||||
if (since_dt is not None or before_dt is not None) and not filed_at_in_window(
|
||||
meta.get("filed_at"), since_dt, before_dt
|
||||
):
|
||||
continue
|
||||
full_source = meta.get("source_file", "") or ""
|
||||
bm25_extra.append(
|
||||
{
|
||||
|
|
@ -1048,6 +1178,26 @@ def _merge_bm25_union_candidates(
|
|||
seen.add(key)
|
||||
|
||||
|
||||
def _candidate_pool_size(n_results: int, date_window_active: bool) -> int:
|
||||
"""Rerank-pool size for the drawer vector query.
|
||||
|
||||
Without a date window this is the historical ``n_results * 3``
|
||||
over-fetch. With one, the window filters the pool AFTER retrieval
|
||||
(ChromaDB rejects string operands for ``$gte``/``$lt``, so ``filed_at``
|
||||
can't be range-filtered server-side), and a narrow window over a large
|
||||
palace would starve a 3x pool even though matching drawers exist —
|
||||
recall is the design requirement. Widen to ``n_results * 15``, capped
|
||||
at 500 (the ceiling the filter-fallback path already uses) — except
|
||||
the pool never drops below ``n_results`` itself, or an oversized
|
||||
request could return fewer rows than an unfiltered query would.
|
||||
``date_filter_pool_truncated`` in the response flags a full pool so a
|
||||
capped result is never silent.
|
||||
"""
|
||||
if not date_window_active:
|
||||
return n_results * 3
|
||||
return max(min(n_results * 15, 500), n_results)
|
||||
|
||||
|
||||
# Strategy dispatch — keeps search_memories' branch count under the
|
||||
# project's complexity ceiling (C901 max-complexity=25). New strategies
|
||||
# register here.
|
||||
|
|
@ -1080,6 +1230,8 @@ def _apply_candidate_strategy(
|
|||
n_results: int,
|
||||
max_distance: float = 0.0,
|
||||
source_file: str = None,
|
||||
since_dt=None,
|
||||
before_dt=None,
|
||||
) -> None:
|
||||
"""Dispatch to the registered merger for ``strategy``.
|
||||
|
||||
|
|
@ -1097,6 +1249,8 @@ def _apply_candidate_strategy(
|
|||
n_results,
|
||||
max_distance=max_distance,
|
||||
source_file=source_file,
|
||||
since_dt=since_dt,
|
||||
before_dt=before_dt,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1112,6 +1266,8 @@ def _finalize_candidate_hits(
|
|||
max_distance: float,
|
||||
source_file: str = None,
|
||||
stop_words: frozenset = frozenset(),
|
||||
since_dt=None,
|
||||
before_dt=None,
|
||||
) -> tuple:
|
||||
try:
|
||||
_apply_candidate_strategy(
|
||||
|
|
@ -1124,6 +1280,8 @@ def _finalize_candidate_hits(
|
|||
n_results,
|
||||
max_distance=max_distance,
|
||||
source_file=source_file,
|
||||
since_dt=since_dt,
|
||||
before_dt=before_dt,
|
||||
)
|
||||
except UnsupportedCapabilityError:
|
||||
return [], _search_error_result(
|
||||
|
|
@ -1173,6 +1331,149 @@ def _unknown_backend_result(error: KeyError) -> dict:
|
|||
)
|
||||
|
||||
|
||||
def _search_result_envelope(
|
||||
*,
|
||||
query: str,
|
||||
wing,
|
||||
room,
|
||||
source_file,
|
||||
since,
|
||||
before,
|
||||
hits: list,
|
||||
candidates_fetched: int,
|
||||
pool_size: int,
|
||||
date_window_active: bool,
|
||||
) -> dict:
|
||||
"""Assemble the ``search_memories`` response dict.
|
||||
|
||||
When a date window is active and the widened candidate pool came back
|
||||
full, drawers beyond the pool never got a chance to match the window —
|
||||
``date_filter_pool_truncated`` flags it so a thin result under a date
|
||||
filter is never mistaken for "that's all there was".
|
||||
"""
|
||||
result = {
|
||||
"query": query,
|
||||
"filters": {
|
||||
"wing": wing,
|
||||
"room": room,
|
||||
"source_file": source_file,
|
||||
"since": since,
|
||||
"before": before,
|
||||
},
|
||||
"total_before_filter": candidates_fetched,
|
||||
"results": hits,
|
||||
}
|
||||
if date_window_active and candidates_fetched >= pool_size:
|
||||
result["date_filter_pool_truncated"] = True
|
||||
return result
|
||||
|
||||
|
||||
def _window_and_fallback_gate(
|
||||
since,
|
||||
before,
|
||||
vector_disabled: bool,
|
||||
*,
|
||||
query: str,
|
||||
palace_path: str,
|
||||
wing,
|
||||
room,
|
||||
n_results: int,
|
||||
collection_name,
|
||||
source_file,
|
||||
stop_words: frozenset = frozenset(),
|
||||
):
|
||||
"""Front gate for ``search_memories``: parse the window, route the fallback.
|
||||
|
||||
Returns ``(since_dt, before_dt, active, short_circuit)``.
|
||||
``short_circuit`` is a complete response to return verbatim — the
|
||||
``{"error": ...}`` payload for an invalid/inverted window, or the
|
||||
BM25-only fallback result when ``vector_disabled`` is set — and ``None``
|
||||
when the vector path should proceed. Extracted so the window plumbing
|
||||
doesn't push ``search_memories`` over the C901 complexity ceiling.
|
||||
"""
|
||||
try:
|
||||
since_dt, before_dt = parse_window(since, before)
|
||||
except ValueError as e:
|
||||
return None, None, False, {"error": str(e)}
|
||||
active = since_dt is not None or before_dt is not None
|
||||
if vector_disabled:
|
||||
return (
|
||||
since_dt,
|
||||
before_dt,
|
||||
active,
|
||||
_vector_disabled_with_window(
|
||||
query=query,
|
||||
palace_path=palace_path,
|
||||
wing=wing,
|
||||
room=room,
|
||||
n_results=n_results,
|
||||
collection_name=collection_name,
|
||||
source_file=source_file,
|
||||
since=since,
|
||||
before=before,
|
||||
since_dt=since_dt,
|
||||
before_dt=before_dt,
|
||||
stop_words=stop_words,
|
||||
),
|
||||
)
|
||||
return since_dt, before_dt, active, None
|
||||
|
||||
|
||||
def _candidate_out_of_scope(dist, meta, max_distance, since_dt, before_dt) -> bool:
|
||||
"""True when a drawer candidate fails the distance or date-window gate.
|
||||
|
||||
Distance is checked on the raw value before rounding to avoid precision
|
||||
loss (pre-existing behavior); the date window applies whenever a bound
|
||||
is set, with the shared ``[since, before)`` semantics.
|
||||
"""
|
||||
if max_distance > 0.0 and dist > max_distance:
|
||||
return True
|
||||
if (since_dt is not None or before_dt is not None) and not filed_at_in_window(
|
||||
meta.get("filed_at"), since_dt, before_dt
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _vector_disabled_with_window(
|
||||
*,
|
||||
query: str,
|
||||
palace_path: str,
|
||||
wing: str,
|
||||
room: str,
|
||||
n_results: int,
|
||||
collection_name: str,
|
||||
source_file: str,
|
||||
since: str,
|
||||
before: str,
|
||||
since_dt,
|
||||
before_dt,
|
||||
stop_words: frozenset = frozenset(),
|
||||
) -> dict:
|
||||
"""Run the BM25-only route and echo the raw window strings.
|
||||
|
||||
The fallback helper takes parsed bounds; the caller's raw ``since``/
|
||||
``before`` strings are stitched into the ``filters`` envelope here so
|
||||
both search paths report the same shape.
|
||||
"""
|
||||
result = _vector_disabled_search(
|
||||
query=query,
|
||||
palace_path=palace_path,
|
||||
wing=wing,
|
||||
room=room,
|
||||
n_results=n_results,
|
||||
collection_name=collection_name,
|
||||
source_file=source_file,
|
||||
since_dt=since_dt,
|
||||
before_dt=before_dt,
|
||||
stop_words=stop_words,
|
||||
)
|
||||
if "filters" in result:
|
||||
result["filters"]["since"] = since
|
||||
result["filters"]["before"] = before
|
||||
return result
|
||||
|
||||
|
||||
def _vector_disabled_search(
|
||||
*,
|
||||
query: str,
|
||||
|
|
@ -1183,6 +1484,8 @@ def _vector_disabled_search(
|
|||
collection_name: str,
|
||||
source_file: str = None,
|
||||
stop_words: frozenset = frozenset(),
|
||||
since_dt=None,
|
||||
before_dt=None,
|
||||
) -> dict:
|
||||
try:
|
||||
backend_name = resolve_backend_name(palace_path)
|
||||
|
|
@ -1206,6 +1509,8 @@ def _vector_disabled_search(
|
|||
n_results=n_results,
|
||||
collection_name=collection_name,
|
||||
stop_words=stop_words,
|
||||
since_dt=since_dt,
|
||||
before_dt=before_dt,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1299,6 +1604,8 @@ def search_memories(
|
|||
wing: str = None,
|
||||
room: str = None,
|
||||
source_file: str = None,
|
||||
since: str = None,
|
||||
before: str = None,
|
||||
n_results: int = 5,
|
||||
max_distance: float = 0.0,
|
||||
vector_disabled: bool = False,
|
||||
|
|
@ -1317,6 +1624,16 @@ def search_memories(
|
|||
room: Optional room filter.
|
||||
source_file: Optional exact source_file filter. Matches the full
|
||||
stored source_file value verbatim (#1815).
|
||||
since: Optional inclusive ISO date/datetime lower bound on a
|
||||
drawer's ``filed_at`` (ingest time, the ``created_at`` shown in
|
||||
results) — ``[since, before)`` window semantics shared with
|
||||
``list_drawers`` (#1128): wall-clock naive comparison, drawers
|
||||
with missing/unparseable ``filed_at`` excluded while a bound is
|
||||
active. Filtering happens after retrieval (ChromaDB rejects
|
||||
string operands for ``$gte``/``$lt``), so the candidate pool is
|
||||
widened via ``_candidate_pool_size`` — see
|
||||
``date_filter_pool_truncated`` in the response.
|
||||
before: Optional exclusive ISO upper bound; see ``since``.
|
||||
n_results: Max results to return.
|
||||
max_distance: Max cosine distance threshold. The palace collection uses
|
||||
cosine distance (hnsw:space=cosine) — 0 = identical, 2 = opposite.
|
||||
|
|
@ -1359,17 +1676,21 @@ def search_memories(
|
|||
# candidate gather) tokenizes against the same locale.
|
||||
stop_words = _resolve_stop_words(lang)
|
||||
|
||||
if vector_disabled:
|
||||
return _vector_disabled_search(
|
||||
query=query,
|
||||
palace_path=palace_path,
|
||||
wing=wing,
|
||||
room=room,
|
||||
n_results=n_results,
|
||||
collection_name=collection_name,
|
||||
source_file=source_file,
|
||||
stop_words=stop_words,
|
||||
)
|
||||
since_dt, before_dt, date_window_active, short_circuit = _window_and_fallback_gate(
|
||||
since,
|
||||
before,
|
||||
vector_disabled,
|
||||
query=query,
|
||||
palace_path=palace_path,
|
||||
wing=wing,
|
||||
room=room,
|
||||
n_results=n_results,
|
||||
collection_name=collection_name,
|
||||
source_file=source_file,
|
||||
stop_words=stop_words,
|
||||
)
|
||||
if short_circuit is not None:
|
||||
return short_circuit
|
||||
|
||||
drawers_col, open_error = _open_search_collection(palace_path, collection_name)
|
||||
if open_error:
|
||||
|
|
@ -1385,10 +1706,11 @@ def search_memories(
|
|||
# This avoids the "weak-closets regression" where narrative content
|
||||
# produces low-signal closets (regex extraction matches few topics)
|
||||
# and closet-first routing hides drawers that direct search would find.
|
||||
pool_size = _candidate_pool_size(n_results, date_window_active)
|
||||
try:
|
||||
dkwargs = {
|
||||
"query_texts": [query],
|
||||
"n_results": n_results * 3, # over-fetch for re-ranking
|
||||
"n_results": pool_size, # over-fetch for re-ranking
|
||||
"include": ["documents", "metadatas", "distances"],
|
||||
}
|
||||
if where:
|
||||
|
|
@ -1443,8 +1765,7 @@ def search_memories(
|
|||
):
|
||||
meta = meta or {}
|
||||
doc = doc or ""
|
||||
# Filter on raw distance before rounding to avoid precision loss.
|
||||
if max_distance > 0.0 and dist > max_distance:
|
||||
if _candidate_out_of_scope(dist, meta, max_distance, since_dt, before_dt):
|
||||
continue
|
||||
|
||||
meta = meta or {}
|
||||
|
|
@ -1574,16 +1895,24 @@ def search_memories(
|
|||
max_distance=max_distance,
|
||||
source_file=source_file,
|
||||
stop_words=stop_words,
|
||||
since_dt=since_dt,
|
||||
before_dt=before_dt,
|
||||
)
|
||||
if strategy_error:
|
||||
return strategy_error
|
||||
|
||||
return {
|
||||
"query": query,
|
||||
"filters": {"wing": wing, "room": room, "source_file": source_file},
|
||||
"total_before_filter": len(_first_or_empty(drawer_results, "documents")),
|
||||
"results": hits,
|
||||
}
|
||||
return _search_result_envelope(
|
||||
query=query,
|
||||
wing=wing,
|
||||
room=room,
|
||||
source_file=source_file,
|
||||
since=since,
|
||||
before=before,
|
||||
hits=hits,
|
||||
candidates_fetched=len(_first_or_empty(drawer_results, "documents")),
|
||||
pool_size=pool_size,
|
||||
date_window_active=date_window_active,
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -124,7 +124,13 @@ def test_cmd_status_custom_palace(mock_config_cls):
|
|||
def test_cmd_search_calls_search(mock_config_cls):
|
||||
mock_config_cls.return_value.palace_path = "/fake/palace"
|
||||
args = argparse.Namespace(
|
||||
palace=None, query="test query", wing="mywing", room="myroom", results=3
|
||||
palace=None,
|
||||
query="test query",
|
||||
wing="mywing",
|
||||
room="myroom",
|
||||
results=3,
|
||||
since="2026-04-01",
|
||||
before=None,
|
||||
)
|
||||
with patch("mempalace.searcher.search") as mock_search:
|
||||
cmd_search(args)
|
||||
|
|
@ -134,13 +140,17 @@ def test_cmd_search_calls_search(mock_config_cls):
|
|||
wing="mywing",
|
||||
room="myroom",
|
||||
n_results=3,
|
||||
since="2026-04-01",
|
||||
before=None,
|
||||
)
|
||||
|
||||
|
||||
@patch("mempalace.cli.MempalaceConfig")
|
||||
def test_cmd_search_error_exits(mock_config_cls):
|
||||
mock_config_cls.return_value.palace_path = "/fake/palace"
|
||||
args = argparse.Namespace(palace=None, query="q", wing=None, room=None, results=5)
|
||||
args = argparse.Namespace(
|
||||
palace=None, query="q", wing=None, room=None, results=5, since=None, before=None
|
||||
)
|
||||
from mempalace.searcher import SearchError
|
||||
|
||||
with patch("mempalace.searcher.search", side_effect=SearchError("fail")):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
"""Unit tests for mempalace.date_window — shared since/before parsing (#463).
|
||||
|
||||
The same helpers back the ``list_drawers`` filter (#1128) via aliases in
|
||||
``mcp_server``; behavioral coverage for that surface lives in
|
||||
``test_mcp_server.py``. These tests pin the module contract directly.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from mempalace.date_window import filed_at_in_window, parse_date_bound, parse_window
|
||||
|
||||
|
||||
class TestParseDateBound:
|
||||
def test_none_and_blank_mean_no_filter(self):
|
||||
assert parse_date_bound(None) is None
|
||||
assert parse_date_bound("") is None
|
||||
assert parse_date_bound(" ") is None
|
||||
|
||||
def test_date_only(self):
|
||||
assert parse_date_bound("2026-04-01") == datetime(2026, 4, 1)
|
||||
|
||||
def test_naive_datetime(self):
|
||||
assert parse_date_bound("2026-04-01T09:30:00") == datetime(2026, 4, 1, 9, 30)
|
||||
|
||||
def test_fractional_seconds(self):
|
||||
assert parse_date_bound("2026-04-01T09:30:00.250000") == datetime(
|
||||
2026, 4, 1, 9, 30, 0, 250000
|
||||
)
|
||||
|
||||
def test_zulu_datetime_parses_on_39_floor(self):
|
||||
assert parse_date_bound("2026-04-01T09:30:00Z") == datetime(2026, 4, 1, 9, 30)
|
||||
|
||||
def test_zulu_date_only(self):
|
||||
# Regression: appending "+00:00" instead of stripping Z broke this
|
||||
# exact shape on Python 3.9/3.10 (caught by review on #1891).
|
||||
assert parse_date_bound("2026-04-01Z") == datetime(2026, 4, 1)
|
||||
|
||||
def test_offset_dropped_wall_clock(self):
|
||||
assert parse_date_bound("2026-04-01T09:30:00+05:00") == datetime(2026, 4, 1, 9, 30)
|
||||
|
||||
def test_non_string_rejected(self):
|
||||
with pytest.raises(ValueError, match="since"):
|
||||
parse_date_bound(20260401, "since")
|
||||
|
||||
def test_garbage_rejected_names_field(self):
|
||||
with pytest.raises(ValueError, match="before"):
|
||||
parse_date_bound("next tuesday", "before")
|
||||
|
||||
|
||||
class TestParseWindow:
|
||||
def test_both_none(self):
|
||||
assert parse_window() == (None, None)
|
||||
|
||||
def test_valid_window(self):
|
||||
since_dt, before_dt = parse_window("2026-04-01", "2026-04-10")
|
||||
assert since_dt == datetime(2026, 4, 1)
|
||||
assert before_dt == datetime(2026, 4, 10)
|
||||
|
||||
def test_inverted_window_rejected(self):
|
||||
with pytest.raises(ValueError, match="must be earlier than"):
|
||||
parse_window("2026-04-10", "2026-04-01")
|
||||
|
||||
def test_equal_bounds_rejected(self):
|
||||
with pytest.raises(ValueError, match="must be earlier than"):
|
||||
parse_window("2026-04-01", "2026-04-01")
|
||||
|
||||
def test_invalid_since_names_field(self):
|
||||
with pytest.raises(ValueError, match="since"):
|
||||
parse_window("nope", None)
|
||||
|
||||
|
||||
class TestFiledAtInWindow:
|
||||
def test_no_bounds_accepts_anything(self):
|
||||
assert filed_at_in_window("2026-01-01T00:00:00", None, None)
|
||||
|
||||
def test_since_inclusive(self):
|
||||
since = datetime(2026, 1, 2)
|
||||
assert filed_at_in_window("2026-01-02T00:00:00", since, None)
|
||||
assert not filed_at_in_window("2026-01-01T23:59:59", since, None)
|
||||
|
||||
def test_before_exclusive(self):
|
||||
before = datetime(2026, 1, 4)
|
||||
assert filed_at_in_window("2026-01-03T23:59:59", None, before)
|
||||
assert not filed_at_in_window("2026-01-04T00:00:00", None, before)
|
||||
|
||||
def test_missing_filed_at_excluded_when_bound_active(self):
|
||||
assert not filed_at_in_window(None, datetime(2026, 1, 1), None)
|
||||
assert not filed_at_in_window("", datetime(2026, 1, 1), None)
|
||||
|
||||
def test_unparseable_filed_at_excluded(self):
|
||||
assert not filed_at_in_window("not-a-date", datetime(2026, 1, 1), None)
|
||||
|
||||
def test_aware_filed_at_compared_wall_clock(self):
|
||||
# diary_ingest stamps aware UTC ("+00:00"); the offset is dropped and
|
||||
# the wall-clock fields are compared.
|
||||
since = datetime(2026, 1, 5)
|
||||
assert filed_at_in_window("2026-01-05T00:00:00+00:00", since, None)
|
||||
assert not filed_at_in_window("2026-01-04T23:59:59+00:00", since, None)
|
||||
|
|
@ -7619,3 +7619,71 @@ def test_ensure_sqlite_integrity_status_joins_inflight_probe(monkeypatch):
|
|||
release_probe.set()
|
||||
background.join(5)
|
||||
consumer_thread.join(5)
|
||||
|
||||
|
||||
class TestSearchDateFilters:
|
||||
"""tool_search since/before window (#463) — MCP surface.
|
||||
|
||||
Window semantics and helpers are shared with list_drawers (#1128) via
|
||||
mempalace.date_window; seeded filed_at values are 2026-01-01..01-04.
|
||||
"""
|
||||
|
||||
BROAD = "authentication database frontend sprint planning"
|
||||
|
||||
def test_search_since_inclusive(self, monkeypatch, config, palace_path, seeded_collection, kg):
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace.mcp_server import tool_search
|
||||
|
||||
result = tool_search(self.BROAD, limit=10, since="2026-01-03")
|
||||
assert "error" not in result
|
||||
got = sorted(r["created_at"][:10] for r in result["results"])
|
||||
assert got == ["2026-01-03", "2026-01-04"]
|
||||
|
||||
def test_search_window_composes_with_wing(
|
||||
self, monkeypatch, config, palace_path, seeded_collection, kg
|
||||
):
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace.mcp_server import tool_search
|
||||
|
||||
result = tool_search(self.BROAD, limit=10, wing="project", before="2026-01-03")
|
||||
got = {(r["wing"], r["created_at"][:10]) for r in result["results"]}
|
||||
assert got == {("project", "2026-01-01"), ("project", "2026-01-02")}
|
||||
|
||||
def test_search_invalid_since_is_clean_error(
|
||||
self, monkeypatch, config, palace_path, seeded_collection, kg
|
||||
):
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace.mcp_server import tool_search
|
||||
|
||||
result = tool_search("anything", since="next tuesday")
|
||||
assert set(result) == {"error"}
|
||||
assert "since" in result["error"]
|
||||
|
||||
def test_search_inverted_window_is_clean_error(
|
||||
self, monkeypatch, config, palace_path, seeded_collection, kg
|
||||
):
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace.mcp_server import tool_search
|
||||
|
||||
result = tool_search("anything", since="2026-01-04", before="2026-01-01")
|
||||
assert set(result) == {"error"}
|
||||
assert "must be earlier than" in result["error"]
|
||||
|
||||
def test_search_filters_envelope_includes_window(
|
||||
self, monkeypatch, config, palace_path, seeded_collection, kg
|
||||
):
|
||||
_patch_mcp_server(monkeypatch, config, kg)
|
||||
from mempalace.mcp_server import tool_search
|
||||
|
||||
result = tool_search(self.BROAD, since="2026-01-02", before="2026-01-04")
|
||||
assert result["filters"]["since"] == "2026-01-02"
|
||||
assert result["filters"]["before"] == "2026-01-04"
|
||||
|
||||
def test_search_schema_declares_window_properties(self):
|
||||
from mempalace.mcp_server import TOOLS
|
||||
|
||||
schema = TOOLS["mempalace_search"]["input_schema"]
|
||||
assert "since" in schema["properties"]
|
||||
assert "before" in schema["properties"]
|
||||
assert schema["properties"]["since"]["type"] == "string"
|
||||
assert schema["properties"]["before"]["type"] == "string"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ plus mock-based tests for error paths.
|
|||
"""
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -638,6 +639,53 @@ class TestSearchCLI:
|
|||
|
||||
assert seen["stop_words"] == frozenset({"the"})
|
||||
|
||||
def test_search_forwards_date_window_to_bm25_fallback_when_hnsw_diverged(
|
||||
self, fake_palace_path
|
||||
):
|
||||
"""A `--since`/`--before` window must survive the diverged-index detour.
|
||||
|
||||
The fence returns BM25-only results before the vector path runs, and
|
||||
that fallback reads drawers straight from sqlite. Unless the window
|
||||
travels with it, the CLI answers a wider question than the caller
|
||||
asked — silently, with no notice that the filter was dropped. A
|
||||
degraded index may cost ranking quality; it must never cost the
|
||||
filter.
|
||||
"""
|
||||
seen = {}
|
||||
|
||||
def _spy_bm25(**kwargs):
|
||||
seen.update(kwargs)
|
||||
return {"query": "anything", "filters": {}, "total_before_filter": 0, "results": []}
|
||||
|
||||
with (
|
||||
patch("mempalace.searcher.resolve_backend_name", return_value="chroma"),
|
||||
patch(
|
||||
"mempalace.backends.chroma.hnsw_capacity_status",
|
||||
return_value={"diverged": True, "message": "test divergence"},
|
||||
),
|
||||
patch("mempalace.searcher._bm25_only_via_sqlite", side_effect=_spy_bm25),
|
||||
):
|
||||
search("anything", fake_palace_path, since="2026-01-01", before="2026-02-01")
|
||||
|
||||
assert seen["since_dt"] == datetime(2026, 1, 1)
|
||||
assert seen["before_dt"] == datetime(2026, 2, 1)
|
||||
|
||||
def test_search_rejects_inverted_window_before_probing_a_diverged_index(self, fake_palace_path):
|
||||
"""An inverted window is a caller error, so it must raise the same way
|
||||
whether the index is healthy or diverged. If the fence ran first it
|
||||
would swallow the mistake and answer with unfiltered BM25 results."""
|
||||
with (
|
||||
patch("mempalace.searcher.resolve_backend_name", return_value="chroma"),
|
||||
patch(
|
||||
"mempalace.backends.chroma.hnsw_capacity_status",
|
||||
return_value={"diverged": True, "message": "test divergence"},
|
||||
),
|
||||
patch("mempalace.searcher._bm25_only_via_sqlite") as mock_bm25,
|
||||
):
|
||||
with pytest.raises(SearchError, match="must be earlier than"):
|
||||
search("anything", fake_palace_path, since="2026-02-01", before="2026-01-01")
|
||||
mock_bm25.assert_not_called()
|
||||
|
||||
def test_search_does_not_run_chroma_probe_for_other_backends(self, fake_palace_path, capsys):
|
||||
"""The HNSW guard is Chroma-specific and must not fence other backends."""
|
||||
mock_col = MagicMock()
|
||||
|
|
@ -1047,3 +1095,441 @@ class TestResultDrawerId:
|
|||
|
||||
def test_missing_metadata_falls_back_to_stored_id(self):
|
||||
assert _result_drawer_id(None, "drawer_abc") == "drawer_abc"
|
||||
|
||||
|
||||
# ── since/before date window (#463) ────────────────────────────────────
|
||||
|
||||
|
||||
class TestSearchMemoriesDateFilter:
|
||||
"""search_memories accepts since/before ISO bounds filtered on filed_at.
|
||||
|
||||
Window semantics mirror list_drawers (#1128): since inclusive, before
|
||||
exclusive, wall-clock naive comparison, undated drawers excluded while
|
||||
a bound is active. Seeded filed_at values: aaa=01-01, bbb=01-02,
|
||||
ccc=01-03, ddd=01-04 (see conftest seeded_collection).
|
||||
"""
|
||||
|
||||
BROAD = "authentication database frontend sprint planning"
|
||||
|
||||
def test_since_narrows_to_newer_drawers(self, palace_path, seeded_collection):
|
||||
result = search_memories(self.BROAD, palace_path, n_results=10, since="2026-01-03")
|
||||
assert result["results"], "expected in-window hits"
|
||||
assert all(r["created_at"] >= "2026-01-03" for r in result["results"])
|
||||
|
||||
def test_before_narrows_to_older_drawers(self, palace_path, seeded_collection):
|
||||
result = search_memories(self.BROAD, palace_path, n_results=10, before="2026-01-02")
|
||||
assert result["results"]
|
||||
assert all(r["created_at"] < "2026-01-02" for r in result["results"])
|
||||
|
||||
def test_window_both_bounds(self, palace_path, seeded_collection):
|
||||
result = search_memories(
|
||||
self.BROAD, palace_path, n_results=10, since="2026-01-02", before="2026-01-04"
|
||||
)
|
||||
got = sorted(r["created_at"][:10] for r in result["results"])
|
||||
assert got == ["2026-01-02", "2026-01-03"]
|
||||
|
||||
def test_since_boundary_inclusive(self, palace_path, seeded_collection):
|
||||
result = search_memories(self.BROAD, palace_path, n_results=10, since="2026-01-04")
|
||||
assert [r["created_at"][:10] for r in result["results"]] == ["2026-01-04"]
|
||||
|
||||
def test_before_boundary_exclusive(self, palace_path, seeded_collection):
|
||||
result = search_memories(self.BROAD, palace_path, n_results=10, before="2026-01-04")
|
||||
assert "2026-01-04" not in [r["created_at"][:10] for r in result["results"]]
|
||||
assert len(result["results"]) == 3
|
||||
|
||||
def test_invalid_since_returns_error(self, palace_path, seeded_collection):
|
||||
result = search_memories(self.BROAD, palace_path, since="next tuesday")
|
||||
assert "error" in result
|
||||
assert "since" in result["error"]
|
||||
|
||||
def test_inverted_window_returns_error(self, palace_path, seeded_collection):
|
||||
result = search_memories(self.BROAD, palace_path, since="2026-01-04", before="2026-01-01")
|
||||
assert "error" in result
|
||||
assert "must be earlier than" in result["error"]
|
||||
|
||||
def test_undated_drawer_excluded_while_bound_active(self, palace_path, seeded_collection):
|
||||
seeded_collection.upsert(
|
||||
ids=["undated1"],
|
||||
documents=["Undated planning note about authentication frontend database."],
|
||||
metadatas=[{"wing": "notes", "room": "planning", "source_file": "undated.md"}],
|
||||
)
|
||||
result = search_memories(self.BROAD, palace_path, n_results=10, since="2026-01-01")
|
||||
assert "undated.md" not in [r["source_file"] for r in result["results"]]
|
||||
# ...but with no bound it is searchable as before.
|
||||
result = search_memories(self.BROAD, palace_path, n_results=10)
|
||||
assert "undated.md" in [r["source_file"] for r in result["results"]]
|
||||
|
||||
def test_aware_filed_at_matches_wall_clock(self, palace_path, seeded_collection):
|
||||
# diary_ingest stamps aware UTC; the window compares wall-clock fields.
|
||||
seeded_collection.upsert(
|
||||
ids=["aware1"],
|
||||
documents=["Aware planning entry about database sprint authentication."],
|
||||
metadatas=[
|
||||
{
|
||||
"wing": "notes",
|
||||
"room": "planning",
|
||||
"source_file": "aware.md",
|
||||
"filed_at": "2026-01-05T12:00:00+00:00",
|
||||
}
|
||||
],
|
||||
)
|
||||
result = search_memories(self.BROAD, palace_path, n_results=10, since="2026-01-05")
|
||||
assert [r["source_file"] for r in result["results"]] == ["aware.md"]
|
||||
|
||||
def test_filters_envelope_echoes_window(self, palace_path, seeded_collection):
|
||||
result = search_memories(self.BROAD, palace_path, since="2026-01-02", before="2026-01-03")
|
||||
assert result["filters"]["since"] == "2026-01-02"
|
||||
assert result["filters"]["before"] == "2026-01-03"
|
||||
|
||||
def test_no_window_keeps_filters_none(self, palace_path, seeded_collection):
|
||||
result = search_memories("authentication", palace_path)
|
||||
assert result["filters"]["since"] is None
|
||||
assert result["filters"]["before"] is None
|
||||
|
||||
def test_window_composes_with_wing_filter(self, palace_path, seeded_collection):
|
||||
result = search_memories(
|
||||
self.BROAD, palace_path, n_results=10, wing="project", since="2026-01-02"
|
||||
)
|
||||
got = {(r["wing"], r["created_at"][:10]) for r in result["results"]}
|
||||
assert got == {("project", "2026-01-02"), ("project", "2026-01-03")}
|
||||
|
||||
def test_max_distance_zero_results_stay_empty_not_error(self, palace_path, seeded_collection):
|
||||
result = search_memories("zebra quantum blockchain", palace_path, since="2027-01-01")
|
||||
assert "error" not in result
|
||||
assert result["results"] == []
|
||||
|
||||
def test_date_window_widens_candidate_pool(self, palace_path):
|
||||
# 12 near-duplicate drawers embed closest to the query; the one
|
||||
# in-window drawer is textually farther, so the historical 3x pool
|
||||
# (n_results=2 -> 6) would never contain it. The widened window
|
||||
# pool must recover it: recall is the design requirement.
|
||||
from mempalace.palace import get_collection
|
||||
|
||||
col = get_collection(palace_path, create=True)
|
||||
ids, docs, metas = [], [], []
|
||||
for i in range(12):
|
||||
ids.append(f"near{i}")
|
||||
docs.append(f"Weekly budget meeting notes revision {i} about spending review.")
|
||||
metas.append(
|
||||
{
|
||||
"wing": "fin",
|
||||
"room": "budget",
|
||||
"source_file": f"near{i}.md",
|
||||
"filed_at": "2026-02-01T00:00:00",
|
||||
}
|
||||
)
|
||||
ids.append("target")
|
||||
docs.append("Quarterly offsite retrospective and travel logistics summary.")
|
||||
metas.append(
|
||||
{
|
||||
"wing": "fin",
|
||||
"room": "budget",
|
||||
"source_file": "target.md",
|
||||
"filed_at": "2026-03-01T00:00:00",
|
||||
}
|
||||
)
|
||||
col.upsert(ids=ids, documents=docs, metadatas=metas)
|
||||
result = search_memories(
|
||||
"budget meeting spending review",
|
||||
palace_path,
|
||||
n_results=2,
|
||||
since="2026-02-15",
|
||||
)
|
||||
assert [r["source_file"] for r in result["results"]] == ["target.md"]
|
||||
|
||||
def test_bm25_fallback_respects_window(self, palace_path, seeded_collection):
|
||||
result = search_memories(
|
||||
"authentication database frontend sprint planning tokens",
|
||||
palace_path,
|
||||
n_results=10,
|
||||
vector_disabled=True,
|
||||
since="2026-01-02",
|
||||
before="2026-01-04",
|
||||
)
|
||||
assert result.get("fallback") == "bm25_only_via_sqlite"
|
||||
assert result["results"], "expected in-window bm25 hits"
|
||||
assert sorted(r["created_at"][:10] for r in result["results"]) == [
|
||||
"2026-01-02",
|
||||
"2026-01-03",
|
||||
]
|
||||
assert result["filters"]["since"] == "2026-01-02"
|
||||
|
||||
def test_bm25_fallback_includes_bare_date_filed_at_on_since_boundary(
|
||||
self, palace_path, seeded_collection
|
||||
):
|
||||
# A bare-date filed_at equal to the since day is in-window
|
||||
# (since is inclusive, parsed as midnight). The SQL prefilter must
|
||||
# not drop it before the authoritative Python check: lexicographic
|
||||
# "2026-01-02" < "2026-01-02T00:00:00", so a full-isoformat lower
|
||||
# bound would exclude it at the SQL layer where Python can't
|
||||
# recover it (review finding on the #463 change).
|
||||
seeded_collection.upsert(
|
||||
ids=["bare1", "space1", "zulu1"],
|
||||
documents=[
|
||||
"Bare-date drawer about the database sprint planning.",
|
||||
"Space-separated drawer about the database sprint planning.",
|
||||
"Zulu drawer about the database sprint planning.",
|
||||
],
|
||||
metadatas=[
|
||||
{
|
||||
"wing": "notes",
|
||||
"room": "planning",
|
||||
"source_file": "bare.md",
|
||||
"filed_at": "2026-01-02",
|
||||
},
|
||||
{
|
||||
"wing": "notes",
|
||||
"room": "planning",
|
||||
"source_file": "space.md",
|
||||
# sqlite CURRENT_TIMESTAMP style: space separator sorts
|
||||
# before "T" and a full-isoformat SQL bound would drop
|
||||
# the whole boundary day.
|
||||
"filed_at": "2026-01-02 09:30:00",
|
||||
},
|
||||
{
|
||||
"wing": "notes",
|
||||
"room": "planning",
|
||||
"source_file": "zulu.md",
|
||||
# "Z" sorts after a fractional bound; the day-granular
|
||||
# upper prefilter must keep it for the Python check.
|
||||
"filed_at": "2026-01-02T10:00:00Z",
|
||||
},
|
||||
],
|
||||
)
|
||||
result = search_memories(
|
||||
"database sprint planning",
|
||||
palace_path,
|
||||
n_results=10,
|
||||
vector_disabled=True,
|
||||
since="2026-01-02",
|
||||
before="2026-01-02T10:00:00.500000",
|
||||
)
|
||||
assert result.get("fallback") == "bm25_only_via_sqlite"
|
||||
got = [r["source_file"] for r in result["results"]]
|
||||
assert "bare.md" in got
|
||||
assert "space.md" in got
|
||||
assert "zulu.md" in got
|
||||
|
||||
def test_bm25_fallback_invalid_since_errors(self, palace_path, seeded_collection):
|
||||
result = search_memories(
|
||||
"authentication", palace_path, vector_disabled=True, since="garbage"
|
||||
)
|
||||
assert "error" in result
|
||||
assert "since" in result["error"]
|
||||
|
||||
def test_pool_truncated_flag_set_when_widened_pool_full(self, palace_path):
|
||||
# n_results=1 -> widened pool = 15; seed 16 in-window drawers so the
|
||||
# backend returns a full pool and the honesty flag must fire.
|
||||
from mempalace.palace import get_collection
|
||||
|
||||
col = get_collection(palace_path, create=True)
|
||||
ids, docs, metas = [], [], []
|
||||
for i in range(16):
|
||||
ids.append(f"flag{i}")
|
||||
docs.append(f"standup summary entry number {i} about deploy status.")
|
||||
metas.append(
|
||||
{
|
||||
"wing": "ops",
|
||||
"room": "standup",
|
||||
"source_file": f"flag{i}.md",
|
||||
"filed_at": "2026-05-01T00:00:00",
|
||||
}
|
||||
)
|
||||
col.upsert(ids=ids, documents=docs, metadatas=metas)
|
||||
result = search_memories(
|
||||
"standup deploy status", palace_path, n_results=1, since="2026-04-01"
|
||||
)
|
||||
assert result.get("date_filter_pool_truncated") is True
|
||||
assert result["total_before_filter"] >= 15
|
||||
|
||||
def test_pool_truncated_flag_absent_on_small_corpus(self, palace_path, seeded_collection):
|
||||
result = search_memories(
|
||||
"authentication database", palace_path, n_results=5, since="2026-01-01"
|
||||
)
|
||||
assert "date_filter_pool_truncated" not in result
|
||||
|
||||
def test_bm25_fallback_survives_calendar_ceiling_before(self, palace_path, seeded_collection):
|
||||
# before="9999-12-31" is a plausible open-ended sentinel; the
|
||||
# day-granular SQL prefilter must not overflow past datetime.max
|
||||
# on the resilience path (it degrades to no SQL narrowing and the
|
||||
# Python filter decides).
|
||||
result = search_memories(
|
||||
"authentication database frontend sprint planning",
|
||||
palace_path,
|
||||
n_results=10,
|
||||
vector_disabled=True,
|
||||
since="2026-01-01",
|
||||
before="9999-12-31",
|
||||
)
|
||||
assert "error" not in result
|
||||
assert result.get("fallback") == "bm25_only_via_sqlite"
|
||||
assert len(result["results"]) == 4
|
||||
|
||||
def test_bm25_fallback_pool_truncated_flag(self, palace_path, seeded_collection):
|
||||
# Direct call with a tiny max_candidates: the FTS page comes back
|
||||
# full under an active window -> the same honesty flag as the
|
||||
# vector path; without a window the key stays absent.
|
||||
from datetime import datetime
|
||||
|
||||
from mempalace.searcher import _bm25_only_via_sqlite
|
||||
|
||||
truncated = _bm25_only_via_sqlite(
|
||||
"authentication database frontend sprint",
|
||||
palace_path,
|
||||
n_results=2,
|
||||
max_candidates=2,
|
||||
since_dt=datetime(2026, 1, 1),
|
||||
before_dt=None,
|
||||
)
|
||||
assert truncated.get("date_filter_pool_truncated") is True
|
||||
unwindowed = _bm25_only_via_sqlite(
|
||||
"authentication database frontend sprint",
|
||||
palace_path,
|
||||
n_results=2,
|
||||
max_candidates=2,
|
||||
)
|
||||
assert "date_filter_pool_truncated" not in unwindowed
|
||||
|
||||
def test_union_strategy_respects_window(self, palace_path, seeded_collection):
|
||||
# The out-of-window drawer has the strongest lexical signal for the
|
||||
# query; union mode must not smuggle it past the window.
|
||||
seeded_collection.upsert(
|
||||
ids=["lex1"],
|
||||
documents=["passkeys passkeys passkeys rollout checklist."],
|
||||
metadatas=[
|
||||
{
|
||||
"wing": "notes",
|
||||
"room": "planning",
|
||||
"source_file": "lex.md",
|
||||
"filed_at": "2026-01-01T00:00:00",
|
||||
}
|
||||
],
|
||||
)
|
||||
result = search_memories(
|
||||
"passkeys rollout",
|
||||
palace_path,
|
||||
n_results=5,
|
||||
candidate_strategy="union",
|
||||
since="2026-01-02",
|
||||
)
|
||||
assert "error" not in result
|
||||
assert "lex.md" not in [r["source_file"] for r in result["results"]]
|
||||
|
||||
def test_chunked_entry_is_windowed_whole_and_reported_logically(
|
||||
self, palace_path, seeded_collection
|
||||
):
|
||||
"""A chunked entry meets the window as one unit and keeps its logical id.
|
||||
|
||||
Chunk rows are stored one per physical chunk and each carries the
|
||||
group's ``filed_at``, so the window decides the whole entry; the hit
|
||||
itself resolves to ``parent_entry_id`` (#2185). Neither side of that
|
||||
pair is exercised by the other's tests: a physical chunk id leaking
|
||||
into a date-filtered result would not round-trip through
|
||||
``mempalace_get_drawer``, and a chunk row read as undated would drop
|
||||
a dated entry out of an active window.
|
||||
"""
|
||||
seeded_collection.upsert(
|
||||
ids=[
|
||||
"diary_notes_ana_20260105_1_chunk_000000",
|
||||
"diary_notes_ana_20260105_1_chunk_000001",
|
||||
],
|
||||
documents=[
|
||||
"Passkey rollout retro, part one: what the migration plan got right.",
|
||||
"Passkey rollout retro, part two: what the fallback plan missed.",
|
||||
],
|
||||
metadatas=[
|
||||
{
|
||||
"wing": "notes",
|
||||
"room": "diary",
|
||||
"chunk_index": index,
|
||||
"parent_entry_id": "diary_notes_ana_20260105_1",
|
||||
"filed_at": "2026-01-05T00:00:00",
|
||||
}
|
||||
for index in range(2)
|
||||
],
|
||||
)
|
||||
|
||||
inside = search_memories(
|
||||
"passkey rollout retro", palace_path, n_results=10, since="2026-01-05"
|
||||
)
|
||||
returned = [r["drawer_id"] for r in inside["results"]]
|
||||
assert "diary_notes_ana_20260105_1" in returned
|
||||
assert not any("_chunk_" in drawer_id for drawer_id in returned)
|
||||
|
||||
outside = search_memories(
|
||||
"passkey rollout retro", palace_path, n_results=10, before="2026-01-05"
|
||||
)
|
||||
excluded = [r["drawer_id"] for r in outside["results"]]
|
||||
assert "diary_notes_ana_20260105_1" not in excluded
|
||||
assert not any("_chunk_" in drawer_id for drawer_id in excluded)
|
||||
|
||||
|
||||
class TestCliSearchDateFilter:
|
||||
"""The printing CLI path accepts the same since/before window."""
|
||||
|
||||
def test_cli_search_since_filters_output(self, palace_path, seeded_collection, capsys):
|
||||
search(
|
||||
"authentication database frontend sprint planning",
|
||||
palace_path,
|
||||
n_results=10,
|
||||
since="2026-01-04",
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
assert "sprint.md" in out
|
||||
assert "auth.py" not in out
|
||||
assert "db.py" not in out
|
||||
|
||||
def test_cli_search_reranks_full_window_pool_before_trim(self, fake_palace_path, capsys):
|
||||
# Regression for the review finding: under an active window the CLI
|
||||
# must hybrid-re-rank ALL in-window survivors and trim to n_results
|
||||
# AFTER the re-rank. A BM25-strong drawer sitting deep in the
|
||||
# vector ordering (position 25 of 30) must still surface in the
|
||||
# printed top-2; trimming before the re-rank would cut it at
|
||||
# position n_results and it could never appear.
|
||||
mock_col = MagicMock()
|
||||
mock_col.metadata = {"hnsw:space": "cosine"}
|
||||
docs, metas, dists = [], [], []
|
||||
for i in range(30):
|
||||
text = "unrelated filler paragraph number {}".format(i)
|
||||
if i == 25:
|
||||
text = "quixotic zephyr baseline report" # exact query tokens
|
||||
docs.append(text)
|
||||
metas.append(
|
||||
{
|
||||
"wing": "w",
|
||||
"room": "r",
|
||||
"source_file": "doc{}.md".format(i),
|
||||
"filed_at": "2026-01-10T00:00:00",
|
||||
}
|
||||
)
|
||||
dists.append(0.30 + i * 0.01) # strictly increasing vector distance
|
||||
mock_col.query.return_value = {
|
||||
"documents": [docs],
|
||||
"metadatas": [metas],
|
||||
"distances": [dists],
|
||||
}
|
||||
with patch("mempalace.searcher.get_collection", return_value=mock_col):
|
||||
search(
|
||||
"quixotic zephyr baseline",
|
||||
fake_palace_path,
|
||||
n_results=2,
|
||||
since="2026-01-01",
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
assert "doc25.md" in out
|
||||
# The widened pool was requested from the backend, not just n_results.
|
||||
assert mock_col.query.call_args.kwargs["n_results"] > 2
|
||||
|
||||
def test_cli_search_invalid_since_raises_search_error(
|
||||
self, palace_path, seeded_collection, capsys
|
||||
):
|
||||
import pytest
|
||||
|
||||
with pytest.raises(SearchError, match="since"):
|
||||
search("anything", palace_path, since="garbage")
|
||||
|
||||
def test_cli_search_inverted_window_raises(self, palace_path, seeded_collection):
|
||||
import pytest
|
||||
|
||||
with pytest.raises(SearchError, match="must be earlier than"):
|
||||
search("anything", palace_path, since="2026-01-04", before="2026-01-01")
|
||||
|
|
|
|||
Loading…
Reference in New Issue