diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ee5d4a..546f110 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Uncalibrated recall scores moved to their own name** — `KEYWORD` and + single-route `VECTOR` searches now report their top score as + `recall_top_score_raw`; `recall_top_score` is reserved for the calibrated + methods (`HYBRID` LR sigmoid, `AGENTIC` cross-encoder), whose values share a + comparable `[0, 1]` scale. Langfuse aggregates scores by name, so the previous + single name meant a chart could average an unbounded BM25 score together with + a probability. Every recall score also carries + `metadata = {"method": ..., "calibrated": ...}` now, a structured field that + can be split on, alongside the existing human-readable comment. Dashboards + built on `recall_top_score` for keyword search need to switch to the new name. + ## [1.2.0] - 2026-07-24 ### Added diff --git a/examples/langfuse/README.md b/examples/langfuse/README.md index fdf8b34..2376bac 100644 --- a/examples/langfuse/README.md +++ b/examples/langfuse/README.md @@ -50,10 +50,13 @@ zero tracing overhead. | query / recall embedding | embedding `everos.embedding` | | OME reflection strategies | agent `everos.ome.` (linked to the triggering request's trace) | -`langfuse.session.id` / `langfuse.user.id` group the traces; recall quality is -pushed as Langfuse scores (`recall_top_score` always, and `recall_hit` for -calibrated methods — HYBRID / AGENTIC). Query and memory text are captured only -when `capture_content = true`. +`langfuse.session.id` / `langfuse.user.id` group the traces. Recall quality is +pushed as Langfuse scores, split by whether the method's score is calibrated: +`recall_top_score` plus `recall_hit` for HYBRID / AGENTIC (comparable `[0, 1]`), +and `recall_top_score_raw` for KEYWORD / single-route VECTOR, whose raw BM25 or +cosine values are on a different scale and must not be averaged in with the +calibrated ones. Query and memory text are captured only when +`capture_content = true`. ## Try it diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index 539e68e..eb8eba4 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -161,5 +161,8 @@ capture_content = false # Recall-quality scores pushed to Langfuse (Langfuse-specific REST, off the # OTLP stream). Only fires when langfuse_public_key/secret_key/host are set # (via everos.toml or EVEROS_OBSERVABILITY__LANGFUSE_* — secrets, not shipped here). +# Score names: "recall_top_score" for the calibrated methods (HYBRID / AGENTIC, +# [0, 1] and comparable), "recall_top_score_raw" for the uncalibrated ones +# (KEYWORD BM25 / single-route VECTOR), plus "recall_hit" for calibrated only. emit_recall_scores = true recall_hit_threshold = 0.6 # only meaningful for calibrated methods diff --git a/src/everos/core/observability/tracing/scores.py b/src/everos/core/observability/tracing/scores.py index 6256acb..3879f42 100644 --- a/src/everos/core/observability/tracing/scores.py +++ b/src/everos/core/observability/tracing/scores.py @@ -14,6 +14,9 @@ queue + a single background worker: Attaches to the originating span via ``traceId`` (OTel trace_id, 032x hex) + ``observationId`` (OTel span_id, 016x hex) — exactly the mapping Langfuse's OTLP ingestion uses. + +Calibrated and uncalibrated top scores go out under *different names* — see +:data:`SCORE_TOP_CALIBRATED` / :data:`SCORE_TOP_RAW`. """ from __future__ import annotations @@ -34,6 +37,16 @@ logger = get_logger(__name__) Sender = Callable[[dict], Awaitable[None]] +# A Langfuse chart aggregates scores by *name*, so a single name may only ever +# carry values on one scale. HYBRID / AGENTIC top scores are calibrated to a +# comparable [0, 1]; KEYWORD (unbounded BM25) and single-route VECTOR are not, +# and averaging the two together would be meaningless. Hence two names: a +# dashboard built on ``recall_top_score`` stays comparable across methods and +# over time, and the raw scores remain available under their own name. +SCORE_TOP_CALIBRATED = "recall_top_score" +SCORE_TOP_RAW = "recall_top_score_raw" +SCORE_HIT = "recall_hit" + @dataclass(frozen=True) class ScoreRecord: @@ -44,6 +57,7 @@ class ScoreRecord: name: str value: float comment: str | None + metadata: dict[str, object] | None = None def _to_payload(record: ScoreRecord) -> dict: @@ -57,6 +71,8 @@ def _to_payload(record: ScoreRecord) -> dict: payload["observationId"] = record.observation_id if record.comment: payload["comment"] = record.comment + if record.metadata: + payload["metadata"] = record.metadata return payload @@ -165,24 +181,41 @@ def emit_recall_scores( hit: bool | None, method: str, ) -> None: - """Enqueue recall_top_score (always) + recall_hit (only when ``hit`` is - a verdict); no-op when the sink is off. + """Enqueue a top score (always) + recall_hit (only when ``hit`` is a + verdict); no-op when the sink is off. - ``hit=None`` means the method's score is uncalibrated (KEYWORD/VECTOR), - so no hit verdict is pushed — only the raw top score. + ``hit=None`` means the method's score is uncalibrated (KEYWORD / + single-route VECTOR): no hit verdict is pushed, and the top score is + reported as :data:`SCORE_TOP_RAW` instead of :data:`SCORE_TOP_CALIBRATED` + so the two scales never land under one score name. + + ``method`` rides along as score metadata (a structured field Langfuse + persists) as well as in the human-readable comment. """ if _sink is None: return + calibrated = hit is not None comment = f"method={method}" + metadata: dict[str, object] = {"method": method, "calibrated": calibrated} _sink.enqueue( ScoreRecord( - trace_id, observation_id, "recall_top_score", float(top_score), comment + trace_id, + observation_id, + SCORE_TOP_CALIBRATED if calibrated else SCORE_TOP_RAW, + float(top_score), + comment, + metadata, ) ) if hit is not None: _sink.enqueue( ScoreRecord( - trace_id, observation_id, "recall_hit", 1.0 if hit else 0.0, comment + trace_id, + observation_id, + SCORE_HIT, + 1.0 if hit else 0.0, + comment, + metadata, ) ) diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 14c1d27..4ba874b 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -144,7 +144,8 @@ def _top_score(data: SearchData) -> float: # (HYBRID → LR sigmoid, AGENTIC → cross-encoder), so the recall_hit # threshold is meaningful. KEYWORD (unbounded BM25) and single-route # VECTOR are excluded — a fixed threshold there yields a misleading -# near-constant "hit" that inflates cross-method dashboards. +# near-constant "hit" that inflates cross-method dashboards. Their top +# score is also reported under a separate score name (see ``scores``). _CALIBRATED_METHODS = frozenset({SearchMethod.HYBRID, SearchMethod.AGENTIC}) diff --git a/tests/unit/test_core/test_observability/test_scores.py b/tests/unit/test_core/test_observability/test_scores.py index 40ff0c7..db78bac 100644 --- a/tests/unit/test_core/test_observability/test_scores.py +++ b/tests/unit/test_core/test_observability/test_scores.py @@ -14,8 +14,12 @@ from pydantic import SecretStr from everos.config.settings import ObservabilitySettings from everos.core.observability.tracing.scores import ( + SCORE_HIT, + SCORE_TOP_CALIBRATED, + SCORE_TOP_RAW, RecallScoreSink, ScoreRecord, + emit_recall_scores, init_score_sink, shutdown_score_sink, ) @@ -79,6 +83,93 @@ async def test_worker_sends_payload_in_langfuse_shape() -> None: } +async def test_worker_forwards_score_metadata() -> None: + """``metadata`` is a structured field Langfuse persists, so a dashboard can + split by method without parsing the free-text comment.""" + sent: list[dict] = [] + done = asyncio.Event() + + async def sender(payload: dict) -> None: + sent.append(payload) + done.set() + + sink = RecallScoreSink(sender=sender, max_queue=10) + sink.start() + sink.enqueue( + ScoreRecord("tid", "oid", "n", 0.5, "method=keyword", {"method": "keyword"}) + ) + await asyncio.wait_for(done.wait(), timeout=1.0) + await sink.stop() + + assert sent[0]["metadata"] == {"method": "keyword"} + + +async def _emitted(**kwargs: object) -> list[dict]: + """Install a capturing sink, run ``emit_recall_scores``, return the payloads.""" + from everos.core.observability.tracing import scores as scores_mod + + sent: list[dict] = [] + + async def sender(payload: dict) -> None: + sent.append(payload) + + sink = RecallScoreSink(sender=sender, max_queue=10) + sink.start() + previous, scores_mod._sink = scores_mod._sink, sink + try: + emit_recall_scores(**kwargs) # type: ignore[arg-type] + await sink.stop() # drains before returning + finally: + scores_mod._sink = previous + return sent + + +async def test_calibrated_method_reports_top_score_and_hit() -> None: + sent = await _emitted( + trace_id="tid", + observation_id="oid", + top_score=0.72, + hit=True, + method="hybrid", + ) + + assert [s["name"] for s in sent] == [SCORE_TOP_CALIBRATED, SCORE_HIT] + assert [s["value"] for s in sent] == [0.72, 1.0] + assert all(s["metadata"] == {"method": "hybrid", "calibrated": True} for s in sent) + + +async def test_uncalibrated_method_reports_raw_name_and_no_hit() -> None: + """An unbounded BM25 top score must not land under the same score name as a + calibrated probability — a chart on that name would average both scales.""" + sent = await _emitted( + trace_id="tid", + observation_id="oid", + top_score=8.4, + hit=None, + method="keyword", + ) + + assert [s["name"] for s in sent] == [SCORE_TOP_RAW] + assert sent[0]["value"] == 8.4 + assert sent[0]["metadata"] == {"method": "keyword", "calibrated": False} + + +async def test_emit_is_a_noop_without_a_sink() -> None: + from everos.core.observability.tracing import scores as scores_mod + + previous, scores_mod._sink = scores_mod._sink, None + try: + emit_recall_scores( + trace_id="tid", + observation_id="oid", + top_score=1.0, + hit=True, + method="hybrid", + ) + finally: + scores_mod._sink = previous + + async def test_enqueue_never_blocks_or_raises_when_full() -> None: async def slow(_: dict) -> None: await asyncio.sleep(10) diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index f352575..183e296 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -1146,9 +1146,10 @@ async def test_search_top_score_zero_without_hit_for_keyword( async def test_search_enqueues_recall_scores( _search_spans: Any, monkeypatch: pytest.MonkeyPatch ) -> None: - """When tracing is active, search hands recall_top_score to the score - sink with the retriever span's trace_id (032x) + observation_id (016x). - KEYWORD is uncalibrated so hit is None (no recall_hit is pushed).""" + """When tracing is active, search hands the top score to the score sink + with the retriever span's trace_id (032x) + observation_id (016x). + KEYWORD is uncalibrated so hit is None, which is what makes the sink + report it as recall_top_score_raw and push no recall_hit.""" import everos.memory.search.manager as mgr_mod captured: dict[str, Any] = {}