fix(observability): separate uncalibrated recall scores by name (#368)

Langfuse aggregates scores by name, so one name may only carry values on
one scale. recall_top_score was emitted for every method, mixing HYBRID's
LR-sigmoid probability and AGENTIC's cross-encoder score (both comparable
in [0, 1]) with KEYWORD's unbounded BM25 and single-route VECTOR's cosine.
A chart on that name averaged the two scales, and in practice a keyword
score can read numerically higher than a calibrated one while meaning less.

Uncalibrated methods now report recall_top_score_raw, leaving
recall_top_score comparable across methods and over time. Every recall
score also carries metadata = {method, calibrated}: a structured field
Langfuse persists and can split on, which the free-text comment could not
serve. The comment stays for reading individual scores.

Breaking for anyone charting recall_top_score for keyword search; 1.2.0 is
four days old, so this is the cheapest moment to correct the naming.


Claude-Session: https://claude.ai/code/session_01UyKinsWs1MgoARPoB9R4NW

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- **Uncalibrated recall scores moved to their own name** — `KEYWORD` and
+  single-route `VECTOR` searches now report their top score as
+  `recall_top_score_raw`; `recall_top_score` is reserved for the calibrated
+  methods (`HYBRID` LR sigmoid, `AGENTIC` cross-encoder), whose values share a
+  comparable `[0, 1]` scale. Langfuse aggregates scores by name, so the previous
+  single name meant a chart could average an unbounded BM25 score together with
+  a probability. Every recall score also carries
+  `metadata = {"method": ..., "calibrated": ...}` now, a structured field that
+  can be split on, alongside the existing human-readable comment. Dashboards
+  built on `recall_top_score` for keyword search need to switch to the new name.
+
 ## [1.2.0] - 2026-07-24
 
 ### Added

examples/langfuse/README.md

@@ -50,10 +50,13 @@ zero tracing overhead.
 | query / recall embedding | embedding `everos.embedding` |
 | OME reflection strategies | agent `everos.ome.<strategy>` (linked to the triggering request's trace) |
 
-`langfuse.session.id` / `langfuse.user.id` group the traces; recall quality is
-pushed as Langfuse scores (`recall_top_score` always, and `recall_hit` for
-calibrated methods — HYBRID / AGENTIC). Query and memory text are captured only
-when `capture_content = true`.
+`langfuse.session.id` / `langfuse.user.id` group the traces. Recall quality is
+pushed as Langfuse scores, split by whether the method's score is calibrated:
+`recall_top_score` plus `recall_hit` for HYBRID / AGENTIC (comparable `[0, 1]`),
+and `recall_top_score_raw` for KEYWORD / single-route VECTOR, whose raw BM25 or
+cosine values are on a different scale and must not be averaged in with the
+calibrated ones. Query and memory text are captured only when
+`capture_content = true`.
 
 ## Try it
 

src/everos/config/default.toml

@@ -161,5 +161,8 @@ capture_content = false
 # Recall-quality scores pushed to Langfuse (Langfuse-specific REST, off the
 # OTLP stream). Only fires when langfuse_public_key/secret_key/host are set
 # (via everos.toml or EVEROS_OBSERVABILITY__LANGFUSE_* — secrets, not shipped here).
+# Score names: "recall_top_score" for the calibrated methods (HYBRID / AGENTIC,
+# [0, 1] and comparable), "recall_top_score_raw" for the uncalibrated ones
+# (KEYWORD BM25 / single-route VECTOR), plus "recall_hit" for calibrated only.
 emit_recall_scores = true
 recall_hit_threshold = 0.6       # only meaningful for calibrated methods

src/everos/core/observability/tracing/scores.py

@@ -14,6 +14,9 @@ queue + a single background worker:
 Attaches to the originating span via ``traceId`` (OTel trace_id, 032x hex) +
 ``observationId`` (OTel span_id, 016x hex) — exactly the mapping Langfuse's
 OTLP ingestion uses.
+
+Calibrated and uncalibrated top scores go out under *different names* — see
+:data:`SCORE_TOP_CALIBRATED` / :data:`SCORE_TOP_RAW`.
 """
 
 from __future__ import annotations
@@ -34,6 +37,16 @@ logger = get_logger(__name__)
 
 Sender = Callable[[dict], Awaitable[None]]
 
+# A Langfuse chart aggregates scores by *name*, so a single name may only ever
+# carry values on one scale. HYBRID / AGENTIC top scores are calibrated to a
+# comparable [0, 1]; KEYWORD (unbounded BM25) and single-route VECTOR are not,
+# and averaging the two together would be meaningless. Hence two names: a
+# dashboard built on ``recall_top_score`` stays comparable across methods and
+# over time, and the raw scores remain available under their own name.
+SCORE_TOP_CALIBRATED = "recall_top_score"
+SCORE_TOP_RAW = "recall_top_score_raw"
+SCORE_HIT = "recall_hit"
+
 
 @dataclass(frozen=True)
 class ScoreRecord:
@@ -44,6 +57,7 @@ class ScoreRecord:
     name: str
     value: float
     comment: str | None
+    metadata: dict[str, object] | None = None
 
 
 def _to_payload(record: ScoreRecord) -> dict:
@@ -57,6 +71,8 @@ def _to_payload(record: ScoreRecord) -> dict:
         payload["observationId"] = record.observation_id
     if record.comment:
         payload["comment"] = record.comment
+    if record.metadata:
+        payload["metadata"] = record.metadata
     return payload
 
 
@@ -165,24 +181,41 @@ def emit_recall_scores(
     hit: bool | None,
     method: str,
 ) -> None:
-    """Enqueue recall_top_score (always) + recall_hit (only when ``hit`` is
-    a verdict); no-op when the sink is off.
+    """Enqueue a top score (always) + recall_hit (only when ``hit`` is a
+    verdict); no-op when the sink is off.
 
-    ``hit=None`` means the method's score is uncalibrated (KEYWORD/VECTOR),
-    so no hit verdict is pushed — only the raw top score.
+    ``hit=None`` means the method's score is uncalibrated (KEYWORD /
+    single-route VECTOR): no hit verdict is pushed, and the top score is
+    reported as :data:`SCORE_TOP_RAW` instead of :data:`SCORE_TOP_CALIBRATED`
+    so the two scales never land under one score name.
+
+    ``method`` rides along as score metadata (a structured field Langfuse
+    persists) as well as in the human-readable comment.
     """
     if _sink is None:
         return
+    calibrated = hit is not None
     comment = f"method={method}"
+    metadata: dict[str, object] = {"method": method, "calibrated": calibrated}
     _sink.enqueue(
         ScoreRecord(
-            trace_id, observation_id, "recall_top_score", float(top_score), comment
+            trace_id,
+            observation_id,
+            SCORE_TOP_CALIBRATED if calibrated else SCORE_TOP_RAW,
+            float(top_score),
+            comment,
+            metadata,
         )
     )
     if hit is not None:
         _sink.enqueue(
             ScoreRecord(
-                trace_id, observation_id, "recall_hit", 1.0 if hit else 0.0, comment
+                trace_id,
+                observation_id,
+                SCORE_HIT,
+                1.0 if hit else 0.0,
+                comment,
+                metadata,
             )
         )
 

src/everos/memory/search/manager.py

@@ -144,7 +144,8 @@ def _top_score(data: SearchData) -> float:
 # (HYBRID → LR sigmoid, AGENTIC → cross-encoder), so the recall_hit
 # threshold is meaningful. KEYWORD (unbounded BM25) and single-route
 # VECTOR are excluded — a fixed threshold there yields a misleading
-# near-constant "hit" that inflates cross-method dashboards.
+# near-constant "hit" that inflates cross-method dashboards. Their top
+# score is also reported under a separate score name (see ``scores``).
 _CALIBRATED_METHODS = frozenset({SearchMethod.HYBRID, SearchMethod.AGENTIC})
 
 

tests/unit/test_core/test_observability/test_scores.py

@@ -14,8 +14,12 @@ from pydantic import SecretStr
 
 from everos.config.settings import ObservabilitySettings
 from everos.core.observability.tracing.scores import (
+    SCORE_HIT,
+    SCORE_TOP_CALIBRATED,
+    SCORE_TOP_RAW,
     RecallScoreSink,
     ScoreRecord,
+    emit_recall_scores,
     init_score_sink,
     shutdown_score_sink,
 )
@@ -79,6 +83,93 @@ async def test_worker_sends_payload_in_langfuse_shape() -> None:
     }
 
 
+async def test_worker_forwards_score_metadata() -> None:
+    """``metadata`` is a structured field Langfuse persists, so a dashboard can
+    split by method without parsing the free-text comment."""
+    sent: list[dict] = []
+    done = asyncio.Event()
+
+    async def sender(payload: dict) -> None:
+        sent.append(payload)
+        done.set()
+
+    sink = RecallScoreSink(sender=sender, max_queue=10)
+    sink.start()
+    sink.enqueue(
+        ScoreRecord("tid", "oid", "n", 0.5, "method=keyword", {"method": "keyword"})
+    )
+    await asyncio.wait_for(done.wait(), timeout=1.0)
+    await sink.stop()
+
+    assert sent[0]["metadata"] == {"method": "keyword"}
+
+
+async def _emitted(**kwargs: object) -> list[dict]:
+    """Install a capturing sink, run ``emit_recall_scores``, return the payloads."""
+    from everos.core.observability.tracing import scores as scores_mod
+
+    sent: list[dict] = []
+
+    async def sender(payload: dict) -> None:
+        sent.append(payload)
+
+    sink = RecallScoreSink(sender=sender, max_queue=10)
+    sink.start()
+    previous, scores_mod._sink = scores_mod._sink, sink
+    try:
+        emit_recall_scores(**kwargs)  # type: ignore[arg-type]
+        await sink.stop()  # drains before returning
+    finally:
+        scores_mod._sink = previous
+    return sent
+
+
+async def test_calibrated_method_reports_top_score_and_hit() -> None:
+    sent = await _emitted(
+        trace_id="tid",
+        observation_id="oid",
+        top_score=0.72,
+        hit=True,
+        method="hybrid",
+    )
+
+    assert [s["name"] for s in sent] == [SCORE_TOP_CALIBRATED, SCORE_HIT]
+    assert [s["value"] for s in sent] == [0.72, 1.0]
+    assert all(s["metadata"] == {"method": "hybrid", "calibrated": True} for s in sent)
+
+
+async def test_uncalibrated_method_reports_raw_name_and_no_hit() -> None:
+    """An unbounded BM25 top score must not land under the same score name as a
+    calibrated probability — a chart on that name would average both scales."""
+    sent = await _emitted(
+        trace_id="tid",
+        observation_id="oid",
+        top_score=8.4,
+        hit=None,
+        method="keyword",
+    )
+
+    assert [s["name"] for s in sent] == [SCORE_TOP_RAW]
+    assert sent[0]["value"] == 8.4
+    assert sent[0]["metadata"] == {"method": "keyword", "calibrated": False}
+
+
+async def test_emit_is_a_noop_without_a_sink() -> None:
+    from everos.core.observability.tracing import scores as scores_mod
+
+    previous, scores_mod._sink = scores_mod._sink, None
+    try:
+        emit_recall_scores(
+            trace_id="tid",
+            observation_id="oid",
+            top_score=1.0,
+            hit=True,
+            method="hybrid",
+        )
+    finally:
+        scores_mod._sink = previous
+
+
 async def test_enqueue_never_blocks_or_raises_when_full() -> None:
     async def slow(_: dict) -> None:
         await asyncio.sleep(10)

tests/unit/test_memory/test_search/test_manager.py

@@ -1146,9 +1146,10 @@ async def test_search_top_score_zero_without_hit_for_keyword(
 async def test_search_enqueues_recall_scores(
     _search_spans: Any, monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    """When tracing is active, search hands recall_top_score to the score
-    sink with the retriever span's trace_id (032x) + observation_id (016x).
-    KEYWORD is uncalibrated so hit is None (no recall_hit is pushed)."""
+    """When tracing is active, search hands the top score to the score sink
+    with the retriever span's trace_id (032x) + observation_id (016x).
+    KEYWORD is uncalibrated so hit is None, which is what makes the sink
+    report it as recall_top_score_raw and push no recall_hit."""
     import everos.memory.search.manager as mgr_mod
 
     captured: dict[str, Any] = {}