xiaoxue_admin
/
EverOS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

docs(examples): make Langfuse wrapper degrade cleanly on a real server (#342) 

* docs(examples): make Langfuse wrapper degrade cleanly on a real server

The wrapper synthesized child spans (extraction, embedding, hybrid
recall, rerank, index sync, consolidation) from a mock-only `_detail`
field. Against a real EverOS server that field is absent, so those spans
rendered with placeholder data — hardcoded model names, token=0, fixed
sleep durations — and recall scores fell to 0.

Now the per-stage child spans are emitted only when `_detail` is present
(the mock, or future native in-core instrumentation). Against a live
server only the top-level span per operation is emitted, with real
latency and output — no fabricated data. Recall quality
(recall_top_score / recall_hit) is derived from the real search
response, which already carries a per-hit score, so it works against a
live server today, not just the mock.

Verified: mock path unchanged (full trace tree, real scores); real-ish
path (no `_detail`) emits only top-level spans plus a real recall score.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* docs(examples): count empty recalls as a miss in Langfuse hit-rate

When a search returns nothing scored, record recall_hit=0 (span attribute +
Langfuse score) instead of omitting it, so genuine empty recalls still show
up in recall hit-rate. No top_score is emitted (there is no hit to score).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dani 2026-07-14 19:02:16 -04:00 committed by GitHub
parent a1e21ca676
commit d3a9f9e394
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 133 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

220
examples/langfuse/everos_langfuse.py
View File

@ -205,6 +205,20 @@ def _j(obj: Any) -> str:
return s if len(s) <= _TRUNC else s[:_TRUNC] + "…" return s if len(s) <= _TRUNC else s[:_TRUNC] + "…"
def _top_score_from_data(data: dict) -> float | None:
"""Best hit score across all scored result arrays in a real search
response. Each array is already sorted desc by the server, so the top
hit is derivable from the public API output alone — no server-internal
detail needed. Returns None when nothing scored came back (a miss)."""
scores = [
float(item["score"])
for key in ("episodes", "profiles", "agent_cases", "agent_skills")
for item in (data.get(key) or [])
if item.get("score") is not None
]
return max(scores) if scores else None
class InstrumentedEverOS: class InstrumentedEverOS:
"""Wraps an EverOS transport (real HTTP server or mock) and emits the """Wraps an EverOS transport (real HTTP server or mock) and emits the
spans that the proposed server-side instrumentation would emit. spans that the proposed server-side instrumentation would emit.
@ -266,28 +280,35 @@ class InstrumentedEverOS:
}) })
detail = resp.get("_detail", {}) detail = resp.get("_detail", {})
# 1. LLM extraction, a *generation*: model + token usage. # Extraction (generation w/ model+tokens), markdown persist, and the
# EverOS does not compute cost; Langfuse derives it from # async index trace all describe server-internal facts the HTTP API
# model + usage in its model-usage views. # doesn't expose yet. Emit them with the mock / once native
with self._tracer.start_as_current_span("everos.extract") as g: # instrumentation ships; skip on a real server rather than fabricate.
self._common(g, session_id=session_id, user_id=user_id, # The top-level everos.memory.flush span (real latency + output) is
app_id=app_id, project_id=project_id, # always emitted.
obs_type="generation", op="extract") if detail:
g.set_attribute("gen_ai.request.model", detail.get("model", "gpt-4.1-mini")) # 1. LLM extraction, a *generation*: model + token usage.
g.set_attribute("langfuse.observation.input", _j(detail.get("buffered_messages", []))) # EverOS does not compute cost; Langfuse derives it from
g.set_attribute("langfuse.observation.output", _j(detail.get("memory_cell", {}))) # model + usage in its model-usage views.
usage = detail.get("usage", {}) with self._tracer.start_as_current_span("everos.extract") as g:
g.set_attribute("gen_ai.usage.input_tokens", usage.get("input", 0)) self._common(g, session_id=session_id, user_id=user_id,
g.set_attribute("gen_ai.usage.output_tokens", usage.get("output", 0)) app_id=app_id, project_id=project_id,
time.sleep(detail.get("extract_s", 0.05)) obs_type="generation", op="extract")
g.set_attribute("gen_ai.request.model", detail.get("model", "gpt-4.1-mini"))
g.set_attribute("langfuse.observation.input", _j(detail.get("buffered_messages", [])))
g.set_attribute("langfuse.observation.output", _j(detail.get("memory_cell", {})))
usage = detail.get("usage", {})
g.set_attribute("gen_ai.usage.input_tokens", usage.get("input", 0))
g.set_attribute("gen_ai.usage.output_tokens", usage.get("output", 0))
time.sleep(detail.get("extract_s", 0.05))
# 2. Markdown persistence (atomic tmp+fsync+rename), strong consistency # 2. Markdown persistence (atomic tmp+fsync+rename), strong consistency
with self._tracer.start_as_current_span("everos.persist.markdown") as p: with self._tracer.start_as_current_span("everos.persist.markdown") as p:
self._common(p, session_id=session_id, user_id=user_id, self._common(p, session_id=session_id, user_id=user_id,
app_id=app_id, project_id=project_id, op="persist") app_id=app_id, project_id=project_id, op="persist")
p.set_attribute("langfuse.observation.output", p.set_attribute("langfuse.observation.output",
_j({"md_files": detail.get("md_files", [])})) _j({"md_files": detail.get("md_files", [])}))
time.sleep(0.008) time.sleep(0.008)
span.set_attribute("langfuse.observation.output", _j(resp["data"])) span.set_attribute("langfuse.observation.output", _j(resp["data"]))
@ -295,16 +316,18 @@ class InstrumentedEverOS:
# "cascade" daemon (file watcher + debounce + entry diff -> LanceDB). # "cascade" daemon (file watcher + debounce + entry diff -> LanceDB).
# It is therefore emitted as its OWN short-lived trace, correlated # It is therefore emitted as its OWN short-lived trace, correlated
# to the originating write by session_id, not as a child span. # to the originating write by session_id, not as a child span.
with self._tracer.start_as_current_span("everos.cascade.index") as ix: # Server-internal, so mock / native only.
self._common(ix, session_id=session_id, user_id=user_id, if detail:
app_id=app_id, project_id=project_id, op="index") with self._tracer.start_as_current_span("everos.cascade.index") as ix:
ix.set_attribute("langfuse.observation.input", self._common(ix, session_id=session_id, user_id=user_id,
_j({"triggered_by": "markdown change", app_id=app_id, project_id=project_id, op="index")
"correlates_to_session": session_id})) ix.set_attribute("langfuse.observation.input",
ix.set_attribute("langfuse.observation.output", _j({"triggered_by": "markdown change",
_j({"rows_indexed": detail.get("rows_indexed", 0), "correlates_to_session": session_id}))
"index_lag_ms": detail.get("index_lag_ms", 500)})) ix.set_attribute("langfuse.observation.output",
time.sleep(0.02) _j({"rows_indexed": detail.get("rows_indexed", 0),
"index_lag_ms": detail.get("index_lag_ms", 500)}))
time.sleep(0.02)
return resp return resp
@ -332,58 +355,76 @@ class InstrumentedEverOS:
resp = self._t("/api/v1/memory/search", payload) resp = self._t("/api/v1/memory/search", payload)
detail = resp.get("_detail", {}) detail = resp.get("_detail", {})
# 1. Query embedding # embed / hybrid_recall / rerank describe INTERNAL pipeline stages the
with self._tracer.start_as_current_span("everos.search.embed_query") as e: # HTTP API doesn't expose yet. Emit them with the mock / once native
self._common(e, session_id=session_id, user_id=user_id, agent_id=agent_id, # instrumentation ships; skip on a real server rather than fabricate.
app_id=app_id, project_id=project_id, if detail:
obs_type="embedding", op="embed") # 1. Query embedding
e.set_attribute("gen_ai.request.model", with self._tracer.start_as_current_span("everos.search.embed_query") as e:
detail.get("embed_model", "Qwen/Qwen3-Embedding-4B")) self._common(e, session_id=session_id, user_id=user_id, agent_id=agent_id,
e.set_attribute("langfuse.observation.input", _j(query)) app_id=app_id, project_id=project_id,
# compact output — never dump the raw vector into telemetry obs_type="embedding", op="embed")
e.set_attribute("langfuse.observation.output", e.set_attribute("gen_ai.request.model",
_j({"embedding_dims": detail.get("embed_dims", 2560)})) detail.get("embed_model", "Qwen/Qwen3-Embedding-4B"))
e.set_attribute("gen_ai.usage.input_tokens", detail.get("embed_tokens", 0)) e.set_attribute("langfuse.observation.input", _j(query))
time.sleep(detail.get("embed_s", 0.03)) # compact output — never dump the raw vector into telemetry
e.set_attribute("langfuse.observation.output",
_j({"embedding_dims": detail.get("embed_dims", 2560)}))
e.set_attribute("gen_ai.usage.input_tokens", detail.get("embed_tokens", 0))
time.sleep(detail.get("embed_s", 0.03))
# 2. Hybrid recall: single LanceDB query = BM25 + vector ANN + filter # 2. Hybrid recall: single LanceDB query = BM25 + vector ANN + filter
with self._tracer.start_as_current_span("everos.search.hybrid_recall") as h: with self._tracer.start_as_current_span("everos.search.hybrid_recall") as h:
self._common(h, session_id=session_id, user_id=user_id, agent_id=agent_id, self._common(h, session_id=session_id, user_id=user_id, agent_id=agent_id,
app_id=app_id, project_id=project_id, app_id=app_id, project_id=project_id,
obs_type="retriever", op="recall") obs_type="retriever", op="recall")
h.set_attribute("langfuse.observation.input", h.set_attribute("langfuse.observation.input",
_j({"bm25": True, "vector_ann": True, "filters": None})) _j({"bm25": True, "vector_ann": True, "filters": None}))
h.set_attribute("langfuse.observation.output", h.set_attribute("langfuse.observation.output",
_j({"candidates": detail.get("candidates", 0)})) _j({"candidates": detail.get("candidates", 0)}))
time.sleep(detail.get("recall_s", 0.03)) time.sleep(detail.get("recall_s", 0.03))
# 3. Rerank (cross-encoder) — scores become Langfuse scores # 3. Rerank (cross-encoder)
with self._tracer.start_as_current_span("everos.search.rerank") as r: with self._tracer.start_as_current_span("everos.search.rerank") as r:
self._common(r, session_id=session_id, user_id=user_id, agent_id=agent_id, self._common(r, session_id=session_id, user_id=user_id, agent_id=agent_id,
app_id=app_id, project_id=project_id, op="rerank") app_id=app_id, project_id=project_id, op="rerank")
r.set_attribute("langfuse.observation.metadata.rerank_model", r.set_attribute("langfuse.observation.metadata.rerank_model",
detail.get("rerank_model", "Qwen/Qwen3-Reranker-4B")) detail.get("rerank_model", "Qwen/Qwen3-Reranker-4B"))
r.set_attribute("langfuse.observation.output", _j(detail.get("ranked", []))) r.set_attribute("langfuse.observation.output", _j(detail.get("ranked", [])))
time.sleep(detail.get("rerank_s", 0.05)) time.sleep(detail.get("rerank_s", 0.05))
# Compact result summary on the retriever span # Recall quality is derivable from the REAL response — every hit
hits = detail.get("ranked", []) # carries a fused/reranked score — so it works against a live server
top_score = float(hits[0]["score"]) if hits else 0.0 # today, not just the mock. None means a miss (nothing scored).
top_score = _top_score_from_data(resp["data"])
span.set_attribute("langfuse.observation.output", _j(resp["data"])) span.set_attribute("langfuse.observation.output", _j(resp["data"]))
span.set_attribute("everos.search.top_score", top_score) if top_score is not None:
span.set_attribute("everos.search.hit", top_score >= hit_threshold) span.set_attribute("everos.search.top_score", top_score)
span.set_attribute("everos.search.hit", top_score >= hit_threshold)
else:
# Nothing scored came back: a genuine miss. Record hit so it
# still counts in recall hit-rate; no top_score (no hit to score).
span.set_attribute("everos.search.hit", False)
# Recall-quality -> Langfuse scores (visible in evals/dashboards). # Recall-quality -> Langfuse scores (visible in evals/dashboards).
# Pushed AFTER the span closes so exporter/network time never # Pushed AFTER the span closes so exporter/network time never
# inflates the measured search latency. # inflates the measured search latency.
pushed = push_score(trace_id_hex, "recall_top_score", top_score, if top_score is not None:
observation_id=retriever_obs_id, pushed = push_score(trace_id_hex, "recall_top_score", top_score,
comment="fused+reranked score of top memory hit") observation_id=retriever_obs_id,
push_score(trace_id_hex, "recall_hit", comment="fused+reranked score of top memory hit")
1.0 if top_score >= hit_threshold else 0.0, push_score(trace_id_hex, "recall_hit",
observation_id=retriever_obs_id, 1.0 if top_score >= hit_threshold else 0.0,
comment=f"top_score >= {hit_threshold}") observation_id=retriever_obs_id,
resp["_scores_pushed"] = pushed comment=f"top_score >= {hit_threshold}")
resp["_scores_pushed"] = pushed
else:
# Miss: record hit=0 so empty recalls still count in hit-rate;
# no top_score is pushed (there is no hit to score).
resp["_scores_pushed"] = push_score(
trace_id_hex, "recall_hit", 0.0,
observation_id=retriever_obs_id,
comment=f"no hit >= {hit_threshold} (empty recall)")
resp["_trace_id"] = trace_id_hex resp["_trace_id"] = trace_id_hex
return resp return resp
@ -398,18 +439,23 @@ class InstrumentedEverOS:
resp = self._t("/api/v1/ome/trigger", {"name": strategy, "force": True}) resp = self._t("/api/v1/ome/trigger", {"name": strategy, "force": True})
detail = resp.get("_detail", {}) detail = resp.get("_detail", {})
with self._tracer.start_as_current_span("everos.reflect.consolidate") as g: # The consolidation generation (model + tokens) is server-internal;
self._common(g, session_id=session_id, user_id=user_id, # emit it with the mock / once native instrumentation ships, skip on
obs_type="generation", op="consolidate") # a real server. The top-level everos.ome.<strategy> agent span (real
g.set_attribute("gen_ai.request.model", detail.get("model", "gpt-4.1-mini")) # latency + output) is always emitted.
g.set_attribute("langfuse.observation.input", if detail:
_j(detail.get("episodes_in", []))) with self._tracer.start_as_current_span("everos.reflect.consolidate") as g:
g.set_attribute("langfuse.observation.output", self._common(g, session_id=session_id, user_id=user_id,
_j(detail.get("consolidated", {}))) obs_type="generation", op="consolidate")
usage = detail.get("usage", {}) g.set_attribute("gen_ai.request.model", detail.get("model", "gpt-4.1-mini"))
g.set_attribute("gen_ai.usage.input_tokens", usage.get("input", 0)) g.set_attribute("langfuse.observation.input",
g.set_attribute("gen_ai.usage.output_tokens", usage.get("output", 0)) _j(detail.get("episodes_in", [])))
time.sleep(detail.get("reflect_s", 0.08)) g.set_attribute("langfuse.observation.output",
_j(detail.get("consolidated", {})))
usage = detail.get("usage", {})
g.set_attribute("gen_ai.usage.input_tokens", usage.get("input", 0))
g.set_attribute("gen_ai.usage.output_tokens", usage.get("output", 0))
time.sleep(detail.get("reflect_s", 0.08))
span.set_attribute("langfuse.observation.output", _j(resp["data"])) span.set_attribute("langfuse.observation.output", _j(resp["data"]))
return resp return resp