feat(observability): OTel tracing chassis + request-id propagation

Optional, off-by-default native OpenTelemetry export (the [otel] extra),
wired as chassis so call sites never branch on config:

- ObservabilitySettings ([observability]) — enabled / endpoint / headers /
  sample_rate / capture_content / langfuse_* / recall_hit_threshold; langfuse
  creds derive the OTLP endpoint + Basic-auth header.
- core.observability.tracing: TracerProvider lifecycle (module-local, no-op
  when off/absent), memory_span helper stamping the langfuse.* contract,
  set_generation_usage, privacy-gated capture_input/output (redaction +
  truncation), non-blocking recall-score sink, W3C traceparent in/out helpers.
- TracingLifespanProvider builds provider + score sink at startup.
- RequestIdMiddleware: per-request id (state + contextvar + structlog +
  X-Request-Id) and continues an upstream traceparent when present; managers
  read the propagated id via resolve_request_id.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.claude/rules/logging-observability.md

@@ -21,3 +21,11 @@ paths:
 - **Metrics** go through `core.observability.metrics` (Prometheus); don't invent
   ad-hoc counters. Histograms/counters/gauges have registry helpers.
 - Don't log secrets, API keys, or full memory content at `info`/above.
+- **Tracing** (optional, `[otel]` extra, **off by default**): open spans with
+  `memory_span(...)` from `core.observability.tracing` — it stamps the Langfuse
+  `langfuse.*` attributes and is a no-op until `[observability] enabled`, so call
+  sites never branch on config. LLM / embedding token usage rides
+  `set_generation_usage` onto the active span (Langfuse computes cost).
+  Request/response content is emitted only when `capture_content` is on
+  (redaction hook + truncation). `request_id` is kept independent of the OTel
+  `trace_id`; an upstream `traceparent` header is continued when present.

pyproject.toml

@@ -70,6 +70,12 @@ dependencies = [
 
 [project.optional-dependencies]
 multimodal = ["everalgo-parser[svg]>=0.2.1"]  # [svg] bundles cairosvg → SVG works by default
+# Native OpenTelemetry tracing export. Optional — EverOS never imports these
+# unless [observability] is enabled. Install with: pip install everos[otel]
+otel = [
+    "opentelemetry-sdk>=1.27.0",
+    "opentelemetry-exporter-otlp-proto-http>=1.27.0",
+]
 
 [project.urls]
 Homepage = "https://evermind.ai"
@@ -255,4 +261,8 @@ dev = [
     "pre-commit>=4.0.0",
     "ipdb>=0.13.13",
     "pyinstrument>=5.0.0",
+    # Tracing tests must actually run (no skip-when-absent), so the optional
+    # [otel] stack is always present in the dev / CI environment.
+    "opentelemetry-sdk>=1.27.0",
+    "opentelemetry-exporter-otlp-proto-http>=1.27.0",
 ]

src/everos/config/default.toml

@@ -145,3 +145,21 @@ session_lock_timeout_seconds = 360.0
 threshold = 0.65
 time_window_days = 7.0
 
+
+[observability]
+# OpenTelemetry tracing export. Off by default; pure OTLP/HTTP, vendor-neutral
+# (Langfuse, an OTel Collector, or any OTLP backend). EverOS ships no vendor SDK.
+# Override via EVEROS_OBSERVABILITY__ENABLED, EVEROS_OBSERVABILITY__ENDPOINT, etc.
+enabled = false
+exporter = "otlp_http"          # "otlp_http" | "none"
+endpoint = ""                    # e.g. https://us.cloud.langfuse.com/api/public/otel/v1/traces
+service_name = "everos"
+sample_rate = 1.0                # 0.0 to 1.0
+# Privacy: false (default) = metadata only; true also emits query / extracted
+# memory / .md paths as span input/output (redacted + truncated).
+capture_content = false
+# Recall-quality scores pushed to Langfuse (Langfuse-specific REST, off the
+# OTLP stream). Only fires when langfuse_public_key/secret_key/host are set
+# (via everos.toml or EVEROS_OBSERVABILITY__LANGFUSE_* — secrets, not shipped here).
+emit_recall_scores = true
+recall_hit_threshold = 0.6       # only meaningful for calibrated methods

src/everos/config/settings.py

@@ -350,6 +350,53 @@ class KnowledgeSettings(BaseModel):
     search: KnowledgeSearchSettings = KnowledgeSearchSettings()
 
 
+class ObservabilitySettings(BaseModel):
+    """``[observability]`` — OpenTelemetry tracing export.
+
+    Off by default. When ``enabled`` is true a ``TracerProvider`` is built
+    once at startup and standard OTLP/HTTP spans are exported to
+    ``endpoint``. The signal is pure OpenTelemetry — vendor-neutral — so it
+    works with any OTLP backend (Langfuse, an OTel Collector, ...); EverOS
+    does not depend on any vendor SDK.
+
+    ``langfuse_*`` are convenience credentials for pushing recall-quality
+    *scores* to Langfuse (a Langfuse-specific REST call, independent of the
+    OTLP span stream). Leave unset for a pure vendor-neutral OTLP export.
+
+    Env binding:
+        EVEROS_OBSERVABILITY__ENABLED
+        EVEROS_OBSERVABILITY__EXPORTER
+        EVEROS_OBSERVABILITY__ENDPOINT
+        EVEROS_OBSERVABILITY__SERVICE_NAME
+        EVEROS_OBSERVABILITY__SAMPLE_RATE
+        EVEROS_OBSERVABILITY__LANGFUSE_PUBLIC_KEY / __LANGFUSE_SECRET_KEY
+        EVEROS_OBSERVABILITY__LANGFUSE_HOST
+        EVEROS_OBSERVABILITY__EMIT_RECALL_SCORES
+        EVEROS_OBSERVABILITY__RECALL_HIT_THRESHOLD
+    """
+
+    enabled: bool = False
+    exporter: Literal["otlp_http", "none"] = "otlp_http"
+    endpoint: str = ""
+    headers: dict[str, str] = Field(default_factory=dict)
+    service_name: str = "everos"
+    sample_rate: float = Field(default=1.0, ge=0.0, le=1.0)
+    # Privacy: when False (default) spans carry metadata only — no query text,
+    # extracted memory, or .md paths. Set True to also emit request/response
+    # content as span input/output (redacted + truncated).
+    capture_content: bool = False
+
+    # Langfuse scores (recall-quality feedback) — optional, Langfuse-specific.
+    langfuse_public_key: str | None = None
+    langfuse_secret_key: SecretStr | None = None
+    langfuse_host: str | None = None
+    emit_recall_scores: bool = True
+    # ``hit`` threshold: only meaningful for calibrated-score methods
+    # (HYBRID LR / rerank / agentic). Not bounded to [0, 1] because raw
+    # BM25 scores are unbounded; tune per method on the eval side.
+    recall_hit_threshold: float = 0.6
+
+
 class Settings(BaseSettings):
     """Top-level application settings."""
 
@@ -365,6 +412,7 @@ class Settings(BaseSettings):
     clustering: ClusteringSettings = ClusteringSettings()
     multimodal: MultimodalSettings = MultimodalSettings()
     knowledge: KnowledgeSettings = KnowledgeSettings()
+    observability: ObservabilitySettings = ObservabilitySettings()
 
     model_config = SettingsConfigDict(
         env_prefix="EVEROS_",

src/everos/core/context/__init__.py

@@ -0,0 +1,22 @@
+"""core.context — request-scoped context propagation (contextvars).
+
+External usage::
+
+    from everos.core.context import (
+        get_request_id,
+        set_request_id,
+        reset_request_id,
+    )
+"""
+
+from .request import get_request_id as get_request_id
+from .request import reset_request_id as reset_request_id
+from .request import resolve_request_id as resolve_request_id
+from .request import set_request_id as set_request_id
+
+__all__ = [
+    "get_request_id",
+    "reset_request_id",
+    "resolve_request_id",
+    "set_request_id",
+]

src/everos/core/context/request.py

@@ -0,0 +1,39 @@
+"""Request-scoped context propagation via ``contextvars``.
+
+The request id is stored in a module-level ``ContextVar`` so it survives
+``await`` boundaries and is readable anywhere in the call chain (service,
+infra, log processors) without being threaded through call signatures.
+"""
+
+from __future__ import annotations
+
+from contextvars import ContextVar, Token
+
+from everos.core.observability.tracing import gen_request_id
+
+_request_id: ContextVar[str | None] = ContextVar("everos_request_id", default=None)
+
+
+def get_request_id() -> str | None:
+    """Return the request id bound to the current context, or ``None``."""
+    return _request_id.get()
+
+
+def set_request_id(value: str | None) -> Token[str | None]:
+    """Bind ``value`` as the current request id; return a reset token."""
+    return _request_id.set(value)
+
+
+def reset_request_id(token: Token[str | None]) -> None:
+    """Restore the request id to what it was before the matching ``set``."""
+    _request_id.reset(token)
+
+
+def resolve_request_id() -> str:
+    """Return the propagated request id, or mint a fresh W3C-compatible one.
+
+    Call sites that need an id (search / get managers) use this so an id
+    injected upstream by ``RequestIdMiddleware`` flows through to the
+    response, while direct / CLI callers still get a freshly minted id.
+    """
+    return get_request_id() or gen_request_id()

src/everos/core/lifespan/__init__.py

@@ -19,9 +19,11 @@ External usage:
 from .base import LifespanProvider as LifespanProvider
 from .factory import build_lifespan as build_lifespan
 from .metrics_lifespan import MetricsLifespanProvider as MetricsLifespanProvider
+from .tracing_lifespan import TracingLifespanProvider as TracingLifespanProvider
 
 __all__ = [
     "LifespanProvider",
     "MetricsLifespanProvider",
+    "TracingLifespanProvider",
     "build_lifespan",
 ]

src/everos/core/lifespan/tracing_lifespan.py

@@ -0,0 +1,50 @@
+"""Tracing lifespan provider.
+
+Builds the OpenTelemetry ``TracerProvider`` at startup (from the
+``[observability]`` settings) and flushes + tears it down at shutdown.
+Chassis-level (backend-agnostic), so it lives here alongside
+``MetricsLifespanProvider`` rather than under the API entrypoint.
+
+Registered with a low ``order`` so the tracer is live before other
+providers start and can themselves be traced.
+"""
+
+from __future__ import annotations
+
+from fastapi import FastAPI
+
+from everos.config import load_settings
+from everos.core.observability.logging import get_logger
+from everos.core.observability.tracing import (
+    init_score_sink,
+    init_tracing,
+    shutdown_score_sink,
+    shutdown_tracing,
+)
+
+from .base import LifespanProvider
+
+logger = get_logger(__name__)
+
+
+class TracingLifespanProvider(LifespanProvider):
+    """Manages the OTel tracer provider + recall-score sink over the app life."""
+
+    def __init__(self, order: int = 1) -> None:
+        super().__init__(name="tracing", order=order)
+
+    async def startup(self, app: FastAPI) -> bool:
+        """Install the tracer provider + recall-score sink when configured.
+
+        Returns True if tracing was enabled and a provider installed.
+        """
+        settings = load_settings().observability
+        enabled = init_tracing(settings)
+        scores = init_score_sink(settings)
+        logger.info("tracing_lifespan_startup", enabled=enabled, scores=scores)
+        return enabled
+
+    async def shutdown(self, app: FastAPI) -> None:
+        await shutdown_score_sink()
+        shutdown_tracing()
+        logger.info("tracing_lifespan_shutdown")

src/everos/core/middleware/__init__.py

@@ -18,6 +18,7 @@ from .cors import DEFAULT_CORS_ALLOW_METHODS as DEFAULT_CORS_ALLOW_METHODS
 from .cors import DEFAULT_CORS_ORIGINS as DEFAULT_CORS_ORIGINS
 from .profile import ProfileMiddleware as ProfileMiddleware
 from .prometheus import PrometheusMiddleware as PrometheusMiddleware
+from .request_id import RequestIdMiddleware as RequestIdMiddleware
 
 __all__ = [
     "DEFAULT_CORS_ALLOW_CREDENTIALS",
@@ -26,4 +27,5 @@ __all__ = [
     "DEFAULT_CORS_ORIGINS",
     "ProfileMiddleware",
     "PrometheusMiddleware",
+    "RequestIdMiddleware",
 ]

src/everos/core/middleware/request_id.py

@@ -0,0 +1,56 @@
+"""Request-context middleware.
+
+Establishes per-request context at HTTP entry:
+
+* Mints a W3C-compatible ``request_id`` and binds it for the request's
+  lifetime — via ``request.state``, the ``core.context`` contextvar
+  (readable by service / infra), and structlog contextvars (so every log
+  line carries it). Echoed on the ``X-Request-Id`` response header.
+* Continues an upstream **distributed trace**: if the request carries a
+  W3C ``traceparent`` header, our first span nests under that trace instead
+  of rooting a new one. Absent → we root our own trace (the common case).
+
+This is the single place a request id enters the system; ``extract_request_id``
+(``entrypoints/api/utils.py``) reads what this middleware sets.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+import structlog
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+from everos.core.context import reset_request_id, set_request_id
+from everos.core.observability.tracing import gen_request_id, use_traceparent
+
+_HEADER = "X-Request-Id"
+
+
+class RequestIdMiddleware(BaseHTTPMiddleware):
+    """Assigns a request id + continues any upstream trace for the request."""
+
+    async def dispatch(
+        self,
+        request: Request,
+        call_next: Callable[[Request], Awaitable[Response]],
+    ) -> Response:
+        request_id = gen_request_id()
+        request.state.request_id = request_id
+        # Bind before ``call_next`` so the downstream task (endpoint) inherits
+        # the value; reset in ``finally`` so it never leaks to the next request.
+        token = set_request_id(request_id)
+        structlog.contextvars.bind_contextvars(request_id=request_id)
+        try:
+            # Continue the upstream trace when a traceparent header is present
+            # (no-op otherwise). Attached before call_next so downstream spans
+            # inherit it across the middleware task boundary.
+            with use_traceparent(request.headers.get("traceparent")):
+                response = await call_next(request)
+            response.headers[_HEADER] = request_id
+            return response
+        finally:
+            structlog.contextvars.unbind_contextvars("request_id")
+            reset_request_id(token)

src/everos/core/observability/tracing/__init__.py

@@ -1,32 +1,56 @@
-"""Tracing utilities — W3C-compatible request id generation.
+"""Tracing — W3C id generation + OpenTelemetry tracer lifecycle.
 
 External usage::
 
-    from everos.core.observability.tracing import gen_request_id
+    from everos.core.observability.tracing import (
+        gen_request_id,
+        get_tracer,
+        init_tracing,
+        shutdown_tracing,
+        force_flush,
+    )
+
+``get_tracer`` is safe to call unconditionally: it returns a no-op tracer
+until ``init_tracing`` installs a provider (and when the optional ``[otel]``
+extra is not installed), so call sites never need to branch on config.
 """
 
 from __future__ import annotations
 
-from uuid import uuid4
+from .attributes import capture_input as capture_input
+from .attributes import capture_output as capture_output
+from .attributes import current_trace_ids as current_trace_ids
+from .attributes import current_traceparent as current_traceparent
+from .attributes import memory_span as memory_span
+from .attributes import set_capture_content as set_capture_content
+from .attributes import set_generation_usage as set_generation_usage
+from .attributes import set_redactor as set_redactor
+from .attributes import use_traceparent as use_traceparent
+from .ids import gen_request_id as gen_request_id
+from .provider import force_flush as force_flush
+from .provider import get_tracer as get_tracer
+from .provider import init_tracing as init_tracing
+from .provider import shutdown_tracing as shutdown_tracing
+from .scores import emit_recall_scores as emit_recall_scores
+from .scores import init_score_sink as init_score_sink
+from .scores import shutdown_score_sink as shutdown_score_sink
 
-
-def gen_request_id() -> str:
-    """Generate a request id matching the W3C trace-context spec.
-
-    Returns 32 lowercase hex characters (128-bit, no prefix) — the same
-    format as a W3C ``trace_id`` / OpenTelemetry trace identifier. Routes
-    and services that mint a fresh request id (when one wasn't injected
-    by upstream middleware) should call this helper rather than rolling
-    their own uuid / prefix format, so the id layer stays compatible
-    with OpenTelemetry exporters and standard APM tooling.
-
-    Example::
-
-        >>> rid = gen_request_id()
-        >>> len(rid)
-        32
-    """
-    return uuid4().hex
-
-
-__all__ = ["gen_request_id"]
+__all__ = [
+    "capture_input",
+    "capture_output",
+    "current_trace_ids",
+    "current_traceparent",
+    "emit_recall_scores",
+    "force_flush",
+    "gen_request_id",
+    "get_tracer",
+    "init_score_sink",
+    "init_tracing",
+    "memory_span",
+    "set_capture_content",
+    "set_generation_usage",
+    "set_redactor",
+    "shutdown_score_sink",
+    "shutdown_tracing",
+    "use_traceparent",
+]

src/everos/core/observability/tracing/attributes.py

@@ -0,0 +1,195 @@
+"""Span helpers implementing the Langfuse / OpenTelemetry attribute contract.
+
+``memory_span`` opens a span under the shared ``everos`` tracer and stamps
+the ``langfuse.*`` trace/observation attributes (observation type, session /
+user ids, trace metadata, tags). ``set_generation_usage`` writes the
+``gen_ai.*`` model + token attributes onto the *current* span, so an LLM
+client wrapper can record usage without knowing which span is active.
+
+Request/response content (``langfuse.observation.input/output``) is
+privacy-gated: ``capture_input`` / ``capture_output`` only emit it when
+``capture_content`` is on, after a redaction hook + truncation. Off by
+default, so spans carry metadata only.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from contextlib import contextmanager
+from typing import Any
+
+from .provider import get_tracer
+
+# ── langfuse.* trace/observation attribute keys ──────────────────────────
+LF_OBSERVATION_TYPE = "langfuse.observation.type"
+LF_SESSION_ID = "langfuse.session.id"
+LF_USER_ID = "langfuse.user.id"
+LF_TAGS = "langfuse.trace.tags"
+LF_METADATA_PREFIX = "langfuse.trace.metadata."
+
+# ── gen_ai.* generation attribute keys (Langfuse computes cost from these) ─
+GEN_AI_REQUEST_MODEL = "gen_ai.request.model"
+GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"
+GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
+
+# ── content capture (privacy-gated) ──────────────────────────────────────
+LF_OBSERVATION_INPUT = "langfuse.observation.input"
+LF_OBSERVATION_OUTPUT = "langfuse.observation.output"
+_MAX_CONTENT_CHARS = 4096
+
+DEFAULT_TAGS: tuple[str, ...] = ("everos", "memory")
+
+# Off by default: no request/response content leaves the process unless
+# capture_content is turned on (set once at init_tracing from settings).
+_capture_content = False
+_redactor: Callable[[str], str] = lambda text: text  # noqa: E731 - overridable hook
+
+
+def set_capture_content(enabled: bool) -> None:
+    """Toggle content capture (called from init_tracing / shutdown)."""
+    global _capture_content
+    _capture_content = enabled
+
+
+def set_redactor(redactor: Callable[[str], str] | None) -> None:
+    """Install a redaction hook applied to captured content; None resets it."""
+    global _redactor
+    _redactor = redactor if redactor is not None else (lambda text: text)
+
+
+def _prepare_content(value: Any) -> str:
+    """Serialize, redact, then truncate content for a span attribute."""
+    text = value if isinstance(value, str) else json.dumps(value, default=str)
+    text = _redactor(text)
+    return text[:_MAX_CONTENT_CHARS]
+
+
+def capture_input(span: Any, value: Any) -> None:
+    """Set ``langfuse.observation.input`` — only when capture_content is on."""
+    if _capture_content and value is not None:
+        span.set_attribute(LF_OBSERVATION_INPUT, _prepare_content(value))
+
+
+def capture_output(span: Any, value: Any) -> None:
+    """Set ``langfuse.observation.output`` — only when capture_content is on."""
+    if _capture_content and value is not None:
+        span.set_attribute(LF_OBSERVATION_OUTPUT, _prepare_content(value))
+
+
+try:
+    from opentelemetry import trace as _otel_trace
+
+    _OTEL_AVAILABLE = True
+except ImportError:  # pragma: no cover - only without the [otel] extra
+    _OTEL_AVAILABLE = False
+
+
+@contextmanager
+def memory_span(
+    name: str,
+    *,
+    observation_type: str,
+    session_id: str | None = None,
+    user_id: str | None = None,
+    metadata: Mapping[str, Any] | None = None,
+    tags: Sequence[str] = DEFAULT_TAGS,
+) -> Iterator[Any]:
+    """Open a span named ``name`` and stamp the langfuse.* attributes.
+
+    Args:
+        name: Span name (e.g. ``everos.memory.search``).
+        observation_type: ``langfuse.observation.type`` — span / generation /
+            embedding / retriever / agent.
+        session_id / user_id: Grouping ids (dropped when None).
+        metadata: Flat mapping → ``langfuse.trace.metadata.<key>``; None
+            values are dropped rather than emitted as the string "None".
+        tags: ``langfuse.trace.tags`` list.
+    """
+    tracer = get_tracer("everos")
+    with tracer.start_as_current_span(name) as span:
+        span.set_attribute(LF_OBSERVATION_TYPE, observation_type)
+        if session_id:
+            span.set_attribute(LF_SESSION_ID, session_id)
+        if user_id:
+            span.set_attribute(LF_USER_ID, user_id)
+        if tags:
+            span.set_attribute(LF_TAGS, list(tags))
+        for key, value in (metadata or {}).items():
+            if value is not None:
+                span.set_attribute(f"{LF_METADATA_PREFIX}{key}", value)
+        yield span
+
+
+def current_traceparent() -> str | None:
+    """W3C ``traceparent`` for the current span, or None when there is none.
+
+    Captured where a request's span is active (e.g. OME enqueue) and carried
+    across the async / process boundary so a background strategy span can
+    re-attach to the originating trace.
+    """
+    if not _OTEL_AVAILABLE:
+        return None
+    from opentelemetry.propagate import inject
+
+    carrier: dict[str, str] = {}
+    inject(carrier)
+    return carrier.get("traceparent")
+
+
+@contextmanager
+def use_traceparent(traceparent: str | None) -> Iterator[None]:
+    """Attach ``traceparent`` as the current context for the block.
+
+    Spans opened inside become children of that (remote) trace. No-op when
+    the traceparent is absent or OTel is not installed — the span then roots
+    its own trace.
+    """
+    if not _OTEL_AVAILABLE or not traceparent:
+        yield
+        return
+    from opentelemetry import context as otel_context
+    from opentelemetry.propagate import extract
+
+    token = otel_context.attach(extract({"traceparent": traceparent}))
+    try:
+        yield
+    finally:
+        otel_context.detach(token)
+
+
+def current_trace_ids() -> tuple[str, str] | None:
+    """Return ``(trace_id_hex_032x, span_id_hex_016x)`` of the current span.
+
+    Returns None when OTel is absent or there is no valid recording span —
+    the exact hex mapping Langfuse's OTLP ingestion uses for traceId /
+    observationId, so recall scores attach to the right observation.
+    """
+    if not _OTEL_AVAILABLE:
+        return None
+    ctx = _otel_trace.get_current_span().get_span_context()
+    if not ctx.is_valid:
+        return None
+    return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x")
+
+
+def set_generation_usage(
+    *,
+    model: str | None = None,
+    input_tokens: int | None = None,
+    output_tokens: int | None = None,
+) -> None:
+    """Record ``gen_ai.*`` model + token attributes on the current span.
+
+    No-op when OTel is absent or there is no active recording span, so LLM
+    client wrappers can call it unconditionally.
+    """
+    if not _OTEL_AVAILABLE:
+        return
+    span = _otel_trace.get_current_span()
+    if model is not None:
+        span.set_attribute(GEN_AI_REQUEST_MODEL, model)
+    if input_tokens is not None:
+        span.set_attribute(GEN_AI_USAGE_INPUT_TOKENS, input_tokens)
+    if output_tokens is not None:
+        span.set_attribute(GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens)

src/everos/core/observability/tracing/ids.py

@@ -0,0 +1,24 @@
+"""W3C-compatible request/trace id generation."""
+
+from __future__ import annotations
+
+from uuid import uuid4
+
+
+def gen_request_id() -> str:
+    """Generate a request id matching the W3C trace-context spec.
+
+    Returns 32 lowercase hex characters (128-bit, no prefix) — the same
+    format as a W3C ``trace_id`` / OpenTelemetry trace identifier. Routes
+    and services that mint a fresh request id (when one wasn't injected
+    by upstream middleware) should call this helper rather than rolling
+    their own uuid / prefix format, so the id layer stays compatible
+    with OpenTelemetry exporters and standard APM tooling.
+
+    Example::
+
+        >>> rid = gen_request_id()
+        >>> len(rid)
+        32
+    """
+    return uuid4().hex

src/everos/core/observability/tracing/provider.py

@@ -0,0 +1,178 @@
+"""OpenTelemetry tracer provider lifecycle + a no-op-safe tracer facade.
+
+OpenTelemetry is an optional dependency (the ``[otel]`` extra). This module
+never fails to import when it is absent: the SDK imports are guarded, and
+``get_tracer`` returns a no-op tracer until ``init_tracing`` installs a real
+provider.
+
+The provider is held here (module-local) rather than on the OTel *global*
+so it can be built and torn down repeatedly — in tests and across restarts —
+without tripping OTel's "set global provider once" guard. Span context
+propagation (parent/child nesting) still works: that rides OTel's context
+vars, which are independent of which provider produced the tracer.
+"""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any
+
+from everos.core.observability.logging import get_logger
+
+logger = get_logger(__name__)
+
+try:
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter,
+    )
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
+
+    _OTEL_AVAILABLE = True
+except ImportError:  # pragma: no cover - only without the [otel] extra
+    _OTEL_AVAILABLE = False
+
+if TYPE_CHECKING:
+    from opentelemetry.sdk.trace.export import SpanProcessor
+
+    from everos.config.settings import ObservabilitySettings
+
+# Our TracerProvider, deliberately kept off the OTel global (see module docstring).
+_provider: Any = None
+
+
+class _NoopSpan:
+    """Span stand-in used when tracing is off or OTel is not installed."""
+
+    def set_attribute(self, key: str, value: object) -> None: ...
+
+    def set_attributes(self, attributes: dict[str, object]) -> None: ...
+
+    def record_exception(self, exception: BaseException) -> None: ...
+
+    def set_status(self, *args: object, **kwargs: object) -> None: ...
+
+    def end(self) -> None: ...
+
+
+class _NoopTracer:
+    """Tracer stand-in whose spans do nothing (zero-overhead when off)."""
+
+    @contextmanager
+    def start_as_current_span(self, name: str, **kwargs: object) -> Iterator[_NoopSpan]:
+        yield _NoopSpan()
+
+
+_NOOP_TRACER = _NoopTracer()
+
+
+def _resolve_otlp_target(
+    settings: ObservabilitySettings,
+) -> tuple[str, dict[str, str]]:
+    """Endpoint + headers for the OTLP exporter.
+
+    Convenience: when ``langfuse_*`` creds are set, derive the OTLP traces
+    endpoint and a Basic-auth header from them — but explicit ``endpoint`` /
+    ``headers`` always win, so a plain vendor-neutral export is unaffected.
+    """
+    endpoint = settings.endpoint
+    headers = dict(settings.headers)
+    pk = settings.langfuse_public_key
+    sk = settings.langfuse_secret_key
+    host = settings.langfuse_host
+    if host and pk and sk:
+        if not endpoint:
+            endpoint = host.rstrip("/") + "/api/public/otel/v1/traces"
+        if "Authorization" not in headers:
+            token = base64.b64encode(f"{pk}:{sk.get_secret_value()}".encode()).decode()
+            headers["Authorization"] = f"Basic {token}"
+    return endpoint, headers
+
+
+def init_tracing(
+    settings: ObservabilitySettings,
+    *,
+    span_processor: SpanProcessor | None = None,
+) -> bool:
+    """Build and install the TracerProvider.
+
+    Args:
+        settings: Observability config. When ``enabled`` is false or
+            ``exporter`` is ``"none"``, this is a no-op.
+        span_processor: Injectable processor (tests pass an in-memory one).
+            Defaults to a ``BatchSpanProcessor`` over an OTLP/HTTP exporter.
+
+    Returns:
+        True if a real provider was installed; False when disabled, the
+        exporter is ``none``, or the ``[otel]`` extra is not installed.
+    """
+    global _provider
+    if not settings.enabled or settings.exporter == "none":
+        return False
+    if not _OTEL_AVAILABLE:
+        logger.warning("observability_enabled_but_otel_not_installed")
+        return False
+
+    from everos import __version__
+
+    resource = Resource.create(
+        {"service.name": settings.service_name, "service.version": __version__}
+    )
+    provider = TracerProvider(
+        resource=resource,
+        sampler=ParentBased(TraceIdRatioBased(settings.sample_rate)),
+    )
+    endpoint, headers = _resolve_otlp_target(settings)
+    processor = span_processor or BatchSpanProcessor(
+        OTLPSpanExporter(endpoint=endpoint, headers=headers)
+    )
+    provider.add_span_processor(processor)
+    _provider = provider
+
+    from .attributes import set_capture_content
+
+    set_capture_content(settings.capture_content)
+    logger.info(
+        "tracing_initialized",
+        service_name=settings.service_name,
+        capture_content=settings.capture_content,
+    )
+    return True
+
+
+def get_tracer(name: str) -> Any:
+    """Return a tracer for ``name`` — the real one if initialized, else no-op."""
+    if _provider is None:
+        return _NOOP_TRACER
+    return _provider.get_tracer(name)
+
+
+def force_flush(timeout_millis: int = 5000) -> None:
+    """Flush pending spans. No-op (and never raises) when uninitialized."""
+    if _provider is None:
+        return
+    try:
+        _provider.force_flush(timeout_millis)
+    except Exception:  # pragma: no cover - telemetry must never break callers
+        logger.warning("tracing_force_flush_failed", exc_info=True)
+
+
+def shutdown_tracing() -> None:
+    """Flush + tear down the provider; safe to call when uninitialized."""
+    global _provider
+    if _provider is None:
+        return
+    try:
+        _provider.force_flush()
+        _provider.shutdown()
+    except Exception:  # pragma: no cover - telemetry must never break callers
+        logger.warning("tracing_shutdown_failed", exc_info=True)
+    finally:
+        _provider = None
+        from .attributes import set_capture_content
+
+        set_capture_content(False)

src/everos/core/observability/tracing/scores.py

@@ -0,0 +1,183 @@
+"""Non-blocking recall-score push to Langfuse.
+
+Recall-quality scores are a Langfuse-specific REST object (POST
+``/api/public/scores``), independent of the OTLP span stream. To keep the
+search request path free of any network time, scores go through a bounded
+queue + a single background worker:
+
+- ``enqueue`` is O(1) and never blocks or raises — when the queue is full it
+  drops + counts (back-pressure never reaches the caller).
+- the worker drains the queue and POSTs one score at a time (matching the
+  Langfuse scores API + the reference prototype); every send is wrapped so a
+  network failure only logs and the worker keeps going.
+
+Attaches to the originating span via ``traceId`` (OTel trace_id, 032x hex) +
+``observationId`` (OTel span_id, 016x hex) — exactly the mapping Langfuse's
+OTLP ingestion uses.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import contextlib
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from everos.core.observability.logging import get_logger
+
+if TYPE_CHECKING:
+    from everos.config.settings import ObservabilitySettings
+
+logger = get_logger(__name__)
+
+Sender = Callable[[dict], Awaitable[None]]
+
+
+@dataclass(frozen=True)
+class ScoreRecord:
+    """One recall score bound for the Langfuse scores API."""
+
+    trace_id: str
+    observation_id: str | None
+    name: str
+    value: float
+    comment: str | None
+
+
+def _to_payload(record: ScoreRecord) -> dict:
+    payload: dict = {
+        "traceId": record.trace_id,
+        "name": record.name,
+        "value": record.value,
+        "dataType": "NUMERIC",
+    }
+    if record.observation_id:
+        payload["observationId"] = record.observation_id
+    if record.comment:
+        payload["comment"] = record.comment
+    return payload
+
+
+class RecallScoreSink:
+    """Bounded queue + background worker that POSTs scores out-of-band."""
+
+    def __init__(
+        self,
+        *,
+        sender: Sender,
+        closer: Callable[[], Awaitable[None]] | None = None,
+        max_queue: int = 1000,
+    ) -> None:
+        self._sender = sender
+        self._closer = closer
+        self._queue: asyncio.Queue[ScoreRecord] = asyncio.Queue(maxsize=max_queue)
+        self._task: asyncio.Task[None] | None = None
+        self.dropped = 0
+
+    def enqueue(self, record: ScoreRecord) -> None:
+        """Hand a score to the worker; never blocks or raises."""
+        try:
+            self._queue.put_nowait(record)
+        except asyncio.QueueFull:
+            self.dropped += 1
+            logger.warning("recall_score_dropped_queue_full", dropped=self.dropped)
+
+    def start(self) -> None:
+        self._task = asyncio.create_task(self._run())
+
+    async def _run(self) -> None:
+        while True:
+            record = await self._queue.get()
+            try:
+                await self._sender(_to_payload(record))
+            except Exception:  # telemetry must never break; log + continue
+                logger.warning("recall_score_send_failed", exc_info=True)
+            finally:
+                self._queue.task_done()
+
+    async def stop(self, *, drain_timeout: float = 5.0) -> None:
+        """Drain pending scores (bounded by ``drain_timeout``), then tear down."""
+        try:
+            await asyncio.wait_for(self._queue.join(), timeout=drain_timeout)
+        except TimeoutError:
+            logger.warning("recall_score_drain_timeout")
+        if self._task is not None:
+            self._task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._task
+            self._task = None
+        if self._closer is not None:
+            with contextlib.suppress(Exception):
+                await self._closer()
+
+
+# ── Module-level lifecycle (mirrors the tracer provider pattern) ─────────
+_sink: RecallScoreSink | None = None
+
+
+def init_score_sink(settings: ObservabilitySettings) -> bool:
+    """Build + start the sink when Langfuse creds + emit_recall_scores are set.
+
+    Returns True if a sink was installed, False otherwise (disabled, scores
+    off, or missing creds) — in which case ``emit_recall_scores`` is a no-op.
+    """
+    global _sink
+    if not settings.enabled or not settings.emit_recall_scores:
+        return False
+    pk = settings.langfuse_public_key
+    sk = settings.langfuse_secret_key
+    host = settings.langfuse_host
+    if not (pk and sk and host):
+        return False
+
+    import httpx
+
+    endpoint = host.rstrip("/") + "/api/public/scores"
+    token = base64.b64encode(f"{pk}:{sk.get_secret_value()}".encode()).decode()
+    auth = f"Basic {token}"
+    client = httpx.AsyncClient(timeout=5.0)
+
+    async def sender(payload: dict) -> None:
+        resp = await client.post(
+            endpoint, json=payload, headers={"Authorization": auth}
+        )
+        resp.raise_for_status()
+
+    _sink = RecallScoreSink(sender=sender, closer=client.aclose)
+    _sink.start()
+    logger.info("recall_score_sink_started", endpoint=endpoint)
+    return True
+
+
+def emit_recall_scores(
+    *,
+    trace_id: str,
+    observation_id: str | None,
+    top_score: float,
+    hit: bool,
+    method: str,
+) -> None:
+    """Enqueue recall_top_score + recall_hit; no-op when the sink is off."""
+    if _sink is None:
+        return
+    comment = f"method={method}"
+    _sink.enqueue(
+        ScoreRecord(
+            trace_id, observation_id, "recall_top_score", float(top_score), comment
+        )
+    )
+    _sink.enqueue(
+        ScoreRecord(
+            trace_id, observation_id, "recall_hit", 1.0 if hit else 0.0, comment
+        )
+    )
+
+
+async def shutdown_score_sink() -> None:
+    """Drain + tear down the sink; safe when uninitialized."""
+    global _sink
+    if _sink is not None:
+        await _sink.stop()
+        _sink = None

src/everos/entrypoints/api/app.py

@@ -14,6 +14,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from everos.core.lifespan import (
     LifespanProvider,
     MetricsLifespanProvider,
+    TracingLifespanProvider,
     build_lifespan,
 )
 from everos.core.middleware import (
@@ -23,6 +24,7 @@ from everos.core.middleware import (
     DEFAULT_CORS_ORIGINS,
     ProfileMiddleware,
     PrometheusMiddleware,
+    RequestIdMiddleware,
 )
 from everos.core.observability.logging import get_logger
 
@@ -68,7 +70,8 @@ def create_app(
         cors_allow_methods: Allowed CORS methods (default: ``["*"]``).
         cors_allow_headers: Allowed CORS headers (default: ``["*"]``).
         lifespan_providers: Optional list of LifespanProvider; defaults to
-            ``[MetricsLifespanProvider(), SqliteLifespanProvider(),
+            ``[TracingLifespanProvider(), MetricsLifespanProvider(),
+            LLMLifespanProvider(), SqliteLifespanProvider(),
             LanceDBLifespanProvider(), CascadeLifespanProvider(),
             OmeLifespanProvider()]``.
 
@@ -79,6 +82,7 @@ def create_app(
 
     if lifespan_providers is None:
         lifespan_providers = [
+            TracingLifespanProvider(),
             MetricsLifespanProvider(),
             LLMLifespanProvider(),
             SqliteLifespanProvider(),
@@ -113,6 +117,9 @@ def create_app(
     )
     app.add_middleware(PrometheusMiddleware)
     app.add_middleware(ProfileMiddleware)
+    # Outermost: every request gets a request id before any other middleware
+    # or handler runs, so all logs + the response header carry it.
+    app.add_middleware(RequestIdMiddleware)
 
     # Routes.
     app.include_router(health.router)

src/everos/memory/get/manager.py

@@ -21,8 +21,8 @@ import json
 from typing import TYPE_CHECKING, Any
 
 from everos.component.utils.datetime import to_display_tz
+from everos.core.context import resolve_request_id
 from everos.core.observability.logging import get_logger
-from everos.core.observability.tracing import gen_request_id
 
 from .dto import (
     GetAgentCaseItem,
@@ -68,7 +68,7 @@ class GetManager:
     # ── Public entry ─────────────────────────────────────────────────
 
     async def get(self, req: GetRequest) -> GetResponse:
-        request_id = gen_request_id()
+        request_id = resolve_request_id()
         descending = req.sort_order == "desc"
         where = compile_filters_for_get(
             req.filters,

tests/unit/test_config/test_observability_settings.py

@@ -0,0 +1,86 @@
+"""Unit tests for ``ObservabilitySettings`` (OpenTelemetry tracing config)."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import pytest
+from pydantic import ValidationError
+
+from everos.config import Settings, load_settings
+from everos.config.settings import ObservabilitySettings
+
+
+@pytest.fixture(autouse=True)
+def _isolate_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    for key in list(os.environ):
+        if key.startswith("EVEROS_"):
+            monkeypatch.delenv(key, raising=False)
+    monkeypatch.setenv("EVEROS_ROOT", str(tmp_path))
+    monkeypatch.chdir(tmp_path)
+    load_settings.cache_clear()
+
+
+def test_observability_defaults_are_off_and_neutral() -> None:
+    obs = load_settings().observability
+    assert obs.enabled is False
+    assert obs.exporter == "otlp_http"
+    assert obs.endpoint == ""
+    assert obs.headers == {}
+    assert obs.service_name == "everos"
+    assert obs.sample_rate == 1.0
+
+
+def test_env_overrides_observability(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    root = tmp_path / "r"
+    root.mkdir()
+    monkeypatch.setenv("EVEROS_OBSERVABILITY__ENABLED", "true")
+    monkeypatch.setenv(
+        "EVEROS_OBSERVABILITY__ENDPOINT", "https://otlp.example/v1/traces"
+    )
+    monkeypatch.setenv("EVEROS_OBSERVABILITY__SERVICE_NAME", "everos-test")
+    s = Settings(_everos_root=root)
+    assert s.observability.enabled is True
+    assert s.observability.endpoint == "https://otlp.example/v1/traces"
+    assert s.observability.service_name == "everos-test"
+
+
+def test_sample_rate_out_of_range_rejected() -> None:
+    with pytest.raises(ValidationError):
+        ObservabilitySettings(sample_rate=1.5)
+    with pytest.raises(ValidationError):
+        ObservabilitySettings(sample_rate=-0.1)
+
+
+def test_recall_score_defaults() -> None:
+    obs = load_settings().observability
+    assert obs.langfuse_public_key is None
+    assert obs.langfuse_secret_key is None
+    assert obs.langfuse_host is None
+    assert obs.emit_recall_scores is True
+    assert obs.recall_hit_threshold == 0.6
+
+
+def test_secret_key_is_not_leaked_in_repr() -> None:
+    obs = ObservabilitySettings(langfuse_secret_key="sk-lf-supersecret")
+    # SecretStr masks the value in repr/str.
+    assert "sk-lf-supersecret" not in repr(obs)
+    assert obs.langfuse_secret_key is not None
+    assert obs.langfuse_secret_key.get_secret_value() == "sk-lf-supersecret"
+
+
+def test_env_overrides_recall_score_fields(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    root = tmp_path / "r"
+    root.mkdir()
+    monkeypatch.setenv("EVEROS_OBSERVABILITY__LANGFUSE_HOST", "https://lf.example")
+    monkeypatch.setenv("EVEROS_OBSERVABILITY__EMIT_RECALL_SCORES", "false")
+    monkeypatch.setenv("EVEROS_OBSERVABILITY__RECALL_HIT_THRESHOLD", "0.8")
+    s = Settings(_everos_root=root)
+    assert s.observability.langfuse_host == "https://lf.example"
+    assert s.observability.emit_recall_scores is False
+    assert s.observability.recall_hit_threshold == 0.8

tests/unit/test_core/test_context.py

@@ -0,0 +1,48 @@
+"""``core.context`` — request-scoped contextvar propagation.
+
+The request id lives in a ``ContextVar`` so it flows across ``await``
+boundaries (HTTP middleware → service → infra → logs) without being
+threaded through every call signature.
+"""
+
+from __future__ import annotations
+
+from everos.core.context import (
+    get_request_id,
+    reset_request_id,
+    resolve_request_id,
+    set_request_id,
+)
+
+
+def test_get_request_id_defaults_to_none() -> None:
+    assert get_request_id() is None
+
+
+def test_set_request_id_roundtrip() -> None:
+    token = set_request_id("abc123")
+    try:
+        assert get_request_id() == "abc123"
+    finally:
+        reset_request_id(token)
+
+
+def test_reset_request_id_restores_previous() -> None:
+    token = set_request_id("first")
+    assert get_request_id() == "first"
+    reset_request_id(token)
+    assert get_request_id() is None
+
+
+def test_resolve_returns_bound_id_when_present() -> None:
+    token = set_request_id("deadbeef" * 4)
+    try:
+        assert resolve_request_id() == "deadbeef" * 4
+    finally:
+        reset_request_id(token)
+
+
+def test_resolve_mints_32hex_when_absent() -> None:
+    rid = resolve_request_id()
+    assert len(rid) == 32
+    assert all(c in "0123456789abcdef" for c in rid)

tests/unit/test_core/test_lifespan/test_tracing_lifespan.py

@@ -0,0 +1,92 @@
+"""``TracingLifespanProvider`` — reads [observability] and manages the
+tracer provider over the app lifespan.
+
+Startup returns whether tracing was enabled; shutdown always flushes and
+tears down without raising. Span-capture behavior itself is covered in
+``test_observability/test_tracing.py``; here we assert the lifespan wiring
+contract only.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+from fastapi import FastAPI
+
+from everos.config import load_settings
+from everos.core.lifespan import TracingLifespanProvider
+from everos.core.observability.tracing import shutdown_tracing
+
+
+@pytest.fixture(autouse=True)
+def _isolate(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]:
+    for key in list(os.environ):
+        if key.startswith("EVEROS_"):
+            monkeypatch.delenv(key, raising=False)
+    monkeypatch.setenv("EVEROS_ROOT", str(tmp_path))
+    monkeypatch.chdir(tmp_path)
+    load_settings.cache_clear()
+    shutdown_tracing()
+    yield
+    shutdown_tracing()
+
+
+async def test_startup_returns_true_when_enabled(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("EVEROS_OBSERVABILITY__ENABLED", "true")
+    monkeypatch.setenv("EVEROS_OBSERVABILITY__ENDPOINT", "http://collector.invalid")
+    load_settings.cache_clear()
+    provider = TracingLifespanProvider()
+    app = FastAPI()
+    result = await provider.startup(app)
+    assert result is True
+    await provider.shutdown(app)  # must not raise
+
+
+async def test_startup_returns_false_when_disabled() -> None:
+    provider = TracingLifespanProvider()
+    app = FastAPI()
+    result = await provider.startup(app)
+    assert result is False
+    await provider.shutdown(app)  # must not raise
+
+
+def test_provider_has_low_order_to_start_first() -> None:
+    # Tracer must be live before other providers start.
+    assert TracingLifespanProvider().order < 5  # MetricsLifespanProvider is 5
+
+
+async def test_startup_installs_score_sink_with_creds(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """With Langfuse creds + emit_recall_scores, startup installs the score
+    sink; shutdown drains and tears it down."""
+    from pydantic import SecretStr
+
+    import everos.core.observability.tracing.scores as scores_mod
+    from everos.config import Settings
+    from everos.config.settings import ObservabilitySettings
+
+    obs = ObservabilitySettings(
+        enabled=True,
+        endpoint="http://collector.invalid",
+        langfuse_public_key="pk-lf",
+        langfuse_secret_key=SecretStr("sk-lf"),
+        langfuse_host="https://lf.example",
+    )
+    monkeypatch.setattr(
+        "everos.core.lifespan.tracing_lifespan.load_settings",
+        lambda: Settings(observability=obs),
+    )
+    provider = TracingLifespanProvider()
+    app = FastAPI()
+    await provider.startup(app)
+    try:
+        assert scores_mod._sink is not None
+    finally:
+        await provider.shutdown(app)
+    assert scores_mod._sink is None

tests/unit/test_core/test_middleware/test_request_id.py

@@ -0,0 +1,113 @@
+"""``RequestIdMiddleware`` — mints a W3C-compatible request id per request,
+propagates it to the endpoint via both ``request.state`` and the
+``core.context`` contextvar, and echoes it on the ``X-Request-Id``
+response header.
+
+The contextvar assertion is the load-bearing one: it proves the id set in
+the middleware crosses Starlette's ``BaseHTTPMiddleware`` task boundary and
+is visible to downstream handlers / loggers.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+
+import pytest
+from fastapi import FastAPI, Request
+from httpx import ASGITransport, AsyncClient
+
+from everos.core.context import get_request_id
+from everos.core.middleware import RequestIdMiddleware
+
+
+def _build_app() -> FastAPI:
+    app = FastAPI()
+    app.add_middleware(RequestIdMiddleware)
+
+    @app.get("/echo")
+    async def echo(request: Request) -> dict[str, str | None]:
+        return {
+            "from_state": getattr(request.state, "request_id", None),
+            "from_contextvar": get_request_id(),
+        }
+
+    return app
+
+
+@pytest.fixture
+async def client() -> AsyncIterator[AsyncClient]:
+    app = _build_app()
+    async with AsyncClient(
+        transport=ASGITransport(app=app), base_url="http://test"
+    ) as c:
+        yield c
+
+
+async def test_sets_request_id_on_state_and_response_header(
+    client: AsyncClient,
+) -> None:
+    resp = await client.get("/echo")
+    assert resp.status_code == 200
+    rid = resp.headers["x-request-id"]
+    assert len(rid) == 32  # W3C trace-id shape (gen_request_id)
+    assert resp.json()["from_state"] == rid
+
+
+async def test_request_id_visible_to_endpoint_via_contextvar(
+    client: AsyncClient,
+) -> None:
+    resp = await client.get("/echo")
+    body = resp.json()
+    assert body["from_contextvar"] is not None
+    assert body["from_contextvar"] == resp.headers["x-request-id"]
+
+
+async def test_each_request_gets_distinct_id(client: AsyncClient) -> None:
+    r1 = await client.get("/echo")
+    r2 = await client.get("/echo")
+    assert r1.headers["x-request-id"] != r2.headers["x-request-id"]
+
+
+async def test_inbound_traceparent_continues_upstream_trace() -> None:
+    """When the request carries a W3C traceparent header, our first span
+    continues that upstream trace (distributed tracing). Absent → own root."""
+    from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+    from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+        InMemorySpanExporter,
+    )
+
+    from everos.config.settings import ObservabilitySettings
+    from everos.core.observability.tracing import (
+        force_flush,
+        init_tracing,
+        memory_span,
+        shutdown_tracing,
+    )
+
+    exporter = InMemorySpanExporter()
+    shutdown_tracing()
+    init_tracing(
+        ObservabilitySettings(enabled=True, endpoint="http://collector.invalid"),
+        span_processor=SimpleSpanProcessor(exporter),
+    )
+    app = FastAPI()
+    app.add_middleware(RequestIdMiddleware)
+
+    @app.get("/s")
+    async def s() -> dict[str, str]:
+        with memory_span("everos.memory.search", observation_type="retriever"):
+            pass
+        return {"ok": "1"}
+
+    upstream = "1234567890abcdef1234567890abcdef"
+    tp = f"00-{upstream}-1111111111111111-01"
+    try:
+        async with AsyncClient(
+            transport=ASGITransport(app=app), base_url="http://test"
+        ) as c:
+            await c.get("/s", headers={"traceparent": tp})
+        force_flush()
+        span = exporter.get_finished_spans()[0]
+        assert format(span.context.trace_id, "032x") == upstream
+    finally:
+        shutdown_tracing()

tests/unit/test_core/test_observability/test_scores.py

@@ -0,0 +1,89 @@
+"""``RecallScoreSink`` — non-blocking recall-score push.
+
+The sink is the piece that guarantees the Langfuse scores REST call never
+touches the search request path: ``enqueue`` is O(1) and never blocks / raises
+(drops + counts when full); a background worker drains and sends. The network
+``sender`` is injected so these tests assert the queue/worker contract offline.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from everos.core.observability.tracing.scores import RecallScoreSink, ScoreRecord
+
+
+async def test_worker_sends_payload_in_langfuse_shape() -> None:
+    sent: list[dict] = []
+    done = asyncio.Event()
+
+    async def sender(payload: dict) -> None:
+        sent.append(payload)
+        done.set()
+
+    sink = RecallScoreSink(sender=sender, max_queue=10)
+    sink.start()
+    sink.enqueue(
+        ScoreRecord(
+            trace_id="tid",
+            observation_id="oid",
+            name="recall_top_score",
+            value=0.8,
+            comment="method=hybrid",
+        )
+    )
+    await asyncio.wait_for(done.wait(), timeout=1.0)
+    await sink.stop()
+
+    assert sent[0] == {
+        "traceId": "tid",
+        "observationId": "oid",
+        "name": "recall_top_score",
+        "value": 0.8,
+        "dataType": "NUMERIC",
+        "comment": "method=hybrid",
+    }
+
+
+async def test_enqueue_never_blocks_or_raises_when_full() -> None:
+    async def slow(_: dict) -> None:
+        await asyncio.sleep(10)
+
+    sink = RecallScoreSink(sender=slow, max_queue=1)
+    # No worker started → queue fills. enqueue must stay non-blocking.
+    sink.enqueue(ScoreRecord("t", "o", "n", 1.0, None))  # fills the single slot
+    sink.enqueue(ScoreRecord("t", "o", "n", 1.0, None))  # dropped, no raise
+    assert sink.dropped == 1
+
+
+async def test_sender_failure_does_not_crash_worker() -> None:
+    calls: list[dict] = []
+    second = asyncio.Event()
+
+    async def flaky(payload: dict) -> None:
+        calls.append(payload)
+        if len(calls) == 1:
+            raise RuntimeError("boom")
+        second.set()
+
+    sink = RecallScoreSink(sender=flaky, max_queue=10)
+    sink.start()
+    sink.enqueue(ScoreRecord("t", "o", "n1", 1.0, None))  # sender raises
+    sink.enqueue(ScoreRecord("t", "o", "n2", 2.0, None))  # worker must survive
+    await asyncio.wait_for(second.wait(), timeout=1.0)
+    await sink.stop()
+    assert len(calls) == 2  # first failed but worker kept going
+
+
+async def test_stop_drains_pending() -> None:
+    sent: list[dict] = []
+
+    async def sender(payload: dict) -> None:
+        sent.append(payload)
+
+    sink = RecallScoreSink(sender=sender, max_queue=10)
+    sink.start()
+    for i in range(3):
+        sink.enqueue(ScoreRecord("t", "o", f"n{i}", float(i), None))
+    await sink.stop()  # should drain the queue before returning
+    assert len(sent) == 3

tests/unit/test_core/test_observability/test_span_attributes.py

@@ -0,0 +1,138 @@
+"""``memory_span`` / ``set_generation_usage`` — the langfuse.* + gen_ai.*
+attribute contract applied to spans.
+
+Captured via an in-memory exporter so the emitted attribute keys/values
+are asserted directly against §4 of the implementation plan.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pytest
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+    InMemorySpanExporter,
+)
+
+from everos.config.settings import ObservabilitySettings
+from everos.core.observability.tracing import (
+    capture_input,
+    capture_output,
+    force_flush,
+    init_tracing,
+    memory_span,
+    set_capture_content,
+    set_generation_usage,
+    set_redactor,
+    shutdown_tracing,
+)
+
+
+@pytest.fixture(autouse=True)
+def _reset() -> Iterator[None]:
+    shutdown_tracing()
+    yield
+    shutdown_tracing()
+
+
+@pytest.fixture
+def captured() -> Iterator[InMemorySpanExporter]:
+    exporter = InMemorySpanExporter()
+    init_tracing(
+        ObservabilitySettings(enabled=True, endpoint="http://collector.invalid"),
+        span_processor=SimpleSpanProcessor(exporter),
+    )
+    yield exporter
+
+
+def test_memory_span_sets_langfuse_attributes(
+    captured: InMemorySpanExporter,
+) -> None:
+    with memory_span(
+        "everos.memory.search",
+        observation_type="retriever",
+        session_id="s1",
+        user_id="u1",
+        metadata={"app_id": "a", "project_id": "p", "agent_id": None},
+    ):
+        pass
+    force_flush()
+    span = captured.get_finished_spans()[0]
+    attrs = span.attributes
+    assert span.name == "everos.memory.search"
+    assert attrs["langfuse.observation.type"] == "retriever"
+    assert attrs["langfuse.session.id"] == "s1"
+    assert attrs["langfuse.user.id"] == "u1"
+    assert attrs["langfuse.trace.metadata.app_id"] == "a"
+    assert attrs["langfuse.trace.metadata.project_id"] == "p"
+    # None-valued metadata is dropped, not emitted as "None".
+    assert "langfuse.trace.metadata.agent_id" not in attrs
+    assert list(attrs["langfuse.trace.tags"]) == ["everos", "memory"]
+
+
+def test_set_generation_usage_annotates_current_span(
+    captured: InMemorySpanExporter,
+) -> None:
+    with memory_span("everos.extract", observation_type="generation"):
+        set_generation_usage(model="gpt-x", input_tokens=11, output_tokens=22)
+    force_flush()
+    attrs = captured.get_finished_spans()[0].attributes
+    assert attrs["gen_ai.request.model"] == "gpt-x"
+    assert attrs["gen_ai.usage.input_tokens"] == 11
+    assert attrs["gen_ai.usage.output_tokens"] == 22
+
+
+def test_set_generation_usage_outside_span_is_noop() -> None:
+    # No active span → must not raise (and nothing to record).
+    set_generation_usage(model="x", input_tokens=1, output_tokens=2)
+
+
+def test_content_dropped_when_capture_off(captured: InMemorySpanExporter) -> None:
+    # Default: capture_content off → no observation.input/output emitted.
+    with memory_span("everos.extract", observation_type="generation") as span:
+        capture_input(span, {"query": "sensitive query"})
+        capture_output(span, "secret memory text")
+    force_flush()
+    attrs = captured.get_finished_spans()[0].attributes
+    assert "langfuse.observation.input" not in attrs
+    assert "langfuse.observation.output" not in attrs
+
+
+def test_content_emitted_when_capture_on(captured: InMemorySpanExporter) -> None:
+    set_capture_content(True)
+    try:
+        with memory_span("everos.memory.search", observation_type="retriever") as span:
+            capture_input(span, {"query": "hello"})
+            capture_output(span, "world")
+    finally:
+        set_capture_content(False)
+    force_flush()
+    attrs = captured.get_finished_spans()[0].attributes
+    import json
+
+    assert json.loads(attrs["langfuse.observation.input"]) == {"query": "hello"}
+    assert attrs["langfuse.observation.output"] == "world"
+
+
+def test_content_redacted_and_truncated(captured: InMemorySpanExporter) -> None:
+    calls: list[str] = []
+
+    def redact(text: str) -> str:
+        calls.append(text)
+        return text.replace("SECRET", "***")
+
+    set_redactor(redact)
+    set_capture_content(True)
+    try:
+        with memory_span("everos.extract", observation_type="generation") as span:
+            capture_output(span, "SECRET " + "a" * 6000)
+    finally:
+        set_capture_content(False)
+        set_redactor(None)
+    force_flush()
+    val = captured.get_finished_spans()[0].attributes["langfuse.observation.output"]
+    assert "SECRET" not in val
+    assert "***" in val
+    assert len(val) <= 4096  # truncated
+    assert calls  # redaction hook was invoked

tests/unit/test_core/test_observability/test_tracing.py

@@ -0,0 +1,170 @@
+"""``core.observability.tracing`` — provider lifecycle + tracer facade.
+
+Spans are captured via an in-memory exporter (a ``SimpleSpanProcessor``
+injected into ``init_tracing``) so assertions run offline, with no OTLP
+endpoint. The provider is kept off the OTel *global* on purpose — the
+module holds its own reference — so tests can init / shutdown repeatedly
+without hitting OTel's set-global-once guard.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pytest
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+    InMemorySpanExporter,
+)
+
+from everos.config.settings import ObservabilitySettings
+from everos.core.observability.tracing import (
+    force_flush,
+    get_tracer,
+    init_tracing,
+    shutdown_tracing,
+)
+
+
+@pytest.fixture(autouse=True)
+def _reset_tracing() -> Iterator[None]:
+    """Ensure each test starts and ends with no provider installed."""
+    shutdown_tracing()
+    yield
+    shutdown_tracing()
+
+
+@pytest.fixture
+def captured_spans() -> Iterator[InMemorySpanExporter]:
+    exporter = InMemorySpanExporter()
+    settings = ObservabilitySettings(enabled=True, endpoint="http://collector.invalid")
+    init_tracing(settings, span_processor=SimpleSpanProcessor(exporter))
+    yield exporter
+
+
+def test_disabled_tracer_is_noop_and_never_raises() -> None:
+    # No init_tracing → no provider → get_tracer returns a no-op tracer.
+    tracer = get_tracer("everos.test")
+    with tracer.start_as_current_span("everos.noop") as span:
+        span.set_attribute("k", "v")  # must not raise
+
+
+def test_init_returns_false_when_disabled() -> None:
+    assert init_tracing(ObservabilitySettings(enabled=False)) is False
+
+
+def test_init_returns_true_when_enabled(captured_spans: InMemorySpanExporter) -> None:
+    # captured_spans fixture already called init_tracing(enabled=True).
+    tracer = get_tracer("x")
+    with tracer.start_as_current_span("s"):
+        pass
+    force_flush()
+    assert len(captured_spans.get_finished_spans()) == 1
+
+
+def test_span_captured_when_enabled(captured_spans: InMemorySpanExporter) -> None:
+    tracer = get_tracer("everos.test")
+    with tracer.start_as_current_span("everos.memory.search"):
+        pass
+    force_flush()
+    names = [s.name for s in captured_spans.get_finished_spans()]
+    assert "everos.memory.search" in names
+
+
+def test_child_span_nests_under_parent(
+    captured_spans: InMemorySpanExporter,
+) -> None:
+    tracer = get_tracer("everos.test")
+    with (
+        tracer.start_as_current_span("parent"),
+        tracer.start_as_current_span("child"),
+    ):
+        pass
+    force_flush()
+    spans = {s.name: s for s in captured_spans.get_finished_spans()}
+    assert spans["child"].parent is not None
+    assert spans["child"].parent.span_id == spans["parent"].context.span_id
+
+
+def test_resource_carries_service_name(
+    captured_spans: InMemorySpanExporter,
+) -> None:
+    tracer = get_tracer("x")
+    with tracer.start_as_current_span("s"):
+        pass
+    force_flush()
+    span = captured_spans.get_finished_spans()[0]
+    assert span.resource.attributes["service.name"] == "everos"
+
+
+async def test_child_spans_nest_across_asyncio_gather(
+    captured_spans: InMemorySpanExporter,
+) -> None:
+    """OTel context must survive the asyncio.gather task boundary: child
+    spans started inside gathered coroutines nest under the parent span,
+    not as siblings/roots. This is the mechanism SearchManager.search
+    relies on (plan cross-cutting note #1)."""
+    import asyncio
+
+    tracer = get_tracer("everos.test")
+
+    async def _child(name: str) -> None:
+        with tracer.start_as_current_span(name):
+            await asyncio.sleep(0)
+
+    with tracer.start_as_current_span("parent"):
+        await asyncio.gather(_child("child_a"), _child("child_b"))
+    force_flush()
+
+    spans = {s.name: s for s in captured_spans.get_finished_spans()}
+    parent_span_id = spans["parent"].context.span_id
+    trace_id = spans["parent"].context.trace_id
+    for child in ("child_a", "child_b"):
+        assert spans[child].parent is not None
+        assert spans[child].parent.span_id == parent_span_id
+        assert spans[child].context.trace_id == trace_id
+
+
+def test_resolve_otlp_target_derives_from_langfuse_creds() -> None:
+    import base64
+
+    from pydantic import SecretStr
+
+    from everos.core.observability.tracing.provider import _resolve_otlp_target
+
+    settings = ObservabilitySettings(
+        enabled=True,
+        langfuse_public_key="pk",
+        langfuse_secret_key=SecretStr("sk"),
+        langfuse_host="https://us.cloud.langfuse.com",
+    )
+    endpoint, headers = _resolve_otlp_target(settings)
+    assert endpoint == "https://us.cloud.langfuse.com/api/public/otel/v1/traces"
+    assert headers["Authorization"] == "Basic " + base64.b64encode(b"pk:sk").decode()
+
+
+def test_resolve_otlp_target_explicit_values_win() -> None:
+    from pydantic import SecretStr
+
+    from everos.core.observability.tracing.provider import _resolve_otlp_target
+
+    settings = ObservabilitySettings(
+        enabled=True,
+        endpoint="http://explicit/v1/traces",
+        headers={"Authorization": "Basic explicit"},
+        langfuse_public_key="pk",
+        langfuse_secret_key=SecretStr("sk"),
+        langfuse_host="https://us.cloud.langfuse.com",
+    )
+    endpoint, headers = _resolve_otlp_target(settings)
+    assert endpoint == "http://explicit/v1/traces"
+    assert headers["Authorization"] == "Basic explicit"
+
+
+def test_resolve_otlp_target_no_langfuse_returns_as_is() -> None:
+    from everos.core.observability.tracing.provider import _resolve_otlp_target
+
+    settings = ObservabilitySettings(enabled=True, endpoint="http://x/v1/traces")
+    endpoint, headers = _resolve_otlp_target(settings)
+    assert endpoint == "http://x/v1/traces"
+    assert "Authorization" not in headers

uv.lock

@@ -595,11 +595,17 @@ dependencies = [
 multimodal = [
     { name = "everalgo-parser", extra = ["svg"] },
 ]
+otel = [
+    { name = "opentelemetry-exporter-otlp-proto-http" },
+    { name = "opentelemetry-sdk" },
+]
 
 [package.dev-dependencies]
 dev = [
     { name = "import-linter" },
     { name = "ipdb" },
+    { name = "opentelemetry-exporter-otlp-proto-http" },
+    { name = "opentelemetry-sdk" },
     { name = "pre-commit" },
     { name = "pyinstrument" },
     { name = "pytest" },
@@ -625,6 +631,8 @@ requires-dist = [
     { name = "jieba", specifier = ">=0.42.1,<1.0" },
     { name = "lancedb", specifier = ">=0.13.0" },
     { name = "openai", specifier = ">=1.0.0" },
+    { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.27.0" },
+    { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.27.0" },
     { name = "portalocker", specifier = ">=2.8.2" },
     { name = "prometheus-client", specifier = ">=0.20.0" },
     { name = "pydantic", specifier = ">=2.7.1" },
@@ -639,12 +647,14 @@ requires-dist = [
     { name = "watchdog", specifier = ">=4.0.0" },
     { name = "watchfiles", specifier = ">=0.21.0" },
 ]
-provides-extras = ["multimodal"]
+provides-extras = ["multimodal", "otel"]
 
 [package.metadata.requires-dev]
 dev = [
     { name = "import-linter", specifier = ">=2.0" },
     { name = "ipdb", specifier = ">=0.13.13" },
+    { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.27.0" },
+    { name = "opentelemetry-sdk", specifier = ">=1.27.0" },
     { name = "pre-commit", specifier = ">=4.0.0" },
     { name = "pyinstrument", specifier = ">=5.0.0" },
     { name = "pytest", specifier = ">=8.4.0" },
@@ -688,6 +698,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" },
 ]
 
+[[package]]
+name = "googleapis-common-protos"
+version = "1.75.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" },
+]
+
 [[package]]
 name = "greenlet"
 version = "3.5.3"
@@ -1339,6 +1361,87 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9d/1c/5d43735b2553baae2a5e899dcbcd0670a86930d993184d72ca909bf11c9b/openai-2.36.0-py3-none-any.whl", hash = "sha256:143f6194b548dbc2c921af1f1b03b9f14c85fed8a75b5b516f5bcc11a2a50c63", size = 1302361, upload-time = "2026-05-07T17:33:15.063Z" },
 ]
 
+[[package]]
+name = "opentelemetry-api"
+version = "1.44.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" },
+]
+
+[[package]]
+name = "opentelemetry-exporter-otlp-proto-common"
+version = "1.44.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "opentelemetry-proto" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" },
+]
+
+[[package]]
+name = "opentelemetry-exporter-otlp-proto-http"
+version = "1.44.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "googleapis-common-protos" },
+    { name = "opentelemetry-api" },
+    { name = "opentelemetry-exporter-otlp-proto-common" },
+    { name = "opentelemetry-proto" },
+    { name = "opentelemetry-sdk" },
+    { name = "requests" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" },
+]
+
+[[package]]
+name = "opentelemetry-proto"
+version = "1.44.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" },
+]
+
+[[package]]
+name = "opentelemetry-sdk"
+version = "1.44.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "opentelemetry-api" },
+    { name = "opentelemetry-semantic-conventions" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" },
+]
+
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.65b0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "opentelemetry-api" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" },
+]
+
 [[package]]
 name = "packaging"
 version = "26.2"
@@ -1505,6 +1608,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" },
 ]
 
+[[package]]
+name = "protobuf"
+version = "7.35.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" },
+    { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" },
+    { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" },
+]
+
 [[package]]
 name = "psutil"
 version = "7.2.2"