diff --git a/.claude/rules/logging-observability.md b/.claude/rules/logging-observability.md index 46faf51..d7055d2 100644 --- a/.claude/rules/logging-observability.md +++ b/.claude/rules/logging-observability.md @@ -21,3 +21,11 @@ paths: - **Metrics** go through `core.observability.metrics` (Prometheus); don't invent ad-hoc counters. Histograms/counters/gauges have registry helpers. - Don't log secrets, API keys, or full memory content at `info`/above. +- **Tracing** (optional, `[otel]` extra, **off by default**): open spans with + `memory_span(...)` from `core.observability.tracing` — it stamps the Langfuse + `langfuse.*` attributes and is a no-op until `[observability] enabled`, so call + sites never branch on config. LLM / embedding token usage rides + `set_generation_usage` onto the active span (Langfuse computes cost). + Request/response content is emitted only when `capture_content` is on + (redaction hook + truncation). `request_id` is kept independent of the OTel + `trace_id`; an upstream `traceparent` header is continued when present. diff --git a/pyproject.toml b/pyproject.toml index 216b247..b9693c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,12 @@ dependencies = [ [project.optional-dependencies] multimodal = ["everalgo-parser[svg]>=0.2.1"] # [svg] bundles cairosvg → SVG works by default +# Native OpenTelemetry tracing export. Optional — EverOS never imports these +# unless [observability] is enabled. Install with: pip install everos[otel] +otel = [ + "opentelemetry-sdk>=1.27.0", + "opentelemetry-exporter-otlp-proto-http>=1.27.0", +] [project.urls] Homepage = "https://evermind.ai" @@ -255,4 +261,8 @@ dev = [ "pre-commit>=4.0.0", "ipdb>=0.13.13", "pyinstrument>=5.0.0", + # Tracing tests must actually run (no skip-when-absent), so the optional + # [otel] stack is always present in the dev / CI environment. + "opentelemetry-sdk>=1.27.0", + "opentelemetry-exporter-otlp-proto-http>=1.27.0", ] diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index 68fab81..539e68e 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -145,3 +145,21 @@ session_lock_timeout_seconds = 360.0 threshold = 0.65 time_window_days = 7.0 + +[observability] +# OpenTelemetry tracing export. Off by default; pure OTLP/HTTP, vendor-neutral +# (Langfuse, an OTel Collector, or any OTLP backend). EverOS ships no vendor SDK. +# Override via EVEROS_OBSERVABILITY__ENABLED, EVEROS_OBSERVABILITY__ENDPOINT, etc. +enabled = false +exporter = "otlp_http" # "otlp_http" | "none" +endpoint = "" # e.g. https://us.cloud.langfuse.com/api/public/otel/v1/traces +service_name = "everos" +sample_rate = 1.0 # 0.0 to 1.0 +# Privacy: false (default) = metadata only; true also emits query / extracted +# memory / .md paths as span input/output (redacted + truncated). +capture_content = false +# Recall-quality scores pushed to Langfuse (Langfuse-specific REST, off the +# OTLP stream). Only fires when langfuse_public_key/secret_key/host are set +# (via everos.toml or EVEROS_OBSERVABILITY__LANGFUSE_* — secrets, not shipped here). +emit_recall_scores = true +recall_hit_threshold = 0.6 # only meaningful for calibrated methods diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 5ae2c8d..354d6c5 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -350,6 +350,53 @@ class KnowledgeSettings(BaseModel): search: KnowledgeSearchSettings = KnowledgeSearchSettings() +class ObservabilitySettings(BaseModel): + """``[observability]`` — OpenTelemetry tracing export. + + Off by default. When ``enabled`` is true a ``TracerProvider`` is built + once at startup and standard OTLP/HTTP spans are exported to + ``endpoint``. The signal is pure OpenTelemetry — vendor-neutral — so it + works with any OTLP backend (Langfuse, an OTel Collector, ...); EverOS + does not depend on any vendor SDK. + + ``langfuse_*`` are convenience credentials for pushing recall-quality + *scores* to Langfuse (a Langfuse-specific REST call, independent of the + OTLP span stream). Leave unset for a pure vendor-neutral OTLP export. + + Env binding: + EVEROS_OBSERVABILITY__ENABLED + EVEROS_OBSERVABILITY__EXPORTER + EVEROS_OBSERVABILITY__ENDPOINT + EVEROS_OBSERVABILITY__SERVICE_NAME + EVEROS_OBSERVABILITY__SAMPLE_RATE + EVEROS_OBSERVABILITY__LANGFUSE_PUBLIC_KEY / __LANGFUSE_SECRET_KEY + EVEROS_OBSERVABILITY__LANGFUSE_HOST + EVEROS_OBSERVABILITY__EMIT_RECALL_SCORES + EVEROS_OBSERVABILITY__RECALL_HIT_THRESHOLD + """ + + enabled: bool = False + exporter: Literal["otlp_http", "none"] = "otlp_http" + endpoint: str = "" + headers: dict[str, str] = Field(default_factory=dict) + service_name: str = "everos" + sample_rate: float = Field(default=1.0, ge=0.0, le=1.0) + # Privacy: when False (default) spans carry metadata only — no query text, + # extracted memory, or .md paths. Set True to also emit request/response + # content as span input/output (redacted + truncated). + capture_content: bool = False + + # Langfuse scores (recall-quality feedback) — optional, Langfuse-specific. + langfuse_public_key: str | None = None + langfuse_secret_key: SecretStr | None = None + langfuse_host: str | None = None + emit_recall_scores: bool = True + # ``hit`` threshold: only meaningful for calibrated-score methods + # (HYBRID LR / rerank / agentic). Not bounded to [0, 1] because raw + # BM25 scores are unbounded; tune per method on the eval side. + recall_hit_threshold: float = 0.6 + + class Settings(BaseSettings): """Top-level application settings.""" @@ -365,6 +412,7 @@ class Settings(BaseSettings): clustering: ClusteringSettings = ClusteringSettings() multimodal: MultimodalSettings = MultimodalSettings() knowledge: KnowledgeSettings = KnowledgeSettings() + observability: ObservabilitySettings = ObservabilitySettings() model_config = SettingsConfigDict( env_prefix="EVEROS_", diff --git a/src/everos/core/context/__init__.py b/src/everos/core/context/__init__.py index e69de29..3070c2d 100644 --- a/src/everos/core/context/__init__.py +++ b/src/everos/core/context/__init__.py @@ -0,0 +1,22 @@ +"""core.context — request-scoped context propagation (contextvars). + +External usage:: + + from everos.core.context import ( + get_request_id, + set_request_id, + reset_request_id, + ) +""" + +from .request import get_request_id as get_request_id +from .request import reset_request_id as reset_request_id +from .request import resolve_request_id as resolve_request_id +from .request import set_request_id as set_request_id + +__all__ = [ + "get_request_id", + "reset_request_id", + "resolve_request_id", + "set_request_id", +] diff --git a/src/everos/core/context/request.py b/src/everos/core/context/request.py new file mode 100644 index 0000000..f598536 --- /dev/null +++ b/src/everos/core/context/request.py @@ -0,0 +1,39 @@ +"""Request-scoped context propagation via ``contextvars``. + +The request id is stored in a module-level ``ContextVar`` so it survives +``await`` boundaries and is readable anywhere in the call chain (service, +infra, log processors) without being threaded through call signatures. +""" + +from __future__ import annotations + +from contextvars import ContextVar, Token + +from everos.core.observability.tracing import gen_request_id + +_request_id: ContextVar[str | None] = ContextVar("everos_request_id", default=None) + + +def get_request_id() -> str | None: + """Return the request id bound to the current context, or ``None``.""" + return _request_id.get() + + +def set_request_id(value: str | None) -> Token[str | None]: + """Bind ``value`` as the current request id; return a reset token.""" + return _request_id.set(value) + + +def reset_request_id(token: Token[str | None]) -> None: + """Restore the request id to what it was before the matching ``set``.""" + _request_id.reset(token) + + +def resolve_request_id() -> str: + """Return the propagated request id, or mint a fresh W3C-compatible one. + + Call sites that need an id (search / get managers) use this so an id + injected upstream by ``RequestIdMiddleware`` flows through to the + response, while direct / CLI callers still get a freshly minted id. + """ + return get_request_id() or gen_request_id() diff --git a/src/everos/core/lifespan/__init__.py b/src/everos/core/lifespan/__init__.py index dd75abb..b9fe386 100644 --- a/src/everos/core/lifespan/__init__.py +++ b/src/everos/core/lifespan/__init__.py @@ -19,9 +19,11 @@ External usage: from .base import LifespanProvider as LifespanProvider from .factory import build_lifespan as build_lifespan from .metrics_lifespan import MetricsLifespanProvider as MetricsLifespanProvider +from .tracing_lifespan import TracingLifespanProvider as TracingLifespanProvider __all__ = [ "LifespanProvider", "MetricsLifespanProvider", + "TracingLifespanProvider", "build_lifespan", ] diff --git a/src/everos/core/lifespan/tracing_lifespan.py b/src/everos/core/lifespan/tracing_lifespan.py new file mode 100644 index 0000000..29fc4f7 --- /dev/null +++ b/src/everos/core/lifespan/tracing_lifespan.py @@ -0,0 +1,50 @@ +"""Tracing lifespan provider. + +Builds the OpenTelemetry ``TracerProvider`` at startup (from the +``[observability]`` settings) and flushes + tears it down at shutdown. +Chassis-level (backend-agnostic), so it lives here alongside +``MetricsLifespanProvider`` rather than under the API entrypoint. + +Registered with a low ``order`` so the tracer is live before other +providers start and can themselves be traced. +""" + +from __future__ import annotations + +from fastapi import FastAPI + +from everos.config import load_settings +from everos.core.observability.logging import get_logger +from everos.core.observability.tracing import ( + init_score_sink, + init_tracing, + shutdown_score_sink, + shutdown_tracing, +) + +from .base import LifespanProvider + +logger = get_logger(__name__) + + +class TracingLifespanProvider(LifespanProvider): + """Manages the OTel tracer provider + recall-score sink over the app life.""" + + def __init__(self, order: int = 1) -> None: + super().__init__(name="tracing", order=order) + + async def startup(self, app: FastAPI) -> bool: + """Install the tracer provider + recall-score sink when configured. + + Returns True if tracing was enabled and a provider installed. + """ + settings = load_settings().observability + enabled = init_tracing(settings) + scores = init_score_sink(settings) + logger.info("tracing_lifespan_startup", enabled=enabled, scores=scores) + return enabled + + async def shutdown(self, app: FastAPI) -> None: + await shutdown_score_sink() + shutdown_tracing() + logger.info("tracing_lifespan_shutdown") diff --git a/src/everos/core/middleware/__init__.py b/src/everos/core/middleware/__init__.py index c6949c3..d298ad9 100644 --- a/src/everos/core/middleware/__init__.py +++ b/src/everos/core/middleware/__init__.py @@ -18,6 +18,7 @@ from .cors import DEFAULT_CORS_ALLOW_METHODS as DEFAULT_CORS_ALLOW_METHODS from .cors import DEFAULT_CORS_ORIGINS as DEFAULT_CORS_ORIGINS from .profile import ProfileMiddleware as ProfileMiddleware from .prometheus import PrometheusMiddleware as PrometheusMiddleware +from .request_id import RequestIdMiddleware as RequestIdMiddleware __all__ = [ "DEFAULT_CORS_ALLOW_CREDENTIALS", @@ -26,4 +27,5 @@ __all__ = [ "DEFAULT_CORS_ORIGINS", "ProfileMiddleware", "PrometheusMiddleware", + "RequestIdMiddleware", ] diff --git a/src/everos/core/middleware/request_id.py b/src/everos/core/middleware/request_id.py new file mode 100644 index 0000000..51a7c5f --- /dev/null +++ b/src/everos/core/middleware/request_id.py @@ -0,0 +1,56 @@ +"""Request-context middleware. + +Establishes per-request context at HTTP entry: + +* Mints a W3C-compatible ``request_id`` and binds it for the request's + lifetime — via ``request.state``, the ``core.context`` contextvar + (readable by service / infra), and structlog contextvars (so every log + line carries it). Echoed on the ``X-Request-Id`` response header. +* Continues an upstream **distributed trace**: if the request carries a + W3C ``traceparent`` header, our first span nests under that trace instead + of rooting a new one. Absent → we root our own trace (the common case). + +This is the single place a request id enters the system; ``extract_request_id`` +(``entrypoints/api/utils.py``) reads what this middleware sets. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +import structlog +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +from everos.core.context import reset_request_id, set_request_id +from everos.core.observability.tracing import gen_request_id, use_traceparent + +_HEADER = "X-Request-Id" + + +class RequestIdMiddleware(BaseHTTPMiddleware): + """Assigns a request id + continues any upstream trace for the request.""" + + async def dispatch( + self, + request: Request, + call_next: Callable[[Request], Awaitable[Response]], + ) -> Response: + request_id = gen_request_id() + request.state.request_id = request_id + # Bind before ``call_next`` so the downstream task (endpoint) inherits + # the value; reset in ``finally`` so it never leaks to the next request. + token = set_request_id(request_id) + structlog.contextvars.bind_contextvars(request_id=request_id) + try: + # Continue the upstream trace when a traceparent header is present + # (no-op otherwise). Attached before call_next so downstream spans + # inherit it across the middleware task boundary. + with use_traceparent(request.headers.get("traceparent")): + response = await call_next(request) + response.headers[_HEADER] = request_id + return response + finally: + structlog.contextvars.unbind_contextvars("request_id") + reset_request_id(token) diff --git a/src/everos/core/observability/tracing/__init__.py b/src/everos/core/observability/tracing/__init__.py index 4f3f263..e437cf8 100644 --- a/src/everos/core/observability/tracing/__init__.py +++ b/src/everos/core/observability/tracing/__init__.py @@ -1,32 +1,56 @@ -"""Tracing utilities — W3C-compatible request id generation. +"""Tracing — W3C id generation + OpenTelemetry tracer lifecycle. External usage:: - from everos.core.observability.tracing import gen_request_id + from everos.core.observability.tracing import ( + gen_request_id, + get_tracer, + init_tracing, + shutdown_tracing, + force_flush, + ) + +``get_tracer`` is safe to call unconditionally: it returns a no-op tracer +until ``init_tracing`` installs a provider (and when the optional ``[otel]`` +extra is not installed), so call sites never need to branch on config. """ from __future__ import annotations -from uuid import uuid4 +from .attributes import capture_input as capture_input +from .attributes import capture_output as capture_output +from .attributes import current_trace_ids as current_trace_ids +from .attributes import current_traceparent as current_traceparent +from .attributes import memory_span as memory_span +from .attributes import set_capture_content as set_capture_content +from .attributes import set_generation_usage as set_generation_usage +from .attributes import set_redactor as set_redactor +from .attributes import use_traceparent as use_traceparent +from .ids import gen_request_id as gen_request_id +from .provider import force_flush as force_flush +from .provider import get_tracer as get_tracer +from .provider import init_tracing as init_tracing +from .provider import shutdown_tracing as shutdown_tracing +from .scores import emit_recall_scores as emit_recall_scores +from .scores import init_score_sink as init_score_sink +from .scores import shutdown_score_sink as shutdown_score_sink - -def gen_request_id() -> str: - """Generate a request id matching the W3C trace-context spec. - - Returns 32 lowercase hex characters (128-bit, no prefix) — the same - format as a W3C ``trace_id`` / OpenTelemetry trace identifier. Routes - and services that mint a fresh request id (when one wasn't injected - by upstream middleware) should call this helper rather than rolling - their own uuid / prefix format, so the id layer stays compatible - with OpenTelemetry exporters and standard APM tooling. - - Example:: - - >>> rid = gen_request_id() - >>> len(rid) - 32 - """ - return uuid4().hex - - -__all__ = ["gen_request_id"] +__all__ = [ + "capture_input", + "capture_output", + "current_trace_ids", + "current_traceparent", + "emit_recall_scores", + "force_flush", + "gen_request_id", + "get_tracer", + "init_score_sink", + "init_tracing", + "memory_span", + "set_capture_content", + "set_generation_usage", + "set_redactor", + "shutdown_score_sink", + "shutdown_tracing", + "use_traceparent", +] diff --git a/src/everos/core/observability/tracing/attributes.py b/src/everos/core/observability/tracing/attributes.py new file mode 100644 index 0000000..46d2f5a --- /dev/null +++ b/src/everos/core/observability/tracing/attributes.py @@ -0,0 +1,195 @@ +"""Span helpers implementing the Langfuse / OpenTelemetry attribute contract. + +``memory_span`` opens a span under the shared ``everos`` tracer and stamps +the ``langfuse.*`` trace/observation attributes (observation type, session / +user ids, trace metadata, tags). ``set_generation_usage`` writes the +``gen_ai.*`` model + token attributes onto the *current* span, so an LLM +client wrapper can record usage without knowing which span is active. + +Request/response content (``langfuse.observation.input/output``) is +privacy-gated: ``capture_input`` / ``capture_output`` only emit it when +``capture_content`` is on, after a redaction hook + truncation. Off by +default, so spans carry metadata only. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager +from typing import Any + +from .provider import get_tracer + +# ── langfuse.* trace/observation attribute keys ────────────────────────── +LF_OBSERVATION_TYPE = "langfuse.observation.type" +LF_SESSION_ID = "langfuse.session.id" +LF_USER_ID = "langfuse.user.id" +LF_TAGS = "langfuse.trace.tags" +LF_METADATA_PREFIX = "langfuse.trace.metadata." + +# ── gen_ai.* generation attribute keys (Langfuse computes cost from these) ─ +GEN_AI_REQUEST_MODEL = "gen_ai.request.model" +GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" +GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" + +# ── content capture (privacy-gated) ────────────────────────────────────── +LF_OBSERVATION_INPUT = "langfuse.observation.input" +LF_OBSERVATION_OUTPUT = "langfuse.observation.output" +_MAX_CONTENT_CHARS = 4096 + +DEFAULT_TAGS: tuple[str, ...] = ("everos", "memory") + +# Off by default: no request/response content leaves the process unless +# capture_content is turned on (set once at init_tracing from settings). +_capture_content = False +_redactor: Callable[[str], str] = lambda text: text # noqa: E731 - overridable hook + + +def set_capture_content(enabled: bool) -> None: + """Toggle content capture (called from init_tracing / shutdown).""" + global _capture_content + _capture_content = enabled + + +def set_redactor(redactor: Callable[[str], str] | None) -> None: + """Install a redaction hook applied to captured content; None resets it.""" + global _redactor + _redactor = redactor if redactor is not None else (lambda text: text) + + +def _prepare_content(value: Any) -> str: + """Serialize, redact, then truncate content for a span attribute.""" + text = value if isinstance(value, str) else json.dumps(value, default=str) + text = _redactor(text) + return text[:_MAX_CONTENT_CHARS] + + +def capture_input(span: Any, value: Any) -> None: + """Set ``langfuse.observation.input`` — only when capture_content is on.""" + if _capture_content and value is not None: + span.set_attribute(LF_OBSERVATION_INPUT, _prepare_content(value)) + + +def capture_output(span: Any, value: Any) -> None: + """Set ``langfuse.observation.output`` — only when capture_content is on.""" + if _capture_content and value is not None: + span.set_attribute(LF_OBSERVATION_OUTPUT, _prepare_content(value)) + + +try: + from opentelemetry import trace as _otel_trace + + _OTEL_AVAILABLE = True +except ImportError: # pragma: no cover - only without the [otel] extra + _OTEL_AVAILABLE = False + + +@contextmanager +def memory_span( + name: str, + *, + observation_type: str, + session_id: str | None = None, + user_id: str | None = None, + metadata: Mapping[str, Any] | None = None, + tags: Sequence[str] = DEFAULT_TAGS, +) -> Iterator[Any]: + """Open a span named ``name`` and stamp the langfuse.* attributes. + + Args: + name: Span name (e.g. ``everos.memory.search``). + observation_type: ``langfuse.observation.type`` — span / generation / + embedding / retriever / agent. + session_id / user_id: Grouping ids (dropped when None). + metadata: Flat mapping → ``langfuse.trace.metadata.``; None + values are dropped rather than emitted as the string "None". + tags: ``langfuse.trace.tags`` list. + """ + tracer = get_tracer("everos") + with tracer.start_as_current_span(name) as span: + span.set_attribute(LF_OBSERVATION_TYPE, observation_type) + if session_id: + span.set_attribute(LF_SESSION_ID, session_id) + if user_id: + span.set_attribute(LF_USER_ID, user_id) + if tags: + span.set_attribute(LF_TAGS, list(tags)) + for key, value in (metadata or {}).items(): + if value is not None: + span.set_attribute(f"{LF_METADATA_PREFIX}{key}", value) + yield span + + +def current_traceparent() -> str | None: + """W3C ``traceparent`` for the current span, or None when there is none. + + Captured where a request's span is active (e.g. OME enqueue) and carried + across the async / process boundary so a background strategy span can + re-attach to the originating trace. + """ + if not _OTEL_AVAILABLE: + return None + from opentelemetry.propagate import inject + + carrier: dict[str, str] = {} + inject(carrier) + return carrier.get("traceparent") + + +@contextmanager +def use_traceparent(traceparent: str | None) -> Iterator[None]: + """Attach ``traceparent`` as the current context for the block. + + Spans opened inside become children of that (remote) trace. No-op when + the traceparent is absent or OTel is not installed — the span then roots + its own trace. + """ + if not _OTEL_AVAILABLE or not traceparent: + yield + return + from opentelemetry import context as otel_context + from opentelemetry.propagate import extract + + token = otel_context.attach(extract({"traceparent": traceparent})) + try: + yield + finally: + otel_context.detach(token) + + +def current_trace_ids() -> tuple[str, str] | None: + """Return ``(trace_id_hex_032x, span_id_hex_016x)`` of the current span. + + Returns None when OTel is absent or there is no valid recording span — + the exact hex mapping Langfuse's OTLP ingestion uses for traceId / + observationId, so recall scores attach to the right observation. + """ + if not _OTEL_AVAILABLE: + return None + ctx = _otel_trace.get_current_span().get_span_context() + if not ctx.is_valid: + return None + return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x") + + +def set_generation_usage( + *, + model: str | None = None, + input_tokens: int | None = None, + output_tokens: int | None = None, +) -> None: + """Record ``gen_ai.*`` model + token attributes on the current span. + + No-op when OTel is absent or there is no active recording span, so LLM + client wrappers can call it unconditionally. + """ + if not _OTEL_AVAILABLE: + return + span = _otel_trace.get_current_span() + if model is not None: + span.set_attribute(GEN_AI_REQUEST_MODEL, model) + if input_tokens is not None: + span.set_attribute(GEN_AI_USAGE_INPUT_TOKENS, input_tokens) + if output_tokens is not None: + span.set_attribute(GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens) diff --git a/src/everos/core/observability/tracing/ids.py b/src/everos/core/observability/tracing/ids.py new file mode 100644 index 0000000..73f8ab8 --- /dev/null +++ b/src/everos/core/observability/tracing/ids.py @@ -0,0 +1,24 @@ +"""W3C-compatible request/trace id generation.""" + +from __future__ import annotations + +from uuid import uuid4 + + +def gen_request_id() -> str: + """Generate a request id matching the W3C trace-context spec. + + Returns 32 lowercase hex characters (128-bit, no prefix) — the same + format as a W3C ``trace_id`` / OpenTelemetry trace identifier. Routes + and services that mint a fresh request id (when one wasn't injected + by upstream middleware) should call this helper rather than rolling + their own uuid / prefix format, so the id layer stays compatible + with OpenTelemetry exporters and standard APM tooling. + + Example:: + + >>> rid = gen_request_id() + >>> len(rid) + 32 + """ + return uuid4().hex diff --git a/src/everos/core/observability/tracing/provider.py b/src/everos/core/observability/tracing/provider.py new file mode 100644 index 0000000..95630cb --- /dev/null +++ b/src/everos/core/observability/tracing/provider.py @@ -0,0 +1,178 @@ +"""OpenTelemetry tracer provider lifecycle + a no-op-safe tracer facade. + +OpenTelemetry is an optional dependency (the ``[otel]`` extra). This module +never fails to import when it is absent: the SDK imports are guarded, and +``get_tracer`` returns a no-op tracer until ``init_tracing`` installs a real +provider. + +The provider is held here (module-local) rather than on the OTel *global* +so it can be built and torn down repeatedly — in tests and across restarts — +without tripping OTel's "set global provider once" guard. Span context +propagation (parent/child nesting) still works: that rides OTel's context +vars, which are independent of which provider produced the tracer. +""" + +from __future__ import annotations + +import base64 +from collections.abc import Iterator +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any + +from everos.core.observability.logging import get_logger + +logger = get_logger(__name__) + +try: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased + + _OTEL_AVAILABLE = True +except ImportError: # pragma: no cover - only without the [otel] extra + _OTEL_AVAILABLE = False + +if TYPE_CHECKING: + from opentelemetry.sdk.trace.export import SpanProcessor + + from everos.config.settings import ObservabilitySettings + +# Our TracerProvider, deliberately kept off the OTel global (see module docstring). +_provider: Any = None + + +class _NoopSpan: + """Span stand-in used when tracing is off or OTel is not installed.""" + + def set_attribute(self, key: str, value: object) -> None: ... + + def set_attributes(self, attributes: dict[str, object]) -> None: ... + + def record_exception(self, exception: BaseException) -> None: ... + + def set_status(self, *args: object, **kwargs: object) -> None: ... + + def end(self) -> None: ... + + +class _NoopTracer: + """Tracer stand-in whose spans do nothing (zero-overhead when off).""" + + @contextmanager + def start_as_current_span(self, name: str, **kwargs: object) -> Iterator[_NoopSpan]: + yield _NoopSpan() + + +_NOOP_TRACER = _NoopTracer() + + +def _resolve_otlp_target( + settings: ObservabilitySettings, +) -> tuple[str, dict[str, str]]: + """Endpoint + headers for the OTLP exporter. + + Convenience: when ``langfuse_*`` creds are set, derive the OTLP traces + endpoint and a Basic-auth header from them — but explicit ``endpoint`` / + ``headers`` always win, so a plain vendor-neutral export is unaffected. + """ + endpoint = settings.endpoint + headers = dict(settings.headers) + pk = settings.langfuse_public_key + sk = settings.langfuse_secret_key + host = settings.langfuse_host + if host and pk and sk: + if not endpoint: + endpoint = host.rstrip("/") + "/api/public/otel/v1/traces" + if "Authorization" not in headers: + token = base64.b64encode(f"{pk}:{sk.get_secret_value()}".encode()).decode() + headers["Authorization"] = f"Basic {token}" + return endpoint, headers + + +def init_tracing( + settings: ObservabilitySettings, + *, + span_processor: SpanProcessor | None = None, +) -> bool: + """Build and install the TracerProvider. + + Args: + settings: Observability config. When ``enabled`` is false or + ``exporter`` is ``"none"``, this is a no-op. + span_processor: Injectable processor (tests pass an in-memory one). + Defaults to a ``BatchSpanProcessor`` over an OTLP/HTTP exporter. + + Returns: + True if a real provider was installed; False when disabled, the + exporter is ``none``, or the ``[otel]`` extra is not installed. + """ + global _provider + if not settings.enabled or settings.exporter == "none": + return False + if not _OTEL_AVAILABLE: + logger.warning("observability_enabled_but_otel_not_installed") + return False + + from everos import __version__ + + resource = Resource.create( + {"service.name": settings.service_name, "service.version": __version__} + ) + provider = TracerProvider( + resource=resource, + sampler=ParentBased(TraceIdRatioBased(settings.sample_rate)), + ) + endpoint, headers = _resolve_otlp_target(settings) + processor = span_processor or BatchSpanProcessor( + OTLPSpanExporter(endpoint=endpoint, headers=headers) + ) + provider.add_span_processor(processor) + _provider = provider + + from .attributes import set_capture_content + + set_capture_content(settings.capture_content) + logger.info( + "tracing_initialized", + service_name=settings.service_name, + capture_content=settings.capture_content, + ) + return True + + +def get_tracer(name: str) -> Any: + """Return a tracer for ``name`` — the real one if initialized, else no-op.""" + if _provider is None: + return _NOOP_TRACER + return _provider.get_tracer(name) + + +def force_flush(timeout_millis: int = 5000) -> None: + """Flush pending spans. No-op (and never raises) when uninitialized.""" + if _provider is None: + return + try: + _provider.force_flush(timeout_millis) + except Exception: # pragma: no cover - telemetry must never break callers + logger.warning("tracing_force_flush_failed", exc_info=True) + + +def shutdown_tracing() -> None: + """Flush + tear down the provider; safe to call when uninitialized.""" + global _provider + if _provider is None: + return + try: + _provider.force_flush() + _provider.shutdown() + except Exception: # pragma: no cover - telemetry must never break callers + logger.warning("tracing_shutdown_failed", exc_info=True) + finally: + _provider = None + from .attributes import set_capture_content + + set_capture_content(False) diff --git a/src/everos/core/observability/tracing/scores.py b/src/everos/core/observability/tracing/scores.py new file mode 100644 index 0000000..ac480eb --- /dev/null +++ b/src/everos/core/observability/tracing/scores.py @@ -0,0 +1,183 @@ +"""Non-blocking recall-score push to Langfuse. + +Recall-quality scores are a Langfuse-specific REST object (POST +``/api/public/scores``), independent of the OTLP span stream. To keep the +search request path free of any network time, scores go through a bounded +queue + a single background worker: + +- ``enqueue`` is O(1) and never blocks or raises — when the queue is full it + drops + counts (back-pressure never reaches the caller). +- the worker drains the queue and POSTs one score at a time (matching the + Langfuse scores API + the reference prototype); every send is wrapped so a + network failure only logs and the worker keeps going. + +Attaches to the originating span via ``traceId`` (OTel trace_id, 032x hex) + +``observationId`` (OTel span_id, 016x hex) — exactly the mapping Langfuse's +OTLP ingestion uses. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from everos.core.observability.logging import get_logger + +if TYPE_CHECKING: + from everos.config.settings import ObservabilitySettings + +logger = get_logger(__name__) + +Sender = Callable[[dict], Awaitable[None]] + + +@dataclass(frozen=True) +class ScoreRecord: + """One recall score bound for the Langfuse scores API.""" + + trace_id: str + observation_id: str | None + name: str + value: float + comment: str | None + + +def _to_payload(record: ScoreRecord) -> dict: + payload: dict = { + "traceId": record.trace_id, + "name": record.name, + "value": record.value, + "dataType": "NUMERIC", + } + if record.observation_id: + payload["observationId"] = record.observation_id + if record.comment: + payload["comment"] = record.comment + return payload + + +class RecallScoreSink: + """Bounded queue + background worker that POSTs scores out-of-band.""" + + def __init__( + self, + *, + sender: Sender, + closer: Callable[[], Awaitable[None]] | None = None, + max_queue: int = 1000, + ) -> None: + self._sender = sender + self._closer = closer + self._queue: asyncio.Queue[ScoreRecord] = asyncio.Queue(maxsize=max_queue) + self._task: asyncio.Task[None] | None = None + self.dropped = 0 + + def enqueue(self, record: ScoreRecord) -> None: + """Hand a score to the worker; never blocks or raises.""" + try: + self._queue.put_nowait(record) + except asyncio.QueueFull: + self.dropped += 1 + logger.warning("recall_score_dropped_queue_full", dropped=self.dropped) + + def start(self) -> None: + self._task = asyncio.create_task(self._run()) + + async def _run(self) -> None: + while True: + record = await self._queue.get() + try: + await self._sender(_to_payload(record)) + except Exception: # telemetry must never break; log + continue + logger.warning("recall_score_send_failed", exc_info=True) + finally: + self._queue.task_done() + + async def stop(self, *, drain_timeout: float = 5.0) -> None: + """Drain pending scores (bounded by ``drain_timeout``), then tear down.""" + try: + await asyncio.wait_for(self._queue.join(), timeout=drain_timeout) + except TimeoutError: + logger.warning("recall_score_drain_timeout") + if self._task is not None: + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + if self._closer is not None: + with contextlib.suppress(Exception): + await self._closer() + + +# ── Module-level lifecycle (mirrors the tracer provider pattern) ───────── +_sink: RecallScoreSink | None = None + + +def init_score_sink(settings: ObservabilitySettings) -> bool: + """Build + start the sink when Langfuse creds + emit_recall_scores are set. + + Returns True if a sink was installed, False otherwise (disabled, scores + off, or missing creds) — in which case ``emit_recall_scores`` is a no-op. + """ + global _sink + if not settings.enabled or not settings.emit_recall_scores: + return False + pk = settings.langfuse_public_key + sk = settings.langfuse_secret_key + host = settings.langfuse_host + if not (pk and sk and host): + return False + + import httpx + + endpoint = host.rstrip("/") + "/api/public/scores" + token = base64.b64encode(f"{pk}:{sk.get_secret_value()}".encode()).decode() + auth = f"Basic {token}" + client = httpx.AsyncClient(timeout=5.0) + + async def sender(payload: dict) -> None: + resp = await client.post( + endpoint, json=payload, headers={"Authorization": auth} + ) + resp.raise_for_status() + + _sink = RecallScoreSink(sender=sender, closer=client.aclose) + _sink.start() + logger.info("recall_score_sink_started", endpoint=endpoint) + return True + + +def emit_recall_scores( + *, + trace_id: str, + observation_id: str | None, + top_score: float, + hit: bool, + method: str, +) -> None: + """Enqueue recall_top_score + recall_hit; no-op when the sink is off.""" + if _sink is None: + return + comment = f"method={method}" + _sink.enqueue( + ScoreRecord( + trace_id, observation_id, "recall_top_score", float(top_score), comment + ) + ) + _sink.enqueue( + ScoreRecord( + trace_id, observation_id, "recall_hit", 1.0 if hit else 0.0, comment + ) + ) + + +async def shutdown_score_sink() -> None: + """Drain + tear down the sink; safe when uninitialized.""" + global _sink + if _sink is not None: + await _sink.stop() + _sink = None diff --git a/src/everos/entrypoints/api/app.py b/src/everos/entrypoints/api/app.py index 1583671..16c2464 100644 --- a/src/everos/entrypoints/api/app.py +++ b/src/everos/entrypoints/api/app.py @@ -14,6 +14,7 @@ from fastapi.middleware.cors import CORSMiddleware from everos.core.lifespan import ( LifespanProvider, MetricsLifespanProvider, + TracingLifespanProvider, build_lifespan, ) from everos.core.middleware import ( @@ -23,6 +24,7 @@ from everos.core.middleware import ( DEFAULT_CORS_ORIGINS, ProfileMiddleware, PrometheusMiddleware, + RequestIdMiddleware, ) from everos.core.observability.logging import get_logger @@ -68,7 +70,8 @@ def create_app( cors_allow_methods: Allowed CORS methods (default: ``["*"]``). cors_allow_headers: Allowed CORS headers (default: ``["*"]``). lifespan_providers: Optional list of LifespanProvider; defaults to - ``[MetricsLifespanProvider(), SqliteLifespanProvider(), + ``[TracingLifespanProvider(), MetricsLifespanProvider(), + LLMLifespanProvider(), SqliteLifespanProvider(), LanceDBLifespanProvider(), CascadeLifespanProvider(), OmeLifespanProvider()]``. @@ -79,6 +82,7 @@ def create_app( if lifespan_providers is None: lifespan_providers = [ + TracingLifespanProvider(), MetricsLifespanProvider(), LLMLifespanProvider(), SqliteLifespanProvider(), @@ -113,6 +117,9 @@ def create_app( ) app.add_middleware(PrometheusMiddleware) app.add_middleware(ProfileMiddleware) + # Outermost: every request gets a request id before any other middleware + # or handler runs, so all logs + the response header carry it. + app.add_middleware(RequestIdMiddleware) # Routes. app.include_router(health.router) diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index 1000cbd..472033c 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -21,8 +21,8 @@ import json from typing import TYPE_CHECKING, Any from everos.component.utils.datetime import to_display_tz +from everos.core.context import resolve_request_id from everos.core.observability.logging import get_logger -from everos.core.observability.tracing import gen_request_id from .dto import ( GetAgentCaseItem, @@ -68,7 +68,7 @@ class GetManager: # ── Public entry ───────────────────────────────────────────────── async def get(self, req: GetRequest) -> GetResponse: - request_id = gen_request_id() + request_id = resolve_request_id() descending = req.sort_order == "desc" where = compile_filters_for_get( req.filters, diff --git a/tests/unit/test_config/test_observability_settings.py b/tests/unit/test_config/test_observability_settings.py new file mode 100644 index 0000000..8939fc6 --- /dev/null +++ b/tests/unit/test_config/test_observability_settings.py @@ -0,0 +1,86 @@ +"""Unit tests for ``ObservabilitySettings`` (OpenTelemetry tracing config).""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from everos.config import Settings, load_settings +from everos.config.settings import ObservabilitySettings + + +@pytest.fixture(autouse=True) +def _isolate_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + for key in list(os.environ): + if key.startswith("EVEROS_"): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + monkeypatch.chdir(tmp_path) + load_settings.cache_clear() + + +def test_observability_defaults_are_off_and_neutral() -> None: + obs = load_settings().observability + assert obs.enabled is False + assert obs.exporter == "otlp_http" + assert obs.endpoint == "" + assert obs.headers == {} + assert obs.service_name == "everos" + assert obs.sample_rate == 1.0 + + +def test_env_overrides_observability( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + root = tmp_path / "r" + root.mkdir() + monkeypatch.setenv("EVEROS_OBSERVABILITY__ENABLED", "true") + monkeypatch.setenv( + "EVEROS_OBSERVABILITY__ENDPOINT", "https://otlp.example/v1/traces" + ) + monkeypatch.setenv("EVEROS_OBSERVABILITY__SERVICE_NAME", "everos-test") + s = Settings(_everos_root=root) + assert s.observability.enabled is True + assert s.observability.endpoint == "https://otlp.example/v1/traces" + assert s.observability.service_name == "everos-test" + + +def test_sample_rate_out_of_range_rejected() -> None: + with pytest.raises(ValidationError): + ObservabilitySettings(sample_rate=1.5) + with pytest.raises(ValidationError): + ObservabilitySettings(sample_rate=-0.1) + + +def test_recall_score_defaults() -> None: + obs = load_settings().observability + assert obs.langfuse_public_key is None + assert obs.langfuse_secret_key is None + assert obs.langfuse_host is None + assert obs.emit_recall_scores is True + assert obs.recall_hit_threshold == 0.6 + + +def test_secret_key_is_not_leaked_in_repr() -> None: + obs = ObservabilitySettings(langfuse_secret_key="sk-lf-supersecret") + # SecretStr masks the value in repr/str. + assert "sk-lf-supersecret" not in repr(obs) + assert obs.langfuse_secret_key is not None + assert obs.langfuse_secret_key.get_secret_value() == "sk-lf-supersecret" + + +def test_env_overrides_recall_score_fields( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + root = tmp_path / "r" + root.mkdir() + monkeypatch.setenv("EVEROS_OBSERVABILITY__LANGFUSE_HOST", "https://lf.example") + monkeypatch.setenv("EVEROS_OBSERVABILITY__EMIT_RECALL_SCORES", "false") + monkeypatch.setenv("EVEROS_OBSERVABILITY__RECALL_HIT_THRESHOLD", "0.8") + s = Settings(_everos_root=root) + assert s.observability.langfuse_host == "https://lf.example" + assert s.observability.emit_recall_scores is False + assert s.observability.recall_hit_threshold == 0.8 diff --git a/tests/unit/test_core/test_context.py b/tests/unit/test_core/test_context.py new file mode 100644 index 0000000..2005ab1 --- /dev/null +++ b/tests/unit/test_core/test_context.py @@ -0,0 +1,48 @@ +"""``core.context`` — request-scoped contextvar propagation. + +The request id lives in a ``ContextVar`` so it flows across ``await`` +boundaries (HTTP middleware → service → infra → logs) without being +threaded through every call signature. +""" + +from __future__ import annotations + +from everos.core.context import ( + get_request_id, + reset_request_id, + resolve_request_id, + set_request_id, +) + + +def test_get_request_id_defaults_to_none() -> None: + assert get_request_id() is None + + +def test_set_request_id_roundtrip() -> None: + token = set_request_id("abc123") + try: + assert get_request_id() == "abc123" + finally: + reset_request_id(token) + + +def test_reset_request_id_restores_previous() -> None: + token = set_request_id("first") + assert get_request_id() == "first" + reset_request_id(token) + assert get_request_id() is None + + +def test_resolve_returns_bound_id_when_present() -> None: + token = set_request_id("deadbeef" * 4) + try: + assert resolve_request_id() == "deadbeef" * 4 + finally: + reset_request_id(token) + + +def test_resolve_mints_32hex_when_absent() -> None: + rid = resolve_request_id() + assert len(rid) == 32 + assert all(c in "0123456789abcdef" for c in rid) diff --git a/tests/unit/test_core/test_lifespan/test_tracing_lifespan.py b/tests/unit/test_core/test_lifespan/test_tracing_lifespan.py new file mode 100644 index 0000000..f6bd230 --- /dev/null +++ b/tests/unit/test_core/test_lifespan/test_tracing_lifespan.py @@ -0,0 +1,92 @@ +"""``TracingLifespanProvider`` — reads [observability] and manages the +tracer provider over the app lifespan. + +Startup returns whether tracing was enabled; shutdown always flushes and +tears down without raising. Span-capture behavior itself is covered in +``test_observability/test_tracing.py``; here we assert the lifespan wiring +contract only. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterator +from pathlib import Path + +import pytest +from fastapi import FastAPI + +from everos.config import load_settings +from everos.core.lifespan import TracingLifespanProvider +from everos.core.observability.tracing import shutdown_tracing + + +@pytest.fixture(autouse=True) +def _isolate(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]: + for key in list(os.environ): + if key.startswith("EVEROS_"): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + monkeypatch.chdir(tmp_path) + load_settings.cache_clear() + shutdown_tracing() + yield + shutdown_tracing() + + +async def test_startup_returns_true_when_enabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("EVEROS_OBSERVABILITY__ENABLED", "true") + monkeypatch.setenv("EVEROS_OBSERVABILITY__ENDPOINT", "http://collector.invalid") + load_settings.cache_clear() + provider = TracingLifespanProvider() + app = FastAPI() + result = await provider.startup(app) + assert result is True + await provider.shutdown(app) # must not raise + + +async def test_startup_returns_false_when_disabled() -> None: + provider = TracingLifespanProvider() + app = FastAPI() + result = await provider.startup(app) + assert result is False + await provider.shutdown(app) # must not raise + + +def test_provider_has_low_order_to_start_first() -> None: + # Tracer must be live before other providers start. + assert TracingLifespanProvider().order < 5 # MetricsLifespanProvider is 5 + + +async def test_startup_installs_score_sink_with_creds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """With Langfuse creds + emit_recall_scores, startup installs the score + sink; shutdown drains and tears it down.""" + from pydantic import SecretStr + + import everos.core.observability.tracing.scores as scores_mod + from everos.config import Settings + from everos.config.settings import ObservabilitySettings + + obs = ObservabilitySettings( + enabled=True, + endpoint="http://collector.invalid", + langfuse_public_key="pk-lf", + langfuse_secret_key=SecretStr("sk-lf"), + langfuse_host="https://lf.example", + ) + monkeypatch.setattr( + "everos.core.lifespan.tracing_lifespan.load_settings", + lambda: Settings(observability=obs), + ) + provider = TracingLifespanProvider() + app = FastAPI() + await provider.startup(app) + try: + assert scores_mod._sink is not None + finally: + await provider.shutdown(app) + assert scores_mod._sink is None diff --git a/tests/unit/test_core/test_middleware/test_request_id.py b/tests/unit/test_core/test_middleware/test_request_id.py new file mode 100644 index 0000000..e91253c --- /dev/null +++ b/tests/unit/test_core/test_middleware/test_request_id.py @@ -0,0 +1,113 @@ +"""``RequestIdMiddleware`` — mints a W3C-compatible request id per request, +propagates it to the endpoint via both ``request.state`` and the +``core.context`` contextvar, and echoes it on the ``X-Request-Id`` +response header. + +The contextvar assertion is the load-bearing one: it proves the id set in +the middleware crosses Starlette's ``BaseHTTPMiddleware`` task boundary and +is visible to downstream handlers / loggers. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator + +import pytest +from fastapi import FastAPI, Request +from httpx import ASGITransport, AsyncClient + +from everos.core.context import get_request_id +from everos.core.middleware import RequestIdMiddleware + + +def _build_app() -> FastAPI: + app = FastAPI() + app.add_middleware(RequestIdMiddleware) + + @app.get("/echo") + async def echo(request: Request) -> dict[str, str | None]: + return { + "from_state": getattr(request.state, "request_id", None), + "from_contextvar": get_request_id(), + } + + return app + + +@pytest.fixture +async def client() -> AsyncIterator[AsyncClient]: + app = _build_app() + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as c: + yield c + + +async def test_sets_request_id_on_state_and_response_header( + client: AsyncClient, +) -> None: + resp = await client.get("/echo") + assert resp.status_code == 200 + rid = resp.headers["x-request-id"] + assert len(rid) == 32 # W3C trace-id shape (gen_request_id) + assert resp.json()["from_state"] == rid + + +async def test_request_id_visible_to_endpoint_via_contextvar( + client: AsyncClient, +) -> None: + resp = await client.get("/echo") + body = resp.json() + assert body["from_contextvar"] is not None + assert body["from_contextvar"] == resp.headers["x-request-id"] + + +async def test_each_request_gets_distinct_id(client: AsyncClient) -> None: + r1 = await client.get("/echo") + r2 = await client.get("/echo") + assert r1.headers["x-request-id"] != r2.headers["x-request-id"] + + +async def test_inbound_traceparent_continues_upstream_trace() -> None: + """When the request carries a W3C traceparent header, our first span + continues that upstream trace (distributed tracing). Absent → own root.""" + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + + from everos.config.settings import ObservabilitySettings + from everos.core.observability.tracing import ( + force_flush, + init_tracing, + memory_span, + shutdown_tracing, + ) + + exporter = InMemorySpanExporter() + shutdown_tracing() + init_tracing( + ObservabilitySettings(enabled=True, endpoint="http://collector.invalid"), + span_processor=SimpleSpanProcessor(exporter), + ) + app = FastAPI() + app.add_middleware(RequestIdMiddleware) + + @app.get("/s") + async def s() -> dict[str, str]: + with memory_span("everos.memory.search", observation_type="retriever"): + pass + return {"ok": "1"} + + upstream = "1234567890abcdef1234567890abcdef" + tp = f"00-{upstream}-1111111111111111-01" + try: + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as c: + await c.get("/s", headers={"traceparent": tp}) + force_flush() + span = exporter.get_finished_spans()[0] + assert format(span.context.trace_id, "032x") == upstream + finally: + shutdown_tracing() diff --git a/tests/unit/test_core/test_observability/test_scores.py b/tests/unit/test_core/test_observability/test_scores.py new file mode 100644 index 0000000..6cd80f7 --- /dev/null +++ b/tests/unit/test_core/test_observability/test_scores.py @@ -0,0 +1,89 @@ +"""``RecallScoreSink`` — non-blocking recall-score push. + +The sink is the piece that guarantees the Langfuse scores REST call never +touches the search request path: ``enqueue`` is O(1) and never blocks / raises +(drops + counts when full); a background worker drains and sends. The network +``sender`` is injected so these tests assert the queue/worker contract offline. +""" + +from __future__ import annotations + +import asyncio + +from everos.core.observability.tracing.scores import RecallScoreSink, ScoreRecord + + +async def test_worker_sends_payload_in_langfuse_shape() -> None: + sent: list[dict] = [] + done = asyncio.Event() + + async def sender(payload: dict) -> None: + sent.append(payload) + done.set() + + sink = RecallScoreSink(sender=sender, max_queue=10) + sink.start() + sink.enqueue( + ScoreRecord( + trace_id="tid", + observation_id="oid", + name="recall_top_score", + value=0.8, + comment="method=hybrid", + ) + ) + await asyncio.wait_for(done.wait(), timeout=1.0) + await sink.stop() + + assert sent[0] == { + "traceId": "tid", + "observationId": "oid", + "name": "recall_top_score", + "value": 0.8, + "dataType": "NUMERIC", + "comment": "method=hybrid", + } + + +async def test_enqueue_never_blocks_or_raises_when_full() -> None: + async def slow(_: dict) -> None: + await asyncio.sleep(10) + + sink = RecallScoreSink(sender=slow, max_queue=1) + # No worker started → queue fills. enqueue must stay non-blocking. + sink.enqueue(ScoreRecord("t", "o", "n", 1.0, None)) # fills the single slot + sink.enqueue(ScoreRecord("t", "o", "n", 1.0, None)) # dropped, no raise + assert sink.dropped == 1 + + +async def test_sender_failure_does_not_crash_worker() -> None: + calls: list[dict] = [] + second = asyncio.Event() + + async def flaky(payload: dict) -> None: + calls.append(payload) + if len(calls) == 1: + raise RuntimeError("boom") + second.set() + + sink = RecallScoreSink(sender=flaky, max_queue=10) + sink.start() + sink.enqueue(ScoreRecord("t", "o", "n1", 1.0, None)) # sender raises + sink.enqueue(ScoreRecord("t", "o", "n2", 2.0, None)) # worker must survive + await asyncio.wait_for(second.wait(), timeout=1.0) + await sink.stop() + assert len(calls) == 2 # first failed but worker kept going + + +async def test_stop_drains_pending() -> None: + sent: list[dict] = [] + + async def sender(payload: dict) -> None: + sent.append(payload) + + sink = RecallScoreSink(sender=sender, max_queue=10) + sink.start() + for i in range(3): + sink.enqueue(ScoreRecord("t", "o", f"n{i}", float(i), None)) + await sink.stop() # should drain the queue before returning + assert len(sent) == 3 diff --git a/tests/unit/test_core/test_observability/test_span_attributes.py b/tests/unit/test_core/test_observability/test_span_attributes.py new file mode 100644 index 0000000..ee3a89f --- /dev/null +++ b/tests/unit/test_core/test_observability/test_span_attributes.py @@ -0,0 +1,138 @@ +"""``memory_span`` / ``set_generation_usage`` — the langfuse.* + gen_ai.* +attribute contract applied to spans. + +Captured via an in-memory exporter so the emitted attribute keys/values +are asserted directly against §4 of the implementation plan. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +import pytest +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +from everos.config.settings import ObservabilitySettings +from everos.core.observability.tracing import ( + capture_input, + capture_output, + force_flush, + init_tracing, + memory_span, + set_capture_content, + set_generation_usage, + set_redactor, + shutdown_tracing, +) + + +@pytest.fixture(autouse=True) +def _reset() -> Iterator[None]: + shutdown_tracing() + yield + shutdown_tracing() + + +@pytest.fixture +def captured() -> Iterator[InMemorySpanExporter]: + exporter = InMemorySpanExporter() + init_tracing( + ObservabilitySettings(enabled=True, endpoint="http://collector.invalid"), + span_processor=SimpleSpanProcessor(exporter), + ) + yield exporter + + +def test_memory_span_sets_langfuse_attributes( + captured: InMemorySpanExporter, +) -> None: + with memory_span( + "everos.memory.search", + observation_type="retriever", + session_id="s1", + user_id="u1", + metadata={"app_id": "a", "project_id": "p", "agent_id": None}, + ): + pass + force_flush() + span = captured.get_finished_spans()[0] + attrs = span.attributes + assert span.name == "everos.memory.search" + assert attrs["langfuse.observation.type"] == "retriever" + assert attrs["langfuse.session.id"] == "s1" + assert attrs["langfuse.user.id"] == "u1" + assert attrs["langfuse.trace.metadata.app_id"] == "a" + assert attrs["langfuse.trace.metadata.project_id"] == "p" + # None-valued metadata is dropped, not emitted as "None". + assert "langfuse.trace.metadata.agent_id" not in attrs + assert list(attrs["langfuse.trace.tags"]) == ["everos", "memory"] + + +def test_set_generation_usage_annotates_current_span( + captured: InMemorySpanExporter, +) -> None: + with memory_span("everos.extract", observation_type="generation"): + set_generation_usage(model="gpt-x", input_tokens=11, output_tokens=22) + force_flush() + attrs = captured.get_finished_spans()[0].attributes + assert attrs["gen_ai.request.model"] == "gpt-x" + assert attrs["gen_ai.usage.input_tokens"] == 11 + assert attrs["gen_ai.usage.output_tokens"] == 22 + + +def test_set_generation_usage_outside_span_is_noop() -> None: + # No active span → must not raise (and nothing to record). + set_generation_usage(model="x", input_tokens=1, output_tokens=2) + + +def test_content_dropped_when_capture_off(captured: InMemorySpanExporter) -> None: + # Default: capture_content off → no observation.input/output emitted. + with memory_span("everos.extract", observation_type="generation") as span: + capture_input(span, {"query": "sensitive query"}) + capture_output(span, "secret memory text") + force_flush() + attrs = captured.get_finished_spans()[0].attributes + assert "langfuse.observation.input" not in attrs + assert "langfuse.observation.output" not in attrs + + +def test_content_emitted_when_capture_on(captured: InMemorySpanExporter) -> None: + set_capture_content(True) + try: + with memory_span("everos.memory.search", observation_type="retriever") as span: + capture_input(span, {"query": "hello"}) + capture_output(span, "world") + finally: + set_capture_content(False) + force_flush() + attrs = captured.get_finished_spans()[0].attributes + import json + + assert json.loads(attrs["langfuse.observation.input"]) == {"query": "hello"} + assert attrs["langfuse.observation.output"] == "world" + + +def test_content_redacted_and_truncated(captured: InMemorySpanExporter) -> None: + calls: list[str] = [] + + def redact(text: str) -> str: + calls.append(text) + return text.replace("SECRET", "***") + + set_redactor(redact) + set_capture_content(True) + try: + with memory_span("everos.extract", observation_type="generation") as span: + capture_output(span, "SECRET " + "a" * 6000) + finally: + set_capture_content(False) + set_redactor(None) + force_flush() + val = captured.get_finished_spans()[0].attributes["langfuse.observation.output"] + assert "SECRET" not in val + assert "***" in val + assert len(val) <= 4096 # truncated + assert calls # redaction hook was invoked diff --git a/tests/unit/test_core/test_observability/test_tracing.py b/tests/unit/test_core/test_observability/test_tracing.py new file mode 100644 index 0000000..5e3518a --- /dev/null +++ b/tests/unit/test_core/test_observability/test_tracing.py @@ -0,0 +1,170 @@ +"""``core.observability.tracing`` — provider lifecycle + tracer facade. + +Spans are captured via an in-memory exporter (a ``SimpleSpanProcessor`` +injected into ``init_tracing``) so assertions run offline, with no OTLP +endpoint. The provider is kept off the OTel *global* on purpose — the +module holds its own reference — so tests can init / shutdown repeatedly +without hitting OTel's set-global-once guard. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +import pytest +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +from everos.config.settings import ObservabilitySettings +from everos.core.observability.tracing import ( + force_flush, + get_tracer, + init_tracing, + shutdown_tracing, +) + + +@pytest.fixture(autouse=True) +def _reset_tracing() -> Iterator[None]: + """Ensure each test starts and ends with no provider installed.""" + shutdown_tracing() + yield + shutdown_tracing() + + +@pytest.fixture +def captured_spans() -> Iterator[InMemorySpanExporter]: + exporter = InMemorySpanExporter() + settings = ObservabilitySettings(enabled=True, endpoint="http://collector.invalid") + init_tracing(settings, span_processor=SimpleSpanProcessor(exporter)) + yield exporter + + +def test_disabled_tracer_is_noop_and_never_raises() -> None: + # No init_tracing → no provider → get_tracer returns a no-op tracer. + tracer = get_tracer("everos.test") + with tracer.start_as_current_span("everos.noop") as span: + span.set_attribute("k", "v") # must not raise + + +def test_init_returns_false_when_disabled() -> None: + assert init_tracing(ObservabilitySettings(enabled=False)) is False + + +def test_init_returns_true_when_enabled(captured_spans: InMemorySpanExporter) -> None: + # captured_spans fixture already called init_tracing(enabled=True). + tracer = get_tracer("x") + with tracer.start_as_current_span("s"): + pass + force_flush() + assert len(captured_spans.get_finished_spans()) == 1 + + +def test_span_captured_when_enabled(captured_spans: InMemorySpanExporter) -> None: + tracer = get_tracer("everos.test") + with tracer.start_as_current_span("everos.memory.search"): + pass + force_flush() + names = [s.name for s in captured_spans.get_finished_spans()] + assert "everos.memory.search" in names + + +def test_child_span_nests_under_parent( + captured_spans: InMemorySpanExporter, +) -> None: + tracer = get_tracer("everos.test") + with ( + tracer.start_as_current_span("parent"), + tracer.start_as_current_span("child"), + ): + pass + force_flush() + spans = {s.name: s for s in captured_spans.get_finished_spans()} + assert spans["child"].parent is not None + assert spans["child"].parent.span_id == spans["parent"].context.span_id + + +def test_resource_carries_service_name( + captured_spans: InMemorySpanExporter, +) -> None: + tracer = get_tracer("x") + with tracer.start_as_current_span("s"): + pass + force_flush() + span = captured_spans.get_finished_spans()[0] + assert span.resource.attributes["service.name"] == "everos" + + +async def test_child_spans_nest_across_asyncio_gather( + captured_spans: InMemorySpanExporter, +) -> None: + """OTel context must survive the asyncio.gather task boundary: child + spans started inside gathered coroutines nest under the parent span, + not as siblings/roots. This is the mechanism SearchManager.search + relies on (plan cross-cutting note #1).""" + import asyncio + + tracer = get_tracer("everos.test") + + async def _child(name: str) -> None: + with tracer.start_as_current_span(name): + await asyncio.sleep(0) + + with tracer.start_as_current_span("parent"): + await asyncio.gather(_child("child_a"), _child("child_b")) + force_flush() + + spans = {s.name: s for s in captured_spans.get_finished_spans()} + parent_span_id = spans["parent"].context.span_id + trace_id = spans["parent"].context.trace_id + for child in ("child_a", "child_b"): + assert spans[child].parent is not None + assert spans[child].parent.span_id == parent_span_id + assert spans[child].context.trace_id == trace_id + + +def test_resolve_otlp_target_derives_from_langfuse_creds() -> None: + import base64 + + from pydantic import SecretStr + + from everos.core.observability.tracing.provider import _resolve_otlp_target + + settings = ObservabilitySettings( + enabled=True, + langfuse_public_key="pk", + langfuse_secret_key=SecretStr("sk"), + langfuse_host="https://us.cloud.langfuse.com", + ) + endpoint, headers = _resolve_otlp_target(settings) + assert endpoint == "https://us.cloud.langfuse.com/api/public/otel/v1/traces" + assert headers["Authorization"] == "Basic " + base64.b64encode(b"pk:sk").decode() + + +def test_resolve_otlp_target_explicit_values_win() -> None: + from pydantic import SecretStr + + from everos.core.observability.tracing.provider import _resolve_otlp_target + + settings = ObservabilitySettings( + enabled=True, + endpoint="http://explicit/v1/traces", + headers={"Authorization": "Basic explicit"}, + langfuse_public_key="pk", + langfuse_secret_key=SecretStr("sk"), + langfuse_host="https://us.cloud.langfuse.com", + ) + endpoint, headers = _resolve_otlp_target(settings) + assert endpoint == "http://explicit/v1/traces" + assert headers["Authorization"] == "Basic explicit" + + +def test_resolve_otlp_target_no_langfuse_returns_as_is() -> None: + from everos.core.observability.tracing.provider import _resolve_otlp_target + + settings = ObservabilitySettings(enabled=True, endpoint="http://x/v1/traces") + endpoint, headers = _resolve_otlp_target(settings) + assert endpoint == "http://x/v1/traces" + assert "Authorization" not in headers diff --git a/uv.lock b/uv.lock index 5620515..94a8541 100644 --- a/uv.lock +++ b/uv.lock @@ -595,11 +595,17 @@ dependencies = [ multimodal = [ { name = "everalgo-parser", extra = ["svg"] }, ] +otel = [ + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, +] [package.dev-dependencies] dev = [ { name = "import-linter" }, { name = "ipdb" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "pre-commit" }, { name = "pyinstrument" }, { name = "pytest" }, @@ -625,6 +631,8 @@ requires-dist = [ { name = "jieba", specifier = ">=0.42.1,<1.0" }, { name = "lancedb", specifier = ">=0.13.0" }, { name = "openai", specifier = ">=1.0.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.27.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.27.0" }, { name = "portalocker", specifier = ">=2.8.2" }, { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, @@ -639,12 +647,14 @@ requires-dist = [ { name = "watchdog", specifier = ">=4.0.0" }, { name = "watchfiles", specifier = ">=0.21.0" }, ] -provides-extras = ["multimodal"] +provides-extras = ["multimodal", "otel"] [package.metadata.requires-dev] dev = [ { name = "import-linter", specifier = ">=2.0" }, { name = "ipdb", specifier = ">=0.13.13" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.27.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.27.0" }, { name = "pre-commit", specifier = ">=4.0.0" }, { name = "pyinstrument", specifier = ">=5.0.0" }, { name = "pytest", specifier = ">=8.4.0" }, @@ -688,6 +698,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, +] + [[package]] name = "greenlet" version = "3.5.3" @@ -1339,6 +1361,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/1c/5d43735b2553baae2a5e899dcbcd0670a86930d993184d72ca909bf11c9b/openai-2.36.0-py3-none-any.whl", hash = "sha256:143f6194b548dbc2c921af1f1b03b9f14c85fed8a75b5b516f5bcc11a2a50c63", size = 1302361, upload-time = "2026-05-07T17:33:15.063Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -1505,6 +1608,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, ] +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + [[package]] name = "psutil" version = "7.2.2"