diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 3edb96ed8d9..17011bb8db7 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -216,7 +216,13 @@ lives in [`plumbing/`](./plumbing): `TracerProvider` so one logger serves many tenants. The cache is a bounded LRU that flushes + shuts down evicted providers, since the key derives from request-supplied credentials and must not grow (or leak threads) without limit. -- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments. +- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments. The + six `gen_ai.client.*` histograms are recorded through the meter resolved by + `providers.resolve_meter_provider`: an injected provider wins (tests/DI), + otherwise the operator's globally configured `MeterProvider` is reused so its + readers/exporters receive them alongside the server metrics, and one is built + and registered as the global only when none is set (mirroring how V2 owns trace + export). ### Adapter diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index a8378b6a043..1869e9ca388 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -42,9 +42,10 @@ create_genai_metrics, ) from litellm.integrations.otel.plumbing.providers import ( - build_meter_provider, build_tracer_provider, + get_meter, get_tracer, + resolve_meter_provider, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service @@ -127,17 +128,15 @@ def __init__( def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | None": """Create the six GenAI histograms when metrics are enabled, else ``None``. - ``meter_provider`` is an explicit override (tests inject one); otherwise a - provider is built from the config's exporter selection. + ``meter_provider`` is an explicit override (tests inject one); otherwise the + provider is resolved from the OTel global so the operator's configured + readers/exporters receive the metrics, building and registering one only + when no global provider is set. """ if not self.config.enable_metrics: return None - provider = ( - meter_provider - if meter_provider is not None - else build_meter_provider(self.config) - ) - meter = provider.get_meter(LITELLM_TRACER_NAME) + provider = resolve_meter_provider(self.config, meter_provider) + meter = get_meter(provider, LITELLM_TRACER_NAME) return GenAIMetricRecorder(create_genai_metrics(meter), self.callback_name) # ====================================================================== # diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index a4362f05e86..6d0710397a3 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -2,8 +2,10 @@ from typing import TYPE_CHECKING, Any, Callable, Iterable -from opentelemetry import baggage +from opentelemetry import baggage, metrics from opentelemetry.context import Context +from opentelemetry.metrics import MeterProvider, NoOpMeterProvider +from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider from opentelemetry.sdk.trace.export import ( @@ -26,7 +28,7 @@ from litellm.integrations.otel.model.utils import parse_headers as parse_headers if TYPE_CHECKING: - from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.metrics import Meter from opentelemetry.sdk.metrics.export import MetricReader _SPAN_KIND_BY_ROLE_KIND: dict[LiteLLMSpanKind, SpanKind] = { @@ -233,17 +235,46 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": def build_meter_provider( config: OpenTelemetryV2Config, metric_reader: "MetricReader | None" = None, -) -> "MeterProvider": +) -> SDKMeterProvider: """Build the :class:`MeterProvider` for GenAI metrics. ``metric_reader`` is an explicit override (tests inject an ``InMemoryMetricReader``); otherwise the reader is selected from the config's exporter kind via :func:`build_metric_reader`. """ - from opentelemetry.sdk.metrics import MeterProvider - reader = metric_reader if metric_reader is not None else build_metric_reader(config) - return MeterProvider(metric_readers=[reader], resource=build_resource(config)) + return SDKMeterProvider(metric_readers=[reader], resource=build_resource(config)) + + +def resolve_meter_provider( + config: OpenTelemetryV2Config, + meter_provider: MeterProvider | None = None, +) -> MeterProvider: + """Resolve the :class:`MeterProvider` GenAI metrics record through. + + An injected provider wins (DI/tests). Otherwise reuse whatever the operator has + configured as the global, whether a real SDK provider or an explicit + ``NoOpMeterProvider``, so the GenAI histograms ride the operator's + readers/exporters and an explicit opt-out is honored. Only when the global is + still the default proxy placeholder does V2 build one from the config and + publish it as the global, mirroring how V2 owns trace export. The built + provider is the one returned, so its reader thread is always live, never + orphaned. + """ + if meter_provider is not None: + return meter_provider + + existing = metrics.get_meter_provider() + if isinstance(existing, (SDKMeterProvider, NoOpMeterProvider)): + return existing + + provider = build_meter_provider(config) + metrics.set_meter_provider(provider) + return provider + + +def get_meter(provider: MeterProvider, name: str = "litellm") -> "Meter": + return provider.get_meter(name, litellm_version) def build_resource(config: OpenTelemetryV2Config) -> Resource: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py index caf2947ac2d..29067f91b5a 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py @@ -33,6 +33,9 @@ GenAIMetricRecorder, create_genai_metrics, ) +from litellm.integrations.otel.plumbing.providers import ( # noqa: E402 + resolve_meter_provider, +) OPERATION_DURATION = "gen_ai.client.operation.duration" TOKEN_USAGE = "gen_ai.client.token.usage" @@ -250,6 +253,50 @@ def test_no_filter_keeps_high_cardinality_keys(): assert expected.issubset(set(dp.attributes.keys())) +def test_metrics_reach_operator_configured_global_provider(monkeypatch): + """Regression: with no meter provider injected, the six gen_ai.client.* + histograms must record through the operator's globally configured + MeterProvider so its readers/exporters receive them. Before the fix the logger + built an isolated provider and the operator's reader saw nothing.""" + from opentelemetry import metrics + + reader = InMemoryMetricReader() + operator_provider = MeterProvider(metric_readers=[reader]) + monkeypatch.setattr(metrics, "get_meter_provider", lambda: operator_provider) + + logger = OpenTelemetryV2( + config=OpenTelemetryV2Config(exporter="in_memory", enable_metrics=True), + ) + kwargs, response_obj, start, end = _build_call() + asyncio.run(logger.async_log_success_event(kwargs, response_obj, start, end)) + + assert set(_metrics_by_name(reader).keys()) == set(ALL_METRICS) + operator_provider.shutdown() + + +def test_resolve_meter_provider_prefers_injected(): + """An injected provider is used verbatim, never replaced by the global.""" + injected = MeterProvider(metric_readers=[InMemoryMetricReader()]) + resolved = resolve_meter_provider( + OpenTelemetryV2Config(exporter="in_memory"), injected + ) + assert resolved is injected + injected.shutdown() + + +def test_resolve_meter_provider_honors_operator_noop(monkeypatch): + """An operator that disabled metrics with a NoOpMeterProvider is not silently + overridden by a freshly built provider.""" + from opentelemetry import metrics + from opentelemetry.metrics import NoOpMeterProvider + + noop = NoOpMeterProvider() + monkeypatch.setattr(metrics, "get_meter_provider", lambda: noop) + + resolved = resolve_meter_provider(OpenTelemetryV2Config(exporter="in_memory")) + assert resolved is noop + + def _recorder(monkeypatch, attributes): """A recorder wired to a fresh in-memory meter, with callback_settings carrying `attributes`. record() resolves the filter lazily from there, so a misconfig