chore(release): backport #32542, #32655 to stable/1.91.x and cut 1.91.3

litellm/integrations/otel/README.md

@@ -223,6 +223,15 @@ lives in [`plumbing/`](./plumbing):
   readers/exporters receive them alongside the server metrics, and one is built
   and registered as the global only when none is set (mirroring how V2 owns trace
   export).
+- [`events.py`](./plumbing/events.py) — GenAI client events. Gated on
+  `enable_events` (`LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS`), a failed LLM call
+  records the semconv `gen_ai.client.operation.exception` log event at severity
+  WARN, carrying `exception.type` / `exception.message` / `exception.stacktrace`
+  and correlated to the failed span through the trace and span ids. The
+  `LoggerProvider` is resolved like the meter provider, except that an explicit
+  `NoOpLoggerProvider` global is an operator opt-out that builds no recorder at
+  all. The deprecated `error.*` span attributes and the `exception` span event
+  are still stamped by the emitter for backwards compatibility.
 
 ### Adapter
 

litellm/integrations/otel/emitter.py

@@ -17,6 +17,7 @@
     ServiceSpanData,
     SpanError,
 )
+from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
 from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
 from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError
 from litellm.integrations.otel.model.spans import (
@@ -74,9 +75,11 @@ def __init__(
         tracer: Tracer,
         config: OpenTelemetryV2Config,
         mappers: Sequence[AttributeMapper] | None = None,
+        event_recorder: GenAIEventRecorder | None = None,
     ) -> None:
         self._tracer = tracer
         self._config = config
+        self._event_recorder = event_recorder
         # The mapper chain is the sole source of span attributes. When not
         # passed in, resolve it from the config so there's one source of truth.
         self._mappers: list[AttributeMapper] = (
@@ -208,6 +211,14 @@ def finish_span(
                 ExceptionEvent.NAME,
                 {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
             )
+            if self._event_recorder is not None and role is SpanRole.LLM_CALL:
+                self._event_recorder.record_operation_exception(
+                    span_context=span.get_span_context(),
+                    error_type=error_type,
+                    message=message,
+                    stack_trace=error.stack_trace,
+                    timestamp_ns=end_time_ns,
+                )
         # On success leave the status UNSET (the semconv default) rather than
         # forcing OK — that matches the FastAPI server span and avoids implying a
         # span-level health signal litellm doesn't actually evaluate. Only a

litellm/integrations/otel/logger.py

@@ -6,6 +6,7 @@
 from typing import TYPE_CHECKING, Any, Callable, Iterator, Mapping, Sequence, cast
 
 from opentelemetry.context import attach, get_current
+from opentelemetry.sdk._logs import LoggerProvider
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.trace import Span, Tracer, get_current_span, use_span
 
@@ -37,14 +38,17 @@
     SpanError,
     is_mcp_tool_call,
 )
+from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
 from litellm.integrations.otel.plumbing.metrics import (
     GenAIMetricRecorder,
     create_genai_metrics,
 )
 from litellm.integrations.otel.plumbing.providers import (
     build_tracer_provider,
+    get_event_logger,
     get_meter,
     get_tracer,
+    resolve_logger_provider,
     resolve_meter_provider,
 )
 from litellm.integrations.otel.plumbing.routing import TenantTracerCache
@@ -101,7 +105,7 @@ def __init__(
         config: OpenTelemetryV2Config | None = None,
         callback_name: str | None = None,
         tracer_provider: TracerProvider | None = None,
-        logger_provider: Any | None = None,  # reserved for OTel logs
+        logger_provider: LoggerProvider | None = None,
         meter_provider: Any | None = None,
         **kwargs: Any,
     ) -> None:
@@ -114,7 +118,12 @@ def __init__(
         self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
         self._metrics_recorder = self._init_metrics(meter_provider)
         self._metric_filter_error_logged = False
-        self._emitter = SpanEmitter(self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names))
+        self._emitter = SpanEmitter(
+            self.tracer,
+            self.config,
+            mappers=resolve_mappers(self.config.mapper_names),
+            event_recorder=self._init_events(logger_provider),
+        )
         self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
         self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict()
         self._init_otel_logger_on_litellm_proxy()
@@ -133,6 +142,22 @@ def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | No
         meter = get_meter(provider, LITELLM_TRACER_NAME)
         return GenAIMetricRecorder(create_genai_metrics(meter), self.callback_name)
 
+    def _init_events(self, logger_provider: LoggerProvider | None) -> "GenAIEventRecorder | None":
+        """Create the GenAI event recorder when events are enabled, else ``None``.
+
+        ``logger_provider`` is an explicit override (tests inject one); otherwise the
+        provider is resolved from the OTel global so an operator-configured logs
+        pipeline receives the events, building and registering one only when no
+        global provider is set. A ``None`` resolution means the operator opted out
+        of the logs signal, so no recorder is built.
+        """
+        if not self.config.enable_events:
+            return None
+        provider = resolve_logger_provider(self.config, logger_provider)
+        if provider is None:
+            return None
+        return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME))
+
     # ====================================================================== #
     #  Proxy global registration
     # ====================================================================== #

litellm/integrations/otel/model/semconv.py

@@ -179,6 +179,19 @@ class ExceptionEvent:
     NAME: Final = "exception"
     TYPE: Final = "exception.type"
     MESSAGE: Final = "exception.message"
+    STACKTRACE: Final = "exception.stacktrace"
+
+
+class GenAIEvent:
+    """GenAI semconv event names, from the GenAI registry's *events* section.
+
+    ``gen_ai.client.operation.exception`` is defined as a log-based event
+    (severity WARN) carrying the ``exception.*`` trio, correlated to the failed
+    span via the trace/span ids — the semconv-compliant home for GenAI failure
+    details, unlike the deprecated ``error.message`` span attribute.
+    """
+
+    OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception"
 
 
 class Server:

litellm/integrations/otel/plumbing/events.py

@@ -0,0 +1,52 @@
+"""GenAI client events: the ``gen_ai.client.operation.exception`` log event.
+
+The GenAI semantic conventions define exception recording for client
+operations as a log-based event (severity WARN) carrying the ``exception.*``
+attribute trio, correlated to the failed span through the trace/span ids —
+not as a span attribute or span event. This module owns building and
+emitting that event; the exporter pipeline it rides is built in
+:mod:`litellm.integrations.otel.plumbing.providers`.
+"""
+
+from dataclasses import dataclass
+
+from opentelemetry._events import Event, EventLogger
+from opentelemetry._logs.severity import SeverityNumber
+from opentelemetry.trace import SpanContext
+
+from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent
+
+
+@dataclass(frozen=True, slots=True)
+class GenAIEventRecorder:
+    event_logger: EventLogger
+
+    def record_operation_exception(
+        self,
+        span_context: SpanContext,
+        error_type: str,
+        message: str,
+        stack_trace: str | None,
+        timestamp_ns: int | None,
+    ) -> None:
+        # ``exception.type`` and ``exception.message`` are the semconv-required
+        # pair and always ride the event; only the recommended stacktrace is
+        # conditional on the payload carrying one.
+        stacktrace = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else ()
+        self.event_logger.emit(
+            Event(
+                name=GenAIEvent.OPERATION_EXCEPTION,
+                timestamp=timestamp_ns,
+                trace_id=span_context.trace_id,
+                span_id=span_context.span_id,
+                trace_flags=span_context.trace_flags,
+                severity_number=SeverityNumber.WARN,
+                attributes=dict(
+                    (
+                        (ExceptionEvent.TYPE, error_type),
+                        (ExceptionEvent.MESSAGE, message),
+                        *stacktrace,
+                    )
+                ),
+            )
+        )

litellm/integrations/otel/plumbing/providers.py

@@ -2,9 +2,20 @@
 
 from typing import TYPE_CHECKING, Any, Callable, Iterable
 
-from opentelemetry import baggage, metrics
+from opentelemetry import _logs, baggage, metrics
+from opentelemetry._events import EventLogger
+from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider
 from opentelemetry.context import Context
 from opentelemetry.metrics import MeterProvider, NoOpMeterProvider
+from opentelemetry.sdk._events import EventLoggerProvider
+from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider
+from opentelemetry.sdk._logs.export import (
+    BatchLogRecordProcessor,
+    ConsoleLogExporter,
+    InMemoryLogExporter,
+    LogExporter,
+    SimpleLogRecordProcessor,
+)
 from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
@@ -224,6 +235,112 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
     return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
 
 
+def _otlp_logs_endpoint(endpoint: str | None) -> str | None:
+    """Point an OTLP/HTTP base endpoint at the ``/v1/logs`` signal path.
+
+    The OTLP/HTTP exporter only appends ``/v1/logs`` when it reads
+    ``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used
+    verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint``
+    for the logs signal (rewriting a sibling signal path when present).
+    """
+    if not endpoint:
+        return endpoint
+    endpoint = endpoint.rstrip("/")
+    if endpoint.endswith("/v1/logs"):
+        return endpoint
+    for other_signal in ("/v1/traces", "/v1/metrics"):
+        if endpoint.endswith(other_signal):
+            return endpoint[: -len(other_signal)] + "/v1/logs"
+    return endpoint + "/v1/logs"
+
+
+def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter:
+    """Build a log exporter mirroring the exporter selection of the other signals.
+
+    ``console`` (and any unrecognized kind) exports to the console; ``otlp_http``
+    and ``otlp_grpc`` export over OTLP with the configured endpoint/headers;
+    ``in_memory`` buffers for tests. Like GenAI metrics, events ride the
+    single-destination shorthand fields, not the multi-exporter ``exporters`` list.
+    """
+    kind = (config.exporter or "console").lower()
+    if kind in ("in_memory", "inmemory", "memory"):
+        return InMemoryLogExporter()
+    if kind in ("otlp_http", "http", "http/protobuf", "http/json"):
+        from opentelemetry.exporter.otlp.proto.http._log_exporter import (
+            OTLPLogExporter as HTTPLogExporter,
+        )
+
+        return HTTPLogExporter(
+            endpoint=_otlp_logs_endpoint(config.endpoint),
+            headers=parse_headers(config.headers),
+        )
+    if kind in ("otlp_grpc", "grpc"):
+        try:
+            from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
+                OTLPLogExporter as GRPCLogExporter,
+            )
+        except ImportError as exc:
+            raise ImportError(
+                "OpenTelemetry OTLP gRPC log exporter is not available. Install "
+                "`opentelemetry-exporter-otlp` and `grpcio` (or `litellm[grpc]`)."
+            ) from exc
+
+        return GRPCLogExporter(endpoint=config.endpoint, headers=parse_headers(config.headers))
+    return ConsoleLogExporter()
+
+
+def build_logger_provider(
+    config: OpenTelemetryV2Config,
+    log_exporter: LogExporter | None = None,
+) -> SDKLoggerProvider:
+    """Build the :class:`LoggerProvider` GenAI events export through.
+
+    ``log_exporter`` is an explicit override (tests inject an
+    ``InMemoryLogExporter``); otherwise the exporter is selected from the config's
+    exporter kind via :func:`build_log_exporter`. Console and in-memory exporters
+    get a Simple processor (synchronous export, which tests rely on), everything
+    else a Batch processor — the same split as span processing.
+    """
+    exporter = log_exporter if log_exporter is not None else build_log_exporter(config)
+    provider = SDKLoggerProvider(resource=build_resource(config))
+    use_simple = isinstance(exporter, (ConsoleLogExporter, InMemoryLogExporter))
+    provider.add_log_record_processor(
+        SimpleLogRecordProcessor(exporter) if use_simple else BatchLogRecordProcessor(exporter)
+    )
+    return provider
+
+
+def resolve_logger_provider(
+    config: OpenTelemetryV2Config,
+    logger_provider: SDKLoggerProvider | None = None,
+) -> SDKLoggerProvider | None:
+    """Resolve the :class:`LoggerProvider` GenAI events record through, or ``None``
+    when the operator has opted out of the logs signal.
+
+    Same resolution order as :func:`resolve_meter_provider`: an injected provider
+    wins (DI/tests); an operator-configured SDK global is reused so events ride
+    their pipeline; an explicit ``NoOpLoggerProvider`` global is an opt-out and
+    yields ``None``, so no event is ever built. Only the default placeholder
+    global makes V2 build a provider from the config and publish it as the global.
+    """
+    if logger_provider is not None:
+        return logger_provider
+
+    existing: LoggerProvider = _logs.get_logger_provider()
+    if isinstance(existing, SDKLoggerProvider):
+        return existing
+    if isinstance(existing, NoOpLoggerProvider):
+        return None
+
+    provider = build_logger_provider(config)
+    _logs.set_logger_provider(provider)
+    return provider
+
+
+def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger:
+    return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version)
+
+
 def build_meter_provider(
     config: OpenTelemetryV2Config,
     metric_reader: "MetricReader | None" = None,