fix(otel): configurable baggage, pass-through + service spans, typed guardrails

litellm/_service_logger.py

@@ -24,6 +24,22 @@
     UserAPIKeyAuth = Any
 
 
+def _get_otel_v2_class() -> Optional[type]:
+    """Return the ``OpenTelemetryV2`` class, or ``None`` if the OTel SDK is absent.
+
+    Imported lazily: ``litellm.integrations.otel.logger`` imports the OpenTelemetry
+    SDK at module scope, so importing it eagerly would break installs without the
+    SDK. The V2 logger only exists when ``LITELLM_OTEL_V2`` is enabled (which
+    requires the SDK), so a failed import simply means "no V2 logger in play".
+    """
+    try:
+        from litellm.integrations.otel.logger import OpenTelemetryV2
+
+        return OpenTelemetryV2
+    except Exception:
+        return None
+
+
 class ServiceLogging(CustomLogger):
     """
     Separate class used for monitoring health of litellm-adjacent services (redis/postgres).
@@ -38,6 +54,37 @@ def __init__(self, mock_testing: bool = False) -> None:
         if "prometheus_system" in litellm.service_callback:
             self.prometheusServicesLogger = PrometheusServicesLogger()
 
+    def _resolve_otel_service_logger(self, callback: Any) -> Optional[Any]:
+        """Resolve the OTel logger (legacy or V2) to emit a service span on.
+
+        Returns the logger instance whose ``async_service_*_hook`` should fire for
+        this ``callback``, or ``None`` when ``callback`` is not an OTel callback.
+
+        The V2 ``OpenTelemetryV2`` logger is a plain ``CustomLogger`` and is NOT a
+        subclass of the legacy ``OpenTelemetry``, so the legacy ``isinstance``
+        check alone misses it — which is why redis/postgres service spans never
+        showed up under ``LITELLM_OTEL_V2``. Match both the legacy and V2 types,
+        whether the callback is the logger instance itself or the ``"otel"`` string
+        (which routes to the proxy's registered ``open_telemetry_logger``).
+        """
+        otel_v2_cls = _get_otel_v2_class()
+
+        def _is_otel_logger(obj: Any) -> bool:
+            if isinstance(obj, OpenTelemetry):
+                return True
+            return otel_v2_cls is not None and isinstance(obj, otel_v2_cls)
+
+        if _is_otel_logger(callback):
+            return callback
+        if callback == "otel":
+            from litellm.proxy.proxy_server import open_telemetry_logger
+
+            if open_telemetry_logger is not None and _is_otel_logger(
+                open_telemetry_logger
+            ):
+                return open_telemetry_logger
+        return None
+
     def service_success_hook(
         self,
         service: ServiceTypes,
@@ -144,18 +191,8 @@ async def async_service_success_hook(
                     end_time=end_time,
                     event_metadata=event_metadata,
                 )
-            elif callback == "otel" or isinstance(callback, OpenTelemetry):
-                _otel_logger_to_use: Optional[OpenTelemetry] = None
-                if isinstance(callback, OpenTelemetry):
-                    _otel_logger_to_use = callback
-                else:
-                    from litellm.proxy.proxy_server import open_telemetry_logger
-
-                    if open_telemetry_logger is not None and isinstance(
-                        open_telemetry_logger, OpenTelemetry
-                    ):
-                        _otel_logger_to_use = open_telemetry_logger
-
+            else:
+                _otel_logger_to_use = self._resolve_otel_service_logger(callback)
                 if _otel_logger_to_use is not None and parent_otel_span is not None:
                     await _otel_logger_to_use.async_service_success_hook(
                         payload=payload,
@@ -255,17 +292,8 @@ async def async_service_failure_hook(
                     end_time=end_time,
                     event_metadata=event_metadata,
                 )
-            elif callback == "otel" or isinstance(callback, OpenTelemetry):
-                _otel_logger_to_use: Optional[OpenTelemetry] = None
-                if isinstance(callback, OpenTelemetry):
-                    _otel_logger_to_use = callback
-                else:
-                    from litellm.proxy.proxy_server import open_telemetry_logger
-
-                    if open_telemetry_logger is not None and isinstance(
-                        open_telemetry_logger, OpenTelemetry
-                    ):
-                        _otel_logger_to_use = open_telemetry_logger
+            else:
+                _otel_logger_to_use = self._resolve_otel_service_logger(callback)
 
                 if not isinstance(error, str):
                     error = str(error)

litellm/integrations/otel/README.md

@@ -63,9 +63,19 @@ becomes the global, so server spans export to that backend too.
    `standard_logging_object` and hands it to the engine, which creates the LLM
    span as a child of the server span. Emission is **async-only**; the
    synchronous callback runs in a worker thread without the request context and
-   is a no-op.
+   is a no-op. **Pass-through** endpoints dispatch their logging from a detached
+   `asyncio.create_task` whose copied context may no longer carry the server
+   span, so the proxy also threads the span explicitly as
+   `litellm_parent_otel_span`; the adapter falls back to it when the ambient
+   context has no recordable span, so the pass-through LLM-call span still nests
+   under the request instead of becoming its own root trace.
 5. **Guardrails / services**: the post-call and service hooks emit guardrail and
-   service spans the same way — typed data → engine → span.
+   service spans the same way — typed data → engine → span. Service spans
+   (Redis/Postgres) are dispatched by `litellm/_service_logger.py`, which
+   recognizes the V2 `OpenTelemetryV2` logger (a plain `CustomLogger`, not a
+   subclass of the legacy `OpenTelemetry`). Guardrail span data is built from the
+   typed, provider-agnostic `StandardLoggingGuardrailInformation` — no single
+   provider's field shape is assumed.
 6. **Export**: each span ends and is handed to the provider's span processors,
    which export to the configured backends (OTLP, console, in-memory, …).
 
@@ -87,7 +97,12 @@ be imported anywhere:
 - [`config.py`](./config.py) — `OpenTelemetryV2Config`, a pydantic-settings
   model that reads `OTEL_*` / `LITELLM_OTEL_*` env vars, plus the feature gate.
   `capture_span_content` gates whether prompt/response bodies may be written as
-  span attributes; it defaults **off** (`no_content`).
+  span attributes; it defaults **off** (`no_content`). The Baggage allowlists are
+  configurable, not hard-coded: set `LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS` /
+  `LITELLM_OTEL_BAGGAGE_METADATA_KEYS` (comma-separated) as env vars, or
+  `baggage_promoted_keys` / `baggage_metadata_keys` (YAML lists) under
+  `callback_settings.otel` in `config.yaml` — the latter reach the config through
+  the logger's constructor kwargs.
 
 ### Engine
 

litellm/integrations/otel/config.py

@@ -1,7 +1,10 @@
 """Typed configuration for the OpenTelemetry instrumentation."""
 
-from pydantic import AliasChoices, BaseModel, Field, model_validator
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing import Any, List
+
+from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
+from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
+from typing_extensions import Annotated
 
 from litellm.integrations.otel.baggage import (
     BAGGAGE_PROMOTED_KEYS,
@@ -122,7 +125,7 @@ class OpenTelemetryV2Config(BaseSettings):
         ),
     )
 
-    mapper_names: list[str] = Field(
+    mapper_names: Annotated[List[str], NoDecode] = Field(
         default_factory=lambda: ["genai"],
         description=(
             "Ordered attribute vocabularies to emit. ``genai`` is the "
@@ -140,12 +143,51 @@ class OpenTelemetryV2Config(BaseSettings):
         ),
     )
 
-    baggage_promoted_keys: list[str] = Field(
-        default_factory=lambda: list(BAGGAGE_PROMOTED_KEYS)
+    baggage_promoted_keys: Annotated[List[str], NoDecode] = Field(
+        default_factory=lambda: list(BAGGAGE_PROMOTED_KEYS),
+        validation_alias=AliasChoices(
+            "baggage_promoted_keys", "LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS"
+        ),
+        description=(
+            "Identity attribute keys written into Baggage and stamped on every "
+            "child span (e.g. ``litellm.team.id``). Configure via the "
+            "``LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS`` env var (comma-separated) or "
+            "``callback_settings.otel.baggage_promoted_keys`` in config.yaml (a "
+            "YAML list)."
+        ),
     )
-    baggage_metadata_keys: list[str] = Field(
-        default_factory=lambda: list(DEFAULT_BAGGAGE_METADATA_KEYS)
+    baggage_metadata_keys: Annotated[List[str], NoDecode] = Field(
+        default_factory=lambda: list(DEFAULT_BAGGAGE_METADATA_KEYS),
+        validation_alias=AliasChoices(
+            "baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"
+        ),
+        description=(
+            "Metadata sub-keys promoted under the ``litellm.metadata.*`` "
+            "namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` "
+            "env var (comma-separated) or "
+            "``callback_settings.otel.baggage_metadata_keys`` in config.yaml."
+        ),
+    )
+
+    @field_validator(
+        "baggage_promoted_keys",
+        "baggage_metadata_keys",
+        "mapper_names",
+        mode="before",
     )
+    @classmethod
+    def _split_csv(cls, value: Any) -> Any:
+        """Accept a comma-separated string for list fields.
+
+        Env vars are strings, but these fields are lists. Pydantic-settings would
+        otherwise require JSON for a list env var; splitting on commas here lets
+        an operator write ``LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS=litellm.team.id,litellm.api_key.hash``.
+        YAML lists (from ``callback_settings.otel.*``) and real lists pass through
+        unchanged.
+        """
+        if isinstance(value, str):
+            return [item.strip() for item in value.split(",") if item.strip()]
+        return value
 
     @model_validator(mode="after")
     def _normalize(self) -> "OpenTelemetryV2Config":

litellm/integrations/otel/logger.py

@@ -17,7 +17,7 @@
 """
 
 from datetime import datetime
-from typing import Any, Mapping, cast
+from typing import TYPE_CHECKING, Any, Mapping, cast
 
 from opentelemetry.context import attach, get_current
 from opentelemetry.sdk.trace import TracerProvider
@@ -46,6 +46,9 @@
 from litellm.integrations.otel.spans import SpanRole
 from litellm.integrations.otel.utils import to_ns
 
+if TYPE_CHECKING:
+    from litellm.types.utils import StandardLoggingGuardrailInformation
+
 LITELLM_TRACER_NAME = "litellm"
 LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
 
@@ -57,6 +60,30 @@
 )
 
 
+def _threaded_parent_span(kwargs: Mapping[str, Any]) -> Span | None:
+    """The proxy SERVER span threaded through request metadata, if any.
+
+    Normally the LLM-call span parents to the ambient OTel context (the active
+    server span). But pass-through logging runs in a detached
+    ``asyncio.create_task`` whose copied context may no longer carry that span,
+    so the proxy also threads it explicitly as ``litellm_parent_otel_span`` (see
+    ``litellm_pre_call_utils`` for proxy routes and the pass-through endpoint for
+    catch-all routes). This reads it back so the call span can fall back to it.
+    """
+    litellm_params = kwargs.get("litellm_params")
+    candidates: list[Any] = []
+    if isinstance(litellm_params, Mapping):
+        candidates.append(litellm_params.get("metadata"))
+        candidates.append(litellm_params.get("litellm_metadata"))
+    candidates.append(kwargs.get("metadata"))
+    for meta in candidates:
+        if isinstance(meta, Mapping):
+            span = meta.get("litellm_parent_otel_span")
+            if span is not None:
+                return cast("Span", span)
+    return None
+
+
 def _pre_call_guardrail_blocked(payload: Mapping[str, Any]) -> bool:
     """True when a pre-call guardrail blocked the request (no LLM call happened).
 
@@ -103,7 +130,11 @@ def __init__(
         **kwargs: Any,
     ) -> None:
         super().__init__(**kwargs)
-        self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config()
+        # Build the config from any settings passed through ``kwargs`` so
+        # ``callback_settings.otel.*`` in config.yaml (e.g. ``baggage_promoted_keys``,
+        # ``capture_message_content``) configures the logger. ``OpenTelemetryV2Config``
+        # ignores extra keys, so unrelated kwargs are dropped harmlessly.
+        self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs)
         self.callback_name = callback_name
         self._tracer_provider: TracerProvider = (
             tracer_provider
@@ -187,8 +218,17 @@ def _emit_llm_call(
             cast("Any", payload), capture_content=self.config.capture_span_content
         )
         # Parent is the ambient context (the instrumentor's server span,
-        # restored by the logging worker); no span is threaded through metadata.
+        # restored by the logging worker). When the ambient context has lost the
+        # server span — e.g. pass-through logging fired from a detached task —
+        # fall back to the server span the proxy threaded through metadata so the
+        # call span still nests under the request instead of being dropped.
         parent_ctx = get_current()
+        if not is_recordable_span(get_current_span(parent_ctx)):
+            threaded_parent = _threaded_parent_span(kwargs)
+            if is_recordable_span(threaded_parent):
+                parent_ctx = context_from_span(
+                    cast("Span", threaded_parent), context=parent_ctx
+                )
         # Write identity into Baggage so child spans (guardrails, services)
         # inherit it.
         bag = promoted_baggage(
@@ -361,7 +401,10 @@ def _emit_guardrail_spans(self, request_data: Mapping[str, Any]) -> None:
             if not isinstance(entry, dict):
                 continue
             self._emitter.emit(
-                SpanRole.GUARDRAIL, GuardrailSpanData.from_logging_entry(entry)
+                SpanRole.GUARDRAIL,
+                GuardrailSpanData.from_logging_entry(
+                    cast("StandardLoggingGuardrailInformation", entry)
+                ),
             )
 
     # ====================================================================== #

litellm/integrations/otel/mappers/genai.py

@@ -73,6 +73,9 @@ class GenAIMapper:
         LiteLLM.GUARDRAIL_RISK_SCORE: lambda d: d.risk_score,
         LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT: lambda d: d.masked_entity_count,
         LiteLLM.GUARDRAIL_DURATION: lambda d: d.duration,
+        LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id,
+        LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template,
+        LiteLLM.GUARDRAIL_DETECTION_METHOD: lambda d: d.detection_method,
     }
 
     _SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = {