chore(release): backport #32256, #32405, #32524 to stable/1.91.x and cut 1.91.1

litellm/integrations/otel/__init__.py

@@ -50,6 +50,7 @@ class (from :mod:`logger`).
     GenAIProvider,
     JsonRpc,
     LiteLLM,
+    LiteLLMError,
     MCPMethod,
     Metric,
     Network,
@@ -85,6 +86,7 @@ class (from :mod:`logger`).
     "HTTP",
     "JsonRpc",
     "LiteLLM",
+    "LiteLLMError",
     "MCP",
     "MCPMethod",
     "Metric",

litellm/integrations/otel/emitter.py

@@ -15,9 +15,10 @@
     LLMCallSpanData,
     MCPToolCallSpanData,
     ServiceSpanData,
+    SpanError,
 )
 from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
-from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
+from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError
 from litellm.integrations.otel.model.spans import (
     SPAN_REGISTRY,
     SpanRole,
@@ -46,6 +47,27 @@
 _DEDUP_CACHE_MAX = 10_000
 
 
+def _stamp_otel_error_attributes(span: Span, error_type: str, resolved_message: str) -> None:
+    """Stamp the OTel-semconv error attributes (``error.type`` + ``error.message``).
+    ``error_type`` and ``resolved_message`` are ``finish_span``'s already-computed
+    fallback chains, so the pair on the status, event, and attributes stays in
+    lockstep."""
+    span.set_attribute(Error.TYPE, error_type)
+    span.set_attribute(Error.MESSAGE, resolved_message)
+
+
+def _stamp_litellm_error_attributes(span: Span, error: SpanError) -> None:
+    """Stamp litellm-specific error detail attributes. Emitted only when the
+    corresponding field is populated so guardrail-shape errors carrying only a
+    message aren't polluted with empty detail keys."""
+    if error.code:
+        span.set_attribute(LiteLLMError.CODE, error.code)
+    if error.stack_trace:
+        span.set_attribute(LiteLLMError.STACK_TRACE, error.stack_trace)
+    if error.llm_provider:
+        span.set_attribute(LiteLLMError.LLM_PROVIDER, error.llm_provider)
+
+
 class SpanEmitter:
     def __init__(
         self,
@@ -175,12 +197,13 @@ def finish_span(
         if error and (error.error_type or error.message):
             error_type = error.error_type or "error"
             message = error.message or error.error_type or "error"
-            span.set_attribute(Error.TYPE, error_type)
+            _stamp_otel_error_attributes(span, error_type, message)
+            _stamp_litellm_error_attributes(span, error)
             span.set_status(Status(StatusCode.ERROR, message))
-            # Carry the full message on the standard ``exception`` event so backends
-            # map it as full text under ``exception.message``. Setting it as a bare
-            # string attribute instead lets backends like Elasticsearch dynamic-map
-            # it to a ``keyword`` capped at 1024 chars, truncating the message.
+            # Also emit the semconv ``exception`` event so backends that
+            # dynamic-map unknown string span attrs to ``keyword`` (e.g.
+            # Elasticsearch with a 1024-char ``ignore_above``) still see the
+            # full untruncated message on the recognized event field.
             span.add_event(
                 ExceptionEvent.NAME,
                 {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},

litellm/integrations/otel/model/payloads.py

@@ -139,6 +139,9 @@ def from_breakdown(cls, breakdown: Mapping[str, object] | None) -> "LLMCost":
 class SpanError:
     error_type: str | None = None
     message: str | None = None
+    code: str | None = None
+    stack_trace: str | None = None
+    llm_provider: str | None = None
 
 
 @dataclass(frozen=True)
@@ -528,6 +531,9 @@ def _parse_error(payload: "StandardLoggingPayload") -> SpanError | None:
     return SpanError(
         error_type=as_str(info.get("error_class")) or as_str(info.get("error_code")),
         message=as_str(info.get("error_message")) or as_str(payload.get("error_str")),
+        code=as_str(info.get("error_code")),
+        stack_trace=as_str(info.get("traceback")),
+        llm_provider=as_str(info.get("llm_provider")),
     )
 
 

litellm/integrations/otel/model/semconv.py

@@ -143,7 +143,27 @@ class Client:
 
 
 class Error:
+    """OTel-defined error attribute keys, from the semconv ``error.*`` registry.
+    ``MESSAGE`` is marked *Deprecated* upstream in favor of domain-specific
+    error message keys plus ``exception.message`` on the exception event, but
+    is still defined and stamped by litellm's v1 integration; keeping it here
+    for byte-for-byte parity."""
+
     TYPE: Final = "error.type"
+    MESSAGE: Final = "error.message"
+
+
+class LiteLLMError:
+    """LiteLLM-specific error attribute keys. Emitted under the ``error.*``
+    namespace (not ``litellm.*``) for byte-for-byte compat with the v1
+    integration in ``opentelemetry.py``; consumers reading these keys on v1
+    spans read the same keys on v2 spans. OTel semconv does not define any of
+    these three, and per its extension rules a namespace may carry additional
+    vendor keys as long as they don't collide with defined names."""
+
+    CODE: Final = "error.code"
+    STACK_TRACE: Final = "error.stack_trace"
+    LLM_PROVIDER: Final = "error.llm_provider"
 
 
 class ExceptionEvent:

litellm/proxy/proxy_server.py

@@ -1074,33 +1074,6 @@ def _generate_stable_operation_id(route: Any) -> str:
 # `_SSO_SENSITIVE_FIELDS` / `_CACHE_SENSITIVE_FIELDS` constants in the SSO
 # and cache endpoint files.
 _ALERTING_SENSITIVE_VARS: Set[str] = {"SLACK_WEBHOOK_URL", "SMTP_PASSWORD"}
-_DB_LITELLM_PARAM_ENV_REF_KEYS = frozenset(
-    {
-        "api_key",
-        "client_secret",
-        "vertex_credentials",
-        "vertex_ai_credentials",
-        "aws_access_key_id",
-        "aws_secret_access_key",
-    }
-)
-
-
-def _db_model_is_team_scoped(model: object) -> bool:
-    model_info = getattr(model, "model_info", None)
-    if isinstance(model_info, BaseModel):
-        return getattr(model_info, "team_id", None) is not None
-    if isinstance(model_info, str):
-        try:
-            model_info = json.loads(model_info)
-        except (TypeError, ValueError):
-            model_info = None
-    if isinstance(model_info, dict) and model_info.get("team_id") is not None:
-        return True
-    if getattr(model_info, "team_id", None) is not None:
-        return True
-    model_name = getattr(model, "model_name", None)
-    return isinstance(model_name, str) and model_name.startswith("model_name_")
 
 
 def _strip_operation_id_method_suffix(operation_id: str) -> str:
@@ -4903,17 +4876,12 @@ async def _delete_deployment(self, db_models: list) -> int:
                     deleted_deployments += 1
         return deleted_deployments
 
-    def _resolve_db_litellm_param(self, key: str, value: object, resolve_env_refs: bool = True) -> object:
+    def _resolve_db_litellm_param(self, key: str, value: object) -> object:
         if not isinstance(value, str):
             return value
 
         decrypted_value = decrypt_value_helper(value=value, key=key, return_original_value=True)
-        if (
-            resolve_env_refs
-            and key in _DB_LITELLM_PARAM_ENV_REF_KEYS
-            and isinstance(decrypted_value, str)
-            and decrypted_value.startswith("os.environ/")
-        ):
+        if isinstance(decrypted_value, str) and decrypted_value.startswith("os.environ/"):
             return get_secret(decrypted_value)
         return decrypted_value
 
@@ -4934,13 +4902,10 @@ def _add_deployment(self, db_models: list) -> int:
         ## ADD MODEL LOGIC
         for m in db_models:
             _litellm_params = m.litellm_params
-            resolve_env_refs = not _db_model_is_team_scoped(m)
             if isinstance(_litellm_params, dict):
                 # decrypt values
                 for k, v in _litellm_params.items():
-                    _litellm_params[k] = self._resolve_db_litellm_param(
-                        key=k, value=v, resolve_env_refs=resolve_env_refs
-                    )
+                    _litellm_params[k] = self._resolve_db_litellm_param(key=k, value=v)
                 _litellm_params = LiteLLM_Params(**_litellm_params)
 
             else:
@@ -4966,15 +4931,12 @@ def decrypt_model_list_from_db(self, new_models: list) -> list:
         _model_list: list = []
         for m in new_models:
             _litellm_params = m.litellm_params
-            resolve_env_refs = not _db_model_is_team_scoped(m)
             if isinstance(_litellm_params, BaseModel):
                 _litellm_params = _litellm_params.model_dump()
             if isinstance(_litellm_params, dict):
                 # decrypt values
                 for k, v in _litellm_params.items():
-                    _litellm_params[k] = self._resolve_db_litellm_param(
-                        key=k, value=v, resolve_env_refs=resolve_env_refs
-                    )
+                    _litellm_params[k] = self._resolve_db_litellm_param(key=k, value=v)
                 _litellm_params = LiteLLM_Params(**_litellm_params)
             else:
                 verbose_proxy_logger.error(

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.91.0"
+version = "1.91.1"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.14"
@@ -269,7 +269,7 @@ source-exclude = [
 profile = "black"
 
 [tool.commitizen]
-version = "1.91.0"
+version = "1.91.1"
 version_files = [
     "pyproject.toml:^version",
 ]

tests/test_litellm/integrations/otel/test_otel_v2_components.py

@@ -579,13 +579,10 @@ def _exception_event(span):
 
 
 def test_error_message_recorded_as_full_exception_event_untruncated():
-    """Regression for the Elasticsearch keyword/ignore_above:1024 truncation.
-
-    A long error message must survive intact on the standard ``exception``
-    event under ``exception.message`` — not get dropped onto a bare string
-    attribute that backends dynamic-map to a 1024-char ``keyword``. The SDK
-    must not truncate it either, so a 5000-char message stays 5000 chars.
-    """
+    """The ``exception`` event carries the full untruncated message under
+    ``exception.message`` so backends that dynamic-map unknown string span
+    attrs to ``keyword`` (e.g. Elasticsearch with a 1024-char ``ignore_above``)
+    still see it in full via the semconv-recognized event field."""
     from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
 
     long_message = "boom: " + "x" * 5000
@@ -596,13 +593,108 @@ def test_error_message_recorded_as_full_exception_event_untruncated():
     assert len(event.attributes[ExceptionEvent.MESSAGE]) == len(long_message) > 1024
     assert event.attributes[ExceptionEvent.TYPE] == "litellm.APIError"
 
-    # error.type stays a low-cardinality attribute; the message does NOT become a
-    # bare string attribute (which is what got truncated).
+    # error.type stays a low-cardinality attribute; the exception EVENT field
+    # ``exception.message`` never becomes a bare string attribute.
     assert span.attributes[Error.TYPE] == "litellm.APIError"
     assert ExceptionEvent.MESSAGE not in span.attributes
     assert span.status.description == long_message
 
 
+def test_error_details_stamped_as_span_attributes_for_labels_ingest():
+    """OTel-defined keys and litellm-specific detail keys both ride span
+    attributes so backends that flatten attrs into label indexes (Elastic APM
+    ``labels.*``, Datadog span tags) render them. The exception event with the
+    full untruncated message stays alongside — both places, matching v1's
+    shape."""
+    from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError
+    from litellm.integrations.otel.emitter import SpanEmitter
+
+    cfg = OpenTelemetryV2Config(exporter="in_memory")
+    provider, exporter = providers.in_memory_provider(cfg)
+    engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
+    data = LLMCallSpanData(
+        operation=GenAIOperation.CHAT,
+        provider="openai",
+        request_model="gpt-4o",
+        response_model=None,
+        response_id=None,
+        request_params=LLMRequestParams(),
+        usage=LLMUsage(),
+        finish_reasons=(),
+        error=SpanError(
+            error_type="litellm.BadRequestError",
+            message="400: violated moderation policy",
+            code="400",
+            stack_trace="File proxy_server.py line 8570 ...",
+            llm_provider="openai",
+        ),
+        response_cost=None,
+        server=None,
+        identity=RequestIdentity(call_id=None),
+    )
+    engine.emit(SpanRole.LLM_CALL, data)
+    (span,) = exporter.get_finished_spans()
+
+    # OTel-defined keys (from the ``error.*`` semconv registry).
+    assert span.attributes[Error.TYPE] == "litellm.BadRequestError"
+    assert span.attributes[Error.MESSAGE] == "400: violated moderation policy"
+    # LiteLLM-specific detail keys — vendor-namespaced under ``error.*``
+    # for v1-parity, not defined by OTel semconv.
+    assert span.attributes[LiteLLMError.CODE] == "400"
+    assert span.attributes[LiteLLMError.STACK_TRACE] == "File proxy_server.py line 8570 ..."
+    assert span.attributes[LiteLLMError.LLM_PROVIDER] == "openai"
+
+    # The exception event carries the same message on the span too.
+    event = _exception_event(span)
+    assert event.attributes[ExceptionEvent.MESSAGE] == "400: violated moderation policy"
+
+
+def test_error_details_omitted_when_span_error_carries_only_message():
+    """A guardrail-shape error (message only, no code/traceback/provider) must
+    not pollute the span with empty-string detail attributes. Only the keys
+    that carry real data land."""
+    from litellm.integrations.otel.model.semconv import Error, LiteLLMError
+
+    span = _emit_error_span("guardrail rejected", error_type="ContentFilter")
+
+    assert span.attributes[Error.TYPE] == "ContentFilter"
+    assert span.attributes[Error.MESSAGE] == "guardrail rejected"
+    # LiteLLM-specific detail keys aren't stamped when the SpanError doesn't
+    # carry them.
+    assert LiteLLMError.CODE not in span.attributes
+    assert LiteLLMError.STACK_TRACE not in span.attributes
+    assert LiteLLMError.LLM_PROVIDER not in span.attributes
+
+
+def test_v2_error_attribute_keys_match_v1_error_attributes_byte_for_byte():
+    """v1 (``opentelemetry.py``) and v2 (``otel/`` package) stamp identical
+    span-attribute keys so consumers reading ``labels.error_message`` don't
+    care which integration produced the span. Renaming either side is a
+    breaking change for downstream dashboards; this test locks the vocabulary."""
+    from litellm.integrations._types.open_inference import ErrorAttributes
+    from litellm.integrations.otel.model.semconv import Error, LiteLLMError
+
+    assert Error.TYPE == ErrorAttributes.ERROR_TYPE
+    assert Error.MESSAGE == ErrorAttributes.ERROR_MESSAGE
+    assert LiteLLMError.CODE == ErrorAttributes.ERROR_CODE
+    assert LiteLLMError.STACK_TRACE == ErrorAttributes.ERROR_STACK_TRACE
+    assert LiteLLMError.LLM_PROVIDER == ErrorAttributes.ERROR_LLM_PROVIDER
+
+
+def test_error_message_falls_back_to_error_type_when_message_absent():
+    """A ``SpanError(error_type=..., message=None)`` still renders on the span:
+    the resolved message is the error_type, and it lands on ``error.message``,
+    the exception event, and the span-status description in lockstep so a
+    single-source-of-truth view isn't inconsistent."""
+    from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
+
+    span = _emit_error_span(message=None, error_type="RateLimitError")
+
+    assert span.attributes[Error.MESSAGE] == "RateLimitError"
+    assert _exception_event(span).attributes[ExceptionEvent.MESSAGE] == "RateLimitError"
+    assert span.status.description == "RateLimitError"
+
+
 def test_success_span_records_no_exception_event():
     from litellm.integrations.otel.emitter import SpanEmitter
     from litellm.integrations.otel.model.semconv import ExceptionEvent