fix(otel): stamp http.response.status_code on all error responses

litellm/integrations/opentelemetry.py

@@ -702,6 +702,14 @@ async def async_post_call_failure_hook(
                 },
             )
 
+            # _record_exception_on_span only stamps when error_code is set;
+            # bare TypeError etc. has none, and the span is about to be ended.
+            error_code = (
+                error_information.get("error_code") if error_information else None
+            )
+            if not error_code:
+                self.set_response_status_code_attribute(parent_otel_span, 500)
+
             # Pre-request latency (request_data carries the propagated
             # metadata on the failure path; omitted if it failed before handoff).
             self.set_preprocessing_duration_attribute(parent_otel_span, request_data)
@@ -750,11 +758,6 @@ async def async_post_call_success_hook(
             # Pre-request latency on the SERVER span (success path).
             self.set_preprocessing_duration_attribute(parent_span, kwargs)
 
-            # http.response.status_code on the SERVER span (success path).
-            # A successful proxy response is HTTP 200; the failure path sets
-            # this from the error code in _record_exception_on_span.
-            self.set_response_status_code_attribute(parent_span, 200)
-
             # 3. Guardrail span
             self._create_guardrail_span(kwargs=kwargs, context=ctx)
 
@@ -937,7 +940,15 @@ def _end_proxy_span_from_kwargs(self, kwargs: dict, end_time) -> None:
             and hasattr(proxy_span, "is_recording")
             and proxy_span.is_recording()
         ):
-            proxy_span.end(end_time=self._to_ns(end_time))
+            self._close_proxy_span_ok(proxy_span, end_time)
+
+    def _close_proxy_span_ok(self, span: Span, end_time) -> None:
+        """Stamp http.response.status_code=200 + status=OK, then end the span."""
+        from opentelemetry.trace import Status, StatusCode
+
+        self.set_response_status_code_attribute(span, 200)
+        span.set_status(Status(StatusCode.OK))
+        span.end(end_time=self._to_ns(end_time))
 
     def _handle_success(self, kwargs, response_obj, start_time, end_time):
         """Create the litellm_request span then close the proxy span."""
@@ -1023,8 +1034,10 @@ def _handle_success(self, kwargs, response_obj, start_time, end_time):
             parent_span is not None
             and hasattr(parent_span, "name")
             and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
+            and hasattr(parent_span, "is_recording")
+            and parent_span.is_recording()
         ):
-            parent_span.end(end_time=self._to_ns(end_time))
+            self._close_proxy_span_ok(parent_span, end_time)
 
         # Stamp team attributes onto the SERVER (root) span before it is
         # closed, so the trace root carries them like every child span.
@@ -2962,6 +2975,11 @@ async def async_management_endpoint_success_hook(
             management_endpoint_span.set_status(Status(StatusCode.OK))
             management_endpoint_span.end(end_time=_end_time_ns)
 
+            # The management wrapper has no other hook that closes the SERVER span.
+            self.set_response_status_code_attribute(parent_otel_span, 200)
+            parent_otel_span.set_status(Status(StatusCode.OK))
+            parent_otel_span.end(end_time=_end_time_ns)
+
     async def async_management_endpoint_failure_hook(
         self,
         logging_payload: ManagementEndpointLoggingPayload,
@@ -3012,6 +3030,24 @@ async def async_management_endpoint_failure_hook(
             management_endpoint_span.set_status(Status(StatusCode.ERROR))
             management_endpoint_span.end(end_time=_end_time_ns)
 
+            # The management wrapper has no other hook that closes the SERVER span.
+            from litellm.litellm_core_utils.litellm_logging import (
+                StandardLoggingPayloadSetup,
+            )
+
+            error_information = StandardLoggingPayloadSetup.get_error_information(
+                original_exception=_exception,
+            )
+            parent_otel_span.set_status(Status(StatusCode.ERROR))
+            self._record_exception_on_span(
+                span=parent_otel_span,
+                kwargs={
+                    "exception": _exception,
+                    "standard_logging_object": {"error_information": error_information},
+                },
+            )
+            parent_otel_span.end(end_time=_end_time_ns)
+
     def create_litellm_proxy_request_started_span(
         self,
         start_time: datetime,

litellm/litellm_core_utils/litellm_logging.py

@@ -5143,13 +5143,17 @@ def get_error_information(
     ) -> StandardLoggingPayloadErrorInformation:
         from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
 
-        # Check for 'code' first (used by ProxyException), then fall back to 'status_code' (used by LiteLLM exceptions)
-        # Ensure error_code is always a string for Prisma Python JSON field compatibility
+        # ProxyException uses .code, LiteLLM exceptions use .status_code,
+        # httpx.HTTPStatusError exposes status only as .response.status_code.
+        # Stringified for Prisma JSON compatibility.
         error_code_attr = getattr(original_exception, "code", None)
         if error_code_attr is not None and str(error_code_attr) not in ("", "None"):
             error_status: str = str(error_code_attr)
         else:
             status_code_attr = getattr(original_exception, "status_code", None)
+            if status_code_attr is None:
+                response_attr = getattr(original_exception, "response", None)
+                status_code_attr = getattr(response_attr, "status_code", None)
             error_status = str(status_code_attr) if status_code_attr is not None else ""
         error_class: str = (
             str(original_exception.__class__.__name__) if original_exception else ""

litellm/proxy/auth/user_api_key_auth.py

@@ -656,6 +656,37 @@ async def _resolve_jwt_to_virtual_key(
         return None
 
 
+def _ensure_parent_otel_span_on_request_state(request: Request) -> None:
+    """Idempotently create the OTEL SERVER span and stash it on
+    ``request.state.parent_otel_span``. Safe to call multiple times.
+
+    Called both at the top of ``user_api_key_auth`` (so body-parse failures
+    have a span to close) and inside ``_user_api_key_auth_builder`` (for
+    callers that bypass ``user_api_key_auth``, e.g. MCP).
+    """
+    from litellm.proxy.proxy_server import open_telemetry_logger
+
+    if open_telemetry_logger is None:
+        return
+    if getattr(request.state, "parent_otel_span", None) is not None:
+        return
+    start_time = datetime.now()
+    try:
+        request.state.litellm_received_at = start_time
+    except Exception:
+        pass
+    parent_otel_span = open_telemetry_logger.create_litellm_proxy_request_started_span(
+        start_time=start_time,
+        headers=_safe_get_request_headers(request),
+    )
+    open_telemetry_logger.set_proxy_request_route_attributes(
+        parent_otel_span,
+        url_path=get_request_route(request=request),
+        http_route=get_request_route_template(request),
+    )
+    request.state.parent_otel_span = parent_otel_span
+
+
 async def _user_api_key_auth_builder(  # noqa: PLR0915
     request: Request,
     api_key: str,
@@ -682,9 +713,10 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
     )
 
     parent_otel_span: Optional[Span] = None
-    start_time = datetime.now()
-    # Stash the proxy-receive instant for the pre-request latency calc —
-    # the OTel Span API exposes no start-time getter, so propagate it.
+    # Prefer the receive-instant stamped by the early helper in
+    # user_api_key_auth (before body parse) — overwriting it would shorten
+    # the preprocessing-duration measurement by the body-parse window.
+    start_time = getattr(request.state, "litellm_received_at", None) or datetime.now()
     try:
         request.state.litellm_received_at = start_time
     except Exception:
@@ -724,18 +756,11 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
             )
 
         if open_telemetry_logger is not None:
-            parent_otel_span = (
-                open_telemetry_logger.create_litellm_proxy_request_started_span(
-                    start_time=start_time,
-                    headers=_safe_get_request_headers(request),
-                )
-            )
-            # `route` is the literal path; template from the matched route.
-            open_telemetry_logger.set_proxy_request_route_attributes(
-                parent_otel_span,
-                url_path=route,
-                http_route=get_request_route_template(request),
-            )
+            # Reuse the span created by user_api_key_auth (before body parse)
+            # so it survives _read_request_body failures. For callers that
+            # bypass user_api_key_auth (e.g. MCP), create it lazily.
+            _ensure_parent_otel_span_on_request_state(request)
+            parent_otel_span = getattr(request.state, "parent_otel_span", None)
 
         ### USER-DEFINED AUTH FUNCTION ###
         if enterprise_custom_auth is not None:
@@ -2112,6 +2137,12 @@ async def user_api_key_auth(
     Parent function to authenticate user api key / jwt token.
     """
 
+    # Create the SERVER span and stash it on request.state BEFORE reading the
+    # body. _read_request_body can raise ProxyException for malformed JSON;
+    # without this, that path leaves no span for the exception handler to
+    # close, and the trace never reaches the backend.
+    _ensure_parent_otel_span_on_request_state(request)
+
     request_data = await _read_request_body(request=request)
     request_data = populate_request_with_path_params(
         request_data=request_data, request=request

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -3222,6 +3222,7 @@ async def info_key_fn_v2(
 @router.get(
     "/key/info", tags=["key management"], dependencies=[Depends(user_api_key_auth)]
 )
+@management_endpoint_wrapper
 async def info_key_fn(
     key: Optional[str] = fastapi.Query(
         default=None, description="Key in the request parameters"

litellm/proxy/management_helpers/utils.py

@@ -518,19 +518,23 @@ async def wrapper(*args, **kwargs):
                         _request_body: dict = await _read_request_body(
                             request=_http_request
                         )
-                        logging_payload = ManagementEndpointLoggingPayload(
-                            route=_route,
-                            request_data=_request_body,
-                            response=None,
-                            start_time=start_time,
-                            end_time=end_time,
-                            exception=e,
-                        )
-
-                        await open_telemetry_logger.async_management_endpoint_failure_hook(  # type: ignore
-                            logging_payload=logging_payload,
-                            parent_otel_span=parent_otel_span,
-                        )
+                    else:
+                        _route = func.__name__
+                        _request_body = {}
+
+                    logging_payload = ManagementEndpointLoggingPayload(
+                        route=_route,
+                        request_data=_request_body,
+                        response=None,
+                        start_time=start_time,
+                        end_time=end_time,
+                        exception=e,
+                    )
+
+                    await open_telemetry_logger.async_management_endpoint_failure_hook(  # type: ignore
+                        logging_payload=logging_payload,
+                        parent_otel_span=parent_otel_span,
+                    )
 
             raise e
 

litellm/proxy/proxy_server.py

@@ -550,6 +550,7 @@ def generate_feedback_box():
     status,
 )
 from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.openapi.docs import get_swagger_ui_html
 from fastapi.openapi.utils import get_openapi
@@ -1209,15 +1210,69 @@ async def openai_exception_handler(request: Request, exc: ProxyException):
     # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions
     headers = exc.headers
     error_dict = exc.to_dict()
+    status_code = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR
+    _close_dangling_otel_server_span(request, status_code)
     return JSONResponse(
-        status_code=(
-            int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR
-        ),
+        status_code=status_code,
         content={"error": error_dict},
         headers=headers,
     )
 
 
+def _close_dangling_otel_server_span(request: Request, status_code: int) -> None:
+    parent_otel_span = getattr(request.state, "parent_otel_span", None)
+    if parent_otel_span is None:
+        return
+    if open_telemetry_logger is None:
+        return
+    try:
+        from opentelemetry.trace import Status, StatusCode
+
+        open_telemetry_logger.set_response_status_code_attribute(
+            parent_otel_span, status_code
+        )
+        parent_otel_span.set_status(
+            Status(StatusCode.ERROR if status_code >= 400 else StatusCode.OK)
+        )
+        parent_otel_span.end()
+    except Exception as e:
+        verbose_proxy_logger.debug(
+            "Error closing dangling OTEL SERVER span: %s", str(e)
+        )
+    finally:
+        request.state.parent_otel_span = None
+
+
+@app.exception_handler(RequestValidationError)
+async def otel_request_validation_exception_handler(
+    request: Request, exc: RequestValidationError
+):
+    _close_dangling_otel_server_span(request, 422)
+    return JSONResponse(
+        status_code=422,
+        content={"detail": jsonable_encoder(exc.errors())},
+    )
+
+
+@app.exception_handler(Exception)
+async def otel_unhandled_exception_handler(request: Request, exc: Exception):
+    if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)):
+        raise exc
+    verbose_proxy_logger.exception(
+        "Unhandled exception in request: %s", type(exc).__name__
+    )
+    _close_dangling_otel_server_span(request, 500)
+    return JSONResponse(
+        status_code=500,
+        content={
+            "error": {
+                "message": "Internal server error",
+                "type": "internal_server_error",
+            }
+        },
+    )
+
+
 router = APIRouter()