fix(proxy): preserve spend metadata on auth failures

litellm/proxy/auth/auth_exception_handler.py

@@ -24,6 +24,9 @@
     is_invalid_virtual_key_error,
     mark_invalid_virtual_key_error,
 )
+from litellm.proxy.common_utils.http_parsing_utils import (
+    _safe_get_request_headers,  # pyright: ignore[reportPrivateUsage]  # canonical non-throwing request header reader
+)
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 from litellm.types.services import ServiceTypes
 
@@ -74,17 +77,35 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
     )
 
 
-def _with_requester_ip_address(request_data: dict[str, object], requester_ip: str | None) -> dict[str, object]:
-    """Auth gate rejections are raised before `add_litellm_data_to_request` records the
-    caller IP, so their failure logs would otherwise carry no IP nor key/user identity."""
+def _with_auth_failure_metadata(
+    request_data: dict[str, object],
+    requester_ip: str | None,
+    spend_logs_metadata: Mapping[str, object] | None,
+) -> dict[str, object]:
+    """Add metadata normally populated after auth without mutating the request body."""
+    raw_spend_metadata: Final = request_data.get("metadata")
+    spend_metadata_base: Final[Mapping[str, object]] = (  # pyright: ignore[reportUnknownVariableType]  # request JSON mappings have string keys
+        raw_spend_metadata if isinstance(raw_spend_metadata, Mapping) else EMPTY_MAPPING
+    )
+    request_data_with_spend_metadata: Final = (
+        {
+            **request_data,
+            "metadata": {**spend_metadata_base, "spend_logs_metadata": spend_logs_metadata},
+        }
+        if spend_logs_metadata is not None
+        else request_data
+    )  # mutable-ok: logging hooks require plain dictionaries
     if not requester_ip:
-        return request_data
-    key: Final = "litellm_metadata" if "litellm_metadata" in request_data else "metadata"
-    metadata: Final = request_data.get(key)
+        return request_data_with_spend_metadata
+    key: Final = "litellm_metadata" if "litellm_metadata" in request_data_with_spend_metadata else "metadata"
+    metadata: Final = request_data_with_spend_metadata.get(key)
     base: Final[Mapping[str, object]] = metadata if isinstance(metadata, Mapping) else EMPTY_MAPPING
     if base.get("requester_ip_address"):
-        return request_data
-    return {**request_data, key: {**base, "requester_ip_address": requester_ip}}  # mutable-ok: logging needs dicts
+        return request_data_with_spend_metadata
+    return {
+        **request_data_with_spend_metadata,
+        key: {**base, "requester_ip_address": requester_ip},
+    }  # mutable-ok: logging needs dicts
 
 
 class UserAPIKeyAuthExceptionHandler:
@@ -199,8 +220,17 @@ async def _handle_authentication_error(
                 )
 
             # Allow callbacks to transform the error response
+            from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
+
+            spend_logs_metadata: Final = LiteLLMProxyRequestSetup.get_spend_logs_metadata_from_request_headers(
+                _safe_get_request_headers(request)
+            )
             transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
-                request_data=_with_requester_ip_address(request_data, requester_ip),
+                request_data=_with_auth_failure_metadata(
+                    request_data=request_data,
+                    requester_ip=requester_ip,
+                    spend_logs_metadata=spend_logs_metadata,
+                ),
                 original_exception=e,
                 user_api_key_dict=user_api_key_dict,
                 error_type=ProxyErrorTypes.auth_error,

litellm/proxy/db/health_check_latest.py

@@ -74,10 +74,14 @@ def _assume_utc(cls, value: datetime) -> datetime:
 _ROWS_ADAPTER: Final = TypeAdapter(tuple[LatestHealthCheckRow, ...])
 
 
+async def query_latest_health_checks(prisma_client: PrismaClient) -> tuple[LatestHealthCheckRow, ...]:
+    rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_SQL)
+    return _ROWS_ADAPTER.validate_python(rows)
+
+
 async def fetch_latest_health_checks(prisma_client: PrismaClient) -> tuple[LatestHealthCheckRow, ...]:
     try:
-        rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_SQL)
-        return _ROWS_ADAPTER.validate_python(rows)
+        return await query_latest_health_checks(prisma_client)
     except Exception as query_err:  # noqa: BLE001  # health decorates other reads; a driver error must not fail them
         verbose_proxy_logger.error("Error getting all latest health checks: %s", query_err)
         return ()

litellm/proxy/dd_span_tagger.py

@@ -36,11 +36,15 @@ def tag_request(
         - ``litellm.key_alias``      — human-readable alias for the API key
         - ``litellm.key_hash``       — hashed API key (safe to log; never the raw secret)
         - ``litellm.requested_model``— model name as sent by the client
+        - ``litellm.user_email``     — email of the authenticated user
 
         Use cases:
         - Trace all requests from a specific user/key: filter by ``litellm.key_alias`` or
           ``litellm.key_hash``.
         - Trace all requests for a specific model: filter by ``litellm.requested_model``.
+        - Trace all requests from a specific person under JWT auth, where there is no key_alias:
+          filter by ``@litellm.user_email:"user@example.com"``. The value comes from the JWT claim
+          mapped by ``litellm_jwtauth.user_email_jwt_field``, or from the user row for virtual keys.
 
         Note: key_alias / key_hash are not available for unauthenticated (e.g. 401) requests.
         """
@@ -51,8 +55,10 @@ def tag_request(
                 set_active_span_tag("litellm.key_hash", str(user_api_key_dict.token))
             if requested_model:
                 set_active_span_tag("litellm.requested_model", str(requested_model))
+            if user_api_key_dict.user_email:
+                set_active_span_tag("litellm.user_email", str(user_api_key_dict.user_email))
         except Exception:
             verbose_proxy_logger.debug(
-                "Failed to tag active ddtrace span with key/model tags",
+                "Failed to tag active ddtrace span with key/model/user tags",
                 exc_info=True,
             )

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -45,7 +45,10 @@
 from litellm.proxy.auth.model_checks import get_key_models
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
-from litellm.proxy.db.health_check_latest import LatestHealthCheckRow
+from litellm.proxy.db.health_check_latest import (
+    LatestHealthCheckRow,
+    query_latest_health_checks,
+)
 from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers
 from litellm.proxy.health_check import (
     ADMIN_ONLY_HEALTH_DISPLAY_PARAMS,
@@ -876,7 +879,7 @@ async def _save_background_health_checks_to_db(
         )
 
         # Step 3: Get latest health checks for all models in one query to compare status
-        latest_checks: Final = await prisma_client.get_all_latest_health_checks()
+        latest_checks: Final = await query_latest_health_checks(prisma_client)
         latest_checks_map: Final = {}
         for check in latest_checks:
             # Use model_id as primary key, fallback to model_name

litellm/proxy/litellm_pre_call_utils.py

@@ -90,6 +90,7 @@
 _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
 
 _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
+_SPEND_LOGS_METADATA_ADAPTER: Final = TypeAdapter(dict[str, object])
 
 # W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/
 # e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
@@ -1256,16 +1257,19 @@ def _get_num_retries_from_request(headers: dict) -> int | None:
         return None
 
     @staticmethod
-    def _get_spend_logs_metadata_from_request_headers(headers: dict) -> dict | None:
+    def get_spend_logs_metadata_from_request_headers(
+        headers: Mapping[str, object],
+    ) -> dict[str, object] | None:  # mutable-ok: request metadata is stored in a TypedDict
         """
         Get the `spend_logs_metadata` from the request headers.
         """
-        from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-
         spend_logs_metadata_header: Final = headers.get("x-litellm-spend-logs-metadata", None)
-        if spend_logs_metadata_header is not None:
-            return safe_json_loads(spend_logs_metadata_header)
-        return None
+        if not isinstance(spend_logs_metadata_header, str):
+            return None
+        try:
+            return _SPEND_LOGS_METADATA_ADAPTER.validate_json(spend_logs_metadata_header)
+        except PydanticValidationError:
+            return None
 
     @staticmethod
     def _get_forwardable_headers(
@@ -1497,7 +1501,7 @@ def add_litellm_metadata_from_request_headers(
         from litellm.proxy._types import LitellmMetadataFromRequestHeaders
 
         metadata_from_headers: Final = LitellmMetadataFromRequestHeaders()
-        spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers)
+        spend_logs_metadata: Final = LiteLLMProxyRequestSetup.get_spend_logs_metadata_from_request_headers(headers)
         if spend_logs_metadata is not None:
             metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata
 

litellm/router.py

@@ -10683,6 +10683,10 @@ def _set_model_group_info(self, model_group: str, user_facing_model_group_name:
         model_list: Final = self.get_model_list(model_name=model_group)
         if model_list is None:
             return None
+        model_group_description: Final[str | None] = next(
+            (d for d in (m.get("model_info", {}).get("description") for m in model_list) if isinstance(d, str)),
+            None,
+        )
         for model in model_list:
             is_match = False
             if (
@@ -10787,6 +10791,7 @@ def _set_model_group_info(self, model_group: str, user_facing_model_group_name:
                     **{
                         "model_group": user_facing_model_group_name,
                         "providers": [llm_provider],
+                        "description": model_group_description,
                         **model_info,
                         "supported_reasoning_efforts": None,
                     }

litellm/types/router.py

@@ -721,6 +721,7 @@ class ModelGroupInfo(BaseModel):
     supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None)
     supported_openai_params: list[str] | None = Field(default=[])
     configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
+    description: str | None = None
 
     def __init__(self, **data) -> None:
         for field_name, field_type in _resolved_annotations(self.__class__).items():

tests/test_litellm/proxy/auth/test_auth_exception_handler.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+from copy import deepcopy
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import httpx
@@ -620,6 +621,94 @@ def _http_request(client_host: str | None = "10.1.2.3", headers: dict[str, str]
     )
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "header_value, has_body_metadata, expected_spend_logs_metadata",
+    [
+        pytest.param(
+            json.dumps({"my_request_id": "req_rejected_2"}),
+            True,
+            {"my_request_id": "req_rejected_2"},
+            id="valid_header_overrides_body",
+        ),
+        pytest.param(
+            json.dumps({"my_request_id": "req_rejected_2"}),
+            False,
+            {"my_request_id": "req_rejected_2"},
+            id="valid_header_without_body_metadata",
+        ),
+        pytest.param(
+            "{invalid-json",
+            True,
+            {"source": "request-body"},
+            id="invalid_header_keeps_body",
+        ),
+        pytest.param(
+            "null",
+            True,
+            {"source": "request-body"},
+            id="null_header_keeps_body",
+        ),
+        pytest.param(
+            None,
+            True,
+            {"source": "request-body"},
+            id="missing_headers_scope_keeps_body",
+        ),
+    ],
+)
+async def test_budget_auth_failure_logs_spend_metadata_from_request_header(
+    header_value: str | None,
+    has_body_metadata: bool,
+    expected_spend_logs_metadata: dict[str, str],
+) -> None:
+    request_data: dict[str, object] = {"model": "gpt-4o"}
+    if has_body_metadata:
+        request_data["metadata"] = {
+            "existing": "keep-me",
+            "spend_logs_metadata": {"source": "request-body"},
+        }
+    original_request_data = deepcopy(request_data)
+    request = _http_request(
+        headers={"x-litellm-spend-logs-metadata": header_value} if header_value is not None else None
+    )
+    if header_value is None:
+        request.scope.pop("headers")
+
+    with (
+        patch(  # test-quality-ok: identity seeding is outside the failure-log payload contract
+            "litellm.proxy.auth.auth_exception_handler.seed_request_identity"
+        ),
+        patch(  # test-quality-ok: capture the exact payload sent to the spend logger
+            "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook",
+            new_callable=AsyncMock,
+            return_value=None,
+        ) as mock_hook,
+        patch(  # test-quality-ok: disable the unrelated database-outage fallback
+            "litellm.proxy.proxy_server.general_settings",
+            {"allow_requests_on_db_unavailable": False},
+        ),
+    ):
+        with pytest.raises(ProxyException) as exc_info:
+            await UserAPIKeyAuthExceptionHandler._handle_authentication_error(
+                BudgetExceededError(message="Budget exceeded", current_cost=100, max_budget=100),
+                request,
+                request_data,
+                "/v1/chat/completions",
+                None,
+                "sk-over-budget",
+            )
+
+    assert exc_info.value.code == str(status.HTTP_429_TOO_MANY_REQUESTS)
+    assert exc_info.value.type == ProxyErrorTypes.budget_exceeded
+    logged_request_data = mock_hook.call_args.kwargs["request_data"]
+    assert logged_request_data["metadata"]["spend_logs_metadata"] == expected_spend_logs_metadata
+    assert logged_request_data["metadata"]["requester_ip_address"] == "10.1.2.3"
+    if has_body_metadata:
+        assert logged_request_data["metadata"]["existing"] == "keep-me"
+    assert request_data == original_request_data
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "auth_error, general_settings, request_kwargs, expected_ip",

tests/test_litellm/proxy/db/test_health_check_latest.py

@@ -8,6 +8,7 @@
     LATEST_HEALTH_CHECKS_SQL,
     fetch_latest_health_checks,
     fetch_latest_health_checks_for_models,
+    query_latest_health_checks,
 )
 
 
@@ -83,6 +84,15 @@ async def test_fetch_all_degrades_to_no_rows_when_the_query_fails():
     assert await fetch_latest_health_checks(prisma) == ()
 
 
+@pytest.mark.asyncio
+async def test_query_all_raises_when_the_query_fails_instead_of_reading_as_an_empty_table():
+    """The background save decides what to write from this read; a failure has to be told apart from no rows."""
+    prisma = _prisma([])
+    prisma.db.query_raw.side_effect = RuntimeError("db down")
+    with pytest.raises(RuntimeError, match="db down"):
+        await query_latest_health_checks(prisma)
+
+
 @pytest.mark.asyncio
 async def test_fetch_all_degrades_to_no_rows_for_a_malformed_row():
     assert await fetch_latest_health_checks(_prisma([{"unexpected": "shape"}])) == ()

tests/test_litellm/proxy/test_common_request_processing.py

@@ -3177,12 +3177,13 @@ def test_streaming_overhead_header_in_custom_headers_from_stream_hidden_params(
 class TestDDSpanTaggerTagRequest:
     """Tests for DDSpanTagger.tag_request - key/model DD span tagging."""
 
-    def _make_user_api_key_dict(self, key_alias=None, token=None):
+    def _make_user_api_key_dict(self, key_alias=None, token=None, user_email=None):
         from litellm.proxy._types import UserAPIKeyAuth
 
         d = UserAPIKeyAuth()
         d.key_alias = key_alias
         d.token = token
+        d.user_email = user_email
         return d
 
     def test_tags_key_alias_and_model(self):
@@ -3223,6 +3224,30 @@ def test_only_model_tagged_when_no_key_info(self):
 
         mock_set_tag.assert_called_once_with("litellm.requested_model", "claude-3-5-sonnet")
 
+    def test_tags_user_email(self):
+        """user_email is tagged so JWT-authenticated requests are traceable per person."""
+        user_key = self._make_user_api_key_dict(user_email="user@example.com")
+
+        with patch("litellm.proxy.dd_span_tagger.set_active_span_tag") as mock_set_tag:
+            DDSpanTagger.tag_request(
+                user_api_key_dict=user_key,
+                requested_model=None,
+            )
+
+        mock_set_tag.assert_called_once_with("litellm.user_email", "user@example.com")
+
+    def test_no_user_email_tag_when_absent(self):
+        """No user email tag when the authenticated identity has no email."""
+        user_key = self._make_user_api_key_dict(key_alias="my-prod-key", user_email=None)
+
+        with patch("litellm.proxy.dd_span_tagger.set_active_span_tag") as mock_set_tag:
+            DDSpanTagger.tag_request(
+                user_api_key_dict=user_key,
+                requested_model="gpt-4o",
+            )
+
+        assert all(call.args[0] != "litellm.user_email" for call in mock_set_tag.call_args_list)
+
 
 class TestHasAttributeErrorInChain:
     """Tests for _has_attribute_error_in_chain helper."""