fix(spend-tracking): keep batch spend keys joinable after v1.99 provenance gate

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -161,6 +161,7 @@ async def _build_creator_attribution_metadata(
         metadata: dict[str, object] = {
             "user_api_key_user_id": job.created_by,
             "user_api_key": api_key,
+            "user_api_key_hash": api_key,
             "user_api_key_team_id": team_id,
             **(await self._get_user_info(batch_id, job.created_by)),
         }

litellm/integrations/cloudzero/database.py

@@ -94,8 +94,18 @@ async def get_usage_data(
 
         try:
             db_response: Final = await client.db.query_raw(query, *params)
-            # Convert the response to polars DataFrame with full schema inference
-            # This prevents schema mismatch errors when data types vary across rows
-            return pl.DataFrame(db_response, infer_schema_length=None)
+            from litellm.proxy.spend_tracking.key_metadata_recovery import (
+                fill_missing_api_key_aliases,
+            )
+
+            usage_rows: Final = (
+                db_response.to_dicts()
+                if isinstance(db_response, pl.DataFrame)
+                else db_response
+                if isinstance(db_response, list)
+                else []
+            )
+            recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows)
+            return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None)
         except Exception as e:
             raise Exception(f"Error retrieving usage data: {e}")

litellm/integrations/focus/database.py

@@ -96,7 +96,19 @@ async def get_usage_data(
 
         try:
             db_response: Final = await client.db.query_raw(query, *query_params)
-            return pl.DataFrame(db_response, infer_schema_length=None)
+            from litellm.proxy.spend_tracking.key_metadata_recovery import (
+                fill_missing_api_key_aliases,
+            )
+
+            usage_rows: Final = (
+                db_response.to_dicts()
+                if isinstance(db_response, pl.DataFrame)
+                else db_response
+                if isinstance(db_response, list)
+                else []
+            )
+            recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows)
+            return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None)
         except Exception as exc:
             raise RuntimeError(f"Error retrieving usage data: {exc}") from exc
 

litellm/proxy/_lazy_openapi_snapshot.json

@@ -3151,6 +3151,17 @@
                 }
               ],
               "title": "Team Id"
+            },
+            "user_email": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "User Email"
             }
           },
           "title": "KeyMetadata",

litellm/proxy/logging_endpoints/callback_logs_endpoints.py

@@ -88,8 +88,10 @@ def _build_logging_obj(payload: dict[str, Any]) -> LiteLLMLogging:
         )
 
         metadata: Final[dict[str, Any]] = payload.get("metadata") or {}
+        user_api_key_hash: Final = metadata.get("user_api_key_hash")
         litellm_metadata: Final[dict[str, Any]] = {
-            "user_api_key": metadata.get("user_api_key_hash"),
+            "user_api_key": user_api_key_hash,
+            "user_api_key_hash": user_api_key_hash,
             "user_api_key_alias": metadata.get("user_api_key_alias"),
             "user_api_key_user_id": metadata.get("user_api_key_user_id"),
             "user_api_key_team_id": metadata.get("user_api_key_team_id"),

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -2,15 +2,19 @@
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from collections.abc import Set as AbstractSet
 from datetime import datetime, timedelta, timezone
-from types import SimpleNamespace
+from types import MappingProxyType, SimpleNamespace
 from typing import TYPE_CHECKING, Final, Protocol
 
 from fastapi import HTTPException, status
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import PTU_SENTINEL_API_KEY
 from litellm.proxy._types import CommonProxyErrors
+from litellm.proxy.spend_tracking.key_metadata_recovery import (
+    attach_user_emails,
+    recover_double_hashed_key_metadata,
+)
 from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled
 from litellm.proxy.utils import PrismaClient
 from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
@@ -111,8 +115,19 @@ def failed_requests(self) -> int: ...
 
 
 class _KeyMetadataDict(TypedDict, total=False):
-    key_alias: str | None
-    team_id: str | None
+    key_alias: ReadOnly[str | None]
+    team_id: ReadOnly[str | None]
+    user_id: ReadOnly[str | None]
+    user_email: ReadOnly[str | None]
+
+
+def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata:
+    meta: Final = api_key_metadata.get(api_key, {})
+    return KeyMetadata(
+        key_alias=meta.get("key_alias"),
+        team_id=meta.get("team_id"),
+        user_email=meta.get("user_email"),
+    )
 
 
 _WhereValue = str | dict[str, object]
@@ -283,10 +298,7 @@ def update_breakdown_metrics(
             if record.api_key not in breakdown.models[model_key].api_key_breakdown:
                 breakdown.models[model_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
                     metrics=SpendMetrics(),
-                    metadata=KeyMetadata(
-                        key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
-                        team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
-                    ),
+                    metadata=_key_metadata(api_key_metadata, record.api_key),
                 )
             breakdown.models[model_key].api_key_breakdown[record.api_key].metrics = update_metrics(
                 breakdown.models[model_key].api_key_breakdown[record.api_key].metrics,
@@ -310,10 +322,7 @@ def update_breakdown_metrics(
             if record.api_key not in breakdown.model_groups[model_group_key].api_key_breakdown:
                 breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
                     metrics=SpendMetrics(),
-                    metadata=KeyMetadata(
-                        key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
-                        team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
-                    ),
+                    metadata=_key_metadata(api_key_metadata, record.api_key),
                 )
             breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics = update_metrics(
                 breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics,
@@ -335,10 +344,7 @@ def update_breakdown_metrics(
             breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key] = (
                 KeyMetricWithMetadata(
                     metrics=SpendMetrics(),
-                    metadata=KeyMetadata(
-                        key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
-                        team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
-                    ),
+                    metadata=_key_metadata(api_key_metadata, record.api_key),
                 )
             )
 
@@ -363,10 +369,7 @@ def update_breakdown_metrics(
         if record.api_key not in breakdown.providers[provider].api_key_breakdown:
             breakdown.providers[provider].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
                 metrics=SpendMetrics(),
-                metadata=KeyMetadata(
-                    key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
-                    team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
-                ),
+                metadata=_key_metadata(api_key_metadata, record.api_key),
             )
         breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = update_metrics(
             breakdown.providers[provider].api_key_breakdown[record.api_key].metrics,
@@ -388,10 +391,7 @@ def update_breakdown_metrics(
         if record.api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown:
             breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
                 metrics=SpendMetrics(),
-                metadata=KeyMetadata(
-                    key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
-                    team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
-                ),
+                metadata=_key_metadata(api_key_metadata, record.api_key),
             )
         breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics = update_metrics(
             breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics,
@@ -403,10 +403,7 @@ def update_breakdown_metrics(
         if record.api_key not in breakdown.api_keys:
             breakdown.api_keys[record.api_key] = KeyMetricWithMetadata(
                 metrics=SpendMetrics(),
-                metadata=KeyMetadata(
-                    key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
-                    team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
-                ),  # Add any api_key-specific metadata here
+                metadata=_key_metadata(api_key_metadata, record.api_key),
             )
         breakdown.api_keys[record.api_key].metrics = update_metrics(breakdown.api_keys[record.api_key].metrics, record)
 
@@ -426,10 +423,7 @@ def update_breakdown_metrics(
             if record.api_key not in breakdown.entities[entity_value].api_key_breakdown:
                 breakdown.entities[entity_value].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
                     metrics=SpendMetrics(),
-                    metadata=KeyMetadata(
-                        key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
-                        team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
-                    ),
+                    metadata=_key_metadata(api_key_metadata, record.api_key),
                 )
             breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = update_metrics(
                 breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics,
@@ -442,17 +436,23 @@ def update_breakdown_metrics(
 async def get_api_key_metadata(
     prisma_client: PrismaClient,
     api_keys: AbstractSet[str],
-) -> dict[str, _KeyMetadataDict]:
+) -> Mapping[str, _KeyMetadataDict]:
     """Get api key metadata, falling back to deleted keys table for keys not found in active table.
 
     This ensures that key_alias and team_id are preserved in historical activity logs
-    even after a key is deleted or regenerated.
+    even after a key is deleted or regenerated. Also recovers aliases for api_key
+    values that were double-hashed by the v1.99 spend-log provenance gate.
     """
     key_records: Sequence[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many(
         where={"token": {"in": list(api_keys)}}
     )
     result: Final[dict[str, _KeyMetadataDict]] = {
-        k.token: {"key_alias": k.key_alias, "team_id": k.team_id} for k in key_records
+        k.token: {
+            "key_alias": k.key_alias,
+            "team_id": k.team_id,
+            "user_id": getattr(k, "user_id", None),
+        }
+        for k in key_records
     }
 
     # For any keys not found in the active table, check the deleted keys table
@@ -471,6 +471,7 @@ async def get_api_key_metadata(
                     result[k.token] = {
                         "key_alias": k.key_alias,
                         "team_id": k.team_id,
+                        "user_id": getattr(k, "user_id", None),
                     }
         except Exception as e:
             verbose_proxy_logger.warning(
@@ -479,7 +480,13 @@ async def get_api_key_metadata(
                 e,
             )
 
-    return result
+    still_missing: Final = api_keys - frozenset(result)
+    combined: Final = (
+        result
+        if not still_missing
+        else MappingProxyType({**result, **(await recover_double_hashed_key_metadata(prisma_client, still_missing))})
+    )
+    return await attach_user_emails(prisma_client, combined)
 
 
 def _adjust_dates_for_timezone(
@@ -951,11 +958,6 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
     )
 
 
-def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata:
-    meta: Final = api_key_metadata.get(api_key, {})
-    return KeyMetadata(key_alias=meta.get("key_alias"), team_id=meta.get("team_id"))
-
-
 def _aggregate_grouping_sets_records_sync(
     *,
     records: Sequence[_GroupingSetsRow],

litellm/proxy/proxy_server.py

@@ -15207,6 +15207,7 @@ async def async_queue_request(
             # extra_body); see above for the same guard upstream.
             data["metadata"] = {}
         data["metadata"]["user_api_key"] = user_api_key_dict.api_key
+        data["metadata"]["user_api_key_hash"] = user_api_key_dict.api_key
         data["metadata"]["user_api_key_metadata"] = strip_callback_config(user_api_key_dict.metadata)
         _headers: Final = _safe_get_request_headers(request).copy()
         _headers.pop("authorization", None)  # do not store the original `sk-..` api key in the db