fix: proxy failure cases, now log ip and user agent, key hash, name

litellm/integrations/prometheus.py

@@ -1478,35 +1478,19 @@ def _is_invalid_api_key_request(
         """
         Determine if a request has an invalid API key based on status code and exception.
 
-        This method prevents invalid authentication attempts from being recorded in
-        Prometheus metrics. A 401 status code is the definitive indicator of authentication
-        failure. Additionally, we check exception messages for authentication error patterns
-        to catch cases where the exception hasn't been converted to a ProxyException yet.
+        Returns True only when we truly cannot record useful metrics (e.g. missing required
+        data). We no longer skip 401/invalid-key requests - all requests including
+        authentication failures and bad requests must be tracked for debugging, security
+        auditing, abuse detection, and capacity planning.
 
         Args:
-            status_code: HTTP status code (401 indicates authentication error)
-            exception: Exception object to check for auth-related error messages
+            status_code: HTTP status code
+            exception: Exception object (unused, kept for API compatibility)
 
         Returns:
-            True if the request has an invalid API key and metrics should be skipped,
+            True if metrics should be skipped (currently always False - track all requests),
             False otherwise
         """
-        if status_code == 401:
-            return True
-
-        # Handle cases where AssertionError is raised before conversion to ProxyException
-        if exception is not None:
-            exception_str = str(exception).lower()
-            auth_error_patterns = [
-                "virtual key expected",
-                "expected to start with 'sk-'",
-                "authentication error",
-                "invalid api key",
-                "api key not valid",
-            ]
-            if any(pattern in exception_str for pattern in auth_error_patterns):
-                return True
-
         return False
 
     def _should_skip_metrics_for_invalid_key(

litellm/proxy/auth/auth_checks.py

@@ -128,7 +128,7 @@ async def common_checks(
                 message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}",
                 type=ProxyErrorTypes.team_model_access_denied,
                 param="model",
-                code=status.HTTP_401_UNAUTHORIZED,
+                code=status.HTTP_400_BAD_REQUEST,
             )
 
     ## 2.1 If user can call model (if personal key)
@@ -1983,7 +1983,7 @@ def _can_object_call_model(
             object_type=object_type
         ),
         param="model",
-        code=status.HTTP_401_UNAUTHORIZED,
+        code=status.HTTP_400_BAD_REQUEST,
     )
 
 
@@ -2084,7 +2084,7 @@ async def can_user_call_model(
             message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}",
             type=ProxyErrorTypes.key_model_access_denied,
             param="model",
-            code=status.HTTP_401_UNAUTHORIZED,
+            code=status.HTTP_400_BAD_REQUEST,
         )
 
     return _can_object_call_model(
@@ -2666,7 +2666,7 @@ def _can_object_call_vector_stores(
                     object_type
                 ),
                 param="vector_store",
-                code=status.HTTP_401_UNAUTHORIZED,
+                code=status.HTTP_400_BAD_REQUEST,
             )
 
     return True

litellm/proxy/auth/auth_exception_handler.py

@@ -9,7 +9,10 @@
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
-from litellm.proxy.auth.auth_utils import _get_request_ip_address
+from litellm.proxy.auth.auth_utils import (
+    _get_request_ip_address,
+    add_client_context_to_request_data,
+)
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 from litellm.types.services import ServiceTypes
 
@@ -30,6 +33,7 @@ async def _handle_authentication_error(
         route: str,
         parent_otel_span: Optional[Span],
         api_key: str,
+        valid_token: Optional[UserAPIKeyAuth] = None,
     ) -> UserAPIKeyAuth:
         """
         Handles Connection Errors when reading a Virtual Key from LiteLLM DB
@@ -71,30 +75,63 @@ async def _handle_authentication_error(
             )
         else:
             # raise the exception to the caller
+            use_x_forwarded_for = general_settings.get("use_x_forwarded_for", False)
             requester_ip = _get_request_ip_address(
                 request=request,
-                use_x_forwarded_for=general_settings.get("use_x_forwarded_for", False),
+                use_x_forwarded_for=use_x_forwarded_for,
+            )
+            user_agent = request.headers.get("user-agent", "") if request else ""
+
+            # Ensure request_data has client context for callbacks (Prometheus, etc.)
+            add_client_context_to_request_data(
+                request=request,
+                request_data=request_data,
+                use_x_forwarded_for=use_x_forwarded_for,
+            )
+
+            key_name = (
+                valid_token.key_alias or getattr(valid_token, "key_name", None)
+                if valid_token
+                else "<unknown-key>"
             )
             verbose_proxy_logger.exception(
-                "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {}\nRequester IP Address:{}".format(
+                "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {}\nRequester IP Address:{}\nUser-Agent:{}\nKey Hash:{}\nKey Name:{}".format(
                     str(e),
-                    requester_ip,
+                    requester_ip or "<unknown>",
+                    user_agent or "<unknown>",
+                    api_key or "<unknown>",
+                    key_name,
                 ),
-                extra={"requester_ip": requester_ip},
+                extra={
+                    "requester_ip": requester_ip,
+                    "user_agent": user_agent,
+                    "key_hash": api_key,
+                    "key_name": key_name,
+                },
             )
 
-            # Log this exception to OTEL, Datadog etc
-            user_api_key_dict = UserAPIKeyAuth(
-                parent_otel_span=parent_otel_span,
-                api_key=api_key,
-                request_route=route,
-            )
+            # Log this exception to OTEL, Datadog etc - use valid_token when available (e.g. model access denied)
+            if valid_token is not None:
+                user_api_key_dict = valid_token
+            else:
+                user_api_key_dict = UserAPIKeyAuth(
+                    parent_otel_span=parent_otel_span,
+                    api_key=api_key,
+                    request_route=route,
+                    key_alias="<unknown-key>",
+                )
             # Allow callbacks to transform the error response
+            error_type = ProxyErrorTypes.auth_error
+            if isinstance(e, ProxyException) and hasattr(e, "type"):
+                try:
+                    error_type = ProxyErrorTypes(e.type)
+                except (ValueError, TypeError):
+                    pass
             transformed_exception = await proxy_logging_obj.post_call_failure_hook(
                 request_data=request_data,
                 original_exception=e,
                 user_api_key_dict=user_api_key_dict,
-                error_type=ProxyErrorTypes.auth_error,
+                error_type=error_type,
                 route=route,
             )
             # Use transformed exception if callback returned one, otherwise use original

litellm/proxy/auth/auth_utils.py

@@ -26,6 +26,28 @@ def _get_request_ip_address(
     return client_ip
 
 
+def add_client_context_to_request_data(
+    request: Request, request_data: dict, use_x_forwarded_for: bool = False
+) -> None:
+    """
+    Add client_ip (requester_ip_address) and User-Agent to request_data metadata early.
+    Call this at the start of the request pipeline so failures have client context for logging.
+    """
+    if "metadata" not in request_data:
+        request_data["metadata"] = {}
+    metadata = request_data["metadata"]
+
+    requester_ip = _get_request_ip_address(
+        request=request, use_x_forwarded_for=use_x_forwarded_for
+    )
+    metadata["requester_ip_address"] = requester_ip or ""
+
+    user_agent = ""
+    if hasattr(request, "headers") and "user-agent" in request.headers:
+        user_agent = request.headers.get("user-agent", "")
+    metadata["user_agent"] = user_agent
+
+
 def _check_valid_ip(
     allowed_ips: Optional[List[str]],
     request: Request,
@@ -314,17 +336,17 @@ def get_request_route(request: Request) -> str:
 def normalize_request_route(route: str) -> str:
     """
     Normalize request routes by replacing dynamic path parameters with placeholders.
-    
+
     This prevents high cardinality in Prometheus metrics by collapsing routes like:
     - /v1/responses/1234567890 -> /v1/responses/{response_id}
     - /v1/threads/thread_123 -> /v1/threads/{thread_id}
-    
+
     Args:
         route: The request route path
-        
+
     Returns:
         Normalized route with dynamic parameters replaced by placeholders
-        
+
     Examples:
         >>> normalize_request_route("/v1/responses/abc123")
         '/v1/responses/{response_id}'
@@ -337,58 +359,90 @@ def normalize_request_route(route: str) -> str:
     # Format: (regex_pattern, replacement_template)
     patterns = [
         # Responses API - must come before generic patterns
-        (r'^(/(?:openai/)?v1/responses)/([^/]+)(/input_items)$', r'\1/{response_id}\3'),
-        (r'^(/(?:openai/)?v1/responses)/([^/]+)(/cancel)$', r'\1/{response_id}\3'),
-        (r'^(/(?:openai/)?v1/responses)/([^/]+)$', r'\1/{response_id}'),
-        (r'^(/responses)/([^/]+)(/input_items)$', r'\1/{response_id}\3'),
-        (r'^(/responses)/([^/]+)(/cancel)$', r'\1/{response_id}\3'),
-        (r'^(/responses)/([^/]+)$', r'\1/{response_id}'),
-
+        (r"^(/(?:openai/)?v1/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"),
+        (r"^(/(?:openai/)?v1/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"),
+        (r"^(/(?:openai/)?v1/responses)/([^/]+)$", r"\1/{response_id}"),
+        (r"^(/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"),
+        (r"^(/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"),
+        (r"^(/responses)/([^/]+)$", r"\1/{response_id}"),
         # Threads API
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)/([^/]+)$', r'\1/{thread_id}\3/{run_id}\5/{step_id}'),
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)$', r'\1/{thread_id}\3/{run_id}\5'),
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/cancel)$', r'\1/{thread_id}\3/{run_id}\5'),
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/submit_tool_outputs)$', r'\1/{thread_id}\3/{run_id}\5'),
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)$', r'\1/{thread_id}\3/{run_id}'),
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)$', r'\1/{thread_id}\3'),
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/messages)/([^/]+)$', r'\1/{thread_id}\3/{message_id}'),
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/messages)$', r'\1/{thread_id}\3'),
-        (r'^(/(?:openai/)?v1/threads)/([^/]+)$', r'\1/{thread_id}'),
-
+        (
+            r"^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)/([^/]+)$",
+            r"\1/{thread_id}\3/{run_id}\5/{step_id}",
+        ),
+        (
+            r"^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)$",
+            r"\1/{thread_id}\3/{run_id}\5",
+        ),
+        (
+            r"^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/cancel)$",
+            r"\1/{thread_id}\3/{run_id}\5",
+        ),
+        (
+            r"^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/submit_tool_outputs)$",
+            r"\1/{thread_id}\3/{run_id}\5",
+        ),
+        (
+            r"^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)$",
+            r"\1/{thread_id}\3/{run_id}",
+        ),
+        (r"^(/(?:openai/)?v1/threads)/([^/]+)(/runs)$", r"\1/{thread_id}\3"),
+        (
+            r"^(/(?:openai/)?v1/threads)/([^/]+)(/messages)/([^/]+)$",
+            r"\1/{thread_id}\3/{message_id}",
+        ),
+        (r"^(/(?:openai/)?v1/threads)/([^/]+)(/messages)$", r"\1/{thread_id}\3"),
+        (r"^(/(?:openai/)?v1/threads)/([^/]+)$", r"\1/{thread_id}"),
         # Vector Stores API
-        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/files)/([^/]+)$', r'\1/{vector_store_id}\3/{file_id}'),
-        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/files)$', r'\1/{vector_store_id}\3'),
-        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/file_batches)/([^/]+)$', r'\1/{vector_store_id}\3/{batch_id}'),
-        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/file_batches)$', r'\1/{vector_store_id}\3'),
-        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)$', r'\1/{vector_store_id}'),
-
+        (
+            r"^(/(?:openai/)?v1/vector_stores)/([^/]+)(/files)/([^/]+)$",
+            r"\1/{vector_store_id}\3/{file_id}",
+        ),
+        (
+            r"^(/(?:openai/)?v1/vector_stores)/([^/]+)(/files)$",
+            r"\1/{vector_store_id}\3",
+        ),
+        (
+            r"^(/(?:openai/)?v1/vector_stores)/([^/]+)(/file_batches)/([^/]+)$",
+            r"\1/{vector_store_id}\3/{batch_id}",
+        ),
+        (
+            r"^(/(?:openai/)?v1/vector_stores)/([^/]+)(/file_batches)$",
+            r"\1/{vector_store_id}\3",
+        ),
+        (r"^(/(?:openai/)?v1/vector_stores)/([^/]+)$", r"\1/{vector_store_id}"),
         # Assistants API
-        (r'^(/(?:openai/)?v1/assistants)/([^/]+)$', r'\1/{assistant_id}'),
-
+        (r"^(/(?:openai/)?v1/assistants)/([^/]+)$", r"\1/{assistant_id}"),
         # Files API
-        (r'^(/(?:openai/)?v1/files)/([^/]+)(/content)$', r'\1/{file_id}\3'),
-        (r'^(/(?:openai/)?v1/files)/([^/]+)$', r'\1/{file_id}'),
-
+        (r"^(/(?:openai/)?v1/files)/([^/]+)(/content)$", r"\1/{file_id}\3"),
+        (r"^(/(?:openai/)?v1/files)/([^/]+)$", r"\1/{file_id}"),
         # Batches API
-        (r'^(/(?:openai/)?v1/batches)/([^/]+)(/cancel)$', r'\1/{batch_id}\3'),
-        (r'^(/(?:openai/)?v1/batches)/([^/]+)$', r'\1/{batch_id}'),
-
+        (r"^(/(?:openai/)?v1/batches)/([^/]+)(/cancel)$", r"\1/{batch_id}\3"),
+        (r"^(/(?:openai/)?v1/batches)/([^/]+)$", r"\1/{batch_id}"),
         # Fine-tuning API
-        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/events)$', r'\1/{fine_tuning_job_id}\3'),
-        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/cancel)$', r'\1/{fine_tuning_job_id}\3'),
-        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/checkpoints)$', r'\1/{fine_tuning_job_id}\3'),
-        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)$', r'\1/{fine_tuning_job_id}'),
-
+        (
+            r"^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/events)$",
+            r"\1/{fine_tuning_job_id}\3",
+        ),
+        (
+            r"^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/cancel)$",
+            r"\1/{fine_tuning_job_id}\3",
+        ),
+        (
+            r"^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/checkpoints)$",
+            r"\1/{fine_tuning_job_id}\3",
+        ),
+        (r"^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)$", r"\1/{fine_tuning_job_id}"),
         # Models API
-        (r'^(/(?:openai/)?v1/models)/([^/]+)$', r'\1/{model}'),
+        (r"^(/(?:openai/)?v1/models)/([^/]+)$", r"\1/{model}"),
     ]
-    
+
     # Apply patterns in order
     for pattern, replacement in patterns:
         normalized = re.sub(pattern, replacement, route)
         if normalized != route:
             return normalized
-    
+
     # Return original route if no pattern matched
     return route
 
@@ -644,6 +698,7 @@ def get_customer_user_header_from_mapping(user_id_mapping) -> Optional[str]:
             return header_name
     return None
 
+
 def _get_customer_id_from_standard_headers(
     request_headers: Optional[dict],
 ) -> Optional[str]:
@@ -679,7 +734,9 @@ def get_end_user_id_from_request_body(
     from litellm.proxy.proxy_server import general_settings
 
     # Check 1: Standard customer ID headers (always checked, no configuration required)
-    customer_id = _get_customer_id_from_standard_headers(request_headers=request_headers)
+    customer_id = _get_customer_id_from_standard_headers(
+        request_headers=request_headers
+    )
     if customer_id is not None:
         return customer_id