fix(router): label router-side rate-limit errors as litellm_rate_limit (was silently vendor)

litellm/__init__.py

@@ -1255,6 +1255,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
     NotFoundError,
     PermissionDeniedError,
     RateLimitError,
+    RateLimitErrorCategory,
+    RateLimitType,
     ServiceUnavailableError,
     BadGatewayError,
     OpenAIError,

litellm/exceptions.py

@@ -9,13 +9,86 @@
 
 ## LiteLLM versions of the OpenAI Exception Types
 
-from typing import Any, Dict, Optional
+import enum
+from typing import Any, Dict, Optional, Union
 
 import httpx
 import openai
 
 from litellm.types.utils import LiteLLMCommonStrings
 
+
+class RateLimitErrorCategory(str, enum.Enum):
+    """
+    Category of a rate limit error, allowing callers to distinguish where the rate
+    limit originated. Exposed on every :class:`RateLimitError` instance via the
+    ``category`` attribute.
+
+    Use these values to switch on the rate limit source, e.g.::
+
+        try:
+            ...
+        except litellm.RateLimitError as e:
+            if e.category == RateLimitErrorCategory.LITELLM_RATE_LIMIT:
+                ...  # litellm's own limiter (key/team/user/model RPM/TPM/budget)
+            elif e.category == RateLimitErrorCategory.VENDOR_RATE_LIMIT:
+                ...  # the upstream LLM provider returned 429
+    """
+
+    VENDOR_RATE_LIMIT = "vendor_rate_limit"
+    """The upstream LLM provider returned a rate-limit response (e.g. OpenAI 429)."""
+
+    VENDOR_BATCH_RATE_LIMIT = "vendor_batch_rate_limit"
+    """The upstream LLM provider returned a rate-limit response on a batch endpoint."""
+
+    LITELLM_RATE_LIMIT = "litellm_rate_limit"
+    """LiteLLM's own rate limiter (key/team/user/model RPM/TPM, budget, parallel-requests, etc.) blocked the request."""
+
+    LITELLM_BATCH_RATE_LIMIT = "litellm_batch_rate_limit"
+    """LiteLLM's own batch rate limiter (token/request budget across a batch input file) blocked the request."""
+
+    UNKNOWN_RATE_LIMIT = "unknown_rate_limit"
+    """Default for callers that did not explicitly classify the rate-limit source.
+
+    New code SHOULD pass an explicit category; this value exists so silent
+    miscategorization (vs. the previous ``VENDOR_RATE_LIMIT`` default) becomes
+    visible in dashboards rather than a confidently-wrong label.
+    """
+
+
+class RateLimitType(str, enum.Enum):
+    """
+    The dimension that was exceeded when a rate-limit error fired.
+
+    This is orthogonal to :class:`RateLimitErrorCategory` — *category* tells
+    callers **who** rate-limited the request (the upstream vendor vs. one of
+    litellm's own limiters), while *type* tells them **which limit dimension**
+    was exceeded (an RPM ceiling, a TPM ceiling, a max-parallel-requests
+    ceiling, a budget cap, or a max-iterations cap).
+
+    Surfaced both on every :class:`RateLimitError` instance via the
+    ``rate_limit_type`` attribute and on the structured
+    ``StandardLoggingPayload.error_information.error_rate_limit_type`` field
+    so custom callbacks / metrics consumers can split rate-limit failures by
+    cause without parsing free-text error messages.
+    """
+
+    REQUESTS = "requests"
+    """Requests-per-minute (RPM) or requests-per-window ceiling exceeded."""
+
+    TOKENS = "tokens"
+    """Tokens-per-minute (TPM) or tokens-per-window ceiling exceeded."""
+
+    CONCURRENT_REQUESTS = "concurrent_requests"
+    """``max_parallel_requests`` — too many in-flight requests at once."""
+
+    BUDGET = "budget"
+    """Spend budget cap reached (key, team, user, or per-session)."""
+
+    MAX_ITERATIONS = "max_iterations"
+    """Per-session max-iterations cap reached (agent-style flows)."""
+
+
 _MINIMAL_ERROR_RESPONSE: Optional[httpx.Response] = None
 
 
@@ -321,6 +394,18 @@ def __repr__(self):
 
 
 class RateLimitError(openai.RateLimitError):  # type: ignore
+    """
+    Unified rate-limit error.
+
+    Every rate-limit condition surfaced by litellm — whether it originated from
+    an upstream LLM provider, a vendor batch endpoint, or one of litellm's own
+    proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,
+    max-iterations, etc.) — is raised as an instance of this class.
+
+    The :attr:`category` attribute lets callers distinguish the source. See
+    :class:`RateLimitErrorCategory` for the available values.
+    """
+
     def __init__(
         self,
         message,
@@ -330,6 +415,19 @@ def __init__(
         litellm_debug_info: Optional[str] = None,
         max_retries: Optional[int] = None,
         num_retries: Optional[int] = None,
+        # An explicit ``category`` is now required for correct labeling — every
+        # callsite (vendor mappers in ``exception_mapping_utils.py``, proxy-side
+        # hooks, router-side throttles) is expected to pass the value that
+        # matches its source. The default below is an honest "unknown" sentinel,
+        # NOT a vendor assumption: silently inheriting it makes the omission
+        # surface in dashboards (as ``unknown_rate_limit``) instead of a
+        # confidently-wrong vendor label.
+        category: Union[str, RateLimitErrorCategory] = (
+            RateLimitErrorCategory.UNKNOWN_RATE_LIMIT
+        ),
+        rate_limit_type: Optional[Union[str, RateLimitType]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        detail: Any = None,
     ):
         self.status_code = 429
         self.message = "litellm.RateLimitError: {}".format(message)
@@ -338,9 +436,39 @@ def __init__(
         self.litellm_debug_info = litellm_debug_info
         self.max_retries = max_retries
         self.num_retries = num_retries
+        self.category = (
+            category.value if isinstance(category, RateLimitErrorCategory) else category
+        )
+        # Which dimension was exceeded — request count, token count, parallel
+        # requests, budget, max iterations. None when the source didn't
+        # classify the failure (e.g. legacy vendor 429 with no header hints).
+        self.rate_limit_type: Optional[str] = (
+            rate_limit_type.value
+            if isinstance(rate_limit_type, RateLimitType)
+            else rate_limit_type
+        )
+        # Headers explicitly attached to the error (e.g. retry-after,
+        # rate_limit_type, reset_at). Preserved across the proxy boundary so
+        # clients can react appropriately.
+        #
+        # IMPORTANT: we deliberately do NOT auto-populate self.headers from
+        # response.headers when only `response` is provided. A vendor 429 can
+        # set arbitrary response headers (Set-Cookie, CORS overrides, …); if
+        # those leaked into e.headers and a downstream proxy serializer
+        # forwarded them to the client, a malicious upstream could inject
+        # browser-interpreted headers for the proxy origin. Vendor response
+        # headers stay reachable on `e.response.headers` for callers that
+        # explicitly want them; only the proxy-supplied `headers=` kwarg
+        # makes it onto `self.headers`.
         _response_headers = (
             getattr(response, "headers", None) if response is not None else None
         )
+        self.headers: Optional[Dict[str, str]] = (
+            {k: str(v) for k, v in headers.items()} if headers else None
+        )
+        # Mirrors FastAPI HTTPException.detail so the same instance can be
+        # serialized through both the ProxyException and HTTPException paths.
+        self.detail = detail if detail is not None else self.message
         self.response = httpx.Response(
             status_code=429,
             headers=_response_headers,

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -22,6 +22,7 @@
     NotFoundError,
     PermissionDeniedError,
     RateLimitError,
+    RateLimitErrorCategory,
     ServiceUnavailableError,
     Timeout,
     UnprocessableEntityError,
@@ -403,6 +404,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         model=model,
                         llm_provider=custom_llm_provider,
                         response=getattr(original_exception, "response", None),
+                        category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                     )
                 elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
                     exception_mapping_worked = True
@@ -510,6 +512,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider=custom_llm_provider,
                         response=getattr(original_exception, "response", None),
                         litellm_debug_info=extra_information,
+                        category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                     )
                 elif (
                     "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"
@@ -591,6 +594,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider=custom_llm_provider,
                             response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 500:
                         exception_mapping_worked = True
@@ -730,6 +734,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AnthropicException - {error_str}",
                             llm_provider="anthropic",
                             model=model,
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif (
                         original_exception.status_code == 500
@@ -798,6 +803,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider="replicate",
                         model=model,
                         response=getattr(original_exception, "response", None),
+                        category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                     )
                 elif hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 401:
@@ -849,6 +855,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="replicate",
                             model=model,
                             response=getattr(original_exception, "response", None),
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 500:
                         exception_mapping_worked = True
@@ -905,6 +912,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider=custom_llm_provider,
                         model=model,
                         response=getattr(original_exception, "response", None),
+                        category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                     )
                 elif (
                     "The server received an invalid response from an upstream server."
@@ -981,6 +989,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             model=model,
                             llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 503:
                         exception_mapping_worked = True
@@ -1071,6 +1080,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         model=model,
                         llm_provider="bedrock",
                         response=getattr(original_exception, "response", None),
+                        category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                     )
                 elif (
                     "Connect timeout on endpoint URL" in error_str
@@ -1152,6 +1162,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider=custom_llm_provider,
                             response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 503:
                         exception_mapping_worked = True
@@ -1271,6 +1282,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider=custom_llm_provider,
                             response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 503:
                         exception_mapping_worked = True
@@ -1407,6 +1419,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                                 url=" https://cloud.google.com/vertex-ai/",
                             ),
                         ),
+                        category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                     )
                 elif (
                     "500 Internal Server Error" in error_str
@@ -1485,6 +1498,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                                     url=" https://cloud.google.com/vertex-ai/",
                                 ),
                             ),
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     if original_exception.status_code == 500:
                         exception_mapping_worked = True
@@ -1604,6 +1618,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider="cohere",
                         model=model,
                         response=getattr(original_exception, "response", None),
+                        category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                     )
                 elif "invalid type:" in error_str:
                     exception_mapping_worked = True
@@ -1656,6 +1671,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider="huggingface",
                         model=model,
                         response=getattr(original_exception, "response", None),
+                        category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                     )
                 if hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 401:
@@ -1688,6 +1704,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="huggingface",
                             model=model,
                             response=getattr(original_exception, "response", None),
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 503:
                         exception_mapping_worked = True
@@ -1755,6 +1772,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="ai21",
                             model=model,
                             response=getattr(original_exception, "response", None),
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     else:
                         exception_mapping_worked = True
@@ -1839,6 +1857,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="nlp_cloud",
                             model=model,
                             response=getattr(original_exception, "response", None),
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif (
                         original_exception.status_code == 500
@@ -1964,6 +1983,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="together_ai",
                             model=model,
                             response=getattr(original_exception, "response", None),
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 524:
                         exception_mapping_worked = True
@@ -2027,6 +2047,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="aleph_alpha",
                             model=model,
                             response=getattr(original_exception, "response", None),
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 500:
                         exception_mapping_worked = True
@@ -2270,6 +2291,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="azure",
                             litellm_debug_info=extra_information,
                             response=getattr(original_exception, "response", None),
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 502:
                         exception_mapping_worked = True
@@ -2374,6 +2396,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider=custom_llm_provider,
                             response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
+                            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
                         )
                     elif original_exception.status_code == 503:
                         exception_mapping_worked = True

litellm/litellm_core_utils/litellm_logging.py

@@ -5159,12 +5159,32 @@ def get_error_information(
         # Get additional error details
         error_message = str(original_exception)
 
+        # For rate-limit errors (litellm.RateLimitError + the proxy-side
+        # ProxyRateLimitError subclass), surface the unified `category` and
+        # `rate_limit_type` fields so callbacks can distinguish vendor vs.
+        # litellm rate limits AND split by the dimension that was exceeded
+        # (requests / tokens / concurrent_requests / budget / max_iterations)
+        # without reaching for the raw exception object.
+        is_rate_limit_error = isinstance(original_exception, litellm.RateLimitError)
+        rate_limit_category: Optional[str] = (
+            getattr(original_exception, "category", None)
+            if is_rate_limit_error
+            else None
+        )
+        rate_limit_type: Optional[str] = (
+            getattr(original_exception, "rate_limit_type", None)
+            if is_rate_limit_error
+            else None
+        )
+
         return StandardLoggingPayloadErrorInformation(
             error_code=error_status,
             error_class=error_class,
             llm_provider=_llm_provider_in_exception,
             traceback=traceback_info,
             error_message=error_message if original_exception else "",
+            error_rate_limit_category=rate_limit_category,
+            error_rate_limit_type=rate_limit_type,
         )
 
     @staticmethod