feat: standardize rate limit errors with category, rate_limit_type, model, and llm_provider fields

litellm/__init__.py

@@ -442,6 +442,13 @@ def _dev_env_hot_reload_enabled() -> bool:
 custom_prometheus_tags: List[str] = []
 prometheus_metrics_config: Optional[List] = None
 prometheus_emit_stream_label: bool = False
+# Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
+# `litellm_proxy_failed_requests_metric`. Off by default to preserve the
+# pre-unification label set so existing dashboards / recording rules keyed on
+# that metric keep matching after upgrade. Enable when downstream consumers
+# are ready to split 429s by source (vendor vs. litellm) and dimension
+# (RPM/TPM/concurrent/budget).
+prometheus_emit_rate_limit_labels: bool = False
 prometheus_user_budget_label_include_email_alias: bool = False
 prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
 prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
@@ -1303,6 +1310,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
     NotFoundError,
     PermissionDeniedError,
     RateLimitError,
+    RateLimitErrorCategory,
+    RateLimitType,
     ServiceUnavailableError,
     BadGatewayError,
     OpenAIError,

litellm/exceptions.py

@@ -9,13 +9,109 @@
 
 ## LiteLLM versions of the OpenAI Exception Types
 
-from typing import Any, Dict, Optional
+import enum
+from typing import Any, Dict, Optional, Union
 
 import httpx
 import openai
 
 from litellm.types.utils import LiteLLMCommonStrings
 
+
+class RateLimitErrorCategory(str, enum.Enum):
+    """
+    Category of a rate limit error, allowing callers to distinguish where the rate
+    limit originated. Exposed on every :class:`RateLimitError` instance via the
+    ``category`` attribute.
+
+    Use these values to switch on the rate limit source, e.g.::
+
+        try:
+            ...
+        except litellm.RateLimitError as e:
+            if e.category == RateLimitErrorCategory.LITELLM_RATE_LIMIT:
+                ...  # litellm's own limiter (key/team/user/model RPM/TPM/budget)
+            elif e.category == RateLimitErrorCategory.VENDOR_RATE_LIMIT:
+                ...  # the upstream LLM provider returned 429
+    """
+
+    VENDOR_RATE_LIMIT = "vendor_rate_limit"
+    """The upstream LLM provider returned a rate-limit response (e.g. OpenAI 429)."""
+
+    VENDOR_BATCH_RATE_LIMIT = "vendor_batch_rate_limit"
+    """The upstream LLM provider returned a rate-limit response on a batch endpoint."""
+
+    LITELLM_RATE_LIMIT = "litellm_rate_limit"
+    """LiteLLM's own rate limiter (key/team/user/model RPM/TPM, budget, parallel-requests, etc.) blocked the request."""
+
+    LITELLM_BATCH_RATE_LIMIT = "litellm_batch_rate_limit"
+    """LiteLLM's own batch rate limiter (token/request budget across a batch input file) blocked the request."""
+
+
+class RateLimitType(str, enum.Enum):
+    """
+    The dimension that was exceeded when a rate-limit error fired.
+
+    This is orthogonal to :class:`RateLimitErrorCategory` — *category* tells
+    callers **who** rate-limited the request (the upstream vendor vs. one of
+    litellm's own limiters), while *type* tells them **which limit dimension**
+    was exceeded (an RPM ceiling, a TPM ceiling, a max-parallel-requests
+    ceiling, a budget cap, or a max-iterations cap).
+
+    Surfaced both on every :class:`RateLimitError` instance via the
+    ``rate_limit_type`` attribute and on the structured
+    ``StandardLoggingPayload.error_information.error_rate_limit_type`` field
+    so custom callbacks / metrics consumers can split rate-limit failures by
+    cause without parsing free-text error messages.
+    """
+
+    REQUESTS = "requests"
+    """Requests-per-minute (RPM) or requests-per-window ceiling exceeded."""
+
+    TOKENS = "tokens"
+    """Tokens-per-minute (TPM) or tokens-per-window ceiling exceeded."""
+
+    CONCURRENT_REQUESTS = "concurrent_requests"
+    """``max_parallel_requests`` — too many in-flight requests at once."""
+
+    BUDGET = "budget"
+    """Spend budget cap reached (key, team, user, or per-session)."""
+
+    MAX_ITERATIONS = "max_iterations"
+    """Per-session max-iterations cap reached (agent-style flows)."""
+
+
+_RATE_LIMIT_CATEGORY_VALUES = frozenset(c.value for c in RateLimitErrorCategory)
+_RATE_LIMIT_TYPE_VALUES = frozenset(t.value for t in RateLimitType)
+
+
+def validate_rate_limit_category(value: Any) -> Optional[str]:
+    """Return ``value`` only if it matches a known :class:`RateLimitErrorCategory`.
+
+    Used at duck-typed read sites (StandardLoggingPayload extraction, Prometheus
+    labels) to reject `.category` strings set by unrelated third-party exceptions
+    — otherwise those would leak into custom-callback payloads and Prometheus
+    label cardinality.
+    """
+    if isinstance(value, RateLimitErrorCategory):
+        return value.value
+    if isinstance(value, str) and value in _RATE_LIMIT_CATEGORY_VALUES:
+        return value
+    return None
+
+
+def validate_rate_limit_type(value: Any) -> Optional[str]:
+    """Return ``value`` only if it matches a known :class:`RateLimitType`.
+
+    See :func:`validate_rate_limit_category` for the rationale.
+    """
+    if isinstance(value, RateLimitType):
+        return value.value
+    if isinstance(value, str) and value in _RATE_LIMIT_TYPE_VALUES:
+        return value
+    return None
+
+
 _MINIMAL_ERROR_RESPONSE: Optional[httpx.Response] = None
 
 
@@ -321,6 +417,18 @@ def __repr__(self):
 
 
 class RateLimitError(openai.RateLimitError):  # type: ignore
+    """
+    Unified rate-limit error.
+
+    Every rate-limit condition surfaced by litellm — whether it originated from
+    an upstream LLM provider, a vendor batch endpoint, or one of litellm's own
+    proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,
+    max-iterations, etc.) — is raised as an instance of this class.
+
+    The :attr:`category` attribute lets callers distinguish the source. See
+    :class:`RateLimitErrorCategory` for the available values.
+    """
+
     def __init__(
         self,
         message,
@@ -330,6 +438,12 @@ def __init__(
         litellm_debug_info: Optional[str] = None,
         max_retries: Optional[int] = None,
         num_retries: Optional[int] = None,
+        category: Union[str, RateLimitErrorCategory] = (
+            RateLimitErrorCategory.VENDOR_RATE_LIMIT
+        ),
+        rate_limit_type: Optional[Union[str, RateLimitType]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        detail: Any = None,
     ):
         self.status_code = 429
         self.message = "litellm.RateLimitError: {}".format(message)
@@ -338,9 +452,39 @@ def __init__(
         self.litellm_debug_info = litellm_debug_info
         self.max_retries = max_retries
         self.num_retries = num_retries
+        self.category = (
+            category.value if isinstance(category, RateLimitErrorCategory) else category
+        )
+        # Which dimension was exceeded — request count, token count, parallel
+        # requests, budget, max iterations. None when the source didn't
+        # classify the failure (e.g. legacy vendor 429 with no header hints).
+        self.rate_limit_type: Optional[str] = (
+            rate_limit_type.value
+            if isinstance(rate_limit_type, RateLimitType)
+            else rate_limit_type
+        )
+        # Headers explicitly attached to the error (e.g. retry-after,
+        # rate_limit_type, reset_at). Preserved across the proxy boundary so
+        # clients can react appropriately.
+        #
+        # IMPORTANT: we deliberately do NOT auto-populate self.headers from
+        # response.headers when only `response` is provided. A vendor 429 can
+        # set arbitrary response headers (Set-Cookie, CORS overrides, …); if
+        # those leaked into e.headers and a downstream proxy serializer
+        # forwarded them to the client, a malicious upstream could inject
+        # browser-interpreted headers for the proxy origin. Vendor response
+        # headers stay reachable on `e.response.headers` for callers that
+        # explicitly want them; only the proxy-supplied `headers=` kwarg
+        # makes it onto `self.headers`.
         _response_headers = (
             getattr(response, "headers", None) if response is not None else None
         )
+        self.headers: Optional[Dict[str, str]] = (
+            {k: str(v) for k, v in headers.items()} if headers else None
+        )
+        # Mirrors FastAPI HTTPException.detail so the same instance can be
+        # serialized through both the ProxyException and HTTPException paths.
+        self.detail = detail if detail is not None else self.message
         self.response = httpx.Response(
             status_code=429,
             headers=_response_headers,
@@ -843,11 +987,24 @@ def __init__(
 
 class BudgetExceededError(Exception):
     def __init__(
-        self, current_cost: float, max_budget: float, message: Optional[str] = None
+        self,
+        current_cost: float,
+        max_budget: float,
+        message: Optional[str] = None,
+        llm_provider: Optional[str] = None,
     ):
         self.current_cost = current_cost
         self.max_budget = max_budget
         self.status_code = 429
+        self.llm_provider = llm_provider or ""
+        # Surface unified rate-limit fields without joining the RateLimitError
+        # hierarchy so existing `except BudgetExceededError:` handlers keep
+        # working; custom callbacks reading StandardLoggingPayload pick these
+        # up via the same `category` / `rate_limit_type` attributes the rest
+        # of the unified rate-limit error path uses. Stored as plain strings
+        # to match the normalization RateLimitError.__init__ performs.
+        self.category: str = RateLimitErrorCategory.LITELLM_RATE_LIMIT.value
+        self.rate_limit_type: str = RateLimitType.BUDGET.value
         message = (
             message
             or f"Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}"

litellm/integrations/prometheus.py

@@ -24,6 +24,10 @@
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
+from litellm.exceptions import (
+    validate_rate_limit_category,
+    validate_rate_limit_type,
+)
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
     BoundedPrometheusSeriesTracker,
@@ -78,6 +82,20 @@ def __init__(  # noqa: PLR0915
             # Always initialize label_filters, even for non-premium users
             self.label_filters = self._parse_prometheus_config()
 
+            # Cache resolved label sets per metric. Several entries in
+            # ``PrometheusMetricLabels.get_labels`` read module-level toggles
+            # (e.g. ``litellm.prometheus_emit_stream_label``,
+            # ``litellm.prometheus_emit_rate_limit_labels``) that can be
+            # changed at runtime. Prometheus counters/gauges/histograms are
+            # created with a *fixed* ``labelnames`` set; if a runtime call
+            # to ``get_labels_for_metric`` returned a different set, the
+            # subsequent ``counter.labels(**_labels)`` would raise a
+            # ``ValueError`` from the prometheus client. Snapshotting at
+            # logger init time pins the label set for the lifetime of the
+            # logger so toggling these flags only takes effect after a
+            # restart, keeping init-time and runtime label sets in sync.
+            self._cached_metric_labels: Dict[str, List[str]] = {}
+
             _custom_buckets = litellm.prometheus_latency_buckets
             self.latency_buckets = (
                 tuple(_custom_buckets)
@@ -1033,13 +1051,27 @@ def get_labels_for_metric(
         self, metric_name: DEFINED_PROMETHEUS_METRICS
     ) -> List[str]:
         """
-        Get the labels for a metric, filtered if configured
+        Get the labels for a metric, filtered if configured.
+
+        The result is cached on the instance so the label set used to
+        construct each Prometheus metric at ``__init__`` time stays in lock
+        step with the label set passed to ``counter.labels(...)`` at
+        runtime, even if the underlying module-level toggles consulted by
+        :meth:`PrometheusMetricLabels.get_labels` (e.g.
+        ``litellm.prometheus_emit_rate_limit_labels``,
+        ``litellm.prometheus_emit_stream_label``) are flipped after the
+        logger has been created.
         """
+        cached = self._cached_metric_labels.get(metric_name)
+        if cached is not None:
+            return cached
+
         # Get default labels for this metric from PrometheusMetricLabels
         default_labels = PrometheusMetricLabels.get_labels(metric_name)
 
         # If no label filtering is configured for this metric, use default labels
         if metric_name not in self.label_filters:
+            self._cached_metric_labels[metric_name] = default_labels
             return default_labels
 
         # Get configured labels for this metric
@@ -1050,6 +1082,7 @@ def get_labels_for_metric(
             label for label in default_labels if label in configured_labels
         ]
 
+        self._cached_metric_labels[metric_name] = filtered_labels
         return filtered_labels
 
     def _track_end_user_metric_series(
@@ -2029,14 +2062,8 @@ async def async_post_call_failure_hook(
 
         Proxy level tracking - failed client side requests
 
-        labelnames=[
-                    "end_user",
-                    "hashed_api_key",
-                    "api_key_alias",
-                    REQUESTED_MODEL,
-                    "team",
-                    "team_alias",
-                ] + EXCEPTION_LABELS,
+        See :attr:`PrometheusMetricLabels.litellm_proxy_failed_requests_metric`
+        for the authoritative list of labels emitted on this metric.
         """
         from litellm.litellm_core_utils.litellm_logging import (
             StandardLoggingPayloadSetup,
@@ -2059,6 +2086,9 @@ async def async_post_call_failure_hook(
             model_id = _metadata.get("model_info", {}).get("id") or request_data.get(
                 "model_info", {}
             ).get("id")
+            rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(
+                original_exception
+            )
             enum_values = UserAPIKeyLabelValues(
                 end_user=user_api_key_dict.end_user_id,
                 user=user_api_key_dict.user_id,
@@ -2073,6 +2103,8 @@ async def async_post_call_failure_hook(
                 status_code=str(status_code),
                 exception_status=str(status_code),
                 exception_class=self._get_exception_class_name(original_exception),
+                rate_limit_category=rate_limit_category,
+                rate_limit_type=rate_limit_type,
                 tags=_tags,
                 route=user_api_key_dict.request_route,
                 client_ip=_metadata.get("requester_ip_address"),
@@ -2843,6 +2875,33 @@ def record_check_batch_cost_error(self, error_type: str):
 
     @staticmethod
     def _get_exception_class_name(exception: Exception) -> str:
+        # Some exception types pin the ``exception_class`` label to a legacy
+        # value for back-compat with existing dashboards (e.g. proxy-side 429s
+        # keep reporting as "HTTPException"). Honor that opt-in marker before
+        # deriving the label from the runtime class name. Reading it via
+        # ``getattr`` keeps this core integrations module free of a transitive
+        # ``fastapi`` dependency.
+        legacy_class_name = getattr(exception, "prometheus_exception_class_name", None)
+        if isinstance(legacy_class_name, str) and legacy_class_name:
+            return legacy_class_name
+
+        # Same back-compat reasoning for ``BudgetExceededError``: the unified
+        # rate-limit error work attached ``.llm_provider`` to budget errors
+        # too (so callbacks reading ``StandardLoggingPayload`` get provider
+        # attribution). Without this short-circuit, the provider prefix below
+        # would silently flip the label from "BudgetExceededError" to e.g.
+        # "Openai.BudgetExceededError" and break dashboards keyed on the
+        # original value.
+        try:
+            from litellm.exceptions import BudgetExceededError
+        except ImportError:
+            BudgetExceededError = None  # type: ignore[assignment,misc]
+
+        if BudgetExceededError is not None and isinstance(
+            exception, BudgetExceededError
+        ):
+            return "BudgetExceededError"
+
         exception_class_name = ""
         if hasattr(exception, "llm_provider"):
             exception_class_name = getattr(exception, "llm_provider") or ""
@@ -2857,6 +2916,27 @@ def _get_exception_class_name(exception: Exception) -> str:
         exception_class_name += exception.__class__.__name__
         return exception_class_name
 
+    @staticmethod
+    def _extract_rate_limit_labels(
+        exception: Optional[Exception],
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Pull the unified ``category`` / ``rate_limit_type`` fields off any
+        exception that declares them (``litellm.RateLimitError`` and bare-
+        Exception subclasses like ``BudgetExceededError``).
+
+        Values are validated against the :class:`RateLimitErrorCategory` /
+        :class:`RateLimitType` enums so unrelated third-party exceptions that
+        happen to declare ``.category`` / ``.rate_limit_type`` string attributes
+        can't leak garbage into Prometheus label cardinality.
+        """
+        if exception is None:
+            return None, None
+        return (
+            validate_rate_limit_category(getattr(exception, "category", None)),
+            validate_rate_limit_type(getattr(exception, "rate_limit_type", None)),
+        )
+
     async def log_success_fallback_event(
         self, original_model_group: str, kwargs: dict, original_exception: Exception
     ):