From 3550c4b7394d64100f8ac9177981c78b49b6031a Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Mon, 25 May 2026 17:48:08 -0700 Subject: [PATCH 1/7] test(prometheus): add regression tests for LIT-2577 v3 remaining-headers fallback --- ...est_prometheus_virtual_key_v3_remaining.py | 300 ++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 tests/test_litellm/integrations/test_prometheus_virtual_key_v3_remaining.py diff --git a/tests/test_litellm/integrations/test_prometheus_virtual_key_v3_remaining.py b/tests/test_litellm/integrations/test_prometheus_virtual_key_v3_remaining.py new file mode 100644 index 000000000000..3e547442a044 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_virtual_key_v3_remaining.py @@ -0,0 +1,300 @@ +""" +Regression tests for LIT-2577. + +Until this fix, ``PrometheusLogger._set_virtual_key_rate_limit_metrics`` +only consulted the v1 ``parallel_request_limiter`` keys +(``litellm-key-remaining-{requests,tokens}-{model_group}`` in ``metadata``). +The active ``parallel_request_limiter_v3`` does not write those; it writes +to ``response._hidden_params["additional_headers"]`` under +``x-ratelimit-{model_per_key,key}-remaining-{requests,tokens}``. Result: +``litellm_remaining_api_key_{requests,tokens}_for_model`` always reported +``sys.maxsize`` (~9.22e18) in DataDog/Prometheus. + +These tests pin the v3 fallback so the bug stays fixed. +""" +from __future__ import annotations + +import sys +from typing import Optional + +import pytest +from prometheus_client import REGISTRY + +from litellm.integrations.prometheus import ( + PrometheusLogger, + _get_additional_headers_from_kwargs, + _get_remaining_from_v3_headers, +) + +MODEL_GROUP = "gpt-4o-mini" + + +def _clear_prometheus_registry() -> None: + for collector in list(REGISTRY._collector_to_names.keys()): + REGISTRY.unregister(collector) + + +@pytest.fixture +def prometheus_logger() -> PrometheusLogger: + _clear_prometheus_registry() + return PrometheusLogger() + + +def _slp_with_additional_headers(additional_headers: Optional[dict]) -> dict: + """Minimal standard_logging_object shape ``_set_virtual_key_rate_limit_metrics`` touches.""" + payload: dict = { + "model_id": "model-123", + "model_group": MODEL_GROUP, + "metadata": { + "user_api_key_hash": "test-hash", + "user_api_key_alias": "test-alias", + "user_api_key_team_id": None, + "user_api_key_team_alias": None, + "user_api_key_user_id": None, + "user_api_key_user_email": None, + "user_api_key_org_id": None, + "requester_metadata": None, + "user_api_key_auth_metadata": None, + "spend_logs_metadata": None, + }, + } + if additional_headers is not None: + payload["hidden_params"] = {"additional_headers": additional_headers} + return payload + + +def _sample(metric_name: str): + for metric in REGISTRY.collect(): + for sample in metric.samples: + if sample.name == metric_name: + return sample + return None + + +def test_v3_additional_headers_populate_remaining_gauges(prometheus_logger): + """v3 path: only ``additional_headers`` populated -> gauges read from there.""" + metadata = {"model_group": MODEL_GROUP} # no legacy litellm-key-remaining-* keys + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers( + { + "x-ratelimit-model_per_key-remaining-requests": 7, + "x-ratelimit-model_per_key-remaining-tokens": 1234, + "x-ratelimit-model_per_key-limit-requests": 10, + "x-ratelimit-model_per_key-limit-tokens": 2000, + } + ), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + req = _sample("litellm_remaining_api_key_requests_for_model") + tok = _sample("litellm_remaining_api_key_tokens_for_model") + assert req is not None and req.value == 7 + assert tok is not None and tok.value == 1234 + + +def test_v3_key_descriptor_used_when_model_per_key_missing(prometheus_logger): + """v3 path: per-(key, model) header missing -> fall back to per-key header.""" + metadata = {"model_group": MODEL_GROUP} + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers( + { + "x-ratelimit-key-remaining-requests": 42, + "x-ratelimit-key-remaining-tokens": 9001, + } + ), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + assert _sample("litellm_remaining_api_key_requests_for_model").value == 42 + assert _sample("litellm_remaining_api_key_tokens_for_model").value == 9001 + + +def test_model_per_key_preferred_over_key(prometheus_logger): + """When both descriptors are present, ``model_per_key`` (more specific) wins.""" + metadata = {"model_group": MODEL_GROUP} + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers( + { + "x-ratelimit-key-remaining-requests": 100, + "x-ratelimit-model_per_key-remaining-requests": 5, + "x-ratelimit-key-remaining-tokens": 5000, + "x-ratelimit-model_per_key-remaining-tokens": 100, + } + ), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + assert _sample("litellm_remaining_api_key_requests_for_model").value == 5 + assert _sample("litellm_remaining_api_key_tokens_for_model").value == 100 + + +def test_v1_metadata_keys_still_take_precedence(prometheus_logger): + """v1 metadata keys are the existing public contract -> still win over v3 fallback.""" + metadata = { + "model_group": MODEL_GROUP, + "litellm-key-remaining-requests-gpt-4o-mini": 11, + "litellm-key-remaining-tokens-gpt-4o-mini": 22, + } + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers( + { + "x-ratelimit-model_per_key-remaining-requests": 999, + "x-ratelimit-model_per_key-remaining-tokens": 999, + } + ), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + assert _sample("litellm_remaining_api_key_requests_for_model").value == 11 + assert _sample("litellm_remaining_api_key_tokens_for_model").value == 22 + + +def test_v3_zero_remaining_preserved(prometheus_logger): + """``0`` is a meaningful value (key exhausted) and must not get coerced to sys.maxsize.""" + metadata = {"model_group": MODEL_GROUP} + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers( + { + "x-ratelimit-model_per_key-remaining-requests": 0, + "x-ratelimit-model_per_key-remaining-tokens": 0, + } + ), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + assert _sample("litellm_remaining_api_key_requests_for_model").value == 0 + assert _sample("litellm_remaining_api_key_tokens_for_model").value == 0 + + +def test_falls_back_to_maxsize_when_neither_path_has_data(prometheus_logger): + """No v1 keys, no v3 keys -> existing behavior (sys.maxsize) preserved.""" + metadata = {"model_group": MODEL_GROUP} + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers({}), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + assert _sample("litellm_remaining_api_key_requests_for_model").value == pytest.approx( + float(sys.maxsize) + ) + assert _sample("litellm_remaining_api_key_tokens_for_model").value == pytest.approx( + float(sys.maxsize) + ) + + +def test_string_values_in_additional_headers_are_coerced(prometheus_logger): + """Headers travelling through HTTP/serialization can land as strings -> coerce to int.""" + metadata = {"model_group": MODEL_GROUP} + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers( + { + "x-ratelimit-model_per_key-remaining-requests": "8", + "x-ratelimit-model_per_key-remaining-tokens": "777", + } + ), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + assert _sample("litellm_remaining_api_key_requests_for_model").value == 8 + assert _sample("litellm_remaining_api_key_tokens_for_model").value == 777 + + +# ----- helper unit tests ------------------------------------------------- + + +def test_get_additional_headers_tolerates_missing_layers(): + """Helper must not raise on partially-populated standard_logging_object.""" + assert _get_additional_headers_from_kwargs({}) == {} + assert _get_additional_headers_from_kwargs({"standard_logging_object": None}) == {} + assert ( + _get_additional_headers_from_kwargs( + {"standard_logging_object": {"hidden_params": None}} + ) + == {} + ) + assert ( + _get_additional_headers_from_kwargs( + {"standard_logging_object": {"hidden_params": {"additional_headers": None}}} + ) + == {} + ) + # non-dict types at each layer + assert _get_additional_headers_from_kwargs({"standard_logging_object": "oops"}) == {} + assert ( + _get_additional_headers_from_kwargs( + {"standard_logging_object": {"hidden_params": "oops"}} + ) + == {} + ) + + +def test_get_remaining_from_v3_headers_returns_none_when_absent(): + assert _get_remaining_from_v3_headers({}, "requests") is None + assert ( + _get_remaining_from_v3_headers({"x-ratelimit-foo-remaining-requests": 1}, "requests") + is None + ) + + +def test_get_remaining_from_v3_headers_prefers_model_per_key_over_key(): + headers = { + "x-ratelimit-key-remaining-requests": 100, + "x-ratelimit-model_per_key-remaining-requests": 5, + } + assert _get_remaining_from_v3_headers(headers, "requests") == 5 From 3e526604dd114a724b47b484b7f94d285ebf4bb8 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Mon, 25 May 2026 18:06:17 -0700 Subject: [PATCH 2/7] fix(prometheus): install v3 rate-limit remaining-headers fallback (LIT-2577) --- .../_prometheus_v3_remaining_fix.py | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 litellm/integrations/_prometheus_v3_remaining_fix.py diff --git a/litellm/integrations/_prometheus_v3_remaining_fix.py b/litellm/integrations/_prometheus_v3_remaining_fix.py new file mode 100644 index 000000000000..72b4624c2617 --- /dev/null +++ b/litellm/integrations/_prometheus_v3_remaining_fix.py @@ -0,0 +1,172 @@ +""" +LIT-2577: install v3 rate-limit remaining-headers fallback on PrometheusLogger. + +The legacy v1 ``parallel_request_limiter`` writes per-(key, model) remaining +values into request ``metadata`` under + ``litellm-key-remaining-{requests,tokens}-{model_group}`` + +The active v3 limiter (``parallel_request_limiter_v3.py``) instead writes them +into ``response._hidden_params["additional_headers"]`` (mirrored into the +standard_logging_payload's ``hidden_params.additional_headers``) under + ``x-ratelimit-{model_per_key,key}-remaining-{requests,tokens}`` + +``PrometheusLogger._set_virtual_key_rate_limit_metrics`` in +``litellm/integrations/prometheus.py`` only reads the v1 metadata keys, so +when v3 is in use the gauges fall through to ``sys.maxsize`` (~9.22e18) and +DataDog / Prometheus dashboards show nonsense values. This module wraps the +method so it reads the v3 ``additional_headers`` as a fallback before falling +through to ``sys.maxsize``; v1 metadata keys still take precedence so the +existing public contract is preserved. + +This patch is installed at import time by ``litellm/integrations/__init__.py``. +A future cleanup can inline this logic directly into ``prometheus.py``; the +indirection here exists because the size of ``prometheus.py`` (~156 KB) blocks +the single-tool-call push path we have available today. +""" +from __future__ import annotations + +from typing import Any, Dict, Optional, Union + + +def _get_additional_headers_from_kwargs(kwargs: dict) -> Dict[str, Any]: + """ + Pull ``additional_headers`` out of ``standard_logging_object.hidden_params``. + + Returns an empty dict on any missing/non-dict layer so callers can + ``.get(...)`` freely without try/except. + """ + standard_logging_payload = kwargs.get("standard_logging_object") or {} + if not isinstance(standard_logging_payload, dict): + return {} + hidden_params = standard_logging_payload.get("hidden_params") or {} + if not isinstance(hidden_params, dict): + return {} + additional_headers = hidden_params.get("additional_headers") or {} + if not isinstance(additional_headers, dict): + return {} + return additional_headers + + +def _get_remaining_from_v3_headers( + additional_headers: Dict[str, Any], + rate_limit_type: str, +) -> Optional[Union[int, float]]: + """ + Return the per-(key, model) remaining value emitted by the v3 rate limiter. + + The v3 post-call hook writes per-descriptor headers of the form + ``x-ratelimit-{descriptor_key}-remaining-{rate_limit_type}``. For the + Prometheus virtual-key gauges we want the per-(key, model) value so we + prefer ``model_per_key`` (which already scopes by both the API key and the + model_group) and fall back to ``key`` (per-key, all-models) if it is + absent. Returns ``None`` if neither descriptor is present. + """ + if not additional_headers: + return None + for descriptor in ("model_per_key", "key"): + value = additional_headers.get( + f"x-ratelimit-{descriptor}-remaining-{rate_limit_type}" + ) + if value is not None: + try: + # v3 values are ints but tolerate strings/floats just in case + return int(value) + except (TypeError, ValueError): + return value + return None + + +def _install_patch() -> None: + """ + Apply the v3 fallback to ``PrometheusLogger._set_virtual_key_rate_limit_metrics``. + + Idempotent — second/Nth call is a no-op. + + Strategy: wrap the existing method. If either of the v1 metadata keys + (``litellm-key-remaining-{requests,tokens}-{model_group}``) is absent on + the inbound ``metadata`` dict, look the value up in the v3 + ``additional_headers`` and inject it into a *shallow copy* of metadata + before delegating to the original. The original's existing + ``sys.maxsize`` fallback continues to handle the "neither limiter + populated anything" case unchanged. + """ + # NOTE: import lazily. Importing this module is done from + # ``litellm/integrations/__init__.py``; resolving + # ``litellm.integrations.prometheus`` here works because Python returns + # the in-progress integrations package object for the parent reference + # and then loads ``prometheus.py`` to completion as a submodule. + from litellm.integrations import prometheus + + if getattr(prometheus.PrometheusLogger, "_lit2577_patched", False): + return + + from litellm.proxy.common_utils.callback_utils import ( + get_model_group_from_litellm_kwargs, + ) + + original = prometheus.PrometheusLogger._set_virtual_key_rate_limit_metrics + + def patched( + self, + user_api_key, + user_api_key_alias, + kwargs, + metadata, + model_id=None, + ): + model_group = get_model_group_from_litellm_kwargs(kwargs) + rk = f"litellm-key-remaining-requests-{model_group}" + tk = f"litellm-key-remaining-tokens-{model_group}" + + # Only do work if at least one v1 key is missing + v1_requests = metadata.get(rk) if isinstance(metadata, dict) else None + v1_tokens = metadata.get(tk) if isinstance(metadata, dict) else None + if v1_requests is not None and v1_tokens is not None: + return original( + self, user_api_key, user_api_key_alias, kwargs, metadata, model_id + ) + + headers = _get_additional_headers_from_kwargs(kwargs) + if not headers: + return original( + self, user_api_key, user_api_key_alias, kwargs, metadata, model_id + ) + + # Build a shallow copy of metadata with v3-derived values filled in. + # We never overwrite values the caller already provided. + patched_metadata: Optional[dict] = None + if v1_requests is None: + v = _get_remaining_from_v3_headers(headers, "requests") + if v is not None: + patched_metadata = dict(metadata) if isinstance(metadata, dict) else {} + patched_metadata[rk] = v + if v1_tokens is None: + v = _get_remaining_from_v3_headers(headers, "tokens") + if v is not None: + if patched_metadata is None: + patched_metadata = ( + dict(metadata) if isinstance(metadata, dict) else {} + ) + patched_metadata[tk] = v + + if patched_metadata is not None: + metadata = patched_metadata + + return original( + self, user_api_key, user_api_key_alias, kwargs, metadata, model_id + ) + + prometheus.PrometheusLogger._set_virtual_key_rate_limit_metrics = patched + prometheus.PrometheusLogger._lit2577_patched = True + + # Expose helpers as module-level attributes on ``prometheus`` so the + # regression tests (which import them from + # ``litellm.integrations.prometheus``) work without callers having to + # know about this patch module. + prometheus._get_additional_headers_from_kwargs = _get_additional_headers_from_kwargs + prometheus._get_remaining_from_v3_headers = _get_remaining_from_v3_headers + + +# Apply on first import. The integrations package __init__ imports us, so this +# runs before any user code touches ``PrometheusLogger``. +_install_patch() From 3826a6a0ea9d67e516d12a629d5c8a049ceb7a78 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Mon, 25 May 2026 18:06:32 -0700 Subject: [PATCH 3/7] fix(prometheus): wire _prometheus_v3_remaining_fix at package import (LIT-2577) --- litellm/integrations/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/litellm/integrations/__init__.py b/litellm/integrations/__init__.py index b6e690fd5914..e073cf6e5739 100644 --- a/litellm/integrations/__init__.py +++ b/litellm/integrations/__init__.py @@ -1 +1,6 @@ from . import * + +# LIT-2577: install v3 rate-limit remaining-headers fallback on PrometheusLogger +# at package import time. See `_prometheus_v3_remaining_fix.py` for the +# full rationale; the import below has the side-effect of running the patch. +from . import _prometheus_v3_remaining_fix # noqa: F401 From 6c590c77aa9bea32a148c5c47bcb346cd73794b8 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Mon, 25 May 2026 18:09:27 -0700 Subject: [PATCH 4/7] fix(prometheus): defer patch installation via meta-path hook to avoid circular import (LIT-2577) --- .../_prometheus_v3_remaining_fix.py | 211 +++++++++++------- 1 file changed, 134 insertions(+), 77 deletions(-) diff --git a/litellm/integrations/_prometheus_v3_remaining_fix.py b/litellm/integrations/_prometheus_v3_remaining_fix.py index 72b4624c2617..16a2231b5554 100644 --- a/litellm/integrations/_prometheus_v3_remaining_fix.py +++ b/litellm/integrations/_prometheus_v3_remaining_fix.py @@ -18,22 +18,35 @@ through to ``sys.maxsize``; v1 metadata keys still take precedence so the existing public contract is preserved. -This patch is installed at import time by ``litellm/integrations/__init__.py``. +This module is imported by ``litellm/integrations/__init__.py``. To avoid a +circular import (``prometheus.py`` -> ``litellm.proxy._types`` -> ``litellm`` +which is still mid-init), we DO NOT touch ``prometheus`` at import time. +Instead we install a one-shot ``sys.meta_path`` finder that intercepts the +first import of ``litellm.integrations.prometheus`` and installs the patch +after that module finishes loading. By that point the litellm package has +fully initialized, so the proxy helper import in the wrapped method works. + A future cleanup can inline this logic directly into ``prometheus.py``; the -indirection here exists because the size of ``prometheus.py`` (~156 KB) blocks -the single-tool-call push path we have available today. +indirection here exists because the size of ``prometheus.py`` (~156 KB) +blocks the single-tool-call push path we have available today. """ from __future__ import annotations +import importlib +import importlib.abc +import importlib.util +import sys from typing import Any, Dict, Optional, Union +_PROMETHEUS_FQN = "litellm.integrations.prometheus" + def _get_additional_headers_from_kwargs(kwargs: dict) -> Dict[str, Any]: """ Pull ``additional_headers`` out of ``standard_logging_object.hidden_params``. - Returns an empty dict on any missing/non-dict layer so callers can - ``.get(...)`` freely without try/except. + Returns an empty dict on any missing / non-dict layer so callers can + ``.get(...)`` freely. """ standard_logging_payload = kwargs.get("standard_logging_object") or {} if not isinstance(standard_logging_payload, dict): @@ -54,11 +67,8 @@ def _get_remaining_from_v3_headers( """ Return the per-(key, model) remaining value emitted by the v3 rate limiter. - The v3 post-call hook writes per-descriptor headers of the form - ``x-ratelimit-{descriptor_key}-remaining-{rate_limit_type}``. For the - Prometheus virtual-key gauges we want the per-(key, model) value so we - prefer ``model_per_key`` (which already scopes by both the API key and the - model_group) and fall back to ``key`` (per-key, all-models) if it is + Prefers ``model_per_key`` (which already scopes by both the API key and + the model_group) and falls back to ``key`` (per-key, all-models) if absent. Returns ``None`` if neither descriptor is present. """ if not additional_headers: @@ -69,42 +79,25 @@ def _get_remaining_from_v3_headers( ) if value is not None: try: - # v3 values are ints but tolerate strings/floats just in case return int(value) except (TypeError, ValueError): return value return None -def _install_patch() -> None: +def _install_patch_on_module(prometheus_module: Any) -> None: """ - Apply the v3 fallback to ``PrometheusLogger._set_virtual_key_rate_limit_metrics``. - - Idempotent — second/Nth call is a no-op. + Apply the v3 fallback wrap to ``PrometheusLogger._set_virtual_key_rate_limit_metrics``. - Strategy: wrap the existing method. If either of the v1 metadata keys - (``litellm-key-remaining-{requests,tokens}-{model_group}``) is absent on - the inbound ``metadata`` dict, look the value up in the v3 - ``additional_headers`` and inject it into a *shallow copy* of metadata - before delegating to the original. The original's existing - ``sys.maxsize`` fallback continues to handle the "neither limiter - populated anything" case unchanged. + Safe to call multiple times — second/Nth call is a no-op. """ - # NOTE: import lazily. Importing this module is done from - # ``litellm/integrations/__init__.py``; resolving - # ``litellm.integrations.prometheus`` here works because Python returns - # the in-progress integrations package object for the parent reference - # and then loads ``prometheus.py`` to completion as a submodule. - from litellm.integrations import prometheus - - if getattr(prometheus.PrometheusLogger, "_lit2577_patched", False): + PrometheusLogger = getattr(prometheus_module, "PrometheusLogger", None) + if PrometheusLogger is None: + return + if getattr(PrometheusLogger, "_lit2577_patched", False): return - from litellm.proxy.common_utils.callback_utils import ( - get_model_group_from_litellm_kwargs, - ) - - original = prometheus.PrometheusLogger._set_virtual_key_rate_limit_metrics + original = PrometheusLogger._set_virtual_key_rate_limit_metrics def patched( self, @@ -114,59 +107,123 @@ def patched( metadata, model_id=None, ): + # NOTE: this import is intentionally lazy. Doing it at module load + # creates a circular import because ``callback_utils`` imports from + # ``litellm`` which is still mid-init when integrations is imported. + from litellm.proxy.common_utils.callback_utils import ( + get_model_group_from_litellm_kwargs, + ) + model_group = get_model_group_from_litellm_kwargs(kwargs) rk = f"litellm-key-remaining-requests-{model_group}" tk = f"litellm-key-remaining-tokens-{model_group}" - # Only do work if at least one v1 key is missing v1_requests = metadata.get(rk) if isinstance(metadata, dict) else None v1_tokens = metadata.get(tk) if isinstance(metadata, dict) else None - if v1_requests is not None and v1_tokens is not None: - return original( - self, user_api_key, user_api_key_alias, kwargs, metadata, model_id - ) - - headers = _get_additional_headers_from_kwargs(kwargs) - if not headers: - return original( - self, user_api_key, user_api_key_alias, kwargs, metadata, model_id - ) - - # Build a shallow copy of metadata with v3-derived values filled in. - # We never overwrite values the caller already provided. - patched_metadata: Optional[dict] = None - if v1_requests is None: - v = _get_remaining_from_v3_headers(headers, "requests") - if v is not None: - patched_metadata = dict(metadata) if isinstance(metadata, dict) else {} - patched_metadata[rk] = v - if v1_tokens is None: - v = _get_remaining_from_v3_headers(headers, "tokens") - if v is not None: - if patched_metadata is None: - patched_metadata = ( - dict(metadata) if isinstance(metadata, dict) else {} - ) - patched_metadata[tk] = v - - if patched_metadata is not None: - metadata = patched_metadata + + if v1_requests is None or v1_tokens is None: + headers = _get_additional_headers_from_kwargs(kwargs) + if headers: + patched_metadata: Optional[dict] = None + if v1_requests is None: + v = _get_remaining_from_v3_headers(headers, "requests") + if v is not None: + patched_metadata = ( + dict(metadata) if isinstance(metadata, dict) else {} + ) + patched_metadata[rk] = v + if v1_tokens is None: + v = _get_remaining_from_v3_headers(headers, "tokens") + if v is not None: + if patched_metadata is None: + patched_metadata = ( + dict(metadata) if isinstance(metadata, dict) else {} + ) + patched_metadata[tk] = v + if patched_metadata is not None: + metadata = patched_metadata return original( self, user_api_key, user_api_key_alias, kwargs, metadata, model_id ) - prometheus.PrometheusLogger._set_virtual_key_rate_limit_metrics = patched - prometheus.PrometheusLogger._lit2577_patched = True + PrometheusLogger._set_virtual_key_rate_limit_metrics = patched + PrometheusLogger._lit2577_patched = True + + # Expose the helpers as module-level attributes on ``prometheus`` so the + # regression tests can ``from litellm.integrations.prometheus import + # _get_additional_headers_from_kwargs, _get_remaining_from_v3_headers`` + # without needing to know about this patch module. + prometheus_module._get_additional_headers_from_kwargs = ( + _get_additional_headers_from_kwargs + ) + prometheus_module._get_remaining_from_v3_headers = ( + _get_remaining_from_v3_headers + ) - # Expose helpers as module-level attributes on ``prometheus`` so the - # regression tests (which import them from - # ``litellm.integrations.prometheus``) work without callers having to - # know about this patch module. - prometheus._get_additional_headers_from_kwargs = _get_additional_headers_from_kwargs - prometheus._get_remaining_from_v3_headers = _get_remaining_from_v3_headers +class _PrometheusPostImportHook(importlib.abc.MetaPathFinder): + """ + One-shot meta-path finder: intercepts the first import of + ``litellm.integrations.prometheus``, lets the normal loader run it to + completion, then installs our patch. + + After firing it removes itself from ``sys.meta_path``. + """ -# Apply on first import. The integrations package __init__ imports us, so this -# runs before any user code touches ``PrometheusLogger``. -_install_patch() + def find_spec(self, fullname, path=None, target=None): + if fullname != _PROMETHEUS_FQN: + return None + try: + sys.meta_path.remove(self) + except ValueError: + pass + # Walk the remaining finders so we don't deadlock on ourselves. + for finder in list(sys.meta_path): + spec = finder.find_spec(fullname, path, target) + if spec is None: + continue + original_loader = spec.loader + if original_loader is None or not hasattr( + original_loader, "exec_module" + ): + return spec + + class _WrappedLoader(importlib.abc.Loader): + def create_module(self, spec_): + if hasattr(original_loader, "create_module"): + return original_loader.create_module(spec_) + return None + + def exec_module(self, module): + original_loader.exec_module(module) + try: + _install_patch_on_module(module) + except Exception: + # Patch failure must NOT break Prometheus logging. + # The original method still works; we just don't get + # the v3 fallback. Worse than a fix, better than a crash. + import logging + + logging.getLogger(__name__).warning( + "LIT-2577: failed to install Prometheus v3 fallback patch", + exc_info=True, + ) + + spec.loader = _WrappedLoader() + return spec + return None + + +# If prometheus is already imported (unusual: would mean something pulled it +# in before integrations.__init__), patch it directly. Otherwise install the +# meta-path hook for the first future import. +_existing = sys.modules.get(_PROMETHEUS_FQN) +if _existing is not None: + _install_patch_on_module(_existing) +else: + _hook = _PrometheusPostImportHook() + if not any( + isinstance(f, _PrometheusPostImportHook) for f in sys.meta_path + ): + sys.meta_path.insert(0, _hook) From 897349d8017456a13cc3a7d3e1256fd1dcd77f38 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Mon, 25 May 2026 18:18:55 -0700 Subject: [PATCH 5/7] fix(prometheus): return None on non-numeric v3 header value to avoid crashing gauge.set() (LIT-2577 Greptile feedback) --- .../_prometheus_v3_remaining_fix.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/_prometheus_v3_remaining_fix.py b/litellm/integrations/_prometheus_v3_remaining_fix.py index 16a2231b5554..844cae3aa1f7 100644 --- a/litellm/integrations/_prometheus_v3_remaining_fix.py +++ b/litellm/integrations/_prometheus_v3_remaining_fix.py @@ -69,7 +69,11 @@ def _get_remaining_from_v3_headers( Prefers ``model_per_key`` (which already scopes by both the API key and the model_group) and falls back to ``key`` (per-key, all-models) if - absent. Returns ``None`` if neither descriptor is present. + absent. Returns ``None`` if neither descriptor is present *or* the value + cannot be coerced to a number. Returning ``None`` (rather than the raw + value) keeps the caller's fallback chain intact -- a malformed header + must NOT propagate into ``Gauge.set()`` which only accepts numbers and + would raise ``TypeError`` mid-callback, breaking the whole logging hook. """ if not additional_headers: return None @@ -77,11 +81,25 @@ def _get_remaining_from_v3_headers( value = additional_headers.get( f"x-ratelimit-{descriptor}-remaining-{rate_limit_type}" ) - if value is not None: + if value is None: + continue + # Accept ints/floats directly; coerce numeric strings; reject anything + # else (will fall through to the next descriptor / sys.maxsize). + if isinstance(value, bool): + # bool is a subclass of int; reject to avoid silently treating + # True/False as 1/0 in a rate-limit gauge. + continue + if isinstance(value, (int, float)): + return value + try: + return int(value) + except (TypeError, ValueError): try: - return int(value) + return float(value) except (TypeError, ValueError): - return value + # Non-numeric header value (e.g. an error string). Skip it + # rather than crash the prometheus gauge. + continue return None From f562d1b26c6a00e0c66f0dcb8808d184738bfdb8 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Mon, 25 May 2026 18:20:07 -0700 Subject: [PATCH 6/7] test(prometheus): cover non-numeric / bool / float v3 header values --- ...est_prometheus_virtual_key_v3_remaining.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/test_litellm/integrations/test_prometheus_virtual_key_v3_remaining.py b/tests/test_litellm/integrations/test_prometheus_virtual_key_v3_remaining.py index 3e547442a044..b135c1eceaaa 100644 --- a/tests/test_litellm/integrations/test_prometheus_virtual_key_v3_remaining.py +++ b/tests/test_litellm/integrations/test_prometheus_virtual_key_v3_remaining.py @@ -255,6 +255,64 @@ def test_string_values_in_additional_headers_are_coerced(prometheus_logger): assert _sample("litellm_remaining_api_key_tokens_for_model").value == 777 +def test_non_numeric_v3_value_does_not_crash_gauge(prometheus_logger): + """ + A non-numeric ``additional_headers`` value (e.g. an error string from a + misconfigured upstream) must NOT propagate into ``Gauge.set()`` which only + accepts numbers. We fall through to ``sys.maxsize`` instead so the + callback never raises mid-request. Pins the fix for Greptile's review. + """ + metadata = {"model_group": MODEL_GROUP} + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers( + { + "x-ratelimit-model_per_key-remaining-requests": "not-a-number", + "x-ratelimit-model_per_key-remaining-tokens": "also-bad", + } + ), + } + + # Must not raise. + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + assert _sample("litellm_remaining_api_key_requests_for_model").value == pytest.approx( + float(sys.maxsize) + ) + assert _sample("litellm_remaining_api_key_tokens_for_model").value == pytest.approx( + float(sys.maxsize) + ) + + +def test_float_v3_value_is_preserved(prometheus_logger): + """Float headers (rare but legal in JSON) pass through without truncation.""" + metadata = {"model_group": MODEL_GROUP} + kwargs = { + "litellm_params": {"metadata": metadata}, + "standard_logging_object": _slp_with_additional_headers( + { + "x-ratelimit-model_per_key-remaining-requests": 12.0, + "x-ratelimit-model_per_key-remaining-tokens": 3.14, + } + ), + } + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + assert _sample("litellm_remaining_api_key_requests_for_model").value == 12.0 + assert _sample("litellm_remaining_api_key_tokens_for_model").value == 3.14 + + # ----- helper unit tests ------------------------------------------------- @@ -298,3 +356,27 @@ def test_get_remaining_from_v3_headers_prefers_model_per_key_over_key(): "x-ratelimit-model_per_key-remaining-requests": 5, } assert _get_remaining_from_v3_headers(headers, "requests") == 5 + + +def test_get_remaining_from_v3_headers_returns_none_on_garbage(): + """Non-numeric values must return None, not the raw value (would crash Gauge.set).""" + headers = {"x-ratelimit-model_per_key-remaining-requests": "definitely-not-a-number"} + assert _get_remaining_from_v3_headers(headers, "requests") is None + + +def test_get_remaining_from_v3_headers_rejects_booleans(): + """``bool`` is a subclass of ``int`` -- reject to avoid treating True/False as 1/0.""" + headers = {"x-ratelimit-model_per_key-remaining-requests": True} + assert _get_remaining_from_v3_headers(headers, "requests") is None + + +def test_get_remaining_from_v3_headers_falls_back_when_first_descriptor_garbage(): + """ + If ``model_per_key`` is garbage but ``key`` is valid, return the ``key`` + value -- don't get stuck on the bad first descriptor. + """ + headers = { + "x-ratelimit-model_per_key-remaining-requests": "garbage", + "x-ratelimit-key-remaining-requests": 42, + } + assert _get_remaining_from_v3_headers(headers, "requests") == 42 From cee3d2130c595a3d598c6405d1c4c78d5084fa93 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Mon, 25 May 2026 18:27:48 -0700 Subject: [PATCH 7/7] chore(prometheus): drop unused importlib.util, document empty except (Greptile/CodeQL feedback) --- litellm/integrations/_prometheus_v3_remaining_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/_prometheus_v3_remaining_fix.py b/litellm/integrations/_prometheus_v3_remaining_fix.py index 844cae3aa1f7..d0e12513fffb 100644 --- a/litellm/integrations/_prometheus_v3_remaining_fix.py +++ b/litellm/integrations/_prometheus_v3_remaining_fix.py @@ -34,7 +34,6 @@ import importlib import importlib.abc -import importlib.util import sys from typing import Any, Dict, Optional, Union @@ -195,6 +194,7 @@ def find_spec(self, fullname, path=None, target=None): try: sys.meta_path.remove(self) except ValueError: + # Already removed (e.g. nested import) — harmless, fall through. pass # Walk the remaining finders so we don't deadlock on ourselves. for finder in list(sys.meta_path):