From 366d8d1db99b58b76773d0e2eeb40189b6238ea1 Mon Sep 17 00:00:00 2001 From: Varshith Date: Tue, 19 May 2026 10:46:54 -0500 Subject: [PATCH 1/4] fix(fallbacks): preserve fallback model in response when using SDK-level fallbacks --- litellm/litellm_core_utils/core_helpers.py | 6 ++ litellm/litellm_core_utils/fallback_utils.py | 10 ++- .../litellm_core_utils/test_fallback_utils.py | 87 ++++++++++++++++++- 3 files changed, 100 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e984df821403..47392621ae3d 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -256,6 +256,12 @@ def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> di "llm_provider-" ): # return raw provider headers (incl. openai-compatible ones) processed_headers[k] = v + elif k.startswith("x-litellm-"): + # LiteLLM's own internal headers (e.g. x-litellm-attempted-fallbacks, + # x-litellm-model-group) are not LLM provider headers and must not be + # prefixed. Downstream consumers (proxy override, callers checking + # whether a fallback happened) look up the bare key. + processed_headers[k] = v else: additional_headers["{}-{}".format("llm_provider", k)] = v diff --git a/litellm/litellm_core_utils/fallback_utils.py b/litellm/litellm_core_utils/fallback_utils.py index daacca85c8af..1606b53e1f98 100644 --- a/litellm/litellm_core_utils/fallback_utils.py +++ b/litellm/litellm_core_utils/fallback_utils.py @@ -7,6 +7,9 @@ safe_deep_copy, filter_internal_params, ) +from litellm.router_utils.add_retry_fallback_headers import ( + add_fallback_headers_to_response, +) from .asyncify import run_async_function @@ -42,7 +45,7 @@ async def async_completion_with_fallbacks(**kwargs): # Try each fallback model most_recent_exception_str: Optional[str] = None - for fallback in fallbacks: + for attempted_fallbacks, fallback in enumerate(fallbacks): try: completion_kwargs = safe_deep_copy(base_kwargs) # Handle dictionary fallback configurations @@ -63,7 +66,10 @@ async def async_completion_with_fallbacks(**kwargs): ) if response is not None: - return response + return add_fallback_headers_to_response( + response=response, + attempted_fallbacks=attempted_fallbacks, + ) except Exception as e: verbose_logger.exception( diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_utils.py b/tests/test_litellm/litellm_core_utils/test_fallback_utils.py index 0c542ff6a1b4..8c73f5cc864b 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_utils.py @@ -1,7 +1,12 @@ +"""Tests for litellm.litellm_core_utils.fallback_utils.""" + import pytest import litellm -from litellm.litellm_core_utils.fallback_utils import async_completion_with_fallbacks +from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.fallback_utils import ( + async_completion_with_fallbacks, +) @pytest.mark.asyncio @@ -41,3 +46,83 @@ async def _fake_acompletion(*, model: str, **kwargs): "primary-model", "fallback-model", ] + + +@pytest.mark.asyncio +async def test_async_completion_with_fallbacks_sets_attempted_fallbacks_header(): + """ + When a fallback succeeds, the response must carry the + `x-litellm-attempted-fallbacks` header so the proxy and other callers can + detect that a fallback occurred. Without it, + `_override_openai_response_model` stamps the requested model back over the + fallback model used. See issue #28241. + """ + response = await async_completion_with_fallbacks( + model="openai/primary-llm", + messages=[{"role": "user", "content": "hi"}], + api_key="fake-key", + mock_response=Exception("forced failure"), + kwargs={ + "fallbacks": [ + { + "model": "openai/backup-llm", + "api_key": "fake-key", + "mock_response": "backup-resp", + } + ] + }, + ) + + hidden_params = getattr(response, "_hidden_params", None) + assert isinstance(hidden_params, dict) + headers = hidden_params.get("additional_headers") or {} + assert headers.get("x-litellm-attempted-fallbacks") == 1 + + +@pytest.mark.asyncio +async def test_async_completion_with_fallbacks_header_is_zero_when_primary_succeeds(): + """ + When the primary model succeeds on the first attempt, the header should be + `0` (no fallback was used). This mirrors the existing router-level + semantics in `async_function_with_fallbacks`. + """ + response = await async_completion_with_fallbacks( + model="openai/primary-llm", + messages=[{"role": "user", "content": "hi"}], + api_key="fake-key", + mock_response="primary-resp", + kwargs={ + "fallbacks": [ + { + "model": "openai/backup-llm", + "api_key": "fake-key", + "mock_response": "backup-resp", + } + ] + }, + ) + + hidden_params = getattr(response, "_hidden_params", None) + assert isinstance(hidden_params, dict) + headers = hidden_params.get("additional_headers") or {} + assert headers.get("x-litellm-attempted-fallbacks") == 0 + assert response.choices[0].message.content == "primary-resp" + + +def test_process_response_headers_preserves_x_litellm_headers(): + """ + `process_response_headers` must not add the `llm_provider-` prefix to + LiteLLM's own internal headers (anything starting with `x-litellm-`). + These are markers set by LiteLLM (e.g. fallback / retry headers); the + proxy and other callers look up the bare key. + """ + result = process_response_headers( + { + "x-litellm-attempted-fallbacks": 1, + "x-litellm-model-group": "gpt-4", + "x-stainless-arch": "arm64", + } + ) + assert result["x-litellm-attempted-fallbacks"] == 1 + assert result["x-litellm-model-group"] == "gpt-4" + assert result["llm_provider-x-stainless-arch"] == "arm64" From 068e4ade1bdde972b48e2c3fe3455035090cfa9f Mon Sep 17 00:00:00 2001 From: Varshith Date: Tue, 19 May 2026 13:06:55 -0500 Subject: [PATCH 2/4] fix(fallbacks): gate x-litellm-* passthrough to trusted callers only The previous patch unconditionally let `x-litellm-*` keys bypass the `llm_provider-` prefix in `process_response_headers`. That function is also called on raw upstream-provider response headers (e.g. from `llm_http_handler.py`), so a malicious provider could return `x-litellm-attempted-fallbacks` and spoof a LiteLLM-internal marker, bypassing the proxy model-override guard. Add a `preserve_litellm_internal_headers` flag (default False). Only `response_metadata.py`, which re-processes the already-built `_hidden_params["additional_headers"]` dict (LiteLLM-owned), passes True. Raw provider header callsites keep the default False, so upstream `x-litellm-*` still gets the `llm_provider-` prefix. Adds a regression test for the spoofing case and renames the existing preserve test to make the trusted-path semantics explicit. --- litellm/litellm_core_utils/core_helpers.py | 15 +++++++-- .../llm_response_utils/response_metadata.py | 3 +- .../litellm_core_utils/test_fallback_utils.py | 31 ++++++++++++++++--- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 47392621ae3d..536ac309f429 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -242,7 +242,18 @@ def _get_parent_otel_span_from_kwargs( return None -def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> dict: +def process_response_headers( + response_headers: Union[httpx.Headers, dict], + preserve_litellm_internal_headers: bool = False, +) -> dict: + """ + `preserve_litellm_internal_headers` must only be True when the input dict is + a LiteLLM-owned structure (e.g. `_hidden_params["additional_headers"]` that + has already been through one round of processing). For raw upstream + provider headers it must remain False, otherwise a malicious provider + returning `x-litellm-*` could spoof LiteLLM-internal markers + (e.g. `x-litellm-attempted-fallbacks`). + """ from litellm.types.utils import OPENAI_RESPONSE_HEADERS openai_headers = {} @@ -256,7 +267,7 @@ def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> di "llm_provider-" ): # return raw provider headers (incl. openai-compatible ones) processed_headers[k] = v - elif k.startswith("x-litellm-"): + elif preserve_litellm_internal_headers and k.startswith("x-litellm-"): # LiteLLM's own internal headers (e.g. x-litellm-attempted-fallbacks, # x-litellm-model-group) are not LLM provider headers and must not be # prefixed. Downstream consumers (proxy override, callers checking diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 06933a6fbcb0..ba870eb9459b 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -49,7 +49,8 @@ def set_hidden_params( result=self.result, litellm_model_name=model, router_model_id=model_id ), "additional_headers": process_response_headers( - self._get_value_from_hidden_params("additional_headers") or {} + self._get_value_from_hidden_params("additional_headers") or {}, + preserve_litellm_internal_headers=True, ), "litellm_model_name": model, } diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_utils.py b/tests/test_litellm/litellm_core_utils/test_fallback_utils.py index 8c73f5cc864b..142602ccb27c 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_utils.py @@ -109,20 +109,41 @@ async def test_async_completion_with_fallbacks_header_is_zero_when_primary_succe assert response.choices[0].message.content == "primary-resp" -def test_process_response_headers_preserves_x_litellm_headers(): +def test_process_response_headers_preserves_x_litellm_headers_when_internal(): """ `process_response_headers` must not add the `llm_provider-` prefix to - LiteLLM's own internal headers (anything starting with `x-litellm-`). - These are markers set by LiteLLM (e.g. fallback / retry headers); the - proxy and other callers look up the bare key. + LiteLLM's own internal headers (anything starting with `x-litellm-`) when + the caller has marked the input as LiteLLM-owned. These are markers set by + LiteLLM (e.g. fallback / retry headers); the proxy and other callers look + up the bare key. """ result = process_response_headers( { "x-litellm-attempted-fallbacks": 1, "x-litellm-model-group": "gpt-4", "x-stainless-arch": "arm64", - } + }, + preserve_litellm_internal_headers=True, ) assert result["x-litellm-attempted-fallbacks"] == 1 assert result["x-litellm-model-group"] == "gpt-4" assert result["llm_provider-x-stainless-arch"] == "arm64" + + +def test_process_response_headers_prefixes_x_litellm_from_raw_provider(): + """ + On raw upstream-provider headers (default `preserve_litellm_internal_headers=False`), + a header whose name starts with `x-litellm-` MUST still get the + `llm_provider-` prefix. Otherwise a malicious provider could return + `x-litellm-attempted-fallbacks` and spoof a LiteLLM-internal marker, + bypassing the proxy model-override guard. + """ + result = process_response_headers( + { + "x-litellm-attempted-fallbacks": 99, + "x-stainless-arch": "arm64", + } + ) + assert "x-litellm-attempted-fallbacks" not in result + assert result["llm_provider-x-litellm-attempted-fallbacks"] == 99 + assert result["llm_provider-x-stainless-arch"] == "arm64" From 3fa2bd34bb0a044a1b0dfb45913f85cf0837f78f Mon Sep 17 00:00:00 2001 From: Varshith Date: Fri, 22 May 2026 09:52:03 -0500 Subject: [PATCH 3/4] fix(fallbacks): ignore preserve_litellm_internal_headers for raw httpx.Headers inputs --- litellm/litellm_core_utils/core_helpers.py | 24 +++++++++++++------ .../litellm_core_utils/test_fallback_utils.py | 20 ++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 536ac309f429..8b96d8355de8 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -247,15 +247,25 @@ def process_response_headers( preserve_litellm_internal_headers: bool = False, ) -> dict: """ - `preserve_litellm_internal_headers` must only be True when the input dict is - a LiteLLM-owned structure (e.g. `_hidden_params["additional_headers"]` that - has already been through one round of processing). For raw upstream - provider headers it must remain False, otherwise a malicious provider - returning `x-litellm-*` could spoof LiteLLM-internal markers - (e.g. `x-litellm-attempted-fallbacks`). + `preserve_litellm_internal_headers` must only be True when the input is a + LiteLLM-owned dict (e.g. `_hidden_params["additional_headers"]` that has + already been through one round of processing). For raw upstream provider + headers — whether passed as `httpx.Headers` or a plain dict — it must + remain False, otherwise a malicious provider returning `x-litellm-*` could + spoof LiteLLM-internal markers (e.g. `x-litellm-attempted-fallbacks`). + + When the input is an `httpx.Headers` object the flag is always treated as + False regardless of what the caller requested, because `httpx.Headers` is + always a raw provider response and can never be LiteLLM-owned. """ from litellm.types.utils import OPENAI_RESPONSE_HEADERS + # Raw httpx.Headers objects come directly from provider HTTP responses and + # must never be treated as LiteLLM-owned, regardless of caller intent. + _preserve = preserve_litellm_internal_headers and isinstance( + response_headers, dict + ) + openai_headers = {} processed_headers = {} additional_headers = {} @@ -267,7 +277,7 @@ def process_response_headers( "llm_provider-" ): # return raw provider headers (incl. openai-compatible ones) processed_headers[k] = v - elif preserve_litellm_internal_headers and k.startswith("x-litellm-"): + elif _preserve and k.startswith("x-litellm-"): # LiteLLM's own internal headers (e.g. x-litellm-attempted-fallbacks, # x-litellm-model-group) are not LLM provider headers and must not be # prefixed. Downstream consumers (proxy override, callers checking diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_utils.py b/tests/test_litellm/litellm_core_utils/test_fallback_utils.py index 142602ccb27c..90a61696e9d9 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_utils.py @@ -1,6 +1,7 @@ """Tests for litellm.litellm_core_utils.fallback_utils.""" import pytest +import httpx import litellm from litellm.litellm_core_utils.core_helpers import process_response_headers @@ -147,3 +148,22 @@ def test_process_response_headers_prefixes_x_litellm_from_raw_provider(): assert "x-litellm-attempted-fallbacks" not in result assert result["llm_provider-x-litellm-attempted-fallbacks"] == 99 assert result["llm_provider-x-stainless-arch"] == "arm64" + + +def test_process_response_headers_ignores_preserve_flag_for_httpx_headers(): + """ + Some providers store raw httpx.Headers directly in _hidden_params["additional_headers"] + without a prior normalization pass. If preserve_litellm_internal_headers=True were + honored for httpx.Headers inputs, a provider returning x-litellm-attempted-fallbacks + could spoof it as a bare LiteLLM-internal marker and make the proxy skip + stamping the correct response model. The flag must be ignored for httpx.Headers. + """ + raw = httpx.Headers( + { + "x-litellm-attempted-fallbacks": "1", + "content-type": "application/json", + } + ) + result = process_response_headers(raw, preserve_litellm_internal_headers=True) + assert "x-litellm-attempted-fallbacks" not in result + assert result["llm_provider-x-litellm-attempted-fallbacks"] == "1" From 1f45398d61fc116c92610f48dfb0352542264155 Mon Sep 17 00:00:00 2001 From: Varshith Date: Sat, 23 May 2026 21:08:24 -0500 Subject: [PATCH 4/4] style(core_helpers): apply black formatting --- litellm/litellm_core_utils/core_helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 8b96d8355de8..98b792efa59e 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -262,9 +262,7 @@ def process_response_headers( # Raw httpx.Headers objects come directly from provider HTTP responses and # must never be treated as LiteLLM-owned, regardless of caller intent. - _preserve = preserve_litellm_internal_headers and isinstance( - response_headers, dict - ) + _preserve = preserve_litellm_internal_headers and isinstance(response_headers, dict) openai_headers = {} processed_headers = {}