From 85497740ff4b88d6a5d8b38fd7b3f29afa156c96 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Tue, 14 Apr 2026 05:40:42 +0200 Subject: [PATCH] fix(caching): add Responses API params to cache key allow-list --- .../litellm_core_utils/model_param_helper.py | 23 +++++++++ tests/local_testing/test_unit_test_caching.py | 51 +++++++++++++++++++ tests/test_litellm/test_model_param_helper.py | 29 +++++++++++ 3 files changed, 103 insertions(+) diff --git a/litellm/litellm_core_utils/model_param_helper.py b/litellm/litellm_core_utils/model_param_helper.py index 66b174feac4..35e744be3a6 100644 --- a/litellm/litellm_core_utils/model_param_helper.py +++ b/litellm/litellm_core_utils/model_param_helper.py @@ -11,6 +11,10 @@ CompletionCreateParamsStreaming as TextCompletionCreateParamsStreaming, ) from openai.types.embedding_create_params import EmbeddingCreateParams +from openai.types.responses.response_create_params import ( + ResponseCreateParamsNonStreaming, + ResponseCreateParamsStreaming, +) from litellm._logging import verbose_logger from litellm.types.rerank import RerankRequest @@ -65,6 +69,9 @@ def _get_all_llm_api_params() -> Set[str]: ModelParamHelper._get_litellm_supported_transcription_kwargs() ) rerank_kwargs = ModelParamHelper._get_litellm_supported_rerank_kwargs() + responses_api_kwargs = ( + ModelParamHelper._get_litellm_supported_responses_api_kwargs() + ) exclude_kwargs = ModelParamHelper._get_exclude_kwargs() combined_kwargs = chat_completion_kwargs.union( @@ -72,6 +79,7 @@ def _get_all_llm_api_params() -> Set[str]: embedding_kwargs, transcription_kwargs, rerank_kwargs, + responses_api_kwargs, ) combined_kwargs = combined_kwargs.difference(exclude_kwargs) return combined_kwargs @@ -167,6 +175,21 @@ def _get_litellm_supported_transcription_kwargs() -> Set[str]: verbose_logger.debug("Error getting transcription kwargs %s", str(e)) return set() + @staticmethod + def _get_litellm_supported_responses_api_kwargs() -> Set[str]: + """ + Get the litellm supported responses API kwargs + + This follows the OpenAI API Spec + """ + non_streaming_params: Set[str] = set( + getattr(ResponseCreateParamsNonStreaming, "__annotations__", {}).keys() + ) + streaming_params: Set[str] = set( + getattr(ResponseCreateParamsStreaming, "__annotations__", {}).keys() + ) + return non_streaming_params.union(streaming_params) + @staticmethod def _get_exclude_kwargs() -> Set[str]: """ diff --git a/tests/local_testing/test_unit_test_caching.py b/tests/local_testing/test_unit_test_caching.py index fa5cf802546..e4ee65a2aa2 100644 --- a/tests/local_testing/test_unit_test_caching.py +++ b/tests/local_testing/test_unit_test_caching.py @@ -131,6 +131,57 @@ def test_get_cache_key_text_completion(): assert cache_key_2 == cache_key_3 +def test_get_cache_key_responses_api(): + """ + Regression test: two /v1/responses calls that differ only in + `instructions` (or any Responses-API-only param) must produce + different cache keys. Mirrors the chat / embedding / text-completion + cache-key tests above. + """ + cache = Cache() + + base_kwargs = { + "model": "openai/gpt-4.1", + "input": [{"role": "user", "content": "what is the weather"}], + "temperature": 0.3, + } + + kwargs_a = { + **base_kwargs, + "instructions": "summarize the weather on 10th May", + } + kwargs_b = { + **base_kwargs, + "instructions": "summarize the weather on 7th May", + } + + key_a = cache.get_cache_key(**kwargs_a) + key_b = cache.get_cache_key(**kwargs_b) + + assert isinstance(key_a, str) and len(key_a) > 0 + assert key_a != key_b, ( + "instructions must be part of the Responses API cache key" + ) + + # Sanity: identical payloads must still collide (cache hits still work) + key_a_again = cache.get_cache_key(**kwargs_a) + assert key_a == key_a_again + + # Spot-check a handful of other Responses-only params individually. + for param, value_x, value_y in [ + ("previous_response_id", "resp_aaa", "resp_bbb"), + ("reasoning", {"effort": "low"}, {"effort": "high"}), + ("include", ["reasoning.encrypted_content"], []), + ("max_output_tokens", 100, 500), + ("background", True, False), + ]: + kx = {**base_kwargs, param: value_x} + ky = {**base_kwargs, param: value_y} + assert cache.get_cache_key(**kx) != cache.get_cache_key(**ky), ( + f"Responses-API param `{param}` is not part of the cache key" + ) + + def test_get_hashed_cache_key(): cache = Cache() cache_key = "model:gpt-3.5-turbo,messages:Hello world" diff --git a/tests/test_litellm/test_model_param_helper.py b/tests/test_litellm/test_model_param_helper.py index c6e4b864a22..2012abec547 100644 --- a/tests/test_litellm/test_model_param_helper.py +++ b/tests/test_litellm/test_model_param_helper.py @@ -31,3 +31,32 @@ def test_get_standard_logging_model_parameters_excludes_prompt_content(): assert "prompt" not in result assert "input" not in result assert result == {"temperature": 0.5} + + +def test_get_all_llm_api_params_includes_responses_api(): + """ + Regression guard for the Responses API cache-key bug: + Responses-API-only kwargs must be present in the cache-key allow-list, + otherwise Cache.get_cache_key() silently drops them and two requests + that differ only in (e.g.) `instructions` collide on the same key. + """ + all_params = ModelParamHelper._get_all_llm_api_params() + responses_only_params = { + "instructions", + "previous_response_id", + "reasoning", + "include", + "store", + "background", + "max_output_tokens", + "max_tool_calls", + "prompt_cache_key", + "prompt_cache_retention", + "context_management", + "conversation", + "safety_identifier", + } + missing = responses_only_params - all_params + assert missing == set(), ( + f"Responses-API kwargs missing from cache-key allow-list: {sorted(missing)}" + )