diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 48d8a03d549..89066e33cbc 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -484,10 +484,14 @@ def _prepare_completion_kwargs( if "output_config" in extra_kwargs: request_data["output_config"] = extra_kwargs["output_config"] + custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider") ( openai_request, tool_name_mapping, - ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(request_data) + ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( + request_data, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + ) if openai_request is None: raise ValueError("Failed to translate request to OpenAI format") @@ -526,6 +530,10 @@ def _prepare_completion_kwargs( if key not in excluded_keys and key not in completion_kwargs and value is not None: completion_kwargs[key] = value + explicit_prompt_cache_key: Final = extra_kwargs.get("prompt_cache_key") + if explicit_prompt_cache_key is not None: + completion_kwargs["prompt_cache_key"] = explicit_prompt_cache_key + # Normalize reasoning_effort based on model capabilities # (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported) # Must run BEFORE _route_openai_thinking, which prepends "responses/" diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index e45414b4a73..974a3b4bb73 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -4,8 +4,10 @@ from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Literal, cast +import litellm from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, + prompt_cache_key_from_user_id, ) # OpenAI has a 64-character limit for function/tool names @@ -13,6 +15,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64 TOOL_NAME_HASH_LENGTH: Final = 8 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55 +PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"}) def truncate_tool_name(name: str) -> str: @@ -148,7 +151,7 @@ def translate_completion_input_params(self, kwargs) -> ChatCompletionRequest | N return result def translate_completion_input_params_with_tool_mapping( - self, kwargs + self, kwargs, *, custom_llm_provider: str | None = None ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -179,7 +182,10 @@ def translate_completion_input_params_with_tool_mapping( ( translated_body, tool_name_mapping, - ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(anthropic_message_request=request_body) + ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request=request_body, + custom_llm_provider=custom_llm_provider, + ) return translated_body, tool_name_mapping @@ -907,16 +913,34 @@ def _add_system_message_to_messages( ChatCompletionSystemMessage(role="system", content=openai_system_content), ) + @staticmethod + def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool: + if not model or not custom_llm_provider: + return False + if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: + return False + supported_params: Final = litellm.get_supported_openai_params( + model=model, custom_llm_provider=custom_llm_provider + ) + return "prompt_cache_key" in (supported_params or ()) + def _translate_metadata_to_openai( self, anthropic_message_request: AnthropicMessagesRequest, new_kwargs: ChatCompletionRequest, + *, + custom_llm_provider: str | None = None, ) -> None: """Translate metadata fields from Anthropic request to OpenAI request.""" if "metadata" in anthropic_message_request: metadata: Final = anthropic_message_request["metadata"] if metadata and "user_id" in metadata: new_kwargs["user"] = metadata["user_id"] + prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"]) + if prompt_cache_key is not None and self._supports_prompt_cache_key( + anthropic_message_request.get("model"), custom_llm_provider + ): + new_kwargs["prompt_cache_key"] = prompt_cache_key if "litellm_metadata" in anthropic_message_request: # metadata will be passed to litellm.acompletion(), it's a litellm_param @@ -1069,7 +1093,10 @@ def _copy_untranslated_anthropic_params( new_kwargs[k] = v def translate_anthropic_to_openai( - self, anthropic_message_request: AnthropicMessagesRequest + self, + anthropic_message_request: AnthropicMessagesRequest, + *, + custom_llm_provider: str | None = None, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1103,6 +1130,7 @@ def translate_anthropic_to_openai( self._translate_metadata_to_openai( anthropic_message_request=anthropic_message_request, new_kwargs=new_kwargs, + custom_llm_provider=custom_llm_provider, ) ## CONVERT TOOL CHOICE self._translate_tool_choice_to_openai( diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index e6d8686b466..843cda249c5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -105,7 +105,8 @@ def _build_responses_kwargs( # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) excluded: Final = {"anthropic_messages"} - for key, value in _forwarded_kwargs(extra_kwargs).items(): + forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + for key, value in forwarded_kwargs.items(): if key == "litellm_logging_obj" and value is not None: from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObject, @@ -121,6 +122,10 @@ def _build_responses_kwargs( elif key not in excluded and key not in responses_kwargs and value is not None: responses_kwargs[key] = value + explicit_prompt_cache_key: Final = forwarded_kwargs.get("prompt_cache_key") + if explicit_prompt_cache_key is not None: + responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key + return responses_kwargs diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 21a8cb9501e..9238433151a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -18,6 +18,7 @@ ) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, + prompt_cache_key_from_user_id, ) from litellm.types.llms.anthropic import ( AllAnthropicPassThroughMessageValues, @@ -452,10 +453,13 @@ def translate_request( if openai_cm is not None: responses_kwargs["context_management"] = openai_cm - # metadata user_id -> user + # metadata user_id -> user and prompt_cache_key metadata: Final = anthropic_request.get("metadata") if isinstance(metadata, dict) and "user_id" in metadata: responses_kwargs["user"] = str(metadata["user_id"])[:64] + prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"]) + if prompt_cache_key is not None: + responses_kwargs["prompt_cache_key"] = prompt_cache_key return responses_kwargs diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py index 46091cd89a2..c5abcf8c04c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -1,8 +1,17 @@ import os +from typing import Final import litellm from litellm.types.utils import ModelInfo +OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH: Final = 64 + + +def prompt_cache_key_from_user_id(user_id: object) -> str | None: + if user_id is None: + return None + return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None + def is_reasoning_auto_summary_enabled() -> bool: """Check whether the default 'summary: detailed' injection is enabled (opt-in).""" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index edfc50c99f6..beb6612497c 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -917,6 +917,7 @@ class ChatCompletionRequest(TypedDict, total=False): seed: int service_tier: str safety_identifier: str + prompt_cache_key: str # writable-ok: the /v1/messages adapter assigns it after construction stop: str | list[str] stream_options: dict temperature: float diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 1185893d428..6edb3610bab 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -4,6 +4,8 @@ import pytest +import litellm + sys.path.insert(0, os.path.abspath("../../../../..")) @@ -635,6 +637,94 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system(): ] +def _translate_with_metadata( + model: str, metadata: dict[str, Any], custom_llm_provider: str | None +) -> dict[str, Any]: + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": model, + "max_tokens": 100, + "metadata": metadata, + "messages": [{"role": "user", "content": "hi"}], + }, + custom_llm_provider=custom_llm_provider, + ) + return cast(dict[str, Any], openai_request) + + +def test_translate_anthropic_to_openai_maps_user_id_to_prompt_cache_key_for_openai(): + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, "openai") + assert openai_request["user"] == "session-abc" + assert openai_request["prompt_cache_key"] == "session-abc" + + +def test_translate_anthropic_to_openai_truncates_prompt_cache_key_but_keeps_full_user(): + long_id = "".join(str(i % 10) for i in range(100)) + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": long_id}, "openai") + assert openai_request["user"] == long_id + assert openai_request["prompt_cache_key"] == long_id[:64] + assert len(openai_request["prompt_cache_key"]) == 64 + + +@pytest.mark.parametrize("model", ["azure/my-gpt-5-deployment", "my-gpt-5-deployment"]) +def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: str): + openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, "azure") + assert openai_request["prompt_cache_key"] == "session-abc" + + +@pytest.mark.parametrize( + "model, custom_llm_provider", + [ + ("gemini/gemini-2.5-pro", "gemini"), + ("vertex_ai/gemini-2.5-pro", "vertex_ai"), + ("anthropic/claude-sonnet-4-5", "anthropic"), + ("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", "bedrock"), + ("no-such-model-lit5875", "no-such-provider-lit5875"), + ], +) +def test_translate_anthropic_to_openai_skips_prompt_cache_key_when_provider_lacks_it( + model: str, custom_llm_provider: str +): + openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, custom_llm_provider) + assert openai_request["user"] == "session-abc" + assert "prompt_cache_key" not in openai_request + + +def test_translate_anthropic_to_openai_skips_prompt_cache_key_for_chained_litellm_proxy(): + assert "prompt_cache_key" in litellm.get_supported_openai_params( + model="xai", custom_llm_provider="litellm_proxy" + ) + openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": "session-abc"}, "litellm_proxy") + assert openai_request["user"] == "session-abc" + assert "prompt_cache_key" not in openai_request + + +def test_translate_anthropic_to_openai_skips_prompt_cache_key_without_provider(): + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, None) + assert openai_request["user"] == "session-abc" + assert "prompt_cache_key" not in openai_request + + +@pytest.mark.parametrize("user_id", ["", None]) +def test_translate_anthropic_to_openai_skips_prompt_cache_key_for_empty_or_null_user_id(user_id: str | None): + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": user_id}, "openai") + assert openai_request["user"] == user_id + assert "prompt_cache_key" not in openai_request + + +def test_translate_anthropic_to_openai_without_metadata_sets_neither_user_nor_prompt_cache_key(): + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "openai/gpt-5.6-luna", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + custom_llm_provider="openai", + ) + assert "user" not in openai_request + assert "prompt_cache_key" not in openai_request + + def test_translate_openai_content_to_anthropic_empty_function_arguments(): """Test that empty function arguments are handled safely and don't cause JSON parsing errors.""" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py new file mode 100644 index 00000000000..5b7f2a60f68 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py @@ -0,0 +1,70 @@ +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) + +from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, +) + +MESSAGES = [{"role": "user", "content": "hello"}] + + +def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, object] | None = None): + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model=model, + metadata={"user_id": "session-abc"}, + thinking=thinking, + extra_kwargs=extra_kwargs, + ) + return completion_kwargs + + +def test_prepare_completion_kwargs_derives_prompt_cache_key_for_openai_provider(): + completion_kwargs = _prepare("openai/gpt-5.6-luna", {"custom_llm_provider": "openai"}) + assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["prompt_cache_key"] == "session-abc" + + +def test_prepare_completion_kwargs_prefers_explicit_prompt_cache_key_over_derived(): + completion_kwargs = _prepare( + "openai/gpt-5.6-luna", + {"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"}, + ) + assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["prompt_cache_key"] == "explicit-key" + + +@pytest.mark.parametrize( + "model, extra_kwargs", + [ + ("gemini/gemini-2.5-pro", {"custom_llm_provider": "gemini"}), + ("openai/gpt-5.6-luna", {}), + ], +) +def test_prepare_completion_kwargs_skips_prompt_cache_key_without_provider_support( + model: str, extra_kwargs: dict[str, object] +): + completion_kwargs = _prepare(model, extra_kwargs) + assert completion_kwargs["user"] == "session-abc" + assert "prompt_cache_key" not in completion_kwargs + + +def test_prepare_completion_kwargs_skips_prompt_cache_key_for_chained_litellm_proxy(): + completion_kwargs = _prepare("litellm_proxy/xai", {"custom_llm_provider": "litellm_proxy"}) + assert completion_kwargs["user"] == "session-abc" + assert "prompt_cache_key" not in completion_kwargs + + +def test_prepare_completion_kwargs_keeps_prompt_cache_key_through_responses_reroute(): + completion_kwargs = _prepare( + "openai/gpt-5.6-luna", + {"custom_llm_provider": "openai"}, + thinking={"type": "enabled", "budget_tokens": 1024}, + ) + assert completion_kwargs["model"] == "responses/openai/gpt-5.6-luna" + assert completion_kwargs["prompt_cache_key"] == "session-abc" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py new file mode 100644 index 00000000000..7ef3077f9d7 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -0,0 +1,45 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) + +from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler import ( + _build_responses_kwargs, +) + +MESSAGES = [{"role": "user", "content": "hello"}] + + +def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": "session-abc"}, + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert responses_kwargs["user"] == "session-abc" + assert responses_kwargs["prompt_cache_key"] == "session-abc" + + +def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": "session-abc"}, + extra_kwargs={"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"}, + ) + assert responses_kwargs["user"] == "session-abc" + assert responses_kwargs["prompt_cache_key"] == "explicit-key" + + +def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert "user" not in responses_kwargs + assert "prompt_cache_key" not in responses_kwargs diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 876213eda3f..297f2052b83 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -992,6 +992,29 @@ def test_metadata_user_id_truncated_to_64_chars(self): kwargs = _ADAPTER.translate_request(req) assert len(kwargs["user"]) == 64 + def test_metadata_user_id_mapped_to_prompt_cache_key(self): + req = _make_request(metadata={"user_id": "user-42"}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["prompt_cache_key"] == "user-42" + + def test_metadata_user_id_prompt_cache_key_truncated_to_first_64_chars(self): + long_id = "".join(str(i % 10) for i in range(100)) + req = _make_request(metadata={"user_id": long_id}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["prompt_cache_key"] == long_id[:64] + assert len(kwargs["prompt_cache_key"]) == 64 + + def test_metadata_empty_user_id_sets_no_prompt_cache_key(self): + req = _make_request(metadata={"user_id": ""}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["user"] == "" + assert "prompt_cache_key" not in kwargs + + def test_metadata_null_user_id_sets_no_prompt_cache_key(self): + req = _make_request(metadata={"user_id": None}) + kwargs = _ADAPTER.translate_request(req) + assert "prompt_cache_key" not in kwargs + def test_no_optional_fields_does_not_add_spurious_keys(self): req = _make_request() kwargs = _ADAPTER.translate_request(req) @@ -1005,6 +1028,7 @@ def test_no_optional_fields_does_not_add_spurious_keys(self): "text", "context_management", "user", + "prompt_cache_key", ): assert key not in kwargs, f"unexpected key: {key}"