From 31d8724f8fb5731092f2406ef9c8c6e83162c37b Mon Sep 17 00:00:00 2001 From: Benjamin Nuernberger Date: Fri, 16 Jan 2026 11:07:34 -0800 Subject: [PATCH 1/3] feat: add support for anthropic_messages call type in prompt caching --- .../prompt_caching_deployment_check.py | 3 +- .../test_prompt_caching_deployment_check.py | 71 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/router_unit_tests/test_prompt_caching_deployment_check.py diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index d3d237d9f288..e9c4b69d8ef9 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -61,9 +61,10 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti if ( call_type != CallTypes.completion.value and call_type != CallTypes.acompletion.value + and call_type != CallTypes.anthropic_messages.value ): # only use prompt caching for completion calls verbose_logger.debug( - "litellm.router_utils.pre_call_checks.prompt_caching_deployment_check: skipping adding model id to prompt caching cache, CALL TYPE IS NOT COMPLETION" + "litellm.router_utils.pre_call_checks.prompt_caching_deployment_check: skipping adding model id to prompt caching cache, CALL TYPE IS NOT COMPLETION or ANTHROPIC MESSAGE" ) return diff --git a/tests/router_unit_tests/test_prompt_caching_deployment_check.py b/tests/router_unit_tests/test_prompt_caching_deployment_check.py new file mode 100644 index 000000000000..21a523ca1154 --- /dev/null +++ b/tests/router_unit_tests/test_prompt_caching_deployment_check.py @@ -0,0 +1,71 @@ +import sys +import os +import asyncio + +sys.path.insert(0, os.path.abspath("../..")) + +import pytest +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ( + PromptCachingDeploymentCheck, +) +from litellm.router_utils.prompt_caching_cache import PromptCachingCache +from litellm.caching.dual_cache import DualCache +from litellm.types.utils import CallTypes +from create_mock_standard_logging_payload import create_standard_logging_payload + + +@pytest.mark.asyncio +async def test_anthropic_messages_call_type_is_cached(): + """ + Regression test: Verify that anthropic_messages call type is allowed + in PromptCachingDeploymentCheck.async_log_success_event. + """ + cache = DualCache() + deployment_check = PromptCachingDeploymentCheck(cache=cache) + prompt_cache = PromptCachingCache(cache=cache) + + # Create messages with enough tokens to pass the caching threshold + test_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "test long message here" * 1024, + "cache_control": { + "type": "ephemeral", + "ttl": "5m" + } + } + ] + } + ] + test_model_id = "test-model-id-123" + + # Create a payload with anthropic_messages call type + payload = create_standard_logging_payload() + payload["call_type"] = CallTypes.anthropic_messages.value + payload["messages"] = test_messages + payload["model"] = "anthropic/claude-3-5-sonnet-20240620" + payload["model_id"] = test_model_id + + # Log the success event (should cache the model_id) + await deployment_check.async_log_success_event( + kwargs={"standard_logging_object": payload}, + response_obj={}, + start_time=1234567890.0, + end_time=1234567891.0, + ) + + # Small delay to ensure cache write completes + await asyncio.sleep(0.1) + + # Verify that the model_id was actually cached + cached_result = await prompt_cache.async_get_model_id( + messages=test_messages, + tools=None, + ) + + # This assertion will FAIL if anthropic_messages is filtered out + assert cached_result is not None, "Model ID should be cached for anthropic_messages call type" + assert cached_result["model_id"] == test_model_id, f"Expected {test_model_id}, got {cached_result['model_id']}" \ No newline at end of file From 8bd307b7f741a8f58648dd9f888eacb71fd0aca7 Mon Sep 17 00:00:00 2001 From: Benjamin Nuernberger Date: Fri, 16 Jan 2026 11:57:26 -0800 Subject: [PATCH 2/3] test: move anthropic_messages prompt caching test to main router test file --- .../test_prompt_caching_deployment_check.py | 71 ---------- tests/test_litellm/test_router.py | 122 ++++++++++++++++++ 2 files changed, 122 insertions(+), 71 deletions(-) delete mode 100644 tests/router_unit_tests/test_prompt_caching_deployment_check.py diff --git a/tests/router_unit_tests/test_prompt_caching_deployment_check.py b/tests/router_unit_tests/test_prompt_caching_deployment_check.py deleted file mode 100644 index 21a523ca1154..000000000000 --- a/tests/router_unit_tests/test_prompt_caching_deployment_check.py +++ /dev/null @@ -1,71 +0,0 @@ -import sys -import os -import asyncio - -sys.path.insert(0, os.path.abspath("../..")) - -import pytest -from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ( - PromptCachingDeploymentCheck, -) -from litellm.router_utils.prompt_caching_cache import PromptCachingCache -from litellm.caching.dual_cache import DualCache -from litellm.types.utils import CallTypes -from create_mock_standard_logging_payload import create_standard_logging_payload - - -@pytest.mark.asyncio -async def test_anthropic_messages_call_type_is_cached(): - """ - Regression test: Verify that anthropic_messages call type is allowed - in PromptCachingDeploymentCheck.async_log_success_event. - """ - cache = DualCache() - deployment_check = PromptCachingDeploymentCheck(cache=cache) - prompt_cache = PromptCachingCache(cache=cache) - - # Create messages with enough tokens to pass the caching threshold - test_messages = [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "test long message here" * 1024, - "cache_control": { - "type": "ephemeral", - "ttl": "5m" - } - } - ] - } - ] - test_model_id = "test-model-id-123" - - # Create a payload with anthropic_messages call type - payload = create_standard_logging_payload() - payload["call_type"] = CallTypes.anthropic_messages.value - payload["messages"] = test_messages - payload["model"] = "anthropic/claude-3-5-sonnet-20240620" - payload["model_id"] = test_model_id - - # Log the success event (should cache the model_id) - await deployment_check.async_log_success_event( - kwargs={"standard_logging_object": payload}, - response_obj={}, - start_time=1234567890.0, - end_time=1234567891.0, - ) - - # Small delay to ensure cache write completes - await asyncio.sleep(0.1) - - # Verify that the model_id was actually cached - cached_result = await prompt_cache.async_get_model_id( - messages=test_messages, - tools=None, - ) - - # This assertion will FAIL if anthropic_messages is filtered out - assert cached_result is not None, "Model ID should be cached for anthropic_messages call type" - assert cached_result["model_id"] == test_model_id, f"Expected {test_model_id}, got {cached_result['model_id']}" \ No newline at end of file diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 12fc65d8b060..2dbaea325ccf 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -2253,3 +2253,125 @@ def test_resolve_model_name_from_model_id_multiple_wildcards(): custom_llm_provider="bedrock", ) assert result is None, f"Expected None for non-matching provider, got '{result}'" + + +@pytest.mark.asyncio +async def test_anthropic_messages_call_type_is_cached(): + """ + Regression test: Verify that anthropic_messages call type is allowed + in PromptCachingDeploymentCheck.async_log_success_event. + """ + import asyncio + from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ( + PromptCachingDeploymentCheck, + ) + from litellm.router_utils.prompt_caching_cache import PromptCachingCache + from litellm.caching.dual_cache import DualCache + from litellm.types.utils import CallTypes + from litellm.types.utils import ( + StandardLoggingPayload, + StandardLoggingModelInformation, + StandardLoggingMetadata, + StandardLoggingHiddenParams, + ) + + # Create mock standard logging payload inline + def create_standard_logging_payload() -> StandardLoggingPayload: + return StandardLoggingPayload( + id="test_id", + call_type="completion", + response_cost=0.1, + response_cost_failure_debug_info=None, + status="success", + total_tokens=30, + prompt_tokens=20, + completion_tokens=10, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=1234567890.5, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-3.5-turbo", model_map_value=None + ), + model="gpt-3.5-turbo", + model_id="model-123", + model_group="openai-gpt", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_org_id=None, + user_api_key_alias="test_alias", + user_api_key_team_id="test_team", + user_api_key_user_id="test_user", + user_api_key_team_alias="test_team_alias", + spend_logs_metadata=None, + requester_ip_address="127.0.0.1", + requester_metadata=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address="127.0.0.1", + messages=[{"role": "user", "content": "Hello, world!"}], + response={"choices": [{"message": {"content": "Hi there!"}}]}, + error_str=None, + model_parameters={"stream": True}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.1", + additional_headers=None, + ), + ) + + cache = DualCache() + deployment_check = PromptCachingDeploymentCheck(cache=cache) + prompt_cache = PromptCachingCache(cache=cache) + + # Create messages with enough tokens to pass the caching threshold + test_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "test long message here" * 1024, + "cache_control": { + "type": "ephemeral", + "ttl": "5m" + } + } + ] + } + ] + test_model_id = "test-model-id-123" + + # Create a payload with anthropic_messages call type + payload = create_standard_logging_payload() + payload["call_type"] = CallTypes.anthropic_messages.value + payload["messages"] = test_messages + payload["model"] = "anthropic/claude-3-5-sonnet-20240620" + payload["model_id"] = test_model_id + + # Log the success event (should cache the model_id) + await deployment_check.async_log_success_event( + kwargs={"standard_logging_object": payload}, + response_obj={}, + start_time=1234567890.0, + end_time=1234567891.0, + ) + + # Small delay to ensure cache write completes + await asyncio.sleep(0.1) + + # Verify that the model_id was actually cached + cached_result = await prompt_cache.async_get_model_id( + messages=test_messages, + tools=None, + ) + + # This assertion will FAIL if anthropic_messages is filtered out + assert cached_result is not None, "Model ID should be cached for anthropic_messages call type" + assert cached_result["model_id"] == test_model_id, f"Expected {test_model_id}, got {cached_result['model_id']}" From eeeed58e198b7e7901302ab1bbd6540792bd6818 Mon Sep 17 00:00:00 2001 From: Benjamin Nuernberger Date: Tue, 20 Jan 2026 12:23:05 -0800 Subject: [PATCH 3/3] add tutorial on using claude code with prompt cache routing --- .../claude_code_prompt_cache_routing.md | 43 +++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 44 insertions(+) create mode 100644 docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md diff --git a/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md b/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md new file mode 100644 index 000000000000..bbb294898564 --- /dev/null +++ b/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md @@ -0,0 +1,43 @@ +# Claude Code - Prompt Cache Routing + +Claude's [Prompt Caching](https://platform.claude.com/docs/en/build-with-claude/prompt-caching) feature helps to optimize API usage through attempting to cache prompts and re-use cached prompts during subsequent API calls. This feature is used by Claude Code. + +When LiteLLM [load balancing](../proxy/load_balancing.md) is enabled, to ensure this prompt caching feature still works with Claude Code, LiteLLM needs to be configured to use the `PromptCachingDeploymentCheck` pre-call check. This pre-call check will ensure that API calls that used prompt caching are remembered and that subsequent API calls that try to use that prompt caching are routed to the same model deployment where a cache write occurred. + +## Set Up + +1. Configure the router so that it uses the `PromptCachingDeploymentCheck` (via setting the `optional_pre_call_checks` property), and configure the models so that they can access multiple deployments of Claude; below, we show an example for multiple AWS accounts (referred to as `account-1` and `account-2`, using the `aws_profile_name` property): +```yaml +router_settings: + optional_pre_call_checks: ["prompt_caching"] + +model_list: +- litellm_params: + model: us.anthropic.claude-sonnet-4-5-20250929-v1:0 + aws_profile_name: account-1 + aws_region_name: us-west-2 + model_info: + litellm_provider: bedrock + model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0 +- litellm_params: + model: us.anthropic.claude-sonnet-4-5-20250929-v1:0 + aws_profile_name: account-2 + aws_region_name: us-west-2 + model_info: + litellm_provider: bedrock + model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0 +``` +2. Utilize Claude Code: + 1. Launch Claude Code, which will do a warm-up API call that tries to cache its warm-up prompt and its system prompt. + 2. Wait a few seconds, then quit Claude Code and re-open it. + 3. You'll notice that the warm-up API call successfully gets a cache hit (if using Claude Code in an IDE like VS Code, ensure that you don't do anything between step 2.1 and 2.2 here, otherwise there may not be a cache hit): + 1. Go to the [LiteLLM Request Logs page](../proxy/ui_logs.md) in the Admin UI + 2. Click on the individual requests to see (a) the cache creation and cache read tokens; and (b) the Model ID. In particular, the API call from step 2.1 should show a cache write, and the API call from step 2.2 should show a cache read; in addition, the Model ID should be equal (meaning the API call is getting forwarded to the same AWS account). + +## Related + +- [Claude Code - Quickstart](./claude_responses_api.md) +- [Claude Code - Customer Tracking](./claude_code_customer_tracking.md) +- [Claude Code - Plugin Marketplace](./claude_code_plugin_marketplace.md) +- [Claude Code - WebSearch](./claude_code_websearch.md) +- [Proxy - Load Balancing](../proxy/load_balancing.md) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index dc4d7ce61958..13e5f3d26a9d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -122,6 +122,7 @@ const sidebars = { items: [ "tutorials/claude_responses_api", "tutorials/claude_code_customer_tracking", + "tutorials/claude_code_prompt_cache_routing", "tutorials/claude_code_websearch", "tutorials/claude_mcp", "tutorials/claude_non_anthropic_models",