From 04afc962b10d905e2ceabdfe121c65367941d513 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 12:29:43 -0700 Subject: [PATCH 1/4] feat(anthropic): add enable_anthropic_prompt_caching for automatic cache_control injection Anthropic only caches a prompt when the request carries explicit cache_control breakpoints, unlike OpenAI where prompt caching is automatic and needs no configuration. Today litellm can inject those breakpoints server-side, but only when an admin hand-writes cache_control_injection_points into a model's litellm_params (or router_settings.default_litellm_params). Clients such as Claude Code and Claude Desktop never set cache_control themselves, and the admin recipe is easy to miss, so Anthropic traffic through the proxy silently pays full price on every repeated prefix. This adds an opt-in litellm_settings flag, enable_anthropic_prompt_caching. When it is on and the request has no injection points configured and no client-supplied cache_control, litellm synthesizes a default pair of breakpoints (the system prompt and the trailing turn) so the stable prefix is cached while the breakpoint advances with the conversation. It is wired into both surfaces: /chat/completions seeds the points before the existing prompt-management gate, and /v1/messages resolves them in maybe_inject_cache_control, so the existing AnthropicCacheControlHook applies them unchanged and keeps its four-block cap and its refusal to overwrite client breakpoints. The default is off, so no existing deployment changes behavior. Injection is gated to providers that actually consume cache_control markers (anthropic and bedrock) and to models the cost map flags as supporting prompt caching; note that supports_prompt_caching alone is not a sufficient gate, since OpenAI, Azure and Gemini models report it as well but never take cache_control markers. The default ttl is Anthropic's 5 minute ephemeral cache, with an optional anthropic_prompt_caching_ttl of "5m" or "1h"; ttl is also added to ChatCompletionCachedContent, which the bedrock and anthropic transforms already read at runtime but the type never declared Resolves LIT-4478 --- litellm/__init__.py | 2 + .../anthropic_cache_control_hook.py | 117 ++++++++++++++- .../messages/handler.py | 8 +- litellm/main.py | 25 ++++ litellm/types/llms/openai.py | 1 + .../test_anthropic_cache_control_hook.py | 136 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 7 files changed, 291 insertions(+), 3 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 6e2a03b7c7c5..c2a98497d624 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -315,6 +315,8 @@ def _dev_env_hot_reload_enabled() -> bool: disable_add_transform_inline_image_block: bool = False disable_add_user_agent_to_request_tags: bool = False disable_anthropic_gemini_context_caching_transform: bool = False +enable_anthropic_prompt_caching: bool = False +anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = None disable_vertex_batch_output_transformation: bool = False extra_spend_tag_headers: Optional[List[str]] = None in_memory_llm_clients_cache: "LLMClientCache" diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 608fdebc1d96..026d8b8e82e1 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -296,18 +296,133 @@ def apply_to_anthropic_messages_request( return processed_messages, processed_system, remaining_points + @staticmethod + def _default_control() -> ChatCompletionCachedContent: + """Build the cache_control block for auto-injected breakpoints. + + Defaults to Anthropic's 5-minute ephemeral cache; honors the optional + ``litellm.anthropic_prompt_caching_ttl`` override ("5m" or "1h"). + """ + import litellm + + ttl = litellm.anthropic_prompt_caching_ttl + if ttl == "5m" or ttl == "1h": + return ChatCompletionCachedContent(type="ephemeral", ttl=ttl) + return ChatCompletionCachedContent(type="ephemeral") + + @staticmethod + def _request_has_cache_control(messages: list[AllMessageValues], system: Optional[Union[str, list]]) -> bool: + """Return True if the request already carries any client-supplied cache_control. + + When the client (e.g. Claude Code) already marks its own breakpoints we + stand down entirely rather than add more, per the auto-caching contract. + """ + if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): + return True + if isinstance(system, list): + return any(isinstance(block, dict) and block.get("cache_control") is not None for block in system) + return False + + @staticmethod + def get_default_injection_points( + messages: list[AllMessageValues], + system: Optional[Union[str, list]], + model: str, + custom_llm_provider: Optional[str], + ) -> list[CacheControlInjectionPoint]: + """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. + + Caches the system prompt and the trailing turn, so the stable prefix + (system + tools + history) is reused while the breakpoint advances with + the conversation. Returns [] (stand down) when the flag is off, the + provider does not consume cache_control breakpoints (only anthropic / + bedrock do), the model lacks prompt-caching support, or the request + already carries client-supplied cache_control. + """ + import litellm + + if litellm.enable_anthropic_prompt_caching is not True: + return [] + + provider = custom_llm_provider + if provider is None: + from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider, + ) + + try: + _, provider, _, _ = get_llm_provider(model=model) + except Exception: # noqa: BLE001 # unroutable model must never block the call, just skip auto-caching + return [] + + if provider not in ("anthropic", "bedrock"): + return [] + + from litellm.utils import supports_prompt_caching + + if not supports_prompt_caching(model=model, custom_llm_provider=provider): + return [] + + if AnthropicCacheControlHook._request_has_cache_control(messages, system): + return [] + + control = AnthropicCacheControlHook._default_control() + points: list[CacheControlInjectionPoint] = [ + CacheControlMessageInjectionPoint(location="message", role="system", index=None, control=control), + CacheControlMessageInjectionPoint(location="message", role=None, index=-1, control=control), + ] + return points + + @staticmethod + def maybe_seed_default_injection_points( + non_default_params: dict[str, Any], + messages: list[AllMessageValues], + model: str, + custom_llm_provider: Optional[str], + ) -> None: + """For /chat/completions: add default injection points to the request params. + + No-op when injection points are already configured (explicit config wins). + Seeding the param lets the existing prompt-management gate and the + AnthropicCacheControlHook run unchanged. + """ + if non_default_params.get("cache_control_injection_points"): + return + points = AnthropicCacheControlHook.get_default_injection_points( + messages=messages, + system=None, + model=model, + custom_llm_provider=custom_llm_provider, + ) + if points: + non_default_params["cache_control_injection_points"] = points + @staticmethod def maybe_inject_cache_control( messages: List[Dict], system: str | list | None, kwargs: Dict[str, Any], + model: Optional[str] = None, + custom_llm_provider: Optional[str] = None, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. + When none are configured but ``litellm.enable_anthropic_prompt_caching`` + is on, synthesize default breakpoints for the native /v1/messages path. Pops the key from kwargs; if remaining (non-message) points exist they are written back so downstream transforms can handle them. """ - injection_points = kwargs.pop("cache_control_injection_points", None) + configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list + Optional[list[CacheControlInjectionPoint]], kwargs.pop("cache_control_injection_points", None) + ) + injection_points: list[CacheControlInjectionPoint] = configured or [] + if not injection_points and model is not None: + injection_points = AnthropicCacheControlHook.get_default_injection_points( + messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages + system=system, + model=model, + custom_llm_provider=custom_llm_provider, + ) if not injection_points: return messages, system diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index dd983f0c344d..c205d7516e6a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -236,7 +236,9 @@ async def anthropic_messages( AnthropicCacheControlHook, ) - messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs) + messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider + ) original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False) @@ -425,7 +427,9 @@ def anthropic_messages_handler( AnthropicCacheControlHook, ) - messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs) + messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider + ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index 6fd68921fb02..4a9b5bdc76f1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -510,6 +510,19 @@ async def acompletion( ######################################################### ######################################################### litellm_logging_obj = kwargs.get("litellm_logging_obj", None) + + from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + ) + from litellm.types.llms.openai import AllMessageValues + + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=kwargs, + messages=cast(list[AllMessageValues], messages), # cast-ok: acompletion types messages as a bare List + model=model, + custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs + ) + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=kwargs.get("prompt_id", None), @@ -5055,6 +5068,18 @@ def completion( # type: ignore litellm_params = {} # used to prevent unbound var errors ## PROMPT MANAGEMENT HOOKS ## + from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + ) + from litellm.types.llms.openai import AllMessageValues + + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=non_default_params, + messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List + model=model, + custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs + ) + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=prompt_id, non_default_params=non_default_params diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index daac1e4506f4..9f689a2dd318 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -529,6 +529,7 @@ class ChatCompletionDeltaToolCallChunk(TypedDict, total=False): class ChatCompletionCachedContent(TypedDict): type: Literal["ephemeral"] + ttl: NotRequired[Literal["5m", "1h"]] class ChatCompletionThinkingBlock(TypedDict, total=False): diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 4664cc863038..ef63555bdac6 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1533,3 +1533,139 @@ def test_existing_cache_control_counted_toward_limit(self): sys_blocks = sum(1 for b in (result_sys or []) if isinstance(b, dict) and b.get("cache_control") is not None) total_blocks = sys_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(m) for m in result_msgs) assert total_blocks <= 4 + + +class TestEnableAnthropicPromptCaching: + """Auto-injected default breakpoints via litellm.enable_anthropic_prompt_caching.""" + + MESSAGES: List[AllMessageValues] = [ + {"role": "system", "content": "a long system prompt"}, + {"role": "user", "content": "first turn"}, + {"role": "assistant", "content": "a reply"}, + {"role": "user", "content": "latest turn"}, + ] + + def _points(self, model="claude-sonnet-4-5", provider="anthropic", messages=None, system=None): + return AnthropicCacheControlHook.get_default_injection_points( + messages=copy.deepcopy(self.MESSAGES) if messages is None else messages, + system=system, + model=model, + custom_llm_provider=provider, + ) + + def test_disabled_by_default(self): + assert litellm.enable_anthropic_prompt_caching is False + assert self._points() == [] + + def test_injects_system_and_trailing_turn(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert self._points() == [ + {"location": "message", "role": "system", "index": None, "control": {"type": "ephemeral"}}, + {"location": "message", "role": None, "index": -1, "control": {"type": "ephemeral"}}, + ] + + def test_bedrock_claude_is_injected(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + points = self._points(model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", provider="bedrock") + assert [p["index"] for p in points] == [None, -1] + + @pytest.mark.parametrize("model, provider", [("gpt-4o", "openai"), ("gemini-2.0-flash", "gemini")]) + def test_non_anthropic_providers_never_injected(self, monkeypatch, model, provider): + """These report supports_prompt_caching=True but never consume cache_control markers.""" + from litellm.utils import supports_prompt_caching + + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert supports_prompt_caching(model=model, custom_llm_provider=provider) is True + assert self._points(model=model, provider=provider) == [] + + def test_model_without_caching_support_not_injected(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert self._points(model="anthropic.claude-3-5-sonnet-20240620-v1:0", provider="bedrock") == [] + + def test_stands_down_when_client_sent_cache_control(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + messages = [ + {"role": "system", "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": "latest turn"}, + ] + assert self._points(messages=messages) == [] + + def test_stands_down_when_system_block_has_cache_control(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}] + assert self._points(messages=[{"role": "user", "content": "hi"}], system=system) == [] + + def test_default_ttl_is_anthropics_five_minute_cache(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert all(p["control"] == {"type": "ephemeral"} for p in self._points()) + + @pytest.mark.parametrize("ttl", ["5m", "1h"]) + def test_ttl_override_applied(self, monkeypatch, ttl): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + monkeypatch.setattr(litellm, "anthropic_prompt_caching_ttl", ttl) + assert all(p["control"] == {"type": "ephemeral", "ttl": ttl} for p in self._points()) + + def test_seed_does_not_override_configured_points(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + configured = [{"location": "message", "role": "user", "index": 0}] + params = {"cache_control_injection_points": configured} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert params["cache_control_injection_points"] is configured + + def test_seed_adds_defaults_when_enabled(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + params: dict = {} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert [p["index"] for p in params["cache_control_injection_points"]] == [None, -1] + + def test_seed_is_noop_when_disabled(self): + params: dict = {} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert params == {} + + def test_v1_messages_applies_defaults_end_to_end(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + messages = [ + {"role": "user", "content": [{"type": "text", "text": "first"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "reply"}]}, + {"role": "user", "content": [{"type": "text", "text": "latest"}]}, + ] + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, + "a system prompt", + {}, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + + assert result_sys == [{"type": "text", "text": "a system prompt", "cache_control": {"type": "ephemeral"}}] + assert result_msgs[-1]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in result_msgs[0]["content"][-1] + + def test_v1_messages_is_noop_when_disabled(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, + "sys", + {}, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + + assert result_sys == "sys" + assert result_msgs == messages diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 87c760257d1d..501a30110c0d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -21870,6 +21870,11 @@ export interface components { }; /** ChatCompletionCachedContent */ ChatCompletionCachedContent: { + /** + * Ttl + * @enum {string} + */ + ttl?: "5m" | "1h"; /** * Type * @constant From f7a3e22b228f82a551a41249caacde6099f27b87 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 12:43:33 -0700 Subject: [PATCH 2/4] feat(anthropic): allow enabling prompt caching via environment variables Both enable_anthropic_prompt_caching and anthropic_prompt_caching_ttl are now read from LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING and LITELLM_ANTHROPIC_PROMPT_CACHING_TTL at import, so the flag can be turned on without a config file. An unsupported ttl falls back to the provider default rather than reaching the provider verbatim --- litellm/__init__.py | 7 ++- .../test_anthropic_cache_control_hook.py | 52 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index c2a98497d624..2f6643c644c2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -315,8 +315,11 @@ def _dev_env_hot_reload_enabled() -> bool: disable_add_transform_inline_image_block: bool = False disable_add_user_agent_to_request_tags: bool = False disable_anthropic_gemini_context_caching_transform: bool = False -enable_anthropic_prompt_caching: bool = False -anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = None +enable_anthropic_prompt_caching: bool = os.getenv("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", "false").lower() == "true" +_anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL") +anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = ( + "1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None +) disable_vertex_batch_output_transformation: bool = False extra_spend_tag_headers: Optional[List[str]] = None in_memory_llm_clients_cache: "LLMClientCache" diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index ef63555bdac6..6a67f1d66435 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -2,7 +2,9 @@ import datetime import json import os +import subprocess import sys +import textwrap import unittest from typing import List, Optional, Tuple from unittest.mock import ANY, MagicMock, Mock, patch @@ -1669,3 +1671,53 @@ def test_v1_messages_is_noop_when_disabled(self): assert result_sys == "sys" assert result_msgs == messages + + +class TestAnthropicPromptCachingEnvVars: + """Both settings are read from the environment at import, so an admin can enable + auto-caching without a config file. Each case re-imports litellm in a subprocess + so the env is read fresh without contaminating this process's module graph. + """ + + @staticmethod + def _import_litellm_with_env(env_override: dict) -> Tuple[bool, Optional[str]]: + env = os.environ.copy() + env.pop("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", None) + env.pop("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL", None) + env.update(env_override) + script = textwrap.dedent( + """ + import json, litellm + print(json.dumps([litellm.enable_anthropic_prompt_caching, litellm.anthropic_prompt_caching_ttl])) + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, env=env, timeout=300 + ) + assert result.returncode == 0, result.stderr + enabled, ttl = json.loads(result.stdout.strip().splitlines()[-1]) + return enabled, ttl + + def test_unset_env_leaves_auto_caching_off(self): + assert self._import_litellm_with_env({}) == (False, None) + + @pytest.mark.parametrize("value", ["true", "True", "TRUE"]) + def test_env_enables_auto_caching_case_insensitively(self, value): + enabled, _ = self._import_litellm_with_env({"LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING": value}) + assert enabled is True + + @pytest.mark.parametrize("value", ["false", "0", "yes", ""]) + def test_env_only_enables_on_true(self, value): + enabled, _ = self._import_litellm_with_env({"LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING": value}) + assert enabled is False + + @pytest.mark.parametrize("value", ["5m", "1h"]) + def test_ttl_env_is_applied(self, value): + _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) + assert ttl == value + + @pytest.mark.parametrize("value", ["10m", "1H", "3600", "ephemeral"]) + def test_unsupported_ttl_env_falls_back_to_provider_default(self, value): + """An unparseable TTL must fall back to Anthropic's 5m default, never reach the provider verbatim.""" + _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) + assert ttl is None From 53c285a94a233ddb99781a197d36faa5690e3bb3 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 18:33:09 -0700 Subject: [PATCH 3/4] fix(anthropic): stand down when the client caches its tool definitions _request_has_cache_control only looked at messages and system, so a client that marks cache_control on tools alone did not suppress auto-injection. Tool breakpoints count toward the provider's four-block limit, so three of them plus the two injected here is five, which Anthropic rejects. Thread tools through both entry points and treat a client-marked tool as the stand-down signal it already is for messages and system. --- .../anthropic_cache_control_hook.py | 21 ++++++- .../messages/handler.py | 4 +- litellm/main.py | 2 + .../test_anthropic_cache_control_hook.py | 55 ++++++++++++++++++- 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 026d8b8e82e1..79ed48943b33 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -311,16 +311,26 @@ def _default_control() -> ChatCompletionCachedContent: return ChatCompletionCachedContent(type="ephemeral") @staticmethod - def _request_has_cache_control(messages: list[AllMessageValues], system: Optional[Union[str, list]]) -> bool: + def _request_has_cache_control( + messages: list[AllMessageValues], + system: Optional[Union[str, list]], + tools: Optional[list] = None, + ) -> bool: """Return True if the request already carries any client-supplied cache_control. When the client (e.g. Claude Code) already marks its own breakpoints we stand down entirely rather than add more, per the auto-caching contract. + Tools count: they are a breakpoint the client can mark, they count toward + the provider's four-block limit, and caching only the tool definitions is + a common pattern, so injecting alongside them can exceed the cap. """ if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): return True if isinstance(system, list): - return any(isinstance(block, dict) and block.get("cache_control") is not None for block in system) + if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system): + return True + if tools is not None: + return any(isinstance(tool, dict) and tool.get("cache_control") is not None for tool in tools) return False @staticmethod @@ -329,6 +339,7 @@ def get_default_injection_points( system: Optional[Union[str, list]], model: str, custom_llm_provider: Optional[str], + tools: Optional[list] = None, ) -> list[CacheControlInjectionPoint]: """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. @@ -363,7 +374,7 @@ def get_default_injection_points( if not supports_prompt_caching(model=model, custom_llm_provider=provider): return [] - if AnthropicCacheControlHook._request_has_cache_control(messages, system): + if AnthropicCacheControlHook._request_has_cache_control(messages, system, tools): return [] control = AnthropicCacheControlHook._default_control() @@ -379,6 +390,7 @@ def maybe_seed_default_injection_points( messages: list[AllMessageValues], model: str, custom_llm_provider: Optional[str], + tools: Optional[list] = None, ) -> None: """For /chat/completions: add default injection points to the request params. @@ -393,6 +405,7 @@ def maybe_seed_default_injection_points( system=None, model=model, custom_llm_provider=custom_llm_provider, + tools=tools, ) if points: non_default_params["cache_control_injection_points"] = points @@ -404,6 +417,7 @@ def maybe_inject_cache_control( kwargs: Dict[str, Any], model: Optional[str] = None, custom_llm_provider: Optional[str] = None, + tools: Optional[list[dict]] = None, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. @@ -420,6 +434,7 @@ def maybe_inject_cache_control( injection_points = AnthropicCacheControlHook.get_default_injection_points( messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages system=system, + tools=tools, model=model, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index c205d7516e6a..59eedbe4538a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -237,7 +237,7 @@ async def anthropic_messages( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools ) original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False) @@ -428,7 +428,7 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index 4a9b5bdc76f1..3584297b35f1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -521,6 +521,7 @@ async def acompletion( messages=cast(list[AllMessageValues], messages), # cast-ok: acompletion types messages as a bare List model=model, custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs + tools=tools, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5078,6 +5079,7 @@ def completion( # type: ignore messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List model=model, custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs + tools=tools, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 6a67f1d66435..70c1f65b5415 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1547,12 +1547,13 @@ class TestEnableAnthropicPromptCaching: {"role": "user", "content": "latest turn"}, ] - def _points(self, model="claude-sonnet-4-5", provider="anthropic", messages=None, system=None): + def _points(self, model="claude-sonnet-4-5", provider="anthropic", messages=None, system=None, tools=None): return AnthropicCacheControlHook.get_default_injection_points( messages=copy.deepcopy(self.MESSAGES) if messages is None else messages, system=system, model=model, custom_llm_provider=provider, + tools=tools, ) def test_disabled_by_default(self): @@ -1597,6 +1598,58 @@ def test_stands_down_when_system_block_has_cache_control(self, monkeypatch): system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}] assert self._points(messages=[{"role": "user", "content": "hi"}], system=system) == [] + @staticmethod + def _tools(count: int, cached: bool) -> List[dict]: + tool: dict = {"type": "function", "function": {"name": "t", "description": "d", "parameters": {}}} + if cached: + tool["cache_control"] = {"type": "ephemeral"} + return [{**tool, "function": {**tool["function"], "name": f"t{i}"}} for i in range(count)] + + def test_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): + """Caching just the tool definitions is a normal client pattern, and those + breakpoints count toward the provider's four-block limit. Three of them plus + our two would be five, which Anthropic rejects outright.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert self._points(tools=self._tools(3, cached=True)) == [] + + def test_injects_when_tools_carry_no_cache_control(self, monkeypatch): + """Tools alone must not suppress injection; only client-marked ones do.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert [p["index"] for p in self._points(tools=self._tools(3, cached=False))] == [None, -1] + + @pytest.mark.parametrize("tools", [None, []]) + def test_absent_tools_do_not_suppress_injection(self, monkeypatch, tools): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert [p["index"] for p in self._points(tools=tools)] == [None, -1] + + def test_seed_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): + """Same guard on the /chat/completions seeding path.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + params: dict = {} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + tools=self._tools(3, cached=True), + ) + assert "cache_control_injection_points" not in params + + def test_v1_messages_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): + """Same guard on the /v1/messages path, where tools reach the hook directly.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(messages), + "sys", + {}, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + tools=self._tools(3, cached=True), + ) + assert result_sys == "sys" + assert result_msgs == messages + def test_default_ttl_is_anthropics_five_minute_cache(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) assert all(p["control"] == {"type": "ephemeral"} for p in self._points()) From 7bcb3a29e57c4e18083cabf8f5e189e0b77111af Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 17 Jul 2026 10:38:44 -0700 Subject: [PATCH 4/4] refactor(anthropic): use PEP 604 unions in the auto prompt-caching hook --- .../anthropic_cache_control_hook.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 79ed48943b33..94c86e07ff50 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -313,8 +313,8 @@ def _default_control() -> ChatCompletionCachedContent: @staticmethod def _request_has_cache_control( messages: list[AllMessageValues], - system: Optional[Union[str, list]], - tools: Optional[list] = None, + system: str | list | None, + tools: list | None = None, ) -> bool: """Return True if the request already carries any client-supplied cache_control. @@ -336,10 +336,10 @@ def _request_has_cache_control( @staticmethod def get_default_injection_points( messages: list[AllMessageValues], - system: Optional[Union[str, list]], + system: str | list | None, model: str, - custom_llm_provider: Optional[str], - tools: Optional[list] = None, + custom_llm_provider: str | None, + tools: list | None = None, ) -> list[CacheControlInjectionPoint]: """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. @@ -389,8 +389,8 @@ def maybe_seed_default_injection_points( non_default_params: dict[str, Any], messages: list[AllMessageValues], model: str, - custom_llm_provider: Optional[str], - tools: Optional[list] = None, + custom_llm_provider: str | None, + tools: list | None = None, ) -> None: """For /chat/completions: add default injection points to the request params. @@ -415,9 +415,9 @@ def maybe_inject_cache_control( messages: List[Dict], system: str | list | None, kwargs: Dict[str, Any], - model: Optional[str] = None, - custom_llm_provider: Optional[str] = None, - tools: Optional[list[dict]] = None, + model: str | None = None, + custom_llm_provider: str | None = None, + tools: list[dict] | None = None, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. @@ -427,7 +427,7 @@ def maybe_inject_cache_control( are written back so downstream transforms can handle them. """ configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list - Optional[list[CacheControlInjectionPoint]], kwargs.pop("cache_control_injection_points", None) + list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) ) injection_points: list[CacheControlInjectionPoint] = configured or [] if not injection_points and model is not None: