Repository navigation
feat(anthropic): add enable_anthropic_prompt_caching for automatic cache_control injection #33573
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
04afc96
f7a3e22
53c285a
7bcb3a2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -296,18 +296,133 @@ def apply_to_anthropic_messages_request( | |
|
|
||
| return processed_messages, processed_system, remaining_points | ||
|
|
||
| @staticmethod | ||
| def _default_control() -> ChatCompletionCachedContent: | ||
| """Build the cache_control block for auto-injected breakpoints. | ||
|
|
||
| Defaults to Anthropic's 5-minute ephemeral cache; honors the optional | ||
| ``litellm.anthropic_prompt_caching_ttl`` override ("5m" or "1h"). | ||
| """ | ||
| import litellm | ||
|
|
||
| ttl = litellm.anthropic_prompt_caching_ttl | ||
| if ttl == "5m" or ttl == "1h": | ||
| return ChatCompletionCachedContent(type="ephemeral", ttl=ttl) | ||
| return ChatCompletionCachedContent(type="ephemeral") | ||
|
|
||
| @staticmethod | ||
| def _request_has_cache_control(messages: list[AllMessageValues], system: Optional[Union[str, list]]) -> bool: | ||
| """Return True if the request already carries any client-supplied cache_control. | ||
|
|
||
| When the client (e.g. Claude Code) already marks its own breakpoints we | ||
| stand down entirely rather than add more, per the auto-caching contract. | ||
| """ | ||
| if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): | ||
| return True | ||
| if isinstance(system, list): | ||
| return any(isinstance(block, dict) and block.get("cache_control") is not None for block in system) | ||
| return False | ||
|
|
||
| @staticmethod | ||
| def get_default_injection_points( | ||
| messages: list[AllMessageValues], | ||
| system: Optional[Union[str, list]], | ||
| model: str, | ||
| custom_llm_provider: Optional[str], | ||
| ) -> list[CacheControlInjectionPoint]: | ||
| """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. | ||
|
|
||
| Caches the system prompt and the trailing turn, so the stable prefix | ||
| (system + tools + history) is reused while the breakpoint advances with | ||
| the conversation. Returns [] (stand down) when the flag is off, the | ||
| provider does not consume cache_control breakpoints (only anthropic / | ||
| bedrock do), the model lacks prompt-caching support, or the request | ||
| already carries client-supplied cache_control. | ||
| """ | ||
| import litellm | ||
|
|
||
| if litellm.enable_anthropic_prompt_caching is not True: | ||
| return [] | ||
|
|
||
| provider = custom_llm_provider | ||
| if provider is None: | ||
| from litellm.litellm_core_utils.get_llm_provider_logic import ( | ||
| get_llm_provider, | ||
| ) | ||
|
|
||
| try: | ||
| _, provider, _, _ = get_llm_provider(model=model) | ||
| except Exception: # noqa: BLE001 # unroutable model must never block the call, just skip auto-caching | ||
| return [] | ||
|
|
||
| if provider not in ("anthropic", "bedrock"): | ||
| return [] | ||
|
greptile-apps[bot] marked this conversation as resolved.
Resolved
|
||
|
|
||
| from litellm.utils import supports_prompt_caching | ||
|
|
||
| if not supports_prompt_caching(model=model, custom_llm_provider=provider): | ||
| return [] | ||
|
|
||
| if AnthropicCacheControlHook._request_has_cache_control(messages, system): | ||
| return [] | ||
|
|
||
| control = AnthropicCacheControlHook._default_control() | ||
| points: list[CacheControlInjectionPoint] = [ | ||
| CacheControlMessageInjectionPoint(location="message", role="system", index=None, control=control), | ||
| CacheControlMessageInjectionPoint(location="message", role=None, index=-1, control=control), | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Medium: Cross-tenant prompt membership oracle This breakpoint caches the complete request-controlled conversation in the upstream workspace/account cache. Because LiteLLM returns
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good catch, and the exposure change is real, so I have documented it at the point of decision. One correction on the suggested mitigation though Confirmed the flag broadens exposure. With it off, a client that never sends The suggested mitigation does not close it here. Limiting injection to operator-controlled static prefixes assumes the system prompt is operator-controlled, but The underlying property is Anthropic prompt caching itself: a prefix is cached against the upstream credentials that sent it, not per end user. Any caller can already self-serve a Given that, this stays an operator decision, which is why the flag is opt-in and off by default. What was missing was surfacing the tradeoff where the decision is made, so the setting description and the env var docs now state that the provider caches against the upstream credentials rather than per end user, and to leave it off if callers sharing credentials must not learn whether another caller recently sent a given prompt |
||
| ] | ||
| return points | ||
|
|
||
| @staticmethod | ||
| def maybe_seed_default_injection_points( | ||
| non_default_params: dict[str, Any], | ||
| messages: list[AllMessageValues], | ||
| model: str, | ||
| custom_llm_provider: Optional[str], | ||
| ) -> None: | ||
| """For /chat/completions: add default injection points to the request params. | ||
|
|
||
| No-op when injection points are already configured (explicit config wins). | ||
| Seeding the param lets the existing prompt-management gate and the | ||
| AnthropicCacheControlHook run unchanged. | ||
| """ | ||
| if non_default_params.get("cache_control_injection_points"): | ||
| return | ||
| points = AnthropicCacheControlHook.get_default_injection_points( | ||
| messages=messages, | ||
| system=None, | ||
| model=model, | ||
| custom_llm_provider=custom_llm_provider, | ||
| ) | ||
| if points: | ||
| non_default_params["cache_control_injection_points"] = points | ||
|
|
||
| @staticmethod | ||
| def maybe_inject_cache_control( | ||
| messages: List[Dict], | ||
| system: str | list | None, | ||
| kwargs: Dict[str, Any], | ||
| model: Optional[str] = None, | ||
| custom_llm_provider: Optional[str] = None, | ||
| ) -> Tuple[List[Dict], str | list | None]: | ||
| """Extract cache_control_injection_points from kwargs and apply if present. | ||
|
|
||
| When none are configured but ``litellm.enable_anthropic_prompt_caching`` | ||
| is on, synthesize default breakpoints for the native /v1/messages path. | ||
| Pops the key from kwargs; if remaining (non-message) points exist they | ||
| are written back so downstream transforms can handle them. | ||
| """ | ||
| injection_points = kwargs.pop("cache_control_injection_points", None) | ||
| configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list | ||
| Optional[list[CacheControlInjectionPoint]], kwargs.pop("cache_control_injection_points", None) | ||
| ) | ||
| injection_points: list[CacheControlInjectionPoint] = configured or [] | ||
| if not injection_points and model is not None: | ||
| injection_points = AnthropicCacheControlHook.get_default_injection_points( | ||
| messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages | ||
| system=system, | ||
| model=model, | ||
| custom_llm_provider=custom_llm_provider, | ||
| ) | ||
| if not injection_points: | ||
| return messages, system | ||
|
|
||
|
|
||
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Uh oh!
There was an error while loading. Please reload this page.