diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index d7d5c39ebd427..f4a5bd46c2f08 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -2092,6 +2092,13 @@ def anthropic_prompt_cache_policy( gateway implements the Anthropic cache_control contract (MiniMax, Zhipu GLM, LiteLLM's Anthropic proxy mode all do). + LiteLLM proxies exposing the OpenAI-compatible surface instead + (``/v1/chat/completions`` — e.g. ``provider: custom:litellm``) + also honour Anthropic-style ``cache_control`` markers when serving + a Claude model, and get the envelope layout (native_anthropic=False) + like OpenRouter. Without this grant they serve zero cache hits, + re-billing the full prompt on every turn. + Qwen / Alibaba-family models on OpenCode, OpenCode Go, and direct Alibaba (DashScope) also honour Anthropic-style ``cache_control`` markers on OpenAI-wire chat completions. Upstream pi-mono #3392 / @@ -2240,6 +2247,31 @@ def anthropic_prompt_cache_policy( if is_anthropic_wire and is_minimax_route: return True, True + # LiteLLM proxies exposing the OpenAI-compatible surface (e.g. + # ``provider: custom:litellm`` with /v1/chat/completions — /v1/messages + # returns 404) serve Claude models and honour Anthropic-style + # cache_control markers on the OpenAI wire, exactly like OpenRouter. + # Without this branch the OpenAI-wire LiteLLM route matches no grant + # branch and falls through to (False, False) — zero cache hits, the + # full prompt re-billed on every turn (the same silent failure class + # the Qwen branch below addresses). LiteLLM's Anthropic-native route + # (/v1/messages) is already covered by the third-party-gateway branch + # above, which returns the native layout. Detection mirrors the + # MiniMax provider-or-host pattern: a ``litellm`` substring in the + # provider id (``custom:litellm``, ``litellm``) or in the base URL + # host (self-hosted proxies registered as bare ``custom``). The host + # check is host-only via ``base_url_hostname`` so a ``litellm`` path + # segment on an unrelated host (e.g. ``https://api.corp.com/v1/litellm``) + # does not grant cache_control to a provider that may reject the + # marker with HTTP 400. + litellm_host = base_url_hostname(eff_base_url) + is_litellm = ( + "litellm" in provider_lower + or "litellm" in litellm_host + ) + if is_litellm and is_claude and not is_anthropic_wire: + return True, False + # Qwen/Alibaba on OpenCode (Zen/Go) and native DashScope: OpenAI-wire # transport that accepts Anthropic-style cache_control markers and # rewards them with real cache hits. Without this branch diff --git a/tests/run_agent/test_anthropic_prompt_cache_policy.py b/tests/run_agent/test_anthropic_prompt_cache_policy.py index e0a4ae00fa407..3bcfefe9548ae 100644 --- a/tests/run_agent/test_anthropic_prompt_cache_policy.py +++ b/tests/run_agent/test_anthropic_prompt_cache_policy.py @@ -312,6 +312,86 @@ def test_custom_openai_wire_does_not_cache_even_with_claude_name(self): assert agent._anthropic_prompt_cache_policy() == (False, False) +class TestLiteLLMOpenAIWire: + """LiteLLM proxies exposing the OpenAI-compatible surface get cache_control. + + A LiteLLM deployment that serves Claude over /v1/chat/completions + (``provider: custom:litellm``; /v1/messages returns 404) honours + Anthropic-style ``cache_control`` markers on the OpenAI wire. Before + this fix the policy matched no grant branch for this route and fell + through to (False, False) — zero cache hits, the full prompt re-billed + on every turn, with no error or warning. Envelope layout (native=False) + because the wire format is OpenAI chat.completions, like OpenRouter. + """ + + def test_named_litellm_custom_provider_caches_with_envelope_layout(self): + agent = _make_agent( + provider="custom:litellm", + base_url="https://litellm.example.com/v1", + api_mode="chat_completions", + model="claude-opus-5", + ) + should, native = agent._anthropic_prompt_cache_policy() + assert should is True, "LiteLLM OpenAI-wire Claude must cache" + assert native is False, "LiteLLM OpenAI wire uses the envelope layout" + + def test_bare_litellm_provider_caches(self): + agent = _make_agent( + provider="litellm", + base_url="https://litellm.example.com/v1", + api_mode="chat_completions", + model="claude-sonnet-4-6", + ) + assert agent._anthropic_prompt_cache_policy() == (True, False) + + def test_custom_provider_pointed_at_litellm_host_caches(self): + # A bare custom provider registered at a LiteLLM host — host match + # alone is sufficient, mirroring the MiniMax provider-or-host branch. + agent = _make_agent( + provider="custom", + base_url="https://litellm.example.com/v1", + api_mode="chat_completions", + model="claude-sonnet-4-6", + ) + assert agent._anthropic_prompt_cache_policy() == (True, False) + + def test_litellm_anthropic_wire_keeps_native_layout(self): + # LiteLLM's Anthropic-native route (/v1/messages) is covered by the + # third-party-gateway branch: native layout, not envelope. + agent = _make_agent( + provider="custom:litellm", + base_url="https://litellm.example.com/v1", + api_mode="anthropic_messages", + model="claude-opus-5", + ) + assert agent._anthropic_prompt_cache_policy() == (True, True) + + def test_non_claude_on_litellm_does_not_cache(self): + # LiteLLM serving a non-Claude model stays off: we only grant + # cache_control when the served model family is documented to + # honour the markers. + agent = _make_agent( + provider="custom:litellm", + base_url="https://litellm.example.com/v1", + api_mode="chat_completions", + model="openai/gpt-5.4", + ) + assert agent._anthropic_prompt_cache_policy() == (False, False) + + def test_litellm_path_segment_on_unrelated_host_does_not_cache(self): + # Host-only detection: a ``litellm`` substring confined to the URL + # path (e.g. a failover path on an unrelated provider) must NOT grant + # cache_control. Injecting the marker onto a strict OpenAI-wire host + # that rejects unknown keys would fail with HTTP 400 (#84506). + agent = _make_agent( + provider="custom", + base_url="https://api.corp.example.com/v1/litellm-failover", + api_mode="chat_completions", + model="claude-sonnet-4-6", + ) + assert agent._anthropic_prompt_cache_policy() == (False, False) + + class TestQwenAlibabaFamily: """Qwen on OpenCode/OpenCode-Go/Alibaba — needs cache_control even on OpenAI-wire.