From 64e43735cab47c62cb9e9366741eb07a011f82b3 Mon Sep 17 00:00:00 2001 From: James Wiesebron Date: Thu, 28 May 2026 13:30:27 -0700 Subject: [PATCH] Preserve cache_control for Qwen on OpenRouter + Anthropic adapter Three related fixes so cache_control survives end-to-end when routing Anthropic /v1/messages traffic to Qwen/DeepSeek upstreams via OpenRouter (measured ~0% -> ~95% cache hit rate on Alibaba/DeepSeek routes): 1. Add QWEN to CacheControlSupportedModels so the OpenRouter chat handler stops stripping cache_control before the upstream call. Alibaba (Qwen) requires explicit cache_control breakpoints. DeepSeek is intentionally omitted: its caching is automatic/prefix-based and ignores cache_control, so listing it would be inert. 2. Widen is_anthropic_claude_model to recognize qwen so the Anthropic->OpenAI adapter applies cache_control for Qwen models. DeepSeek is likewise omitted here; this also keeps DeepSeek's thinking translated to reasoning_effort rather than passed through raw. 3. Drop the x-anthropic-billing-header system block in the adapter path (mirrors _filter_billing_headers_from_system in the messages adapter). Its per-request hash otherwise invalidates the prefix cache every turn, pinning cache_read_input_tokens at 0. This is model-agnostic and is what restores DeepSeek's automatic prefix caching. Adds unit tests for all paths, including that DeepSeek does not receive a cache_control block and is not treated as cache_control-capable. --- .../adapters/transformation.py | 19 +++++- .../llms/openrouter/chat/transformation.py | 7 +++ ...al_pass_through_adapters_transformation.py | 55 +++++++++++++++++ .../test_openrouter_chat_transformation.py | 60 +++++++++++++++++++ 4 files changed, 139 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 51a1e739a0f3..0d0e852fcc35 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -717,7 +717,11 @@ def is_anthropic_claude_model(model: str) -> bool: - vertex_ai/*claude* models """ model_lower = model.lower() - return "anthropic" in model_lower or "claude" in model_lower + return ( + "anthropic" in model_lower + or "claude" in model_lower + or "qwen" in model_lower + ) @staticmethod def translate_thinking_for_model( @@ -951,9 +955,20 @@ def _add_system_message_to_messages( model_name = anthropic_message_request.get("model", "") for block in system_content: if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "") + # Drop the `x-anthropic-billing-header:` system block. + # Some clients (e.g. Claude Code) inject this block ahead + # of the cache_control marker with a per-request hash; left + # in place it invalidates the upstream prefix-cache key on + # every turn, so cache_read_input_tokens stays 0. The + # sibling messages/transformation.py path already filters it + # via _filter_billing_headers_from_system; this adapter path + # missed it. + if text.startswith("x-anthropic-billing-header:"): + continue text_block: Dict[str, Any] = { "type": "text", - "text": block.get("text", ""), + "text": text, } self._add_cache_control_if_applicable(block, text_block, model_name) openai_system_content.append(text_block) diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 0d7850e8c740..31af3df5bf2e 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -30,6 +30,13 @@ class CacheControlSupportedModels(str, Enum): MINIMAX = "minimax" GLM = "glm" ZAI = "z-ai" + # OpenRouter's Alibaba (Qwen) upstreams require explicit cache_control + # content blocks. See https://openrouter.ai/docs/features/prompt-caching. + # Without this the handler strips cache_control before the upstream call, + # so every turn pays full input rate (cache_read discount never applies). + # DeepSeek is intentionally omitted: its caching is automatic/prefix-based + # and ignores cache_control, so listing it here would be inert. + QWEN = "qwen" class OpenrouterConfig(OpenAIGPTConfig): diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 44530fecebdf..8462428a9ad2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -2472,3 +2472,58 @@ def test_translate_anthropic_tool_choice_none(): result = adapter.translate_anthropic_tool_choice_to_openai({"type": "none"}) assert result == "none" + + +@pytest.mark.parametrize( + "model, expected", + [ + ("anthropic/claude-sonnet-4", True), + ("claude-3-5-sonnet", True), + ("openrouter/qwen/qwen3-max", True), + # DeepSeek caching is automatic/prefix-based and ignores cache_control, + # so it is intentionally NOT recognized here (no cache_control applied). + ("openrouter/deepseek/deepseek-v3", False), + ("openrouter/openai/gpt-4o", False), + ("gemini/gemini-1.5-pro", False), + ], +) +def test_is_anthropic_claude_model_includes_qwen(model, expected): + """ + cache_control passthrough in the Anthropic->OpenAI adapter is gated on + is_anthropic_claude_model. Alibaba (Qwen) upstreams require explicit + cache_control breakpoints on OpenRouter, so qwen must be recognized here; + DeepSeek must not (its caching is automatic and ignores cache_control). + """ + assert LiteLLMAnthropicMessagesAdapter.is_anthropic_claude_model(model) is expected + + +def test_add_system_message_filters_billing_header_block(): + """ + The `x-anthropic-billing-header:` system block carries a per-request hash + that invalidates the upstream prefix cache on every turn. It must be + dropped during the Anthropic->OpenAI system-block translation, mirroring + _filter_billing_headers_from_system in the messages adapter. + """ + adapter = LiteLLMAnthropicMessagesAdapter() + new_messages: list = [] + request = { + "model": "openrouter/qwen/qwen3-max", + "system": [ + {"type": "text", "text": "x-anthropic-billing-header: cch=abcde"}, + { + "type": "text", + "text": "You are a helpful assistant.", + "cache_control": {"type": "ephemeral"}, + }, + ], + } + + adapter._add_system_message_to_messages(new_messages, cast(Any, request)) + + assert len(new_messages) == 1 + content = new_messages[0]["content"] + texts = [block["text"] for block in content] + assert "You are a helpful assistant." in texts + assert all(not t.startswith("x-anthropic-billing-header:") for t in texts) + # cache_control survives for qwen (gated by is_anthropic_claude_model) + assert any(block.get("cache_control") for block in content) diff --git a/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py index f102319d6bf5..14c7c6da7817 100644 --- a/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -553,3 +553,63 @@ def test_openrouter_non_reasoning_models_do_not_add_reasoning_effort(): ) assert "reasoning_effort" not in supported_params + + +def test_openrouter_transform_request_with_cache_control_qwen(): + """ + Alibaba (Qwen) upstreams require explicit cache_control breakpoints on + OpenRouter, so transform_request must preserve the marker (move it to a + content block) rather than stripping it. Without this every turn pays + full input rate. + """ + config = OpenrouterConfig() + + messages = [ + { + "role": "user", + "content": "Analyze this data", + "cache_control": {"type": "ephemeral"}, + } + ] + + transformed_request = config.transform_request( + model="openrouter/qwen/qwen3-max", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + user_message = transformed_request["messages"][0] + assert isinstance(user_message["content"], list) + assert user_message["content"][0]["type"] == "text" + assert user_message["content"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_openrouter_transform_request_drops_cache_control_for_deepseek(): + """ + DeepSeek caching on OpenRouter is automatic/prefix-based and ignores + cache_control, so DeepSeek is intentionally absent from + CacheControlSupportedModels. The handler should strip the marker rather + than emit an inert cache_control content block. + """ + config = OpenrouterConfig() + + messages = [ + { + "role": "user", + "content": "Analyze this data", + "cache_control": {"type": "ephemeral"}, + } + ] + + transformed_request = config.transform_request( + model="openrouter/deepseek/deepseek-v3", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + user_message = transformed_request["messages"][0] + assert "cache_control" not in user_message