feat(prompt-caching): map cache_control_injection_points to OpenAI prompt_cache_breakpoint on GPT-5.6+ targets

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -10,7 +10,7 @@
 from itertools import groupby
 from os import PathLike
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Final, Literal, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
 
 from openai.types.chat.chat_completion_custom_tool_param import (
     CustomFormatGrammar,
@@ -1325,6 +1325,16 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool:
     return False
 
 
+_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
+
+
+def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
+    if marker is None:
+        return target
+    marked: Final = {**target, "prompt_cache_breakpoint": marker}  # mutable-ok: API message payload
+    return cast(_MarkedT, marked)  # cast-ok: same block shape as the input plus the marker key
+
+
 def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
     """
     Filters a value from a dictionary

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -2,7 +2,7 @@
 import hashlib
 import json
 from collections.abc import AsyncIterator, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Final, Literal, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
 
 from litellm.llms.anthropic.experimental_pass_through.utils import (
     is_reasoning_auto_summary_enabled,
@@ -61,6 +61,7 @@ def create_tool_name_mapping(
 
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     parse_tool_call_arguments,
+    with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
     THOUGHT_SIGNATURE_SEPARATOR,
@@ -245,6 +246,9 @@ def translate_completion_output_params_streaming(
         return anthropic_wrapper.anthropic_sse_wrapper()
 
 
+_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object])
+
+
 class LiteLLMAnthropicMessagesAdapter:
     def __init__(self):
         pass
@@ -308,6 +312,12 @@ def _add_cache_control_if_applicable(
                 # Fallback for non-dict objects (shouldn't happen in practice)
                 cast(dict[str, object], target)["cache_control"] = cache_control
 
+    @staticmethod
+    def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT:
+        if isinstance(source, dict) and "prompt_cache_breakpoint" in source:
+            return with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"])
+        return target
+
     def translatable_anthropic_params(self) -> list[str]:
         """
         Which anthropic params, we need to translate to the openai format.
@@ -368,7 +378,9 @@ def translate_anthropic_messages_to_openai(
                         if content.get("type") == "text":
                             text_obj = ChatCompletionTextObject(type="text", text=content.get("text", ""))
                             self._add_cache_control_if_applicable(content, text_obj, model)
-                            new_user_content_list.append(text_obj)
+                            new_user_content_list.append(
+                                self._add_prompt_cache_breakpoint_if_present(content, text_obj)
+                            )
                         elif content.get("type") == "image":
                             # Convert Anthropic image format to OpenAI format
                             source = content.get("source", {})
@@ -378,7 +390,9 @@ def translate_anthropic_messages_to_openai(
                                 image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url)
                                 image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj)
                                 self._add_cache_control_if_applicable(content, image_obj, model)
-                                new_user_content_list.append(image_obj)
+                                new_user_content_list.append(
+                                    self._add_prompt_cache_breakpoint_if_present(content, image_obj)
+                                )
                         elif content.get("type") == "document":
                             # Convert Anthropic document format (PDF, etc.) to OpenAI format
                             source = content.get("source", {})
@@ -869,7 +883,7 @@ def _translate_midturn_system_message_to_openai(
                 continue
             text_obj = ChatCompletionTextObject(type="text", text=text)
             self._add_cache_control_if_applicable(block, text_obj, model)
-            text_parts.append(text_obj)
+            text_parts.append(self._add_prompt_cache_breakpoint_if_present(block, text_obj))
         return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None
 
     def _add_system_message_to_messages(
@@ -900,7 +914,7 @@ def _add_system_message_to_messages(
                         "text": block.get("text", ""),
                     }
                     self._add_cache_control_if_applicable(block, text_block, model_name)
-                    openai_system_content.append(text_block)
+                    openai_system_content.append(self._add_prompt_cache_breakpoint_if_present(block, text_block))
             if openai_system_content:
                 new_messages.insert(
                     0,

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -230,7 +230,7 @@ async def anthropic_messages(
     )
 
     messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
-        messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
+        messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
     )
 
     original_stream: Final = stream or kwargs.get("_websearch_interception_converted_stream", False)
@@ -422,7 +422,7 @@ def anthropic_messages_handler(
     )
 
     messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
-        messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
+        messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
     )
 
     metadata = validate_anthropic_api_metadata(metadata)

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -12,6 +12,7 @@
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     TOOL_RESULT_IMAGE_BOUNDARY,
     TOOL_RESULT_IMAGE_PLACEHOLDER,
+    with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
@@ -82,7 +83,7 @@ def _translate_anthropic_image_source_to_url(source: object) -> str | None:
     @staticmethod
     def _translate_midturn_system_content_to_responses(
         content: str | Iterable[AnthropicSystemMessageContent],
-    ) -> list[dict[str, str]]:  # mutable-ok: API message payload
+    ) -> list[dict[str, object]]:  # mutable-ok: API message payload
         """Convert in-sequence system content to Responses input-text parts."""
         if isinstance(content, str):
             return (
@@ -91,7 +92,9 @@ def _translate_midturn_system_content_to_responses(
         if not isinstance(content, list):
             return []  # mutable-ok: API message payload
         return [  # mutable-ok: API message payload
-            {"type": "input_text", "text": text}  # mutable-ok: API message payload
+            with_prompt_cache_breakpoint(
+                {"type": "input_text", "text": text}, block.get("prompt_cache_breakpoint")
+            )  # mutable-ok: API message payload
             for block in content
             if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text"))  # pyright: ignore[reportUnnecessaryIsInstance]  # untrusted client payload
         ]
@@ -146,11 +149,20 @@ def translate_messages_to_responses_input(
                             continue
                         btype = block.get("type")
                         if btype == "text":
-                            user_parts.append({"type": "input_text", "text": block.get("text", "")})
+                            user_parts.append(
+                                with_prompt_cache_breakpoint(
+                                    {"type": "input_text", "text": block.get("text", "")},
+                                    block.get("prompt_cache_breakpoint"),
+                                )
+                            )
                         elif btype == "image":
                             url = self._translate_anthropic_image_source_to_url(cast(dict, block.get("source", {})))
                             if url:
-                                user_parts.append({"type": "input_image", "image_url": url})
+                                user_parts.append(
+                                    with_prompt_cache_breakpoint(
+                                        {"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint")
+                                    )
+                                )
                         elif btype == "tool_result":
                             tool_use_id = block.get("tool_use_id", "")
                             inner = block.get("content")
@@ -376,19 +388,36 @@ def translate_request(
             anthropic_request["messages"],
         )
 
+        input_items: Final = self.translate_messages_to_responses_input(messages_list)
+        system: Final = anthropic_request.get("system")
+        developer_parts: Final = (
+            self._translate_midturn_system_content_to_responses(system)
+            if isinstance(system, list)
+            and any(isinstance(block, dict) and block.get("prompt_cache_breakpoint") is not None for block in system)
+            else ()
+        )
+        if developer_parts:
+            input_items.insert(
+                0,
+                {  # mutable-ok: API message payload
+                    "type": "message",
+                    "role": "developer",
+                    "content": developer_parts,
+                },
+            )
+
         responses_kwargs: Final[dict[str, Any]] = {
             "model": model,
-            "input": self.translate_messages_to_responses_input(messages_list),
+            "input": input_items,
         }
 
-        # system -> instructions
-        system: Final = anthropic_request.get("system")
-        if system:
+        if system and not developer_parts:
             if isinstance(system, str):
                 responses_kwargs["instructions"] = system
             elif isinstance(system, list):
-                text_parts = [b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"]
-                responses_kwargs["instructions"] = "\n".join(filter(None, text_parts))
+                responses_kwargs["instructions"] = "\n".join(
+                    filter(None, (b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"))
+                )
 
         # max_tokens -> max_output_tokens
         max_tokens: Final = anthropic_request.get("max_tokens")

litellm/main.py

@@ -507,6 +507,7 @@ async def acompletion(
         custom_llm_provider=cast(str | None, custom_llm_provider),  # cast-ok: read from untyped kwargs
         tools=tools,
         enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")),  # cast-ok: untyped kwargs
+        api_base=kwargs.get("api_base") or base_url,
     )
 
     if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
@@ -5171,6 +5172,7 @@ def completion(
         custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")),  # cast-ok: untyped kwargs
         tools=tools,
         enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")),  # cast-ok: untyped kwargs
+        api_base=kwargs.get("api_base") or base_url,
     )
 
     if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (

litellm/model_prices_and_context_window_backup.json

@@ -25368,6 +25368,7 @@
         "supports_none_reasoning_effort": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -25430,6 +25431,7 @@
         "supports_none_reasoning_effort": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -25492,6 +25494,7 @@
         "supports_none_reasoning_effort": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -25554,6 +25557,7 @@
         "supports_none_reasoning_effort": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,

litellm/responses/utils.py

@@ -37,24 +37,45 @@ def normalize_responses_api_stream_options(
     return ResponsesAPIStreamOptions(include_obfuscation=include_obfuscation)
 
 
+def _is_chat_text_part(part: object) -> bool:
+    return isinstance(part, dict) and part.get("type") == "text"
+
+
+def _as_input_text_part(part: object) -> object:
+    if isinstance(part, dict) and part.get("type") == "text":
+        return {**part, "type": "input_text"}  # mutable-ok: fresh part so the caller's block keeps its chat type
+    return part
+
+
 class ResponsesAPIRequestUtils:
     """Helper utils for constructing ResponseAPI requests"""
 
+    @staticmethod
+    def shape_prompt_managed_message_for_responses(message: object) -> object:
+        if not isinstance(message, dict) or message.get("role") == "assistant":
+            return message
+        content: object = message.get("content")
+        if not isinstance(content, list) or not any(_is_chat_text_part(part) for part in content):
+            return message
+        shaped_content: Final = [_as_input_text_part(part) for part in content]  # mutable-ok: Responses-shaped copy
+        return {**message, "content": shaped_content}  # mutable-ok: copy, the hook's message stays untouched
+
     @staticmethod
     def merge_prompt_management_input(
         original_input: str | ResponseInputParam,
         client_input: list[AllMessageValues],
         merged_input: list[AllMessageValues],
     ) -> list[object]:
+        shape: Final = ResponsesAPIRequestUtils.shape_prompt_managed_message_for_responses
         if isinstance(original_input, str):
-            return [*merged_input]
+            return [shape(message) for message in merged_input]
 
         original_items: Final = tuple(original_input)
         client_item_ids: Final = frozenset(id(item) for item in client_input)
         message_positions = tuple(index for index, item in enumerate(original_items) if id(item) in client_item_ids)
 
         if len(message_positions) == len(original_items):
-            return [*merged_input]
+            return [shape(message) for message in merged_input]
         if not message_positions:
             verbose_logger.warning(
                 "Prompt management hook returned messages without Responses API input messages; merged messages were ignored"
@@ -69,7 +90,7 @@ def merge_prompt_management_input(
         if corresponding_messages:
             merged_by_position: Final = dict(zip(message_positions, merged_input))
             return [
-                merged_by_position[index] if index in merged_by_position else item
+                shape(merged_by_position[index]) if index in merged_by_position else item
                 for index, item in enumerate(original_items)
             ]
 
@@ -82,14 +103,14 @@ def merge_prompt_management_input(
                 for index, position in enumerate(message_positions)
             }
             trailing_items: Final = original_items[message_positions[-1] + 1 :]
-            return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), merged)] + list(
+            return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), shape(merged))] + list(
                 trailing_items
             )
 
         verbose_logger.warning(
             "Prompt management hook replaced Responses API messages; non-message input items were dropped"
         )
-        return [*merged_input]
+        return [shape(message) for message in merged_input]
 
     @staticmethod
     def merge_client_forwarded_headers(

litellm/types/integrations/anthropic_cache_control_hook.py

@@ -1,6 +1,6 @@
 from typing import Literal
 
-from typing_extensions import NotRequired, TypedDict
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.types.llms.openai import ChatCompletionCachedContent
 
@@ -13,6 +13,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
     index: int | str | None  # Optional: target by specific index
     control: ChatCompletionCachedContent | None
     _litellm_judged: NotRequired[bool]  # Internal: written back by litellm once the client cache_control judgment ran
+    _litellm_openai_dialect: NotRequired[ReadOnly[bool]]
 
 
 class CacheControlToolConfigInjectionPoint(TypedDict):
@@ -21,6 +22,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
     location: Literal["tool_config"]
     control: ChatCompletionCachedContent | None
     _litellm_judged: NotRequired[bool]  # Internal: written back by litellm once the client cache_control judgment ran
+    _litellm_openai_dialect: NotRequired[ReadOnly[bool]]
 
 
 CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint

litellm/types/llms/anthropic.py

@@ -9,6 +9,7 @@
     ChatCompletionCachedContent,
     ChatCompletionRedactedThinkingBlock,
     ChatCompletionThinkingBlock,
+    PromptCacheBreakpoint,
 )
 
 
@@ -201,6 +202,7 @@ class AnthropicMessagesTextParam(TypedDict, total=False):
     type: Required[Literal["text"]]
     text: Required[str]
     cache_control: dict | ChatCompletionCachedContent | None
+    prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
 
 
 class AnthropicMessagesToolUseParam(TypedDict, total=False):
@@ -261,6 +263,7 @@ class AnthropicMessagesImageParam(TypedDict, total=False):
     type: Required[Literal["image"]]
     source: Required[AnthropicContentParamSource | AnthropicContentParamSourceFileId | AnthropicContentParamSourceUrl]
     cache_control: dict | ChatCompletionCachedContent | None
+    prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
 
 
 class CitationsObject(TypedDict):
@@ -347,6 +350,7 @@ class AnthropicSystemMessageContent(TypedDict, total=False):
     type: str
     text: str
     cache_control: dict | ChatCompletionCachedContent | None
+    prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
 
 
 class AnthropicMessagesSystemMessageParam(TypedDict, total=False):