chore(release): backport #30480, #30787, #30788, #31035, #31133, #31122 to stable/1.85.x and cut 1.85.7

Dockerfile

@@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

docker/Dockerfile.database

@@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

docker/Dockerfile.non_root

@@ -1,6 +1,6 @@
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 

litellm/integrations/anthropic_cache_control_hook.py

@@ -27,6 +27,11 @@
     LiteLLMLoggingObj = Any
 
 
+# Anthropic (and Bedrock Claude) reject requests with more than 4 cache_control
+# breakpoints: "A maximum of 4 blocks with cache_control may be provided."
+MAX_CACHE_CONTROL_BLOCKS = 4
+
+
 class AnthropicCacheControlHook(CustomPromptManagement):
     def get_chat_completion_prompt(
         self,
@@ -61,31 +66,102 @@ def get_chat_completion_prompt(
         processed_messages = copy.deepcopy(messages)
 
         # Separate message-level and non-message-level injection points
-        remaining_points = []
+        message_points: List[CacheControlMessageInjectionPoint] = []
+        remaining_points: List[CacheControlInjectionPoint] = []
         for point in injection_points:
             if point.get("location") == "message":
-                point = cast(CacheControlMessageInjectionPoint, point)
-                processed_messages = self._process_message_injection(
-                    point=point, messages=processed_messages
-                )
+                message_points.append(cast(CacheControlMessageInjectionPoint, point))
             else:
                 remaining_points.append(point)
 
+        # Non-message points (currently Bedrock tool_config) are handled in the
+        # provider transform, where each tool_config point appends at most one
+        # cachePoint to the tools. That block also counts toward Anthropic's
+        # limit, so reserve a slot for it here to leave room.
+        reserved_blocks = (
+            1
+            if any(p.get("location") == "tool_config" for p in remaining_points)
+            else 0
+        )
+
+        processed_messages = self._apply_message_injections(
+            points=message_points,
+            messages=processed_messages,
+            max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
+        )
+
         # Pass through non-message injection points for provider-specific handling
         if remaining_points:
             non_default_params["cache_control_injection_points"] = remaining_points
 
         return model, processed_messages, non_default_params
 
     @staticmethod
-    def _process_message_injection(
-        point: CacheControlMessageInjectionPoint, messages: List[AllMessageValues]
+    def _apply_message_injections(
+        points: List[CacheControlMessageInjectionPoint],
+        messages: List[AllMessageValues],
+        max_blocks: int,
     ) -> List[AllMessageValues]:
-        """Process message-level cache control injection."""
-        control: ChatCompletionCachedContent = point.get(
-            "control", None
-        ) or ChatCompletionCachedContent(type="ephemeral")
+        """Apply message-level cache control injection points in order.
+
+        Anthropic allows at most ``MAX_CACHE_CONTROL_BLOCKS`` cache_control
+        breakpoints per request. Client-supplied breakpoints count toward that
+        limit, so we never inject onto a message that already carries
+        cache_control (preserving the client's TTL) and we stop injecting once
+        ``max_blocks`` is reached. Injection points are honored in config order,
+        so earlier points win when slots are scarce.
+        """
+        used_blocks = sum(
+            AnthropicCacheControlHook._count_cache_control_blocks(msg)
+            for msg in messages
+        )
+
+        limit_reached = False
+        for point in points:
+            if used_blocks >= max_blocks:
+                limit_reached = True
+                break
+
+            control: ChatCompletionCachedContent = point.get(
+                "control", None
+            ) or ChatCompletionCachedContent(type="ephemeral")
+
+            for target_index in AnthropicCacheControlHook._resolve_target_indices(
+                point=point, messages=messages
+            ):
+                if used_blocks >= max_blocks:
+                    limit_reached = True
+                    break
+
+                if AnthropicCacheControlHook._message_has_cache_control(
+                    messages[target_index]
+                ):
+                    # Client already marked this message; don't overwrite it.
+                    continue
+
+                messages[target_index] = (
+                    AnthropicCacheControlHook._safe_insert_cache_control_in_message(
+                        messages[target_index], control
+                    )
+                )
+                used_blocks += 1
+
+            if limit_reached:
+                break
+
+        if limit_reached:
+            verbose_logger.warning(
+                f"AnthropicCacheControlHook: Reached the Anthropic limit of "
+                f"{MAX_CACHE_CONTROL_BLOCKS} cache_control blocks. Skipping further injection."
+            )
 
+        return messages
+
+    @staticmethod
+    def _resolve_target_indices(
+        point: CacheControlMessageInjectionPoint, messages: List[AllMessageValues]
+    ) -> List[int]:
+        """Resolve which message indices an injection point targets."""
         _targetted_index: Optional[Union[int, str]] = point.get("index", None)
         targetted_index: Optional[int] = None
         if isinstance(_targetted_index, str):
@@ -96,36 +172,49 @@ def _process_message_injection(
         else:
             targetted_index = _targetted_index
 
-        targetted_role = point.get("role", None)
-
         # Case 1: Target by specific index
         if targetted_index is not None:
             original_index = targetted_index
-            # Handle negative indices (convert to positive)
             if targetted_index < 0:
                 targetted_index += len(messages)
 
             if 0 <= targetted_index < len(messages):
-                messages[targetted_index] = (
-                    AnthropicCacheControlHook._safe_insert_cache_control_in_message(
-                        messages[targetted_index], control
-                    )
-                )
-            else:
-                verbose_logger.warning(
-                    f"AnthropicCacheControlHook: Provided index {original_index} is out of bounds for message list of length {len(messages)}. "
-                    f"Targeted index was {targetted_index}. Skipping cache control injection for this point."
-                )
+                return [targetted_index]
+
+            verbose_logger.warning(
+                f"AnthropicCacheControlHook: Provided index {original_index} is out of bounds for message list of length {len(messages)}. "
+                f"Targeted index was {targetted_index}. Skipping cache control injection for this point."
+            )
+            return []
+
         # Case 2: Target by role
-        elif targetted_role is not None:
-            for msg in messages:
-                if msg.get("role") == targetted_role:
-                    msg = (
-                        AnthropicCacheControlHook._safe_insert_cache_control_in_message(
-                            message=msg, control=control
-                        )
-                    )
-        return messages
+        targetted_role = point.get("role", None)
+        if targetted_role is not None:
+            return [
+                idx
+                for idx, msg in enumerate(messages)
+                if msg.get("role") == targetted_role
+            ]
+
+        return []
+
+    @staticmethod
+    def _count_cache_control_blocks(message: AllMessageValues) -> int:
+        """Count cache_control breakpoints on a message (message + content level)."""
+        count = 0
+        if message.get("cache_control") is not None:
+            count += 1
+        content = message.get("content")
+        if isinstance(content, list):
+            for block in content:
+                if isinstance(block, dict) and block.get("cache_control") is not None:
+                    count += 1
+        return count
+
+    @staticmethod
+    def _message_has_cache_control(message: AllMessageValues) -> bool:
+        """Return True if the message already carries any cache_control."""
+        return AnthropicCacheControlHook._count_cache_control_blocks(message) > 0
 
     @staticmethod
     def _safe_insert_cache_control_in_message(

litellm/litellm_core_utils/litellm_logging.py

@@ -2944,7 +2944,12 @@ def _failure_handler_helper_fn(
         )
         self.model_call_details["end_time"] = end_time
         self.model_call_details.setdefault("original_response", None)
-        self.model_call_details["response_cost"] = 0
+        # A stream interrupted mid-flight still billed the provider for the
+        # chunks already delivered; the router stashes that recovered usage as
+        # ``combined_usage_object`` and pre-computes its cost, so preserve it
+        # here instead of zeroing the spend on an otherwise-failed request.
+        if self.model_call_details.get("combined_usage_object") is None:
+            self.model_call_details["response_cost"] = 0
 
         if hasattr(exception, "headers") and isinstance(exception.headers, dict):
             self.model_call_details.setdefault("litellm_params", {})

litellm/litellm_core_utils/streaming_handler.py

@@ -1923,11 +1923,29 @@ def __next__(self) -> "ModelResponseStream":  # noqa: PLR0915
 
         except StopIteration:
             if self.sent_last_chunk is True:
-                complete_streaming_response = litellm.stream_chunk_builder(
-                    chunks=self.chunks,
-                    messages=self.messages,
-                    logging_obj=self.logging_obj,
-                )
+                try:
+                    complete_streaming_response = litellm.stream_chunk_builder(
+                        chunks=self.chunks,
+                        messages=self.messages,
+                        logging_obj=self.logging_obj,
+                    )
+                except Exception as e:
+                    # stream_chunk_builder can re-raise (as APIError) on large agentic
+                    # streams. The raise originates inside this except-StopIteration block,
+                    # so the sibling `except Exception` below does not catch it; it would
+                    # escape __next__ and drop the request from SpendLogs. Recover
+                    # best-effort usage from the raw chunks so cost is still tracked
+                    verbose_logger.warning(
+                        "stream_chunk_builder raised at end-of-stream (%s); logging "
+                        "best-effort usage from chunks.",
+                        str(e),
+                    )
+                    try:
+                        complete_streaming_response = self.model_response_creator(
+                            chunk={"usage": calculate_total_usage(chunks=self.chunks)}
+                        )
+                    except Exception:
+                        complete_streaming_response = None
 
                 response = self.model_response_creator()
                 if complete_streaming_response is not None:
@@ -2152,11 +2170,27 @@ async def __anext__(self) -> "ModelResponseStream":  # noqa: PLR0915
         except (StopAsyncIteration, StopIteration):
             if self.sent_last_chunk is True:
                 # log the final chunk with accurate streaming values
-                complete_streaming_response = litellm.stream_chunk_builder(
-                    chunks=self.chunks,
-                    messages=self.messages,
-                    logging_obj=self.logging_obj,
-                )
+                try:
+                    complete_streaming_response = litellm.stream_chunk_builder(
+                        chunks=self.chunks,
+                        messages=self.messages,
+                        logging_obj=self.logging_obj,
+                    )
+                except Exception as e:
+                    # see sync __next__: a raise from stream_chunk_builder inside this
+                    # except handler escapes __anext__ and drops the request from SpendLogs.
+                    # Recover best-effort usage from the raw chunks so cost is still tracked
+                    verbose_logger.warning(
+                        "stream_chunk_builder raised at end-of-stream (%s); logging "
+                        "best-effort usage from chunks.",
+                        str(e),
+                    )
+                    try:
+                        complete_streaming_response = self.model_response_creator(
+                            chunk={"usage": calculate_total_usage(chunks=self.chunks)}
+                        )
+                    except Exception:
+                        complete_streaming_response = None
 
                 response = self.model_response_creator()
                 if complete_streaming_response is not None:
@@ -2233,6 +2267,7 @@ async def __anext__(self) -> "ModelResponseStream":  # noqa: PLR0915
                 litellm.request_timeout
             )
             if self.logging_obj is not None:
+                self._record_partial_usage_for_failure()
                 ## LOGGING
                 threading.Thread(
                     target=self.logging_obj.failure_handler,
@@ -2246,6 +2281,7 @@ async def __anext__(self) -> "ModelResponseStream":  # noqa: PLR0915
         except Exception as e:
             traceback_exception = traceback.format_exc()
             if self.logging_obj is not None:
+                self._record_partial_usage_for_failure()
                 ## LOGGING
                 threading.Thread(
                     target=self.logging_obj.failure_handler,
@@ -2257,6 +2293,33 @@ async def __anext__(self) -> "ModelResponseStream":  # noqa: PLR0915
                 )
             self._handle_stream_fallback_error(e)
 
+    def _record_partial_usage_for_failure(self) -> None:
+        """
+        A stream that breaks mid-flight still billed the provider for the chunks
+        already delivered. Recover that partial usage from the chunks seen so
+        far and stash it, with its cost, on the logging object so the failure
+        handler records the real partial spend instead of zero. A request that
+        later recovers via a router fallback overwrites this with the combined
+        success log on the same request id, so this never double counts.
+        """
+        if self.logging_obj is None or not self.chunks:
+            return
+        try:
+            partial_response = litellm.stream_chunk_builder(chunks=self.chunks)
+            usage = cast(Optional[Usage], getattr(partial_response, "usage", None))
+            if usage is None:
+                return
+            self.logging_obj.model_call_details["combined_usage_object"] = usage
+            self.logging_obj.model_call_details["response_cost"] = (
+                self.logging_obj._response_cost_calculator(result=partial_response)
+                or 0.0
+            )
+        except Exception as recover_error:
+            verbose_logger.debug(
+                "could not recover partial usage for interrupted stream: %s",
+                recover_error,
+            )
+
     def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
         """
         Common error handling for both __next__ and __anext__.

litellm/proxy/auth/auth_checks.py

@@ -3282,9 +3282,18 @@ async def _virtual_key_max_budget_check(
         ####################################
 
         if spend >= valid_token.max_budget:
+            # name the key in the error so operators don't have to reverse-map
+            # spend back to a key; key_name is the masked form (last 4 chars)
+            key_label = valid_token.key_alias or "key"
+            key_descriptor = (
+                f"{key_label} ({valid_token.key_name})"
+                if valid_token.key_name
+                else key_label
+            )
             raise litellm.BudgetExceededError(
                 current_cost=spend,
                 max_budget=valid_token.max_budget,
+                message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}",
             )