chore: sync upstream 2026-07-17

.github/fork-patches.txt

@@ -36,3 +36,4 @@ litellm/llms/anthropic/chat/transformation.py | _translate_legacy_thinking_for_a
 ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx | import { getProviderLogoAndName } from "@/components/provider_info_helpers" | Fork-only provider display-name enhancement (model group / provider columns render getProviderLogoAndName(provider).displayName instead of the raw provider slug, including a sort comparator that sorts by the rendered name). Originally lived in ui/litellm-dashboard/src/components/model_hub_table_columns.tsx; the 2026-07-17 sync's upstream refactor (#33629, shared DataTable migration) deleted that file and moved its columns to this path, dropping the enhancement along the way, so the patch was reapplied at the new location plus the sibling Providers-modal usage in AIHub/ModelHubTable.tsx (same drop-only-the-import pattern). REMOVAL CONDITION: remove once upstream ships equivalent provider display-name rendering in these files itself.
 ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx | import { getProviderLogoAndName } from "@/components/provider_info_helpers" | Fork-only provider display-name enhancement for the model detail modal's Providers badges (renders getProviderLogoAndName(provider).displayName instead of the raw provider slug). Sibling patch to AIHub/ModelHubTableColumns.tsx above; the 2026-07-17 sync merge kept the fork's call site but dropped only the import line, the same drop-only-the-declaration pattern documented for the anthropic legacy-thinking translation. REMOVAL CONDITION: remove once upstream ships equivalent provider display-name rendering in this file itself.
 .github/workflows/codspeed.yml | continue-on-error: true | rayward-external/litellm was never registered with CodSpeed (codspeed.io) — the "Run benchmarks" step's own pytest-codspeed run passes, but the upload step 401s ("Repository not found or the user does not have access to it") on every PR/push run, so continue-on-error keeps a fork-infra gap from blocking the merge gate. First observed 2026-07-16 (PR #123, the first PR whose base commit made this workflow's checks land on a PR at all). REMOVAL CONDITION: remove once an admin registers rayward-external/litellm with CodSpeed (or adds a working CODSPEED_TOKEN) and the upload step succeeds.
+tests/test_litellm/proxy/test_budget_reservation.py | _arelease_max_parallel_requests_on_disconnect = AsyncMock\(\) | Upstream #33736 moved the max_parallel_requests slot release into an async proxy_logging_obj._arelease_max_parallel_requests_on_disconnect() call inside async_streaming_data_generator's disconnect cleanup, but only added coverage in the new tests/test_litellm/proxy/test_common_request_processing.py; this pre-existing sibling file's bare MagicMock() stand-ins for proxy_logging_obj were never given that attribute, so awaiting it raised "TypeError: object MagicMock can't be used in 'await' expression" in 4 tests. Both call sites are IDENTICAL to upstream (verified via `git diff upstream/litellm_internal_staging`), so this reproduces on a clean upstream checkout too — upstream's own CI misses it because the new test file doesn't exercise these older streaming-cancel fixtures. REMOVAL CONDITION: remove once upstream adds the same AsyncMock to test_budget_reservation.py's _drive_streaming_cancel/slow-path fixtures itself.

litellm/__init__.py

@@ -315,6 +315,11 @@ def _dev_env_hot_reload_enabled() -> bool:
 disable_add_transform_inline_image_block: bool = False
 disable_add_user_agent_to_request_tags: bool = False
 disable_anthropic_gemini_context_caching_transform: bool = False
+enable_anthropic_prompt_caching: bool = os.getenv("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", "false").lower() == "true"
+_anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL")
+anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
+    "1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
+)
 disable_vertex_batch_output_transformation: bool = False
 extra_spend_tag_headers: Optional[List[str]] = None
 in_memory_llm_clients_cache: "LLMClientCache"

litellm/integrations/anthropic_cache_control_hook.py

@@ -412,18 +412,148 @@ def apply_to_anthropic_messages_request(
 
         return processed_messages, processed_system, remaining_points
 
+    @staticmethod
+    def _default_control() -> ChatCompletionCachedContent:
+        """Build the cache_control block for auto-injected breakpoints.
+
+        Defaults to Anthropic's 5-minute ephemeral cache; honors the optional
+        ``litellm.anthropic_prompt_caching_ttl`` override ("5m" or "1h").
+        """
+        import litellm
+
+        ttl = litellm.anthropic_prompt_caching_ttl
+        if ttl == "5m" or ttl == "1h":
+            return ChatCompletionCachedContent(type="ephemeral", ttl=ttl)
+        return ChatCompletionCachedContent(type="ephemeral")
+
+    @staticmethod
+    def _request_has_cache_control(
+        messages: list[AllMessageValues],
+        system: str | list | None,
+        tools: list | None = None,
+    ) -> bool:
+        """Return True if the request already carries any client-supplied cache_control.
+
+        When the client (e.g. Claude Code) already marks its own breakpoints we
+        stand down entirely rather than add more, per the auto-caching contract.
+        Tools count: they are a breakpoint the client can mark, they count toward
+        the provider's four-block limit, and caching only the tool definitions is
+        a common pattern, so injecting alongside them can exceed the cap.
+        """
+        if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages):
+            return True
+        if isinstance(system, list):
+            if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system):
+                return True
+        if tools is not None:
+            return any(isinstance(tool, dict) and tool.get("cache_control") is not None for tool in tools)
+        return False
+
+    @staticmethod
+    def get_default_injection_points(
+        messages: list[AllMessageValues],
+        system: str | list | None,
+        model: str,
+        custom_llm_provider: str | None,
+        tools: list | None = None,
+    ) -> list[CacheControlInjectionPoint]:
+        """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
+
+        Caches the system prompt and the trailing turn, so the stable prefix
+        (system + tools + history) is reused while the breakpoint advances with
+        the conversation. Returns [] (stand down) when the flag is off, the
+        provider does not consume cache_control breakpoints (only anthropic /
+        bedrock do), the model lacks prompt-caching support, or the request
+        already carries client-supplied cache_control.
+        """
+        import litellm
+
+        if litellm.enable_anthropic_prompt_caching is not True:
+            return []
+
+        provider = custom_llm_provider
+        if provider is None:
+            from litellm.litellm_core_utils.get_llm_provider_logic import (
+                get_llm_provider,
+            )
+
+            try:
+                _, provider, _, _ = get_llm_provider(model=model)
+            except Exception:  # noqa: BLE001  # unroutable model must never block the call, just skip auto-caching
+                return []
+
+        if provider not in ("anthropic", "bedrock"):
+            return []
+
+        from litellm.utils import supports_prompt_caching
+
+        if not supports_prompt_caching(model=model, custom_llm_provider=provider):
+            return []
+
+        if AnthropicCacheControlHook._request_has_cache_control(messages, system, tools):
+            return []
+
+        control = AnthropicCacheControlHook._default_control()
+        points: list[CacheControlInjectionPoint] = [
+            CacheControlMessageInjectionPoint(location="message", role="system", index=None, control=control),
+            CacheControlMessageInjectionPoint(location="message", role=None, index=-1, control=control),
+        ]
+        return points
+
+    @staticmethod
+    def maybe_seed_default_injection_points(
+        non_default_params: dict[str, Any],
+        messages: list[AllMessageValues],
+        model: str,
+        custom_llm_provider: str | None,
+        tools: list | None = None,
+    ) -> None:
+        """For /chat/completions: add default injection points to the request params.
+
+        No-op when injection points are already configured (explicit config wins).
+        Seeding the param lets the existing prompt-management gate and the
+        AnthropicCacheControlHook run unchanged.
+        """
+        if non_default_params.get("cache_control_injection_points"):
+            return
+        points = AnthropicCacheControlHook.get_default_injection_points(
+            messages=messages,
+            system=None,
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            tools=tools,
+        )
+        if points:
+            non_default_params["cache_control_injection_points"] = points
+
     @staticmethod
     def maybe_inject_cache_control(
         messages: List[Dict],
         system: str | list | None,
         kwargs: Dict[str, Any],
+        model: str | None = None,
+        custom_llm_provider: str | None = None,
+        tools: list[dict] | None = None,
     ) -> Tuple[List[Dict], str | list | None]:
         """Extract cache_control_injection_points from kwargs and apply if present.
 
+        When none are configured but ``litellm.enable_anthropic_prompt_caching``
+        is on, synthesize default breakpoints for the native /v1/messages path.
         Pops the key from kwargs; if remaining (non-message) points exist they
         are written back so downstream transforms can handle them.
         """
-        injection_points = kwargs.pop("cache_control_injection_points", None)
+        configured = cast(  # cast-ok: kwargs is untyped; this key only holds the documented injection-point list
+            list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None)
+        )
+        injection_points: list[CacheControlInjectionPoint] = configured or []
+        if not injection_points and model is not None:
+            injection_points = AnthropicCacheControlHook.get_default_injection_points(
+                messages=cast(list[AllMessageValues], messages),  # cast-ok: Anthropic-shaped dicts from v1/messages
+                system=system,
+                tools=tools,
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+            )
         if not injection_points:
             return messages, system
 

litellm/integrations/opik/utils.py

@@ -1,40 +1,38 @@
 import configparser
 import os
 import time
+import uuid
 from typing import Any, Dict, Final, List, Optional, Tuple
 
 CONFIG_FILE_PATH_DEFAULT: Final[str] = "~/.opik.config"
 
 
-def create_uuid7():
-    ns = time.time_ns()
-    last = [0, 0, 0, 0]
-
-    # Simple uuid7 implementation
-    sixteen_secs = 16_000_000_000
-    t1, rest1 = divmod(ns, sixteen_secs)
-    t2, rest2 = divmod(rest1 << 16, sixteen_secs)
-    t3, _ = divmod(rest2 << 12, sixteen_secs)
-    t3 |= 7 << 12  # Put uuid version in top 4 bits, which are 0 in t3
-
-    # The next two bytes are an int (t4) with two bits for
-    # the variant 2 and a 14 bit sequence counter which increments
-    # if the time is unchanged.
-    if t1 == last[0] and t2 == last[1] and t3 == last[2]:
-        # Stop the seq counter wrapping past 0x3FFF.
-        # This won't happen in practice, but if it does,
-        # uuids after the 16383rd with that same timestamp
-        # will not longer be correctly ordered but
-        # are still unique due to the 6 random bytes.
-        if last[3] < 0x3FFF:
-            last[3] += 1
-    else:
-        last[:] = (t1, t2, t3, 0)
-    t4 = (2 << 14) | last[3]  # Put variant 0b10 in top two bits
-
-    # Six random bytes for the lower part of the uuid
-    rand = os.urandom(6)
-    return f"{t1:>08x}-{t2:>04x}-{t3:>04x}-{t4:>04x}-{rand.hex()}"
+def create_uuid7() -> str:
+    """Generate an RFC 9562 conformant UUIDv7 string.
+
+    The top 48 bits encode the Unix timestamp in milliseconds. Opik's backend
+    validates this embedded timestamp on ingestion (it must fall within a window
+    around "now"), so the encoding has to be correct or trace/span batches are
+    rejected with HTTP 400. Implemented with the standard library only, so no
+    extra dependency is added to litellm. See ``opik.id_helpers`` for the
+    reference implementation.
+    """
+    unix_ts_ms = int(time.time() * 1000)
+
+    # Fill the 16-byte buffer with random data, then overwrite the structured
+    # parts (timestamp, version, variant) defined by the UUIDv7 layout.
+    uuid_bytes = bytearray(os.urandom(16))
+
+    # First 48 bits (6 bytes): Unix timestamp in milliseconds.
+    uuid_bytes[0:6] = unix_ts_ms.to_bytes(6, byteorder="big")
+
+    # Version 7 in the top 4 bits of byte 6.
+    uuid_bytes[6] = 0x70 | (uuid_bytes[6] & 0x0F)
+
+    # Variant 0b10 in the top 2 bits of byte 8.
+    uuid_bytes[8] = 0x80 | (uuid_bytes[8] & 0x3F)
+
+    return str(uuid.UUID(bytes=bytes(uuid_bytes)))
 
 
 def _read_opik_config_file() -> Dict[str, str]:

litellm/litellm_core_utils/litellm_logging.py

@@ -1453,6 +1453,9 @@ def _response_cost_calculator(
             response_cost = litellm.response_cost_calculator(**response_cost_calculator_kwargs)
 
             verbose_logger.debug(f"response_cost: {response_cost}")
+            additional_response_cost: object = self.model_call_details.get("additional_response_cost")
+            if isinstance(additional_response_cost, (int, float)) and additional_response_cost > 0:
+                return (response_cost or 0.0) + additional_response_cost
             return response_cost
         except Exception as e:  # error calculating cost
             debug_info = StandardLoggingModelCostFailureDebugInformation(

litellm/llms/anthropic/common_utils.py

@@ -906,16 +906,17 @@ def strip_advisor_blocks_from_messages(messages: List[Any], replace_with_text: b
 
 def is_anthropic_invalid_thinking_signature_error(error_text: str) -> bool:
     """
-    Detect Anthropic 400 when encrypted thinking signatures in history do not match
-    the current deployment (e.g. user rotated API key or switched model endpoint).
+    Detect Anthropic 400 errors caused by missing or invalid thinking signatures.
 
-    Example API message:
+    Known error formats:
+    {"message":"messages.2.content.0.thinking.signature.str: Input should be a valid string"}
+    messages.N.content.M.thinking.signature.str: Input should be a valid string
     messages.N.content.M: Invalid `signature` in `thinking` block
     """
     if not error_text:
         return False
     lower = error_text.lower()
-    return "invalid" in lower and "signature" in lower and "thinking" in lower and "block" in lower
+    return "thinking" in lower and "signature" in lower and ("invalid" in lower or "valid string" in lower)
 
 
 def strip_thinking_blocks_from_anthropic_messages(messages: List[Any]) -> List[Any]:

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -237,7 +237,9 @@ async def anthropic_messages(
         AnthropicCacheControlHook,
     )
 
-    messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs)
+    messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
+        messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
+    )
 
     original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False)
 
@@ -426,7 +428,9 @@ def anthropic_messages_handler(
         AnthropicCacheControlHook,
     )
 
-    messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs)
+    messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
+        messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
+    )
 
     metadata = validate_anthropic_api_metadata(metadata)
 

litellm/llms/fireworks_ai/cost_calculator.py

@@ -75,10 +75,23 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
         model_info = get_model_info(model=base_model, custom_llm_provider="fireworks_ai")
 
     ## CALCULATE INPUT COST
-
-    prompt_cost: float = usage["prompt_tokens"] * model_info["input_cost_per_token"]
+    prompt_tokens_details = usage.prompt_tokens_details
+    cached_tokens: int = (
+        prompt_tokens_details.cached_tokens
+        if prompt_tokens_details is not None and prompt_tokens_details.cached_tokens is not None
+        else 0
+    )
+    input_cost_per_token: float = model_info["input_cost_per_token"] or 0.0
+    cache_read_input_token_cost = model_info.get("cache_read_input_token_cost")
+    cache_read_cost_per_token: float = (
+        cache_read_input_token_cost if cache_read_input_token_cost is not None else input_cost_per_token
+    )
+    non_cached_prompt_tokens: int = max(usage.prompt_tokens - cached_tokens, 0)
+
+    prompt_cost: float = non_cached_prompt_tokens * input_cost_per_token + cached_tokens * cache_read_cost_per_token
 
     ## CALCULATE OUTPUT COST
-    completion_cost = usage["completion_tokens"] * model_info["output_cost_per_token"]
+    output_cost_per_token: float = model_info["output_cost_per_token"] or 0.0
+    completion_cost: float = usage.completion_tokens * output_cost_per_token
 
     return prompt_cost, completion_cost

litellm/main.py

@@ -510,6 +510,20 @@ async def acompletion(
     #########################################################
     #########################################################
     litellm_logging_obj = kwargs.get("litellm_logging_obj", None)
+
+    from litellm.integrations.anthropic_cache_control_hook import (
+        AnthropicCacheControlHook,
+    )
+    from litellm.types.llms.openai import AllMessageValues
+
+    AnthropicCacheControlHook.maybe_seed_default_injection_points(
+        non_default_params=kwargs,
+        messages=cast(list[AllMessageValues], messages),  # cast-ok: acompletion types messages as a bare List
+        model=model,
+        custom_llm_provider=cast(Optional[str], custom_llm_provider),  # cast-ok: read from untyped kwargs
+        tools=tools,
+    )
+
     if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
         litellm_logging_obj.should_run_prompt_management_hooks(
             prompt_id=kwargs.get("prompt_id", None),
@@ -5055,6 +5069,19 @@ def completion(  # type: ignore
     litellm_params = {}  # used to prevent unbound var errors
     ## PROMPT MANAGEMENT HOOKS ##
 
+    from litellm.integrations.anthropic_cache_control_hook import (
+        AnthropicCacheControlHook,
+    )
+    from litellm.types.llms.openai import AllMessageValues
+
+    AnthropicCacheControlHook.maybe_seed_default_injection_points(
+        non_default_params=non_default_params,
+        messages=cast(list[AllMessageValues], messages),  # cast-ok: completion types messages as a bare List
+        model=model,
+        custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")),  # cast-ok: untyped kwargs
+        tools=tools,
+    )
+
     if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
         litellm_logging_obj.should_run_prompt_management_hooks(
             prompt_id=prompt_id, non_default_params=non_default_params