Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions litellm/experimental_mcp_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -555,7 +555,7 @@ async def call_tool(
Call an MCP Tool.
"""
verbose_logger.info(
f"MCP client calling tool '{call_tool_request_params.name}' with arguments: {call_tool_request_params.arguments}"
f"MCP client calling tool '{call_tool_request_params.name}'"
)

async def on_progress(
Expand Down Expand Up @@ -664,7 +664,7 @@ async def get_prompt(
) -> GetPromptResult:
"""Fetch a prompt definition from the MCP server."""
verbose_logger.info(
f"MCP client fetching prompt '{get_prompt_request_params.name}' with arguments: {get_prompt_request_params.arguments}"
f"MCP client fetching prompt '{get_prompt_request_params.name}'"
)

async def _get_prompt_operation(session: ClientSession):
Expand Down
9 changes: 6 additions & 3 deletions litellm/litellm_core_utils/prompt_templates/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -5547,7 +5547,7 @@ def _bedrock_tools_pt(
]
"""
from litellm.llms.bedrock.common_utils import (
get_bedrock_base_model,
bedrock_converse_supports_strict_tools,
normalize_json_schema_custom_types_to_object,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
Expand All @@ -5556,9 +5556,12 @@ def _bedrock_tools_pt(
("array", "boolean", "integer", "null", "number", "object", "string")
)
# Only Claude on Bedrock honours strict tool schemas; other families
# (Nova, Llama, GPT-OSS) reject the strict field outright.
# (Nova, Llama, GPT-OSS) reject the strict field outright. Opus 4.7/4.8
# also reject `strict` on Bedrock Converse (see #31582) — their validator
# maps toolSpec to the native Anthropic tool shape, which has no strict
# field, even though Anthropic's native API accepts it as a top-level key.
supports_strict_tools = bool(
model and get_bedrock_base_model(model).startswith("anthropic")
model and bedrock_converse_supports_strict_tools(model)
)
tool_block_list: List[BedrockToolBlock] = []
for tool_idx, tool in enumerate(tools):
Expand Down
39 changes: 31 additions & 8 deletions litellm/llms/bedrock/chat/converse_transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
from ..common_utils import (
BedrockError,
BedrockModelInfo,
bedrock_converse_supports_parallel_tool_use_config,
get_anthropic_beta_from_headers,
get_bedrock_tool_name,
is_claude_4_5_on_bedrock,
Expand Down Expand Up @@ -1176,18 +1177,34 @@ def _get_cache_point_block(
if cache_control is None:
return None

cache_point = CachePointBlock(type="default")
if isinstance(cache_control, dict) and "ttl" in cache_control:
ttl = cache_control["ttl"]
if ttl in ["5m", "1h"] and model is not None:
if is_claude_4_5_on_bedrock(model):
cache_point["ttl"] = ttl
cache_point = self._build_cache_point_block(cache_control, model)

if block_type == "system":
return SystemContentBlock(cachePoint=cache_point)
else:
return ContentBlock(cachePoint=cache_point)

@staticmethod
def _build_cache_point_block(
control: Optional[dict], model: Optional[str] = None
) -> CachePointBlock:
"""Build a Bedrock ``cachePoint`` block from an OpenAI-style ``cache_control``/``control`` dict.

``type`` is always ``"default"`` (the only value Bedrock's Converse API
accepts). ``ttl`` is only honored for models that support extended TTL
caching (Claude 4.5 family on Bedrock).
"""
cache_point = CachePointBlock(type="default")
if isinstance(control, dict) and "ttl" in control:
ttl = control["ttl"]
if (
ttl in ["5m", "1h"]
and model is not None
and is_claude_4_5_on_bedrock(model)
):
cache_point["ttl"] = ttl
return cache_point

def _transform_system_message(
self, messages: List[AllMessageValues], model: Optional[str] = None
) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
Expand Down Expand Up @@ -1339,7 +1356,10 @@ def _prepare_request_params(
parallel_tool_use_config = additional_request_params.pop(
"_parallel_tool_use_config", None
)
if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
if (
parallel_tool_use_config is not None
and bedrock_converse_supports_parallel_tool_use_config(model)
):
for key, value in parallel_tool_use_config.items():
if (
key in additional_request_params
Expand Down Expand Up @@ -1651,7 +1671,10 @@ def _transform_request_helper(
if cache_injection_points and len(bedrock_tools) > 0:
for point in cache_injection_points:
if point.get("location") == "tool_config":
bedrock_tools.append({"cachePoint": {"type": "default"}})
cache_point = self._build_cache_point_block(
point.get("control"), model
)
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
break

bedrock_tool_config: Optional[ToolConfigBlock] = None
Expand Down
105 changes: 75 additions & 30 deletions litellm/llms/bedrock/common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
Common utilities used across bedrock chat/embedding/image generation
"""

import contextlib
import functools
import json
import os
import re
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union

if TYPE_CHECKING:
Expand Down Expand Up @@ -650,39 +652,82 @@ def get_bedrock_base_model(model: str) -> str:
return model


def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
return any(
(litellm.model_cost.get(candidate) or {}).get(
"supports_parallel_tool_use_config"
)
is True
for candidate in (model, get_bedrock_base_model(model))
)


def is_claude_4_5_on_bedrock(model: str) -> bool:
"""
Check if the model is a Claude 4.5 model on Bedrock.
Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
Check if the model supports Bedrock prompt caching with an extended '1h' TTL
(in addition to the default 5m TTL).

Backed by the ``cache_creation_input_token_cost_above_1hr`` field in
``model_prices_and_context_window.json`` instead of a hardcoded list of
model-name patterns, so newly released models pick up support as soon as
their pricing entry ships, with no code change required here.
"""
model_lower = model.lower()
claude_4_5_patterns = [
"sonnet-4.5",
"sonnet_4.5",
"sonnet-4-5",
"sonnet_4_5",
"haiku-4.5",
"haiku_4.5",
"haiku-4-5",
"haiku_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
"sonnet-4.6",
"sonnet_4.6",
"sonnet-4-6",
"sonnet_4_6",
"opus-4.6",
"opus_4.6",
"opus-4-6",
"opus_4_6",
"opus-4.7",
"opus_4.7",
"opus-4-7",
"opus_4_7",
]
return any(pattern in model_lower for pattern in claude_4_5_patterns)
return any(
(litellm.model_cost.get(candidate) or {}).get(
"cache_creation_input_token_cost_above_1hr"
)
is not None
for candidate in (model, get_bedrock_base_model(model))
)


_BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$")


def bedrock_converse_supports_strict_tools(model: str) -> bool:
"""
Whether ``toolSpec.strict`` can be forwarded to Bedrock Converse for ``model``.

Non-Anthropic Bedrock families (Nova, Llama, GPT-OSS) reject the field
outright. Anthropic models forward it unless their entry in
``model_prices_and_context_window.json`` sets
``bedrock_converse_supports_strict_tools: false`` — Bedrock routes those
(Opus 4.7/4.8, see #31582) through a stricter validator that rejects the
``strict`` key on ``toolSpec`` even though Anthropic's native API accepts
it as a top-level tool field.
"""
base = get_bedrock_base_model(model)
if not base.startswith("anthropic"):
return False
flag = _get_bedrock_converse_strict_tools_flag(base)
return flag if flag is not None else True


def _get_bedrock_converse_strict_tools_flag(base_model: str) -> Optional[bool]:
candidates = dict.fromkeys(
(base_model, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base_model))
)
for candidate in candidates:
with contextlib.suppress(Exception):
model_info = get_cached_model_info()(
model=candidate,
custom_llm_provider="bedrock",
)

flag = model_info.get("bedrock_converse_supports_strict_tools")
if isinstance(flag, bool):
return flag

model_cost_key = model_info.get("key")
if isinstance(model_cost_key, str):
local_flag = (
_get_local_model_cost_map()
.get(model_cost_key, {})
.get("bedrock_converse_supports_strict_tools")
)
if isinstance(local_flag, bool):
return local_flag
return None
Comment on lines +706 to +730

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Strict-tools lookup uses a different access pattern than its siblings

is_claude_4_5_on_bedrock and bedrock_converse_supports_parallel_tool_use_config both read from litellm.model_cost directly (with a base-model fallback), so they always reflect whatever cost-map is currently loaded — including test overrides. _get_bedrock_converse_strict_tools_flag instead goes through get_model_info(), which is @lru_cache-decorated. If litellm.model_cost is swapped in a test (or hot-reloaded in a long-running proxy process) after the first call, the strict-tools check may silently return a stale cached value while the other two checks immediately reflect the new map. A simpler direct litellm.model_cost.get(candidate) pattern (as the other two helpers use) would avoid this discrepancy and eliminate the need for the _get_local_model_cost_map() fallback path.



def normalize_bedrock_opus_output_config_effort(model: str, output_config: Any) -> None:
Expand Down
Loading
Loading