From ac42eb105351a0aafecb6a94b7fbeacaa1cbdebd Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 28 Sep 2026 17:20:08 +0000 Subject: [PATCH 1/5] feat(bedrock): pass chat through to each model's native openai endpoint on runtime and mantle Co-authored-by: Matthew Lapointe Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/model-catalog/src/model_info.rs | 2 +- .../litellm_core_utils/responses_api_utils.py | 62 +++ .../chat/chat_completions/transformation.py | 39 +- litellm/llms/bedrock/common_utils.py | 163 ++++-- .../bedrock_mantle/chat/transformation.py | 50 +- litellm/llms/bedrock_mantle/common_utils.py | 24 +- litellm/main.py | 96 +++- ...odel_prices_and_context_window_backup.json | 475 +++++++++++++++++- litellm/utils.py | 23 +- model_prices_and_context_window.json | 475 +++++++++++++++++- model_prices_and_context_window.schema.json | 2 +- ...bedrock_chat_completions_transformation.py | 301 +++++++++-- .../llms/bedrock/chat/test_invoke_handler.py | 4 +- .../test_bedrock_mantle_transformation.py | 38 +- tests/unit/test_main.py | 115 ++++- tests/unit/test_model_prices_schema.py | 73 +++ tests/unit/test_utils.py | 2 +- 17 files changed, 1748 insertions(+), 196 deletions(-) create mode 100644 litellm/litellm_core_utils/responses_api_utils.py diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 8577eb8af682..af724c2f3ae0 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -434,7 +434,7 @@ pub struct ModelInfo { #[serde(skip_serializing_if = "Option::is_none")] pub supports_bedrock_runtime_chat_completions_response_format: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option, + pub supports_bedrock_chat_completions_tools_with_reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] pub supports_computer_use: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/litellm/litellm_core_utils/responses_api_utils.py b/litellm/litellm_core_utils/responses_api_utils.py new file mode 100644 index 000000000000..7ea8f5367d12 --- /dev/null +++ b/litellm/litellm_core_utils/responses_api_utils.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final, Protocol, cast, runtime_checkable + + +@runtime_checkable +class _FunctionToolLike(Protocol): + type: str + + +def has_function_tool(tools: object) -> bool: + if not isinstance(tools, Sequence): + return False + typed_tools: Final = cast(Sequence[object], tools) # cast-ok: a runtime Sequence contains Python objects + return any(_is_function_tool(tool) for tool in typed_tools) + + +def _is_function_tool(tool: object) -> bool: + if not isinstance(tool, Mapping): + return isinstance(tool, _FunctionToolLike) and tool.type == "function" + tool_mapping: Final = cast(Mapping[str, object], tool) # cast-ok: OpenAI tool mappings have string keys + function: Final = tool_mapping.get("function") + return tool_mapping.get("type") == "function" and (isinstance(function, Mapping) or "name" in tool_mapping) + + +def reasoning_effort_is_active( + reasoning_effort: object, + supports_none_reasoning_effort: bool | None = None, +) -> bool: + if isinstance(reasoning_effort, Mapping): + effort_mapping: Final = cast( # cast-ok: reasoning payload keys are strings + Mapping[str, object], reasoning_effort + ) + return effort_mapping.get("effort") != "none" or effort_mapping.get("summary") is not None + return reasoning_effort != "none" or supports_none_reasoning_effort is False + + +def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> object | None: + if "reasoningSummary" in optional_params: + return optional_params["reasoningSummary"] + if "reasoning_summary" in optional_params: + return optional_params["reasoning_summary"] + extra_body: Final = optional_params.get("extra_body") + if isinstance(extra_body, Mapping): + extra_body_mapping: Final = cast( # cast-ok: extra_body uses JSON object keys + Mapping[str, object], extra_body + ) + if "reasoningSummary" in extra_body_mapping: + return extra_body_mapping["reasoningSummary"] + if "reasoning_summary" in extra_body_mapping: + return extra_body_mapping["reasoning_summary"] + return None + + +def filter_additional_drop_params( + request_params: Mapping[str, object], + additional_drop_params: Sequence[str] | None, +) -> Mapping[str, object]: + dropped: Final = frozenset(additional_drop_params or ()) + return MappingProxyType({key: value for key, value in request_params.items() if key not in dropped}) diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 381ce7922d7b..2de979c7554f 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Final, Literal import httpx +from pydantic import TypeAdapter from typing_extensions import assert_never import litellm @@ -29,7 +30,12 @@ ) from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM -from litellm.llms.bedrock.common_utils import BedrockError, split_bedrock_region_path +from litellm.llms.bedrock.common_utils import ( + BedrockError, + bedrock_chat_completions_serves_tools_with_reasoning, + split_bedrock_region_path, + with_max_completion_tokens, +) from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues @@ -41,6 +47,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj REASONING_OPEN_TAG: Final = "" +_SUPPORTED_OPENAI_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) REASONING_CLOSE_TAG: Final = "" CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY: Final = MappingProxyType( @@ -200,24 +207,6 @@ def chunk_parser(self, chunk: dict) -> ModelResponseStream: # mutable-ok: BaseM return parsed -def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, object]: - """ - Send the caller's ``max_tokens`` as ``max_completion_tokens``. - - Every model on this surface accepts ``max_completion_tokens`` and the GPT-5.6 family - rejects ``max_tokens``; an explicit ``max_completion_tokens`` wins when both are set. - """ - if "max_tokens" not in params: - return params - return MappingProxyType( - { - key: value - for key, value in (("max_completion_tokens", params["max_tokens"]), *params.items()) - if key != "max_tokens" - } - ) - - class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None: super().__init__() @@ -394,15 +383,19 @@ def transform_response( choice.message.content = content return response - def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseConfig signature + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseConfig signature refused: Final = frozenset(("n", *chat_completions_params_refused_for(model))) - base_params: Final = tuple( - param for param in super().get_supported_openai_params(model) if param not in refused + supported_params: Final = _SUPPORTED_OPENAI_PARAMS_ADAPTER.validate_python( + super().get_supported_openai_params(model) # pyright: ignore[reportUnknownMemberType] # base untyped ) + base_params: Final = tuple(param for param in supported_params if param not in refused) reasoning_param: Final = ( ("reasoning_effort",) if "reasoning_effort" not in base_params - and litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider) + and ( + bedrock_chat_completions_serves_tools_with_reasoning(model) + or litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider) + ) else () ) return [*base_params, *reasoning_param] # mutable-ok: BaseConfig signature returns a list diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 6bd4a8d634c5..1fde32cb25a7 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -11,7 +11,7 @@ import re from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, cast if TYPE_CHECKING: from botocore.model import Shape @@ -24,6 +24,12 @@ import litellm from litellm import verbose_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.litellm_core_utils.responses_api_utils import ( + filter_additional_drop_params, + has_function_tool, + peek_reasoning_summary_aliases, + reasoning_effort_is_active, +) from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -829,18 +835,43 @@ def split_bedrock_region_path(model: str) -> tuple[str | None, str]: return None, stripped +def _bedrock_mantle_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: + mantle_model: Final = model.removeprefix("bedrock_mantle/") + region, model_id = split_bedrock_region_path(mantle_model) + candidates: Final = ( + f"bedrock_mantle/{mantle_model}", + f"bedrock_mantle/{region}/{model_id}" if region is not None else f"bedrock_mantle/{model_id}", + f"bedrock_mantle/{model_id}", + ) + model_cost: Final = cast( # cast-ok: model price rows use string keys + Mapping[str, Mapping[str, object]], litellm.model_cost + ) + return tuple(model_cost.get(key) for key in candidates) + + BEDROCK_RUNTIME_PRICE_MAP_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse")) def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: + model_cost: Final = cast( # cast-ok: model price rows use string keys + Mapping[str, Mapping[str, object]], litellm.model_cost + ) return tuple( - litellm.model_cost.get(key) - for key in (model, strip_bedrock_routing_prefix(model), split_bedrock_region_path(model)[1]) + model_cost.get(key) for key in (model, strip_bedrock_routing_prefix(model), split_bedrock_region_path(model)[1]) ) -def _bedrock_price_map_flag(model: str, flag: str) -> bool: - return any(entry is not None and entry.get(flag) is True for entry in _bedrock_price_map_entries(model)) +def _bedrock_price_map_flag( + model: str, + flag: str, + custom_llm_provider: Literal["bedrock", "bedrock_mantle"] = "bedrock", +) -> bool: + entries: Final = ( + _bedrock_mantle_price_map_entries(model) + if custom_llm_provider == "bedrock_mantle" + else _bedrock_price_map_entries(model) + ) + return any(entry is not None and entry.get(flag) is True for entry in entries) def _bedrock_runtime_row_lists_chat_completions(entry: Mapping[str, object]) -> bool: @@ -869,14 +900,48 @@ def uses_bedrock_runtime_chat_completions(model: str) -> bool: ) -def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool: +def bedrock_chat_completions_serves_tools_with_reasoning( + model: str, + custom_llm_provider: Literal["bedrock", "bedrock_mantle"] = "bedrock", +) -> bool: """Whether AWS's native Chat Completions serves this model's function tools with any ``reasoning_effort``. - Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` - flag (gpt-oss, Grok). Without it AWS only takes tools with ``reasoning_effort="none"`` - (the GPT-5.6 family), and Converse serves tools with any effort, so those requests fall back to it. + Data-driven from the price-map ``supports_bedrock_chat_completions_tools_with_reasoning`` + flag. Without it, function tools with active reasoning require Responses when available. """ - return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_tools_with_reasoning") + return _bedrock_price_map_flag( + model, + "supports_bedrock_chat_completions_tools_with_reasoning", + custom_llm_provider, + ) + + +def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, object]: + if "max_tokens" not in params: + return params + return MappingProxyType( + { + key: value + for key, value in (("max_completion_tokens", params["max_tokens"]), *params.items()) + if key != "max_tokens" + } + ) + + +def bedrock_reasoning_effort_is_active( + model: str, + reasoning_effort: object, + custom_llm_provider: Literal["bedrock", "bedrock_mantle"] = "bedrock", +) -> bool: + entries: Final = ( + _bedrock_mantle_price_map_entries(model) + if custom_llm_provider == "bedrock_mantle" + else _bedrock_price_map_entries(model) + ) + supports_none_reasoning_effort: Final = not any( + entry is not None and entry.get("supports_none_reasoning_effort") is False for entry in entries + ) + return reasoning_effort_is_active(reasoning_effort, supports_none_reasoning_effort) def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> bool: @@ -909,13 +974,27 @@ def _response_format_needs_converse(model: str, response_format: object) -> bool return False if not isinstance(response_format, Mapping): return not bedrock_runtime_chat_completions_enforces_response_format(model) - response_format_type: Final = response_format.get("type") + response_format_mapping: Final = cast( # cast-ok: API response-format keys are strings + Mapping[str, object], response_format + ) + response_format_type: Final = response_format_mapping.get("type") if response_format_type == "text": return False - is_json_schema: Final = response_format_type == "json_schema" and "json_schema" in response_format + is_json_schema: Final = response_format_type == "json_schema" and "json_schema" in response_format_mapping return not (is_json_schema and bedrock_runtime_chat_completions_enforces_response_format(model)) +def _bedrock_request_has_non_tool_converse_trigger( + model: str, + request_params: Mapping[str, object], +) -> bool: + return ( + any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS) + or bedrock_request_metadata_is_owned() + or _response_format_needs_converse(model, request_params.get("response_format")) + ) + + def bedrock_request_needs_converse(model: str, request_params: Mapping[str, object]) -> bool: """Whether a request on a runtime-Chat-Completions model must still be served by Converse. @@ -924,29 +1003,58 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje forwards as ``additionalModelRequestFields`` and ``inferenceConfig``) have no field on AWS's native OpenAI surface, ``stop`` stays on Converse where it fails loudly instead of silently stopping hidden reasoning, operator-owned request metadata is only written onto the Converse body, - function tools (``tools`` or legacy ``functions``) on a model without - ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless - ``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as + function tools with active reasoning on models without + ``supports_bedrock_chat_completions_tools_with_reasoning`` use Responses when available and Converse otherwise. + Legacy ``functions`` with active reasoning still require Converse, and a ``response_format`` goes native only as ``{"type": "json_schema", "json_schema": ...}`` (a pydantic model is converted to that) on a model with ``supports_bedrock_runtime_chat_completions_response_format``: a schema on any other model is only honored by Converse, and every ``json_object`` form (``response_schema`` included) keeps Converse's handling everywhere, since AWS's native surface rejects that type with a 400 unless the prompt mentions json. """ - if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS): + if _bedrock_request_has_non_tool_converse_trigger(model, request_params): return True - if bedrock_request_metadata_is_owned(): - return True - if _response_format_needs_converse(model, request_params.get("response_format")): - return True - if not (request_params.get("tools") or request_params.get("functions")): + has_tools: Final = has_function_tool(request_params.get("tools")) + has_legacy_functions: Final = bool(request_params.get("functions")) + if not (has_tools or has_legacy_functions): + return False + if bedrock_chat_completions_serves_tools_with_reasoning(model): + return False + if not bedrock_reasoning_effort_is_active(model, request_params.get("reasoning_effort")): + return False + model_cost: Final = cast( # cast-ok: model price rows use string keys + Mapping[str, object], litellm.model_cost + ) + return has_legacy_functions or not bedrock_supports_openai_responses(model, model_cost) + + +def bedrock_chat_request_needs_native_responses(model: str, request_params: Mapping[str, object]) -> bool: + if _bedrock_request_has_non_tool_converse_trigger(model, request_params): + return False + model_cost: Final = cast( # cast-ok: model price rows use string keys + Mapping[str, object], litellm.model_cost + ) + if not bedrock_supports_openai_responses(model, model_cost): return False + reasoning_effort: Final = request_params.get("reasoning_effort") + if reasoning_effort is not None and peek_reasoning_summary_aliases(request_params) is not None: + return True return ( - not bedrock_runtime_chat_completions_serves_tools_with_reasoning(model) - and request_params.get("reasoning_effort") != "none" + has_function_tool(request_params.get("tools")) + and not bedrock_chat_completions_serves_tools_with_reasoning(model) + and bedrock_reasoning_effort_is_active(model, reasoning_effort) ) +def bedrock_chat_request_needs_native_responses_for_request( + model: str, + request_params: Mapping[str, object], + additional_drop_params: Sequence[str] | None, +) -> bool: + filtered_request_params: Final = filter_additional_drop_params(request_params, additional_drop_params) + return bedrock_chat_request_needs_native_responses(model, filtered_request_params) + + def bedrock_route_for_request( model: str, request_params: Mapping[str, object], additional_drop_params: Sequence[str] | None ) -> BedrockRoute: @@ -955,9 +1063,8 @@ def bedrock_route_for_request( Param mapping and dispatch both call this with the same inputs, so a request that falls back to Converse is mapped with the Converse config and sent to Converse, never one without the other. """ - dropped: Final = frozenset(additional_drop_params or ()) return BedrockModelInfo.get_bedrock_route( - model, MappingProxyType({key: value for key, value in request_params.items() if key not in dropped}) + model, filter_additional_drop_params(request_params, additional_drop_params) ) @@ -1368,9 +1475,9 @@ def get_bedrock_route( if is_bedrock_application_inference_profile_arn(model): return "converse" - if uses_bedrock_runtime_chat_completions(model) and not ( - request_params is not None and bedrock_request_needs_converse(model, request_params) - ): + if uses_bedrock_runtime_chat_completions(model): + if request_params is not None and bedrock_request_needs_converse(model, request_params): + return "converse" return "chat_completions" base_model: Final = BedrockModelInfo.get_base_model(model) diff --git a/litellm/llms/bedrock_mantle/chat/transformation.py b/litellm/llms/bedrock_mantle/chat/transformation.py index 590919f1fb08..85f83deda968 100644 --- a/litellm/llms/bedrock_mantle/chat/transformation.py +++ b/litellm/llms/bedrock_mantle/chat/transformation.py @@ -11,9 +11,10 @@ """ from collections.abc import AsyncIterator, Iterator -from typing import Any, Final +from typing import Any, Final, cast import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -28,10 +29,17 @@ from litellm.types.router import GenericLiteLLMParams from ...base_llm.chat.transformation import BaseLLMException -from ...bedrock.common_utils import BedrockError +from ...bedrock.common_utils import ( + BedrockError, + bedrock_chat_completions_serves_tools_with_reasoning, + with_max_completion_tokens, +) from ...openai_like.chat.transformation import OpenAILikeChatConfig from ..common_utils import mantle_base_segment, split_mantle_region_prefix +_SUPPORTED_OPENAI_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) +_BOOLEAN_ADAPTER: Final = TypeAdapter(bool) + class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig): """ @@ -107,8 +115,10 @@ def validate_environment( headers["OpenAI-Project"] = project_id return headers - def get_supported_openai_params(self, model: str) -> list: - base_params: Final = super().get_supported_openai_params(model) + def get_supported_openai_params(self, model: str) -> list[str]: + base_params: Final = _SUPPORTED_OPENAI_PARAMS_ADAPTER.validate_python( + super().get_supported_openai_params(model) # pyright: ignore[reportUnknownMemberType] # base untyped + ) extra_params: Final = tuple( param for param, supported in ( @@ -119,9 +129,39 @@ def get_supported_openai_params(self, model: str) -> list: ) return [*base_params, *extra_params] # mutable-ok: fresh list required by the inherited signature + def map_openai_params( + self, + non_default_params: dict[str, object], # mutable-ok: BaseConfig signature + optional_params: dict[str, object], # mutable-ok: BaseConfig signature + model: str, + drop_params: bool, + replace_max_completion_tokens_with_max_tokens: bool = True, + ) -> dict[str, object]: # mutable-ok: BaseConfig signature + model_cost: Final = cast(dict[str, dict[str, object]], litellm.model_cost) + uses_openai_path: Final = mantle_base_segment(model, model_cost) == "openai/v1" + mapped: Final = TypeAdapter(dict[str, object]).validate_python( + super().map_openai_params( # pyright: ignore[reportUnknownMemberType] # parent mapper is untyped + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + replace_max_completion_tokens_with_max_tokens=( + replace_max_completion_tokens_with_max_tokens and not uses_openai_path + ), + ) + ) + if not uses_openai_path: + return mapped + return dict(with_max_completion_tokens(mapped)) # mutable-ok: BaseConfig signature + def _supports_reasoning(self, model: str) -> bool: try: - return litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider) + return bedrock_chat_completions_serves_tools_with_reasoning( + model, + custom_llm_provider="bedrock_mantle", + ) or _BOOLEAN_ADAPTER.validate_python( + litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider) + ) except Exception as e: verbose_logger.debug("BedrockMantleChatConfig: error checking reasoning support: %s", e) return False diff --git a/litellm/llms/bedrock_mantle/common_utils.py b/litellm/llms/bedrock_mantle/common_utils.py index 9892ef6224e4..f470abe356de 100644 --- a/litellm/llms/bedrock_mantle/common_utils.py +++ b/litellm/llms/bedrock_mantle/common_utils.py @@ -13,8 +13,8 @@ """ import re -from collections.abc import Mapping -from typing import Final +from collections.abc import Mapping, Sequence +from typing import Final, cast from botocore.exceptions import ( CredentialRetrievalError, @@ -127,7 +127,7 @@ def sign_request( ) from e -def mantle_supports_responses(model: str | None, model_cost: dict) -> bool: +def mantle_supports_responses(model: str | None, model_cost: Mapping[str, object]) -> bool: """Whether a Bedrock Mantle model can serve the native Responses API. Purely data-driven from the model's price-map capability signal -- either @@ -139,10 +139,20 @@ def mantle_supports_responses(model: str | None, model_cost: dict) -> bool: gpt-oss substring), so a substring gate would be wrong. A model absent from model_cost simply has no signal and returns False (chat-completions emulation). """ - entry: Final = model_cost.get(f"bedrock_mantle/{split_mantle_region_prefix(model)[1]}", {}) if model else {} - if "/v1/responses" in (entry.get("supported_endpoints") or []): - return True - return entry.get("mode") == "responses" + entry: Final = model_cost.get(f"bedrock_mantle/{split_mantle_region_prefix(model)[1]}") if model else None + if not isinstance(entry, Mapping): + return False + typed_entry: Final = cast( # cast-ok: the entry passed the Mapping check + Mapping[str, object], entry + ) + supported_endpoints: Final = typed_entry.get("supported_endpoints") + return ( + isinstance(supported_endpoints, Sequence) + and "/v1/responses" + in cast( # cast-ok: model-price endpoints are strings + Sequence[object], supported_endpoints + ) + ) or typed_entry.get("mode") == "responses" def mantle_base_segment(model: str | None, model_cost: dict) -> str: diff --git a/litellm/main.py b/litellm/main.py index 468e3a2f66d1..f0a8936fb802 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -103,6 +103,11 @@ from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) +from litellm.litellm_core_utils.responses_api_utils import ( + filter_additional_drop_params, + has_function_tool, + reasoning_effort_is_active, +) from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, @@ -112,7 +117,13 @@ from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) -from litellm.llms.bedrock.common_utils import BedrockModelInfo, bedrock_route_for_request +from litellm.llms.bedrock.common_utils import ( + BedrockModelInfo, + bedrock_chat_completions_serves_tools_with_reasoning, + bedrock_chat_request_needs_native_responses_for_request, + bedrock_reasoning_effort_is_active, + bedrock_route_for_request, +) from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config @@ -1065,7 +1076,9 @@ def responses_api_bridge_check( reasoning_effort: str | Mapping[str, object] | None = None, reasoning_summary: object | None = None, api_base: str | None = None, -) -> tuple[dict, str]: + request_params: Mapping[str, object] | None = None, + additional_drop_params: Sequence[str] | None = None, +) -> tuple[dict[str, object], str]: model_info: dict[str, object] = {} # Global flag: route ALL OpenAI chat completions through Responses API. @@ -1077,7 +1090,7 @@ def responses_api_bridge_check( try: model_info = cast( - dict, + dict[str, object], _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider), ) if model_info.get("mode") is None and model.startswith("responses/"): @@ -1121,18 +1134,8 @@ def responses_api_bridge_check( # azure_ai gate keys on those measured boundaries instead of gpt-5.4+. # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). - has_function_tool: Final = any( - ( - tool.get("type") == "function" and (isinstance(tool.get("function"), dict) or "name" in tool) - if isinstance(tool, dict) - else getattr(tool, "type", None) == "function" - ) - for tool in (tools or ()) - ) - if isinstance(reasoning_effort, dict): - reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None - else: - reasoning_active = reasoning_effort != "none" + has_function_tool_in_request: Final = has_function_tool(tools) + reasoning_active: Final = reasoning_effort_is_active(reasoning_effort) # The reasoning+tools constraint is enforced by the real OpenAI backend behind any api.openai.com # host (the default URL or a PrivateLink hostname such as .privatelink.api.openai.com) and # by Azure OpenAI through the azure provider. Resolve the effective OpenAI base arg>global>env>default @@ -1147,7 +1150,7 @@ def responses_api_bridge_check( custom_llm_provider == "azure" or resolved_api_base == "" or _is_openai_backed_api_base(resolved_api_base) ) chat_rejects_function_tools: Final = ( - has_function_tool + has_function_tool_in_request and reasoning_active and ( foundry_chat_rejects_function_tools_while_reasoning(model, reasoning_effort) @@ -1158,6 +1161,46 @@ def responses_api_bridge_check( ) ) ) + if ( + custom_llm_provider == "bedrock" + and request_params is not None + and model_info.get("mode") != "responses" + and bedrock_chat_request_needs_native_responses_for_request( + model, + request_params, + additional_drop_params, + ) + ): + model_info["mode"] = "responses" + if custom_llm_provider == "bedrock_mantle" and model_info.get("mode") != "responses": + from litellm.llms.bedrock_mantle.common_utils import mantle_supports_responses + + mantle_request_params: Final = ( + filter_additional_drop_params(request_params, additional_drop_params) + if request_params is not None + else MappingProxyType({"tools": tools, "reasoning_effort": reasoning_effort}) + ) + mantle_reasoning_effort: Final = mantle_request_params.get("reasoning_effort") + mantle_reasoning_summary: Final = ( + peek_reasoning_summary_aliases(mantle_request_params) if request_params is not None else None + ) + mantle_needs_native_responses: Final = ( + has_function_tool(mantle_request_params.get("tools")) + and not bedrock_chat_completions_serves_tools_with_reasoning(model, "bedrock_mantle") + and bedrock_reasoning_effort_is_active( + model, + mantle_reasoning_effort, + "bedrock_mantle", + ) + ) or ( + mantle_reasoning_effort is not None + and (mantle_reasoning_summary is not None or reasoning_summary is not None) + ) + mantle_model_cost: Final = cast( # cast-ok: model price rows use string keys + Mapping[str, object], litellm.model_cost + ) + if mantle_needs_native_responses and mantle_supports_responses(model, mantle_model_cost): + model_info["mode"] = "responses" if ( (custom_llm_provider in ("openai", "azure") or on_foundry_openai_endpoint) and model_info.get("mode") != "responses" @@ -5558,7 +5601,7 @@ def completion( if bridges_to_responses_api else kwargs.get("allowed_openai_params") ) - optional_param_args: Final = { + optional_param_args: Final[dict[str, object]] = { "functions": functions, "function_call": function_call, "temperature": temperature, @@ -5736,7 +5779,17 @@ def completion( # check handles cases like gpt-5.4+ with tools+reasoning_effort or # reasoningSummary/reasoning_summary without tools (AI SDK) that the first # (early) check doesn't cover. - _reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params) + _completion_request_params: Final[Mapping[str, object]] = MappingProxyType( + {**optional_param_args, **non_default_params} + ) + completion_kwargs: Final = cast( # cast-ok: completion's **kwargs are untyped + Mapping[str, object], kwargs + ) + _reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases( + cast( # cast-ok: get_optional_params returns request fields + Mapping[str, object], optional_params + ) + ) if responses_api_model_info.get("mode") != "responses": responses_api_model_info, model = responses_api_bridge_check( model=model, @@ -5746,6 +5799,11 @@ def completion( reasoning_effort=reasoning_effort, reasoning_summary=_reasoning_summary_for_bridge, api_base=api_base, + request_params=_completion_request_params, + additional_drop_params=cast( # cast-ok: completion kwargs may carry drop parameters + Sequence[str] | None, + completion_kwargs.get("additional_drop_params"), + ), ) # Use base_model (the true underlying model) for Azure model-type @@ -5819,7 +5877,7 @@ def completion( optional_params=optional_params, organization=organization, provider_config=provider_config, - request_params=MappingProxyType({**optional_param_args, **non_default_params}), + request_params=_completion_request_params, shared_session=shared_session, stream=stream, temperature=temperature, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 12bdc6b38caf..b5cccb0d2d30 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12678,6 +12678,10 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -12695,6 +12699,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -12711,6 +12719,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -12728,6 +12740,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12739,6 +12755,10 @@ "supports_reasoning": true }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12755,6 +12775,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12771,6 +12795,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12782,6 +12810,10 @@ "supports_reasoning": true }, "bedrock/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12818,6 +12850,10 @@ "output_cost_per_token": 7.2e-07 }, "bedrock/ap-south-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -12835,6 +12871,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -12851,6 +12891,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -12868,6 +12912,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12879,6 +12927,10 @@ "supports_reasoning": true }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12895,6 +12947,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12911,6 +12967,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.09e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -12928,6 +12988,10 @@ "output_cost_per_token": 1.236e-06 }, "bedrock/ap-southeast-3/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -12945,6 +13009,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -12961,6 +13029,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -12978,6 +13050,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12994,6 +13070,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13028,6 +13108,10 @@ "output_cost_per_token": 6.9e-07 }, "bedrock/eu-north-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -13045,6 +13129,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13061,6 +13149,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13078,6 +13170,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13181,6 +13277,10 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13197,6 +13297,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13214,6 +13318,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13248,6 +13356,10 @@ "output_cost_per_token": 6.5e-07 }, "bedrock/eu-west-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13264,6 +13376,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13281,6 +13397,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13315,6 +13435,10 @@ "output_cost_per_token": 7.8e-07 }, "bedrock/eu-west-2/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 4.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13331,6 +13455,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 4.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13365,6 +13493,10 @@ "supports_vision": false }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13412,6 +13544,10 @@ "supports_tool_choice": true }, "bedrock/eu-south-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13428,6 +13564,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-south-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13445,6 +13585,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13497,6 +13641,10 @@ "output_cost_per_token": 1.01e-06 }, "bedrock/sa-east-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -13514,6 +13662,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13530,6 +13682,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13547,6 +13703,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13558,6 +13718,10 @@ "supports_reasoning": true }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13574,6 +13738,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13727,6 +13895,10 @@ "supports_tool_choice": true }, "bedrock/us-east-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -13744,6 +13916,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13760,6 +13936,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", @@ -13776,6 +13956,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13787,6 +13971,10 @@ "supports_reasoning": true }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13803,6 +13991,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13819,6 +14011,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -13836,6 +14032,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13852,6 +14052,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13869,6 +14073,10 @@ "output_cost_per_token": 1.2e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13880,6 +14088,10 @@ "supports_reasoning": true }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13896,6 +14108,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -14357,6 +14573,10 @@ "supports_tool_choice": true }, "bedrock/us-west-2/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -14374,6 +14594,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -14390,6 +14614,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", @@ -14406,6 +14634,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -14417,6 +14649,10 @@ "supports_reasoning": true }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -14433,6 +14669,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -21984,6 +22224,10 @@ "supports_tool_choice": true }, "deepseek.v3-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, @@ -21998,6 +22242,10 @@ "supports_native_structured_output": true }, "deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 164000, @@ -31237,6 +31485,10 @@ "output_cost_per_token": 2e-06 }, "google.gemma-3-12b-it": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -31252,6 +31504,10 @@ "supports_vision": true }, "google.gemma-3-27b-it": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 2.3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -31267,6 +31523,10 @@ "supports_vision": true }, "google.gemma-3-4b-it": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -37500,6 +37760,10 @@ "supports_tool_choice": true }, "minimax.minimax-m2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, @@ -37516,6 +37780,10 @@ "supports_vision": false }, "minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, @@ -37532,6 +37800,10 @@ "supports_vision": false }, "minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock_converse", @@ -37786,6 +38058,10 @@ "supports_tool_choice": true }, "mistral.mistral-large-3-675b-instruct": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -38613,6 +38889,10 @@ "supports_vision": true }, "moonshot.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -38630,6 +38910,10 @@ "supports_vision": false }, "moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -41101,7 +41385,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -41119,7 +41403,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -41134,6 +41418,10 @@ "supports_tool_choice": true }, "openai.gpt-oss-safeguard-120b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -41149,6 +41437,10 @@ "supports_vision": false }, "openai.gpt-oss-safeguard-20b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -44296,6 +44588,10 @@ "supports_reasoning": true }, "qwen.qwen3-coder-480b-a35b-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262000, @@ -44310,6 +44606,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-west-2/index.json" }, "qwen.qwen3-235b-a22b-2507-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 2.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, @@ -44325,6 +44625,10 @@ "output_cost_per_token_batches": 4.4e-07 }, "qwen.qwen3-coder-30b-a3b-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, @@ -44338,6 +44642,10 @@ "supports_native_structured_output": true }, "qwen.qwen3-32b-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 131072, @@ -44351,6 +44659,10 @@ "supports_native_structured_output": true }, "qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -44367,6 +44679,10 @@ "supports_vision": false }, "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44383,6 +44699,10 @@ "supports_vision": false }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44399,6 +44719,10 @@ "supports_vision": false }, "bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.545e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44415,6 +44739,10 @@ "supports_vision": false }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44431,6 +44759,10 @@ "supports_vision": false }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 2.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44447,6 +44779,10 @@ "supports_vision": false }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44463,6 +44799,10 @@ "supports_vision": false }, "qwen.qwen3-vl-235b-a22b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5.3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -44479,6 +44819,10 @@ "supports_response_schema": false }, "qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -46862,7 +47206,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -46879,7 +47223,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -46890,9 +47234,10 @@ "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, "supported_endpoints": [ - "/v1/chat/completions" + "/v1/chat/completions", + "/v1/responses" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, @@ -47105,6 +47450,10 @@ "supports_tool_choice": false }, "us.deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, @@ -47121,6 +47470,10 @@ "supports_vision": false }, "eu.deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, @@ -51848,6 +52201,10 @@ "supports_web_search": true }, "zai.glm-4.7": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, @@ -51865,6 +52222,10 @@ "supports_vision": false }, "zai.glm-5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", @@ -51883,6 +52244,10 @@ "supports_vision": false }, "zai.glm-4.7-flash": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, @@ -57281,7 +57646,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57324,7 +57689,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57432,7 +57797,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57654,6 +58019,7 @@ "text", "image" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_output_modalities": [ "text" ], @@ -57665,6 +58031,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57685,6 +58052,7 @@ "text", "image" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_output_modalities": [ "text" ], @@ -57696,6 +58064,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57716,6 +58085,7 @@ "text", "image" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_output_modalities": [ "text" ], @@ -57727,6 +58097,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57747,6 +58118,7 @@ "text", "image" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_output_modalities": [ "text" ], @@ -57758,6 +58130,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57810,7 +58183,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57848,7 +58221,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57886,7 +58259,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57943,6 +58316,7 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57978,6 +58352,7 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -58013,6 +58388,7 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -58048,10 +58424,15 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "openai.gpt-6-sol": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "cache_creation_input_token_cost": 2.5e-06, @@ -58115,10 +58496,15 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "openai.gpt-6-luna": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "input_cost_per_token": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, "cache_creation_input_token_cost": 1.25e-07, @@ -58182,6 +58568,7 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -58201,12 +58588,13 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_modalities": [ "text", "image" @@ -58241,12 +58629,13 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_modalities": [ "text", "image" @@ -58278,6 +58667,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -58297,6 +58687,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -58316,6 +58707,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -58336,6 +58728,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -58357,6 +58750,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -58398,9 +58792,10 @@ "output_cost_per_token": 6.6e-06, "cache_read_input_token_cost": 5.5e-07, "supported_endpoints": [ - "/v1/chat/completions" + "/v1/chat/completions", + "/v1/responses" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, @@ -58419,9 +58814,10 @@ "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, "supported_endpoints": [ - "/v1/chat/completions" + "/v1/chat/completions", + "/v1/responses" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, @@ -58620,6 +59016,10 @@ ] }, "bedrock/us-east-1/zai.glm-5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", @@ -58638,6 +59038,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/zai.glm-5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", @@ -64379,7 +64783,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64396,7 +64800,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64641,7 +65045,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64658,7 +65062,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64835,7 +65239,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -64870,7 +65274,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -64910,6 +65314,7 @@ "supported_endpoints": [ "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_modalities": [ "text", "image" @@ -64940,6 +65345,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64964,6 +65370,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -64984,6 +65391,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -65003,6 +65411,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -65022,6 +65431,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -65074,6 +65484,7 @@ "supported_endpoints": [ "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_modalities": [ "text", "image" @@ -65107,6 +65518,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -75852,6 +76264,11 @@ "supports_web_search": true }, "moonshotai.kimi-k3": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -75872,6 +76289,11 @@ "supports_vision": true }, "global.moonshotai.kimi-k3": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -75892,6 +76314,11 @@ "supports_vision": true }, "us.moonshotai.kimi-k3": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "cache_creation_input_token_cost": 4.125e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, diff --git a/litellm/utils.py b/litellm/utils.py index de95e153dde0..2f61943e0865 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -88,6 +88,9 @@ match_capability_generalizations, match_fill_missing_generalizations, ) +from litellm.litellm_core_utils.responses_api_utils import ( + peek_reasoning_summary_aliases as peek_reasoning_summary_aliases, # noqa: PLC0414 # public utils compatibility +) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, strip_special_tokens from litellm.rust_bridge import tokenizer as tokenizer_dispatch @@ -10151,7 +10154,7 @@ def get_standard_openai_params(params: Mapping[str, object]) -> dict: return {k: v for k, v in params.items() if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None} -def get_non_default_completion_params(kwargs: Mapping[str, object]) -> dict: +def get_non_default_completion_params(kwargs: Mapping[str, object]) -> dict[str, object]: openai_params: Final = litellm.OPENAI_CHAT_COMPLETION_PARAMS default_params: Final = openai_params + all_litellm_params non_default_params: Final = { @@ -10161,24 +10164,6 @@ def get_non_default_completion_params(kwargs: Mapping[str, object]) -> dict: return non_default_params -def peek_reasoning_summary_aliases(optional_params: dict) -> object | None: - """Read AI-SDK-style reasoning summary from optional_params or nested extra_body. - - Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped. - """ - if "reasoningSummary" in optional_params: - return optional_params["reasoningSummary"] - if "reasoning_summary" in optional_params: - return optional_params["reasoning_summary"] - extra_body: Final = optional_params.get("extra_body") - if isinstance(extra_body, dict): - if "reasoningSummary" in extra_body: - return extra_body["reasoningSummary"] - if "reasoning_summary" in extra_body: - return extra_body["reasoning_summary"] - return None - - def strip_reasoning_summary_aliases_from_optional_params( optional_params: dict, ) -> tuple[dict, object | None]: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 12bdc6b38caf..b5cccb0d2d30 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12678,6 +12678,10 @@ "supports_tool_choice": true }, "bedrock/ap-northeast-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -12695,6 +12699,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -12711,6 +12719,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -12728,6 +12740,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12739,6 +12755,10 @@ "supports_reasoning": true }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12755,6 +12775,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12771,6 +12795,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12782,6 +12810,10 @@ "supports_reasoning": true }, "bedrock/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12818,6 +12850,10 @@ "output_cost_per_token": 7.2e-07 }, "bedrock/ap-south-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -12835,6 +12871,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -12851,6 +12891,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -12868,6 +12912,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12879,6 +12927,10 @@ "supports_reasoning": true }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12895,6 +12947,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12911,6 +12967,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.09e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -12928,6 +12988,10 @@ "output_cost_per_token": 1.236e-06 }, "bedrock/ap-southeast-3/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -12945,6 +13009,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -12961,6 +13029,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -12978,6 +13050,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -12994,6 +13070,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13028,6 +13108,10 @@ "output_cost_per_token": 6.9e-07 }, "bedrock/eu-north-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -13045,6 +13129,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13061,6 +13149,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13078,6 +13170,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13181,6 +13277,10 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13197,6 +13297,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13214,6 +13318,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13248,6 +13356,10 @@ "output_cost_per_token": 6.5e-07 }, "bedrock/eu-west-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13264,6 +13376,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13281,6 +13397,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13315,6 +13435,10 @@ "output_cost_per_token": 7.8e-07 }, "bedrock/eu-west-2/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 4.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13331,6 +13455,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 4.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13365,6 +13493,10 @@ "supports_vision": false }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13412,6 +13544,10 @@ "supports_tool_choice": true }, "bedrock/eu-south-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13428,6 +13564,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-south-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13445,6 +13585,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13497,6 +13641,10 @@ "output_cost_per_token": 1.01e-06 }, "bedrock/sa-east-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -13514,6 +13662,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13530,6 +13682,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13547,6 +13703,10 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13558,6 +13718,10 @@ "supports_reasoning": true }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13574,6 +13738,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13727,6 +13895,10 @@ "supports_tool_choice": true }, "bedrock/us-east-1/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -13744,6 +13916,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13760,6 +13936,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", @@ -13776,6 +13956,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13787,6 +13971,10 @@ "supports_reasoning": true }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13803,6 +13991,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13819,6 +14011,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -13836,6 +14032,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13852,6 +14052,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 1000000, @@ -13869,6 +14073,10 @@ "output_cost_per_token": 1.2e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13880,6 +14088,10 @@ "supports_reasoning": true }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -13896,6 +14108,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -14357,6 +14573,10 @@ "supports_tool_choice": true }, "bedrock/us-west-2/deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, @@ -14374,6 +14594,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -14390,6 +14614,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", @@ -14406,6 +14634,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -14417,6 +14649,10 @@ "supports_reasoning": true }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -14433,6 +14669,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, @@ -21984,6 +22224,10 @@ "supports_tool_choice": true }, "deepseek.v3-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, @@ -21998,6 +22242,10 @@ "supports_native_structured_output": true }, "deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 164000, @@ -31237,6 +31485,10 @@ "output_cost_per_token": 2e-06 }, "google.gemma-3-12b-it": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -31252,6 +31504,10 @@ "supports_vision": true }, "google.gemma-3-27b-it": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 2.3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -31267,6 +31523,10 @@ "supports_vision": true }, "google.gemma-3-4b-it": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -37500,6 +37760,10 @@ "supports_tool_choice": true }, "minimax.minimax-m2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, @@ -37516,6 +37780,10 @@ "supports_vision": false }, "minimax.minimax-m2.1": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, @@ -37532,6 +37800,10 @@ "supports_vision": false }, "minimax.minimax-m2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock_converse", @@ -37786,6 +38058,10 @@ "supports_tool_choice": true }, "mistral.mistral-large-3-675b-instruct": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -38613,6 +38889,10 @@ "supports_vision": true }, "moonshot.kimi-k2-thinking": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -38630,6 +38910,10 @@ "supports_vision": false }, "moonshotai.kimi-k2.5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -41101,7 +41385,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -41119,7 +41403,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -41134,6 +41418,10 @@ "supports_tool_choice": true }, "openai.gpt-oss-safeguard-120b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -41149,6 +41437,10 @@ "supports_vision": false }, "openai.gpt-oss-safeguard-20b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -44296,6 +44588,10 @@ "supports_reasoning": true }, "qwen.qwen3-coder-480b-a35b-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262000, @@ -44310,6 +44606,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-west-2/index.json" }, "qwen.qwen3-235b-a22b-2507-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 2.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, @@ -44325,6 +44625,10 @@ "output_cost_per_token_batches": 4.4e-07 }, "qwen.qwen3-coder-30b-a3b-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, @@ -44338,6 +44642,10 @@ "supports_native_structured_output": true }, "qwen.qwen3-32b-v1:0": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 131072, @@ -44351,6 +44659,10 @@ "supports_native_structured_output": true }, "qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -44367,6 +44679,10 @@ "supports_vision": false }, "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44383,6 +44699,10 @@ "supports_vision": false }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44399,6 +44719,10 @@ "supports_vision": false }, "bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.545e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44415,6 +44739,10 @@ "supports_vision": false }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44431,6 +44759,10 @@ "supports_vision": false }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 2.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44447,6 +44779,10 @@ "supports_vision": false }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -44463,6 +44799,10 @@ "supports_vision": false }, "qwen.qwen3-vl-235b-a22b": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5.3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -44479,6 +44819,10 @@ "supports_response_schema": false }, "qwen.qwen3-coder-next": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -46862,7 +47206,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -46879,7 +47223,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -46890,9 +47234,10 @@ "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, "supported_endpoints": [ - "/v1/chat/completions" + "/v1/chat/completions", + "/v1/responses" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, @@ -47105,6 +47450,10 @@ "supports_tool_choice": false }, "us.deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, @@ -47121,6 +47470,10 @@ "supports_vision": false }, "eu.deepseek.v3.2": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, @@ -51848,6 +52201,10 @@ "supports_web_search": true }, "zai.glm-4.7": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, @@ -51865,6 +52222,10 @@ "supports_vision": false }, "zai.glm-5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", @@ -51883,6 +52244,10 @@ "supports_vision": false }, "zai.glm-4.7-flash": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, @@ -57281,7 +57646,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57324,7 +57689,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57432,7 +57797,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57654,6 +58019,7 @@ "text", "image" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_output_modalities": [ "text" ], @@ -57665,6 +58031,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57685,6 +58052,7 @@ "text", "image" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_output_modalities": [ "text" ], @@ -57696,6 +58064,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57716,6 +58085,7 @@ "text", "image" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_output_modalities": [ "text" ], @@ -57727,6 +58097,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57747,6 +58118,7 @@ "text", "image" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_output_modalities": [ "text" ], @@ -57758,6 +58130,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57810,7 +58183,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57848,7 +58221,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57886,7 +58259,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -57943,6 +58316,7 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57978,6 +58352,7 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -58013,6 +58388,7 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -58048,10 +58424,15 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "openai.gpt-6-sol": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "cache_creation_input_token_cost": 2.5e-06, @@ -58115,10 +58496,15 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "openai.gpt-6-luna": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "input_cost_per_token": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, "cache_creation_input_token_cost": 1.25e-07, @@ -58182,6 +58568,7 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/", "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -58201,12 +58588,13 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_modalities": [ "text", "image" @@ -58241,12 +58629,13 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_modalities": [ "text", "image" @@ -58278,6 +58667,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -58297,6 +58687,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -58316,6 +58707,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -58336,6 +58728,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -58357,6 +58750,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -58398,9 +58792,10 @@ "output_cost_per_token": 6.6e-06, "cache_read_input_token_cost": 5.5e-07, "supported_endpoints": [ - "/v1/chat/completions" + "/v1/chat/completions", + "/v1/responses" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, @@ -58419,9 +58814,10 @@ "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, "supported_endpoints": [ - "/v1/chat/completions" + "/v1/chat/completions", + "/v1/responses" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, @@ -58620,6 +59016,10 @@ ] }, "bedrock/us-east-1/zai.glm-5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", @@ -58638,6 +59038,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/zai.glm-5": { + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", @@ -64379,7 +64783,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64396,7 +64800,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64641,7 +65045,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64658,7 +65062,7 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64835,7 +65239,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -64870,7 +65274,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -64910,6 +65314,7 @@ "supported_endpoints": [ "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_modalities": [ "text", "image" @@ -64940,6 +65345,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -64964,6 +65370,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -64984,6 +65391,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -65003,6 +65411,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -65022,6 +65431,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -65074,6 +65484,7 @@ "supported_endpoints": [ "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supported_modalities": [ "text", "image" @@ -65107,6 +65518,7 @@ "/v1/chat/completions", "/v1/responses" ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -75852,6 +76264,11 @@ "supports_web_search": true }, "moonshotai.kimi-k3": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -75872,6 +76289,11 @@ "supports_vision": true }, "global.moonshotai.kimi-k3": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -75892,6 +76314,11 @@ "supports_vision": true }, "us.moonshotai.kimi-k3": { + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_bedrock_chat_completions_tools_with_reasoning": true, "cache_creation_input_token_cost": 4.125e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index d961dd03c66b..148a4abf4fab 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -941,7 +941,7 @@ "supports_bedrock_runtime_chat_completions_response_format": { "type": "boolean" }, - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": { + "supports_bedrock_chat_completions_tools_with_reasoning": { "type": "boolean" }, "supports_computer_use": { diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index f9f330a73406..4d8de157ad0f 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1,10 +1,12 @@ """Native Bedrock Runtime Chat Completions: Grok, gpt-oss and GPT-5.6 stay on /openai/v1/chat/completions.""" import json +from collections.abc import Callable, Generator, Mapping, Sequence +from typing import Final, Protocol, cast import httpx import pytest -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm.llms.bedrock.chat.chat_completions.transformation import ( @@ -18,21 +20,35 @@ from litellm.llms.bedrock.common_utils import ( BEDROCK_CONVERSE_ONLY_REQUEST_KEYS, BedrockModelInfo, + bedrock_chat_request_needs_native_responses, + bedrock_chat_request_needs_native_responses_for_request, bedrock_request_needs_converse, bedrock_route_for_request, get_bedrock_chat_config, uses_bedrock_runtime_chat_completions, ) from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.types.utils import ModelResponse + + +class _CacheClearable(Protocol): + def cache_clear(self) -> None: ... + + +def _clear_model_info_cache() -> None: + model_info_cache: Final = cast(_CacheClearable, litellm.get_model_info) + model_info_cache.cache_clear() @pytest.fixture -def local_cost_map(monkeypatch): +def local_cost_map(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - litellm.get_model_info.cache_clear() + get_model_cost_map: Final = cast(Callable[..., object], litellm.get_model_cost_map) + cost_map: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python(get_model_cost_map(url="")) + monkeypatch.setattr(litellm, "model_cost", cost_map) + _clear_model_info_cache() yield - litellm.get_model_info.cache_clear() + _clear_model_info_cache() @pytest.mark.parametrize( @@ -139,8 +155,16 @@ def test_transform_request_is_openai_chat_body_not_converse(): assert "messages" in body -def _chat_completion_json(content, model, tool_calls=None): - message = {"role": "assistant", "content": content, **({"tool_calls": tool_calls} if tool_calls else {})} +def _chat_completion_json( + content: str | None, + model: str, + tool_calls: Sequence[Mapping[str, object]] | None = None, +) -> dict[str, object]: + message: Final[dict[str, object]] = { + "role": "assistant", + "content": content, + **({"tool_calls": tool_calls} if tool_calls else {}), + } return { "id": "chatcmpl-test", "object": "chat.completion", @@ -151,15 +175,33 @@ def _chat_completion_json(content, model, tool_calls=None): } -CONVERSE_JSON = { +CONVERSE_JSON: Final[dict[str, object]] = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, } +RESPONSES_JSON: Final[dict[str, object]] = { + "id": "resp_test", + "object": "response", + "created_at": 1733529600, + "status": "completed", + "model": "openai.gpt-5.6-sol", + "output": [ + { + "type": "message", + "id": "msg_test", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, +} + @pytest.fixture -def fake_aws_env(monkeypatch): +def fake_aws_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") monkeypatch.delenv("AWS_BEDROCK_RUNTIME_ENDPOINT", raising=False) monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) @@ -168,12 +210,19 @@ def fake_aws_env(monkeypatch): monkeypatch.setenv("AWS_SESSION_TOKEN", "testing") -def _recording_client(**response_kwargs): +def _recording_client( + *, + json: Mapping[str, object] | None = None, + content: str | bytes | None = None, + headers: Mapping[str, str] | None = None, +) -> tuple[list[httpx.Request], HTTPHandler]: requests: list[httpx.Request] = [] - def handle(request): + def handle(request: httpx.Request) -> httpx.Response: requests.append(request) - return httpx.Response(200, **response_kwargs) + if json is not None: + return httpx.Response(200, json=json) + return httpx.Response(200, content=content or b"", headers=headers) return requests, HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))) @@ -272,7 +321,8 @@ def test_guardrail_config_falls_back_to_converse(local_cost_map, model): @pytest.mark.parametrize( - "model", ["openai.gpt-oss-20b-1:0", "us.xai.grok-4.6", "global.openai.gpt-5.6-sol"] + "model", + ["openai.gpt-oss-20b-1:0", "us.xai.grok-4.6", "global.openai.gpt-5.6-sol"], ) @pytest.mark.parametrize( "request_params", @@ -286,21 +336,152 @@ def test_converse_extension_params_fall_back_to_converse(local_cost_map, model, @pytest.mark.parametrize( - "request_params, expected_route", + "request_params", [ - ({"tools": [GET_WEATHER_TOOL]}, "converse"), - ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "converse"), - ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "converse"), - ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, "chat_completions"), - ({"reasoning_effort": "low"}, "chat_completions"), - ({"tools": None, "reasoning_effort": "low"}, "chat_completions"), - ({"tools": [], "reasoning_effort": "low"}, "chat_completions"), - ({}, "chat_completions"), + {"tools": [GET_WEATHER_TOOL]}, + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, + {"reasoning_effort": "low"}, + {"tools": None, "reasoning_effort": "low"}, + {"tools": [], "reasoning_effort": "low"}, + {}, ], ) -def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route): - assert BedrockModelInfo.get_bedrock_route("global.openai.gpt-5.6-sol", request_params) == expected_route - assert BedrockModelInfo.get_bedrock_route("bedrock/us.openai.gpt-5.6-terra", request_params) == expected_route +def test_gpt56_tools_with_responses_support_stay_on_chat_route( + local_cost_map: None, + request_params: dict[str, object], +) -> None: + assert BedrockModelInfo.get_bedrock_route("global.openai.gpt-5.6-sol", request_params) == "chat_completions" + assert BedrockModelInfo.get_bedrock_route("us.openai.gpt-5.6-terra", request_params) == "chat_completions" + + +@pytest.fixture +def chat_model_without_responses( + local_cost_map: None, + monkeypatch: pytest.MonkeyPatch, +) -> Generator[str, None, None]: + validated_model_cost: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python( + cast(object, litellm.model_cost) + ) + isolated_model_cost: Final = dict(validated_model_cost) + monkeypatch.setattr(litellm, "model_cost", isolated_model_cost) + model: Final = "bedrock-test-no-responses-tools-reasoning" + litellm.register_model( + { + model: { + "litellm_provider": "bedrock_converse", + "supported_endpoints": ["/v1/chat/completions"], + } + }, + persist_across_reloads=False, + ) + _clear_model_info_cache() + try: + yield model + finally: + _clear_model_info_cache() + + +@pytest.mark.parametrize( + "request_params", + [{}, {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "medium"}], + ids=("plain", "tools-with-reasoning"), +) +def test_qwen_native_chat_support_does_not_bridge( + local_cost_map: None, + request_params: dict[str, object], +) -> None: + assert BedrockModelInfo.get_bedrock_route("qwen.qwen3-32b-v1:0", request_params) == "chat_completions" + assert bedrock_chat_request_needs_native_responses("qwen.qwen3-32b-v1:0", request_params) is False + + +def test_gpt56_tools_with_reasoning_bridge_but_keep_native_chat_route(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "medium"} + + assert BedrockModelInfo.get_bedrock_route(model, request_params) == "chat_completions" + assert bedrock_chat_request_needs_native_responses(model, request_params) is True + + +def test_converse_only_guardrail_prevents_gpt56_responses_bridge(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = { + "tools": [GET_WEATHER_TOOL], + "reasoning_effort": "medium", + "guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}, + } + + assert BedrockModelInfo.get_bedrock_route(model, request_params) == "converse" + assert bedrock_chat_request_needs_native_responses(model, request_params) is False + + +def test_response_format_needing_converse_prevents_responses_bridge(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = { + "tools": [GET_WEATHER_TOOL], + "reasoning_effort": "medium", + "response_format": {"type": "json_object"}, + } + + assert BedrockModelInfo.get_bedrock_route(model, request_params) == "converse" + assert bedrock_chat_request_needs_native_responses(model, request_params) is False + + +def test_gpt56_tools_with_reasoning_none_stay_on_native_chat(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"} + + assert BedrockModelInfo.get_bedrock_route(model, request_params) == "chat_completions" + assert bedrock_chat_request_needs_native_responses(model, request_params) is False + + +def test_gpt6_astra_tools_with_reasoning_none_bridge(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-6-astra" + request_params: Final = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"} + + assert BedrockModelInfo.get_bedrock_route(model, request_params) == "chat_completions" + assert bedrock_chat_request_needs_native_responses(model, request_params) is True + + +def test_gpt56_legacy_functions_with_reasoning_stay_on_converse(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = {"functions": [GET_WEATHER_TOOL["function"]], "reasoning_effort": "medium"} + + assert BedrockModelInfo.get_bedrock_route(model, request_params) == "converse" + assert bedrock_chat_request_needs_native_responses(model, request_params) is False + + +def test_function_tools_without_native_responses_use_converse(chat_model_without_responses: str) -> None: + request_params: Final = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "medium"} + + assert BedrockModelInfo.get_bedrock_route(chat_model_without_responses, request_params) == "converse" + assert bedrock_chat_request_needs_native_responses(chat_model_without_responses, request_params) is False + + +def test_dropping_function_tools_prevents_responses_bridge(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "medium"} + + assert bedrock_route_for_request(model, request_params, ["tools"]) == "chat_completions" + assert bedrock_chat_request_needs_native_responses_for_request(model, request_params, ["tools"]) is False + + +def test_reasoning_effort_mapping_with_summary_is_active_for_bridge(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = { + "tools": [GET_WEATHER_TOOL], + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + } + + assert bedrock_chat_request_needs_native_responses(model, request_params) is True + + +def test_reasoning_summary_alias_bridges_to_responses(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = {"reasoning_effort": "medium", "reasoningSummary": "detailed"} + + assert bedrock_chat_request_needs_native_responses(model, request_params) is True @pytest.mark.parametrize("reasoning_effort", ["low", "high", None]) @@ -670,19 +851,65 @@ def test_gpt_oss_completion_hits_chat_completions_and_splits_reasoning(local_cos assert response.choices[0].message.content == "Hi" -def test_gpt56_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env): - requests, client = _recording_client(json=CONVERSE_JSON) - response = litellm.completion( - model="bedrock/global.openai.gpt-5.6-sol", - messages=[{"role": "user", "content": "hello"}], - tools=[GET_WEATHER_TOOL], - reasoning_effort="low", - client=client, +def test_runtime_gpt56_bridge_and_qwen_chat_dispatch_use_native_endpoints( + local_cost_map: None, + fake_aws_env: None, +) -> None: + responses_requests, responses_client = _recording_client(json=RESPONSES_JSON) + responses_response: Final[ModelResponse] = cast( + ModelResponse, + litellm.completion( + model="bedrock/global.openai.gpt-5.6-sol", + messages=[{"role": "user", "content": "hello"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="medium", + client=responses_client, + ), ) - assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-5.6-sol/converse") - assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" - assert response.choices[0].message.content == "ok" + assert len(responses_requests) == 1 + assert str(responses_requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses" + assert responses_response.choices[0].message.content == "ok" + + chat_requests, chat_client = _recording_client(json=_chat_completion_json("ok", "qwen.qwen3-32b-v1:0")) + chat_response: Final[ModelResponse] = cast( + ModelResponse, + litellm.completion( + model="bedrock/qwen.qwen3-32b-v1:0", + messages=[{"role": "user", "content": "hello"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="medium", + client=chat_client, + ), + ) + + assert len(chat_requests) == 1 + assert str(chat_requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + assert chat_response.choices[0].message.content == "ok" + + +def test_minimax_tools_with_reasoning_use_native_chat( + local_cost_map: None, + fake_aws_env: None, +) -> None: + tool_calls: Final = [{"id": "call_0", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}] + requests, client = _recording_client(json=_chat_completion_json(None, "minimax.minimax-m2.5", tool_calls)) + response: Final[ModelResponse] = cast( + ModelResponse, + litellm.completion( + model="bedrock/minimax.minimax-m2.5", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="medium", + client=client, + ), + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + body: Final = TypeAdapter(dict[str, object]).validate_json(requests[0].content) + assert body["reasoning_effort"] == "medium" + assert body["tools"] == [GET_WEATHER_TOOL] + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map, fake_aws_env): @@ -990,7 +1217,7 @@ def test_json_object_keeps_converse_where_aws_would_demand_the_word_json(local_c ({}, {"tools": [GET_WEATHER_TOOL]}, True), ({}, {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, False), ( - {"supports_bedrock_runtime_chat_completions_tools_with_reasoning": True}, + {"supports_bedrock_chat_completions_tools_with_reasoning": True}, {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, False, ), diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index ed8b70239773..55c2939feba3 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -893,7 +893,7 @@ def test_converse_stream_with_an_empty_200_body_raises_instead_of_an_empty_turn( with pytest.raises(MidStreamFallbackError) as exc_info: list( litellm.completion( - model="bedrock/us.moonshotai.kimi-k3", + model="bedrock/converse/us.moonshotai.kimi-k3", messages=[{"role": "user", "content": "hi"}], stream=True, client=client, @@ -916,7 +916,7 @@ async def _aiter_bytes(chunk_size: int | None = None) -> AsyncIterator[bytes]: client.post = AsyncMock(return_value=response) stream: Final = await litellm.acompletion( - model="bedrock/us.moonshotai.kimi-k3", + model="bedrock/converse/us.moonshotai.kimi-k3", messages=[{"role": "user", "content": "hi"}], stream=True, client=client, diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 4bf3dd11fa1d..53e937f00bac 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -5,19 +5,19 @@ API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html """ -import json import asyncio +import json +from typing import Final from unittest.mock import Mock, patch - import httpx import pytest from botocore.auth import SigV4Auth from botocore.awsrequest import AWSRequest import litellm -from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws +from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig from litellm.types.utils import LlmProviders from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe @@ -255,6 +255,38 @@ def test_verbosity_passes_through_for_gpt_5_models(self): ) assert optional_params["verbosity"] == "low" + def test_max_token_mapping_follows_openai_path_metadata(self, local_cost_map: None) -> None: + cfg: Final = BedrockMantleChatConfig() + openai_path_params: Final = cfg.map_openai_params( + non_default_params={"max_tokens": 64}, + optional_params={}, + model="openai.gpt-5.6-sol", + drop_params=False, + ) + v1_path_params: Final = cfg.map_openai_params( + non_default_params={"max_completion_tokens": 64}, + optional_params={}, + model="openai.gpt-oss-120b", + drop_params=False, + ) + + assert openai_path_params == {"max_completion_tokens": 64} + assert v1_path_params == {"max_tokens": 64} + + def test_bedrock_tools_reasoning_flag_supports_reasoning_effort(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + litellm, + "model_cost", + { + "bedrock_mantle/test-model": { + "supports_bedrock_chat_completions_tools_with_reasoning": True, + } + }, + ) + cfg: Final = BedrockMantleChatConfig() + + assert "reasoning_effort" in cfg.get_supported_openai_params("test-model") + class TestBedrockMantleChatAuth: """Chat Completions must use the same Bearer-or-SigV4 auth as the Responses diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 57200a79a8c7..2f92b8c045e5 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -1506,6 +1506,115 @@ def test_responses_api_bridge_check_gpt_5_tools_without_summary_stays_chat(): assert model_info.get("mode") != "responses" +def test_bedrock_runtime_gpt56_tools_with_reasoning_bridge_to_responses(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + function_tools: Final = [{"type": "function", "function": {"name": "get_capital"}}] + request_params: Final = {"tools": function_tools, "reasoning_effort": "medium"} + model_info, model = responses_api_bridge_check( + model="us.openai.gpt-5.6-sol", + custom_llm_provider="bedrock", + tools=function_tools, + reasoning_effort="medium", + request_params=request_params, + ) + + assert model == "us.openai.gpt-5.6-sol" + assert model_info.get("mode") == "responses" + + +def test_bedrock_runtime_qwen_tools_with_reasoning_stay_on_chat(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + function_tools: Final = [{"type": "function", "function": {"name": "get_capital"}}] + request_params: Final = {"tools": function_tools, "reasoning_effort": "medium"} + model_info, model = responses_api_bridge_check( + model="qwen.qwen3-32b-v1:0", + custom_llm_provider="bedrock", + tools=function_tools, + reasoning_effort="medium", + request_params=request_params, + ) + + assert model == "qwen.qwen3-32b-v1:0" + assert model_info.get("mode") != "responses" + + +def test_bedrock_mantle_gpt56_plain_request_stays_on_chat(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + model_info, model = responses_api_bridge_check( + model="openai.gpt-5.6-sol", + custom_llm_provider="bedrock_mantle", + request_params={}, + ) + + assert model == "openai.gpt-5.6-sol" + assert model_info.get("mode") != "responses" + + +def test_bedrock_mantle_gpt56_tools_with_reasoning_bridge_to_responses(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + function_tools: Final = [{"type": "function", "function": {"name": "get_capital"}}] + request_params: Final = {"tools": function_tools, "reasoning_effort": "medium"} + model_info, model = responses_api_bridge_check( + model="openai.gpt-5.6-sol", + custom_llm_provider="bedrock_mantle", + tools=function_tools, + reasoning_effort="medium", + request_params=request_params, + ) + + assert model == "openai.gpt-5.6-sol" + assert model_info.get("mode") == "responses" + + +def test_bedrock_mantle_gpt56_cyber_plain_request_uses_responses(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + model_info, model = responses_api_bridge_check( + model="openai.gpt-5.6-cyber", + custom_llm_provider="bedrock_mantle", + request_params={}, + ) + + assert model == "openai.gpt-5.6-cyber" + assert model_info.get("mode") == "responses" + + +def test_bedrock_mantle_reasoning_summary_alias_bridges_to_responses(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + request_params: Final = {"reasoning_effort": "medium", "reasoningSummary": "detailed"} + model_info, model = responses_api_bridge_check( + model="openai.gpt-5.6-sol", + custom_llm_provider="bedrock_mantle", + reasoning_effort="medium", + request_params=request_params, + ) + + assert model == "openai.gpt-5.6-sol" + assert model_info.get("mode") == "responses" + + +def test_bedrock_mantle_grok46_tools_with_reasoning_stay_on_chat(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + function_tools: Final = [{"type": "function", "function": {"name": "get_capital"}}] + request_params: Final = {"tools": function_tools, "reasoning_effort": "medium"} + model_info, model = responses_api_bridge_check( + model="xai.grok-4.6", + custom_llm_provider="bedrock_mantle", + tools=function_tools, + reasoning_effort="medium", + request_params=request_params, + ) + + assert model == "xai.grok-4.6" + assert model_info.get("mode") != "responses" + + @patch("litellm.completion_extras.responses_api_bridge.completion") def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( mock_responses_completion, @@ -3378,7 +3487,7 @@ async def test_acompletion_forwards_aws_credentials_through_responses_bridge( "object": "response", "created_at": 1760144904, "status": "completed", - "model": "openai.gpt-5.4", + "model": "openai.gpt-5.6-sol", "output": [ { "type": "message", @@ -3392,8 +3501,10 @@ async def test_acompletion_forwards_aws_credentials_through_responses_bridge( ) response = await litellm.acompletion( - model="bedrock_mantle/openai.gpt-5.4", + model="bedrock_mantle/openai.gpt-5.6-sol", messages=[{"role": "user", "content": "hi"}], + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort="medium", api_base="https://bedrock-mantle.us-east-2.api.aws/v1", aws_region_name="us-east-2", num_retries=0, diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index a05c345b5cde..e03c9f1c7415 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -4,12 +4,14 @@ import json import re from collections.abc import Mapping +from itertools import groupby from pathlib import Path from types import MappingProxyType from typing import Final import jsonschema import pytest +from pydantic import TypeAdapter import litellm from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name @@ -338,7 +340,13 @@ def test_azure_gpt_chat_latest_declares_the_one_effort_azure_accepts(prices: dic BEDROCK_OPENAI_GPT_MARKERS: Final = ("openai.gpt-5.4", "openai.gpt-5.5", "openai.gpt-5.6", "openai.gpt-6-astra") BEDROCK_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse", "bedrock_mantle")) +BEDROCK_RUNTIME_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse")) BEDROCK_ROW_PREFIXES: Final = ("bedrock_mantle/", "us.", "global.") +BEDROCK_RUNTIME_CRIS_PREFIXES: Final = ("us-gov.", "global.", "apac.", "eu.", "us.", "jp.", "au.") +BEDROCK_CHAT_ENDPOINT: Final = "/v1/chat/completions" +BEDROCK_TOOLS_REASONING_FLAG: Final = "supports_bedrock_chat_completions_tools_with_reasoning" +BEDROCK_RESPONSE_FORMAT_FLAG: Final = "supports_bedrock_runtime_chat_completions_response_format" +BEDROCK_SUPPORTED_ENDPOINTS_ADAPTER: Final = TypeAdapter(list[str]) GPT_5_4_BEDROCK_LADDER: Final = ("none", "low", "medium", "high", "xhigh") GPT_5_6_BEDROCK_LADDER: Final = ("none", "low", "medium", "high", "xhigh", "max") GPT_6_ASTRA_BEDROCK_LADDER: Final = ("low", "medium", "high", "xhigh", "max") @@ -383,6 +391,71 @@ def test_every_bedrock_openai_gpt_row_advertises_xhigh(prices: dict): assert missing == [] +def bedrock_runtime_base_model_id(name: str) -> str: + without_provider_prefix: Final = name.removeprefix("bedrock_converse/").removeprefix("bedrock/") + first_component, separator, remainder = without_provider_prefix.partition("/") + model_id: Final = ( + remainder + if separator and re.fullmatch(r"[a-z]{2}(?:-[a-z0-9]+)+-\d+", first_component) + else without_provider_prefix + ) + return next( + (model_id.removeprefix(prefix) for prefix in BEDROCK_RUNTIME_CRIS_PREFIXES if model_id.startswith(prefix)), + model_id, + ) + + +def bedrock_supported_endpoints(entry: Mapping[str, object]) -> tuple[str, ...]: + supported_endpoints: Final = entry.get("supported_endpoints") + if supported_endpoints is None: + return () + return tuple(BEDROCK_SUPPORTED_ENDPOINTS_ADAPTER.validate_python(supported_endpoints)) + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_bedrock_endpoint_capabilities_are_consistent(path: Path) -> None: + prices: Final = TypeAdapter(dict[str, dict[str, object]]).validate_json(path.read_text()) + runtime_rows: Final = tuple( + (bedrock_runtime_base_model_id(name), entry) + for name, entry in prices.items() + if entry.get("litellm_provider") in BEDROCK_RUNTIME_PROVIDERS + ) + ordered_runtime_rows: Final = tuple(sorted(runtime_rows, key=lambda row: row[0])) + runtime_groups: Final = tuple( + (base_id, tuple(entry for _, entry in group)) + for base_id, group in groupby(ordered_runtime_rows, key=lambda row: row[0]) + ) + inconsistent_runtime_rows: Final = tuple( + base_id + for base_id, entries in runtime_groups + if len( + frozenset( + json.dumps((bedrock_supported_endpoints(entry), entry.get(BEDROCK_TOOLS_REASONING_FLAG))) + for entry in entries + ) + ) + != 1 + ) + runtime_capabilities_without_chat: Final = tuple( + name + for name, entry in prices.items() + if entry.get("litellm_provider") in BEDROCK_RUNTIME_PROVIDERS + and (entry.get(BEDROCK_TOOLS_REASONING_FLAG) is True or entry.get(BEDROCK_RESPONSE_FORMAT_FLAG) is True) + and BEDROCK_CHAT_ENDPOINT not in bedrock_supported_endpoints(entry) + ) + mantle_responses_with_chat: Final = tuple( + name + for name, entry in prices.items() + if entry.get("litellm_provider") == "bedrock_mantle" + and entry.get("mode") == "responses" + and BEDROCK_CHAT_ENDPOINT in bedrock_supported_endpoints(entry) + ) + + assert inconsistent_runtime_rows == () + assert runtime_capabilities_without_chat == () + assert mantle_responses_with_chat == () + + def is_active_priced_mistral_chat_row(name: str, entry: Mapping[str, object]) -> bool: input_cost: Final = entry.get("input_cost_per_token") return ( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cab4245edda6..747de154ccae 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -939,7 +939,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_video_input": {"type": "boolean"}, "supports_vision": {"type": "boolean"}, "supports_web_search": {"type": "boolean"}, - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, + "supports_bedrock_chat_completions_tools_with_reasoning": {"type": "boolean"}, "supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"}, "supports_url_context": {"type": "boolean"}, "supports_multimodal": {"type": "boolean"}, From f80c5836c0abe0aba24749de8502a062c48213bf Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 28 Sep 2026 17:42:18 +0000 Subject: [PATCH 2/5] fix(bedrock): regenerate model pricing schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- model_prices_and_context_window.schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 148a4abf4fab..596b0b03e096 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -938,10 +938,10 @@ "supports_audio_output": { "type": "boolean" }, - "supports_bedrock_runtime_chat_completions_response_format": { + "supports_bedrock_chat_completions_tools_with_reasoning": { "type": "boolean" }, - "supports_bedrock_chat_completions_tools_with_reasoning": { + "supports_bedrock_runtime_chat_completions_response_format": { "type": "boolean" }, "supports_computer_use": { From bcf0cd93f0b960494c9b6cce6056e3ecf6837499 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 28 Sep 2026 18:05:48 +0000 Subject: [PATCH 3/5] fix(bedrock): preserve legacy functions and bridge web search Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/common_utils.py | 4 ++ litellm/main.py | 37 +++++++--- ...bedrock_chat_completions_transformation.py | 12 ++++ tests/unit/test_main.py | 68 +++++++++++++++++++ 4 files changed, 110 insertions(+), 11 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 1fde32cb25a7..14897032f5ce 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1029,6 +1029,8 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje def bedrock_chat_request_needs_native_responses(model: str, request_params: Mapping[str, object]) -> bool: + if request_params.get("functions"): + return False if _bedrock_request_has_non_tool_converse_trigger(model, request_params): return False model_cost: Final = cast( # cast-ok: model price rows use string keys @@ -1036,6 +1038,8 @@ def bedrock_chat_request_needs_native_responses(model: str, request_params: Mapp ) if not bedrock_supports_openai_responses(model, model_cost): return False + if request_params.get("web_search_options") is not None and _bedrock_price_map_flag(model, "supports_web_search"): + return True reasoning_effort: Final = request_params.get("reasoning_effort") if reasoning_effort is not None and peek_reasoning_summary_aliases(request_params) is not None: return True diff --git a/litellm/main.py b/litellm/main.py index f0a8936fb802..03ea700bfb70 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -119,6 +119,7 @@ ) from litellm.llms.bedrock.common_utils import ( BedrockModelInfo, + _bedrock_price_map_flag, bedrock_chat_completions_serves_tools_with_reasoning, bedrock_chat_request_needs_native_responses_for_request, bedrock_reasoning_effort_is_active, @@ -1178,23 +1179,37 @@ def responses_api_bridge_check( mantle_request_params: Final = ( filter_additional_drop_params(request_params, additional_drop_params) if request_params is not None - else MappingProxyType({"tools": tools, "reasoning_effort": reasoning_effort}) + else MappingProxyType( + { + "tools": tools, + "reasoning_effort": reasoning_effort, + "web_search_options": web_search_options, + } + ) ) mantle_reasoning_effort: Final = mantle_request_params.get("reasoning_effort") mantle_reasoning_summary: Final = ( peek_reasoning_summary_aliases(mantle_request_params) if request_params is not None else None ) - mantle_needs_native_responses: Final = ( - has_function_tool(mantle_request_params.get("tools")) - and not bedrock_chat_completions_serves_tools_with_reasoning(model, "bedrock_mantle") - and bedrock_reasoning_effort_is_active( - model, - mantle_reasoning_effort, - "bedrock_mantle", + mantle_has_legacy_functions: Final = bool(mantle_request_params.get("functions")) + mantle_needs_native_responses: Final = not mantle_has_legacy_functions and ( + ( + has_function_tool(mantle_request_params.get("tools")) + and not bedrock_chat_completions_serves_tools_with_reasoning(model, "bedrock_mantle") + and bedrock_reasoning_effort_is_active( + model, + mantle_reasoning_effort, + "bedrock_mantle", + ) + ) + or ( + mantle_request_params.get("web_search_options") is not None + and _bedrock_price_map_flag(model, "supports_web_search", "bedrock_mantle") + ) + or ( + mantle_reasoning_effort is not None + and (mantle_reasoning_summary is not None or reasoning_summary is not None) ) - ) or ( - mantle_reasoning_effort is not None - and (mantle_reasoning_summary is not None or reasoning_summary is not None) ) mantle_model_cost: Final = cast( # cast-ok: model price rows use string keys Mapping[str, object], litellm.model_cost diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 4d8de157ad0f..63424f091d6c 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -484,6 +484,18 @@ def test_reasoning_summary_alias_bridges_to_responses(local_cost_map: None) -> N assert bedrock_chat_request_needs_native_responses(model, request_params) is True +def test_legacy_functions_with_reasoning_summary_stay_on_converse(local_cost_map: None) -> None: + model: Final = "us.openai.gpt-5.6-sol" + request_params: Final = { + "functions": [GET_WEATHER_TOOL["function"]], + "reasoning_effort": "medium", + "reasoningSummary": "auto", + } + + assert BedrockModelInfo.get_bedrock_route(model, request_params) == "converse" + assert bedrock_chat_request_needs_native_responses(model, request_params) is False + + @pytest.mark.parametrize("reasoning_effort", ["low", "high", None]) def test_gpt_oss_tools_with_any_reasoning_effort_stay_on_chat_completions(local_cost_map, reasoning_effort): params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": reasoning_effort} diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 2f92b8c045e5..665f69dc2068 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -1540,6 +1540,25 @@ def test_bedrock_runtime_qwen_tools_with_reasoning_stay_on_chat(local_cost_map: assert model_info.get("mode") != "responses" +def test_bedrock_runtime_legacy_functions_with_reasoning_summary_stay_on_chat(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + request_params: Final = { + "functions": [{"name": "get_capital", "parameters": {"type": "object"}}], + "reasoning_effort": "medium", + "reasoningSummary": "auto", + } + model_info, model = responses_api_bridge_check( + model="us.openai.gpt-5.6-sol", + custom_llm_provider="bedrock", + reasoning_effort="medium", + request_params=request_params, + ) + + assert model == "us.openai.gpt-5.6-sol" + assert model_info.get("mode") != "responses" + + def test_bedrock_mantle_gpt56_plain_request_stays_on_chat(local_cost_map: None) -> None: from litellm.main import responses_api_bridge_check @@ -1583,6 +1602,55 @@ def test_bedrock_mantle_gpt56_cyber_plain_request_uses_responses(local_cost_map: assert model_info.get("mode") == "responses" +def test_bedrock_mantle_legacy_functions_with_reasoning_summary_stay_on_chat(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + request_params: Final = { + "functions": [{"name": "get_capital", "parameters": {"type": "object"}}], + "reasoning_effort": "medium", + "reasoningSummary": "auto", + } + model_info, model = responses_api_bridge_check( + model="openai.gpt-5.6-sol", + custom_llm_provider="bedrock_mantle", + reasoning_effort="medium", + request_params=request_params, + ) + + assert model == "openai.gpt-5.6-sol" + assert model_info.get("mode") != "responses" + + +def test_bedrock_mantle_gpt56_web_search_bridges_to_responses(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + request_params: Final = {"web_search_options": {}} + model_info, model = responses_api_bridge_check( + model="openai.gpt-5.6-sol", + custom_llm_provider="bedrock_mantle", + web_search_options={}, + request_params=request_params, + ) + + assert model == "openai.gpt-5.6-sol" + assert model_info.get("mode") == "responses" + + +def test_bedrock_mantle_gpt6_web_search_without_metadata_stays_on_chat(local_cost_map: None) -> None: + from litellm.main import responses_api_bridge_check + + request_params: Final = {"web_search_options": {}} + model_info, model = responses_api_bridge_check( + model="openai.gpt-6-luna", + custom_llm_provider="bedrock_mantle", + web_search_options={}, + request_params=request_params, + ) + + assert model == "openai.gpt-6-luna" + assert model_info.get("mode") != "responses" + + def test_bedrock_mantle_reasoning_summary_alias_bridges_to_responses(local_cost_map: None) -> None: from litellm.main import responses_api_bridge_check From 6fc79b0ea3173dbe820c5030db0bc1a4fb8ee2bf Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 28 Sep 2026 18:26:43 +0000 Subject: [PATCH 4/5] refactor(bedrock): centralize Mantle Responses bridge eligibility Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/common_utils.py | 9 ++++- litellm/llms/bedrock_mantle/common_utils.py | 38 +++++++++++++++++- litellm/main.py | 40 ++++--------------- .../test_bedrock_mantle_transformation.py | 13 ++++++ 4 files changed, 65 insertions(+), 35 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 14897032f5ce..4a049c7fc9e6 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -874,6 +874,13 @@ def _bedrock_price_map_flag( return any(entry is not None and entry.get(flag) is True for entry in entries) +def bedrock_supports_web_search( + model: str, + custom_llm_provider: Literal["bedrock", "bedrock_mantle"] = "bedrock", +) -> bool: + return _bedrock_price_map_flag(model, "supports_web_search", custom_llm_provider) + + def _bedrock_runtime_row_lists_chat_completions(entry: Mapping[str, object]) -> bool: endpoints: Final = entry.get("supported_endpoints") return ( @@ -1038,7 +1045,7 @@ def bedrock_chat_request_needs_native_responses(model: str, request_params: Mapp ) if not bedrock_supports_openai_responses(model, model_cost): return False - if request_params.get("web_search_options") is not None and _bedrock_price_map_flag(model, "supports_web_search"): + if request_params.get("web_search_options") is not None and bedrock_supports_web_search(model): return True reasoning_effort: Final = request_params.get("reasoning_effort") if reasoning_effort is not None and peek_reasoning_summary_aliases(request_params) is not None: diff --git a/litellm/llms/bedrock_mantle/common_utils.py b/litellm/llms/bedrock_mantle/common_utils.py index f470abe356de..6e3ef3acede2 100644 --- a/litellm/llms/bedrock_mantle/common_utils.py +++ b/litellm/llms/bedrock_mantle/common_utils.py @@ -23,8 +23,15 @@ ProfileNotFound, ) +import litellm +from litellm.litellm_core_utils.responses_api_utils import has_function_tool, peek_reasoning_summary_aliases from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, SignsRequestsWithAWS -from litellm.llms.bedrock.common_utils import AmazonBedrockGlobalConfig +from litellm.llms.bedrock.common_utils import ( + AmazonBedrockGlobalConfig, + bedrock_chat_completions_serves_tools_with_reasoning, + bedrock_reasoning_effort_is_active, + bedrock_supports_web_search, +) from litellm.secret_managers.main import get_secret_str BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1" @@ -155,6 +162,35 @@ def mantle_supports_responses(model: str | None, model_cost: Mapping[str, object ) or typed_entry.get("mode") == "responses" +def mantle_chat_request_needs_native_responses( + model: str, + request_params: Mapping[str, object], + reasoning_summary: object | None, + *, + read_summary_aliases: bool = True, +) -> bool: + if request_params.get("functions"): + return False + reasoning_effort: Final = request_params.get("reasoning_effort") + reasoning_summary_alias: Final = peek_reasoning_summary_aliases(request_params) if read_summary_aliases else None + needs_native_responses: Final = ( + ( + has_function_tool(request_params.get("tools")) + and not bedrock_chat_completions_serves_tools_with_reasoning(model, "bedrock_mantle") + and bedrock_reasoning_effort_is_active(model, reasoning_effort, "bedrock_mantle") + ) + or ( + request_params.get("web_search_options") is not None + and bedrock_supports_web_search(model, "bedrock_mantle") + ) + or (reasoning_effort is not None and (reasoning_summary_alias is not None or reasoning_summary is not None)) + ) + model_cost: Final = cast( # cast-ok: model price rows use string keys + Mapping[str, object], litellm.model_cost + ) + return needs_native_responses and mantle_supports_responses(model, model_cost) + + def mantle_base_segment(model: str | None, model_cost: dict) -> str: """Return the base path segment for a Bedrock Mantle model's OpenAI surface. diff --git a/litellm/main.py b/litellm/main.py index 03ea700bfb70..77c7dc5f9da8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -119,12 +119,10 @@ ) from litellm.llms.bedrock.common_utils import ( BedrockModelInfo, - _bedrock_price_map_flag, - bedrock_chat_completions_serves_tools_with_reasoning, bedrock_chat_request_needs_native_responses_for_request, - bedrock_reasoning_effort_is_active, bedrock_route_for_request, ) +from litellm.llms.bedrock_mantle.common_utils import mantle_chat_request_needs_native_responses from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config @@ -1174,8 +1172,6 @@ def responses_api_bridge_check( ): model_info["mode"] = "responses" if custom_llm_provider == "bedrock_mantle" and model_info.get("mode") != "responses": - from litellm.llms.bedrock_mantle.common_utils import mantle_supports_responses - mantle_request_params: Final = ( filter_additional_drop_params(request_params, additional_drop_params) if request_params is not None @@ -1187,34 +1183,12 @@ def responses_api_bridge_check( } ) ) - mantle_reasoning_effort: Final = mantle_request_params.get("reasoning_effort") - mantle_reasoning_summary: Final = ( - peek_reasoning_summary_aliases(mantle_request_params) if request_params is not None else None - ) - mantle_has_legacy_functions: Final = bool(mantle_request_params.get("functions")) - mantle_needs_native_responses: Final = not mantle_has_legacy_functions and ( - ( - has_function_tool(mantle_request_params.get("tools")) - and not bedrock_chat_completions_serves_tools_with_reasoning(model, "bedrock_mantle") - and bedrock_reasoning_effort_is_active( - model, - mantle_reasoning_effort, - "bedrock_mantle", - ) - ) - or ( - mantle_request_params.get("web_search_options") is not None - and _bedrock_price_map_flag(model, "supports_web_search", "bedrock_mantle") - ) - or ( - mantle_reasoning_effort is not None - and (mantle_reasoning_summary is not None or reasoning_summary is not None) - ) - ) - mantle_model_cost: Final = cast( # cast-ok: model price rows use string keys - Mapping[str, object], litellm.model_cost - ) - if mantle_needs_native_responses and mantle_supports_responses(model, mantle_model_cost): + if mantle_chat_request_needs_native_responses( + model, + mantle_request_params, + reasoning_summary, + read_summary_aliases=request_params is not None, + ): model_info["mode"] = "responses" if ( (custom_llm_provider in ("openai", "azure") or on_foundry_openai_endpoint) diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 53e937f00bac..44c43cd8015f 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -872,6 +872,19 @@ def test_gemma_4_models_register_under_bedrock_mantle(local_cost_map, model_id): assert resolved_model == model_id +def test_chat_only_safeguard_model_does_not_bridge_tools_to_responses(local_cost_map): + from litellm.llms.bedrock_mantle.common_utils import mantle_chat_request_needs_native_responses + + assert not mantle_chat_request_needs_native_responses( + model="openai.gpt-oss-safeguard-120b", + request_params={ + "tools": [{"type": "function", "function": {"name": "get_capital"}}], + "reasoning_effort": "medium", + }, + reasoning_summary=None, + ) + + @pytest.mark.asyncio async def test_mantle_signing_runs_off_the_event_loop(): """Regression for issue #40165: Mantle signs with SigV4 through a composed BaseAWSLLM, so the From c22b3abafbc956640f76f5cb0ed09fc9c67e8f09 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 28 Sep 2026 20:14:27 +0000 Subject: [PATCH 5/5] fix(bedrock): honor regional Mantle rows and dict none effort in Responses bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/responses_api_utils.py | 6 +- litellm/llms/bedrock/common_utils.py | 9 +- litellm/llms/bedrock_mantle/common_utils.py | 10 +- ...bedrock_chat_completions_transformation.py | 28 ++++- .../test_bedrock_mantle_transformation.py | 100 ++++++++---------- 5 files changed, 89 insertions(+), 64 deletions(-) diff --git a/litellm/litellm_core_utils/responses_api_utils.py b/litellm/litellm_core_utils/responses_api_utils.py index 7ea8f5367d12..ebf2afaefae9 100644 --- a/litellm/litellm_core_utils/responses_api_utils.py +++ b/litellm/litellm_core_utils/responses_api_utils.py @@ -33,7 +33,11 @@ def reasoning_effort_is_active( effort_mapping: Final = cast( # cast-ok: reasoning payload keys are strings Mapping[str, object], reasoning_effort ) - return effort_mapping.get("effort") != "none" or effort_mapping.get("summary") is not None + return ( + effort_mapping.get("effort") != "none" + or effort_mapping.get("summary") is not None + or supports_none_reasoning_effort is False + ) return reasoning_effort != "none" or supports_none_reasoning_effort is False diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 15967f5a90ad..a7bca024e842 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -826,18 +826,21 @@ def split_bedrock_region_path(model: str) -> tuple[str | None, str]: return None, stripped -def _bedrock_mantle_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: +def _bedrock_mantle_price_map_keys(model: str) -> tuple[str, ...]: mantle_model: Final = model.removeprefix("bedrock_mantle/") region, model_id = split_bedrock_region_path(mantle_model) - candidates: Final = ( + return ( f"bedrock_mantle/{mantle_model}", f"bedrock_mantle/{region}/{model_id}" if region is not None else f"bedrock_mantle/{model_id}", f"bedrock_mantle/{model_id}", ) + + +def _bedrock_mantle_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: model_cost: Final = cast( # cast-ok: model price rows use string keys Mapping[str, Mapping[str, object]], litellm.model_cost ) - return tuple(model_cost.get(key) for key in candidates) + return tuple(model_cost.get(key) for key in _bedrock_mantle_price_map_keys(model)) BEDROCK_RUNTIME_PRICE_MAP_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse")) diff --git a/litellm/llms/bedrock_mantle/common_utils.py b/litellm/llms/bedrock_mantle/common_utils.py index 6e3ef3acede2..ff1b2fd38bcf 100644 --- a/litellm/llms/bedrock_mantle/common_utils.py +++ b/litellm/llms/bedrock_mantle/common_utils.py @@ -28,6 +28,7 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, SignsRequestsWithAWS from litellm.llms.bedrock.common_utils import ( AmazonBedrockGlobalConfig, + _bedrock_mantle_price_map_keys, bedrock_chat_completions_serves_tools_with_reasoning, bedrock_reasoning_effort_is_active, bedrock_supports_web_search, @@ -146,7 +147,14 @@ def mantle_supports_responses(model: str | None, model_cost: Mapping[str, object gpt-oss substring), so a substring gate would be wrong. A model absent from model_cost simply has no signal and returns False (chat-completions emulation). """ - entry: Final = model_cost.get(f"bedrock_mantle/{split_mantle_region_prefix(model)[1]}") if model else None + if not model: + return False + return any( + _mantle_price_map_entry_supports_responses(model_cost.get(key)) for key in _bedrock_mantle_price_map_keys(model) + ) + + +def _mantle_price_map_entry_supports_responses(entry: object) -> bool: if not isinstance(entry, Mapping): return False typed_entry: Final = cast( # cast-ok: the entry passed the Mapping check diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 63424f091d6c..d73214aae840 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -444,6 +444,26 @@ def test_gpt6_astra_tools_with_reasoning_none_bridge(local_cost_map: None) -> No assert bedrock_chat_request_needs_native_responses(model, request_params) is True +@pytest.mark.parametrize( + "model, expected", + [ + ("us.openai.gpt-6-sol", True), + ("us.openai.gpt-5.6-sol", False), + ], +) +def test_dict_reasoning_effort_none_respects_model_support( + local_cost_map: None, + model: str, + expected: bool, +) -> None: + request_params: Final[dict[str, object]] = { + "tools": [GET_WEATHER_TOOL], + "reasoning_effort": {"effort": "none"}, + } + + assert bedrock_chat_request_needs_native_responses(model, request_params) is expected + + def test_gpt56_legacy_functions_with_reasoning_stay_on_converse(local_cost_map: None) -> None: model: Final = "us.openai.gpt-5.6-sol" request_params: Final = {"functions": [GET_WEATHER_TOOL["function"]], "reasoning_effort": "medium"} @@ -566,11 +586,9 @@ def _assert_remote_images_inlined(content): def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch): - import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling + from litellm.litellm_core_utils.prompt_templates import image_handling - monkeypatch.setattr( - image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}" - ) + monkeypatch.setattr(image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}") body = AmazonBedrockRuntimeChatCompletionsConfig().transform_request( model="us.xai.grok-4.6", messages=IMAGE_MESSAGES, @@ -583,7 +601,7 @@ def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch async def test_async_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch): - import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling + from litellm.litellm_core_utils.prompt_templates import image_handling async def fake_convert(url): return f"data:image/png;base64,{url}" diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 44c43cd8015f..c75d68b236d5 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -18,6 +18,10 @@ import litellm from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig +from litellm.llms.bedrock_mantle.common_utils import ( + mantle_chat_request_needs_native_responses, + mantle_supports_responses, +) from litellm.types.utils import LlmProviders from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe @@ -98,9 +102,7 @@ def test_malicious_aws_region_name_rejected(self, monkeypatch): cfg._get_openai_compatible_provider_info( None, None, - litellm_params=GenericLiteLLMParams( - aws_region_name="us-east-1.api.aws.attacker.example/" - ), + litellm_params=GenericLiteLLMParams(aws_region_name="us-east-1.api.aws.attacker.example/"), ) def test_get_llm_provider_rejects_malicious_aws_region_name(self, monkeypatch): @@ -113,14 +115,10 @@ def test_get_llm_provider_rejects_malicious_aws_region_name(self, monkeypatch): litellm.get_llm_provider( model="openai.gpt-5.5", custom_llm_provider="bedrock_mantle", - litellm_params=GenericLiteLLMParams( - aws_region_name="us-east-1.api.aws.attacker.example/" - ), + litellm_params=GenericLiteLLMParams(aws_region_name="us-east-1.api.aws.attacker.example/"), ) - def test_get_llm_provider_uses_aws_region_name_for_responses( - self, monkeypatch, local_cost_map - ): + def test_get_llm_provider_uses_aws_region_name_for_responses(self, monkeypatch, local_cost_map): from litellm.types.router import GenericLiteLLMParams monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) @@ -178,18 +176,14 @@ def test_chat_base_for_gpt_oss_uses_v1(self, monkeypatch): monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2") monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info( - None, None, model="openai.gpt-oss-120b" - ) + api_base, _ = cfg._get_openai_compatible_provider_info(None, None, model="openai.gpt-oss-120b") assert api_base == "https://bedrock-mantle.us-east-2.api.aws/v1" @pytest.mark.parametrize( "model_id", ["google.gemma-4-31b", "google.gemma-4-26b-a4b", "google.gemma-4-e2b"], ) - def test_chat_base_for_gemma_4_uses_openai_v1( - self, monkeypatch, local_cost_map, model_id - ): + def test_chat_base_for_gemma_4_uses_openai_v1(self, monkeypatch, local_cost_map, model_id): # The chat-config bug the Gemma 4 cards exposed: gemma-4-* is served on the # /openai/v1 base, not the hardcoded /v1. Driven by the price-map # use_openai_responses_path flag (loaded by local_cost_map). Fails before @@ -197,22 +191,16 @@ def test_chat_base_for_gemma_4_uses_openai_v1( monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2") monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info( - None, None, model=model_id - ) + api_base, _ = cfg._get_openai_compatible_provider_info(None, None, model=model_id) assert api_base == "https://bedrock-mantle.us-east-2.api.aws/openai/v1" - def test_chat_base_explicit_api_base_wins_over_derived( - self, monkeypatch, local_cost_map - ): + def test_chat_base_explicit_api_base_wins_over_derived(self, monkeypatch, local_cost_map): # An explicit api_base must not be overridden by the data-driven default, # even for a model whose default differs (gemma-4 -> openai/v1). monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) custom_base = "https://bedrock-mantle.us-west-2.api.aws/v1" cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info( - custom_base, None, model="google.gemma-4-31b" - ) + api_base, _ = cfg._get_openai_compatible_provider_info(custom_base, None, model="google.gemma-4-31b") assert api_base == custom_base def test_api_key_from_env(self, monkeypatch): @@ -299,9 +287,7 @@ def _signer_that_forbids_credentials(self): from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM signer = BaseAWSLLM() - signer.get_credentials = MagicMock( - side_effect=AssertionError("SigV4 must not run when a Bearer token exists") - ) + signer.get_credentials = MagicMock(side_effect=AssertionError("SigV4 must not run when a Bearer token exists")) return signer def test_bearer_token_skips_sigv4(self, monkeypatch): @@ -418,9 +404,7 @@ def test_sigv4_region_resolved_from_api_base_host(self, monkeypatch): assert "/eu-west-1/bedrock/aws4_request" in headers["Authorization"] - def test_sigv4_scope_matches_api_base_when_aws_region_name_disagrees( - self, monkeypatch - ): + def test_sigv4_scope_matches_api_base_when_aws_region_name_disagrees(self, monkeypatch): # If a caller (e.g. proxy) passes a stale api_base in one region and an # aws_region_name in a different region, the SigV4 credential scope must # match the URL host or Bedrock rejects the request with 401. Without the @@ -508,7 +492,7 @@ def test_no_bearer_and_no_credentials_raises_value_error(self, monkeypatch): signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleChatConfig(aws_signer=signer) - with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: + with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -534,9 +518,7 @@ def test_completion_no_bearer_signs_with_sigv4_end_to_end(self, monkeypatch): ): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE") - monkeypatch.setenv( - "AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0" - ) + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0") monkeypatch.setenv("AWS_REGION", "us-east-2") requests = [] @@ -566,9 +548,7 @@ def mock_post(self, url, data=None, headers=None, **kwargs): request=httpx.Request("POST", url), ) - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post - ): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): response = litellm.completion( model="bedrock_mantle/openai.gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], @@ -612,7 +592,9 @@ def test_completion_per_request_role_reaches_signer_and_not_the_body(self, monke "object": "chat.completion", "created": 1733529600, "model": "google.gemma-4-31b", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, }, request=httpx.Request("POST", url), @@ -678,9 +660,7 @@ def test_completion_sends_openai_project_header_and_clean_body(self): def mock_post(self, url, data=None, headers=None, **kwargs): raw_body = data.decode("utf-8") if isinstance(data, bytes) else data - requests.append( - {"headers": headers or {}, "body": json.loads(raw_body or "{}")} - ) + requests.append({"headers": headers or {}, "body": json.loads(raw_body or "{}")}) return httpx.Response( status_code=200, json={ @@ -704,9 +684,7 @@ def mock_post(self, url, data=None, headers=None, **kwargs): request=httpx.Request("POST", url), ) - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post - ): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): response = litellm.completion( model="bedrock_mantle/openai.gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], @@ -722,20 +700,15 @@ def mock_post(self, url, data=None, headers=None, **kwargs): class TestBedrockMantleProviderResolution: def test_get_llm_provider_resolves_correctly(self): - model, provider, _, _ = litellm.get_llm_provider( - "bedrock_mantle/openai.gpt-oss-120b" - ) + model, provider, _, _ = litellm.get_llm_provider("bedrock_mantle/openai.gpt-oss-120b") assert provider == "bedrock_mantle" assert model == "openai.gpt-oss-120b" def test_get_llm_provider_20b(self): - model, provider, _, _ = litellm.get_llm_provider( - "bedrock_mantle/openai.gpt-oss-20b" - ) + model, provider, _, _ = litellm.get_llm_provider("bedrock_mantle/openai.gpt-oss-20b") assert provider == "bedrock_mantle" assert model == "openai.gpt-oss-20b" - def test_get_llm_provider_strips_region_prefix(self, monkeypatch, local_cost_map): for var in ("BEDROCK_MANTLE_REGION", "BEDROCK_MANTLE_API_BASE", "AWS_REGION", "AWS_REGION_NAME"): monkeypatch.delenv(var, raising=False) @@ -768,7 +741,9 @@ def respond(request: httpx.Request) -> httpx.Response: "object": "chat.completion", "created": 1733529600, "model": "xai.grok-4.3", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], "usage": {"prompt_tokens": 38, "completion_tokens": 20, "total_tokens": 58}, }, request=request, @@ -873,8 +848,6 @@ def test_gemma_4_models_register_under_bedrock_mantle(local_cost_map, model_id): def test_chat_only_safeguard_model_does_not_bridge_tools_to_responses(local_cost_map): - from litellm.llms.bedrock_mantle.common_utils import mantle_chat_request_needs_native_responses - assert not mantle_chat_request_needs_native_responses( model="openai.gpt-oss-safeguard-120b", request_params={ @@ -885,6 +858,25 @@ def test_chat_only_safeguard_model_does_not_bridge_tools_to_responses(local_cost ) +def test_regional_mantle_responses_metadata_enables_tools_bridge(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "us-gov-west-1/fake-id" + model_cost: Final[dict[str, object]] = { + "bedrock_mantle/us-gov-west-1/fake-id": { + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "mode": "chat", + } + } + monkeypatch.setattr(litellm, "model_cost", model_cost) + request_params: Final[dict[str, object]] = { + "tools": [{"type": "function", "function": {"name": "get_weather"}}], + "reasoning_effort": "high", + } + + assert mantle_supports_responses(model, model_cost) is True + assert mantle_supports_responses("fake-id", model_cost) is False + assert mantle_chat_request_needs_native_responses(model, request_params, reasoning_summary=None) is True + + @pytest.mark.asyncio async def test_mantle_signing_runs_off_the_event_loop(): """Regression for issue #40165: Mantle signs with SigV4 through a composed BaseAWSLLM, so the