feat(bedrock)!: bridge OpenAI model function tools with reasoning to native /responses instead of Converse

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -5,7 +5,7 @@
 from collections.abc import AsyncIterable, Coroutine, Iterable
 from typing import TYPE_CHECKING, Any, Final, Union
 
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm.types.llms.openai import ResponsesAPIResponse
 
@@ -23,6 +23,7 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
     logging_obj: "LiteLLMLoggingObj"
     custom_llm_provider: str
     encoding: object
+    client: ReadOnly[object]
 
 
 def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
@@ -152,6 +153,7 @@ def validate_input_kwargs(self, kwargs: dict) -> ResponsesToCompletionBridgeHand
             logging_obj=logging_obj,
             custom_llm_provider=custom_llm_provider,
             encoding=typed_kwargs.get("encoding"),
+            client=typed_kwargs.get("client"),
         )
 
     def completion(
@@ -186,7 +188,7 @@ def completion(
             litellm_params=litellm_params,
             headers=headers,
             litellm_logging_obj=logging_obj,
-            client=kwargs.get("client"),
+            client=validated_kwargs["client"],
         )
 
         # Set on request_data rather than passed as explicit kwargs: the spread of
@@ -282,6 +284,7 @@ async def acompletion(self, *args, **kwargs) -> Union["ModelResponse", "CustomSt
                 litellm_params=litellm_params,
                 headers=headers,
                 litellm_logging_obj=logging_obj,
+                client=validated_kwargs["client"],
             )
         except Exception as e:
             raise e

litellm/llms/bedrock/common_utils.py

@@ -59,6 +59,7 @@
     "openai",
     "mantle",
     "chat_completions",
+    "responses",
 ]
 
 
@@ -997,7 +998,8 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
     written onto the Converse body,
     function tools (``tools`` or legacy ``functions``) on a model without
     ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless
-    ``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as
+    ``reasoning_effort`` is exactly ``"none"`` (function-only ``tools`` on a model listing ``/v1/responses`` are
+    bridged to bedrock-runtime's native Responses API instead), and a ``response_format`` goes native only as
     ``{"type": "json_schema", "json_schema": ...}`` (a pydantic model is converted to that) on a model with
     ``supports_bedrock_runtime_chat_completions_response_format``: a schema on any other model is only
     honored by Converse, and every ``json_object`` form (``response_schema`` included) keeps Converse's
@@ -1015,6 +1017,12 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
         return True
     if _response_format_needs_converse(model, request_params.get("response_format")):
         return True
+    return _chat_completions_rejects_function_tools(model, request_params) and not _bridgeable_to_responses(
+        model, request_params
+    )
+
+
+def _chat_completions_rejects_function_tools(model: str, request_params: Mapping[str, object]) -> bool:
     if not (request_params.get("tools") or request_params.get("functions")):
         return False
     return (
@@ -1023,12 +1031,64 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
     )
 
 
+class _NamedFunction(TypedDict):
+    name: ReadOnly[str]
+
+
+class _FunctionTool(TypedDict):
+    type: ReadOnly[Literal["function"]]
+    function: ReadOnly[_NamedFunction]
+
+
+_FUNCTION_TOOLS_ADAPTER: Final = TypeAdapter(tuple[_FunctionTool, ...])
+_OPENAI_FUNCTION_NAME: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}")
+
+
+def _function_tools_only(tools: object) -> bool:
+    if not isinstance(tools, (list, tuple)):
+        return False
+    try:
+        validated: Final = _FUNCTION_TOOLS_ADAPTER.validate_python(tools)
+    except ValidationError:
+        return False
+    return all(_OPENAI_FUNCTION_NAME.fullmatch(tool["function"]["name"]) for tool in validated)
+
+
+_EXTRA_BODY_ADAPTER: Final = TypeAdapter(dict[str, object])
+
+
+def _extra_body_has_tools(extra_body: object) -> bool:
+    return isinstance(extra_body, dict) and bool(_EXTRA_BODY_ADAPTER.validate_python(extra_body).get("tools"))
+
+
+def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -> bool:
+    return (
+        not request_params.get("functions")
+        and request_params.get("aws_bedrock_client") is None
+        and _function_tools_only(request_params.get("tools"))
+        and not _extra_body_has_tools(request_params.get("extra_body"))
+        and _bedrock_serves_responses(model)
+    )
+
+
+def _bedrock_serves_responses(model: str) -> bool:
+    region, model_id = split_bedrock_region_path(model)
+    return region is None and any(
+        _price_map_entry_lists_endpoint(_model_cost_entry(key), "/v1/responses")
+        for key in (model_id, f"bedrock/{model_id}")
+    )
+
+
 def _chat_completions_unless_converse_needed(
     model: str, request_params: Mapping[str, object] | None
-) -> Literal["converse", "chat_completions"]:
-    if request_params is not None and bedrock_request_needs_converse(model, request_params):
+) -> Literal["converse", "chat_completions", "responses"]:
+    if request_params is None:
+        return "chat_completions"
+    if bedrock_request_needs_converse(model, request_params):
         return "converse"
-    return "chat_completions"
+    if not _chat_completions_rejects_function_tools(model, request_params):
+        return "chat_completions"
+    return "converse" if request_params.get("_litellm_responses_api_bridge_allowed") is False else "responses"
 
 
 def bedrock_route_for_request(
@@ -1474,8 +1534,9 @@ def get_bedrock_route(
         Grok and GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by
         default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
         ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
-        feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
-        model stays on Converse without the prefix.
+        feature only Converse serves, to ``responses`` (the chat-to-Responses bridge onto bedrock-runtime's
+        native ``/openai/v1/responses``) when it carries function tools with reasoning on, and ``converse/``
+        pins a model to Converse. Every other OpenAI-family model stays on Converse without the prefix.
         """
         route_mappings: dict[
             str,

litellm/llms/bedrock/responses/transformation.py

@@ -222,17 +222,13 @@ def get_complete_url(
         api_base: str | None,
         litellm_params: dict,  # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
     ) -> str:
-        region: Final = self._get_aws_region_name(optional_params=litellm_params, model=None)
-        override: Final = (
-            api_base
-            or litellm_params.get("aws_bedrock_runtime_endpoint")
-            or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT")
+        runtime_endpoint: Final = litellm_params.get("aws_bedrock_runtime_endpoint")
+        _, destination = self.get_runtime_endpoint(
+            api_base=api_base or None,
+            aws_bedrock_runtime_endpoint=runtime_endpoint if isinstance(runtime_endpoint, str) else None,
+            aws_region_name=self._get_aws_region_name(optional_params=litellm_params, model=None),
         )
-        # Partition-aware: bedrock-runtime is amazonaws.com.cn in China, and other
-        # suffixes in GovCloud/ISO, so defer to the shared endpoint builder.
-        host: Final = (
-            override or self._select_default_endpoint_url(endpoint_type="runtime", aws_region_name=region)
-        ).rstrip("/")
+        host: Final = destination.rstrip("/")
         base: Final = next(
             (host[: -len(suffix)] for suffix in BEDROCK_RUNTIME_OPENAI_BASE_SUFFIXES if host.endswith(suffix)),
             host,

litellm/main.py

@@ -5723,7 +5723,11 @@ def completion(
             "allowed_openai_params": allowed_openai_params,
             "base_model": base_model,
         }
-        optional_params = get_optional_params(**optional_param_args, **non_default_params)
+        optional_params = get_optional_params(
+            **optional_param_args,
+            **non_default_params,
+            _litellm_responses_api_bridge_allowed=not skip_responses_api_bridge,
+        )
         processed_non_default_params: Final = pre_process_non_default_params(
             model=model,
             passed_params=optional_param_args,
@@ -5878,11 +5882,32 @@ def completion(
                 api_base=api_base,
             )
 
+        request_params: Final = MappingProxyType(
+            _OPTIONAL_PARAMS_ADAPTER.validate_python(
+                {
+                    **optional_param_args,
+                    **non_default_params,
+                    "_litellm_responses_api_bridge_allowed": not skip_responses_api_bridge,
+                }
+            )
+        )
+        bedrock_bridges_to_responses: Final = (
+            custom_llm_provider == "bedrock"
+            and bedrock_route_for_request(
+                model,
+                request_params,
+                _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(kwargs.get("additional_drop_params") or ()),
+            )
+            == "responses"
+        )
+
         # Use base_model (the true underlying model) for Azure model-type
         # detection when the deployment name differs from the model name.
         _azure_detection_model: Final = base_model or model
 
-        if responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge:
+        if (
+            responses_api_model_info.get("mode") == "responses" or bedrock_bridges_to_responses
+        ) and not skip_responses_api_bridge:
             from litellm.completion_extras import responses_api_bridge
 
             optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
@@ -5950,9 +5975,7 @@ def completion(
             optional_params=optional_params,
             organization=organization,
             provider_config=provider_config,
-            request_params=MappingProxyType(
-                _OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **non_default_params})
-            ),
+            request_params=request_params,
             shared_session=shared_session,
             stream=stream,
             temperature=temperature,

litellm/utils.py

@@ -4522,6 +4522,7 @@ def get_optional_params(
     store: bool | None = None,
     prompt_cache_key: str | None = None,
     base_model: str | None = None,
+    _litellm_responses_api_bridge_allowed: bool = True,
     **kwargs: object,
 ):
     drop_params = normalize_drop_params(drop_params)  # rebind-ok: config and DB deployments pass "true" as a string
@@ -4532,6 +4533,7 @@ def get_optional_params(
     # non_default_params / _check_valid_arg — it's a routing hint, not an
     # OpenAI param.
     passed_params.pop("base_model", None)
+    passed_params.pop("_litellm_responses_api_bridge_allowed")
     provider_config: BaseConfig | None = None
     if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]:
         provider_config = ProviderConfigManager.get_provider_chat_config(
@@ -4583,7 +4585,11 @@ def _check_valid_arg(supported_params: list[str]):
                 )
 
     bedrock_route: Final = (
-        _bedrock_route_for_request(model, passed_params, additional_drop_params)
+        _bedrock_route_for_request(
+            model,
+            {**passed_params, "_litellm_responses_api_bridge_allowed": _litellm_responses_api_bridge_allowed},
+            additional_drop_params,
+        )
         if custom_llm_provider == "bedrock"
         else None
     )

tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py

@@ -8,7 +8,7 @@
 import httpx
 import openai
 import pytest
-from integration._support.bedrock_runtime_peer import answer, respond, target_of
+from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of
 from integration._support.client import Gateway, Scenario, eventually
 from integration._support.database import read_rows
 from integration._support.sigv4 import signature
@@ -159,6 +159,19 @@ def _spend_row(identity: str) -> dict[str, JsonValue]:
     return rows[0]
 
 
+def _only_spend_row_of(model: str) -> dict[str, JsonValue]:
+    rows: Final = eventually(
+        lambda: read_rows(
+            'SELECT model_group, status, prompt_tokens, completion_tokens, api_base FROM "LiteLLM_SpendLogs"'
+            " WHERE model_group=%s",
+            (model,),
+        ),
+        lambda found: len(found) == 1,
+        seconds=70,
+    )
+    return rows[0]
+
+
 def _success_row(model: str, api_base: str) -> dict[str, JsonValue]:
     return {"model_group": model, "status": "success", "prompt_tokens": 9, "completion_tokens": 5, "api_base": api_base}
 
@@ -387,17 +400,44 @@ def test_json_schema_response_format_is_forwarded_natively(gateway: Gateway) ->
         assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
 
 
-def test_tools_while_reasoning_keep_converse(gateway: Gateway) -> None:
+def test_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None:
     marker: Final = uuid.uuid4().hex
     with wire_server(respond) as wire, gateway.scenario() as scenario:
         model: Final = _deployment(scenario, wire)
         response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
         payload: Final = _payload(response)
+        assert payload["id"] == f"resp_upstream_{marker}", response.text
+        choices: Final = payload["choices"]
+        assert isinstance(choices, list) and len(choices) == 1, response.text
+        assert choices[0]["finish_reason"] == "stop", response.text
+        assert choices[0]["message"]["content"] == answer(marker), response.text
+        body: Final = _body(_converse_request(wire, NATIVE_RESPONSES))
+        assert body["model"] == GPT, body
+        assert body["reasoning"] == {"effort": "high"}, body
+        assert [(tool["type"], tool["name"], tool["parameters"]) for tool in body["tools"]] == [
+            ("function", "lookup_invoice", dict(TOOL_PARAMETERS))
+        ], body
+        assert _upstream_requests_mentioning(gateway, marker) == [], response.text
+        assert _only_spend_row_of(model) == {
+            **_success_row(model, f"{wire.url}{NATIVE_RESPONSES}"),
+            "prompt_tokens": 30,
+        }
+
+
+def test_tools_while_reasoning_with_a_name_openai_rejects_keep_converse(gateway: Gateway) -> None:
+    marker: Final = uuid.uuid4().hex
+    tool: Final = {**TOOL, "function": {**TOOL["function"], "name": "lookup invoice!"}}
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(scenario, wire)
+        response: Final = _chat(gateway, model, marker, tools=[tool], reasoning_effort="high")
+        payload: Final = _payload(response)
         assert payload["choices"] == [
             {"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
         ], response.text
         body: Final = _body(_converse_request(wire))
-        assert body["toolConfig"] == {"tools": [CONVERSE_TOOL]}, body
+        assert body["toolConfig"] == {
+            "tools": [{"toolSpec": {**CONVERSE_TOOL["toolSpec"], "name": "lookup_invoice_"}}]
+        }, body
         assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body
         assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")