chore(release): backport #40989 to stable/1.101.x and cut 1.101.6

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -17,7 +17,7 @@
 
 import json
 from collections.abc import Mapping
-from typing import Any, Final
+from typing import Any, Final, cast  # noqa: TID251  # map_openai_params returns the filtered params as a bare dict
 
 from typing_extensions import ReadOnly, TypedDict
 
@@ -29,6 +29,7 @@
     BedrockMantleAuthMixin,
 )
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import (
     ResponseInputParam,
@@ -54,8 +55,6 @@
 
 _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 
-_CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
-
 _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
 _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
 _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
@@ -187,17 +186,14 @@ def transform_responses_api_request(
         litellm_params: GenericLiteLLMParams,
         headers: dict,
     ) -> dict:
-        remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
-        normalized_input: Final = self._normalize_codex_input_items(remaining_input)
+        params: Final = cast(  # cast-ok: the base signature leaves the params dict untyped
+            "ResponsesAPIOptionalRequestParams", response_api_optional_request_params
+        )
+        hoisted: Final = hoist_additional_tools(input, params.get("tools"))
+        normalized_input: Final = self._normalize_codex_input_items(hoisted.input)
         request_params: Final = (
-            {
-                **response_api_optional_request_params,
-                "tools": [
-                    *(response_api_optional_request_params.get("tools") or []),
-                    *hoisted_tools,
-                ],
-            }
-            if hoisted_tools
+            self._params_with_hoisted_tools(params, hoisted)
+            if hoisted.hoisted
             else response_api_optional_request_params
         )
         return super().transform_responses_api_request(
@@ -208,41 +204,14 @@ def transform_responses_api_request(
             headers=headers,
         )
 
-    @staticmethod
-    def _is_codex_additional_tools_item(item: Any) -> bool:
-        return isinstance(item, dict) and item.get("type") == _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
-
-    @staticmethod
-    def _tools_of_additional_tools_item(item: "dict[str, Any]") -> "list[Any]":
-        tools: Final = item.get("tools")
-        return tools if isinstance(tools, list) else []
-
     @classmethod
-    def _hoist_codex_additional_tools(
-        cls,
-        input: "str | ResponseInputParam",
-    ) -> "tuple[str | ResponseInputParam, list[Any]]":
-        """Codex's "responses lite" wire mode ships tool definitions inside
-        `input` as {"type": "additional_tools", "role": "developer",
-        "tools": [...]} items. api.openai.com accepts that item type; Mantle
-        rejects the whole request with 400 "Invalid 'input': value did not
-        match any expected variant" but accepts the same tools at the top
-        level, so move them there and strip the items from `input`.
-        """
-        if not isinstance(input, list):
-            return input, []
-        additional_tools_items: Final = [item for item in input if cls._is_codex_additional_tools_item(item)]
-        if not additional_tools_items:
-            return input, []
-        remaining_input: Final = [item for item in input if not cls._is_codex_additional_tools_item(item)]
-        hoisted_tools = [tool for item in additional_tools_items for tool in cls._tools_of_additional_tools_item(item)]
-        verbose_logger.debug(
-            "Bedrock Mantle Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) "
-            "into the top-level tools param (Mantle rejects that input item type).",
-            len(hoisted_tools),
-            len(additional_tools_items),
-        )
-        return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
+    def _params_with_hoisted_tools(
+        cls, params: Mapping[str, object], hoisted: HoistedAdditionalTools
+    ) -> dict[str, object]:
+        supported_tools: Final = cls._filter_unsupported_tools(list(hoisted.tools))
+        if supported_tools:
+            return {**params, "tools": supported_tools}
+        return {key: value for key, value in params.items() if key != "tools"}
 
     @staticmethod
     def _agent_message_text(item: "Mapping[str, object]") -> str:

litellm/llms/openai/responses/guardrail_translation/tool_merge.py

@@ -6,8 +6,10 @@
 from pydantic import BaseModel, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix
 from litellm.responses.litellm_completion_transformation.transformation import (
     NAMESPACE_DESCRIPTION_SEPARATOR,
+    NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS,
     LiteLLMCompletionResponsesConfig,
 )
 
@@ -34,8 +36,8 @@ def _validated_tools(values: Iterable[object]) -> tuple[Tool, ...]:
     return tuple(tool for tool in validated if tool is not None)
 
 
-def _is_function(tool: Tool) -> bool:
-    return tool.get("type") == "function"
+def _has_chat_tool(member: Tool) -> bool:
+    return member.get("type") in NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS
 
 
 def _chat_tool_key(tool: Tool) -> str:
@@ -67,18 +69,19 @@ def _function_fields(tool: Tool) -> Tool:
     return function if function is not None else MappingProxyType({})
 
 
-def _without_namespace_prefix(key: str, value: object, prefix: str) -> object:
-    if key != "description" or not isinstance(value, str) or not value.startswith(prefix):
+def _member_description(key: str, value: object, prefix: str, suffix: str) -> object:
+    if key != "description" or not isinstance(value, str):
         return value
-    return value[len(prefix) :]
+    return value.replace(prefix, "", 1).replace(suffix, "", 1)
 
 
 def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_description: str) -> Tool:
     flattened_function: Final = _function_fields(flattened)
     prefix: Final = f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}" if namespace_description else ""
+    suffix: Final = custom_tool_grammar_suffix(member.get("format")) if member.get("type") == "custom" else ""
     changed_function: Final = MappingProxyType(
         {
-            key: _without_namespace_prefix(key, value, prefix)
+            key: _member_description(key, value, prefix, suffix)
             for key, value in _function_fields(guardrailed).items()
             if flattened_function.get(key) != value
         }
@@ -93,8 +96,8 @@ def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_
     return {**member, **changed_extras, **changed_function}  # mutable-ok: json.dumps rejects MappingProxyType
 
 
-def _rebuilt_function_members(
-    function_members: Sequence[Tool],
+def _rebuilt_flattened_members(
+    flattened_members: Sequence[Tool],
     flattened_group: Sequence[Tool],
     group_keys: Sequence[IndexedKey],
     guardrailed_by_key: Mapping[IndexedKey, Tool],
@@ -106,7 +109,7 @@ def _rebuilt_function_members(
         else member
         if guardrailed_by_key[key] == flattened
         else _rebuilt_member(member, flattened, guardrailed_by_key[key], namespace_description)
-        for member, flattened, key in zip(function_members, flattened_group, group_keys)
+        for member, flattened, key in zip(flattened_members, flattened_group, group_keys)
     )
 
 
@@ -118,9 +121,9 @@ def _rebuilt_namespace(
     guardrailed_by_key: Mapping[IndexedKey, Tool],
 ) -> tuple[Tool, ...]:
     namespace_description: Final = str(original.get("description") or "")
-    rebuilt_functions: Final = iter(
-        _rebuilt_function_members(
-            tuple(member for member in members if _is_function(member)),
+    rebuilt_flattened: Final = iter(
+        _rebuilt_flattened_members(
+            tuple(member for member in members if _has_chat_tool(member)),
             flattened_group,
             group_keys,
             guardrailed_by_key,
@@ -129,7 +132,7 @@ def _rebuilt_namespace(
     )
     rebuilt_members: Final = tuple(
         rebuilt
-        for rebuilt in (next(rebuilt_functions) if _is_function(member) else member for member in members)
+        for rebuilt in (next(rebuilt_flattened) if _has_chat_tool(member) else member for member in members)
         if rebuilt is not None
     )
     if not rebuilt_members:
@@ -149,7 +152,7 @@ def _merged_original(
     if guardrailed_group == tuple(flattened_group):
         return (original,)
     members: Final = _namespace_members(original) if original.get("type") == "namespace" else ()
-    if members and sum(map(_is_function, members)) == len(flattened_group):
+    if members and sum(map(_has_chat_tool, members)) == len(flattened_group):
         return _rebuilt_namespace(original, members, flattened_group, group_keys, guardrailed_by_key)
     if not guardrailed_group:
         return ()

litellm/responses/additional_tools.py

@@ -0,0 +1,65 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Final, cast  # noqa: TID251  # validating the openai tool union strips vendor keys from raw tools
+
+from pydantic import BaseModel, ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam
+
+ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
+
+
+class _InputItemType(BaseModel):
+    type: str = ""
+
+
+class _AdditionalToolsItem(BaseModel):
+    tools: tuple[dict[str, object], ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class HoistedAdditionalTools:
+    input: str | ResponseInputParam
+    tools: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]
+    hoisted: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]
+
+
+def _is_additional_tools_item(item: object) -> bool:
+    try:
+        return _InputItemType.model_validate(item).type == ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
+    except ValidationError:
+        return False
+
+
+def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
+    try:
+        parsed: Final = _AdditionalToolsItem.model_validate(item)
+    except ValidationError:
+        return ()
+    return tuple(
+        cast(
+            "ALL_RESPONSES_API_TOOL_PARAMS", tool
+        )  # cast-ok: nested tools carry the same raw tool JSON as top-level tools
+        for tool in parsed.tools
+    )
+
+
+def hoist_additional_tools(
+    input: str | ResponseInputParam,
+    tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None,
+) -> HoistedAdditionalTools:
+    existing: Final = tuple(tools or ())
+    if isinstance(input, str):
+        return HoistedAdditionalTools(input=input, tools=existing, hoisted=())
+    items: Final = tuple(item for item in input if _is_additional_tools_item(item))
+    if not items:
+        return HoistedAdditionalTools(input=input, tools=existing, hoisted=())
+    hoisted: Final = tuple(tool for item in items for tool in _tools_of_item(item))
+    verbose_logger.debug(
+        "Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) into the top-level tools param.",
+        len(hoisted),
+        len(items),
+    )
+    remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)]
+    return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted)

litellm/responses/litellm_completion_transformation/custom_tools.py

@@ -39,15 +39,38 @@ def openai_shaped_tool_call_item_id(item_type: str, tool_id: str) -> str:
     return f"{prefix}_{tool_id}"
 
 
+class _ToolNameFields(BaseModel):
+    type: str = ""
+    name: str = ""
+    tools: tuple[object, ...] = ()
+
+
+def _tool_name_fields_of(tool: object) -> _ToolNameFields | None:
+    try:
+        return _ToolNameFields.model_validate(tool)
+    except ValidationError:
+        return None
+
+
+def _custom_tool_name_of(tool: object) -> str | None:
+    parsed: Final = _tool_name_fields_of(tool)
+    if parsed is None or parsed.type != "custom" or not parsed.name:
+        return None
+    return parsed.name
+
+
+def _nested_tools_of(tool: object) -> tuple[object, ...]:
+    parsed: Final = _tool_name_fields_of(tool)
+    if parsed is None or parsed.type != "namespace":
+        return ()
+    return parsed.tools
+
+
 def extract_custom_tool_names(tools: Sequence[object] | None) -> set[str]:
-    """Extract names of tools originally defined as ``type: "custom"``."""
-    if not tools:
-        return set()
-    names: Final[set[str]] = set()
-    for tool in tools:
-        if isinstance(tool, dict) and tool.get("type") == "custom" and "name" in tool:
-            names.add(tool["name"])
-    return names
+    """Extract names of ``type: "custom"`` tools, at the top level or one level inside a ``namespace`` tool."""
+    top_level: Final = tuple(tools or ())
+    nested: Final = tuple(nested_tool for tool in top_level for nested_tool in _nested_tools_of(tool))
+    return {name for tool in (*top_level, *nested) if (name := _custom_tool_name_of(tool)) is not None}
 
 
 def is_custom_tool_call(tool_name: str, custom_tool_names: set[str]) -> bool:
@@ -143,7 +166,7 @@ def validated_allowed_callers(value: object) -> list[str] | None:
         raise ValueError("allowed_callers must be a list of strings") from exc
 
 
-def _grammar_suffix(fmt: object) -> str:
+def custom_tool_grammar_suffix(fmt: object) -> str:
     try:
         parsed: Final = _CustomToolFormat.model_validate(fmt)
     except ValidationError:
@@ -167,7 +190,9 @@ def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatComp
     raw_name: Final = tool.get("name")
     name: Final = raw_name if isinstance(raw_name, str) else ""
     raw_description: Final = tool.get("description")
-    description = (raw_description if isinstance(raw_description, str) else "") + _grammar_suffix(tool.get("format"))
+    description: Final = (raw_description if isinstance(raw_description, str) else "") + custom_tool_grammar_suffix(
+        tool.get("format")
+    )
     allowed_callers: Final = validated_allowed_callers(tool.get("allowed_callers"))
     function_chunk: Final = ChatCompletionToolParamFunctionChunk(
         name=name,

litellm/responses/litellm_completion_transformation/handler.py

@@ -6,6 +6,7 @@
 from typing import Final
 
 import litellm
+from litellm.responses.additional_tools import hoist_additional_tools
 from litellm.responses.litellm_completion_transformation.streaming_iterator import (
     LiteLLMCompletionStreamingIterator,
 )
@@ -37,11 +38,16 @@ def response_api_handler(
         | BaseResponsesAPIStreamingIterator
         | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
     ):
+        hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools"))
+        bridged_input: Final = hoisted.input
+        bridged_request: Final[ResponsesAPIOptionalRequestParams] = (
+            {**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request
+        )
         litellm_completion_request: Final[dict] = (
             LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
                 model=model,
-                input=input,
-                responses_api_request=responses_api_request,
+                input=bridged_input,
+                responses_api_request=bridged_request,
                 custom_llm_provider=custom_llm_provider,
                 stream=stream,
                 extra_headers=extra_headers,
@@ -52,8 +58,8 @@ def response_api_handler(
         if _is_async:
             return self.async_response_api_handler(
                 litellm_completion_request=litellm_completion_request,
-                request_input=input,
-                responses_api_request=responses_api_request,
+                request_input=bridged_input,
+                responses_api_request=bridged_request,
                 **kwargs,
             )
 
@@ -70,8 +76,8 @@ def response_api_handler(
             responses_api_response: Final[ResponsesAPIResponse] = (
                 LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
                     chat_completion_response=litellm_completion_response,
-                    request_input=input,
-                    responses_api_request=responses_api_request,
+                    request_input=bridged_input,
+                    responses_api_request=bridged_request,
                 )
             )
 
@@ -81,8 +87,8 @@ def response_api_handler(
             return LiteLLMCompletionStreamingIterator(
                 model=model,
                 litellm_custom_stream_wrapper=litellm_completion_response,
-                request_input=input,
-                responses_api_request=responses_api_request,
+                request_input=bridged_input,
+                responses_api_request=bridged_request,
                 custom_llm_provider=custom_llm_provider,
                 litellm_metadata=kwargs.get("litellm_metadata", {}),
             )