From af16f12117d5e55d78bc2c4bdd86f9bf5deba34d Mon Sep 17 00:00:00 2001 From: Nicholas Gigliotti Date: Sat, 14 Mar 2026 00:34:50 -0400 Subject: [PATCH 1/2] feat(bedrock): support native structured outputs for Invoke API (Claude 4.5+) For Bedrock InvokeModel Claude models that support native structured outputs (Haiku 4.5, Sonnet 4.5, Opus 4.5, Opus 4.6), use output_config.format with json_schema instead of the synthetic json_tool_call workaround. Unsupported models automatically fall back to the existing tool-call approach. Completes the Invoke API portion of #21208 (Converse was merged in #21222). --- .../anthropic_claude3_transformation.py | 93 ++-- .../test_bedrock_completion.py | 382 +++++---------- ...ations_anthropic_claude3_transformation.py | 457 +++++++++++++----- 3 files changed, 540 insertions(+), 392 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 7936b6ea644..51b1dac0467 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -3,6 +3,9 @@ import httpx from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, +) from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) @@ -21,6 +24,18 @@ else: LiteLLMLoggingObj = Any +# Anthropic Claude models that support native structured outputs on Bedrock InvokeModel. +# Maintained separately from the Converse path's BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS +# because Invoke and Converse have independent feature rollouts. +# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html +BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS = { + "claude-haiku-4-5", + "claude-sonnet-4-5", + "claude-sonnet-4-6", + "claude-opus-4-5", + "claude-opus-4-6", +} + class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): """ @@ -49,6 +64,11 @@ def custom_llm_provider(self) -> Optional[str]: def get_supported_openai_params(self, model: str) -> List[str]: return AnthropicConfig.get_supported_openai_params(self, model) + @staticmethod + def _supports_native_structured_outputs(model: str) -> bool: + """Check if the Bedrock Invoke model supports native structured outputs.""" + return any(substring in model for substring in BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS) + def map_openai_params( self, non_default_params: dict, @@ -56,15 +76,30 @@ def map_openai_params( model: str, drop_params: bool, ) -> dict: - # Force tool-based structured outputs for Bedrock Invoke - # (similar to VertexAI fix in #19201) - # Bedrock Invoke doesn't support output_format parameter - original_model = model - if "response_format" in non_default_params: - # Use a model name that forces tool-based approach + response_format = non_default_params.get("response_format") + + # Native path: build output_format directly for Bedrock-supported models + # (includes haiku-4-5 which the Anthropic parent doesn't know about). + if isinstance(response_format, dict) and self._supports_native_structured_outputs(model): + _output_format = self.map_response_format_to_anthropic_output_format(response_format) + if _output_format is not None: + optional_params["output_format"] = _output_format + optional_params["json_mode"] = True + remaining = {k: v for k, v in non_default_params.items() if k != "response_format"} + return AnthropicConfig.map_openai_params( + self, + remaining, + optional_params, + model, + drop_params, + ) + + # Fallback: force tool-based structured outputs for unsupported models + # (or json_object without schema on a supported model). + if response_format is not None: model = "claude-3-sonnet-20240229" - optional_params = AnthropicConfig.map_openai_params( + return AnthropicConfig.map_openai_params( self, non_default_params, optional_params, @@ -72,11 +107,6 @@ def map_openai_params( drop_params, ) - # Restore original model name - model = original_model - - return optional_params - def transform_request( self, model: str, @@ -87,11 +117,7 @@ def transform_request( ) -> dict: # Filter out AWS authentication parameters before passing to Anthropic transformation # AWS params should only be used for signing requests, not included in request body - filtered_params = { - k: v - for k, v in optional_params.items() - if k not in self.aws_authentication_params - } + filtered_params = {k: v for k, v in optional_params.items() if k not in self.aws_authentication_params} filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params) _anthropic_request = AnthropicConfig.transform_request( @@ -105,11 +131,26 @@ def transform_request( _anthropic_request.pop("model", None) _anthropic_request.pop("stream", None) - # Bedrock Invoke doesn't support output_format parameter - _anthropic_request.pop("output_format", None) - # Bedrock Invoke doesn't support output_config parameter - # Fixes: https://github.com/BerriAI/litellm/issues/22797 - _anthropic_request.pop("output_config", None) + + # Convert Anthropic output_format to Bedrock InvokeModel output_config.format + output_format = _anthropic_request.pop("output_format", None) + if output_format and isinstance(output_format, dict) and output_format.get("type") == "json_schema": + schema = output_format.get("schema", {}) + normalized_schema = AmazonConverseConfig._add_additional_properties_to_schema(schema) + # Preserve existing output_config keys (e.g. effort from reasoning_effort) + output_config = _anthropic_request.get("output_config") or {} + output_config["format"] = { + "type": "json_schema", + "schema": normalized_schema, + } + _anthropic_request["output_config"] = output_config + else: + # Non-native path: strip output_config entirely. + # Bedrock Invoke rejects the key itself (not just sub-keys) with + # "extraneous key [output_config] is not permitted" for models + # that don't support native structured outputs. + # Fixes: https://github.com/BerriAI/litellm/issues/22797 + _anthropic_request.pop("output_config", None) if "anthropic_version" not in _anthropic_request: _anthropic_request["anthropic_version"] = self.anthropic_version @@ -135,9 +176,7 @@ def transform_request( ) beta_set.update(auto_betas) - if tool_search_used and not ( - programmatic_tool_calling_used or input_examples_used - ): + if tool_search_used and not (programmatic_tool_calling_used or input_examples_used): beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER) if "opus-4" in model.lower() or "opus_4" in model.lower(): beta_set.add("tool-search-tool-2025-10-19") @@ -166,9 +205,7 @@ def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict: if tool_type == "tool_search_tool_regex_20251119": normalized_tool = tool.copy() normalized_tool["type"] = "tool_search_tool_regex" - normalized_tool["name"] = normalized_tool.get( - "name", "tool_search_tool_regex" - ) + normalized_tool["name"] = normalized_tool.get("name", "tool_search_tool_regex") normalized_tools.append(normalized_tool) continue normalized_tools.append(tool) diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index b71e4e51877..ad8c4bbdbe1 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -16,9 +16,7 @@ import os import json -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path from unittest.mock import AsyncMock, Mock, patch import pytest @@ -157,9 +155,9 @@ def test_completion_bedrock_guardrails(streaming): saw_trace = True print(chunk) - assert ( - saw_trace is True - ), "Did not see trace in response even when trace=enabled sent in the guardrailConfig" + assert saw_trace is True, ( + "Did not see trace in response even when trace=enabled sent in the guardrailConfig" + ) except RateLimitError: pass @@ -292,9 +290,7 @@ def bedrock_session_token_creds(): # For circle-ci testing # aws_role_name = os.environ["AWS_TEMP_ROLE_NAME"] # TODO: This is using ai.moda's IAM role, we should use LiteLLM's IAM role eventually - aws_role_name = ( - "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci" - ) + aws_role_name = "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci" aws_web_identity_token = "test-oidc-token-123" creds = bllm.get_credentials( @@ -664,15 +660,9 @@ def test_bedrock_claude_3_tool_calling(): print(f"response: {response}") # Add any assertions here to check the response assert isinstance(response.choices[0].message.tool_calls[0].function.name, str) - assert isinstance( - response.choices[0].message.tool_calls[0].function.arguments, str - ) - messages.append( - response.choices[0].message.model_dump() - ) # Add assistant tool invokes - tool_result = ( - '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}' - ) + assert isinstance(response.choices[0].message.tool_calls[0].function.arguments, str) + messages.append(response.choices[0].message.model_dump()) # Add assistant tool invokes + tool_result = '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}' # Add user submitted tool results in the OpenAI format messages.append( { @@ -704,9 +694,7 @@ def encode_image(image_path): return base64.b64encode(image_file.read()).decode("utf-8") -@pytest.mark.skip( - reason="we already test claude-3, this is just another way to pass images" -) +@pytest.mark.skip(reason="we already test claude-3, this is just another way to pass images") def test_completion_claude_3_base64(): try: litellm.set_verbose = True @@ -723,9 +711,7 @@ def test_completion_claude_3_base64(): {"type": "text", "text": "Whats in this image?"}, { "type": "image_url", - "image_url": { - "url": "data:image/jpeg;base64," + base64_image - }, + "image_url": {"url": "data:image/jpeg;base64," + base64_image}, }, ], } @@ -790,9 +776,7 @@ def test_bedrock_ptu(): litellm.set_verbose = True from openai.types.chat import ChatCompletion - model_id = ( - "arn:aws:bedrock:us-west-2:888602223428:provisioned-model/8fxff74qyhs3" - ) + model_id = "arn:aws:bedrock:us-west-2:888602223428:provisioned-model/8fxff74qyhs3" try: response = litellm.completion( model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", @@ -843,10 +827,7 @@ async def test_bedrock_custom_api_base(): ) assert "test" in mock_client_post.call_args.kwargs["headers"] assert mock_client_post.call_args.kwargs["headers"]["test"] == "hello world" - assert ( - mock_client_post.call_args.kwargs["headers"]["Authorization"] - == "my-test-key" - ) + assert mock_client_post.call_args.kwargs["headers"]["Authorization"] == "my-test-key" mock_client_post.assert_called_once() @@ -881,10 +862,7 @@ async def test_bedrock_extra_headers(model): print(f"mock_client_post.call_args.kwargs: {mock_client_post.call_args.kwargs}") assert "test" in mock_client_post.call_args.kwargs["headers"] assert mock_client_post.call_args.kwargs["headers"]["test"] == "hello world" - assert ( - mock_client_post.call_args.kwargs["headers"]["Authorization"] - == "my-test-key" - ) + assert mock_client_post.call_args.kwargs["headers"]["Authorization"] == "my-test-key" mock_client_post.assert_called_once() @@ -1023,10 +1001,7 @@ def test_bedrock_tool_calling(): for tool_call in _choice_1.message.tool_calls: _tool_Call_name = tool_call.function.name if _tool_Call_name is not None and "DoSomethingVeryCool" in _tool_Call_name: - assert ( - _tool_Call_name - == "-DoSomethingVeryCool-forLitellm_Testin999229291-0293993" - ) + assert _tool_Call_name == "-DoSomethingVeryCool-forLitellm_Testin999229291-0293993" def test_bedrock_tools_pt_valid_names(): @@ -1149,12 +1124,8 @@ def test_bedrock_tools_transformation_valid_params(): toolJsonSchema = result[0]["toolSpec"]["inputSchema"]["json"] assert toolJsonSchema is not None print("transformed toolJsonSchema keys=", toolJsonSchema.keys()) - print( - "allowed ToolJsonSchemaBlock keys=", ToolJsonSchemaBlock.__annotations__.keys() - ) - assert set(toolJsonSchema.keys()).issubset( - set(ToolJsonSchemaBlock.__annotations__.keys()) - ) + print("allowed ToolJsonSchemaBlock keys=", ToolJsonSchemaBlock.__annotations__.keys()) + assert set(toolJsonSchema.keys()).issubset(set(ToolJsonSchemaBlock.__annotations__.keys())) assert isinstance(result, list) assert len(result) == 1 @@ -1163,10 +1134,7 @@ def test_bedrock_tools_transformation_valid_params(): assert result[0]["toolSpec"]["description"] == "Invalid name test" assert "inputSchema" in result[0]["toolSpec"] assert "json" in result[0]["toolSpec"]["inputSchema"] - assert ( - result[0]["toolSpec"]["inputSchema"]["json"]["properties"]["test"]["type"] - == "string" - ) + assert result[0]["toolSpec"]["inputSchema"]["json"]["properties"]["test"]["type"] == "string" assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"] @@ -1243,24 +1211,18 @@ def test_bedrock_converse_translation_tool_message(): }, ] - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") print(translated_msg) assert translated_msg == [ { "role": "user", "content": [ - { - "text": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses" - }, + {"text": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses"}, { "toolResult": { "content": [ - { - "text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}' - } + {"text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}'} ], "toolUseId": "tooluse_DnqEmD5qR6y2-aJ-Xd05xw", } @@ -1286,11 +1248,7 @@ def test_base_aws_llm_get_credentials(): credentials = session.get_credentials().get_frozen_credentials() end_time = time.time() - print( - "Total time for credentials - {}. Credentials - {}".format( - end_time - start_time, credentials - ) - ) + print("Total time for credentials - {}. Credentials - {}".format(end_time - start_time, credentials)) start_time = time.time() credentials = BaseAWSLLM().get_credentials( @@ -1591,20 +1549,14 @@ def test_bedrock_completion_test_3(): }, ] - transformed_messages = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + transformed_messages = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") print(transformed_messages) assert transformed_messages[-1]["role"] == "user" assert transformed_messages[-1]["content"] == [ { "toolResult": { - "content": [ - { - "text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}' - } - ], + "content": [{"text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}'}], "toolUseId": "tooluse_EF8PwJ1dSMSh6tLGKu9VdA", } } @@ -1840,9 +1792,7 @@ def test_bedrock_completion_test_4(modify_params): } if modify_params: - transformed_messages = _bedrock_converse_messages_pt( - messages=data["messages"], model="", llm_provider="" - ) + transformed_messages = _bedrock_converse_messages_pt(messages=data["messages"], model="", llm_provider="") expected_messages = [ { "role": "user", @@ -1942,9 +1892,7 @@ def test_bedrock_base_model_helper(): assert base_model == "amazon.nova-pro-v1:0" assert ( - BedrockModelInfo.get_base_model( - "invoke/anthropic.claude-3-5-sonnet-20241022-v2:0" - ) + BedrockModelInfo.get_base_model("invoke/anthropic.claude-3-5-sonnet-20241022-v2:0") == "anthropic.claude-3-5-sonnet-20241022-v2:0" ) @@ -1975,9 +1923,7 @@ def test_bedrock_route_detection(model, expected_route): from litellm.llms.bedrock.common_utils import BedrockModelInfo route = BedrockModelInfo.get_bedrock_route(model) - assert ( - route == expected_route - ), f"Expected route '{expected_route}' for model '{model}', but got '{route}'" + assert route == expected_route, f"Expected route '{expected_route}' for model '{model}', but got '{route}'" @pytest.mark.parametrize( @@ -2047,9 +1993,7 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control): ], ) def test_bedrock_supports_tool_call(model, expected_supports_tool_call): - supported_openai_params = ( - litellm.AmazonConverseConfig().get_supported_openai_params(model=model) - ) + supported_openai_params = litellm.AmazonConverseConfig().get_supported_openai_params(model=model) if expected_supports_tool_call: assert "tools" in supported_openai_params else: @@ -2197,10 +2141,7 @@ def test_bedrock_empty_content_handling(messages, continue_message_index): # Verify assistant message with default text was inserted assert formatted_messages[0]["role"] == "user" assert formatted_messages[1]["role"] == "assistant" - assert ( - formatted_messages[continue_message_index]["content"][0]["text"] - == "Please continue." - ) + assert formatted_messages[continue_message_index]["content"][0]["text"] == "Please continue." def test_bedrock_custom_continue_message(): @@ -2249,9 +2190,7 @@ def test_bedrock_no_default_message(): ) # Verify empty message is replaced with placeholder and valid message remains - assistant_messages = [ - msg for msg in formatted_messages if msg["role"] == "assistant" - ] + assistant_messages = [msg for msg in formatted_messages if msg["role"] == "assistant"] assert len(assistant_messages) == 1 assert assistant_messages[0]["content"][0]["text"] == "Valid response" @@ -2273,18 +2212,13 @@ def mock_transform(*args, **kwargs): captured_data = result return result - with patch( - "litellm.AmazonConverseConfig._transform_request", side_effect=mock_transform - ): + with patch("litellm.AmazonConverseConfig._transform_request", side_effect=mock_transform): litellm.completion(**data) # Assert that additionalRequestParameters exists and contains topK assert "additionalModelRequestFields" in captured_data assert "inferenceConfig" in captured_data["additionalModelRequestFields"] - assert ( - captured_data["additionalModelRequestFields"]["inferenceConfig"]["topK"] - == 10 - ) + assert captured_data["additionalModelRequestFields"]["inferenceConfig"]["topK"] == 10 def test_bedrock_cross_region_inference(monkeypatch): @@ -2344,17 +2278,13 @@ def test_bedrock_process_empty_text_blocks(): assert modified_message["content"][0]["text"] == "Please continue." -@pytest.mark.skip( - reason="Skipping test due to bedrock changing their response schema support. Come back to this." -) +@pytest.mark.skip(reason="Skipping test due to bedrock changing their response schema support. Come back to this.") def test_nova_optional_params_tool_choice(): try: litellm.drop_params = True litellm.set_verbose = True litellm.completion( - messages=[ - {"role": "user", "content": "A WWII competitive game for 4-8 players"} - ], + messages=[{"role": "user", "content": "A WWII competitive game for 4-8 players"}], model="bedrock/us.amazon.nova-pro-v1:0", temperature=0.3, tools=[ @@ -2444,9 +2374,7 @@ def test_bedrock_image_embedding_transformation(self): "inference_params": {}, } - transformed_request = ( - AmazonTitanMultimodalEmbeddingG1Config()._transform_request(**args) - ) + transformed_request = AmazonTitanMultimodalEmbeddingG1Config()._transform_request(**args) transformed_request[ "inputImage" ] == "iVBORw0KGgoAAAANSUhEUgAAAGQAAABkBAMAAACCzIhnAAAAG1BMVEURAAD///+ln5/h39/Dv79qX18uHx+If39MPz9oMSdmAAAACXBIWXMAAA7EAAAOxAGVKw4bAAABB0lEQVRYhe2SzWrEIBCAh2A0jxEs4j6GLDS9hqWmV5Flt0cJS+lRwv742DXpEjY1kOZW6HwHFZnPmVEBEARBEARB/jd0KYA/bcUYbPrRLh6amXHJ/K+ypMoyUaGthILzw0l+xI0jsO7ZcmCcm4ILd+QuVYgpHOmDmz6jBeJImdcUCmeBqQpuqRIbVmQsLCrAalrGpfoEqEogqbLTWuXCPCo+Ki1XGqgQ+jVVuhB8bOaHkvmYuzm/b0KYLWwoK58oFqi6XfxQ4Uz7d6WeKpna6ytUs5e8betMcqAv5YPC5EZB2Lm9FIn0/VP6R58+/GEY1X1egVoZ/3bt/EqF6malgSAIgiDIH+QL41409QMY0LMAAAAASUVORK5CYII=" @@ -2510,9 +2438,7 @@ def test_bedrock_error_handling_streaming(): } ) - decoder = AWSEventStreamDecoder( - model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0" - ) + decoder = AWSEventStreamDecoder(model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0") with pytest.raises(Exception) as e: decoder._parse_message_from_event(event) assert isinstance(e.value, BedrockError) @@ -2591,9 +2517,7 @@ def test_bedrock_custom_deepseek(): with patch.object(client, "post") as mock_post: # Mock the response mock_response = Mock() - mock_response.text = json.dumps( - {"generation": "Here's a joke...", "stop_reason": "stop"} - ) + mock_response.text = json.dumps({"generation": "Here's a joke...", "stop_reason": "stop"}) mock_response.status_code = 200 # Add required response attributes mock_response.headers = {"Content-Type": "application/json"} @@ -2641,14 +2565,8 @@ def test_bedrock_custom_deepseek(): ], ) def test_handle_top_k_value_helper(model, expected_output): - assert ( - litellm.AmazonConverseConfig()._handle_top_k_value(model, {"topK": 3}) - == expected_output - ) - assert ( - litellm.AmazonConverseConfig()._handle_top_k_value(model, {"top_k": 3}) - == expected_output - ) + assert litellm.AmazonConverseConfig()._handle_top_k_value(model, {"topK": 3}) == expected_output + assert litellm.AmazonConverseConfig()._handle_top_k_value(model, {"top_k": 3}) == expected_output @pytest.mark.parametrize( @@ -2669,9 +2587,7 @@ def test_bedrock_top_k_param(model, expected_params): mock_response = Mock() if "mistral" in model: - mock_response.text = json.dumps( - {"outputs": [{"text": "Here's a joke...", "stop_reason": "stop"}]} - ) + mock_response.text = json.dumps({"outputs": [{"text": "Here's a joke...", "stop_reason": "stop"}]}) else: mock_response.text = json.dumps( { @@ -2713,9 +2629,7 @@ def test_bedrock_invoke_provider(): == "anthropic" ) assert ( - litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( - "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" - ) + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0") == "anthropic" ) assert ( @@ -2724,40 +2638,12 @@ def test_bedrock_invoke_provider(): ) == "llama" ) - assert ( - litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( - "us.amazon.nova-pro-v1:0" - ) - == "nova" - ) - assert ( - litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-pro-v1:0") - == "nova" - ) - assert ( - litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( - "amazon.nova-lite-v1:0" - ) - == "nova" - ) - assert ( - litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( - "amazon.nova-micro-v1:0" - ) - == "nova" - ) - assert ( - litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( - "amazon.nova-premier-v1:0" - ) - == "nova" - ) - assert ( - litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( - "amazon.nova-2-lite-v1:0" - ) - == "nova" - ) + assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("us.amazon.nova-pro-v1:0") == "nova" + assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-pro-v1:0") == "nova" + assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-lite-v1:0") == "nova" + assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-micro-v1:0") == "nova" + assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-premier-v1:0") == "nova" + assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-2-lite-v1:0") == "nova" def test_bedrock_description_param(): @@ -2770,9 +2656,7 @@ def test_bedrock_description_param(): try: response = completion( model="bedrock/us.amazon.nova-pro-v1:0", - messages=[ - {"role": "user", "content": "What is the meaning of this poem?"} - ], + messages=[{"role": "user", "content": "What is the meaning of this poem?"}], response_format={ "type": "json_schema", "json_schema": { @@ -2794,9 +2678,7 @@ def test_bedrock_description_param(): request_body_str = json.dumps(request_body, indent=4, default=str) print("request_body=", request_body_str) - assert ( - "Find the meaning inside a poem" in request_body_str - ) # assert description is passed + assert "Find the meaning inside a poem" in request_body_str # assert description is passed @pytest.mark.parametrize( @@ -2853,10 +2735,7 @@ async def test_bedrock_thinking_in_assistant_message(sync_mode): print(mock_post.call_args.kwargs) json_data = mock_post.call_args.kwargs["data"] - assert ( - "Alright, let's get started with resolving this issue about implementing" - in json_data - ) + assert "Alright, let's get started with resolving this issue about implementing" in json_data @pytest.mark.asyncio @@ -2900,32 +2779,20 @@ async def test_bedrock_stream_thinking_content_openwebui(): # Assert there's exactly one opening and closing tag assert think_open_pos >= 0, "Opening tag not found" assert think_close_pos > 0, "Closing tag not found" - assert ( - content.count("") == 1 - ), "There should be exactly one opening tag" - assert ( - content.count("") == 1 - ), "There should be exactly one closing tag" + assert content.count("") == 1, "There should be exactly one opening tag" + assert content.count("") == 1, "There should be exactly one closing tag" # Assert the opening tag comes before the closing tag - assert ( - think_open_pos < think_close_pos - ), "Opening tag should come before closing tag" + assert think_open_pos < think_close_pos, "Opening tag should come before closing tag" # Assert there's content between the tags thinking_content = content[think_open_pos + 7 : think_close_pos] - assert ( - len(thinking_content.strip()) > 0 - ), "There should be content between thinking tags" + assert len(thinking_content.strip()) > 0, "There should be content between thinking tags" # Assert there's content after the closing tag - assert ( - len(content) > think_close_pos + 8 - ), "There should be content after the thinking tags" + assert len(content) > think_close_pos + 8, "There should be content after the thinking tags" response_content = content[think_close_pos + 8 :].strip() - assert ( - len(response_content) > 0 - ), "There should be non-empty content after thinking tags" + assert len(response_content) > 0, "There should be non-empty content after thinking tags" def test_bedrock_application_inference_profile(): @@ -2958,9 +2825,7 @@ def test_bedrock_application_inference_profile(): } ] - with patch.object(client, "post") as mock_post, patch.object( - client2, "post" - ) as mock_post2: + with patch.object(client, "post") as mock_post, patch.object(client2, "post") as mock_post2: try: resp = completion( model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", @@ -2986,9 +2851,7 @@ def test_bedrock_application_inference_profile(): mock_post2.assert_called_once() print(mock_post.call_args.kwargs) json_data = mock_post.call_args.kwargs["data"] - assert mock_post.call_args.kwargs["url"].startswith( - "https://bedrock-runtime.eu-central-1.amazonaws.com/" - ) + assert mock_post.call_args.kwargs["url"].startswith("https://bedrock-runtime.eu-central-1.amazonaws.com/") assert mock_post2.call_args.kwargs["url"] == mock_post.call_args.kwargs["url"] @@ -3121,9 +2984,7 @@ async def test_bedrock_passthrough(sync_mode: bool): } ], "temperature": 0, - "metadata": { - "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84" - }, + "metadata": {"user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"}, "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": ["claude-code-20250219"], } @@ -3182,9 +3043,7 @@ async def test_bedrock_passthrough_router(): } ], "temperature": 0, - "metadata": { - "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84" - }, + "metadata": {"user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"}, "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": ["claude-code-20250219"], } @@ -3220,11 +3079,7 @@ class MockCustomLogger(CustomLogger): "messages": [ { "role": "user", - "content": [ - { - "text": "Write an article about impact of high inflation to GDP of a country" - } - ], + "content": [{"text": "Write an article about impact of high inflation to GDP of a country"}], } ], "system": [{"text": "You are an economist with access to lots of data"}], @@ -3276,9 +3131,7 @@ class MockCustomLogger(CustomLogger): } ], "temperature": 0, - "metadata": { - "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84" - }, + "metadata": {"user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"}, "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": ["claude-code-20250219"], } @@ -3328,9 +3181,7 @@ class MockCustomLogger(CustomLogger): } ], "temperature": 0, - "metadata": { - "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84" - }, + "metadata": {"user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"}, "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": ["claude-code-20250219"], } @@ -3412,9 +3263,7 @@ def test_bedrock_openai_imported_model(): url = mock_post.call_args.kwargs["url"] print(f"URL: {url}") assert "bedrock-runtime.us-east-1.amazonaws.com" in url - assert ( - "arn:aws:bedrock:us-east-1:117159858402:imported-model%2Fm4gc1mrfuddy" in url - ) + assert "arn:aws:bedrock:us-east-1:117159858402:imported-model%2Fm4gc1mrfuddy" in url assert "/invoke" in url # Validate request body follows OpenAI format @@ -3443,9 +3292,7 @@ def test_bedrock_openai_imported_model(): # Check image_url content assert user_msg["content"][1]["type"] == "image_url" assert "image_url" in user_msg["content"][1] - assert user_msg["content"][1]["image_url"]["url"].startswith( - "data:image/jpeg;base64," - ) + assert user_msg["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,") assert user_msg["content"][2]["type"] == "image_url" assert "image_url" in user_msg["content"][2] @@ -3481,9 +3328,7 @@ def test_bedrock_nova_provider_detection(): for model, expected in nova_test_cases: provider = BaseAWSLLM.get_bedrock_invoke_provider(model) - assert ( - provider == expected - ), f"Failed for model: {model}, expected: {expected}, got: {provider}" + assert provider == expected, f"Failed for model: {model}, expected: {expected}, got: {provider}" # Verify that Amazon Titan models still return "amazon" titan_test_cases = [ @@ -3493,9 +3338,7 @@ def test_bedrock_nova_provider_detection(): for model, expected in titan_test_cases: provider = BaseAWSLLM.get_bedrock_invoke_provider(model) - assert ( - provider == expected - ), f"Failed for model: {model}, expected: {expected}, got: {provider}" + assert provider == expected, f"Failed for model: {model}, expected: {expected}, got: {provider}" def test_bedrock_openai_provider_detection(): @@ -3512,9 +3355,7 @@ def test_bedrock_openai_provider_detection(): for model in test_cases: provider = BaseAWSLLM.get_bedrock_invoke_provider(model) - assert ( - provider == "openai" - ), f"Failed for model: {model}, got provider: {provider}" + assert provider == "openai", f"Failed for model: {model}, got provider: {provider}" print(f"✓ Provider detection works for: {model}") @@ -3524,14 +3365,10 @@ def test_bedrock_openai_model_id_extraction(): """ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM - model = ( - "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-model-123" - ) + model = "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-model-123" provider = BaseAWSLLM.get_bedrock_invoke_provider(model) - model_id = BaseAWSLLM.get_bedrock_model_id( - model=model, provider=provider, optional_params={} - ) + model_id = BaseAWSLLM.get_bedrock_model_id(model=model, provider=provider, optional_params={}) # The ARN should be double URL encoded assert "arn" in model_id @@ -3718,27 +3555,15 @@ def test_bedrock_openai_explicit_route_check(): from litellm.llms.bedrock.common_utils import BedrockModelInfo # Test with openai/ prefix + assert BedrockModelInfo._explicit_openai_route("openai/arn:aws:bedrock:us-east-1:123:imported-model/test") is True assert ( - BedrockModelInfo._explicit_openai_route( - "openai/arn:aws:bedrock:us-east-1:123:imported-model/test" - ) - is True - ) - assert ( - BedrockModelInfo._explicit_openai_route( - "bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test" - ) + BedrockModelInfo._explicit_openai_route("bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test") is True ) # Test without openai/ prefix assert BedrockModelInfo._explicit_openai_route("anthropic.claude-3-sonnet") is False - assert ( - BedrockModelInfo._explicit_openai_route( - "arn:aws:bedrock:us-east-1:123:imported-model/test" - ) - is False - ) + assert BedrockModelInfo._explicit_openai_route("arn:aws:bedrock:us-east-1:123:imported-model/test") is False print("✓ Explicit route check works correctly") @@ -3850,10 +3675,12 @@ def test_bedrock_openai_error_handling(): assert exc_info.value.status_code == 422 print("✓ Error handling works correctly") + # ============================================================================ # Nova Grounding (web_search_options) Unit Tests (Mocked) # ============================================================================ + def test_bedrock_nova_grounding_web_search_options_non_streaming(): """ Unit test for Nova grounding using web_search_options parameter (non-streaming). @@ -4094,8 +3921,8 @@ def test_bedrock_nova_grounding_request_transformation(): json=lambda: { "output": {"message": {"role": "assistant", "content": [{"text": "Test"}]}}, "stopReason": "end_turn", - "usage": {"inputTokens": 10, "outputTokens": 5} - } + "usage": {"inputTokens": 10, "outputTokens": 5}, + }, ) try: @@ -4131,3 +3958,60 @@ def test_bedrock_nova_grounding_request_transformation(): assert system_tool_found, "systemTool with nova_grounding should be present" print("✓ web_search_options correctly transformed to systemTool") + + +@pytest.mark.parametrize( + "model", + [ + "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "bedrock/invoke/us.anthropic.claude-sonnet-4-6", + "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", + "bedrock/invoke/us.anthropic.claude-opus-4-6-v1", + ], +) +def test_bedrock_invoke_native_structured_output(model): + """ + Integration test: verify native structured outputs via Bedrock InvokeModel + for each supported Claude model. + + Uses output_config.format (Bedrock InvokeModel native API) instead of + the synthetic json_tool_call workaround. + """ + response = completion( + model=model, + messages=[ + { + "role": "user", + "content": "Classify the sentiment of 'I love this product!' as positive, negative, or neutral.", + } + ], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "sentiment", + "schema": { + "type": "object", + "properties": { + "sentiment": { + "type": "string", + "enum": ["positive", "negative", "neutral"], + } + }, + "required": ["sentiment"], + "additionalProperties": False, + }, + }, + }, + max_tokens=100, + ) + print(f"Response for {model}: {response}") + + # Validate the response is a ModelResponse with valid JSON content + assert isinstance(response, ModelResponse) + content = response.choices[0].message.content + assert content is not None + + parsed = json.loads(content) + assert "sentiment" in parsed + assert parsed["sentiment"] in ["positive", "negative", "neutral"] diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index cb05531c2f8..551b3789217 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -25,27 +25,24 @@ def test_get_supported_params_thinking(): def test_aws_params_filtered_from_request_body(): """ Test that AWS authentication parameters are filtered out from the request body. - + This is a security test to ensure AWS credentials are not leaked in the request body sent to Bedrock. AWS params should only be used for request signing. - - Regression test for: AWS params (aws_role_name, aws_session_name, etc.) + + Regression test for: AWS params (aws_role_name, aws_session_name, etc.) being included in the Bedrock InvokeModel request body. """ config = AmazonAnthropicClaudeConfig() - + # Test messages - messages = [ - {"role": "user", "content": "Hello, how are you?"} - ] - + messages = [{"role": "user", "content": "Hello, how are you?"}] + # Optional params with AWS authentication parameters that should be filtered out optional_params = { # Regular Anthropic params - these SHOULD be in the request "max_tokens": 100, "temperature": 0.7, "top_p": 0.9, - # AWS authentication params - these should NOT be in the request body "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", @@ -59,7 +56,7 @@ def test_aws_params_filtered_from_request_body(): "aws_bedrock_runtime_endpoint": "https://bedrock-runtime.us-west-2.amazonaws.com", "aws_external_id": "external-id-123", } - + # Transform the request result = config.transform_request( model="anthropic.claude-haiku-4-5-20251001-v1:0", @@ -68,39 +65,69 @@ def test_aws_params_filtered_from_request_body(): litellm_params={}, headers={}, ) - + # Convert result to JSON string to check what would be sent in the request result_json = json.dumps(result) - + # Verify AWS authentication params are NOT in the request body - assert "aws_access_key_id" not in result_json, "AWS access key should not be in request body" - assert "aws_secret_access_key" not in result_json, "AWS secret key should not be in request body" - assert "aws_session_token" not in result_json, "AWS session token should not be in request body" - assert "aws_region_name" not in result_json, "AWS region should not be in request body" - assert "aws_role_name" not in result_json, "AWS role name should not be in request body" - assert "aws_session_name" not in result_json, "AWS session name should not be in request body" - assert "aws_profile_name" not in result_json, "AWS profile name should not be in request body" - assert "aws_web_identity_token" not in result_json, "AWS web identity token should not be in request body" - assert "aws_sts_endpoint" not in result_json, "AWS STS endpoint should not be in request body" - assert "aws_bedrock_runtime_endpoint" not in result_json, "AWS bedrock endpoint should not be in request body" - assert "aws_external_id" not in result_json, "AWS external ID should not be in request body" - + assert ( + "aws_access_key_id" not in result_json + ), "AWS access key should not be in request body" + assert ( + "aws_secret_access_key" not in result_json + ), "AWS secret key should not be in request body" + assert ( + "aws_session_token" not in result_json + ), "AWS session token should not be in request body" + assert ( + "aws_region_name" not in result_json + ), "AWS region should not be in request body" + assert ( + "aws_role_name" not in result_json + ), "AWS role name should not be in request body" + assert ( + "aws_session_name" not in result_json + ), "AWS session name should not be in request body" + assert ( + "aws_profile_name" not in result_json + ), "AWS profile name should not be in request body" + assert ( + "aws_web_identity_token" not in result_json + ), "AWS web identity token should not be in request body" + assert ( + "aws_sts_endpoint" not in result_json + ), "AWS STS endpoint should not be in request body" + assert ( + "aws_bedrock_runtime_endpoint" not in result_json + ), "AWS bedrock endpoint should not be in request body" + assert ( + "aws_external_id" not in result_json + ), "AWS external ID should not be in request body" + # Also check that the sensitive values themselves are not in the response - assert "AKIAIOSFODNN7EXAMPLE" not in result_json, "AWS access key value leaked in request body" - assert "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" not in result_json, "AWS secret key value leaked in request body" - assert "arn:aws:iam::123456789012:role/test-role" not in result_json, "AWS role ARN leaked in request body" + assert ( + "AKIAIOSFODNN7EXAMPLE" not in result_json + ), "AWS access key value leaked in request body" + assert ( + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" not in result_json + ), "AWS secret key value leaked in request body" + assert ( + "arn:aws:iam::123456789012:role/test-role" not in result_json + ), "AWS role ARN leaked in request body" assert "test-session" not in result_json, "AWS session name leaked in request body" - + # Verify normal params ARE still in the request body assert result["max_tokens"] == 100, "max_tokens should be in request body" assert result["temperature"] == 0.7, "temperature should be in request body" assert result["top_p"] == 0.9, "top_p should be in request body" - + # Verify Bedrock-specific params are added - assert result["anthropic_version"] == "bedrock-2023-05-31", "anthropic_version should be set" + assert ( + result["anthropic_version"] == "bedrock-2023-05-31" + ), "anthropic_version should be set" assert "model" not in result, "model should be removed for Bedrock Invoke API" assert "stream" not in result, "stream should be removed for Bedrock Invoke API" - + # Verify messages are present assert "messages" in result, "messages should be in request body" assert len(result["messages"]) == 1, "should have 1 message" @@ -109,41 +136,41 @@ def test_aws_params_filtered_from_request_body(): def test_output_format_conversion_to_inline_schema(): """ Test that output_format is converted to inline schema in message content for Bedrock Invoke. - + Bedrock Invoke doesn't support the output_format parameter, so LiteLLM converts it by embedding the schema directly into the user message content. """ from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) - + config = AmazonAnthropicClaudeMessagesConfig() - + # Test messages messages = [ - {"role": "user", "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan."} + { + "role": "user", + "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan.", + } ] - + # Output format with schema output_format_schema = { "type": "object", "properties": { "name": {"type": "string"}, "email": {"type": "string"}, - "plan_interest": {"type": "string"} + "plan_interest": {"type": "string"}, }, "required": ["name", "email", "plan_interest"], - "additionalProperties": False + "additionalProperties": False, } - + anthropic_messages_optional_request_params = { "max_tokens": 1024, - "output_format": { - "type": "json_schema", - "schema": output_format_schema - } + "output_format": {"type": "json_schema", "schema": output_format_schema}, } - + # Transform the request result = config.transform_anthropic_messages_request( model="anthropic.claude-sonnet-4-20250514-v1:0", @@ -152,27 +179,29 @@ def test_output_format_conversion_to_inline_schema(): litellm_params={}, headers={}, ) - + # Verify output_format was removed from the request - assert "output_format" not in result, "output_format should be removed from request body" - + assert ( + "output_format" not in result + ), "output_format should be removed from request body" + # Verify the schema was added to the last user message content assert "messages" in result last_user_message = result["messages"][0] assert last_user_message["role"] == "user" - + content = last_user_message["content"] assert isinstance(content, list), "content should be a list" assert len(content) == 2, "content should have 2 items (original text + schema)" - + # Check original text is preserved assert content[0]["type"] == "text" assert "John Smith" in content[0]["text"] - + # Check schema was added as JSON string assert content[1]["type"] == "text" schema_text = content[1]["text"] - + # Parse the schema JSON parsed_schema = json.loads(schema_text) assert parsed_schema["type"] == "object" @@ -180,7 +209,7 @@ def test_output_format_conversion_to_inline_schema(): assert "email" in parsed_schema["properties"] assert "plan_interest" in parsed_schema["properties"] assert parsed_schema["required"] == ["name", "email", "plan_interest"] - + # Verify other params are preserved assert result["max_tokens"] == 1024 assert result["anthropic_version"] == "bedrock-2023-05-31" @@ -193,29 +222,22 @@ def test_output_format_conversion_with_string_content(): from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) - + config = AmazonAnthropicClaudeMessagesConfig() - + # Test messages with string content - messages = [ - {"role": "user", "content": "What is 2+2?"} - ] - + messages = [{"role": "user", "content": "What is 2+2?"}] + output_format_schema = { "type": "object", - "properties": { - "result": {"type": "integer"} - } + "properties": {"result": {"type": "integer"}}, } - + anthropic_messages_optional_request_params = { "max_tokens": 100, - "output_format": { - "type": "json_schema", - "schema": output_format_schema - } + "output_format": {"type": "json_schema", "schema": output_format_schema}, } - + # Transform the request result = config.transform_anthropic_messages_request( model="anthropic.claude-sonnet-4-20250514-v1:0", @@ -224,17 +246,17 @@ def test_output_format_conversion_with_string_content(): litellm_params={}, headers={}, ) - + # Verify the content was converted to list format last_user_message = result["messages"][0] content = last_user_message["content"] assert isinstance(content, list), "content should be converted to list" assert len(content) == 2, "content should have 2 items" - + # Check original text assert content[0]["type"] == "text" assert content[0]["text"] == "What is 2+2?" - + # Check schema was added assert content[1]["type"] == "text" parsed_schema = json.loads(content[1]["text"]) @@ -248,21 +270,19 @@ def test_output_format_with_no_schema(): from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) - + config = AmazonAnthropicClaudeMessagesConfig() - - messages = [ - {"role": "user", "content": "Hello"} - ] - + + messages = [{"role": "user", "content": "Hello"}] + anthropic_messages_optional_request_params = { "max_tokens": 100, "output_format": { "type": "json_schema" # No schema field - } + }, } - + # Transform the request result = config.transform_anthropic_messages_request( model="anthropic.claude-sonnet-4-20250514-v1:0", @@ -271,11 +291,11 @@ def test_output_format_with_no_schema(): litellm_params={}, headers={}, ) - + # Verify output_format was removed but no schema was added assert "output_format" not in result last_user_message = result["messages"][0] - + # Content should remain as string (not converted to list) assert isinstance(last_user_message["content"], str) assert last_user_message["content"] == "Hello" @@ -289,9 +309,9 @@ def test_opus_4_5_model_detection(): from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) - + config = AmazonAnthropicClaudeMessagesConfig() - + # Test various Opus 4.5 naming patterns opus_4_5_models = [ "anthropic.claude-opus-4-5-20250514-v1:0", @@ -301,11 +321,10 @@ def test_opus_4_5_model_detection(): "us.anthropic.claude-opus-4-5-20250514-v1:0", "ANTHROPIC.CLAUDE-OPUS-4-5-20250514-V1:0", # Case insensitive ] - + for model in opus_4_5_models: - assert config._is_claude_opus_4_5(model), \ - f"Should detect {model} as Opus 4.5" - + assert config._is_claude_opus_4_5(model), f"Should detect {model} as Opus 4.5" + # Test non-Opus 4.5 models non_opus_4_5_models = [ "anthropic.claude-sonnet-4-5-20250929-v1:0", @@ -313,29 +332,30 @@ def test_opus_4_5_model_detection(): "anthropic.claude-opus-4-1-20250514-v1:0", # Opus 4.1, not 4.5 "anthropic.claude-haiku-4-5-20251001-v1:0", ] - + for model in non_opus_4_5_models: - assert not config._is_claude_opus_4_5(model), \ - f"Should not detect {model} as Opus 4.5" + assert not config._is_claude_opus_4_5( + model + ), f"Should not detect {model} as Opus 4.5" # def test_structured_outputs_beta_header_filtered_for_bedrock_invoke(): # """ # Test that unsupported beta headers are filtered out for Bedrock Invoke API. - + # Bedrock Invoke API only supports a specific whitelist of beta flags and returns # "invalid beta flag" error for others (e.g., structured-outputs, mcp-servers). # This test ensures unsupported headers are filtered while keeping supported ones. - + # Fixes: https://github.com/BerriAI/litellm/issues/16726 # """ # config = AmazonAnthropicClaudeConfig() - + # messages = [{"role": "user", "content": "test"}] - + # # Test 1: structured-outputs beta header (unsupported) # headers = {"anthropic-beta": "structured-outputs-2025-11-13"} - + # result = config.transform_request( # model="anthropic.claude-4-0-sonnet-20250514-v1:0", # messages=messages, @@ -343,15 +363,15 @@ def test_opus_4_5_model_detection(): # litellm_params={}, # headers=headers, # ) - + # # Verify structured-outputs beta is filtered out # anthropic_beta = result.get("anthropic_beta", []) # assert not any("structured-outputs" in beta for beta in anthropic_beta), \ # f"structured-outputs beta should be filtered, got: {anthropic_beta}" - + # # Test 2: mcp-servers beta header (unsupported - the main issue from #16726) # headers = {"anthropic-beta": "mcp-servers-2025-12-04"} - + # result = config.transform_request( # model="anthropic.claude-4-0-sonnet-20250514-v1:0", # messages=messages, @@ -359,15 +379,15 @@ def test_opus_4_5_model_detection(): # litellm_params={}, # headers=headers, # ) - + # # Verify mcp-servers beta is filtered out # anthropic_beta = result.get("anthropic_beta", []) # assert not any("mcp-servers" in beta for beta in anthropic_beta), \ # f"mcp-servers beta should be filtered, got: {anthropic_beta}" - + # # Test 3: Mix of supported and unsupported beta headers # headers = {"anthropic-beta": "computer-use-2024-10-22,mcp-servers-2025-12-04,structured-outputs-2025-11-13"} - + # result = config.transform_request( # model="anthropic.claude-4-0-sonnet-20250514-v1:0", # messages=messages, @@ -375,7 +395,7 @@ def test_opus_4_5_model_detection(): # litellm_params={}, # headers=headers, # ) - + # # Verify only supported betas are kept # anthropic_beta = result.get("anthropic_beta", []) # assert not any("structured-outputs" in beta for beta in anthropic_beta), \ @@ -413,9 +433,9 @@ def test_output_config_removed_from_bedrock_chat_invoke_request(): headers={}, ) - assert "output_config" not in result, ( - f"output_config should be stripped for Bedrock Chat Invoke, got keys: {list(result.keys())}" - ) + assert ( + "output_config" not in result + ), f"output_config should be stripped for Bedrock Chat Invoke, got keys: {list(result.keys())}" # Verify normal params survive assert result["max_tokens"] == 100 @@ -423,20 +443,18 @@ def test_output_config_removed_from_bedrock_chat_invoke_request(): def test_output_format_removed_from_bedrock_invoke_request(): """ Test that output_format parameter is removed from Bedrock Invoke requests. - + Bedrock Invoke API doesn't support the output_format parameter (only supported in Anthropic Messages API). This test ensures it's removed to prevent errors. """ config = AmazonAnthropicClaudeConfig() - + messages = [{"role": "user", "content": "test"}] - + # Create a request with output_format via map_openai_params - non_default_params = { - "response_format": {"type": "json_object"} - } + non_default_params = {"response_format": {"type": "json_object"}} optional_params = {} - + # This should trigger tool-based structured outputs optional_params = config.map_openai_params( non_default_params=non_default_params, @@ -444,7 +462,7 @@ def test_output_format_removed_from_bedrock_invoke_request(): model="anthropic.claude-4-0-sonnet-20250514-v1:0", drop_params=False, ) - + result = config.transform_request( model="anthropic.claude-4-0-sonnet-20250514-v1:0", messages=messages, @@ -452,7 +470,216 @@ def test_output_format_removed_from_bedrock_invoke_request(): litellm_params={}, headers={}, ) - + # Verify output_format is not in the request - assert "output_format" not in result, \ - f"output_format should be removed for Bedrock Invoke, got keys: {result.keys()}" + assert ( + "output_format" not in result + ), f"output_format should be removed for Bedrock Invoke, got keys: {result.keys()}" + + +# --------------------------------------------------------------------------- +# Native structured output tests +# --------------------------------------------------------------------------- + +SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT = { + "type": "json_schema", + "json_schema": { + "name": "contact_info", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"}, + "address": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "zip": {"type": "string"}, + }, + "required": ["city", "zip"], + }, + }, + "required": ["name", "email", "address"], + }, + }, +} + + +def test_supports_native_structured_outputs(): + """Verify model substring matching for native structured output support.""" + supported = [ + "anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "anthropic.claude-sonnet-4-6-20260301-v1:0", + "us.anthropic.claude-opus-4-5-20250514-v1:0", + "anthropic.claude-opus-4-6-20260101-v1:0", + ] + unsupported = [ + "anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic.claude-sonnet-4-20250514-v1:0", + "anthropic.claude-3-haiku-20240307-v1:0", + ] + for m in supported: + assert AmazonAnthropicClaudeConfig._supports_native_structured_outputs( + m + ), f"Expected {m} to be supported" + for m in unsupported: + assert not AmazonAnthropicClaudeConfig._supports_native_structured_outputs( + m + ), f"Expected {m} to be unsupported" + + +def test_native_structured_output_supported_model(): + """Supported model should produce output_format, not tools.""" + config = AmazonAnthropicClaudeConfig() + optional_params = config.map_openai_params( + non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT}, + optional_params={}, + model="anthropic.claude-sonnet-4-5-20250929-v1:0", + drop_params=False, + ) + + assert "output_format" in optional_params + assert optional_params["output_format"]["type"] == "json_schema" + assert optional_params.get("json_mode") is True + # Should NOT inject a synthetic tool + assert "tools" not in optional_params + assert "tool_choice" not in optional_params + + +def test_native_structured_output_unsupported_model(): + """Unsupported model should fall back to tool-call approach.""" + config = AmazonAnthropicClaudeConfig() + optional_params = config.map_openai_params( + non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT}, + optional_params={}, + model="anthropic.claude-3-5-sonnet-20241022-v2:0", + drop_params=False, + ) + + # Should use tool-based approach + assert "tools" in optional_params + assert "output_format" not in optional_params + + +def test_native_structured_output_haiku_4_5(): + """ + Haiku 4.5 is Bedrock-only (not on the direct Anthropic API), so it needs + special handling in the Bedrock invoke class rather than the parent. + """ + config = AmazonAnthropicClaudeConfig() + optional_params = config.map_openai_params( + non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT}, + optional_params={}, + model="anthropic.claude-haiku-4-5-20251001-v1:0", + drop_params=False, + ) + + assert "output_format" in optional_params + assert optional_params["output_format"]["type"] == "json_schema" + assert optional_params.get("json_mode") is True + assert "tools" not in optional_params + + +def test_native_structured_output_transform_request(): + """ + End-to-end: map_openai_params + transform_request should produce + output_config.format in the final request body. + """ + config = AmazonAnthropicClaudeConfig() + model = "anthropic.claude-sonnet-4-5-20250929-v1:0" + messages = [{"role": "user", "content": "Extract contact info."}] + + optional_params = config.map_openai_params( + non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT}, + optional_params={}, + model=model, + drop_params=False, + ) + + result = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # output_config.format should be present + assert ( + "output_config" in result + ), f"Expected output_config, got keys: {list(result.keys())}" + assert result["output_config"]["format"]["type"] == "json_schema" + assert "schema" in result["output_config"]["format"] + # output_format should NOT leak into the request + assert "output_format" not in result + + +def test_native_structured_output_schema_normalization(): + """additionalProperties: false should be recursively added to all object nodes.""" + config = AmazonAnthropicClaudeConfig() + model = "anthropic.claude-sonnet-4-5-20250929-v1:0" + messages = [{"role": "user", "content": "test"}] + + optional_params = config.map_openai_params( + non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT}, + optional_params={}, + model=model, + drop_params=False, + ) + + result = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + schema = result["output_config"]["format"]["schema"] + # Top-level object + assert schema.get("additionalProperties") is False + # Nested object (address) + assert schema["properties"]["address"].get("additionalProperties") is False + + +def test_native_structured_output_unsupported_model_no_output_config(): + """Unsupported model should NOT have output_config in the final request.""" + config = AmazonAnthropicClaudeConfig() + model = "anthropic.claude-3-5-sonnet-20241022-v2:0" + messages = [{"role": "user", "content": "test"}] + + optional_params = config.map_openai_params( + non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT}, + optional_params={}, + model=model, + drop_params=False, + ) + + result = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "output_config" not in result + assert "output_format" not in result + + +def test_native_structured_output_json_object_fallback(): + """ + json_object with no schema on a supported model should NOT produce + native output_format (there is no schema to constrain). The parent + also skips json_object-without-schema, so nothing is injected. + """ + config = AmazonAnthropicClaudeConfig() + optional_params = config.map_openai_params( + non_default_params={"response_format": {"type": "json_object"}}, + optional_params={}, + model="anthropic.claude-sonnet-4-5-20250929-v1:0", + drop_params=False, + ) + + assert "output_format" not in optional_params From f7287f065638c5bfd4be674efe10ee22d3448a1d Mon Sep 17 00:00:00 2001 From: Nicholas Gigliotti Date: Mon, 16 Mar 2026 18:10:57 -0400 Subject: [PATCH 2/2] refactor: extract schema helper to common_utils, clean up tests - Move _add_additional_properties_to_schema from AmazonConverseConfig to bedrock/common_utils.py as a shared top-level function. Both Converse and Invoke paths now import from the shared location, removing the cross-class private method dependency. - Remove 65-line commented-out test block (test_structured_outputs_beta_header_filtered_for_bedrock_invoke). - Update stale docstring on test_output_config_removed_from_bedrock_chat_invoke_request to reflect that native-path models now keep output_config. --- .../bedrock/chat/converse_transformation.py | 78 +++++++------------ .../anthropic_claude3_transformation.py | 45 ++++++++--- litellm/llms/bedrock/common_utils.py | 48 +++++++++++- ...ations_anthropic_claude3_transformation.py | 75 +----------------- 4 files changed, 108 insertions(+), 138 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 229457a73b4..bc4c1b53c2a 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -493,8 +493,7 @@ def _clamp_thinking_budget_tokens(optional_params: dict) -> None: budget = thinking.get("budget_tokens") if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS: verbose_logger.debug( - "Bedrock requires thinking.budget_tokens >= %d, got %d. " - "Clamping to minimum.", + "Bedrock requires thinking.budget_tokens >= %d, got %d. Clamping to minimum.", BEDROCK_MIN_THINKING_BUDGET_TOKENS, budget, ) @@ -778,39 +777,14 @@ def _add_additional_properties_to_schema(schema: dict) -> dict: Bedrock's native structured-outputs API requires this field to be explicitly set on every object node, otherwise it returns a validation error. - """ - if not isinstance(schema, dict): - return schema - - result = dict(schema) - if result.get("type") == "object" and "additionalProperties" not in result: - result["additionalProperties"] = False - - # Recurse into nested schemas - if "properties" in result and isinstance(result["properties"], dict): - result["properties"] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) - for k, v in result["properties"].items() - } - if "items" in result and isinstance(result["items"], dict): - result["items"] = AmazonConverseConfig._add_additional_properties_to_schema( - result["items"] - ) - for defs_key in ("$defs", "definitions"): - if defs_key in result and isinstance(result[defs_key], dict): - result[defs_key] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) - for k, v in result[defs_key].items() - } - for key in ("anyOf", "allOf", "oneOf"): - if key in result and isinstance(result[key], list): - result[key] = [ - AmazonConverseConfig._add_additional_properties_to_schema(item) - for item in result[key] - ] + Delegates to the shared implementation in ``bedrock/common_utils.py``. + """ + from litellm.llms.bedrock.common_utils import ( + add_additional_properties_to_schema, + ) - return result + return add_additional_properties_to_schema(schema) @staticmethod def _create_output_config_for_response_format( @@ -913,7 +887,9 @@ def map_openai_params( ) if param == "tool_choice": _tool_choice_value = self.map_tool_choice_values( - model=model, tool_choice=value, drop_params=drop_params # type: ignore + model=model, + tool_choice=value, + drop_params=drop_params, # type: ignore ) if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value @@ -1748,9 +1724,7 @@ def apply_tool_call_transformation_if_needed( return message, returned_finish_reason - def _translate_message_content( - self, content_blocks: List[ContentBlock] - ) -> Tuple[ + def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[ str, List[ChatCompletionToolCallChunk], Optional[List[BedrockConverseReasoningContentBlock]], @@ -1767,9 +1741,9 @@ def _translate_message_content( """ content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[ - List[BedrockConverseReasoningContentBlock] - ] = None + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( + None + ) citationsContentBlocks: Optional[List[CitationsContentBlock]] = None for idx, content in enumerate(content_blocks): """ @@ -1980,9 +1954,9 @@ def _transform_response( # noqa: PLR0915 chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[ - List[BedrockConverseReasoningContentBlock] - ] = None + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( + None + ) citationsContentBlocks: Optional[List[CitationsContentBlock]] = None if message is not None: @@ -2001,17 +1975,17 @@ def _transform_response( # noqa: PLR0915 provider_specific_fields["citationsContent"] = citationsContentBlocks if provider_specific_fields: - chat_completion_message[ - "provider_specific_fields" - ] = provider_specific_fields + chat_completion_message["provider_specific_fields"] = ( + provider_specific_fields + ) if reasoningContentBlocks is not None: - chat_completion_message[ - "reasoning_content" - ] = self._transform_reasoning_content(reasoningContentBlocks) - chat_completion_message[ - "thinking_blocks" - ] = self._transform_thinking_blocks(reasoningContentBlocks) + chat_completion_message["reasoning_content"] = ( + self._transform_reasoning_content(reasoningContentBlocks) + ) + chat_completion_message["thinking_blocks"] = ( + self._transform_thinking_blocks(reasoningContentBlocks) + ) chat_completion_message["content"] = content_str filtered_tools = self._filter_json_mode_tools( json_mode=json_mode, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 51b1dac0467..fe4d7a25cf2 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -3,13 +3,11 @@ import httpx from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.bedrock.chat.converse_transformation import ( - AmazonConverseConfig, -) from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import ( + add_additional_properties_to_schema, get_anthropic_beta_from_headers, remove_custom_field_from_tools, ) @@ -67,7 +65,10 @@ def get_supported_openai_params(self, model: str) -> List[str]: @staticmethod def _supports_native_structured_outputs(model: str) -> bool: """Check if the Bedrock Invoke model supports native structured outputs.""" - return any(substring in model for substring in BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS) + return any( + substring in model + for substring in BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS + ) def map_openai_params( self, @@ -80,12 +81,20 @@ def map_openai_params( # Native path: build output_format directly for Bedrock-supported models # (includes haiku-4-5 which the Anthropic parent doesn't know about). - if isinstance(response_format, dict) and self._supports_native_structured_outputs(model): - _output_format = self.map_response_format_to_anthropic_output_format(response_format) + if isinstance( + response_format, dict + ) and self._supports_native_structured_outputs(model): + _output_format = self.map_response_format_to_anthropic_output_format( + response_format + ) if _output_format is not None: optional_params["output_format"] = _output_format optional_params["json_mode"] = True - remaining = {k: v for k, v in non_default_params.items() if k != "response_format"} + remaining = { + k: v + for k, v in non_default_params.items() + if k != "response_format" + } return AnthropicConfig.map_openai_params( self, remaining, @@ -117,7 +126,11 @@ def transform_request( ) -> dict: # Filter out AWS authentication parameters before passing to Anthropic transformation # AWS params should only be used for signing requests, not included in request body - filtered_params = {k: v for k, v in optional_params.items() if k not in self.aws_authentication_params} + filtered_params = { + k: v + for k, v in optional_params.items() + if k not in self.aws_authentication_params + } filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params) _anthropic_request = AnthropicConfig.transform_request( @@ -134,9 +147,13 @@ def transform_request( # Convert Anthropic output_format to Bedrock InvokeModel output_config.format output_format = _anthropic_request.pop("output_format", None) - if output_format and isinstance(output_format, dict) and output_format.get("type") == "json_schema": + if ( + output_format + and isinstance(output_format, dict) + and output_format.get("type") == "json_schema" + ): schema = output_format.get("schema", {}) - normalized_schema = AmazonConverseConfig._add_additional_properties_to_schema(schema) + normalized_schema = add_additional_properties_to_schema(schema) # Preserve existing output_config keys (e.g. effort from reasoning_effort) output_config = _anthropic_request.get("output_config") or {} output_config["format"] = { @@ -176,7 +193,9 @@ def transform_request( ) beta_set.update(auto_betas) - if tool_search_used and not (programmatic_tool_calling_used or input_examples_used): + if tool_search_used and not ( + programmatic_tool_calling_used or input_examples_used + ): beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER) if "opus-4" in model.lower() or "opus_4" in model.lower(): beta_set.add("tool-search-tool-2025-10-19") @@ -205,7 +224,9 @@ def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict: if tool_type == "tool_search_tool_regex_20251119": normalized_tool = tool.copy() normalized_tool["type"] = "tool_search_tool_regex" - normalized_tool["name"] = normalized_tool.get("name", "tool_search_tool_regex") + normalized_tool["name"] = normalized_tool.get( + "name", "tool_search_tool_regex" + ) normalized_tools.append(normalized_tool) continue normalized_tools.append(tool) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9666aa68c99..99c02be9b95 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -49,6 +49,46 @@ def get_cached_model_info(): return _get_model_info +def add_additional_properties_to_schema(schema: dict) -> dict: + """ + Recursively ensure all object types in a JSON schema have + ``"additionalProperties": false``. + + Bedrock's native structured-outputs API requires this field to be + explicitly set on every object node, otherwise it returns a + validation error. + """ + if not isinstance(schema, dict): + return schema + + result = dict(schema) + + if result.get("type") == "object" and "additionalProperties" not in result: + result["additionalProperties"] = False + + # Recurse into nested schemas + if "properties" in result and isinstance(result["properties"], dict): + result["properties"] = { + k: add_additional_properties_to_schema(v) + for k, v in result["properties"].items() + } + if "items" in result and isinstance(result["items"], dict): + result["items"] = add_additional_properties_to_schema(result["items"]) + for defs_key in ("$defs", "definitions"): + if defs_key in result and isinstance(result[defs_key], dict): + result[defs_key] = { + k: add_additional_properties_to_schema(v) + for k, v in result[defs_key].items() + } + for key in ("anyOf", "allOf", "oneOf"): + if key in result and isinstance(result[key], list): + result[key] = [ + add_additional_properties_to_schema(item) for item in result[key] + ] + + return result + + def remove_custom_field_from_tools(request_body: dict) -> None: """ Remove ``custom`` field from each tool in the request body. @@ -1062,9 +1102,11 @@ def sign_aws_request( return ( dict(prepped.headers), - request_data.encode("utf-8") - if isinstance(request_data, str) - else request_data, + ( + request_data.encode("utf-8") + if isinstance(request_data, str) + else request_data + ), ) def generate_unique_job_name(self, model: str, prefix: str = "litellm") -> str: diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 551b3789217..ab80a63d63d 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -339,79 +339,12 @@ def test_opus_4_5_model_detection(): ), f"Should not detect {model} as Opus 4.5" -# def test_structured_outputs_beta_header_filtered_for_bedrock_invoke(): -# """ -# Test that unsupported beta headers are filtered out for Bedrock Invoke API. - -# Bedrock Invoke API only supports a specific whitelist of beta flags and returns -# "invalid beta flag" error for others (e.g., structured-outputs, mcp-servers). -# This test ensures unsupported headers are filtered while keeping supported ones. - -# Fixes: https://github.com/BerriAI/litellm/issues/16726 -# """ -# config = AmazonAnthropicClaudeConfig() - -# messages = [{"role": "user", "content": "test"}] - -# # Test 1: structured-outputs beta header (unsupported) -# headers = {"anthropic-beta": "structured-outputs-2025-11-13"} - -# result = config.transform_request( -# model="anthropic.claude-4-0-sonnet-20250514-v1:0", -# messages=messages, -# optional_params={}, -# litellm_params={}, -# headers=headers, -# ) - -# # Verify structured-outputs beta is filtered out -# anthropic_beta = result.get("anthropic_beta", []) -# assert not any("structured-outputs" in beta for beta in anthropic_beta), \ -# f"structured-outputs beta should be filtered, got: {anthropic_beta}" - -# # Test 2: mcp-servers beta header (unsupported - the main issue from #16726) -# headers = {"anthropic-beta": "mcp-servers-2025-12-04"} - -# result = config.transform_request( -# model="anthropic.claude-4-0-sonnet-20250514-v1:0", -# messages=messages, -# optional_params={}, -# litellm_params={}, -# headers=headers, -# ) - -# # Verify mcp-servers beta is filtered out -# anthropic_beta = result.get("anthropic_beta", []) -# assert not any("mcp-servers" in beta for beta in anthropic_beta), \ -# f"mcp-servers beta should be filtered, got: {anthropic_beta}" - -# # Test 3: Mix of supported and unsupported beta headers -# headers = {"anthropic-beta": "computer-use-2024-10-22,mcp-servers-2025-12-04,structured-outputs-2025-11-13"} - -# result = config.transform_request( -# model="anthropic.claude-4-0-sonnet-20250514-v1:0", -# messages=messages, -# optional_params={}, -# litellm_params={}, -# headers=headers, -# ) - -# # Verify only supported betas are kept -# anthropic_beta = result.get("anthropic_beta", []) -# assert not any("structured-outputs" in beta for beta in anthropic_beta), \ -# f"structured-outputs beta should be filtered, got: {anthropic_beta}" -# assert not any("mcp-servers" in beta for beta in anthropic_beta), \ -# f"mcp-servers beta should be filtered, got: {anthropic_beta}" -# assert any("computer-use" in beta for beta in anthropic_beta), \ -# f"computer-use beta should be kept, got: {anthropic_beta}" - - def test_output_config_removed_from_bedrock_chat_invoke_request(): """ - Test that output_config parameter is stripped from Bedrock Chat Invoke requests. - - Bedrock Invoke API doesn't support the output_config parameter (Anthropic-only). - Ensures the chat/invoke path mirrors the messages/invoke path fix. + Test that output_config is stripped for models that do not support native + structured outputs on Bedrock Invoke. Models that *do* support it (e.g. + claude-sonnet-4-5) keep output_config -- see the native structured output + tests below. Fixes: https://github.com/BerriAI/litellm/issues/22797 """