diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py
index 229457a73b4..bc4c1b53c2a 100644
--- a/litellm/llms/bedrock/chat/converse_transformation.py
+++ b/litellm/llms/bedrock/chat/converse_transformation.py
@@ -493,8 +493,7 @@ def _clamp_thinking_budget_tokens(optional_params: dict) -> None:
budget = thinking.get("budget_tokens")
if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS:
verbose_logger.debug(
- "Bedrock requires thinking.budget_tokens >= %d, got %d. "
- "Clamping to minimum.",
+ "Bedrock requires thinking.budget_tokens >= %d, got %d. Clamping to minimum.",
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
budget,
)
@@ -778,39 +777,14 @@ def _add_additional_properties_to_schema(schema: dict) -> dict:
Bedrock's native structured-outputs API requires this field to be
explicitly set on every object node, otherwise it returns a
validation error.
- """
- if not isinstance(schema, dict):
- return schema
-
- result = dict(schema)
- if result.get("type") == "object" and "additionalProperties" not in result:
- result["additionalProperties"] = False
-
- # Recurse into nested schemas
- if "properties" in result and isinstance(result["properties"], dict):
- result["properties"] = {
- k: AmazonConverseConfig._add_additional_properties_to_schema(v)
- for k, v in result["properties"].items()
- }
- if "items" in result and isinstance(result["items"], dict):
- result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(
- result["items"]
- )
- for defs_key in ("$defs", "definitions"):
- if defs_key in result and isinstance(result[defs_key], dict):
- result[defs_key] = {
- k: AmazonConverseConfig._add_additional_properties_to_schema(v)
- for k, v in result[defs_key].items()
- }
- for key in ("anyOf", "allOf", "oneOf"):
- if key in result and isinstance(result[key], list):
- result[key] = [
- AmazonConverseConfig._add_additional_properties_to_schema(item)
- for item in result[key]
- ]
+ Delegates to the shared implementation in ``bedrock/common_utils.py``.
+ """
+ from litellm.llms.bedrock.common_utils import (
+ add_additional_properties_to_schema,
+ )
- return result
+ return add_additional_properties_to_schema(schema)
@staticmethod
def _create_output_config_for_response_format(
@@ -913,7 +887,9 @@ def map_openai_params(
)
if param == "tool_choice":
_tool_choice_value = self.map_tool_choice_values(
- model=model, tool_choice=value, drop_params=drop_params # type: ignore
+ model=model,
+ tool_choice=value,
+ drop_params=drop_params, # type: ignore
)
if _tool_choice_value is not None:
optional_params["tool_choice"] = _tool_choice_value
@@ -1748,9 +1724,7 @@ def apply_tool_call_transformation_if_needed(
return message, returned_finish_reason
- def _translate_message_content(
- self, content_blocks: List[ContentBlock]
- ) -> Tuple[
+ def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
str,
List[ChatCompletionToolCallChunk],
Optional[List[BedrockConverseReasoningContentBlock]],
@@ -1767,9 +1741,9 @@ def _translate_message_content(
"""
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
- reasoningContentBlocks: Optional[
- List[BedrockConverseReasoningContentBlock]
- ] = None
+ reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
+ None
+ )
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
for idx, content in enumerate(content_blocks):
"""
@@ -1980,9 +1954,9 @@ def _transform_response( # noqa: PLR0915
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
- reasoningContentBlocks: Optional[
- List[BedrockConverseReasoningContentBlock]
- ] = None
+ reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
+ None
+ )
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
if message is not None:
@@ -2001,17 +1975,17 @@ def _transform_response( # noqa: PLR0915
provider_specific_fields["citationsContent"] = citationsContentBlocks
if provider_specific_fields:
- chat_completion_message[
- "provider_specific_fields"
- ] = provider_specific_fields
+ chat_completion_message["provider_specific_fields"] = (
+ provider_specific_fields
+ )
if reasoningContentBlocks is not None:
- chat_completion_message[
- "reasoning_content"
- ] = self._transform_reasoning_content(reasoningContentBlocks)
- chat_completion_message[
- "thinking_blocks"
- ] = self._transform_thinking_blocks(reasoningContentBlocks)
+ chat_completion_message["reasoning_content"] = (
+ self._transform_reasoning_content(reasoningContentBlocks)
+ )
+ chat_completion_message["thinking_blocks"] = (
+ self._transform_thinking_blocks(reasoningContentBlocks)
+ )
chat_completion_message["content"] = content_str
filtered_tools = self._filter_json_mode_tools(
json_mode=json_mode,
diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
index 7936b6ea644..fe4d7a25cf2 100644
--- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
+++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
@@ -7,6 +7,7 @@
AmazonInvokeConfig,
)
from litellm.llms.bedrock.common_utils import (
+ add_additional_properties_to_schema,
get_anthropic_beta_from_headers,
remove_custom_field_from_tools,
)
@@ -21,6 +22,18 @@
else:
LiteLLMLoggingObj = Any
+# Anthropic Claude models that support native structured outputs on Bedrock InvokeModel.
+# Maintained separately from the Converse path's BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS
+# because Invoke and Converse have independent feature rollouts.
+# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
+BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS = {
+ "claude-haiku-4-5",
+ "claude-sonnet-4-5",
+ "claude-sonnet-4-6",
+ "claude-opus-4-5",
+ "claude-opus-4-6",
+}
+
class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
"""
@@ -49,6 +62,14 @@ def custom_llm_provider(self) -> Optional[str]:
def get_supported_openai_params(self, model: str) -> List[str]:
return AnthropicConfig.get_supported_openai_params(self, model)
+ @staticmethod
+ def _supports_native_structured_outputs(model: str) -> bool:
+ """Check if the Bedrock Invoke model supports native structured outputs."""
+ return any(
+ substring in model
+ for substring in BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS
+ )
+
def map_openai_params(
self,
non_default_params: dict,
@@ -56,15 +77,38 @@ def map_openai_params(
model: str,
drop_params: bool,
) -> dict:
- # Force tool-based structured outputs for Bedrock Invoke
- # (similar to VertexAI fix in #19201)
- # Bedrock Invoke doesn't support output_format parameter
- original_model = model
- if "response_format" in non_default_params:
- # Use a model name that forces tool-based approach
+ response_format = non_default_params.get("response_format")
+
+ # Native path: build output_format directly for Bedrock-supported models
+ # (includes haiku-4-5 which the Anthropic parent doesn't know about).
+ if isinstance(
+ response_format, dict
+ ) and self._supports_native_structured_outputs(model):
+ _output_format = self.map_response_format_to_anthropic_output_format(
+ response_format
+ )
+ if _output_format is not None:
+ optional_params["output_format"] = _output_format
+ optional_params["json_mode"] = True
+ remaining = {
+ k: v
+ for k, v in non_default_params.items()
+ if k != "response_format"
+ }
+ return AnthropicConfig.map_openai_params(
+ self,
+ remaining,
+ optional_params,
+ model,
+ drop_params,
+ )
+
+ # Fallback: force tool-based structured outputs for unsupported models
+ # (or json_object without schema on a supported model).
+ if response_format is not None:
model = "claude-3-sonnet-20240229"
- optional_params = AnthropicConfig.map_openai_params(
+ return AnthropicConfig.map_openai_params(
self,
non_default_params,
optional_params,
@@ -72,11 +116,6 @@ def map_openai_params(
drop_params,
)
- # Restore original model name
- model = original_model
-
- return optional_params
-
def transform_request(
self,
model: str,
@@ -105,11 +144,30 @@ def transform_request(
_anthropic_request.pop("model", None)
_anthropic_request.pop("stream", None)
- # Bedrock Invoke doesn't support output_format parameter
- _anthropic_request.pop("output_format", None)
- # Bedrock Invoke doesn't support output_config parameter
- # Fixes: https://github.com/BerriAI/litellm/issues/22797
- _anthropic_request.pop("output_config", None)
+
+ # Convert Anthropic output_format to Bedrock InvokeModel output_config.format
+ output_format = _anthropic_request.pop("output_format", None)
+ if (
+ output_format
+ and isinstance(output_format, dict)
+ and output_format.get("type") == "json_schema"
+ ):
+ schema = output_format.get("schema", {})
+ normalized_schema = add_additional_properties_to_schema(schema)
+ # Preserve existing output_config keys (e.g. effort from reasoning_effort)
+ output_config = _anthropic_request.get("output_config") or {}
+ output_config["format"] = {
+ "type": "json_schema",
+ "schema": normalized_schema,
+ }
+ _anthropic_request["output_config"] = output_config
+ else:
+ # Non-native path: strip output_config entirely.
+ # Bedrock Invoke rejects the key itself (not just sub-keys) with
+ # "extraneous key [output_config] is not permitted" for models
+ # that don't support native structured outputs.
+ # Fixes: https://github.com/BerriAI/litellm/issues/22797
+ _anthropic_request.pop("output_config", None)
if "anthropic_version" not in _anthropic_request:
_anthropic_request["anthropic_version"] = self.anthropic_version
diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py
index 9666aa68c99..99c02be9b95 100644
--- a/litellm/llms/bedrock/common_utils.py
+++ b/litellm/llms/bedrock/common_utils.py
@@ -49,6 +49,46 @@ def get_cached_model_info():
return _get_model_info
+def add_additional_properties_to_schema(schema: dict) -> dict:
+ """
+ Recursively ensure all object types in a JSON schema have
+ ``"additionalProperties": false``.
+
+ Bedrock's native structured-outputs API requires this field to be
+ explicitly set on every object node, otherwise it returns a
+ validation error.
+ """
+ if not isinstance(schema, dict):
+ return schema
+
+ result = dict(schema)
+
+ if result.get("type") == "object" and "additionalProperties" not in result:
+ result["additionalProperties"] = False
+
+ # Recurse into nested schemas
+ if "properties" in result and isinstance(result["properties"], dict):
+ result["properties"] = {
+ k: add_additional_properties_to_schema(v)
+ for k, v in result["properties"].items()
+ }
+ if "items" in result and isinstance(result["items"], dict):
+ result["items"] = add_additional_properties_to_schema(result["items"])
+ for defs_key in ("$defs", "definitions"):
+ if defs_key in result and isinstance(result[defs_key], dict):
+ result[defs_key] = {
+ k: add_additional_properties_to_schema(v)
+ for k, v in result[defs_key].items()
+ }
+ for key in ("anyOf", "allOf", "oneOf"):
+ if key in result and isinstance(result[key], list):
+ result[key] = [
+ add_additional_properties_to_schema(item) for item in result[key]
+ ]
+
+ return result
+
+
def remove_custom_field_from_tools(request_body: dict) -> None:
"""
Remove ``custom`` field from each tool in the request body.
@@ -1062,9 +1102,11 @@ def sign_aws_request(
return (
dict(prepped.headers),
- request_data.encode("utf-8")
- if isinstance(request_data, str)
- else request_data,
+ (
+ request_data.encode("utf-8")
+ if isinstance(request_data, str)
+ else request_data
+ ),
)
def generate_unique_job_name(self, model: str, prefix: str = "litellm") -> str:
diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py
index b71e4e51877..ad8c4bbdbe1 100644
--- a/tests/llm_translation/test_bedrock_completion.py
+++ b/tests/llm_translation/test_bedrock_completion.py
@@ -16,9 +16,7 @@
import os
import json
-sys.path.insert(
- 0, os.path.abspath("../..")
-) # Adds the parent directory to the system path
+sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path
from unittest.mock import AsyncMock, Mock, patch
import pytest
@@ -157,9 +155,9 @@ def test_completion_bedrock_guardrails(streaming):
saw_trace = True
print(chunk)
- assert (
- saw_trace is True
- ), "Did not see trace in response even when trace=enabled sent in the guardrailConfig"
+ assert saw_trace is True, (
+ "Did not see trace in response even when trace=enabled sent in the guardrailConfig"
+ )
except RateLimitError:
pass
@@ -292,9 +290,7 @@ def bedrock_session_token_creds():
# For circle-ci testing
# aws_role_name = os.environ["AWS_TEMP_ROLE_NAME"]
# TODO: This is using ai.moda's IAM role, we should use LiteLLM's IAM role eventually
- aws_role_name = (
- "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci"
- )
+ aws_role_name = "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci"
aws_web_identity_token = "test-oidc-token-123"
creds = bllm.get_credentials(
@@ -664,15 +660,9 @@ def test_bedrock_claude_3_tool_calling():
print(f"response: {response}")
# Add any assertions here to check the response
assert isinstance(response.choices[0].message.tool_calls[0].function.name, str)
- assert isinstance(
- response.choices[0].message.tool_calls[0].function.arguments, str
- )
- messages.append(
- response.choices[0].message.model_dump()
- ) # Add assistant tool invokes
- tool_result = (
- '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}'
- )
+ assert isinstance(response.choices[0].message.tool_calls[0].function.arguments, str)
+ messages.append(response.choices[0].message.model_dump()) # Add assistant tool invokes
+ tool_result = '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}'
# Add user submitted tool results in the OpenAI format
messages.append(
{
@@ -704,9 +694,7 @@ def encode_image(image_path):
return base64.b64encode(image_file.read()).decode("utf-8")
-@pytest.mark.skip(
- reason="we already test claude-3, this is just another way to pass images"
-)
+@pytest.mark.skip(reason="we already test claude-3, this is just another way to pass images")
def test_completion_claude_3_base64():
try:
litellm.set_verbose = True
@@ -723,9 +711,7 @@ def test_completion_claude_3_base64():
{"type": "text", "text": "Whats in this image?"},
{
"type": "image_url",
- "image_url": {
- "url": "data:image/jpeg;base64," + base64_image
- },
+ "image_url": {"url": "data:image/jpeg;base64," + base64_image},
},
],
}
@@ -790,9 +776,7 @@ def test_bedrock_ptu():
litellm.set_verbose = True
from openai.types.chat import ChatCompletion
- model_id = (
- "arn:aws:bedrock:us-west-2:888602223428:provisioned-model/8fxff74qyhs3"
- )
+ model_id = "arn:aws:bedrock:us-west-2:888602223428:provisioned-model/8fxff74qyhs3"
try:
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
@@ -843,10 +827,7 @@ async def test_bedrock_custom_api_base():
)
assert "test" in mock_client_post.call_args.kwargs["headers"]
assert mock_client_post.call_args.kwargs["headers"]["test"] == "hello world"
- assert (
- mock_client_post.call_args.kwargs["headers"]["Authorization"]
- == "my-test-key"
- )
+ assert mock_client_post.call_args.kwargs["headers"]["Authorization"] == "my-test-key"
mock_client_post.assert_called_once()
@@ -881,10 +862,7 @@ async def test_bedrock_extra_headers(model):
print(f"mock_client_post.call_args.kwargs: {mock_client_post.call_args.kwargs}")
assert "test" in mock_client_post.call_args.kwargs["headers"]
assert mock_client_post.call_args.kwargs["headers"]["test"] == "hello world"
- assert (
- mock_client_post.call_args.kwargs["headers"]["Authorization"]
- == "my-test-key"
- )
+ assert mock_client_post.call_args.kwargs["headers"]["Authorization"] == "my-test-key"
mock_client_post.assert_called_once()
@@ -1023,10 +1001,7 @@ def test_bedrock_tool_calling():
for tool_call in _choice_1.message.tool_calls:
_tool_Call_name = tool_call.function.name
if _tool_Call_name is not None and "DoSomethingVeryCool" in _tool_Call_name:
- assert (
- _tool_Call_name
- == "-DoSomethingVeryCool-forLitellm_Testin999229291-0293993"
- )
+ assert _tool_Call_name == "-DoSomethingVeryCool-forLitellm_Testin999229291-0293993"
def test_bedrock_tools_pt_valid_names():
@@ -1149,12 +1124,8 @@ def test_bedrock_tools_transformation_valid_params():
toolJsonSchema = result[0]["toolSpec"]["inputSchema"]["json"]
assert toolJsonSchema is not None
print("transformed toolJsonSchema keys=", toolJsonSchema.keys())
- print(
- "allowed ToolJsonSchemaBlock keys=", ToolJsonSchemaBlock.__annotations__.keys()
- )
- assert set(toolJsonSchema.keys()).issubset(
- set(ToolJsonSchemaBlock.__annotations__.keys())
- )
+ print("allowed ToolJsonSchemaBlock keys=", ToolJsonSchemaBlock.__annotations__.keys())
+ assert set(toolJsonSchema.keys()).issubset(set(ToolJsonSchemaBlock.__annotations__.keys()))
assert isinstance(result, list)
assert len(result) == 1
@@ -1163,10 +1134,7 @@ def test_bedrock_tools_transformation_valid_params():
assert result[0]["toolSpec"]["description"] == "Invalid name test"
assert "inputSchema" in result[0]["toolSpec"]
assert "json" in result[0]["toolSpec"]["inputSchema"]
- assert (
- result[0]["toolSpec"]["inputSchema"]["json"]["properties"]["test"]["type"]
- == "string"
- )
+ assert result[0]["toolSpec"]["inputSchema"]["json"]["properties"]["test"]["type"] == "string"
assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"]
@@ -1243,24 +1211,18 @@ def test_bedrock_converse_translation_tool_message():
},
]
- translated_msg = _bedrock_converse_messages_pt(
- messages=messages, model="", llm_provider=""
- )
+ translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="")
print(translated_msg)
assert translated_msg == [
{
"role": "user",
"content": [
- {
- "text": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses"
- },
+ {"text": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses"},
{
"toolResult": {
"content": [
- {
- "text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}'
- }
+ {"text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}'}
],
"toolUseId": "tooluse_DnqEmD5qR6y2-aJ-Xd05xw",
}
@@ -1286,11 +1248,7 @@ def test_base_aws_llm_get_credentials():
credentials = session.get_credentials().get_frozen_credentials()
end_time = time.time()
- print(
- "Total time for credentials - {}. Credentials - {}".format(
- end_time - start_time, credentials
- )
- )
+ print("Total time for credentials - {}. Credentials - {}".format(end_time - start_time, credentials))
start_time = time.time()
credentials = BaseAWSLLM().get_credentials(
@@ -1591,20 +1549,14 @@ def test_bedrock_completion_test_3():
},
]
- transformed_messages = _bedrock_converse_messages_pt(
- messages=messages, model="", llm_provider=""
- )
+ transformed_messages = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="")
print(transformed_messages)
assert transformed_messages[-1]["role"] == "user"
assert transformed_messages[-1]["content"] == [
{
"toolResult": {
- "content": [
- {
- "text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}'
- }
- ],
+ "content": [{"text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}'}],
"toolUseId": "tooluse_EF8PwJ1dSMSh6tLGKu9VdA",
}
}
@@ -1840,9 +1792,7 @@ def test_bedrock_completion_test_4(modify_params):
}
if modify_params:
- transformed_messages = _bedrock_converse_messages_pt(
- messages=data["messages"], model="", llm_provider=""
- )
+ transformed_messages = _bedrock_converse_messages_pt(messages=data["messages"], model="", llm_provider="")
expected_messages = [
{
"role": "user",
@@ -1942,9 +1892,7 @@ def test_bedrock_base_model_helper():
assert base_model == "amazon.nova-pro-v1:0"
assert (
- BedrockModelInfo.get_base_model(
- "invoke/anthropic.claude-3-5-sonnet-20241022-v2:0"
- )
+ BedrockModelInfo.get_base_model("invoke/anthropic.claude-3-5-sonnet-20241022-v2:0")
== "anthropic.claude-3-5-sonnet-20241022-v2:0"
)
@@ -1975,9 +1923,7 @@ def test_bedrock_route_detection(model, expected_route):
from litellm.llms.bedrock.common_utils import BedrockModelInfo
route = BedrockModelInfo.get_bedrock_route(model)
- assert (
- route == expected_route
- ), f"Expected route '{expected_route}' for model '{model}', but got '{route}'"
+ assert route == expected_route, f"Expected route '{expected_route}' for model '{model}', but got '{route}'"
@pytest.mark.parametrize(
@@ -2047,9 +1993,7 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control):
],
)
def test_bedrock_supports_tool_call(model, expected_supports_tool_call):
- supported_openai_params = (
- litellm.AmazonConverseConfig().get_supported_openai_params(model=model)
- )
+ supported_openai_params = litellm.AmazonConverseConfig().get_supported_openai_params(model=model)
if expected_supports_tool_call:
assert "tools" in supported_openai_params
else:
@@ -2197,10 +2141,7 @@ def test_bedrock_empty_content_handling(messages, continue_message_index):
# Verify assistant message with default text was inserted
assert formatted_messages[0]["role"] == "user"
assert formatted_messages[1]["role"] == "assistant"
- assert (
- formatted_messages[continue_message_index]["content"][0]["text"]
- == "Please continue."
- )
+ assert formatted_messages[continue_message_index]["content"][0]["text"] == "Please continue."
def test_bedrock_custom_continue_message():
@@ -2249,9 +2190,7 @@ def test_bedrock_no_default_message():
)
# Verify empty message is replaced with placeholder and valid message remains
- assistant_messages = [
- msg for msg in formatted_messages if msg["role"] == "assistant"
- ]
+ assistant_messages = [msg for msg in formatted_messages if msg["role"] == "assistant"]
assert len(assistant_messages) == 1
assert assistant_messages[0]["content"][0]["text"] == "Valid response"
@@ -2273,18 +2212,13 @@ def mock_transform(*args, **kwargs):
captured_data = result
return result
- with patch(
- "litellm.AmazonConverseConfig._transform_request", side_effect=mock_transform
- ):
+ with patch("litellm.AmazonConverseConfig._transform_request", side_effect=mock_transform):
litellm.completion(**data)
# Assert that additionalRequestParameters exists and contains topK
assert "additionalModelRequestFields" in captured_data
assert "inferenceConfig" in captured_data["additionalModelRequestFields"]
- assert (
- captured_data["additionalModelRequestFields"]["inferenceConfig"]["topK"]
- == 10
- )
+ assert captured_data["additionalModelRequestFields"]["inferenceConfig"]["topK"] == 10
def test_bedrock_cross_region_inference(monkeypatch):
@@ -2344,17 +2278,13 @@ def test_bedrock_process_empty_text_blocks():
assert modified_message["content"][0]["text"] == "Please continue."
-@pytest.mark.skip(
- reason="Skipping test due to bedrock changing their response schema support. Come back to this."
-)
+@pytest.mark.skip(reason="Skipping test due to bedrock changing their response schema support. Come back to this.")
def test_nova_optional_params_tool_choice():
try:
litellm.drop_params = True
litellm.set_verbose = True
litellm.completion(
- messages=[
- {"role": "user", "content": "A WWII competitive game for 4-8 players"}
- ],
+ messages=[{"role": "user", "content": "A WWII competitive game for 4-8 players"}],
model="bedrock/us.amazon.nova-pro-v1:0",
temperature=0.3,
tools=[
@@ -2444,9 +2374,7 @@ def test_bedrock_image_embedding_transformation(self):
"inference_params": {},
}
- transformed_request = (
- AmazonTitanMultimodalEmbeddingG1Config()._transform_request(**args)
- )
+ transformed_request = AmazonTitanMultimodalEmbeddingG1Config()._transform_request(**args)
transformed_request[
"inputImage"
] == "iVBORw0KGgoAAAANSUhEUgAAAGQAAABkBAMAAACCzIhnAAAAG1BMVEURAAD///+ln5/h39/Dv79qX18uHx+If39MPz9oMSdmAAAACXBIWXMAAA7EAAAOxAGVKw4bAAABB0lEQVRYhe2SzWrEIBCAh2A0jxEs4j6GLDS9hqWmV5Flt0cJS+lRwv742DXpEjY1kOZW6HwHFZnPmVEBEARBEARB/jd0KYA/bcUYbPrRLh6amXHJ/K+ypMoyUaGthILzw0l+xI0jsO7ZcmCcm4ILd+QuVYgpHOmDmz6jBeJImdcUCmeBqQpuqRIbVmQsLCrAalrGpfoEqEogqbLTWuXCPCo+Ki1XGqgQ+jVVuhB8bOaHkvmYuzm/b0KYLWwoK58oFqi6XfxQ4Uz7d6WeKpna6ytUs5e8betMcqAv5YPC5EZB2Lm9FIn0/VP6R58+/GEY1X1egVoZ/3bt/EqF6malgSAIgiDIH+QL41409QMY0LMAAAAASUVORK5CYII="
@@ -2510,9 +2438,7 @@ def test_bedrock_error_handling_streaming():
}
)
- decoder = AWSEventStreamDecoder(
- model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0"
- )
+ decoder = AWSEventStreamDecoder(model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0")
with pytest.raises(Exception) as e:
decoder._parse_message_from_event(event)
assert isinstance(e.value, BedrockError)
@@ -2591,9 +2517,7 @@ def test_bedrock_custom_deepseek():
with patch.object(client, "post") as mock_post:
# Mock the response
mock_response = Mock()
- mock_response.text = json.dumps(
- {"generation": "Here's a joke...", "stop_reason": "stop"}
- )
+ mock_response.text = json.dumps({"generation": "Here's a joke...", "stop_reason": "stop"})
mock_response.status_code = 200
# Add required response attributes
mock_response.headers = {"Content-Type": "application/json"}
@@ -2641,14 +2565,8 @@ def test_bedrock_custom_deepseek():
],
)
def test_handle_top_k_value_helper(model, expected_output):
- assert (
- litellm.AmazonConverseConfig()._handle_top_k_value(model, {"topK": 3})
- == expected_output
- )
- assert (
- litellm.AmazonConverseConfig()._handle_top_k_value(model, {"top_k": 3})
- == expected_output
- )
+ assert litellm.AmazonConverseConfig()._handle_top_k_value(model, {"topK": 3}) == expected_output
+ assert litellm.AmazonConverseConfig()._handle_top_k_value(model, {"top_k": 3}) == expected_output
@pytest.mark.parametrize(
@@ -2669,9 +2587,7 @@ def test_bedrock_top_k_param(model, expected_params):
mock_response = Mock()
if "mistral" in model:
- mock_response.text = json.dumps(
- {"outputs": [{"text": "Here's a joke...", "stop_reason": "stop"}]}
- )
+ mock_response.text = json.dumps({"outputs": [{"text": "Here's a joke...", "stop_reason": "stop"}]})
else:
mock_response.text = json.dumps(
{
@@ -2713,9 +2629,7 @@ def test_bedrock_invoke_provider():
== "anthropic"
)
assert (
- litellm.AmazonInvokeConfig().get_bedrock_invoke_provider(
- "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
- )
+ litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0")
== "anthropic"
)
assert (
@@ -2724,40 +2638,12 @@ def test_bedrock_invoke_provider():
)
== "llama"
)
- assert (
- litellm.AmazonInvokeConfig().get_bedrock_invoke_provider(
- "us.amazon.nova-pro-v1:0"
- )
- == "nova"
- )
- assert (
- litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-pro-v1:0")
- == "nova"
- )
- assert (
- litellm.AmazonInvokeConfig().get_bedrock_invoke_provider(
- "amazon.nova-lite-v1:0"
- )
- == "nova"
- )
- assert (
- litellm.AmazonInvokeConfig().get_bedrock_invoke_provider(
- "amazon.nova-micro-v1:0"
- )
- == "nova"
- )
- assert (
- litellm.AmazonInvokeConfig().get_bedrock_invoke_provider(
- "amazon.nova-premier-v1:0"
- )
- == "nova"
- )
- assert (
- litellm.AmazonInvokeConfig().get_bedrock_invoke_provider(
- "amazon.nova-2-lite-v1:0"
- )
- == "nova"
- )
+ assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("us.amazon.nova-pro-v1:0") == "nova"
+ assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-pro-v1:0") == "nova"
+ assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-lite-v1:0") == "nova"
+ assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-micro-v1:0") == "nova"
+ assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-premier-v1:0") == "nova"
+ assert litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-2-lite-v1:0") == "nova"
def test_bedrock_description_param():
@@ -2770,9 +2656,7 @@ def test_bedrock_description_param():
try:
response = completion(
model="bedrock/us.amazon.nova-pro-v1:0",
- messages=[
- {"role": "user", "content": "What is the meaning of this poem?"}
- ],
+ messages=[{"role": "user", "content": "What is the meaning of this poem?"}],
response_format={
"type": "json_schema",
"json_schema": {
@@ -2794,9 +2678,7 @@ def test_bedrock_description_param():
request_body_str = json.dumps(request_body, indent=4, default=str)
print("request_body=", request_body_str)
- assert (
- "Find the meaning inside a poem" in request_body_str
- ) # assert description is passed
+ assert "Find the meaning inside a poem" in request_body_str # assert description is passed
@pytest.mark.parametrize(
@@ -2853,10 +2735,7 @@ async def test_bedrock_thinking_in_assistant_message(sync_mode):
print(mock_post.call_args.kwargs)
json_data = mock_post.call_args.kwargs["data"]
- assert (
- "Alright, let's get started with resolving this issue about implementing"
- in json_data
- )
+ assert "Alright, let's get started with resolving this issue about implementing" in json_data
@pytest.mark.asyncio
@@ -2900,32 +2779,20 @@ async def test_bedrock_stream_thinking_content_openwebui():
# Assert there's exactly one opening and closing tag
assert think_open_pos >= 0, "Opening tag not found"
assert think_close_pos > 0, "Closing tag not found"
- assert (
- content.count("") == 1
- ), "There should be exactly one opening tag"
- assert (
- content.count("") == 1
- ), "There should be exactly one closing tag"
+ assert content.count("") == 1, "There should be exactly one opening tag"
+ assert content.count("") == 1, "There should be exactly one closing tag"
# Assert the opening tag comes before the closing tag
- assert (
- think_open_pos < think_close_pos
- ), "Opening tag should come before closing tag"
+ assert think_open_pos < think_close_pos, "Opening tag should come before closing tag"
# Assert there's content between the tags
thinking_content = content[think_open_pos + 7 : think_close_pos]
- assert (
- len(thinking_content.strip()) > 0
- ), "There should be content between thinking tags"
+ assert len(thinking_content.strip()) > 0, "There should be content between thinking tags"
# Assert there's content after the closing tag
- assert (
- len(content) > think_close_pos + 8
- ), "There should be content after the thinking tags"
+ assert len(content) > think_close_pos + 8, "There should be content after the thinking tags"
response_content = content[think_close_pos + 8 :].strip()
- assert (
- len(response_content) > 0
- ), "There should be non-empty content after thinking tags"
+ assert len(response_content) > 0, "There should be non-empty content after thinking tags"
def test_bedrock_application_inference_profile():
@@ -2958,9 +2825,7 @@ def test_bedrock_application_inference_profile():
}
]
- with patch.object(client, "post") as mock_post, patch.object(
- client2, "post"
- ) as mock_post2:
+ with patch.object(client, "post") as mock_post, patch.object(client2, "post") as mock_post2:
try:
resp = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
@@ -2986,9 +2851,7 @@ def test_bedrock_application_inference_profile():
mock_post2.assert_called_once()
print(mock_post.call_args.kwargs)
json_data = mock_post.call_args.kwargs["data"]
- assert mock_post.call_args.kwargs["url"].startswith(
- "https://bedrock-runtime.eu-central-1.amazonaws.com/"
- )
+ assert mock_post.call_args.kwargs["url"].startswith("https://bedrock-runtime.eu-central-1.amazonaws.com/")
assert mock_post2.call_args.kwargs["url"] == mock_post.call_args.kwargs["url"]
@@ -3121,9 +2984,7 @@ async def test_bedrock_passthrough(sync_mode: bool):
}
],
"temperature": 0,
- "metadata": {
- "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"
- },
+ "metadata": {"user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"},
"anthropic_version": "bedrock-2023-05-31",
"anthropic_beta": ["claude-code-20250219"],
}
@@ -3182,9 +3043,7 @@ async def test_bedrock_passthrough_router():
}
],
"temperature": 0,
- "metadata": {
- "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"
- },
+ "metadata": {"user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"},
"anthropic_version": "bedrock-2023-05-31",
"anthropic_beta": ["claude-code-20250219"],
}
@@ -3220,11 +3079,7 @@ class MockCustomLogger(CustomLogger):
"messages": [
{
"role": "user",
- "content": [
- {
- "text": "Write an article about impact of high inflation to GDP of a country"
- }
- ],
+ "content": [{"text": "Write an article about impact of high inflation to GDP of a country"}],
}
],
"system": [{"text": "You are an economist with access to lots of data"}],
@@ -3276,9 +3131,7 @@ class MockCustomLogger(CustomLogger):
}
],
"temperature": 0,
- "metadata": {
- "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"
- },
+ "metadata": {"user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"},
"anthropic_version": "bedrock-2023-05-31",
"anthropic_beta": ["claude-code-20250219"],
}
@@ -3328,9 +3181,7 @@ class MockCustomLogger(CustomLogger):
}
],
"temperature": 0,
- "metadata": {
- "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"
- },
+ "metadata": {"user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84"},
"anthropic_version": "bedrock-2023-05-31",
"anthropic_beta": ["claude-code-20250219"],
}
@@ -3412,9 +3263,7 @@ def test_bedrock_openai_imported_model():
url = mock_post.call_args.kwargs["url"]
print(f"URL: {url}")
assert "bedrock-runtime.us-east-1.amazonaws.com" in url
- assert (
- "arn:aws:bedrock:us-east-1:117159858402:imported-model%2Fm4gc1mrfuddy" in url
- )
+ assert "arn:aws:bedrock:us-east-1:117159858402:imported-model%2Fm4gc1mrfuddy" in url
assert "/invoke" in url
# Validate request body follows OpenAI format
@@ -3443,9 +3292,7 @@ def test_bedrock_openai_imported_model():
# Check image_url content
assert user_msg["content"][1]["type"] == "image_url"
assert "image_url" in user_msg["content"][1]
- assert user_msg["content"][1]["image_url"]["url"].startswith(
- "data:image/jpeg;base64,"
- )
+ assert user_msg["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
assert user_msg["content"][2]["type"] == "image_url"
assert "image_url" in user_msg["content"][2]
@@ -3481,9 +3328,7 @@ def test_bedrock_nova_provider_detection():
for model, expected in nova_test_cases:
provider = BaseAWSLLM.get_bedrock_invoke_provider(model)
- assert (
- provider == expected
- ), f"Failed for model: {model}, expected: {expected}, got: {provider}"
+ assert provider == expected, f"Failed for model: {model}, expected: {expected}, got: {provider}"
# Verify that Amazon Titan models still return "amazon"
titan_test_cases = [
@@ -3493,9 +3338,7 @@ def test_bedrock_nova_provider_detection():
for model, expected in titan_test_cases:
provider = BaseAWSLLM.get_bedrock_invoke_provider(model)
- assert (
- provider == expected
- ), f"Failed for model: {model}, expected: {expected}, got: {provider}"
+ assert provider == expected, f"Failed for model: {model}, expected: {expected}, got: {provider}"
def test_bedrock_openai_provider_detection():
@@ -3512,9 +3355,7 @@ def test_bedrock_openai_provider_detection():
for model in test_cases:
provider = BaseAWSLLM.get_bedrock_invoke_provider(model)
- assert (
- provider == "openai"
- ), f"Failed for model: {model}, got provider: {provider}"
+ assert provider == "openai", f"Failed for model: {model}, got provider: {provider}"
print(f"✓ Provider detection works for: {model}")
@@ -3524,14 +3365,10 @@ def test_bedrock_openai_model_id_extraction():
"""
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
- model = (
- "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-model-123"
- )
+ model = "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-model-123"
provider = BaseAWSLLM.get_bedrock_invoke_provider(model)
- model_id = BaseAWSLLM.get_bedrock_model_id(
- model=model, provider=provider, optional_params={}
- )
+ model_id = BaseAWSLLM.get_bedrock_model_id(model=model, provider=provider, optional_params={})
# The ARN should be double URL encoded
assert "arn" in model_id
@@ -3718,27 +3555,15 @@ def test_bedrock_openai_explicit_route_check():
from litellm.llms.bedrock.common_utils import BedrockModelInfo
# Test with openai/ prefix
+ assert BedrockModelInfo._explicit_openai_route("openai/arn:aws:bedrock:us-east-1:123:imported-model/test") is True
assert (
- BedrockModelInfo._explicit_openai_route(
- "openai/arn:aws:bedrock:us-east-1:123:imported-model/test"
- )
- is True
- )
- assert (
- BedrockModelInfo._explicit_openai_route(
- "bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test"
- )
+ BedrockModelInfo._explicit_openai_route("bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test")
is True
)
# Test without openai/ prefix
assert BedrockModelInfo._explicit_openai_route("anthropic.claude-3-sonnet") is False
- assert (
- BedrockModelInfo._explicit_openai_route(
- "arn:aws:bedrock:us-east-1:123:imported-model/test"
- )
- is False
- )
+ assert BedrockModelInfo._explicit_openai_route("arn:aws:bedrock:us-east-1:123:imported-model/test") is False
print("✓ Explicit route check works correctly")
@@ -3850,10 +3675,12 @@ def test_bedrock_openai_error_handling():
assert exc_info.value.status_code == 422
print("✓ Error handling works correctly")
+
# ============================================================================
# Nova Grounding (web_search_options) Unit Tests (Mocked)
# ============================================================================
+
def test_bedrock_nova_grounding_web_search_options_non_streaming():
"""
Unit test for Nova grounding using web_search_options parameter (non-streaming).
@@ -4094,8 +3921,8 @@ def test_bedrock_nova_grounding_request_transformation():
json=lambda: {
"output": {"message": {"role": "assistant", "content": [{"text": "Test"}]}},
"stopReason": "end_turn",
- "usage": {"inputTokens": 10, "outputTokens": 5}
- }
+ "usage": {"inputTokens": 10, "outputTokens": 5},
+ },
)
try:
@@ -4131,3 +3958,60 @@ def test_bedrock_nova_grounding_request_transformation():
assert system_tool_found, "systemTool with nova_grounding should be present"
print("✓ web_search_options correctly transformed to systemTool")
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
+ "bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ "bedrock/invoke/us.anthropic.claude-sonnet-4-6",
+ "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0",
+ "bedrock/invoke/us.anthropic.claude-opus-4-6-v1",
+ ],
+)
+def test_bedrock_invoke_native_structured_output(model):
+ """
+ Integration test: verify native structured outputs via Bedrock InvokeModel
+ for each supported Claude model.
+
+ Uses output_config.format (Bedrock InvokeModel native API) instead of
+ the synthetic json_tool_call workaround.
+ """
+ response = completion(
+ model=model,
+ messages=[
+ {
+ "role": "user",
+ "content": "Classify the sentiment of 'I love this product!' as positive, negative, or neutral.",
+ }
+ ],
+ response_format={
+ "type": "json_schema",
+ "json_schema": {
+ "name": "sentiment",
+ "schema": {
+ "type": "object",
+ "properties": {
+ "sentiment": {
+ "type": "string",
+ "enum": ["positive", "negative", "neutral"],
+ }
+ },
+ "required": ["sentiment"],
+ "additionalProperties": False,
+ },
+ },
+ },
+ max_tokens=100,
+ )
+ print(f"Response for {model}: {response}")
+
+ # Validate the response is a ModelResponse with valid JSON content
+ assert isinstance(response, ModelResponse)
+ content = response.choices[0].message.content
+ assert content is not None
+
+ parsed = json.loads(content)
+ assert "sentiment" in parsed
+ assert parsed["sentiment"] in ["positive", "negative", "neutral"]
diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
index cb05531c2f8..ab80a63d63d 100644
--- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
+++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
@@ -25,27 +25,24 @@ def test_get_supported_params_thinking():
def test_aws_params_filtered_from_request_body():
"""
Test that AWS authentication parameters are filtered out from the request body.
-
+
This is a security test to ensure AWS credentials are not leaked in the request
body sent to Bedrock. AWS params should only be used for request signing.
-
- Regression test for: AWS params (aws_role_name, aws_session_name, etc.)
+
+ Regression test for: AWS params (aws_role_name, aws_session_name, etc.)
being included in the Bedrock InvokeModel request body.
"""
config = AmazonAnthropicClaudeConfig()
-
+
# Test messages
- messages = [
- {"role": "user", "content": "Hello, how are you?"}
- ]
-
+ messages = [{"role": "user", "content": "Hello, how are you?"}]
+
# Optional params with AWS authentication parameters that should be filtered out
optional_params = {
# Regular Anthropic params - these SHOULD be in the request
"max_tokens": 100,
"temperature": 0.7,
"top_p": 0.9,
-
# AWS authentication params - these should NOT be in the request body
"aws_access_key_id": "AKIAIOSFODNN7EXAMPLE",
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
@@ -59,7 +56,7 @@ def test_aws_params_filtered_from_request_body():
"aws_bedrock_runtime_endpoint": "https://bedrock-runtime.us-west-2.amazonaws.com",
"aws_external_id": "external-id-123",
}
-
+
# Transform the request
result = config.transform_request(
model="anthropic.claude-haiku-4-5-20251001-v1:0",
@@ -68,39 +65,69 @@ def test_aws_params_filtered_from_request_body():
litellm_params={},
headers={},
)
-
+
# Convert result to JSON string to check what would be sent in the request
result_json = json.dumps(result)
-
+
# Verify AWS authentication params are NOT in the request body
- assert "aws_access_key_id" not in result_json, "AWS access key should not be in request body"
- assert "aws_secret_access_key" not in result_json, "AWS secret key should not be in request body"
- assert "aws_session_token" not in result_json, "AWS session token should not be in request body"
- assert "aws_region_name" not in result_json, "AWS region should not be in request body"
- assert "aws_role_name" not in result_json, "AWS role name should not be in request body"
- assert "aws_session_name" not in result_json, "AWS session name should not be in request body"
- assert "aws_profile_name" not in result_json, "AWS profile name should not be in request body"
- assert "aws_web_identity_token" not in result_json, "AWS web identity token should not be in request body"
- assert "aws_sts_endpoint" not in result_json, "AWS STS endpoint should not be in request body"
- assert "aws_bedrock_runtime_endpoint" not in result_json, "AWS bedrock endpoint should not be in request body"
- assert "aws_external_id" not in result_json, "AWS external ID should not be in request body"
-
+ assert (
+ "aws_access_key_id" not in result_json
+ ), "AWS access key should not be in request body"
+ assert (
+ "aws_secret_access_key" not in result_json
+ ), "AWS secret key should not be in request body"
+ assert (
+ "aws_session_token" not in result_json
+ ), "AWS session token should not be in request body"
+ assert (
+ "aws_region_name" not in result_json
+ ), "AWS region should not be in request body"
+ assert (
+ "aws_role_name" not in result_json
+ ), "AWS role name should not be in request body"
+ assert (
+ "aws_session_name" not in result_json
+ ), "AWS session name should not be in request body"
+ assert (
+ "aws_profile_name" not in result_json
+ ), "AWS profile name should not be in request body"
+ assert (
+ "aws_web_identity_token" not in result_json
+ ), "AWS web identity token should not be in request body"
+ assert (
+ "aws_sts_endpoint" not in result_json
+ ), "AWS STS endpoint should not be in request body"
+ assert (
+ "aws_bedrock_runtime_endpoint" not in result_json
+ ), "AWS bedrock endpoint should not be in request body"
+ assert (
+ "aws_external_id" not in result_json
+ ), "AWS external ID should not be in request body"
+
# Also check that the sensitive values themselves are not in the response
- assert "AKIAIOSFODNN7EXAMPLE" not in result_json, "AWS access key value leaked in request body"
- assert "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" not in result_json, "AWS secret key value leaked in request body"
- assert "arn:aws:iam::123456789012:role/test-role" not in result_json, "AWS role ARN leaked in request body"
+ assert (
+ "AKIAIOSFODNN7EXAMPLE" not in result_json
+ ), "AWS access key value leaked in request body"
+ assert (
+ "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" not in result_json
+ ), "AWS secret key value leaked in request body"
+ assert (
+ "arn:aws:iam::123456789012:role/test-role" not in result_json
+ ), "AWS role ARN leaked in request body"
assert "test-session" not in result_json, "AWS session name leaked in request body"
-
+
# Verify normal params ARE still in the request body
assert result["max_tokens"] == 100, "max_tokens should be in request body"
assert result["temperature"] == 0.7, "temperature should be in request body"
assert result["top_p"] == 0.9, "top_p should be in request body"
-
+
# Verify Bedrock-specific params are added
- assert result["anthropic_version"] == "bedrock-2023-05-31", "anthropic_version should be set"
+ assert (
+ result["anthropic_version"] == "bedrock-2023-05-31"
+ ), "anthropic_version should be set"
assert "model" not in result, "model should be removed for Bedrock Invoke API"
assert "stream" not in result, "stream should be removed for Bedrock Invoke API"
-
+
# Verify messages are present
assert "messages" in result, "messages should be in request body"
assert len(result["messages"]) == 1, "should have 1 message"
@@ -109,41 +136,41 @@ def test_aws_params_filtered_from_request_body():
def test_output_format_conversion_to_inline_schema():
"""
Test that output_format is converted to inline schema in message content for Bedrock Invoke.
-
+
Bedrock Invoke doesn't support the output_format parameter, so LiteLLM converts it by
embedding the schema directly into the user message content.
"""
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
)
-
+
config = AmazonAnthropicClaudeMessagesConfig()
-
+
# Test messages
messages = [
- {"role": "user", "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan."}
+ {
+ "role": "user",
+ "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan.",
+ }
]
-
+
# Output format with schema
output_format_schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"email": {"type": "string"},
- "plan_interest": {"type": "string"}
+ "plan_interest": {"type": "string"},
},
"required": ["name", "email", "plan_interest"],
- "additionalProperties": False
+ "additionalProperties": False,
}
-
+
anthropic_messages_optional_request_params = {
"max_tokens": 1024,
- "output_format": {
- "type": "json_schema",
- "schema": output_format_schema
- }
+ "output_format": {"type": "json_schema", "schema": output_format_schema},
}
-
+
# Transform the request
result = config.transform_anthropic_messages_request(
model="anthropic.claude-sonnet-4-20250514-v1:0",
@@ -152,27 +179,29 @@ def test_output_format_conversion_to_inline_schema():
litellm_params={},
headers={},
)
-
+
# Verify output_format was removed from the request
- assert "output_format" not in result, "output_format should be removed from request body"
-
+ assert (
+ "output_format" not in result
+ ), "output_format should be removed from request body"
+
# Verify the schema was added to the last user message content
assert "messages" in result
last_user_message = result["messages"][0]
assert last_user_message["role"] == "user"
-
+
content = last_user_message["content"]
assert isinstance(content, list), "content should be a list"
assert len(content) == 2, "content should have 2 items (original text + schema)"
-
+
# Check original text is preserved
assert content[0]["type"] == "text"
assert "John Smith" in content[0]["text"]
-
+
# Check schema was added as JSON string
assert content[1]["type"] == "text"
schema_text = content[1]["text"]
-
+
# Parse the schema JSON
parsed_schema = json.loads(schema_text)
assert parsed_schema["type"] == "object"
@@ -180,7 +209,7 @@ def test_output_format_conversion_to_inline_schema():
assert "email" in parsed_schema["properties"]
assert "plan_interest" in parsed_schema["properties"]
assert parsed_schema["required"] == ["name", "email", "plan_interest"]
-
+
# Verify other params are preserved
assert result["max_tokens"] == 1024
assert result["anthropic_version"] == "bedrock-2023-05-31"
@@ -193,29 +222,22 @@ def test_output_format_conversion_with_string_content():
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
)
-
+
config = AmazonAnthropicClaudeMessagesConfig()
-
+
# Test messages with string content
- messages = [
- {"role": "user", "content": "What is 2+2?"}
- ]
-
+ messages = [{"role": "user", "content": "What is 2+2?"}]
+
output_format_schema = {
"type": "object",
- "properties": {
- "result": {"type": "integer"}
- }
+ "properties": {"result": {"type": "integer"}},
}
-
+
anthropic_messages_optional_request_params = {
"max_tokens": 100,
- "output_format": {
- "type": "json_schema",
- "schema": output_format_schema
- }
+ "output_format": {"type": "json_schema", "schema": output_format_schema},
}
-
+
# Transform the request
result = config.transform_anthropic_messages_request(
model="anthropic.claude-sonnet-4-20250514-v1:0",
@@ -224,17 +246,17 @@ def test_output_format_conversion_with_string_content():
litellm_params={},
headers={},
)
-
+
# Verify the content was converted to list format
last_user_message = result["messages"][0]
content = last_user_message["content"]
assert isinstance(content, list), "content should be converted to list"
assert len(content) == 2, "content should have 2 items"
-
+
# Check original text
assert content[0]["type"] == "text"
assert content[0]["text"] == "What is 2+2?"
-
+
# Check schema was added
assert content[1]["type"] == "text"
parsed_schema = json.loads(content[1]["text"])
@@ -248,21 +270,19 @@ def test_output_format_with_no_schema():
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
)
-
+
config = AmazonAnthropicClaudeMessagesConfig()
-
- messages = [
- {"role": "user", "content": "Hello"}
- ]
-
+
+ messages = [{"role": "user", "content": "Hello"}]
+
anthropic_messages_optional_request_params = {
"max_tokens": 100,
"output_format": {
"type": "json_schema"
# No schema field
- }
+ },
}
-
+
# Transform the request
result = config.transform_anthropic_messages_request(
model="anthropic.claude-sonnet-4-20250514-v1:0",
@@ -271,11 +291,11 @@ def test_output_format_with_no_schema():
litellm_params={},
headers={},
)
-
+
# Verify output_format was removed but no schema was added
assert "output_format" not in result
last_user_message = result["messages"][0]
-
+
# Content should remain as string (not converted to list)
assert isinstance(last_user_message["content"], str)
assert last_user_message["content"] == "Hello"
@@ -289,9 +309,9 @@ def test_opus_4_5_model_detection():
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
)
-
+
config = AmazonAnthropicClaudeMessagesConfig()
-
+
# Test various Opus 4.5 naming patterns
opus_4_5_models = [
"anthropic.claude-opus-4-5-20250514-v1:0",
@@ -301,11 +321,10 @@ def test_opus_4_5_model_detection():
"us.anthropic.claude-opus-4-5-20250514-v1:0",
"ANTHROPIC.CLAUDE-OPUS-4-5-20250514-V1:0", # Case insensitive
]
-
+
for model in opus_4_5_models:
- assert config._is_claude_opus_4_5(model), \
- f"Should detect {model} as Opus 4.5"
-
+ assert config._is_claude_opus_4_5(model), f"Should detect {model} as Opus 4.5"
+
# Test non-Opus 4.5 models
non_opus_4_5_models = [
"anthropic.claude-sonnet-4-5-20250929-v1:0",
@@ -313,85 +332,19 @@ def test_opus_4_5_model_detection():
"anthropic.claude-opus-4-1-20250514-v1:0", # Opus 4.1, not 4.5
"anthropic.claude-haiku-4-5-20251001-v1:0",
]
-
+
for model in non_opus_4_5_models:
- assert not config._is_claude_opus_4_5(model), \
- f"Should not detect {model} as Opus 4.5"
-
-
-# def test_structured_outputs_beta_header_filtered_for_bedrock_invoke():
-# """
-# Test that unsupported beta headers are filtered out for Bedrock Invoke API.
-
-# Bedrock Invoke API only supports a specific whitelist of beta flags and returns
-# "invalid beta flag" error for others (e.g., structured-outputs, mcp-servers).
-# This test ensures unsupported headers are filtered while keeping supported ones.
-
-# Fixes: https://github.com/BerriAI/litellm/issues/16726
-# """
-# config = AmazonAnthropicClaudeConfig()
-
-# messages = [{"role": "user", "content": "test"}]
-
-# # Test 1: structured-outputs beta header (unsupported)
-# headers = {"anthropic-beta": "structured-outputs-2025-11-13"}
-
-# result = config.transform_request(
-# model="anthropic.claude-4-0-sonnet-20250514-v1:0",
-# messages=messages,
-# optional_params={},
-# litellm_params={},
-# headers=headers,
-# )
-
-# # Verify structured-outputs beta is filtered out
-# anthropic_beta = result.get("anthropic_beta", [])
-# assert not any("structured-outputs" in beta for beta in anthropic_beta), \
-# f"structured-outputs beta should be filtered, got: {anthropic_beta}"
-
-# # Test 2: mcp-servers beta header (unsupported - the main issue from #16726)
-# headers = {"anthropic-beta": "mcp-servers-2025-12-04"}
-
-# result = config.transform_request(
-# model="anthropic.claude-4-0-sonnet-20250514-v1:0",
-# messages=messages,
-# optional_params={},
-# litellm_params={},
-# headers=headers,
-# )
-
-# # Verify mcp-servers beta is filtered out
-# anthropic_beta = result.get("anthropic_beta", [])
-# assert not any("mcp-servers" in beta for beta in anthropic_beta), \
-# f"mcp-servers beta should be filtered, got: {anthropic_beta}"
-
-# # Test 3: Mix of supported and unsupported beta headers
-# headers = {"anthropic-beta": "computer-use-2024-10-22,mcp-servers-2025-12-04,structured-outputs-2025-11-13"}
-
-# result = config.transform_request(
-# model="anthropic.claude-4-0-sonnet-20250514-v1:0",
-# messages=messages,
-# optional_params={},
-# litellm_params={},
-# headers=headers,
-# )
-
-# # Verify only supported betas are kept
-# anthropic_beta = result.get("anthropic_beta", [])
-# assert not any("structured-outputs" in beta for beta in anthropic_beta), \
-# f"structured-outputs beta should be filtered, got: {anthropic_beta}"
-# assert not any("mcp-servers" in beta for beta in anthropic_beta), \
-# f"mcp-servers beta should be filtered, got: {anthropic_beta}"
-# assert any("computer-use" in beta for beta in anthropic_beta), \
-# f"computer-use beta should be kept, got: {anthropic_beta}"
+ assert not config._is_claude_opus_4_5(
+ model
+ ), f"Should not detect {model} as Opus 4.5"
def test_output_config_removed_from_bedrock_chat_invoke_request():
"""
- Test that output_config parameter is stripped from Bedrock Chat Invoke requests.
-
- Bedrock Invoke API doesn't support the output_config parameter (Anthropic-only).
- Ensures the chat/invoke path mirrors the messages/invoke path fix.
+ Test that output_config is stripped for models that do not support native
+ structured outputs on Bedrock Invoke. Models that *do* support it (e.g.
+ claude-sonnet-4-5) keep output_config -- see the native structured output
+ tests below.
Fixes: https://github.com/BerriAI/litellm/issues/22797
"""
@@ -413,9 +366,9 @@ def test_output_config_removed_from_bedrock_chat_invoke_request():
headers={},
)
- assert "output_config" not in result, (
- f"output_config should be stripped for Bedrock Chat Invoke, got keys: {list(result.keys())}"
- )
+ assert (
+ "output_config" not in result
+ ), f"output_config should be stripped for Bedrock Chat Invoke, got keys: {list(result.keys())}"
# Verify normal params survive
assert result["max_tokens"] == 100
@@ -423,20 +376,18 @@ def test_output_config_removed_from_bedrock_chat_invoke_request():
def test_output_format_removed_from_bedrock_invoke_request():
"""
Test that output_format parameter is removed from Bedrock Invoke requests.
-
+
Bedrock Invoke API doesn't support the output_format parameter (only supported
in Anthropic Messages API). This test ensures it's removed to prevent errors.
"""
config = AmazonAnthropicClaudeConfig()
-
+
messages = [{"role": "user", "content": "test"}]
-
+
# Create a request with output_format via map_openai_params
- non_default_params = {
- "response_format": {"type": "json_object"}
- }
+ non_default_params = {"response_format": {"type": "json_object"}}
optional_params = {}
-
+
# This should trigger tool-based structured outputs
optional_params = config.map_openai_params(
non_default_params=non_default_params,
@@ -444,7 +395,7 @@ def test_output_format_removed_from_bedrock_invoke_request():
model="anthropic.claude-4-0-sonnet-20250514-v1:0",
drop_params=False,
)
-
+
result = config.transform_request(
model="anthropic.claude-4-0-sonnet-20250514-v1:0",
messages=messages,
@@ -452,7 +403,216 @@ def test_output_format_removed_from_bedrock_invoke_request():
litellm_params={},
headers={},
)
-
+
# Verify output_format is not in the request
- assert "output_format" not in result, \
- f"output_format should be removed for Bedrock Invoke, got keys: {result.keys()}"
+ assert (
+ "output_format" not in result
+ ), f"output_format should be removed for Bedrock Invoke, got keys: {result.keys()}"
+
+
+# ---------------------------------------------------------------------------
+# Native structured output tests
+# ---------------------------------------------------------------------------
+
+SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT = {
+ "type": "json_schema",
+ "json_schema": {
+ "name": "contact_info",
+ "schema": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "email": {"type": "string"},
+ "address": {
+ "type": "object",
+ "properties": {
+ "city": {"type": "string"},
+ "zip": {"type": "string"},
+ },
+ "required": ["city", "zip"],
+ },
+ },
+ "required": ["name", "email", "address"],
+ },
+ },
+}
+
+
+def test_supports_native_structured_outputs():
+ """Verify model substring matching for native structured output support."""
+ supported = [
+ "anthropic.claude-haiku-4-5-20251001-v1:0",
+ "anthropic.claude-sonnet-4-5-20250929-v1:0",
+ "anthropic.claude-sonnet-4-6-20260301-v1:0",
+ "us.anthropic.claude-opus-4-5-20250514-v1:0",
+ "anthropic.claude-opus-4-6-20260101-v1:0",
+ ]
+ unsupported = [
+ "anthropic.claude-3-5-sonnet-20241022-v2:0",
+ "anthropic.claude-sonnet-4-20250514-v1:0",
+ "anthropic.claude-3-haiku-20240307-v1:0",
+ ]
+ for m in supported:
+ assert AmazonAnthropicClaudeConfig._supports_native_structured_outputs(
+ m
+ ), f"Expected {m} to be supported"
+ for m in unsupported:
+ assert not AmazonAnthropicClaudeConfig._supports_native_structured_outputs(
+ m
+ ), f"Expected {m} to be unsupported"
+
+
+def test_native_structured_output_supported_model():
+ """Supported model should produce output_format, not tools."""
+ config = AmazonAnthropicClaudeConfig()
+ optional_params = config.map_openai_params(
+ non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT},
+ optional_params={},
+ model="anthropic.claude-sonnet-4-5-20250929-v1:0",
+ drop_params=False,
+ )
+
+ assert "output_format" in optional_params
+ assert optional_params["output_format"]["type"] == "json_schema"
+ assert optional_params.get("json_mode") is True
+ # Should NOT inject a synthetic tool
+ assert "tools" not in optional_params
+ assert "tool_choice" not in optional_params
+
+
+def test_native_structured_output_unsupported_model():
+ """Unsupported model should fall back to tool-call approach."""
+ config = AmazonAnthropicClaudeConfig()
+ optional_params = config.map_openai_params(
+ non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT},
+ optional_params={},
+ model="anthropic.claude-3-5-sonnet-20241022-v2:0",
+ drop_params=False,
+ )
+
+ # Should use tool-based approach
+ assert "tools" in optional_params
+ assert "output_format" not in optional_params
+
+
+def test_native_structured_output_haiku_4_5():
+ """
+ Haiku 4.5 is Bedrock-only (not on the direct Anthropic API), so it needs
+ special handling in the Bedrock invoke class rather than the parent.
+ """
+ config = AmazonAnthropicClaudeConfig()
+ optional_params = config.map_openai_params(
+ non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT},
+ optional_params={},
+ model="anthropic.claude-haiku-4-5-20251001-v1:0",
+ drop_params=False,
+ )
+
+ assert "output_format" in optional_params
+ assert optional_params["output_format"]["type"] == "json_schema"
+ assert optional_params.get("json_mode") is True
+ assert "tools" not in optional_params
+
+
+def test_native_structured_output_transform_request():
+ """
+ End-to-end: map_openai_params + transform_request should produce
+ output_config.format in the final request body.
+ """
+ config = AmazonAnthropicClaudeConfig()
+ model = "anthropic.claude-sonnet-4-5-20250929-v1:0"
+ messages = [{"role": "user", "content": "Extract contact info."}]
+
+ optional_params = config.map_openai_params(
+ non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT},
+ optional_params={},
+ model=model,
+ drop_params=False,
+ )
+
+ result = config.transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params={},
+ headers={},
+ )
+
+ # output_config.format should be present
+ assert (
+ "output_config" in result
+ ), f"Expected output_config, got keys: {list(result.keys())}"
+ assert result["output_config"]["format"]["type"] == "json_schema"
+ assert "schema" in result["output_config"]["format"]
+ # output_format should NOT leak into the request
+ assert "output_format" not in result
+
+
+def test_native_structured_output_schema_normalization():
+ """additionalProperties: false should be recursively added to all object nodes."""
+ config = AmazonAnthropicClaudeConfig()
+ model = "anthropic.claude-sonnet-4-5-20250929-v1:0"
+ messages = [{"role": "user", "content": "test"}]
+
+ optional_params = config.map_openai_params(
+ non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT},
+ optional_params={},
+ model=model,
+ drop_params=False,
+ )
+
+ result = config.transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params={},
+ headers={},
+ )
+
+ schema = result["output_config"]["format"]["schema"]
+ # Top-level object
+ assert schema.get("additionalProperties") is False
+ # Nested object (address)
+ assert schema["properties"]["address"].get("additionalProperties") is False
+
+
+def test_native_structured_output_unsupported_model_no_output_config():
+ """Unsupported model should NOT have output_config in the final request."""
+ config = AmazonAnthropicClaudeConfig()
+ model = "anthropic.claude-3-5-sonnet-20241022-v2:0"
+ messages = [{"role": "user", "content": "test"}]
+
+ optional_params = config.map_openai_params(
+ non_default_params={"response_format": SAMPLE_JSON_SCHEMA_RESPONSE_FORMAT},
+ optional_params={},
+ model=model,
+ drop_params=False,
+ )
+
+ result = config.transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params={},
+ headers={},
+ )
+
+ assert "output_config" not in result
+ assert "output_format" not in result
+
+
+def test_native_structured_output_json_object_fallback():
+ """
+ json_object with no schema on a supported model should NOT produce
+ native output_format (there is no schema to constrain). The parent
+ also skips json_object-without-schema, so nothing is injected.
+ """
+ config = AmazonAnthropicClaudeConfig()
+ optional_params = config.map_openai_params(
+ non_default_params={"response_format": {"type": "json_object"}},
+ optional_params={},
+ model="anthropic.claude-sonnet-4-5-20250929-v1:0",
+ drop_params=False,
+ )
+
+ assert "output_format" not in optional_params