Skip to content

feat(bedrock): support native structured outputs for Invoke API (Claude 4.5+) #23778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ndgigliotti wants to merge 2 commits into
base: main
Choose a base branch
from
+613 −495
Draft

feat(bedrock): support native structured outputs for Invoke API (Claude 4.5+) #23778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
af16f12
feat(bedrock): support native structured outputs for Invoke API (Clau…
ndgigliotti Mar 14, 2026
f7287f0
refactor: extract schema helper to common_utils, clean up tests
ndgigliotti Mar 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 26 additions & 52 deletions litellm/llms/bedrock/chat/converse_transformation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,8 +493,7 @@ def _clamp_thinking_budget_tokens(optional_params: dict) -> None:
budget = thinking.get("budget_tokens")
if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS:
verbose_logger.debug(
"Bedrock requires thinking.budget_tokens >= %d, got %d. "
"Clamping to minimum.",
"Bedrock requires thinking.budget_tokens >= %d, got %d. Clamping to minimum.",
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
budget,
)
Expand Down Expand Up @@ -778,39 +777,14 @@ def _add_additional_properties_to_schema(schema: dict) -> dict:
Bedrock's native structured-outputs API requires this field to be
explicitly set on every object node, otherwise it returns a
validation error.
"""
if not isinstance(schema, dict):
return schema

result = dict(schema)

if result.get("type") == "object" and "additionalProperties" not in result:
result["additionalProperties"] = False

# Recurse into nested schemas
if "properties" in result and isinstance(result["properties"], dict):
result["properties"] = {
k: AmazonConverseConfig._add_additional_properties_to_schema(v)
for k, v in result["properties"].items()
}
if "items" in result and isinstance(result["items"], dict):
result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(
result["items"]
)
for defs_key in ("$defs", "definitions"):
if defs_key in result and isinstance(result[defs_key], dict):
result[defs_key] = {
k: AmazonConverseConfig._add_additional_properties_to_schema(v)
for k, v in result[defs_key].items()
}
for key in ("anyOf", "allOf", "oneOf"):
if key in result and isinstance(result[key], list):
result[key] = [
AmazonConverseConfig._add_additional_properties_to_schema(item)
for item in result[key]
]
Delegates to the shared implementation in ``bedrock/common_utils.py``.
"""
from litellm.llms.bedrock.common_utils import (
add_additional_properties_to_schema,
)

return result
return add_additional_properties_to_schema(schema)

@staticmethod
def _create_output_config_for_response_format(
Expand Down Expand Up @@ -913,7 +887,9 @@ def map_openai_params(
)
if param == "tool_choice":
_tool_choice_value = self.map_tool_choice_values(
model=model, tool_choice=value, drop_params=drop_params # type: ignore
model=model,
tool_choice=value,
drop_params=drop_params, # type: ignore
)
if _tool_choice_value is not None:
optional_params["tool_choice"] = _tool_choice_value
Expand Down Expand Up @@ -1748,9 +1724,7 @@ def apply_tool_call_transformation_if_needed(

return message, returned_finish_reason

def _translate_message_content(
self, content_blocks: List[ContentBlock]
) -> Tuple[
def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
str,
List[ChatCompletionToolCallChunk],
Optional[List[BedrockConverseReasoningContentBlock]],
Expand All @@ -1767,9 +1741,9 @@ def _translate_message_content(
"""
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
for idx, content in enumerate(content_blocks):
"""
Expand Down Expand Up @@ -1980,9 +1954,9 @@ def _transform_response( # noqa: PLR0915
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None

if message is not None:
Expand All @@ -2001,17 +1975,17 @@ def _transform_response( # noqa: PLR0915
provider_specific_fields["citationsContent"] = citationsContentBlocks

if provider_specific_fields:
chat_completion_message[
"provider_specific_fields"
] = provider_specific_fields
chat_completion_message["provider_specific_fields"] = (
provider_specific_fields
)

if reasoningContentBlocks is not None:
chat_completion_message[
"reasoning_content"
] = self._transform_reasoning_content(reasoningContentBlocks)
chat_completion_message[
"thinking_blocks"
] = self._transform_thinking_blocks(reasoningContentBlocks)
chat_completion_message["reasoning_content"] = (
self._transform_reasoning_content(reasoningContentBlocks)
)
chat_completion_message["thinking_blocks"] = (
self._transform_thinking_blocks(reasoningContentBlocks)
)
chat_completion_message["content"] = content_str
filtered_tools = self._filter_json_mode_tools(
json_mode=json_mode,
Expand Down
92 changes: 75 additions & 17 deletions litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
AmazonInvokeConfig,
)
from litellm.llms.bedrock.common_utils import (
add_additional_properties_to_schema,
get_anthropic_beta_from_headers,
remove_custom_field_from_tools,
)
Expand All @@ -21,6 +22,18 @@
else:
LiteLLMLoggingObj = Any

# Anthropic Claude models that support native structured outputs on Bedrock InvokeModel.
# Maintained separately from the Converse path's BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS
# because Invoke and Converse have independent feature rollouts.
# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS = {
"claude-haiku-4-5",
"claude-sonnet-4-5",
"claude-sonnet-4-6",
"claude-opus-4-5",
"claude-opus-4-6",
}
Comment on lines +29 to +35
Copy link
Contributor

@greptile-apps greptile-apps bot Mar 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hardcoded model list violates project policy

BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS is a hardcoded set of model name substrings, which means every time AWS adds a new Claude model that supports native structured outputs on the Invoke API, users must upgrade LiteLLM to get support.

The project convention (also violated by the existing BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS in converse_transformation.py) is to store these flags in model_prices_and_context_window.json and read them via get_model_info. This lets users pick up new model support without an SDK upgrade.

The recommended fix is to:

  1. Add a "supports_bedrock_invoke_structured_outputs": true key to each model entry in model_prices_and_context_window.json
  2. Replace _supports_native_structured_outputs with a lookup through get_model_info (similar to how supports_reasoning is used for the reasoning effort feature)
# Instead of:
BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS = {
    "claude-haiku-4-5",
    "claude-sonnet-4-5",
    ...
}

# Do something like:
from litellm.utils import get_model_info

def _supports_native_structured_outputs(model: str) -> bool:
    try:
        info = get_model_info(model=model, custom_llm_provider="bedrock")
        return bool(info.get("supports_bedrock_invoke_structured_outputs"))
    except Exception:
        return False

Note: the same issue exists in converse_transformation.py's BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS, but that's pre-existing. Fixing it here would be a good opportunity to align with the policy.

Rule Used: What: Do not hardcode model-specific flags in the ... (source)

Copy link
Contributor Author

@ndgigliotti ndgigliotti Mar 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Valid point. The model_prices_and_context_window.json approach would be cleaner long-term, but there are a couple of blockers for this PR:

  1. The Invoke path doesn't have its own model entries in the JSON -- there's only one bedrock/invoke/ entry and it's for an old model. The lookup would need to strip the bedrock/invoke/ prefix, handle inference profile IDs (us.anthropic.claude-sonnet-4-6), and fall back to the base Bedrock entry. That model ID resolution logic doesn't exist yet.
  2. The existing Converse path (BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS) uses the same hardcoded set pattern.

Happy to follow up with a separate PR to migrate both Invoke and Converse sets to JSON lookups if the maintainers prefer that approach.

Comment on lines +29 to +35
Copy link
Contributor

@greptile-apps greptile-apps bot Mar 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hardcoded model list should live in model_prices_and_context_window.json

Per project convention, model-capability flags should be stored in model_prices_and_context_window.json and read via get_model_info, not hardcoded here. Hardcoding means users must upgrade LiteLLM every time AWS adds a new Claude model to the Invoke native structured-output feature set.

The same pattern exists for the Converse path (BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS); both should eventually be migrated. The PR author has already noted this as a follow-up concern in the discussion thread.

Rule Used: What: Do not hardcode model-specific flags in the ... (source)



class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
"""
Expand Down Expand Up @@ -49,34 +62,60 @@ def custom_llm_provider(self) -> Optional[str]:
def get_supported_openai_params(self, model: str) -> List[str]:
return AnthropicConfig.get_supported_openai_params(self, model)

@staticmethod
def _supports_native_structured_outputs(model: str) -> bool:
"""Check if the Bedrock Invoke model supports native structured outputs."""
return any(
substring in model
for substring in BEDROCK_INVOKE_NATIVE_STRUCTURED_OUTPUT_MODELS
)

def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
# Force tool-based structured outputs for Bedrock Invoke
# (similar to VertexAI fix in #19201)
# Bedrock Invoke doesn't support output_format parameter
original_model = model
if "response_format" in non_default_params:
# Use a model name that forces tool-based approach
response_format = non_default_params.get("response_format")

# Native path: build output_format directly for Bedrock-supported models
# (includes haiku-4-5 which the Anthropic parent doesn't know about).
if isinstance(
response_format, dict
) and self._supports_native_structured_outputs(model):
_output_format = self.map_response_format_to_anthropic_output_format(
response_format
)
if _output_format is not None:
optional_params["output_format"] = _output_format
optional_params["json_mode"] = True
remaining = {
k: v
for k, v in non_default_params.items()
if k != "response_format"
}
return AnthropicConfig.map_openai_params(
self,
remaining,
optional_params,
model,
drop_params,
)

# Fallback: force tool-based structured outputs for unsupported models
# (or json_object without schema on a supported model).
if response_format is not None:
model = "claude-3-sonnet-20240229"
Comment on lines +106 to 109
Copy link
Contributor

@greptile-apps greptile-apps bot Mar 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fallback silently overrides model for tool injection

Setting model = "claude-3-sonnet-20240229" is a local variable override — it tricks the parent's map_openai_params into choosing the tool-based path by selecting an old model name that is known not to support native outputs. This is a subtle and fragile approach.

If the parent's native-supported model set ever changes (e.g., adds "claude-3-sonnet-20240229" to the native list — unlikely but possible), this fallback would break silently. A more explicit approach would be to directly call map_response_format_to_anthropic_tool on the fallback path rather than relying on an opaque model override.

Copy link
Contributor Author

@ndgigliotti ndgigliotti Mar 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed this is fragile. It's the pre-existing pattern from the code this PR refactored -- the previous implementation also overrode the model name the same way. Calling map_response_format_to_anthropic_tool directly would be cleaner, but that method also handles tool_choice injection and thinking-mode checks that are coupled to the parent's internal state. Extracting just the tool-based path without duplicating logic would require refactoring the parent class, which is out of scope here.

Open to revisiting if the maintainers want to refactor the parent class.


optional_params = AnthropicConfig.map_openai_params(
return AnthropicConfig.map_openai_params(
self,
non_default_params,
optional_params,
model,
drop_params,
)

# Restore original model name
model = original_model

return optional_params

def transform_request(
self,
model: str,
Expand Down Expand Up @@ -105,11 +144,30 @@ def transform_request(

_anthropic_request.pop("model", None)
_anthropic_request.pop("stream", None)
# Bedrock Invoke doesn't support output_format parameter
_anthropic_request.pop("output_format", None)
# Bedrock Invoke doesn't support output_config parameter
# Fixes: https://github.com/BerriAI/litellm/issues/22797
_anthropic_request.pop("output_config", None)

# Convert Anthropic output_format to Bedrock InvokeModel output_config.format
output_format = _anthropic_request.pop("output_format", None)
if (
output_format
and isinstance(output_format, dict)
and output_format.get("type") == "json_schema"
):
schema = output_format.get("schema", {})
normalized_schema = add_additional_properties_to_schema(schema)
# Preserve existing output_config keys (e.g. effort from reasoning_effort)
output_config = _anthropic_request.get("output_config") or {}
output_config["format"] = {
"type": "json_schema",
"schema": normalized_schema,
}
_anthropic_request["output_config"] = output_config
else:
# Non-native path: strip output_config entirely.
# Bedrock Invoke rejects the key itself (not just sub-keys) with
# "extraneous key [output_config] is not permitted" for models
# that don't support native structured outputs.
# Fixes: https://github.com/BerriAI/litellm/issues/22797
_anthropic_request.pop("output_config", None)
if "anthropic_version" not in _anthropic_request:
_anthropic_request["anthropic_version"] = self.anthropic_version

Expand Down
48 changes: 45 additions & 3 deletions litellm/llms/bedrock/common_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,46 @@ def get_cached_model_info():
return _get_model_info


def add_additional_properties_to_schema(schema: dict) -> dict:
"""
Recursively ensure all object types in a JSON schema have
``"additionalProperties": false``.

Bedrock's native structured-outputs API requires this field to be
explicitly set on every object node, otherwise it returns a
validation error.
"""
if not isinstance(schema, dict):
return schema

result = dict(schema)

if result.get("type") == "object" and "additionalProperties" not in result:
result["additionalProperties"] = False

# Recurse into nested schemas
if "properties" in result and isinstance(result["properties"], dict):
result["properties"] = {
k: add_additional_properties_to_schema(v)
for k, v in result["properties"].items()
}
if "items" in result and isinstance(result["items"], dict):
result["items"] = add_additional_properties_to_schema(result["items"])
for defs_key in ("$defs", "definitions"):
if defs_key in result and isinstance(result[defs_key], dict):
result[defs_key] = {
k: add_additional_properties_to_schema(v)
for k, v in result[defs_key].items()
}
for key in ("anyOf", "allOf", "oneOf"):
if key in result and isinstance(result[key], list):
result[key] = [
add_additional_properties_to_schema(item) for item in result[key]
]

return result


def remove_custom_field_from_tools(request_body: dict) -> None:
"""
Remove ``custom`` field from each tool in the request body.
Expand Down Expand Up @@ -1062,9 +1102,11 @@ def sign_aws_request(

return (
dict(prepped.headers),
request_data.encode("utf-8")
if isinstance(request_data, str)
else request_data,
(
request_data.encode("utf-8")
if isinstance(request_data, str)
else request_data
),
)

def generate_unique_job_name(self, model: str, prefix: str = "litellm") -> str:
Expand Down
Loading
Loading