From 27d695bd774e8261b945134213049a059e40f80f Mon Sep 17 00:00:00 2001 From: hzt <3061613175@qq.com> Date: Fri, 20 Feb 2026 14:00:36 +0800 Subject: [PATCH] fix(bedrock): correct modelInput format for Converse API batch models For Bedrock batch file uploads targeting Converse API models (e.g. Nova), the modelInput was incorrectly using raw OpenAI passthrough format. This caused image_url blocks to be kept as-is (not converted to Bedrock image format) and inference params like max_tokens to remain at the top level instead of being wrapped in inferenceConfig. Route Nova (and future Converse-format providers) through AmazonConverseConfig.transform_request() so that: - image_url content blocks are converted to Bedrock image format - max_tokens/temperature/top_p are wrapped in inferenceConfig - messages use the Converse content block structure Fixes #21596 --- litellm/llms/bedrock/files/transformation.py | 76 ++++-- .../expected_bedrock_batch_completions.jsonl | 4 +- .../test_bedrock_files_transformation.py | 220 ++++++++++++++++++ 3 files changed, 276 insertions(+), 24 deletions(-) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index fdcbe1a8242..20ec233808d 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -202,52 +202,84 @@ def map_openai_params( return optional_params + # Providers whose InvokeModel body uses the Converse API format + # (messages + inferenceConfig + image blocks). Nova is the primary + # example; add others here as they adopt the same schema. + CONVERSE_INVOKE_PROVIDERS = ("nova",) + def _map_openai_to_bedrock_params( self, openai_request_body: Dict[str, Any], provider: Optional[str] = None, ) -> Dict[str, Any]: """ - Transform OpenAI request body to Bedrock-compatible modelInput parameters using existing transformation logic + Transform OpenAI request body to Bedrock-compatible modelInput + parameters using existing transformation logic. + + Routes to the correct per-provider transformation so that the + resulting dict matches the InvokeModel body that Bedrock expects + for batch inference. """ from litellm.types.utils import LlmProviders + _model = openai_request_body.get("model", "") messages = openai_request_body.get("messages", []) - - # Use existing Anthropic transformation logic for Anthropic models + optional_params = { + k: v + for k, v in openai_request_body.items() + if k not in ["model", "messages"] + } + + # --- Anthropic: use existing AmazonAnthropicClaudeConfig --- if provider == LlmProviders.ANTHROPIC: from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) - - anthropic_config = AmazonAnthropicClaudeConfig() - - # Extract optional params (everything except model and messages) - optional_params = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]} - mapped_params = anthropic_config.map_openai_params( + + config = AmazonAnthropicClaudeConfig() + mapped_params = config.map_openai_params( non_default_params={}, optional_params=optional_params, model=_model, - drop_params=False + drop_params=False, ) - - # Transform using existing Anthropic logic - bedrock_params = anthropic_config.transform_request( + return config.transform_request( model=_model, messages=messages, optional_params=mapped_params, litellm_params={}, - headers={} + headers={}, ) - return bedrock_params - else: - # For other providers, use basic mapping - bedrock_params = { - "messages": messages, - **{k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]} - } - return bedrock_params + # --- Converse API providers (e.g. Nova): use AmazonConverseConfig + # to correctly convert image_url blocks to Bedrock image format + # and wrap inference params inside inferenceConfig. --- + if provider in self.CONVERSE_INVOKE_PROVIDERS: + from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, + ) + + config = AmazonConverseConfig() + mapped_params = config.map_openai_params( + non_default_params=optional_params, + optional_params={}, + model=_model, + drop_params=False, + ) + return config.transform_request( + model=_model, + messages=messages, + optional_params=mapped_params, + litellm_params={}, + headers={}, + ) + + # --- All other providers: passthrough (OpenAI-compatible models + # like openai.gpt-oss-*, qwen, deepseek, etc.) --- + return { + "messages": messages, + **optional_params, + } def _transform_openai_jsonl_content_to_bedrock_jsonl_content( self, openai_jsonl_content: List[Dict[str, Any]] diff --git a/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl b/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl index c58963bb1de..8bb35ba95d7 100644 --- a/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl +++ b/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl @@ -1,2 +1,2 @@ -{"recordId": "request-1", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are a helpful assistant."}], "anthropic_version": "bedrock-2023-05-31"}} -{"recordId": "request-2", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are an unhelpful assistant."}], "anthropic_version": "bedrock-2023-05-31"}} +{"recordId": "request-1", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are a helpful assistant."}], "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": []}} +{"recordId": "request-2", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are an unhelpful assistant."}], "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": []}} diff --git a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py index 06e72539088..88cac84e438 100644 --- a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py @@ -88,3 +88,223 @@ def test_transform_openai_jsonl_content_to_bedrock_jsonl_content(self): print(f"\n=== Expected output written to: {expected_output_path} ===") + def test_nova_text_only_uses_converse_format(self): + """ + Test that Nova models produce Converse API format in batch modelInput. + + Verifies that: + - max_tokens is wrapped inside inferenceConfig.maxTokens + - messages use Converse content block format + - No raw OpenAI keys (max_tokens, temperature) at the top level + """ + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + + openai_jsonl_content = [ + { + "custom_id": "nova-text-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "us.amazon.nova-pro-v1:0", + "messages": [ + {"role": "user", "content": "What is the capital of France?"} + ], + "max_tokens": 50, + "temperature": 0.7, + }, + } + ] + + result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content + ) + + assert len(result) == 1 + record = result[0] + assert record["recordId"] == "nova-text-1" + + model_input = record["modelInput"] + + # Must have inferenceConfig with maxTokens, NOT top-level max_tokens + assert "inferenceConfig" in model_input, ( + "Nova modelInput must contain inferenceConfig" + ) + assert model_input["inferenceConfig"]["maxTokens"] == 50 + assert model_input["inferenceConfig"]["temperature"] == 0.7 + assert "max_tokens" not in model_input, ( + "max_tokens must NOT be at the top level for Nova" + ) + assert "temperature" not in model_input, ( + "temperature must NOT be at the top level for Nova" + ) + + # Must have messages + assert "messages" in model_input + + def test_nova_image_content_uses_converse_image_blocks(self): + """ + Test that image_url content blocks are converted to Bedrock Converse + image format for Nova models in batch. + + Verifies that: + - image_url blocks are converted to {"image": {"format": ..., "source": {"bytes": ...}}} + - text blocks are converted to {"text": "..."} + - No raw OpenAI image_url type remains + """ + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + + # 1x1 transparent PNG + img_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4" + "2mP8z8BQDwADhQGAWjR9awAAAABJRU5ErkJggg==" + ) + + openai_jsonl_content = [ + { + "custom_id": "nova-img-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "us.amazon.nova-pro-v1:0", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + img_b64 + }, + }, + ], + } + ], + "max_tokens": 100, + }, + } + ] + + result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content + ) + + assert len(result) == 1 + model_input = result[0]["modelInput"] + + # Check inferenceConfig + assert "inferenceConfig" in model_input + assert model_input["inferenceConfig"]["maxTokens"] == 100 + assert "max_tokens" not in model_input + + # Check messages structure + messages = model_input["messages"] + assert len(messages) == 1 + content_blocks = messages[0]["content"] + + # Should have text block and image block in Converse format + has_text = False + has_image = False + for block in content_blocks: + if "text" in block: + has_text = True + if "image" in block: + has_image = True + # Verify Converse image format + assert "format" in block["image"], ( + "Image block must have format field" + ) + assert "source" in block["image"], ( + "Image block must have source field" + ) + assert "bytes" in block["image"]["source"], ( + "Image source must have bytes field" + ) + # Must NOT have OpenAI-style image_url + assert "image_url" not in block, ( + "image_url must not appear in Converse format" + ) + assert block.get("type") != "image_url", ( + "type=image_url must not appear in Converse format" + ) + + assert has_text, "Should have a text content block" + assert has_image, "Should have an image content block" + + def test_anthropic_still_works_after_nova_fix(self): + """ + Regression test: ensure Anthropic models are still correctly + transformed after the Converse API provider changes. + """ + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + + openai_jsonl_content = [ + { + "custom_id": "claude-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "messages": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello!"}, + ], + "max_tokens": 10, + }, + } + ] + + result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content + ) + + assert len(result) == 1 + model_input = result[0]["modelInput"] + + # Anthropic should have anthropic_version + assert "anthropic_version" in model_input + assert "messages" in model_input + assert "max_tokens" in model_input + + def test_openai_passthrough_still_works(self): + """ + Regression test: ensure OpenAI-compatible models (e.g. gpt-oss) + still use passthrough format. + """ + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + + openai_jsonl_content = [ + { + "custom_id": "openai-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "openai.gpt-oss-120b-1:0", + "messages": [ + {"role": "user", "content": "Hello!"}, + ], + "max_tokens": 10, + }, + } + ] + + result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content + ) + + assert len(result) == 1 + model_input = result[0]["modelInput"] + + # OpenAI-compatible should use passthrough: max_tokens at top level + assert "messages" in model_input + assert "max_tokens" in model_input + assert model_input["max_tokens"] == 10 +