Skip to content

Fix bedrock empty content error #7177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
10 commits merged into from
Dec 12, 2024
Merged

Fix bedrock empty content error #7177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c717e37
add resolver
enyst Dec 10, 2024
b69ed3e
handle empty content on bedrock with default content
enyst Dec 11, 2024
cfbeb33
use existing default message, tests
enyst Dec 11, 2024
c1547a3
Update tests/llm_translation/test_bedrock_completion.py
enyst Dec 11, 2024
df5617d
fix tests
enyst Dec 11, 2024
62c41ce
Merge branch 'fix/bedrock-content-block-error' of github.com:enyst/li…
enyst Dec 11, 2024
9083c1a
Revert "add resolver"
enyst Dec 11, 2024
4149715
fallback to empty
enyst Dec 11, 2024
0ab80a7
Merge branch 'main' of github.com:BerriAI/litellm into fix/bedrock-co…
enyst Dec 11, 2024
de4950b
Merge branch 'litellm_dev_12_11_2024_v2' into fix/bedrock-content-blo…
Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions litellm/litellm_core_utils/prompt_templates/factory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2474,8 +2474,22 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
for element in messages[msg_i]["content"]:
if isinstance(element, dict):
if element["type"] == "text":
_part = BedrockContentBlock(text=element["text"])
_parts.append(_part)
if element["text"].strip():
_part = BedrockContentBlock(text=element["text"])
else:
# bedrock requires non-empty content
# insert a default text block if the user provided it, or if modify_params is True
if user_continue_message is not None:
_part = BedrockContentBlock(
text=user_continue_message["content"][0]["text"]
)
elif litellm.modify_params:
_part = BedrockContentBlock(
text=DEFAULT_USER_CONTINUE_MESSAGE["content"][0]["text"]
)
else:
_part = BedrockContentBlock(text="")
_parts.append(_part)
elif element["type"] == "image_url":
if isinstance(element["image_url"], dict):
image_url = element["image_url"]["url"]
Expand All @@ -2494,7 +2508,19 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
_parts.append(_cache_point_block)
user_content.extend(_parts)
else:
_part = BedrockContentBlock(text=messages[msg_i]["content"])
if messages[msg_i]["content"].strip():
# bedrock requires non-empty content
# insert a default text block if the user provided it, or if modify_params is True
if user_continue_message is not None:
_part = BedrockContentBlock(
text=user_continue_message["content"][0]["text"]
)
elif litellm.modify_params:
_part = BedrockContentBlock(
text=DEFAULT_USER_CONTINUE_MESSAGE["content"][0]["text"]
)
else:
_part = BedrockContentBlock(text="")
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
messages[msg_i], block_type="content_block"
Expand Down Expand Up @@ -2566,7 +2592,21 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
for element in messages[msg_i]["content"]:
if isinstance(element, dict):
if element["type"] == "text":
assistants_part = BedrockContentBlock(text=element["text"])
if element["text"].strip():
assistants_part = BedrockContentBlock(text=element["text"])
else:
# bedrock requires non-empty content
# insert a default text block if the user provided it, or if modify_params is True
if assistant_continue_message is not None:
assistants_part = BedrockContentBlock(
text=assistant_continue_message["content"][0]["text"]
)
elif litellm.modify_params:
assistants_part = BedrockContentBlock(
text=DEFAULT_ASSISTANT_CONTINUE_MESSAGE["content"][0]["text"]
)
else:
assistants_part = BedrockContentBlock(text="")
assistants_parts.append(assistants_part)
elif element["type"] == "image_url":
if isinstance(element["image_url"], dict):
Expand Down
107 changes: 107 additions & 0 deletions tests/llm_translation/test_bedrock_completion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2112,6 +2112,112 @@ def get_base_rerank_call_args(self) -> dict:
}


def test_bedrock_empty_content_handling():
"""
Test that empty content in messages is handled correctly with default messages
"""
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Hello!"
}
]
},
{
"role": "assistant",
"content": [
{
"type": "text",
"text": ""
}
]
}
]

# Test with default behavior (modify_params=True)
litellm.modify_params = True
formatted_messages = _bedrock_converse_messages_pt(
messages=messages,
model="anthropic.claude-3-sonnet-20240229-v1:0",
llm_provider="bedrock"
)

# Verify assistant message with default text was inserted
assert formatted_messages[1]["role"] == "assistant"
assert formatted_messages[1]["content"][0].text == "Please continue."

def test_bedrock_custom_continue_message():
"""
Test that custom continue messages are used when provided
"""
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Hello!"
}
]
},
{
"role": "assistant",
"content": [
{
"type": "text",
"text": " "
}
]
}
]

custom_continue = {
"role": "assistant",
"content": [
{
"text": "Custom continue message"
}
]
}

formatted_messages = _bedrock_converse_messages_pt(
messages=messages,
model="anthropic.claude-3-sonnet-20240229-v1:0",
llm_provider="bedrock",
assistant_continue_message=custom_continue
)

# Verify custom message was used
assert formatted_messages[1]["role"] == "assistant"
assert formatted_messages[1]["content"][0].text == "Custom continue message"

def test_bedrock_no_default_message():
"""
Test that empty content is handled correctly when modify_params=False
"""
messages = [
{"role": "user", "content": "Hello!"},
{"role": "assistant", "content": ""},
{"role": "user", "content": "Hi again"},
{"role": "assistant", "content": "Valid response"}
]

litellm.modify_params = False
formatted_messages = _bedrock_converse_messages_pt(
messages=messages,
model="anthropic.claude-3-sonnet-20240229-v1:0",
llm_provider="bedrock"
)

# Verify empty message is present and valid message remains
assistant_messages = [msg for msg in formatted_messages if msg["role"] == "assistant"]
assert len(assistant_messages) == 2 # Both empty and valid messages present
assert assistant_messages[0]["content"][0].text == "" # First message is empty
assert assistant_messages[1]["content"][0].text == "Valid response" # Second message is valid

@pytest.mark.parametrize("top_k_param", ["top_k", "topK"])
def test_bedrock_nova_topk(top_k_param):
litellm.set_verbose = True
Expand All @@ -2121,3 +2227,4 @@ def test_bedrock_nova_topk(top_k_param):
top_k_param: 10,
}
litellm.completion(**data)