diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f4b5de8f7c0..bdcc8ab8c24 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -425,6 +425,15 @@ def strip_bedrock_routing_prefix(model: str) -> str: return model +def strip_bedrock_throughput_suffix(model: str) -> str: + """ Strip throughput tier suffixes from Bedrock model names. """ + import re + + # Pattern matches model:version:throughput where throughput is like 51k, 18k, etc. + # Keep the model:version part, strip the :throughput suffix + return re.sub(r"(:\d+):\d+k$", r"\1", model) + + def get_bedrock_base_model(model: str) -> str: """ Get the base model from the given model name. @@ -432,9 +441,11 @@ def get_bedrock_base_model(model: str) -> str: Handle model names like: - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" - "bedrock/converse/model" -> "model" + - "anthropic.claude-3-5-sonnet-20241022-v2:0:51k" -> "anthropic.claude-3-5-sonnet-20241022-v2:0" """ model = strip_bedrock_routing_prefix(model) model = extract_model_name_from_bedrock_arn(model) + model = strip_bedrock_throughput_suffix(model) potential_region = model.split(".", 1)[0] alt_potential_region = model.split("/", 1)[0] diff --git a/tests/llm_translation/test_bedrock_common_utils.py b/tests/llm_translation/test_bedrock_common_utils.py index 7b6a05b6988..d5ec4967058 100644 --- a/tests/llm_translation/test_bedrock_common_utils.py +++ b/tests/llm_translation/test_bedrock_common_utils.py @@ -12,6 +12,7 @@ get_bedrock_base_model, get_bedrock_cross_region_inference_regions, strip_bedrock_routing_prefix, + strip_bedrock_throughput_suffix, ) from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter @@ -46,6 +47,21 @@ def test_model_with_dots_unchanged(self): ) +class TestStripBedrockThroughputSuffix: + """Tests for strip_bedrock_throughput_suffix function.""" + + @pytest.mark.parametrize("input_model,expected", [ + ("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("model:1:51k", "model:1"), + ("model:123:18k", "model:123"), + ("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-3-sonnet", "anthropic.claude-3-sonnet"), + ]) + def test_strip_throughput_suffix(self, input_model, expected): + assert strip_bedrock_throughput_suffix(input_model) == expected + + class TestExtractModelNameFromBedrockArn: """Tests for extract_model_name_from_bedrock_arn function.""" @@ -118,6 +134,16 @@ def test_combined_bedrock_and_region_prefix(self): == "anthropic.claude-3-sonnet-20240229-v1:0" ) + @pytest.mark.parametrize("input_model,expected", [ + ("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("us.anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ]) + def test_strips_throughput_suffix(self, input_model, expected): + """Test that throughput tier suffixes like :51k are stripped. Issue #19113.""" + assert get_bedrock_base_model(input_model) == expected + class TestBedrockModelInfoWrappers: """Tests that BedrockModelInfo methods correctly wrap standalone functions."""