Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion litellm/llms/bedrock/chat/converse_transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import copy
import json
import re
import time
import types
from typing import Final, Literal, cast, overload
Expand Down Expand Up @@ -92,6 +93,7 @@
"bash_",
"text_editor_",
]
BEDROCK_OPENAI_COMPAT_MIN_MAX_TOKENS: Final = 16

# Beta header patterns that are not supported by Bedrock Converse API
# These will be filtered out to prevent errors
Expand Down Expand Up @@ -280,6 +282,10 @@ def _validate_request_metadata(self, metadata: dict) -> None:
llm_provider="bedrock",
)

@staticmethod
def _requires_min_max_tokens(model: str) -> bool:
return re.search(r"openai\.gpt-\d|xai\.grok-", model) is not None
Comment on lines +285 to +287

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Model capability is hardcoded

This name-pattern check violates the directive to store model-specific flags in model_prices_and_context_window.json and read them through get_model_info; fix before merging

Rule Used: What: Do not hardcode model-specific flags in the codebase. Instead, put them in model_prices_and_context_window.json and then read them in via get_model_info Why: Prevents need for users to upgrade litellm each time a new model supports this featu... (source)

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same shape as main's merged #41870. The pattern matches model families, not individual models, so new GPT or Grok releases need no cost-map entry

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That’s a fair distinction. I rechecked #41870: _requires_min_max_tokens is not identifying a finite catalog of model capabilities; it encodes Bedrock Converse’s shared minimum-token contract for the openai.gpt-* and xai.grok-* model families. Using get_model_info would require adding or updating every future model ID and would make the fix stale, whereas family matching intentionally covers new releases and mirrors the merged implementation on main. I’m withdrawing this finding; no change is needed.


def _is_nova_2_model(self, model: str) -> bool:
"""
Check if the model is a Nova 2 model that supports reasoningConfig.
Expand Down Expand Up @@ -858,7 +864,11 @@ def map_openai_params(
is_thinking_enabled=is_thinking_enabled,
)
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["maxTokens"] = value
optional_params["maxTokens"] = (
max(value, BEDROCK_OPENAI_COMPAT_MIN_MAX_TOKENS)
if isinstance(value, int) and self._requires_min_max_tokens(model)
else value
Comment on lines 878 to +882

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Opaque profiles bypass clamp

Opaque application profile ARNs hide the backing model, so max_tokens=1 remains unclamped and Bedrock rejects compatible-model requests

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Backport of #41870, where this was withdrawn. The regex clamps ARNs embedding the model id; opaque application profiles carry none, so get_model_info cannot resolve them

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That’s fair. Opaque application-profile ARNs contain no model identifier, and get_model_info cannot resolve the backing model from them, so there is no reliable way to apply this model-specific clamp without risking unrelated profiles. The regex intentionally covers only identifiable model IDs, including ARNs that embed them. I agree this finding should be withdrawn.

)
if param == "stream":
optional_params["stream"] = value
if param == "stop":
Expand Down
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "litellm"
version = "1.98.0"
version = "1.98.1"
description = "Library to easily interface with LLM API providers"
readme = "README.md"
requires-python = ">=3.10, <3.15"
Expand Down Expand Up @@ -306,7 +306,7 @@ members = ["enterprise", "litellm-proxy-extras"]
profile = "black"

[tool.commitizen]
version = "1.98.0"
version = "1.98.1"
version_files = [
"pyproject.toml:^version",
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,34 @@ def test_reasoning_with_forced_tool_choice_switches_to_auto():
assert optional_params["tool_choice"] == {"auto": {}}


@pytest.mark.parametrize(
"model, param, value, expected_max_tokens",
[
("us.openai.gpt-6-astra", "max_tokens", 1, 16),
("us.openai.gpt-6-astra", "max_completion_tokens", 1, 16),
("us.openai.gpt-6-astra", "max_tokens", 64, 64),
("us.xai.grok-4.6", "max_tokens", 1, 16),
("global.xai.grok-4.6", "max_completion_tokens", 1, 16),
("us.xai.grok-4.6", "max_tokens", 32, 32),
("anthropic.claude-sonnet-4-5-20250929-v1:0", "max_tokens", 1, 1),
("arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.openai.gpt-6-astra", "max_tokens", 1, 16),
("arn:aws:bedrock:us-east-1:123456789012:inference-profile/global.xai.grok-4.6", "max_tokens", 1, 16),
("arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123xyz", "max_tokens", 1, 1),
],
)
def test_map_openai_params_enforces_minimum_max_tokens_for_openai_compat_models(
model: str, param: str, value: int, expected_max_tokens: int
):
optional_params = AmazonConverseConfig().map_openai_params(
non_default_params={param: value},
optional_params={},
model=model,
drop_params=False,
)

assert optional_params["maxTokens"] == expected_max_tokens


@pytest.mark.parametrize(
"model",
[
Expand Down
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading