diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 73e74c228ba5..26b627f62e61 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -121,6 +121,38 @@ def _prepare_completion_kwargs( Logging as LiteLLMLoggingObject, ) + # Cap max_tokens against the model's known output token limit. + # Without this, requests from clients like Claude Code (which may send + # large max_tokens values valid for other providers) will be rejected by + # models with stricter limits (e.g. Amazon Nova Pro: 10,000 tokens). + # + # By this point get_llm_provider() has already been called in the outer + # anthropic_messages_handler, so `model` is the stripped model name + # (e.g. "converse/us.amazon.nova-pro-v1:0") and `custom_llm_provider` + # is passed via extra_kwargs (e.g. "bedrock"). If no explicit provider + # is available we infer it from the model string. + _custom_llm_provider = (extra_kwargs or {}).get("custom_llm_provider") + try: + _lookup_provider = _custom_llm_provider + if _lookup_provider is None: + _, _lookup_provider, _, _ = litellm.utils.get_llm_provider(model) + model_info = litellm.get_model_info( + model=model, custom_llm_provider=_lookup_provider + ) + model_max_output = model_info.get("max_output_tokens") + if model_max_output is not None and max_tokens > model_max_output: + from litellm._logging import verbose_logger + + verbose_logger.debug( + "Anthropic adapter: capping max_tokens from %d to %d for model=%s", + max_tokens, + model_max_output, + model, + ) + max_tokens = model_max_output + except Exception: + pass + request_data = { "model": model, "messages": messages, @@ -163,7 +195,26 @@ def _prepare_completion_kwargs( "include_usage": True, } + # These params are only understood by Anthropic Claude models. + # When routing to a non-Anthropic backend (e.g. Bedrock Nova Pro, + # Llama, Mistral), they are rejected as unknown fields. We strip + # them here so that the underlying model receives a clean request. + # Note: "thinking" is intentionally excluded from this list because + # some non-Anthropic models (e.g. Qwen) support reasoning/thinking + # and the existing adapter logic already handles translation for those. + _anthropic_only_params = {"output_config"} + _target_provider = (extra_kwargs or {}).get("custom_llm_provider", "") + _is_anthropic_claude = _target_provider in ( + "anthropic", + ) or ( + _target_provider == "bedrock" + and "anthropic.claude" in completion_kwargs.get("model", "") + ) + excluded_keys = {"anthropic_messages"} + if not _is_anthropic_claude: + excluded_keys = excluded_keys | _anthropic_only_params + extra_kwargs = extra_kwargs or {} for key, value in extra_kwargs.items(): if ( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_handler.py new file mode 100644 index 000000000000..c69f73b7aaa4 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_handler.py @@ -0,0 +1,162 @@ +""" +Unit tests for LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs. + +Covers: +- max_tokens capping: prevents HTTP 400 from providers with strict output token limits + (e.g. Amazon Nova Pro: 10,000 tokens). +- output_config stripping: prevents HTTP 400 "extraneous key" errors from non-Anthropic + backends that don't understand Anthropic-specific parameters. +""" +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, +) + + +MESSAGES = [{"role": "user", "content": "hello"}] +MODEL = "converse/us.amazon.nova-pro-v1:0" +PROVIDER = "bedrock" +MODEL_MAX_OUTPUT = 10_000 + + +def _call(max_tokens, extra_kwargs=None): + """Helper: call _prepare_completion_kwargs and return the resolved max_tokens.""" + kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=max_tokens, + messages=MESSAGES, + model=MODEL, + extra_kwargs=extra_kwargs or {"custom_llm_provider": PROVIDER}, + ) + return kwargs["max_tokens"] + + +class TestMaxTokensCapping: + def test_caps_when_exceeds_limit(self): + """max_tokens above the model limit is silently capped to max_output_tokens.""" + with patch("litellm.get_model_info") as mock_info: + mock_info.return_value = {"max_output_tokens": MODEL_MAX_OUTPUT} + result = _call(max_tokens=16_000) + assert result == MODEL_MAX_OUTPUT + + def test_unchanged_when_within_limit(self): + """max_tokens at or below the model limit is left unchanged.""" + with patch("litellm.get_model_info") as mock_info: + mock_info.return_value = {"max_output_tokens": MODEL_MAX_OUTPUT} + result = _call(max_tokens=8_000) + assert result == 8_000 + + def test_unchanged_when_equal_to_limit(self): + """max_tokens exactly equal to the model limit is left unchanged.""" + with patch("litellm.get_model_info") as mock_info: + mock_info.return_value = {"max_output_tokens": MODEL_MAX_OUTPUT} + result = _call(max_tokens=MODEL_MAX_OUTPUT) + assert result == MODEL_MAX_OUTPUT + + def test_fallback_infers_provider_when_not_in_extra_kwargs(self): + """When custom_llm_provider is absent from extra_kwargs, max_tokens is still + capped correctly by inferring the provider from the model string.""" + with patch("litellm.utils.get_llm_provider") as mock_provider, \ + patch("litellm.get_model_info") as mock_info: + mock_provider.return_value = (MODEL, PROVIDER, None, None) + mock_info.return_value = {"max_output_tokens": MODEL_MAX_OUTPUT} + + result = _call(max_tokens=16_000, extra_kwargs={}) + + assert result == MODEL_MAX_OUTPUT + + def test_resilient_when_get_model_info_raises(self): + """If get_model_info raises, max_tokens is passed through unchanged.""" + with patch("litellm.get_model_info", side_effect=Exception("model not found")), \ + patch("litellm.utils.get_llm_provider", side_effect=Exception("no provider")): + result = _call(max_tokens=16_000) + assert result == 16_000 + + def test_no_cap_when_max_output_tokens_missing(self): + """If model_info has no max_output_tokens key, max_tokens is unchanged.""" + with patch("litellm.get_model_info") as mock_info: + mock_info.return_value = {} + result = _call(max_tokens=16_000) + assert result == 16_000 + + def test_no_cap_when_max_output_tokens_is_none(self): + """Explicit None max_output_tokens does not trigger capping.""" + with patch("litellm.get_model_info") as mock_info: + mock_info.return_value = {"max_output_tokens": None} + result = _call(max_tokens=16_000) + assert result == 16_000 + + def test_explicit_provider_used_before_inference(self): + """When custom_llm_provider is present, get_llm_provider is not called.""" + with patch("litellm.utils.get_llm_provider") as mock_provider, \ + patch("litellm.get_model_info") as mock_info: + mock_info.return_value = {"max_output_tokens": MODEL_MAX_OUTPUT} + _call(max_tokens=16_000, extra_kwargs={"custom_llm_provider": PROVIDER}) + mock_provider.assert_not_called() + + +def _call_with_output_config(extra_kwargs=None): + """Helper: call _prepare_completion_kwargs with output_config in extra_kwargs + and return the full completion kwargs dict.""" + kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model=MODEL, + extra_kwargs={ + "output_config": {"type": "text"}, + **(extra_kwargs or {"custom_llm_provider": PROVIDER}), + }, + ) + return kwargs + + +class TestOutputConfigStripping: + def test_stripped_for_bedrock_non_claude(self): + """output_config is stripped when targeting a non-Anthropic Bedrock model.""" + with patch("litellm.get_model_info", return_value={}): + kwargs = _call_with_output_config( + extra_kwargs={"custom_llm_provider": "bedrock"} + ) + assert "output_config" not in kwargs + + def test_stripped_for_non_anthropic_provider(self): + """output_config is stripped for any non-Anthropic provider.""" + with patch("litellm.get_model_info", return_value={}): + kwargs = _call_with_output_config( + extra_kwargs={"custom_llm_provider": "openai"} + ) + assert "output_config" not in kwargs + + def test_passed_through_for_anthropic_provider(self): + """output_config is preserved when targeting the Anthropic provider directly.""" + with patch("litellm.get_model_info", return_value={}): + kwargs = _call_with_output_config( + extra_kwargs={"custom_llm_provider": "anthropic"} + ) + assert "output_config" in kwargs + + def test_passed_through_for_bedrock_claude(self): + """output_config is preserved when targeting an Anthropic Claude model on Bedrock.""" + with patch("litellm.get_model_info", return_value={}): + kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + extra_kwargs={ + "output_config": {"type": "text"}, + "custom_llm_provider": "bedrock", + }, + ) + assert "output_config" in kwargs + + def test_stripped_when_no_provider_specified(self): + """output_config is stripped when no provider is given (defaults to non-Anthropic).""" + with patch("litellm.get_model_info", return_value={}): + kwargs = _call_with_output_config(extra_kwargs={}) + assert "output_config" not in kwargs