From b597463affb64c6e25744cd8f69f22a345340bb9 Mon Sep 17 00:00:00 2001 From: Kent Date: Sun, 21 Jun 2026 09:44:25 +0800 Subject: [PATCH 1/3] fix(bedrock): surface modeled HTTP status for mid-stream error events (#24608) --- litellm/llms/bedrock/chat/invoke_handler.py | 19 +----- litellm/llms/bedrock/common_utils.py | 63 +++++++++++++------ .../test_bedrock_completion.py | 26 ++++++-- 3 files changed, 67 insertions(+), 41 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 75b560b4d6d..9fca7bc61af 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -70,6 +70,7 @@ from ..common_utils import ( BedrockError, ModelResponseIterator, + build_bedrock_stream_error, get_bedrock_response_stream_shape, get_bedrock_tool_name, ) @@ -1841,23 +1842,7 @@ def _parse_message_from_event(self, event) -> Optional[str]: parsed_response = self.parser.parse(response_dict, response_stream_shape) if response_dict["status_code"] != 200: - decoded_body = response_dict["body"].decode() - if isinstance(decoded_body, dict): - error_message = decoded_body.get("message") - elif isinstance(decoded_body, str): - error_message = decoded_body - else: - error_message = "" - exception_status = response_dict["headers"].get(":exception-type") - error_message = exception_status + " " + error_message - raise BedrockError( - status_code=response_dict["status_code"], - message=( - json.dumps(error_message) - if isinstance(error_message, dict) - else error_message - ), - ) + raise build_bedrock_stream_error(response_dict, response_stream_shape) if "chunk" in parsed_response: chunk = parsed_response.get("chunk") if not chunk: diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index bdc5da321c6..6eb43eb5cd2 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -7,9 +7,21 @@ import functools import json import os -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Mapping, + Optional, + TypedDict, + Union, +) if TYPE_CHECKING: + from botocore.model import Shape + from litellm.types.llms.bedrock import BedrockCreateBatchRequest import httpx @@ -1132,6 +1144,37 @@ def get_bedrock_response_stream_shape(): return _load_bedrock_response_stream_shape() +class BedrockEventStreamResponseDict(TypedDict): + status_code: int + headers: Mapping[str, str] + body: bytes + + +def build_bedrock_stream_error( + response_dict: BedrockEventStreamResponseDict, + response_stream_shape: Shape | None, +) -> BedrockError: + """Build a BedrockError for a non-200 event-stream error event. + + botocore hard-codes HTTP 400 on every mid-stream error event, so the modeled + ResponseStream member's httpStatusCode is the real status. Resolve it from the + shape and fall back to the raw status when the type is not modeled. + """ + exception_type = response_dict["headers"].get(":exception-type") + decoded_body = response_dict["body"].decode() + message = f"{exception_type} {decoded_body}" if exception_type else decoded_body + + status_code = response_dict["status_code"] + if exception_type is not None and response_stream_shape is not None: + member = response_stream_shape.members.get(exception_type) + if member is not None: + modeled_status = (member.metadata or {}).get("error", {}).get("httpStatusCode") + if modeled_status is not None: + status_code = int(modeled_status) + + return BedrockError(status_code=status_code, message=message) + + class BedrockEventStreamDecoderBase: """ Base class for event stream decoding for Bedrock @@ -1156,23 +1199,7 @@ def _parse_message_from_event(self, event) -> Optional[str]: parsed_response = self.parser.parse(response_dict, response_stream_shape) if response_dict["status_code"] != 200: - decoded_body = response_dict["body"].decode() - if isinstance(decoded_body, dict): - error_message = decoded_body.get("message") - elif isinstance(decoded_body, str): - error_message = decoded_body - else: - error_message = "" - exception_status = response_dict["headers"].get(":exception-type") - error_message = exception_status + " " + error_message - raise BedrockError( - status_code=response_dict["status_code"], - message=( - json.dumps(error_message) - if isinstance(error_message, dict) - else error_message - ), - ) + raise build_bedrock_stream_error(response_dict, response_stream_shape) if "chunk" in parsed_response: chunk = parsed_response.get("chunk") if not chunk: diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index fa22ff6b392..f4c307e9c8a 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2502,19 +2502,34 @@ async def test_bedrock_image_url_sync_client(): mock_post.assert_called_once() -def test_bedrock_error_handling_streaming(): +@pytest.mark.parametrize( + "exception_type, expected_status_code", + [ + ("internalServerException", 500), + ("serviceUnavailableException", 503), + ("modelTimeoutException", 408), + ("modelStreamErrorException", 424), + ("validationException", 400), + ], +) +def test_bedrock_error_handling_streaming(exception_type, expected_status_code): + """Bedrock event-stream error events arrive with botocore's hard-coded + status_code=400; the decoder must surface the modeled HTTP status instead + (e.g. internalServerException -> 500). For 5xx this is what makes the error + retryable downstream; for all types it replaces the misleading 400 with the + true code. Regression for #24608.""" from litellm.llms.bedrock.chat.invoke_handler import ( AWSEventStreamDecoder, BedrockError, ) - from unittest.mock import patch, Mock + from unittest.mock import Mock event = Mock() event.to_response_dict = Mock( return_value={ "status_code": 400, "headers": { - ":exception-type": "serviceUnavailableException", + ":exception-type": exception_type, ":content-type": "application/json", ":message-type": "exception", }, @@ -2525,11 +2540,10 @@ def test_bedrock_error_handling_streaming(): decoder = AWSEventStreamDecoder( model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0" ) - with pytest.raises(Exception) as e: + with pytest.raises(BedrockError) as e: decoder._parse_message_from_event(event) - assert isinstance(e.value, BedrockError) assert "Bedrock is unable to process your request." in e.value.message - assert e.value.status_code == 400 + assert e.value.status_code == expected_status_code @pytest.mark.parametrize( From 25a0550dc741302b5eec414227dd68e4adfc92cd Mon Sep 17 00:00:00 2001 From: Kent Date: Sun, 21 Jun 2026 09:57:08 +0800 Subject: [PATCH 2/3] test(bedrock): mid-stream server errors trigger streaming fallback (#24608) --- .../test_streaming_handler.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index e88010739c5..e7fc53676fd 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -876,6 +876,114 @@ def _raise_bad_request(**kwargs): assert "invalid maxOutputTokens" in str(excinfo.value) +def _bedrock_error_event(exception_type: str): + """A mocked botocore event-stream error event: status_code is botocore's + hard-coded 400, with the real type in the :exception-type header.""" + event = Mock() + event.to_response_dict = Mock( + return_value={ + "status_code": 400, + "headers": { + ":exception-type": exception_type, + ":content-type": "application/json", + ":message-type": "exception", + }, + "body": b'{"message":"Bedrock had an internal error."}', + } + ) + return event + + +@pytest.mark.asyncio +async def test_bedrock_midstream_internal_server_error_wraps_for_fallback( + logging_obj: Logging, +): + """End-to-end regression for https://github.com/BerriAI/litellm/issues/24608: + a Bedrock mid-stream internalServerException event (botocore stamps it 400) + must flow through the real decoder, gain its modeled 500 status, and wrap + into MidStreamFallbackError so the Router can run streaming fallback. + + Calls the real AWSEventStreamDecoder, so reverting the decoder status fix + makes the decoder raise BedrockError(400) and the gate raises BadRequestError + directly -> this test fails without the fix.""" + from litellm.exceptions import MidStreamFallbackError + from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder + + decoder = AWSEventStreamDecoder(model="anthropic.claude-3-sonnet-20240229-v1:0") + + async def _bedrock_stream(): + decoder._parse_message_from_event( + _bedrock_error_event("internalServerException") + ) + yield # unreachable; the line above raises + + async def _make_call(**kwargs): + return _bedrock_stream() + + response = CustomStreamWrapper( + completion_stream=None, + model="anthropic.claude-3-sonnet-20240229-v1:0", + logging_obj=logging_obj, + custom_llm_provider="bedrock", + make_call=_make_call, + ) + + with pytest.raises(MidStreamFallbackError): + await response.__anext__() + + +@pytest.mark.asyncio +async def test_bedrock_5xx_wraps_for_midstream_fallback(logging_obj: Logging): + """Gate contract: a Bedrock 5xx (here 503 serviceUnavailableException) wraps + into MidStreamFallbackError so the Router can run streaming fallback.""" + from litellm.exceptions import MidStreamFallbackError + from litellm.llms.bedrock.chat.invoke_handler import BedrockError + + async def _raise_503(**kwargs): + raise BedrockError( + status_code=503, + message="serviceUnavailableException Bedrock is unavailable.", + ) + + response = CustomStreamWrapper( + completion_stream=None, + model="anthropic.claude-3-sonnet-20240229-v1:0", + logging_obj=logging_obj, + custom_llm_provider="bedrock", + make_call=_raise_503, + ) + + with pytest.raises(MidStreamFallbackError): + await response.__anext__() + + +@pytest.mark.asyncio +async def test_bedrock_validation_error_raises_directly(logging_obj: Logging): + """Gate contract: a Bedrock validationException (400) is a client error and + must surface directly, never wrapped into MidStreamFallbackError.""" + from litellm.exceptions import MidStreamFallbackError + from litellm.llms.bedrock.chat.invoke_handler import BedrockError + + async def _raise_400(**kwargs): + raise BedrockError( + status_code=400, + message="validationException malformed input.", + ) + + response = CustomStreamWrapper( + completion_stream=None, + model="anthropic.claude-3-sonnet-20240229-v1:0", + logging_obj=logging_obj, + custom_llm_provider="bedrock", + make_call=_raise_400, + ) + + with pytest.raises(Exception) as excinfo: + await response.__anext__() + assert not isinstance(excinfo.value, MidStreamFallbackError) + assert getattr(excinfo.value, "status_code", None) == 400 + + @pytest.mark.asyncio async def test_async_streaming_read_timeout_triggers_midstream_fallback( logging_obj: Logging, From a46108f1e834a07ae60d2d713f5a638e4ce3ef8c Mon Sep 17 00:00:00 2001 From: Kent Date: Sun, 21 Jun 2026 10:10:34 +0800 Subject: [PATCH 3/3] style(bedrock): black-format stream-error helper (#24608) --- litellm/llms/bedrock/common_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 6eb43eb5cd2..9f58e5c0f1c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1168,7 +1168,9 @@ def build_bedrock_stream_error( if exception_type is not None and response_stream_shape is not None: member = response_stream_shape.members.get(exception_type) if member is not None: - modeled_status = (member.metadata or {}).get("error", {}).get("httpStatusCode") + modeled_status = ( + (member.metadata or {}).get("error", {}).get("httpStatusCode") + ) if modeled_status is not None: status_code = int(modeled_status)