From d2754a87cceb52538700a0aa5422a9194cf247e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 16:39:39 +0000 Subject: [PATCH] test(bedrock): make opus-4-7 + batch cells fail loudly and mock image-gen Replace the silent skips added for the new CI account with noisier behavior: - reasoning-effort grid: opus-4-7 cells now fail (when AWS creds are present) instead of skipping, so the missing entitlement stays visible in CI; they still skip when AWS creds are absent (local dev) - Bedrock batch inference tests: drop the skip so they run and fail until batch access is granted - Titan + Nova Canvas image-gen tests: mock the Bedrock HTTP call so the transform + cost-tracking path stays under test without live model access https://claude.ai/code/session_01MT7SWDnXUjv6e6EPG7BDjT --- .../test_bedrock_files_and_batches.py | 3 - .../test_bedrock_image_gen_unit_tests.py | 38 ++++++++--- .../image_gen_tests/test_image_generation.py | 64 +++++++++++++++++-- .../reasoning_effort_grid/grid_spec.py | 7 +- .../test_reasoning_effort_grid.py | 6 +- .../test_bedrock_batches_api.py | 3 - 6 files changed, 92 insertions(+), 29 deletions(-) diff --git a/tests/batches_tests/test_bedrock_files_and_batches.py b/tests/batches_tests/test_bedrock_files_and_batches.py index dc16f5024d4..97c0802ec99 100644 --- a/tests/batches_tests/test_bedrock_files_and_batches.py +++ b/tests/batches_tests/test_bedrock_files_and_batches.py @@ -42,9 +42,6 @@ async def test_async_create_file(): ) -@pytest.mark.skip( - reason="Bedrock batch inference (model-invocation-job) is not authorized on AWS account 941277531214 (requires an AWS support case); re-enable once batch access is granted" -) @pytest.mark.asyncio() async def test_async_file_and_batch(): """ diff --git a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py index b976cbe9201..181691b730d 100644 --- a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py +++ b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py @@ -1,3 +1,4 @@ +import json import logging import os import sys @@ -44,6 +45,9 @@ ) from litellm.llms.bedrock.common_utils import BedrockError +# Base64 placeholder used for mocked Bedrock image responses (a 1x1 PNG). +_MOCK_BEDROCK_IMAGE_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + @pytest.mark.parametrize( "model,expected", @@ -527,21 +531,35 @@ def test_backward_compatibility_regular_nova_model(): assert result["imageGenerationConfig"]["cfg_scale"] == 7 -@pytest.mark.skip( - reason="amazon.titan-image-generator is legacy-gated and unavailable on AWS account 941277531214" -) def test_amazon_titan_image_gen(): - """Test Amazon Titan image generation with cost tracking.""" - from litellm import image_generation + """Test Amazon Titan image generation with cost tracking. + + The Bedrock CI account is not entitled to amazon.titan-image-generator, so + the network call is mocked and only the transform + cost-tracking path is + exercised. + """ + from litellm.llms.custom_httpx.http_handler import HTTPHandler # Use v2 as v1 has reached end of life model_id = "bedrock/amazon.titan-image-generator-v2:0" - response = litellm.image_generation( - model=model_id, - prompt="A serene mountain landscape at sunset with a lake reflection", - aws_region_name="us-east-1", - ) + mock_payload = {"images": [_MOCK_BEDROCK_IMAGE_B64]} + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_payload + mock_response.text = json.dumps(mock_payload) + mock_response.headers = {} + + client = HTTPHandler() + with patch.object(client, "post", return_value=mock_response): + response = litellm.image_generation( + model=model_id, + prompt="A serene mountain landscape at sunset with a lake reflection", + aws_region_name="us-east-1", + aws_access_key_id="fake-access-key-id", + aws_secret_access_key="fake-secret-access-key", + client=client, + ) print(f"response cost: {response._hidden_params['response_cost']}") diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 5d9b48d3846..23a94ef389a 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -7,7 +7,6 @@ import traceback from unittest.mock import AsyncMock, MagicMock, patch - sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path @@ -136,9 +135,51 @@ def get_base_image_generation_call_args(self) -> dict: } -@pytest.mark.skip( - reason="amazon.nova-canvas-v1:0 is legacy-gated and unavailable on AWS account 941277531214" -) +# Base64 placeholder used for mocked Bedrock image responses (a 1x1 PNG). +_MOCK_BEDROCK_IMAGE_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + + +async def _assert_mocked_bedrock_image_generation(call_args: dict) -> None: + """Run ``aimage_generation`` with the Bedrock HTTP call mocked. + + The CI account is not entitled to Nova Canvas, so the network call is + replaced with a canned Bedrock response. This keeps the request transform, + response transform, and cost-tracking path under test without live access. + """ + mock_payload = {"images": [_MOCK_BEDROCK_IMAGE_B64]} + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_payload + mock_response.text = json.dumps(mock_payload) + mock_response.headers = {} + + custom_logger = TestCustomLogger() + litellm.logging_callback_manager._reset_all_callbacks() + litellm.callbacks = [custom_logger] + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + return_value=mock_response, + ): + response = await litellm.aimage_generation( + **call_args, + prompt="A image of a otter", + aws_access_key_id="fake-access-key-id", + aws_secret_access_key="fake-secret-access-key", + ) + + await asyncio.sleep(1) + + assert custom_logger.standard_logging_payload is not None + assert custom_logger.standard_logging_payload["response_cost"] is not None + assert custom_logger.standard_logging_payload["response_cost"] > 0 + assert response.data is not None + for d in response.data: + assert isinstance(d, Image) + assert d.b64_json is not None or d.url is not None + + class TestBedrockNovaCanvasTextToImage(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: litellm.in_memory_llm_clients_cache = InMemoryCache() @@ -151,10 +192,13 @@ def get_base_image_generation_call_args(self) -> dict: "aws_region_name": "us-east-1", } + @pytest.mark.asyncio(scope="module") + async def test_basic_image_generation(self): + await _assert_mocked_bedrock_image_generation( + self.get_base_image_generation_call_args() + ) + -@pytest.mark.skip( - reason="amazon.nova-canvas-v1:0 is legacy-gated and unavailable on AWS account 941277531214" -) class TestBedrockNovaCanvasColorGuidedGeneration(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: litellm.in_memory_llm_clients_cache = InMemoryCache() @@ -168,6 +212,12 @@ def get_base_image_generation_call_args(self) -> dict: "aws_region_name": "us-east-1", } + @pytest.mark.asyncio(scope="module") + async def test_basic_image_generation(self): + await _assert_mocked_bedrock_image_generation( + self.get_base_image_generation_call_args() + ) + class TestOpenAIGPTImage1(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: diff --git a/tests/llm_translation/reasoning_effort_grid/grid_spec.py b/tests/llm_translation/reasoning_effort_grid/grid_spec.py index 2f7101f6ce0..993643e0fc1 100644 --- a/tests/llm_translation/reasoning_effort_grid/grid_spec.py +++ b/tests/llm_translation/reasoning_effort_grid/grid_spec.py @@ -21,7 +21,7 @@ class ModelEntry: extra_params: Tuple[Tuple[str, str], ...] = field(default_factory=tuple) required_env: FrozenSet[str] = field(default_factory=frozenset) caps: FrozenSet[str] = field(default_factory=frozenset) - skip_reason: Optional[str] = None + fail_reason: Optional[str] = None def params(self) -> Dict[str, str]: return dict(self.extra_params) @@ -205,10 +205,11 @@ def expected(model: ModelEntry, effort: str) -> CellExpectation: extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, caps=_CAPS_OPUS_4_7, - skip_reason=( + fail_reason=( "claude-opus-4-7 is not entitled on the Bedrock CI account " "941277531214 (model access requires an AWS Sales request, not " - "self-serve); remove this skip_reason once access is granted" + "self-serve); this cell fails on purpose so it stays loud in CI — " + "remove this fail_reason once access is granted" ), ), ModelEntry( diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py index ca7f68030fd..56121d2ba58 100644 --- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py +++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py @@ -163,13 +163,13 @@ async def test_reasoning_effort_grid( cell: CellExpectation, wire_capture, ) -> None: - if model.skip_reason: - pytest.skip(model.skip_reason) - skip_reason = _required_env_missing(model) if skip_reason: pytest.skip(skip_reason) + if model.fail_reason: + pytest.fail(model.fail_reason) + if route_name == "bedrock_invoke_messages": status, exc = await _call_messages(model, effort) else: diff --git a/tests/openai_endpoints_tests/test_bedrock_batches_api.py b/tests/openai_endpoints_tests/test_bedrock_batches_api.py index 83bd8cf6011..4bb46334968 100644 --- a/tests/openai_endpoints_tests/test_bedrock_batches_api.py +++ b/tests/openai_endpoints_tests/test_bedrock_batches_api.py @@ -10,9 +10,6 @@ BEDROCK_BATCH_MODEL = "bedrock/batch-us.anthropic.claude-haiku-4-5-20251001-v1:0" -@pytest.mark.skip( - reason="Bedrock batch inference (model-invocation-job) is not authorized on AWS account 941277531214 (requires an AWS support case); re-enable once batch access is granted" -) @pytest.mark.asyncio async def test_bedrock_batches_api(): """