fix(vision): clamp image dimensions before inline base64 encode

agent/conversation_compression.py

@@ -504,7 +504,7 @@ def try_shrink_image_parts_in_messages(api_messages: list) -> bool:
         return False
 
     try:
-        from tools.vision_tools import _resize_image_for_vision
+        from tools.vision_tools import _resize_image_for_vision, _is_anthropic_provider
     except Exception as exc:
         logger.warning("image-shrink recovery: vision_tools unavailable — %s", exc)
         return False
@@ -546,6 +546,7 @@ def _shrink_data_url(url: str) -> Optional[str]:
                     Path(tmp.name),
                     mime_type=mime,
                     max_base64_bytes=target_bytes,
+                    clamp_dimensions=_is_anthropic_provider(),
                 )
             finally:
                 try:

tests/run_agent/test_image_shrink_recovery.py

@@ -145,7 +145,7 @@ def test_oversized_image_url_dict_shape_rewritten(self, monkeypatch):
         oversized_url = _big_png_data_url(5000)  # ~5 MB raw → ~6.7 MB b64
         shrunk = "data:image/jpeg;base64," + "A" * 1000  # small
 
-        def _fake_resize(path, mime_type=None, max_base64_bytes=None):
+        def _fake_resize(path, mime_type=None, max_base64_bytes=None, clamp_dimensions=False):
             return shrunk
 
         monkeypatch.setattr(

tests/tools/test_vision_tools.py

@@ -17,8 +17,11 @@
     _image_to_base64_data_url,
     _resize_image_for_vision,
     _is_image_size_error,
+    _image_exceeds_pixel_cap,
+    _get_image_dimensions,
     _MAX_BASE64_BYTES,
     _RESIZE_TARGET_BYTES,
+    _MAX_IMAGE_DIMENSION,
     vision_analyze_tool,
     check_vision_requirements,
 )
@@ -890,6 +893,239 @@ def test_no_pillow_returns_original(self, tmp_path):
                 assert len(result) > 100
 
 
+# ---------------------------------------------------------------------------
+# Pixel-dimension cap — Anthropic rejects >8000 px on either axis as
+# non_retryable_client_error, which permanently bricks the session via the
+# native fast path (image gets inlined into the tool-result envelope before
+# the API call fails).  See `_MAX_IMAGE_DIMENSION`.
+# ---------------------------------------------------------------------------
+
+
+class TestPixelDimensionCap:
+    """Tests for the per-axis pixel cap enforcement."""
+
+    def test_max_dimension_below_anthropic_limit(self):
+        """The cap must stay strictly below Anthropic's 8000 px hard limit."""
+        assert _MAX_IMAGE_DIMENSION < 8000
+
+    def test_get_dimensions_reads_header(self, tmp_path):
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        img = Image.new("RGB", (123, 456), (0, 0, 0))
+        path = tmp_path / "dims.png"
+        img.save(path, "PNG")
+        assert _get_image_dimensions(path) == (123, 456)
+
+    def test_exceeds_cap_wide(self, tmp_path):
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        img = Image.new("RGB", (_MAX_IMAGE_DIMENSION + 100, 200), (0, 0, 0))
+        path = tmp_path / "wide.png"
+        img.save(path, "PNG")
+        assert _image_exceeds_pixel_cap(path) is True
+
+    def test_exceeds_cap_tall(self, tmp_path):
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        img = Image.new("RGB", (200, _MAX_IMAGE_DIMENSION + 100), (0, 0, 0))
+        path = tmp_path / "tall.png"
+        img.save(path, "PNG")
+        assert _image_exceeds_pixel_cap(path) is True
+
+    def test_within_cap(self, tmp_path):
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        img = Image.new("RGB", (4000, 3000), (0, 0, 0))
+        path = tmp_path / "ok.png"
+        img.save(path, "PNG")
+        assert _image_exceeds_pixel_cap(path) is False
+
+    def test_no_pillow_returns_false(self, tmp_path):
+        """Without Pillow, the dimension check is a no-op (byte guard remains)."""
+        path = tmp_path / "fake.png"
+        path.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100)
+        with patch.dict("sys.modules", {"PIL": None, "PIL.Image": None}):
+            assert _image_exceeds_pixel_cap(path) is False
+            assert _get_image_dimensions(path) is None
+
+    def test_resize_clamps_oversized_dimension(self, tmp_path):
+        """A 10000x100 image must come back ≤ _MAX_IMAGE_DIMENSION on the long side.
+
+        This is the regression test for the bug: an image well under 20 MB
+        base64 but >8000 px on one axis would slip through and brick the
+        session.  After the fix, ``_resize_image_for_vision`` must clamp
+        dimensions to the cap regardless of byte size.
+        """
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        # 10000x100 solid-colour PNG compresses to a few KB — well under any
+        # byte cap — but violates the pixel-dimension cap.
+        img = Image.new("RGB", (10000, 100), (50, 100, 150))
+        path = tmp_path / "wide.png"
+        img.save(path, "PNG")
+
+        result = _resize_image_for_vision(path, mime_type="image/png", clamp_dimensions=True)
+        assert result.startswith("data:image/png;base64,")
+
+        # Decode the returned data URL and assert dimensions
+        import base64
+        from io import BytesIO
+        _, b64data = result.split(",", 1)
+        decoded = Image.open(BytesIO(base64.b64decode(b64data)))
+        assert decoded.width <= _MAX_IMAGE_DIMENSION
+        assert decoded.height <= _MAX_IMAGE_DIMENSION
+        # Aspect ratio should be roughly preserved
+        original_ratio = 10000 / 100  # 100:1
+        new_ratio = decoded.width / max(decoded.height, 1)
+        # Proportional clamp ⇒ ratio preserved within rounding tolerance.
+        # Pillow's int rounding can cost ~1% on extreme aspect ratios.
+        assert new_ratio >= original_ratio * 0.95, (
+            f"Aspect ratio drifted: {decoded.width}x{decoded.height} "
+            f"(ratio {new_ratio:.1f}, original {original_ratio:.1f})"
+        )
+
+    def test_resize_clamps_oversized_tall_image(self, tmp_path):
+        """Mirror of the wide case for the tall axis."""
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        img = Image.new("RGB", (100, 10000), (150, 100, 50))
+        path = tmp_path / "tall.png"
+        img.save(path, "PNG")
+
+        result = _resize_image_for_vision(path, mime_type="image/png", clamp_dimensions=True)
+        import base64
+        from io import BytesIO
+        _, b64data = result.split(",", 1)
+        decoded = Image.open(BytesIO(base64.b64decode(b64data)))
+        assert decoded.width <= _MAX_IMAGE_DIMENSION
+        assert decoded.height <= _MAX_IMAGE_DIMENSION
+
+    def test_resize_within_cap_not_dimension_clamped(self, tmp_path):
+        """An image already within the cap shouldn't be dimension-clamped.
+
+        (Byte-size resize may still apply for very large files, but a 4000x3000
+        image small enough in bytes should pass through untouched.)
+        """
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        img = Image.new("RGB", (4000, 3000), (0, 128, 255))
+        path = tmp_path / "ok.png"
+        img.save(path, "PNG")
+        result = _resize_image_for_vision(path, mime_type="image/png", clamp_dimensions=True)
+        import base64
+        from io import BytesIO
+        _, b64data = result.split(",", 1)
+        decoded = Image.open(BytesIO(base64.b64decode(b64data)))
+        assert decoded.size == (4000, 3000)
+
+    def test_clamp_disabled_by_default_preserves_oversized(self, tmp_path):
+        """Non-Anthropic providers (clamp_dimensions=False) keep original dims."""
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        img = Image.new("RGB", (10000, 100), (50, 100, 150))
+        path = tmp_path / "wide.png"
+        img.save(path, "PNG")
+        # Default: clamp_dimensions=False — small enough in bytes, so fast-exit.
+        result = _resize_image_for_vision(path, mime_type="image/png")
+        import base64
+        from io import BytesIO
+        _, b64data = result.split(",", 1)
+        decoded = Image.open(BytesIO(base64.b64decode(b64data)))
+        assert decoded.size == (10000, 100)
+
+
+class TestNativeVisionDimensionWiring:
+    """Wire-up regression: _vision_analyze_native must pass clamp_dimensions
+    matching the active provider to _resize_image_for_vision."""
+
+    def _make_oversized_png(self, tmp_path):
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+        img = Image.new("RGB", (10000, 100), (128, 128, 128))
+        path = tmp_path / "wide.png"
+        img.save(path, "PNG")
+        return path
+
+    def test_anthropic_provider_passes_clamp_true(self, tmp_path):
+        from tools.vision_tools import _vision_analyze_native
+        path = self._make_oversized_png(tmp_path)
+
+        with patch("tools.vision_tools._is_anthropic_provider", return_value=True), \
+             patch("tools.vision_tools._resize_image_for_vision",
+                   return_value="data:image/png;base64,AAAA") as resize_spy, \
+             patch("tools.vision_tools._build_native_vision_tool_result",
+                   return_value={"ok": True}):
+            asyncio.run(_vision_analyze_native(str(path), "describe"))
+
+        resize_spy.assert_called_once()
+        assert resize_spy.call_args.kwargs.get("clamp_dimensions") is True
+
+    def test_non_anthropic_provider_passes_clamp_false(self, tmp_path):
+        from tools.vision_tools import _vision_analyze_native
+        path = self._make_oversized_png(tmp_path)
+
+        # Non-Anthropic: oversized-but-small-bytes image should skip the resize
+        # entirely (byte cap not exceeded, pixel guard gated off).
+        with patch("tools.vision_tools._is_anthropic_provider", return_value=False), \
+             patch("tools.vision_tools._resize_image_for_vision") as resize_spy, \
+             patch("tools.vision_tools._build_native_vision_tool_result",
+                   return_value={"ok": True}):
+            asyncio.run(_vision_analyze_native(str(path), "describe"))
+
+        resize_spy.assert_not_called()
+
+
+class TestIsAnthropicProvider:
+    """_is_anthropic_provider must cover native Anthropic, common aliases,
+    and aggregators that proxy Claude. Same provider set as
+    _supports_media_in_tool_results."""
+
+    @pytest.mark.parametrize("provider", [
+        "anthropic", "claude", "claude-code", "anthropic-direct",
+        "openrouter", "nous", "vertex", "bedrock",
+        "anthropic-vertex", "google-vertex",
+    ])
+    def test_matches(self, provider):
+        from tools.vision_tools import _is_anthropic_provider
+        with patch("agent.auxiliary_client._read_main_provider",
+                   return_value=provider):
+            assert _is_anthropic_provider() is True
+
+    @pytest.mark.parametrize("provider", [
+        "openai", "openai-chat", "openai-codex", "azure-openai",
+        "gemini", "google", "xai", "deepseek", "custom", "",
+    ])
+    def test_does_not_match(self, provider):
+        from tools.vision_tools import _is_anthropic_provider
+        with patch("agent.auxiliary_client._read_main_provider",
+                   return_value=provider):
+            assert _is_anthropic_provider() is False
+
+    def test_uppercase_normalized(self):
+        from tools.vision_tools import _is_anthropic_provider
+        with patch("agent.auxiliary_client._read_main_provider",
+                   return_value="  ANTHROPIC  "):
+            assert _is_anthropic_provider() is True
+
+
 # ---------------------------------------------------------------------------
 # _is_image_size_error — detect size-related API errors
 # ---------------------------------------------------------------------------

tools/browser_tool.py

@@ -3241,6 +3241,7 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
         except Exception as _api_err:
             from tools.vision_tools import (
                 _is_image_size_error, _resize_image_for_vision, _RESIZE_TARGET_BYTES,
+                _is_anthropic_provider,
             )
             if (_is_image_size_error(_api_err)
                     and len(data_url) > _RESIZE_TARGET_BYTES):
@@ -3251,7 +3252,8 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
                     _RESIZE_TARGET_BYTES / (1024 * 1024),
                 )
                 data_url = _resize_image_for_vision(
-                    screenshot_path, mime_type="image/png")
+                    screenshot_path, mime_type="image/png",
+                    clamp_dimensions=_is_anthropic_provider())
                 call_kwargs["messages"][0]["content"][1]["image_url"]["url"] = data_url
                 response = call_llm(**call_kwargs)
             else: