fix(vision): pre-flight shrink oversized images before first API call

tests/tools/test_vision_tools.py

@@ -1318,3 +1318,143 @@ async def fake_native(image_url, question, task_id=None):
             f"analyses were serialized to the cap (peak={calls_peak}); only the "
             "encode burst should be bounded, not the whole call"
         )
+
+
+# ---------------------------------------------------------------------------
+# Pre-flight shrink — first async_call_llm must carry a resized payload
+# ---------------------------------------------------------------------------
+
+
+class TestAuxVisionPreFlightShrink:
+    """Regression: production Anthropic failures were oversized aux-vision
+    images (6+ MB base64) that burned all 3 retries because the shrink was
+    reactive-only. The pre-flight in ``vision_analyze_tool`` must resize
+    BEFORE the first ``async_call_llm`` call.
+
+    These tests mock ``async_call_llm`` and inspect the first call's message
+    payload directly — proving the pre-flight fires end-to-end, not just
+    that ``_resize_image_for_vision`` works in isolation.
+    """
+
+    @pytest.mark.asyncio
+    async def test_first_api_call_payload_is_pre_shrunk(self, tmp_path):
+        """A ~6 MB base64 image must be resized before the first API call."""
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+
+        from tools.vision_tools import _AUX_VISION_TARGET_BYTES
+
+        # Build a real ~6 MB JPEG payload that exceeds the aux target.
+        import io as _io
+        img = Image.new("RGB", (3000, 3000))
+        pixels = img.load()
+        for y in range(0, 3000, 10):
+            for x in range(0, 3000, 10):
+                pixels[x, y] = ((x * 7) % 256, (y * 11) % 256, ((x + y) * 3) % 256)
+        buf = _io.BytesIO()
+        img.save(buf, format="JPEG", quality=95)
+        raw = buf.getvalue()
+
+        big_path = tmp_path / "big.jpg"
+        big_path.write_bytes(raw)
+
+        # Confirm the source really is oversized — otherwise the test is a no-op.
+        import base64 as _b64
+        raw_b64_len = len(_b64.b64encode(raw))
+        if raw_b64_len <= _AUX_VISION_TARGET_BYTES:
+            pytest.skip(
+                f"Generated JPEG is only {raw_b64_len} bytes b64 — "
+                f"not oversized enough to exercise pre-flight path"
+            )
+
+        mock_response = MagicMock()
+        mock_choice = MagicMock()
+        mock_choice.message.content = "shrunk-and-analyzed"
+        mock_response.choices = [mock_choice]
+
+        with (
+            patch("hermes_cli.config.load_config", return_value={}),
+            patch(
+                "tools.vision_tools.async_call_llm",
+                new_callable=AsyncMock,
+                return_value=mock_response,
+            ) as mock_llm,
+        ):
+            result = json.loads(await vision_analyze_tool(
+                str(big_path), "describe", "test/model"))
+
+        # Sanity: the tool succeeded (didn't raise or 400).
+        assert result["success"] is True
+
+        # Load-bearing assertion: the FIRST async_call_llm invocation must
+        # carry an already-shrunk image payload — never full-resolution.
+        assert mock_llm.await_count == 1, (
+            f"expected exactly 1 API call after pre-flight shrink, "
+            f"got {mock_llm.await_count}"
+        )
+        first_call_kwargs = mock_llm.await_args.kwargs
+        messages = first_call_kwargs["messages"]
+        image_url_part = messages[0]["content"][1]["image_url"]["url"]
+        assert image_url_part.startswith("data:image/"), (
+            f"expected data URL, got {image_url_part[:60]!r}"
+        )
+        assert len(image_url_part) <= _AUX_VISION_TARGET_BYTES, (
+            f"first API call carried {len(image_url_part)} bytes "
+            f"— pre-flight shrink didn't fire "
+            f"(target={_AUX_VISION_TARGET_BYTES})"
+        )
+
+    @pytest.mark.asyncio
+    async def test_small_image_untouched_by_preflight(self, tmp_path):
+        """Small images below target must NOT be resized — full resolution
+        preserved so aux-vision quality doesn't degrade unnecessarily."""
+        try:
+            from PIL import Image
+        except ImportError:
+            pytest.skip("Pillow not installed")
+
+        # Small 256x256 solid image — well under the target.
+        import io as _io
+        img = Image.new("RGB", (256, 256), color=(128, 64, 200))
+        buf = _io.BytesIO()
+        img.save(buf, format="PNG")
+        small_path = tmp_path / "small.png"
+        small_path.write_bytes(buf.getvalue())
+
+        original_b64_len = len(base64_encoded_data_url_for_path(small_path))
+
+        mock_response = MagicMock()
+        mock_choice = MagicMock()
+        mock_choice.message.content = "small-image-untouched"
+        mock_response.choices = [mock_choice]
+
+        with (
+            patch("hermes_cli.config.load_config", return_value={}),
+            patch(
+                "tools.vision_tools.async_call_llm",
+                new_callable=AsyncMock,
+                return_value=mock_response,
+            ) as mock_llm,
+        ):
+            result = json.loads(await vision_analyze_tool(
+                str(small_path), "describe", "test/model"))
+
+        assert result["success"] is True
+        first_call_kwargs = mock_llm.await_args.kwargs
+        image_url_part = first_call_kwargs["messages"][0]["content"][1]["image_url"]["url"]
+        # Small image sent at full resolution — length identical to original encode.
+        assert len(image_url_part) == original_b64_len, (
+            f"small image was resized by pre-flight (before={original_b64_len}, "
+            f"after={len(image_url_part)}) — pre-flight should skip small payloads"
+        )
+
+
+def base64_encoded_data_url_for_path(path):
+    """Helper: encode a file to a data URL the same way vision_tools does."""
+    import base64 as _b64
+    from tools.vision_tools import _determine_mime_type
+    raw = path.read_bytes()
+    mime = _determine_mime_type(path)
+    return f"data:{mime};base64,{_b64.b64encode(raw).decode('ascii')}"

tools/browser_tool.py

@@ -4062,6 +4062,19 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
         _screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii")
         data_url = f"data:image/png;base64,{_screenshot_b64}"
 
+        # Proactive aux-LLM pre-flight resize for the non-native path.  Aux
+        # vision providers (Claude Haiku especially) reject payloads over
+        # ~5 MB or 8000px/side with a hard 400.  Full-page screenshots
+        # regularly exceed both — resize BEFORE the first call_llm to
+        # avoid the 3-retry HTTP 400 churn observed in production.
+        # (The native fast-path below does its own embed-cap check.)
+        from tools.vision_tools import (
+            _image_exceeds_dimension as _vt_image_exceeds_dimension,
+            _resize_image_for_vision as _vt_resize_image_for_vision,
+            _AUX_VISION_TARGET_BYTES as _VT_AUX_VISION_TARGET_BYTES,
+            _AUX_VISION_MAX_DIMENSION as _VT_AUX_VISION_MAX_DIMENSION,
+        )
+
         # Fast path: when native image routing is in effect for the active main
         # model, attach the screenshot directly instead of describing it through
         # an auxiliary vision LLM. The model inspects the pixels on its next
@@ -4139,6 +4152,29 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
         }
         if vision_model:
             call_kwargs["model"] = vision_model
+
+        # Proactive pre-flight: shrink oversized screenshots BEFORE the first
+        # call_llm.  A 3000x2000 full-page PNG can easily land 6+ MB base64,
+        # and Anthropic aux vision (Claude Haiku) rejects >5 MB with a hard
+        # 400 that burns all retries.  Mirrors vision_analyze_tool.
+        _bv_over_bytes = len(data_url) > _VT_AUX_VISION_TARGET_BYTES
+        _bv_over_dims = _vt_image_exceeds_dimension(
+            screenshot_path, _VT_AUX_VISION_MAX_DIMENSION)
+        if _bv_over_bytes or _bv_over_dims:
+            logger.info(
+                "browser_vision pre-flight: screenshot is %.1f MB / over-dims=%s "
+                "(targets %.1f MB / %dpx); shrinking before first API call...",
+                len(data_url) / (1024 * 1024), _bv_over_dims,
+                _VT_AUX_VISION_TARGET_BYTES / (1024 * 1024),
+                _VT_AUX_VISION_MAX_DIMENSION,
+            )
+            data_url = _vt_resize_image_for_vision(
+                screenshot_path, mime_type="image/png",
+                max_base64_bytes=_VT_AUX_VISION_TARGET_BYTES,
+                max_dimension=_VT_AUX_VISION_MAX_DIMENSION,
+            )
+            call_kwargs["messages"][0]["content"][1]["image_url"]["url"] = data_url
+
         # Try full-size screenshot; on size-related rejection, downscale and retry.
         try:
             response = call_llm(**call_kwargs)

tools/vision_tools.py

@@ -566,9 +566,19 @@ def _image_to_base64_data_url(image_path: Path, mime_type: Optional[str] = None)
 _EMBED_MAX_DIMENSION = 7900
 
 # Target size when auto-resizing on API failure (5 MB).  After a provider
-# rejects an image, we downscale to this target and retry once.
+# rejects an image, we downscale to this target and retry once.  Only used
+# by the reactive retry paths — the proactive paths use _EMBED_TARGET_BYTES.
 _RESIZE_TARGET_BYTES = 5 * 1024 * 1024
 
+# Aux-LLM byte target — shared by `vision_analyze_tool` and `browser_vision`
+# proactive pre-flight paths.  Set equal to _EMBED_TARGET_BYTES because the
+# same provider constraints apply: aux vision is often Anthropic Claude
+# (Haiku) which enforces the 5 MB per-image ceiling.  Shrinking to 4 MB
+# up front avoids the 3-retry HTTP 400 churn observed in production when
+# oversized screenshots or user-supplied images are sent full-resolution.
+_AUX_VISION_TARGET_BYTES = _EMBED_TARGET_BYTES
+_AUX_VISION_MAX_DIMENSION = _EMBED_MAX_DIMENSION
+
 
 def _is_image_size_error(error: Exception) -> bool:
     """Detect if an API error is related to image or payload size."""
@@ -1170,16 +1180,39 @@ async def vision_analyze_tool(
             temp_image_path = normalized_path
             should_cleanup = True
 
-        # Convert image to base64 — send at full resolution first.
-        # If the provider rejects it as too large, we auto-resize and retry.
-        # Offloaded to the bounded vision CPU executor so a fan-out of encodes
-        # can't saturate every core and starve the event loop.
+        # Convert image to base64.  Offloaded to the bounded vision CPU executor
+        # so a fan-out of encodes can't saturate every core and starve the event
+        # loop.
         logger.info("Converting image to base64...")
         image_data_url = await _run_encode_on_cpu_executor(
             _image_to_base64_data_url, temp_image_path, mime_type=detected_mime_type)
         data_size_kb = len(image_data_url) / 1024
         logger.info("Image converted to base64 (%.1f KB)", data_size_kb)
 
+        # Proactive aux-LLM pre-flight resize.  Aux vision providers (Claude
+        # Haiku especially) reject payloads over ~5 MB or 8000px/side with a
+        # hard 400 — async_call_llm will burn all retries on unrecoverable
+        # size errors before the reactive shrink below gets a chance.
+        # Mirrors the native fast-path proactive check above.
+        _over_bytes = len(image_data_url) > _AUX_VISION_TARGET_BYTES
+        _over_dims = await _run_encode_on_cpu_executor(
+            _image_exceeds_dimension, temp_image_path, _AUX_VISION_MAX_DIMENSION,
+        )
+        if _over_bytes or _over_dims:
+            logger.info(
+                "Aux-vision pre-flight: image is %.1f MB / over-dims=%s "
+                "(targets %.1f MB / %dpx); shrinking before first API call...",
+                len(image_data_url) / (1024 * 1024), _over_dims,
+                _AUX_VISION_TARGET_BYTES / (1024 * 1024),
+                _AUX_VISION_MAX_DIMENSION,
+            )
+            image_data_url = await _run_encode_on_cpu_executor(
+                _resize_image_for_vision,
+                temp_image_path, mime_type=detected_mime_type,
+                max_base64_bytes=_AUX_VISION_TARGET_BYTES,
+                max_dimension=_AUX_VISION_MAX_DIMENSION,
+            )
+
         # Hard limit (20 MB) — no provider accepts payloads this large.
         if len(image_data_url) > _MAX_BASE64_BYTES:
             # Try to resize down to 5 MB before giving up.