fix(413): recover from image-payload 413s instead of bricking the session

agent/conversation_compression.py

@@ -796,6 +796,7 @@ def try_shrink_image_parts_in_messages(
     api_messages: list,
     *,
     max_dimension: int = 8000,
+    target_bytes: Optional[int] = None,
 ) -> bool:
     """Re-encode all native image parts at a smaller size to recover from
     image-too-large errors (Anthropic 5 MB, unknown other providers).
@@ -804,6 +805,17 @@ def try_shrink_image_parts_in_messages(
     actually replaced, False if there were no image parts to shrink or
     Pillow couldn't help (caller should surface the original error).
 
+    ``target_bytes`` overrides the default 4 MB per-image budget.  The
+    per-image default is right for the Anthropic 5 MB *single-image*
+    ceiling, but an aggregate request-body 413 (``payload_too_large``)
+    can be triggered by several individually-under-4 MB images whose
+    *sum* blows the gateway's body limit (observed on GitHub Copilot:
+    a handful of ~500 KB screenshots embedded by native vision).  The
+    413 recovery path passes a much smaller ``target_bytes`` so every
+    embedded image is shrunk hard enough to bring the total under the
+    limit, instead of no-opping because each image is already under
+    4 MB.  See the ``payload_too_large`` branch in ``conversation_loop``.
+
     Strategy: look for ``image_url`` / ``input_image`` parts carrying a
     ``data:image/...;base64,...`` payload, plus Anthropic-native
     ``{"type": "image", "source": {"type": "base64", ...}}`` blocks.
@@ -829,7 +841,16 @@ def try_shrink_image_parts_in_messages(
     # Non-Anthropic providers we haven't observed rejecting are fine with
     # much larger; shrinking to 4 MB here loses quality but only fires
     # after a confirmed provider rejection, so the alternative is failure.
-    target_bytes = 4 * 1024 * 1024
+    #
+    # ``target_bytes`` (when passed by the caller) overrides this for the
+    # aggregate-payload 413 path, where the binding constraint is the SUM
+    # of all embedded images, not any single image's 5 MB ceiling.  A much
+    # smaller per-image target there is what actually brings the total
+    # request body back under the gateway limit.
+    if target_bytes is not None and target_bytes > 0:
+        target_bytes = int(target_bytes)
+    else:
+        target_bytes = 4 * 1024 * 1024
     # Anthropic enforces an 8000px per-side dimension cap independently of
     # the 5 MB byte cap.  In many-image requests, the provider can report a
     # lower cap (observed: 2000px).  The caller passes that parsed ceiling

agent/conversation_loop.py

@@ -2963,6 +2963,55 @@ def _perform_api_call(next_api_kwargs):
                     )
 
                 if is_payload_too_large:
+                    # ── Aggregate-image 413 recovery (runs BEFORE text
+                    # compression) ──────────────────────────────────────
+                    # A 413 is a request-BODY-size error, measured in bytes,
+                    # not a token-count overflow.  When the body is bloated by
+                    # several base64 images embedded via native vision (each
+                    # individually under the 5 MB single-image ceiling, but
+                    # collectively megabytes), compressing TEXT can't shrink
+                    # it — the images dominate the bytes and text compression
+                    # reports "cannot compress further" while the real payload
+                    # is untouched.  Observed on GitHub Copilot: a handful of
+                    # ~500 KB screenshots from vision_analyze 413 a session
+                    # that is only tens of KB of text.
+                    #
+                    # Shrink every embedded image and ``continue`` (re-runs
+                    # _build_api_kwargs at the top of this retry loop with the
+                    # shrunk payload), mirroring the ``image_too_large``
+                    # sibling above.  Two PROGRESSIVE passes: the first 413
+                    # re-encodes images to ≤512 KB each; if the gateway still
+                    # 413s (sum still over its body cap), the second pass goes
+                    # to ≤256 KB.  Only after both image passes are spent —
+                    # i.e. the body is genuinely text-dominated — do we fall
+                    # through to the text-compression path below.
+                    _shrink_target = None
+                    if not _retry.payload_image_shrink_pass1_attempted:
+                        _retry.payload_image_shrink_pass1_attempted = True
+                        _shrink_target = 512 * 1024
+                    elif not _retry.payload_image_shrink_pass2_attempted:
+                        _retry.payload_image_shrink_pass2_attempted = True
+                        _shrink_target = 256 * 1024
+                    if _shrink_target is not None:
+                        if agent._try_shrink_image_parts_in_messages(
+                            api_messages,
+                            max_dimension=2000,
+                            target_bytes=_shrink_target,
+                        ):
+                            agent._buffer_status(
+                                "🖼️  Request too large (413) from embedded images — "
+                                f"shrank attached image(s) to ≤{_shrink_target // 1024} KB "
+                                "each and retrying..."
+                            )
+                            continue
+                        else:
+                            logger.info(
+                                f"{agent.log_prefix}413 payload-image-shrink "
+                                f"(target {_shrink_target // 1024} KB): no further "
+                                "shrinkable image parts; falling through to text "
+                                "compression."
+                            )
+
                     compression_attempts += 1
                     if compression_attempts > max_compression_attempts:
                         # Terminal — surface the buffered retry trace.

agent/turn_retry_state.py

@@ -54,6 +54,17 @@ class TurnRetryState:
     oauth_1m_beta_retry_attempted: bool = False
     llama_cpp_grammar_retry_attempted: bool = False
 
+    # Aggregate-payload (413) image-shrink guards.  Unlike a single-pass
+    # recovery, the 413 image path shrinks PROGRESSIVELY harder: the first
+    # 413 re-encodes every embedded image to ≤512 KB, a second 413 (the new
+    # payload still too big for the gateway) re-encodes to ≤256 KB.  Two
+    # one-shot bools keep the dataclass all-boolean (see the field-set and
+    # all-False contract tests) while giving the loop two escalating passes
+    # before it falls through to text compression.  See the
+    # ``payload_too_large`` branch in ``conversation_loop``.
+    payload_image_shrink_pass1_attempted: bool = False
+    payload_image_shrink_pass2_attempted: bool = False
+
     # ── Transport / rate-limit recovery ──────────────────────────────────
     primary_recovery_attempted: bool = False
     has_retried_429: bool = False

run_agent.py

@@ -4676,12 +4676,14 @@ def _try_shrink_image_parts_in_messages(
         api_messages: list,
         *,
         max_dimension: int = 8000,
+        target_bytes: int = None,
     ) -> bool:
         """Forwarder — see ``agent.conversation_compression.try_shrink_image_parts_in_messages``."""
         from agent.conversation_compression import try_shrink_image_parts_in_messages
         return try_shrink_image_parts_in_messages(
             api_messages,
             max_dimension=max_dimension,
+            target_bytes=target_bytes,
         )
 
     def _try_strip_image_parts_from_tool_messages(self, api_messages: list) -> bool:

tests/agent/test_turn_retry_state.py

@@ -22,6 +22,8 @@
     "thinking_sig_retry_attempted",
     "invalid_encrypted_content_retry_attempted",
     "image_shrink_retry_attempted",
+    "payload_image_shrink_pass1_attempted",
+    "payload_image_shrink_pass2_attempted",
     "multimodal_tool_content_retry_attempted",
     "oauth_1m_beta_retry_attempted",
     "llama_cpp_grammar_retry_attempted",

tests/run_agent/test_image_shrink_recovery.py

@@ -661,3 +661,103 @@ def test_byte_oversized_with_no_dim_cap_accepts_byte_shrink(self, monkeypatch):
         # Default cap (8000) — no explicit max_dimension passed.
         assert agent._try_shrink_image_parts_in_messages(msgs) is True
         assert msgs[0]["content"][0]["image_url"]["url"] == shrunk
+
+
+class TestAggregatePayloadShrink:
+    """Aggregate-413 path: several individually-under-4MB images whose SUM
+    blows a gateway's request-body byte limit (observed on GitHub Copilot
+    with native-vision screenshots).  The default 4 MB per-image budget
+    no-ops on these, so the 413 handler passes a small ``target_bytes`` to
+    force every embedded image down.  Regression for the
+    ``payload_too_large`` image-shrink branch in ``conversation_loop``.
+    """
+
+    def test_default_budget_noops_on_sub_4mb_images(self, monkeypatch):
+        """~500 KB images are under the 4 MB default - shrink must NOT fire."""
+        agent = _make_agent()
+        url1 = _big_png_data_url(500)
+        url2 = _big_png_data_url(480)
+        url3 = _big_png_data_url(520)
+        for u in (url1, url2, url3):
+            assert len(u) < 4 * 1024 * 1024
+
+        resize_hits = {"count": 0}
+        monkeypatch.setattr(
+            "tools.vision_tools._resize_image_for_vision",
+            lambda *a, **kw: resize_hits.__setitem__(
+                "count", resize_hits["count"] + 1
+            ) or "shrunk",
+            raising=False,
+        )
+
+        msgs = [{
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": {"url": url1}},
+                {"type": "image_url", "image_url": {"url": url2}},
+                {"type": "image_url", "image_url": {"url": url3}},
+            ],
+        }]
+        # No target_bytes override -> 4 MB default -> nothing exceeds it.
+        assert agent._try_shrink_image_parts_in_messages(msgs) is False
+        assert resize_hits["count"] == 0
+        assert msgs[0]["content"][0]["image_url"]["url"] == url1
+
+    def test_small_target_bytes_shrinks_sub_4mb_images(self, monkeypatch):
+        """With a 512 KB target_bytes, the same ~500 KB images DO get shrunk."""
+        agent = _make_agent()
+        url1 = _big_png_data_url(500)
+        url2 = _big_png_data_url(520)
+        shrunk = "data:image/jpeg;base64," + "S" * 1000
+
+        seen = {"max_base64_bytes": None}
+
+        def _fake_resize(path, mime_type=None, max_base64_bytes=None, max_dimension=None):
+            seen["max_base64_bytes"] = max_base64_bytes
+            return shrunk
+
+        monkeypatch.setattr(
+            "tools.vision_tools._resize_image_for_vision",
+            _fake_resize,
+            raising=False,
+        )
+
+        msgs = [{
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": {"url": url1}},
+                {"type": "image_url", "image_url": {"url": url2}},
+            ],
+        }]
+        changed = agent._try_shrink_image_parts_in_messages(
+            msgs,
+            max_dimension=2000,
+            target_bytes=512 * 1024,
+        )
+        assert changed is True
+        assert seen["max_base64_bytes"] == 512 * 1024
+        assert msgs[0]["content"][0]["image_url"]["url"] == shrunk
+        assert msgs[0]["content"][1]["image_url"]["url"] == shrunk
+
+    def test_target_bytes_zero_falls_back_to_default(self, monkeypatch):
+        """target_bytes<=0 must fall back to the 4 MB default, not break."""
+        agent = _make_agent()
+        small = _big_png_data_url(500)
+        resize_hits = {"count": 0}
+        monkeypatch.setattr(
+            "tools.vision_tools._resize_image_for_vision",
+            lambda *a, **kw: resize_hits.__setitem__(
+                "count", resize_hits["count"] + 1
+            ) or "shrunk",
+            raising=False,
+        )
+        msgs = [{
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": {"url": small}},
+            ],
+        }]
+        assert agent._try_shrink_image_parts_in_messages(
+            msgs, target_bytes=0,
+        ) is False
+        assert resize_hits["count"] == 0