diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index ba67f0369549..2ed07a98efd0 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -796,6 +796,7 @@ def try_shrink_image_parts_in_messages( api_messages: list, *, max_dimension: int = 8000, + target_bytes: Optional[int] = None, ) -> bool: """Re-encode all native image parts at a smaller size to recover from image-too-large errors (Anthropic 5 MB, unknown other providers). @@ -804,6 +805,17 @@ def try_shrink_image_parts_in_messages( actually replaced, False if there were no image parts to shrink or Pillow couldn't help (caller should surface the original error). + ``target_bytes`` overrides the default 4 MB per-image budget. The + per-image default is right for the Anthropic 5 MB *single-image* + ceiling, but an aggregate request-body 413 (``payload_too_large``) + can be triggered by several individually-under-4 MB images whose + *sum* blows the gateway's body limit (observed on GitHub Copilot: + a handful of ~500 KB screenshots embedded by native vision). The + 413 recovery path passes a much smaller ``target_bytes`` so every + embedded image is shrunk hard enough to bring the total under the + limit, instead of no-opping because each image is already under + 4 MB. See the ``payload_too_large`` branch in ``conversation_loop``. + Strategy: look for ``image_url`` / ``input_image`` parts carrying a ``data:image/...;base64,...`` payload, plus Anthropic-native ``{"type": "image", "source": {"type": "base64", ...}}`` blocks. @@ -829,7 +841,16 @@ def try_shrink_image_parts_in_messages( # Non-Anthropic providers we haven't observed rejecting are fine with # much larger; shrinking to 4 MB here loses quality but only fires # after a confirmed provider rejection, so the alternative is failure. - target_bytes = 4 * 1024 * 1024 + # + # ``target_bytes`` (when passed by the caller) overrides this for the + # aggregate-payload 413 path, where the binding constraint is the SUM + # of all embedded images, not any single image's 5 MB ceiling. A much + # smaller per-image target there is what actually brings the total + # request body back under the gateway limit. + if target_bytes is not None and target_bytes > 0: + target_bytes = int(target_bytes) + else: + target_bytes = 4 * 1024 * 1024 # Anthropic enforces an 8000px per-side dimension cap independently of # the 5 MB byte cap. In many-image requests, the provider can report a # lower cap (observed: 2000px). The caller passes that parsed ceiling diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 3526029ddab1..c7b7605c9f72 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -2963,6 +2963,55 @@ def _perform_api_call(next_api_kwargs): ) if is_payload_too_large: + # ── Aggregate-image 413 recovery (runs BEFORE text + # compression) ────────────────────────────────────── + # A 413 is a request-BODY-size error, measured in bytes, + # not a token-count overflow. When the body is bloated by + # several base64 images embedded via native vision (each + # individually under the 5 MB single-image ceiling, but + # collectively megabytes), compressing TEXT can't shrink + # it — the images dominate the bytes and text compression + # reports "cannot compress further" while the real payload + # is untouched. Observed on GitHub Copilot: a handful of + # ~500 KB screenshots from vision_analyze 413 a session + # that is only tens of KB of text. + # + # Shrink every embedded image and ``continue`` (re-runs + # _build_api_kwargs at the top of this retry loop with the + # shrunk payload), mirroring the ``image_too_large`` + # sibling above. Two PROGRESSIVE passes: the first 413 + # re-encodes images to ≤512 KB each; if the gateway still + # 413s (sum still over its body cap), the second pass goes + # to ≤256 KB. Only after both image passes are spent — + # i.e. the body is genuinely text-dominated — do we fall + # through to the text-compression path below. + _shrink_target = None + if not _retry.payload_image_shrink_pass1_attempted: + _retry.payload_image_shrink_pass1_attempted = True + _shrink_target = 512 * 1024 + elif not _retry.payload_image_shrink_pass2_attempted: + _retry.payload_image_shrink_pass2_attempted = True + _shrink_target = 256 * 1024 + if _shrink_target is not None: + if agent._try_shrink_image_parts_in_messages( + api_messages, + max_dimension=2000, + target_bytes=_shrink_target, + ): + agent._buffer_status( + "🖼️ Request too large (413) from embedded images — " + f"shrank attached image(s) to ≤{_shrink_target // 1024} KB " + "each and retrying..." + ) + continue + else: + logger.info( + f"{agent.log_prefix}413 payload-image-shrink " + f"(target {_shrink_target // 1024} KB): no further " + "shrinkable image parts; falling through to text " + "compression." + ) + compression_attempts += 1 if compression_attempts > max_compression_attempts: # Terminal — surface the buffered retry trace. diff --git a/agent/turn_retry_state.py b/agent/turn_retry_state.py index 34183bd06be5..bd2d50e4f954 100644 --- a/agent/turn_retry_state.py +++ b/agent/turn_retry_state.py @@ -54,6 +54,17 @@ class TurnRetryState: oauth_1m_beta_retry_attempted: bool = False llama_cpp_grammar_retry_attempted: bool = False + # Aggregate-payload (413) image-shrink guards. Unlike a single-pass + # recovery, the 413 image path shrinks PROGRESSIVELY harder: the first + # 413 re-encodes every embedded image to ≤512 KB, a second 413 (the new + # payload still too big for the gateway) re-encodes to ≤256 KB. Two + # one-shot bools keep the dataclass all-boolean (see the field-set and + # all-False contract tests) while giving the loop two escalating passes + # before it falls through to text compression. See the + # ``payload_too_large`` branch in ``conversation_loop``. + payload_image_shrink_pass1_attempted: bool = False + payload_image_shrink_pass2_attempted: bool = False + # ── Transport / rate-limit recovery ────────────────────────────────── primary_recovery_attempted: bool = False has_retried_429: bool = False diff --git a/run_agent.py b/run_agent.py index 8432b0b3db1c..8864b0071c31 100644 --- a/run_agent.py +++ b/run_agent.py @@ -4676,12 +4676,14 @@ def _try_shrink_image_parts_in_messages( api_messages: list, *, max_dimension: int = 8000, + target_bytes: int = None, ) -> bool: """Forwarder — see ``agent.conversation_compression.try_shrink_image_parts_in_messages``.""" from agent.conversation_compression import try_shrink_image_parts_in_messages return try_shrink_image_parts_in_messages( api_messages, max_dimension=max_dimension, + target_bytes=target_bytes, ) def _try_strip_image_parts_from_tool_messages(self, api_messages: list) -> bool: diff --git a/tests/agent/test_turn_retry_state.py b/tests/agent/test_turn_retry_state.py index 21b772d68014..48d150fba22b 100644 --- a/tests/agent/test_turn_retry_state.py +++ b/tests/agent/test_turn_retry_state.py @@ -22,6 +22,8 @@ "thinking_sig_retry_attempted", "invalid_encrypted_content_retry_attempted", "image_shrink_retry_attempted", + "payload_image_shrink_pass1_attempted", + "payload_image_shrink_pass2_attempted", "multimodal_tool_content_retry_attempted", "oauth_1m_beta_retry_attempted", "llama_cpp_grammar_retry_attempted", diff --git a/tests/run_agent/test_image_shrink_recovery.py b/tests/run_agent/test_image_shrink_recovery.py index bdbb905d66e0..311fe9fae714 100644 --- a/tests/run_agent/test_image_shrink_recovery.py +++ b/tests/run_agent/test_image_shrink_recovery.py @@ -661,3 +661,103 @@ def test_byte_oversized_with_no_dim_cap_accepts_byte_shrink(self, monkeypatch): # Default cap (8000) — no explicit max_dimension passed. assert agent._try_shrink_image_parts_in_messages(msgs) is True assert msgs[0]["content"][0]["image_url"]["url"] == shrunk + + +class TestAggregatePayloadShrink: + """Aggregate-413 path: several individually-under-4MB images whose SUM + blows a gateway's request-body byte limit (observed on GitHub Copilot + with native-vision screenshots). The default 4 MB per-image budget + no-ops on these, so the 413 handler passes a small ``target_bytes`` to + force every embedded image down. Regression for the + ``payload_too_large`` image-shrink branch in ``conversation_loop``. + """ + + def test_default_budget_noops_on_sub_4mb_images(self, monkeypatch): + """~500 KB images are under the 4 MB default - shrink must NOT fire.""" + agent = _make_agent() + url1 = _big_png_data_url(500) + url2 = _big_png_data_url(480) + url3 = _big_png_data_url(520) + for u in (url1, url2, url3): + assert len(u) < 4 * 1024 * 1024 + + resize_hits = {"count": 0} + monkeypatch.setattr( + "tools.vision_tools._resize_image_for_vision", + lambda *a, **kw: resize_hits.__setitem__( + "count", resize_hits["count"] + 1 + ) or "shrunk", + raising=False, + ) + + msgs = [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": url1}}, + {"type": "image_url", "image_url": {"url": url2}}, + {"type": "image_url", "image_url": {"url": url3}}, + ], + }] + # No target_bytes override -> 4 MB default -> nothing exceeds it. + assert agent._try_shrink_image_parts_in_messages(msgs) is False + assert resize_hits["count"] == 0 + assert msgs[0]["content"][0]["image_url"]["url"] == url1 + + def test_small_target_bytes_shrinks_sub_4mb_images(self, monkeypatch): + """With a 512 KB target_bytes, the same ~500 KB images DO get shrunk.""" + agent = _make_agent() + url1 = _big_png_data_url(500) + url2 = _big_png_data_url(520) + shrunk = "data:image/jpeg;base64," + "S" * 1000 + + seen = {"max_base64_bytes": None} + + def _fake_resize(path, mime_type=None, max_base64_bytes=None, max_dimension=None): + seen["max_base64_bytes"] = max_base64_bytes + return shrunk + + monkeypatch.setattr( + "tools.vision_tools._resize_image_for_vision", + _fake_resize, + raising=False, + ) + + msgs = [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": url1}}, + {"type": "image_url", "image_url": {"url": url2}}, + ], + }] + changed = agent._try_shrink_image_parts_in_messages( + msgs, + max_dimension=2000, + target_bytes=512 * 1024, + ) + assert changed is True + assert seen["max_base64_bytes"] == 512 * 1024 + assert msgs[0]["content"][0]["image_url"]["url"] == shrunk + assert msgs[0]["content"][1]["image_url"]["url"] == shrunk + + def test_target_bytes_zero_falls_back_to_default(self, monkeypatch): + """target_bytes<=0 must fall back to the 4 MB default, not break.""" + agent = _make_agent() + small = _big_png_data_url(500) + resize_hits = {"count": 0} + monkeypatch.setattr( + "tools.vision_tools._resize_image_for_vision", + lambda *a, **kw: resize_hits.__setitem__( + "count", resize_hits["count"] + 1 + ) or "shrunk", + raising=False, + ) + msgs = [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": small}}, + ], + }] + assert agent._try_shrink_image_parts_in_messages( + msgs, target_bytes=0, + ) is False + assert resize_hits["count"] == 0