From b5ac5b94355068613d445c683461efdb85744a04 Mon Sep 17 00:00:00 2001 From: Marc Cull Date: Tue, 2 Jun 2026 13:24:06 +0000 Subject: [PATCH] fix(agent): shrink oversized inline images before compressing on 413 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 413 'payload too large' on a turn carrying a large base64 image (e.g. a 15–25 MB iPhone photo posted in Slack) used to skip the image-shrink recovery path because that path is only wired into Anthropic's 400 'image exceeds 5 MB maximum' error. Anthropic returns 413 (not 400) when the *whole* request is oversized, so the existing handler went straight to _compress_context — which drops *old* messages but leaves the giant inline image on the current turn intact. Result: 413 → compress (no shrink) → 413 → … → 'Cannot compress further' → session auto-reset. Try _try_shrink_image_parts_in_messages once before counting a compression attempt. If it changes anything, retry immediately; otherwise fall through to compression as before. Symmetric with the existing 400/image_too_large path. Gated by the same image_shrink_retry_attempted flag so a second 413 still falls through to compression instead of looping. Triggered by Marc's items-for-sale workflow on Slack. --- agent/conversation_loop.py | 23 +++++++++ tests/run_agent/test_413_compression.py | 64 +++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 743988b03b0fe..b642ec5814b95 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -2852,6 +2852,29 @@ def _stop_spinner(): ) if is_payload_too_large: + # Before counting a compression attempt, try shrinking + # oversized native image parts in-place. Providers like + # Anthropic return 413 (not 400 "image exceeds N MB") + # when the *whole* request is oversized — usually because + # one or more inline base64 images are huge. Compression + # drops *old* messages, so a fresh user turn with a 20 MB + # iPhone photo would 413, compress (no change), 413, + # compress (no change), … until max attempts. Try shrink + # once; if it changes anything, retry without burning a + # compression attempt. See the symmetric 400 handler + # above (FailoverReason.image_too_large). + if ( + not image_shrink_retry_attempted + and agent._try_shrink_image_parts_in_messages(api_messages) + ): + image_shrink_retry_attempted = True + agent._vprint( + f"{agent.log_prefix}📐 413 payload too large — shrank " + f"oversized image part(s) and retrying before compression...", + force=True, + ) + continue + compression_attempts += 1 if compression_attempts > max_compression_attempts: # Terminal — surface the buffered retry trace. diff --git a/tests/run_agent/test_413_compression.py b/tests/run_agent/test_413_compression.py index cadb26c449b3f..bf17dc52e899a 100644 --- a/tests/run_agent/test_413_compression.py +++ b/tests/run_agent/test_413_compression.py @@ -409,6 +409,70 @@ def test_413_cannot_compress_further(self, agent): assert result.get("partial") is True assert "413" in result["error"] + def test_413_with_oversized_image_shrinks_before_compression(self, agent): + """A 413 on a turn with an oversized inline image should shrink first. + + Regression for the items-for-sale workflow: a 15-25 MB iPhone photo + inlined as a data:base64 URL triggers Anthropic's 413 (whole-request + payload too large, NOT the 400 "image exceeds 5 MB" path). Before + the fix, the 413 handler went straight to _compress_context, which + drops old messages but leaves the huge user image in place → 413 → + compress → 413 → … until max attempts → session auto-reset. + With the fix, _try_shrink_image_parts_in_messages runs first, shrinks + the image, and the retry succeeds without burning a compression slot. + """ + err_413 = _make_413_error() + ok_resp = _mock_response(content="OK after shrink", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [err_413, ok_resp] + + with ( + patch.object( + agent, "_try_shrink_image_parts_in_messages", return_value=True + ) as mock_shrink, + patch.object(agent, "_compress_context") as mock_compress, + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + result = agent.run_conversation("look at this photo") + + # Shrink helper was called and short-circuited the recovery — + # compression must NOT have run on this turn. + assert mock_shrink.call_count >= 1 + mock_compress.assert_not_called() + assert result["completed"] is True + assert result["final_response"] == "OK after shrink" + + def test_413_falls_back_to_compression_when_no_image_to_shrink(self, agent): + """413 without any oversized images: shrink returns False, compression runs.""" + err_413 = _make_413_error() + ok_resp = _mock_response(content="OK after compression", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [err_413, ok_resp] + + prefill = [ + {"role": "user", "content": "previous question"}, + {"role": "assistant", "content": "previous answer"}, + ] + + with ( + patch.object( + agent, "_try_shrink_image_parts_in_messages", return_value=False + ) as mock_shrink, + patch.object(agent, "_compress_context") as mock_compress, + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + mock_compress.return_value = ( + [{"role": "user", "content": "hello"}], + "compressed prompt", + ) + result = agent.run_conversation("hello", conversation_history=prefill) + + assert mock_shrink.call_count >= 1 + mock_compress.assert_called_once() + assert result["completed"] is True + class TestPreflightCompression: """Preflight compression should compress history before the first API call."""