diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 0b0ca6a6aa9c9..60d70c0408348 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -2234,26 +2234,53 @@ def _relocate_orphaned_tool_search_results(messages: List[Dict[str, Any]]) -> No def drop_orphan_server_tool_uses_in_storage( messages: List[Dict[str, Any]], ) -> int: - """Drop any ``server_tool_use`` block whose paired - ``tool_search_tool_*_tool_result`` doesn't exist anywhere in the - message list. + """Drop server-side block orphans in BOTH directions: + + * ``server_tool_use`` whose paired result block + (``tool_search_tool_*_tool_result`` OR ``web_search_tool_result``) + doesn't exist anywhere in the message list. + * ``tool_search_tool_*_tool_result`` / ``web_search_tool_result`` + whose paired ``server_tool_use`` doesn't exist anywhere in the + message list. Why: relocation handles "result split across messages" — the normal Anthropic delivery pattern. But a stream interruption (timeout, - cancel, 5xx mid-response) can land the ``server_tool_use`` on disk - without the result EVER arriving. Every subsequent API call then - 400s with: + cancel, 5xx mid-response) can land either side of the pair on disk + without the other. Every subsequent API call then 400s with: ``tool_search_tool_ tool use with id ... was found - without a corresponding tool_search_tool__tool_result``. + without a corresponding tool_search_tool__tool_result`` + or + ``unexpected `tool_use_id` found in `web_search_tool_result` + blocks: . Each `web_search_tool_result` block must have a + corresponding `server_tool_use` block before it``. The session is permanently wedged until the orphan is removed. - Verified against ``session_20260509_145003_c5e465`` where one - server_tool_use had no result anywhere — dropping it unwedges the - session with no loss of usable data (the unfinished tool search - yielded nothing the model could act on anyway). + Verified against: + * ``session_20260509_145003_c5e465`` — server_tool_use without + result (tool_search side). + * ``session_20260513_093942_d374cc`` — web_search_tool_result + without server_tool_use. The prior version of this function + CAUSED that breakage: it tracked only tool_search results, so a + healthy web_search server_tool_use looked unpaired and got + dropped, leaving the web_search_tool_result orphaned forever. - Returns the number of orphan use blocks removed. + Returns the number of orphan blocks removed (uses + results). + + Extend ``SERVER_TOOL_RESULT_TYPES`` when Anthropic adds new + server-side tools that emit a paired result block so the pairing + audit stays correct. """ + SERVER_TOOL_RESULT_TYPES = ("tool_search_tool_result", "web_search_tool_result") + + def _is_server_tool_result(t: Any) -> bool: + return isinstance(t, str) and ( + t in SERVER_TOOL_RESULT_TYPES + or (t.startswith("tool_search_tool_") and t.endswith("_tool_result")) + ) + + # Phase 1: collect every server-side use-id and result-id that + # actually exists on disk. + use_ids: set[str] = set() result_ids: set[str] = set() for msg in messages: if msg.get("role") != "assistant": @@ -2265,16 +2292,16 @@ def drop_orphan_server_tool_uses_in_storage( if not isinstance(block, dict): continue t = block.get("type") - if not isinstance(t, str): - continue - if ( - t == "tool_search_tool_result" - or (t.startswith("tool_search_tool_") and t.endswith("_tool_result")) - ): + if t == "server_tool_use": + bid = block.get("id") + if isinstance(bid, str): + use_ids.add(bid) + elif _is_server_tool_result(t): tu_id = block.get("tool_use_id") if isinstance(tu_id, str): result_ids.add(tu_id) + # Phase 2: drop orphans in both directions in a single pass. dropped = 0 for msg in messages: if msg.get("role") != "assistant": @@ -2282,19 +2309,29 @@ def drop_orphan_server_tool_uses_in_storage( content = msg.get("anthropic_content_blocks") if not isinstance(content, list): continue - keep = [] + keep: List[Dict[str, Any]] = [] + msg_dropped = 0 for block in content: - if ( - isinstance(block, dict) - and block.get("type") == "server_tool_use" - and isinstance(block.get("id"), str) - and block["id"] not in result_ids - ): - dropped += 1 - continue + if isinstance(block, dict): + t = block.get("type") + if ( + t == "server_tool_use" + and isinstance(block.get("id"), str) + and block["id"] not in result_ids + ): + msg_dropped += 1 + continue + if ( + _is_server_tool_result(t) + and isinstance(block.get("tool_use_id"), str) + and block["tool_use_id"] not in use_ids + ): + msg_dropped += 1 + continue keep.append(block) - if dropped: + if msg_dropped: msg["anthropic_content_blocks"] = keep + dropped += msg_dropped return dropped @@ -3098,13 +3135,34 @@ def convert_messages_to_anthropic( # owns the matching server_tool_use. _relocate_orphaned_tool_search_results(result) - # Drop ``server_tool_use`` blocks whose paired result NEVER arrived - # (stream interruption, timeout, cancel mid-response). Without this, - # the assistant message has a use without a result, and every API - # call replays the orphan and 400s. Runs after relocation so a - # split-but-deliverable pair gets repaired first; only truly - # missing results trigger a drop. Operates on the wire-shape - # ``msg["content"]`` (lists of blocks). + # Drop server-side block orphans in BOTH directions on the + # wire-shape ``msg["content"]`` immediately before send: + # + # * ``server_tool_use`` whose paired result block never arrived + # (stream interruption / timeout / cancel mid-response). + # * ``web_search_tool_result`` / ``tool_search_tool_*_tool_result`` + # whose paired ``server_tool_use`` is missing (compaction cut, + # or a stale on-disk corruption from an earlier Hermes version + # that dropped the wrong side of the pair). + # + # Without either side of this audit, the assistant message has an + # unpaired server-side block and the request 400s with one of: + # * ``tool_search_tool_ tool use with id ... was found + # without a corresponding tool_search_tool__tool_result`` + # * ``unexpected `tool_use_id` found in `web_search_tool_result` + # blocks: . Each `web_search_tool_result` block must have a + # corresponding `server_tool_use` block before it`` + # Runs AFTER ``_relocate_orphaned_tool_search_results`` so a + # split-but-deliverable pair gets repaired first. + _SERVER_RESULT_TYPES_WIRE = ("tool_search_tool_result", "web_search_tool_result") + + def _is_server_tool_result_wire(_t): + return isinstance(_t, str) and ( + _t in _SERVER_RESULT_TYPES_WIRE + or (_t.startswith("tool_search_tool_") and _t.endswith("_tool_result")) + ) + + _use_ids_wire: set = set() _result_ids_wire: set = set() for _m in result: if _m.get("role") != "assistant": @@ -3116,10 +3174,11 @@ def convert_messages_to_anthropic( if not isinstance(_b, dict): continue _t = _b.get("type") - if isinstance(_t, str) and ( - _t == "tool_search_tool_result" - or (_t.startswith("tool_search_tool_") and _t.endswith("_tool_result")) - ): + if _t == "server_tool_use": + _bid = _b.get("id") + if isinstance(_bid, str): + _use_ids_wire.add(_bid) + elif _is_server_tool_result_wire(_t): _ru = _b.get("tool_use_id") if isinstance(_ru, str): _result_ids_wire.add(_ru) @@ -3129,15 +3188,23 @@ def convert_messages_to_anthropic( _c = _m.get("content") if not isinstance(_c, list): continue - _kept = [ - _b for _b in _c - if not ( - isinstance(_b, dict) - and _b.get("type") == "server_tool_use" - and isinstance(_b.get("id"), str) - and _b["id"] not in _result_ids_wire - ) - ] + _kept = [] + for _b in _c: + if isinstance(_b, dict): + _t = _b.get("type") + if ( + _t == "server_tool_use" + and isinstance(_b.get("id"), str) + and _b["id"] not in _result_ids_wire + ): + continue + if ( + _is_server_tool_result_wire(_t) + and isinstance(_b.get("tool_use_id"), str) + and _b["tool_use_id"] not in _use_ids_wire + ): + continue + _kept.append(_b) if len(_kept) != len(_c): _m["content"] = _kept or [{"type": "text", "text": "(empty)"}] diff --git a/tests/agent/test_anthropic_tool_search_roundtrip.py b/tests/agent/test_anthropic_tool_search_roundtrip.py index 241c5cf5aa502..61ba24a17b23e 100644 --- a/tests/agent/test_anthropic_tool_search_roundtrip.py +++ b/tests/agent/test_anthropic_tool_search_roundtrip.py @@ -33,6 +33,7 @@ _normalize_tool_search_result_inner, _relocate_orphaned_tool_search_results, convert_messages_to_anthropic, + drop_orphan_server_tool_uses_in_storage, ) @@ -291,10 +292,46 @@ def test_preserves_outer_cache_control(self): # --------------------------------------------------------------------------- class TestConvertMessagesRoundTrip: def _build_assistant_msg(self, server_tool_blocks): + # Real Anthropic responses always pair a ``server_tool_use`` + # with each ``*_tool_result``. The outbound request-build path + # drops any unpaired result block (would 400 on Anthropic's + # input validator anyway). For each result block in the + # fixture, auto-prepend a matching server_tool_use so the + # message is shape-correct. + synthetic = [] + for b in server_tool_blocks: + if not isinstance(b, dict): + synthetic.append(b) + continue + t = b.get("type") + tu_id = b.get("tool_use_id") + if ( + isinstance(t, str) + and isinstance(tu_id, str) + and ( + t == "tool_search_tool_result" + or t == "web_search_tool_result" + or (t.startswith("tool_search_tool_") and t.endswith("_tool_result")) + ) + ): + # Pick a placeholder tool name that matches the result + # family. Hand-set name so server_tool_use is identifiable. + stu_name = ( + "web_search" + if t == "web_search_tool_result" + else "tool_search_tool_regex" + ) + synthetic.append({ + "type": "server_tool_use", + "id": tu_id, + "name": stu_name, + "input": {}, + }) + synthetic.append(b) return { "role": "assistant", "content": "Looking that up for you.", - "server_tool_blocks": server_tool_blocks, + "server_tool_blocks": synthetic, "tool_calls": [], } @@ -1257,3 +1294,214 @@ def test_end_to_end_via_convert_messages_to_anthropic(self): if isinstance(b, dict) ] assert "tool_result" in next_types + + +# --------------------------------------------------------------------------- +# drop_orphan_server_tool_uses_in_storage — symmetric orphan audit +# Covers regression where the prior version dropped a healthy +# ``server_tool_use`` paired with a ``web_search_tool_result`` because +# only tool_search result types were treated as evidence of pairing +# (session 20260513_093942_d374cc). +# --------------------------------------------------------------------------- +class TestDropOrphanServerToolUsesInStorage: + def _stu(self, tu_id: str, name: str = "web_search"): + return {"type": "server_tool_use", "id": tu_id, "name": name, "input": {}} + + def _wsr(self, tu_id: str): + return { + "type": "web_search_tool_result", + "tool_use_id": tu_id, + "content": [{"type": "web_search_result", "url": "https://x", "title": "t"}], + } + + def _tsr(self, tu_id: str, variant: str = "regex"): + return { + "type": f"tool_search_tool_{variant}_tool_result", + "tool_use_id": tu_id, + "content": {"type": "tool_search_tool_search_result", "tool_references": []}, + } + + def _assistant(self, blocks): + return {"role": "assistant", "anthropic_content_blocks": blocks} + + def test_keeps_healthy_web_search_pair(self): + """REGRESSION GUARD: the prior version dropped the + server_tool_use here because it only recognized + ``tool_search_*_tool_result`` as a paired result, not + ``web_search_tool_result``.""" + msgs = [ + {"role": "user", "content": "hi"}, + self._assistant([ + self._stu("srvtoolu_OK", name="web_search"), + self._wsr("srvtoolu_OK"), + {"type": "text", "text": "done"}, + ]), + ] + dropped = drop_orphan_server_tool_uses_in_storage(msgs) + assert dropped == 0 + types = [b["type"] for b in msgs[1]["anthropic_content_blocks"]] + assert types == ["server_tool_use", "web_search_tool_result", "text"] + + def test_keeps_healthy_tool_search_pair(self): + msgs = [ + self._assistant([ + self._stu("srvtoolu_TS", name="tool_search_tool_regex"), + self._tsr("srvtoolu_TS"), + ]), + ] + dropped = drop_orphan_server_tool_uses_in_storage(msgs) + assert dropped == 0 + types = [b["type"] for b in msgs[0]["anthropic_content_blocks"]] + assert "server_tool_use" in types + assert "tool_search_tool_regex_tool_result" in types + + def test_drops_orphan_server_tool_use_with_no_result(self): + msgs = [ + self._assistant([ + self._stu("srvtoolu_LONELY"), + {"type": "text", "text": "x"}, + ]), + ] + dropped = drop_orphan_server_tool_uses_in_storage(msgs) + assert dropped == 1 + types = [b["type"] for b in msgs[0]["anthropic_content_blocks"]] + assert "server_tool_use" not in types + assert types == ["text"] + + def test_drops_orphan_web_search_result_with_no_use(self): + """The exact shape of session 20260513_093942_d374cc — a + web_search_tool_result block sitting in the message list with + no matching server_tool_use anywhere. The API 400s on this + until it's dropped.""" + msgs = [ + self._assistant([ + {"type": "thinking", "thinking": "...", "signature": "s"}, + self._wsr("srvtoolu_ORPHAN_WSR"), + {"type": "text", "text": "still wrote a reply"}, + ]), + ] + dropped = drop_orphan_server_tool_uses_in_storage(msgs) + assert dropped == 1 + types = [b["type"] for b in msgs[0]["anthropic_content_blocks"]] + assert "web_search_tool_result" not in types + # Other content survives. + assert types == ["thinking", "text"] + + def test_drops_orphan_tool_search_result_with_no_use(self): + msgs = [ + self._assistant([ + self._tsr("srvtoolu_ORPHAN_TSR"), + ]), + ] + dropped = drop_orphan_server_tool_uses_in_storage(msgs) + assert dropped == 1 + # All blocks gone; keep is empty list (function does not inject + # placeholder text here — that's only the outbound wire-build path). + assert msgs[0]["anthropic_content_blocks"] == [] + + def test_mixed_session_drops_only_unpaired_blocks(self): + msgs = [ + self._assistant([ + # healthy web_search pair + self._stu("srvtoolu_GOOD_WS", name="web_search"), + self._wsr("srvtoolu_GOOD_WS"), + # orphan server_tool_use + self._stu("srvtoolu_ORPHAN_USE"), + # orphan web_search_tool_result + self._wsr("srvtoolu_ORPHAN_RES"), + # healthy tool_search pair + self._stu("srvtoolu_GOOD_TS", name="tool_search_tool_regex"), + self._tsr("srvtoolu_GOOD_TS"), + {"type": "text", "text": "tail"}, + ]), + ] + dropped = drop_orphan_server_tool_uses_in_storage(msgs) + assert dropped == 2 # the use orphan + the result orphan + kept_ids = [ + (b.get("type"), b.get("id") or b.get("tool_use_id")) + for b in msgs[0]["anthropic_content_blocks"] + ] + assert ("server_tool_use", "srvtoolu_GOOD_WS") in kept_ids + assert ("web_search_tool_result", "srvtoolu_GOOD_WS") in kept_ids + assert ("server_tool_use", "srvtoolu_GOOD_TS") in kept_ids + assert ("tool_search_tool_regex_tool_result", "srvtoolu_GOOD_TS") in kept_ids + # Orphans are gone. + assert ("server_tool_use", "srvtoolu_ORPHAN_USE") not in kept_ids + assert ("web_search_tool_result", "srvtoolu_ORPHAN_RES") not in kept_ids + + def test_pair_split_across_messages_is_not_dropped(self): + """Relocation handles splits; this sanitizer only fires when a + block is unpaired ANYWHERE in the message list.""" + msgs = [ + self._assistant([ + self._stu("srvtoolu_SPLIT", name="web_search"), + ]), + {"role": "user", "content": "follow-up"}, + self._assistant([ + self._wsr("srvtoolu_SPLIT"), + {"type": "text", "text": "answer"}, + ]), + ] + dropped = drop_orphan_server_tool_uses_in_storage(msgs) + assert dropped == 0 + + def test_outbound_wire_shape_drops_orphan_web_search_result(self): + """End-to-end check: a message persisted with an orphaned + ``web_search_tool_result`` in ``anthropic_content_blocks`` must + not produce a 400-shaped payload after convert_messages_to_anthropic. + + Reproduces session 20260513_093942_d374cc exactly: the API + rejected the very next call with:: + + unexpected `tool_use_id` found in `web_search_tool_result` + blocks: srvtoolu_01XyDgKcEqDSm8udPKWPNBsP. Each + `web_search_tool_result` block must have a corresponding + `server_tool_use` block before it. + """ + # Build a session where the persisted assistant message has a + # web_search_tool_result but its matching server_tool_use was + # (incorrectly) dropped earlier. Use the run_agent path: the + # adapter pulls server-side blocks out of ``server_tool_blocks`` + # on the dict — mimic that. + msg = { + "role": "assistant", + "content": "okay", + "server_tool_blocks": [ + # Note: NO matching server_tool_use. This is the + # broken on-disk shape. + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_BROKEN", + "content": [ + { + "type": "web_search_result", + "url": "https://x", + "title": "t", + } + ], + }, + ], + "tool_calls": [], + } + _, out_msgs = convert_messages_to_anthropic( + [{"role": "user", "content": "hi"}, msg] + ) + # No orphan web_search_tool_result anywhere in the outbound payload. + for m in out_msgs: + content = m.get("content") + if not isinstance(content, list): + continue + wsr_ids = [ + b.get("tool_use_id") + for b in content + if isinstance(b, dict) and b.get("type") == "web_search_tool_result" + ] + stu_ids = [ + b.get("id") + for b in content + if isinstance(b, dict) and b.get("type") == "server_tool_use" + ] + for tid in wsr_ids: + assert tid in stu_ids, ( + f"orphan web_search_tool_result {tid!r} survived to outbound payload" + )