From 4a67f42fcf40a722044db56924385e07cd94d7cb Mon Sep 17 00:00:00 2001 From: KKamJi Date: Mon, 20 Jul 2026 08:39:06 +0900 Subject: [PATCH] fix(gateway): strip hidden-reasoning incomplete tails from replayed live history A retry-exhausted Codex turn ends as an assistant message with finish_reason=incomplete, hidden reasoning only, and no visible answer or tool call. The gateway keeps that turn out of the persisted transcript, but the cached agent still holds it in live _session_messages. When the FTS write-corruption guard (#50502) resurrects the live transcript because disk persistence lagged, the poisoned tail is replayed to the provider and seeds another incomplete continuation loop. Add agent/replay_cleanup.strip_incomplete_reasoning_tail to drop that tail (plus interleaved _CODEX_INCOMPLETE_NUDGE user messages) and apply it in the gateway's FTS-lag reconciliation path before the stale-confirmation expiry. Visible partial answers, completed turns, and tool-call turns are never stripped, so genuine FTS-corruption recovery is preserved. The nudge is matched by prefix to avoid a conversation_loop import cycle; a regression test guards against drift. --- agent/replay_cleanup.py | 100 +++++++++++ gateway/run.py | 7 + tests/agent/test_replay_cleanup.py | 169 ++++++++++++++++++ ...est_incomplete_reasoning_tail_reconcile.py | 110 ++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 tests/gateway/test_incomplete_reasoning_tail_reconcile.py diff --git a/agent/replay_cleanup.py b/agent/replay_cleanup.py index 780fe761bbda..b9f4d479708b 100644 --- a/agent/replay_cleanup.py +++ b/agent/replay_cleanup.py @@ -201,6 +201,106 @@ def sanitize_replay_history( return strip_dangling_tool_call_tail(strip_interrupted_tool_tails(agent_history)) +# ---------------------------------------------------------------------- +# Hidden-reasoning-only incomplete tail expiry +# ---------------------------------------------------------------------- + +# Prefix of agent/conversation_loop.py::_CODEX_INCOMPLETE_NUDGE. Matched by +# prefix here (rather than imported) so replay_cleanup does not import +# conversation_loop and create a cycle. A regression test guards drift. +_CODEX_INCOMPLETE_NUDGE_PREFIX = ( + "[System: Your previous response contained only internal reasoning and" +) + + +def _has_visible_text(content: Any) -> bool: + """Return True if ``content`` carries any user-visible text. + + Accepts a plain string or the structured parts list (``{"type": "text", + "text": ...}`` / bare strings) that vision and post-compaction turns use. + """ + if isinstance(content, str): + return bool(content.strip()) + if isinstance(content, list): + for part in content: + if isinstance(part, dict): + if part.get("type") == "text" and str(part.get("text", "")).strip(): + return True + elif isinstance(part, str) and part.strip(): + return True + return False + return bool(content) + + +def _is_hidden_reasoning_incomplete_assistant(msg: Any) -> bool: + """Return True for a hidden-reasoning-only incomplete assistant turn. + + That is: an ``assistant`` message with ``finish_reason == "incomplete"``, + no ``tool_calls`` (a tool-call turn is handled by the tool-tail strippers, + not erased here), and no user-visible text (only hidden reasoning). This is + the exact shape a Codex continuation-retry-exhausted turn leaves in the + live ``_session_messages``. + """ + if not isinstance(msg, dict) or msg.get("role") != "assistant": + return False + if msg.get("finish_reason") != "incomplete": + return False + if msg.get("tool_calls"): + return False + return not _has_visible_text(msg.get("content")) + + +def _is_codex_incomplete_nudge(msg: Any) -> bool: + """Return True for a gateway-injected Codex "produce your final answer" nudge.""" + return ( + isinstance(msg, dict) + and msg.get("role") == "user" + and isinstance(msg.get("content"), str) + and msg["content"].startswith(_CODEX_INCOMPLETE_NUDGE_PREFIX) + ) + + +def strip_incomplete_reasoning_tail( + agent_history: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Strip a trailing hidden-reasoning-only incomplete assistant tail. + + When a Codex turn exhausts its continuation retries it ends with an + assistant message that has ``finish_reason == "incomplete"`` and only + hidden reasoning (no visible answer, no tool call). The gateway + deliberately keeps that turn OUT of the persisted transcript, but the + cached agent still holds it in its live ``_session_messages``. When the FTS + write-corruption guard resurrects the live transcript because disk + persistence lagged, that poisoned tail is replayed to the provider and + seeds another incomplete continuation loop (the hidden-reasoning-only + incomplete loop reported for the Discord/Mac gateway). + + Remove that tail (and the interleaved ``_CODEX_INCOMPLETE_NUDGE`` user + messages that only exist to prod it) so provider continuation resumes from + the last real turn, matching the state the persisted transcript already + represents. A visible partial answer (any content) or a completed turn is + never stripped, so genuine recovered context in the true FTS-corruption + case survives. Returns the same list object when there is nothing to strip. + """ + if not agent_history: + return agent_history + end = len(agent_history) + while end > 0: + msg = agent_history[end - 1] + if _is_hidden_reasoning_incomplete_assistant(msg) or _is_codex_incomplete_nudge(msg): + end -= 1 + continue + break + if end == len(agent_history): + return agent_history + logger.warning( + "Stripping hidden-reasoning-only incomplete assistant tail from replay " + "history (%d message(s)) so provider continuation does not loop", + len(agent_history) - end, + ) + return agent_history[:end] + + # ────────────────────────────────────────────────────────────────────── # Stale dangerous-confirmation text expiry (#59607) # ────────────────────────────────────────────────────────────────────── diff --git a/gateway/run.py b/gateway/run.py index b071e4854f39..7fea3252a917 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -1122,6 +1122,7 @@ def _last_transcript_timestamp(history: Optional[List[Dict[str, Any]]]) -> Any: is_interrupted_tool_result as _is_interrupted_tool_result, strip_interrupted_tool_tails as _strip_interrupted_tool_tails, strip_dangling_tool_call_tail as _strip_dangling_tool_call_tail, + strip_incomplete_reasoning_tail as _strip_incomplete_reasoning_tail, strip_stale_dangerous_confirmations as _strip_stale_dangerous_confirmations, is_dangerous_confirmation as _is_dangerous_confirmation, ) @@ -20531,6 +20532,12 @@ def _clarify_callback_sync(question: str, choices) -> str: # dangerous confirmation can't slip through this path # either. Idempotent; messages without timestamps are # untouched. + # Also drop a trailing hidden-reasoning-only incomplete + # assistant tail: a retry-exhausted Codex turn is + # suppressed from disk but lingers in live + # _session_messages, so this guard would otherwise + # resurrect it and loop provider continuation forever. + _selected = _strip_incomplete_reasoning_tail(_selected) agent_history = _strip_stale_dangerous_confirmations( _selected, now=time.time() ) diff --git a/tests/agent/test_replay_cleanup.py b/tests/agent/test_replay_cleanup.py index 14b44e8f2b2b..0e18324f30b6 100644 --- a/tests/agent/test_replay_cleanup.py +++ b/tests/agent/test_replay_cleanup.py @@ -9,6 +9,7 @@ from agent.replay_cleanup import ( is_interrupted_tool_result, strip_dangling_tool_call_tail, + strip_incomplete_reasoning_tail, strip_interrupted_tool_tails, sanitize_replay_history, ) @@ -146,3 +147,171 @@ def test_sanitize_replay_history_noop_on_clean_history(): def test_sanitize_replay_history_empty(): assert sanitize_replay_history([]) == [] + + +# --- strip_incomplete_reasoning_tail (hidden-reasoning-only incomplete loop) --- +# +# When a Codex turn exhausts its continuation retries it ends with an +# assistant message carrying finish_reason=="incomplete" and NO visible answer +# (only hidden reasoning). The gateway deliberately keeps that turn OUT of the +# persisted transcript, but the cached agent still holds it in its live +# _session_messages. The FTS-corruption guard then resurrects the live +# transcript when disk lagged and replays that poisoned tail, seeding another +# incomplete loop. This stripper removes it before provider continuation. + +_INCOMPLETE_ASSISTANT = { + "role": "assistant", + "content": "", + "reasoning": "let me think about this", + "finish_reason": "incomplete", +} + + +def _nudge(): + from agent.conversation_loop import _CODEX_INCOMPLETE_NUDGE + + return {"role": "user", "content": _CODEX_INCOMPLETE_NUDGE} + + +def test_strip_incomplete_reasoning_tail_removes_hidden_reasoning_only_tail(): + history = [_user("real question"), dict(_INCOMPLETE_ASSISTANT)] + out = strip_incomplete_reasoning_tail(history) + assert out == [_user("real question")] + + +def test_strip_incomplete_reasoning_tail_removes_interleaved_nudges_and_retries(): + history = [ + _user("real question"), + dict(_INCOMPLETE_ASSISTANT), + _nudge(), + dict(_INCOMPLETE_ASSISTANT), + _nudge(), + dict(_INCOMPLETE_ASSISTANT), + ] + out = strip_incomplete_reasoning_tail(history) + assert out == [_user("real question")] + + +def test_strip_incomplete_reasoning_tail_preserves_visible_incomplete_answer(): + # A partial-but-VISIBLE answer must never be discarded, even if the turn + # was marked incomplete: the user should still receive that text. + visible = { + "role": "assistant", + "content": "Here is a partial answer", + "finish_reason": "incomplete", + } + history = [_user("q"), visible] + assert strip_incomplete_reasoning_tail(history) == history + + +def test_strip_incomplete_reasoning_tail_preserves_completed_answer(): + history = [ + _user("q"), + {"role": "assistant", "content": "done", "finish_reason": "stop"}, + ] + assert strip_incomplete_reasoning_tail(history) == history + + +def test_strip_incomplete_reasoning_tail_only_touches_the_tail(): + # A completed assistant answer earlier in the history is a hard stop: + # nothing before the last real turn is removed. + history = [ + _user("q1"), + {"role": "assistant", "content": "answer 1", "finish_reason": "stop"}, + _user("q2"), + dict(_INCOMPLETE_ASSISTANT), + ] + out = strip_incomplete_reasoning_tail(history) + assert out == history[:3] + + +def test_strip_incomplete_reasoning_tail_ignores_incomplete_tool_call_turn(): + # An assistant turn that issued tool_calls is not "reasoning only": leave + # it for the tool-tail strippers, don't erase it here. + tc_turn = { + "role": "assistant", + "content": "", + "finish_reason": "incomplete", + "tool_calls": [ + {"id": "c1", "function": {"name": "read_file", "arguments": "{}"}} + ], + } + history = [_user("q"), tc_turn] + assert strip_incomplete_reasoning_tail(history) == history + + +def test_strip_incomplete_reasoning_tail_noop_and_identity(): + clean = [ + _user("hi"), + {"role": "assistant", "content": "hey", "finish_reason": "stop"}, + ] + assert strip_incomplete_reasoning_tail(clean) is clean + assert strip_incomplete_reasoning_tail([]) == [] + + +def test_nudge_prefix_stays_in_sync_with_conversation_loop(): + # The stripper matches the nudge by prefix (no import) to avoid a + # conversation_loop <-> replay_cleanup cycle; guard against drift. + from agent.conversation_loop import _CODEX_INCOMPLETE_NUDGE + from agent.replay_cleanup import _CODEX_INCOMPLETE_NUDGE_PREFIX + + assert _CODEX_INCOMPLETE_NUDGE.startswith(_CODEX_INCOMPLETE_NUDGE_PREFIX) + + +# --- structured-list content (vision / post-compaction turns) --- +# +# After a vision turn or post-compaction rewrite, an assistant message's +# ``content`` is a structured parts list (``[{"type": "text", "text": ...}]`` +# or bare strings) instead of a plain string. The hidden-reasoning-only +# detector routes that through _has_visible_text, so the stripper must treat a +# list carrying no visible text exactly like an empty string tail (strip it), +# and a list carrying any visible text like a partial answer (keep it). + + +def _incomplete_with_content(content): + return { + "role": "assistant", + "content": content, + "reasoning": "internal only", + "finish_reason": "incomplete", + } + + +def test_incomplete_tail_with_empty_structured_list_is_stripped(): + history = [_user("q"), _incomplete_with_content([])] + assert strip_incomplete_reasoning_tail(history) == [_user("q")] + + +def test_incomplete_tail_with_only_nontext_parts_is_stripped(): + # A parts list holding only non-text (e.g. reasoning/image) entries carries + # no user-visible answer, so it is a hidden-reasoning-only tail. + history = [ + _user("q"), + _incomplete_with_content([{"type": "reasoning", "text": "thinking"}]), + ] + assert strip_incomplete_reasoning_tail(history) == [_user("q")] + + +def test_incomplete_tail_with_blank_text_part_is_stripped(): + # A text part whose text is empty/whitespace is not visible content. + history = [ + _user("q"), + _incomplete_with_content([{"type": "text", "text": " "}]), + ] + assert strip_incomplete_reasoning_tail(history) == [_user("q")] + + +def test_incomplete_tail_with_visible_text_part_is_preserved(): + # A visible text part in the structured list is a partial answer the user + # should still receive, so it must never be stripped. + history = [ + _user("q"), + _incomplete_with_content([{"type": "text", "text": "partial answer"}]), + ] + assert strip_incomplete_reasoning_tail(history) == history + + +def test_incomplete_tail_with_bare_string_part_is_preserved(): + # Bare non-empty strings in the parts list also count as visible content. + history = [_user("q"), _incomplete_with_content(["visible via bare string"])] + assert strip_incomplete_reasoning_tail(history) == history diff --git a/tests/gateway/test_incomplete_reasoning_tail_reconcile.py b/tests/gateway/test_incomplete_reasoning_tail_reconcile.py new file mode 100644 index 000000000000..05110b66661d --- /dev/null +++ b/tests/gateway/test_incomplete_reasoning_tail_reconcile.py @@ -0,0 +1,110 @@ +"""Regression: persisted-transcript lag must not resurrect a hidden-reasoning- +only incomplete Codex tail and loop provider continuation forever. + +Verified incident (session 20260719_133329_0f161a77): the gateway logged +"Persisted transcript lagged live cached history ... disk=3, memory=4" +immediately before "Codex response remained incomplete after 3 continuation +attempts". Root cause: a retry-exhausted Codex turn is deliberately kept OUT of +the persisted transcript (only the user message is written), but the cached +agent keeps the poisoned reasoning-only assistant tail in its live +_session_messages. On the next turn disk= memory): the guard is a no-op and never rewrites the + # already-cleaned persisted history. + persisted = [ + {"role": "user", "content": "q1"}, + {"role": "assistant", "content": "a1", "finish_reason": "stop"}, + ] + live = [{"role": "user", "content": "q1"}] + + assert _reconcile(persisted, live) is persisted