fix(codex): healthy app-server sessions survive long post-tool reasoning silence (#112928, salvage #112932)

agent/codex_runtime.py

@@ -490,7 +490,8 @@ def run_codex_app_server_turn(agent, *, user_message: str, original_user_message
             final_response=f"Codex app-server turn failed: {exc}. Fall back to default runtime with `/codex-runtime auto`.",
         )
     interrupt = _consume_user_interrupt(agent, turn.interrupted)
-    # Wedged client (deadline blown, watchdog tripped, OAuth refresh died, subprocess exited): retire it.
+    # Wedged client (turn deadline blown, OAuth refresh died, subprocess exited): retire it. Post-tool
+    # silence alone no longer retires — it only logs a warning (#112928).
     if getattr(turn, "should_retire", False):
         logger.warning("codex app-server session retired (turn error: %s)", turn.error)
         _close_codex_session(agent)

agent/transports/codex_app_server_session.py

@@ -52,7 +52,7 @@ class TurnResult:
     token_usage_last: Optional[dict[str, Any]] = None
     model_context_window: Optional[int] = None
     compacted: bool = False
-    # Codex likely wedged (turn timeout, watchdog, token refresh failure): caller respawns next turn.
+    # Codex likely wedged (turn timeout, dead subprocess, token refresh failure): caller respawns next turn.
     should_retire: bool = False
 
 
@@ -330,12 +330,11 @@ def run_turn(
     ) -> TurnResult:
         """Send a user message and block until turn/completed, bridging approvals and projecting items.
 
-        post_tool_quiet_timeout: silence this long after a tool completes fast-fails and retires.
-
         post_tool_quiet_timeout: if codex emits a tool completion and then goes quiet for this many seconds
-        without emitting another item or `turn/completed`, fast-fail and mark the session for retirement.
-        Mirrors openclaw beta.8's post-tool completion watchdog (#81697) so a wedged codex doesn't burn the
-        full turn deadline.
+        without emitting another item or `turn/completed`, log a warning (once per tool result) and keep
+        waiting. Wire silence is not evidence of a wedged process: after a large tool output codex can
+        reason for minutes without emitting a single event while the app-server still answers RPCs
+        (#112928). Only subprocess death or ``turn_timeout`` retires the session.
         """
         result = TurnResult()
         if self._start_for(result):
@@ -359,21 +358,25 @@ def _run_started_turn(
         self, result: TurnResult, ts: dict, turn_timeout: float, notification_poll_timeout: float,
         post_tool_quiet_timeout: float,
     ) -> None:
-        """Drive an accepted ``turn/start`` to completion: watchdog, approvals, projection."""
+        """Drive an accepted ``turn/start`` to completion: quiet warning, approvals, projection."""
         projector = CodexEventProjector()
         result.turn_id = (ts.get("turn") or {}).get("id")
         with self._active_turn_lock:
             self._active_turn_id = result.turn_id
-        # Post-tool watchdog: armed on each tool completion, cleared by any other activity.
+        # Post-tool quiet timer: armed on each tool completion, cleared by any other activity.
+        # Observability only — it never interrupts or retires (see run_turn docstring).
         last_tool_completion_at: Optional[float] = None
 
-        def watchdog_tripped() -> bool:
+        def warn_if_quiet() -> bool:
+            nonlocal last_tool_completion_at
             if last_tool_completion_at is None or (time.monotonic() - last_tool_completion_at) <= post_tool_quiet_timeout:
                 return False
-            self._issue_interrupt(result.turn_id)
-            result.interrupted = True
-            self._retire(result, f"codex went silent for {post_tool_quiet_timeout:.0f}s after a tool result; retiring app-server session.")
-            return True
+            last_tool_completion_at = None
+            logger.warning(
+                "codex has emitted no events for %.0fs after a tool result; still waiting (turn deadline %.0fs)",
+                post_tool_quiet_timeout, turn_timeout,
+            )
+            return False
 
         def on_server_request(sreq: dict) -> bool:
             nonlocal last_tool_completion_at
@@ -392,7 +395,7 @@ def on_server_request(sreq: dict) -> bool:
                     last_tool_completion_at = time.monotonic()
                 turn_complete = turn_complete or aborted
             self._handle_server_request(sreq)
-            # An approval round-trip is live signal — don't let it trip the watchdog.
+            # An approval round-trip is live signal — don't let it trip the quiet warning.
             last_tool_completion_at = None
             return turn_complete
 
@@ -414,7 +417,7 @@ def on_note(note: dict, method: str) -> bool:
 
         self._drive_turn(
             result, turn_timeout=turn_timeout, notification_poll_timeout=notification_poll_timeout,
-            timeout_label="turn", before_poll=watchdog_tripped, on_server_request=on_server_request,
+            timeout_label="turn", before_poll=warn_if_quiet, on_server_request=on_server_request,
             on_note=on_note, accept_final_text_at_deadline=True,
         )
         with self._active_turn_lock:
@@ -429,7 +432,7 @@ def _drive_turn(
     ) -> None:
         """Shared poll loop for run_turn / compact_thread until turn/completed or deadline.
 
-        Per iteration: interrupt -> subprocess death -> ``before_poll`` (watchdog) ->
+        Per iteration: interrupt -> subprocess death -> ``before_poll`` (quiet warning) ->
         server requests (answered first so codex isn't blocked) -> one notification,
         filtered by ``pre_scope_filter`` then turn scope, handed to ``on_note``. Hooks
         return True to complete the turn. Deadline without completion interrupts and

tests/agent/transports/test_codex_app_server_session.py

@@ -7,6 +7,8 @@
 
 from __future__ import annotations
 
+import itertools
+import logging
 import time
 from unittest.mock import patch
 from typing import Any, Optional
@@ -708,8 +710,8 @@ def cb(command, description, *, allow_permanent=True):
 class TestSessionRetirement:
     """Mirrors openclaw beta.8's resilience fixes:
       - retire timed-out app-server clients (should_retire on deadline)
-      - post-tool completion watchdog (don't burn the full deadline after a
-        tool result if codex goes silent)
+      - post-tool silence is a warning, never a retirement: only a dead
+        subprocess or the turn deadline retires (#112928)
       - <turn_aborted> raw marker as terminal (don't wait for turn/completed
         that never comes)
       - OAuth refresh failure classification (suggest `codex login` instead
@@ -747,7 +749,11 @@ def test_final_agent_message_without_turn_completed_is_recovered(self):
         assert not any(method == "turn/interrupt" for method, _ in client.requests)
 
 
-    def test_post_tool_watchdog_uses_monotonic_clock(self):
+    def test_post_tool_silence_warns_but_does_not_retire_a_healthy_turn(self, caplog):
+        """#112928: codex can reason for minutes after a large tool result without
+        emitting a single wire event while the process stays alive. Silence past
+        the quiet threshold must only warn; a later turn/completed ends the turn
+        normally with no turn/interrupt and no retirement."""
         client = FakeClient()
         client.queue_notification(
             "item/completed",
@@ -759,9 +765,15 @@ def test_post_tool_watchdog_uses_monotonic_clock(self):
             },
             threadId="t", turnId="tu1",
         )
+        client.queue_notification(
+            "turn/completed", threadId="t",
+            turn={"id": "tu1", "status": "completed", "error": None},
+        )
         s = make_session(client)
-        monotonic_values = iter([1000.0, 999.0, 999.0, 999.0, 1000.2])
-        with patch.object(
+        # Clock: deadline arm, tool item at 999.0, then every later poll sits
+        # well past the 0.15s quiet threshold until turn/completed is drained.
+        monotonic_values = itertools.chain([1000.0, 999.0, 999.0, 999.0], itertools.repeat(1000.2))
+        with caplog.at_level(logging.WARNING, logger=session_mod.logger.name), patch.object(
             session_mod.time,
             "monotonic",
             side_effect=lambda: next(monotonic_values),
@@ -772,13 +784,16 @@ def test_post_tool_watchdog_uses_monotonic_clock(self):
                 notification_poll_timeout=0.0,
                 post_tool_quiet_timeout=0.15,
             )
-        assert r.interrupted is True
-        assert r.should_retire is True
-        assert r.error and "silent" in r.error
+        assert r.interrupted is False
+        assert r.should_retire is False
+        assert r.error is None
+        assert r.tool_iterations == 1
+        assert not any(method == "turn/interrupt" for method, _ in client.requests)
+        assert any("no events for" in rec.getMessage() for rec in caplog.records)
 
-    def test_post_tool_watchdog_resets_on_further_activity(self):
-        """A tool completion followed by an agent message should NOT trip
-        the watchdog — further activity = codex still alive."""
+    def test_post_tool_activity_clears_the_quiet_timer_and_never_retires(self):
+        """A tool completion followed by an agent message completes normally: further activity clears
+        the post-tool quiet timer, and even when it expires it only warns, never retires."""
         client = FakeClient()
         client.queue_notification(
             "item/completed",
@@ -790,7 +805,7 @@ def test_post_tool_watchdog_resets_on_further_activity(self):
             },
             threadId="t", turnId="tu1",
         )
-        # Non-tool activity immediately after — resets watchdog.
+        # Non-tool activity immediately after — clears the quiet timer.
         client.queue_notification(
             "item/completed",
             item={"type": "agentMessage", "id": "m1", "text": "tool finished"},
@@ -806,7 +821,7 @@ def test_post_tool_watchdog_resets_on_further_activity(self):
             notification_poll_timeout=0.01,
             post_tool_quiet_timeout=0.05,
         )
-        # Tool ran, then text reset the watchdog, then turn/completed.
+        # Tool ran, then text cleared the quiet timer, then turn/completed.
         # Should NOT be a retirement case.
         assert r.tool_iterations == 1
         assert r.final_text == "tool finished"