From bcf60167244758df0429cb25a5aaa89067f9d1f7 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:58:13 -0700 Subject: [PATCH 1/2] fix(codex): healthy app-server sessions survive long post-tool reasoning silence After a tool result, `CodexAppServerSession._run_started_turn` armed a 90s wire-silence watchdog that sent turn/interrupt and retired the session. On large contexts codex legitimately emits no events for minutes while it reasons after a big tool output, and the app-server answers RPCs the whole time (the interrupt was acknowledged in ~25ms). Silence is not evidence of a wedged process, so the watchdog killed healthy work and surfaced as protocol_violation / crashed workers. Keep `post_tool_quiet_timeout` as observability only: past the threshold log one warning per tool result and keep waiting. Retirement still happens on the two real signals the poll loop already checks every iteration -- subprocess death (`_subprocess_died`) and the overall `turn_timeout` deadline -- so a truly wedged codex stays bounded. The existing monotonic-clock watchdog test becomes the invariant for the new behaviour: tool item, silence past the threshold, then turn/completed -> no interrupt, no retirement, a warning logged. Fixes #112928 Co-authored-by: KoNit-K <124019182+KoNit-K@users.noreply.github.com> --- agent/transports/codex_app_server_session.py | 35 ++++++++++--------- .../test_codex_app_server_session.py | 31 +++++++++++----- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/agent/transports/codex_app_server_session.py b/agent/transports/codex_app_server_session.py index 25635069cfa3c..ede339a8008a3 100644 --- a/agent/transports/codex_app_server_session.py +++ b/agent/transports/codex_app_server_session.py @@ -52,7 +52,7 @@ class TurnResult: token_usage_last: Optional[dict[str, Any]] = None model_context_window: Optional[int] = None compacted: bool = False - # Codex likely wedged (turn timeout, watchdog, token refresh failure): caller respawns next turn. + # Codex likely wedged (turn timeout, dead subprocess, token refresh failure): caller respawns next turn. should_retire: bool = False @@ -330,12 +330,11 @@ def run_turn( ) -> TurnResult: """Send a user message and block until turn/completed, bridging approvals and projecting items. - post_tool_quiet_timeout: silence this long after a tool completes fast-fails and retires. - post_tool_quiet_timeout: if codex emits a tool completion and then goes quiet for this many seconds - without emitting another item or `turn/completed`, fast-fail and mark the session for retirement. - Mirrors openclaw beta.8's post-tool completion watchdog (#81697) so a wedged codex doesn't burn the - full turn deadline. + without emitting another item or `turn/completed`, log a warning (once per tool result) and keep + waiting. Wire silence is not evidence of a wedged process: after a large tool output codex can + reason for minutes without emitting a single event while the app-server still answers RPCs + (#112928). Only subprocess death or ``turn_timeout`` retires the session. """ result = TurnResult() if self._start_for(result): @@ -359,21 +358,25 @@ def _run_started_turn( self, result: TurnResult, ts: dict, turn_timeout: float, notification_poll_timeout: float, post_tool_quiet_timeout: float, ) -> None: - """Drive an accepted ``turn/start`` to completion: watchdog, approvals, projection.""" + """Drive an accepted ``turn/start`` to completion: quiet warning, approvals, projection.""" projector = CodexEventProjector() result.turn_id = (ts.get("turn") or {}).get("id") with self._active_turn_lock: self._active_turn_id = result.turn_id - # Post-tool watchdog: armed on each tool completion, cleared by any other activity. + # Post-tool quiet timer: armed on each tool completion, cleared by any other activity. + # Observability only — it never interrupts or retires (see run_turn docstring). last_tool_completion_at: Optional[float] = None - def watchdog_tripped() -> bool: + def warn_if_quiet() -> bool: + nonlocal last_tool_completion_at if last_tool_completion_at is None or (time.monotonic() - last_tool_completion_at) <= post_tool_quiet_timeout: return False - self._issue_interrupt(result.turn_id) - result.interrupted = True - self._retire(result, f"codex went silent for {post_tool_quiet_timeout:.0f}s after a tool result; retiring app-server session.") - return True + last_tool_completion_at = None + logger.warning( + "codex has emitted no events for %.0fs after a tool result; still waiting (turn deadline %.0fs)", + post_tool_quiet_timeout, turn_timeout, + ) + return False def on_server_request(sreq: dict) -> bool: nonlocal last_tool_completion_at @@ -392,7 +395,7 @@ def on_server_request(sreq: dict) -> bool: last_tool_completion_at = time.monotonic() turn_complete = turn_complete or aborted self._handle_server_request(sreq) - # An approval round-trip is live signal — don't let it trip the watchdog. + # An approval round-trip is live signal — don't let it trip the quiet warning. last_tool_completion_at = None return turn_complete @@ -414,7 +417,7 @@ def on_note(note: dict, method: str) -> bool: self._drive_turn( result, turn_timeout=turn_timeout, notification_poll_timeout=notification_poll_timeout, - timeout_label="turn", before_poll=watchdog_tripped, on_server_request=on_server_request, + timeout_label="turn", before_poll=warn_if_quiet, on_server_request=on_server_request, on_note=on_note, accept_final_text_at_deadline=True, ) with self._active_turn_lock: @@ -429,7 +432,7 @@ def _drive_turn( ) -> None: """Shared poll loop for run_turn / compact_thread until turn/completed or deadline. - Per iteration: interrupt -> subprocess death -> ``before_poll`` (watchdog) -> + Per iteration: interrupt -> subprocess death -> ``before_poll`` (quiet warning) -> server requests (answered first so codex isn't blocked) -> one notification, filtered by ``pre_scope_filter`` then turn scope, handed to ``on_note``. Hooks return True to complete the turn. Deadline without completion interrupts and diff --git a/tests/agent/transports/test_codex_app_server_session.py b/tests/agent/transports/test_codex_app_server_session.py index 7548e59e2918d..da1e318f17749 100644 --- a/tests/agent/transports/test_codex_app_server_session.py +++ b/tests/agent/transports/test_codex_app_server_session.py @@ -7,6 +7,8 @@ from __future__ import annotations +import itertools +import logging import time from unittest.mock import patch from typing import Any, Optional @@ -708,8 +710,8 @@ def cb(command, description, *, allow_permanent=True): class TestSessionRetirement: """Mirrors openclaw beta.8's resilience fixes: - retire timed-out app-server clients (should_retire on deadline) - - post-tool completion watchdog (don't burn the full deadline after a - tool result if codex goes silent) + - post-tool silence is a warning, never a retirement: only a dead + subprocess or the turn deadline retires (#112928) - raw marker as terminal (don't wait for turn/completed that never comes) - OAuth refresh failure classification (suggest `codex login` instead @@ -747,7 +749,11 @@ def test_final_agent_message_without_turn_completed_is_recovered(self): assert not any(method == "turn/interrupt" for method, _ in client.requests) - def test_post_tool_watchdog_uses_monotonic_clock(self): + def test_post_tool_silence_warns_but_does_not_retire_a_healthy_turn(self, caplog): + """#112928: codex can reason for minutes after a large tool result without + emitting a single wire event while the process stays alive. Silence past + the quiet threshold must only warn; a later turn/completed ends the turn + normally with no turn/interrupt and no retirement.""" client = FakeClient() client.queue_notification( "item/completed", @@ -759,9 +765,15 @@ def test_post_tool_watchdog_uses_monotonic_clock(self): }, threadId="t", turnId="tu1", ) + client.queue_notification( + "turn/completed", threadId="t", + turn={"id": "tu1", "status": "completed", "error": None}, + ) s = make_session(client) - monotonic_values = iter([1000.0, 999.0, 999.0, 999.0, 1000.2]) - with patch.object( + # Clock: deadline arm, tool item at 999.0, then every later poll sits + # well past the 0.15s quiet threshold until turn/completed is drained. + monotonic_values = itertools.chain([1000.0, 999.0, 999.0, 999.0], itertools.repeat(1000.2)) + with caplog.at_level(logging.WARNING, logger=session_mod.logger.name), patch.object( session_mod.time, "monotonic", side_effect=lambda: next(monotonic_values), @@ -772,9 +784,12 @@ def test_post_tool_watchdog_uses_monotonic_clock(self): notification_poll_timeout=0.0, post_tool_quiet_timeout=0.15, ) - assert r.interrupted is True - assert r.should_retire is True - assert r.error and "silent" in r.error + assert r.interrupted is False + assert r.should_retire is False + assert r.error is None + assert r.tool_iterations == 1 + assert not any(method == "turn/interrupt" for method, _ in client.requests) + assert any("no events for" in rec.getMessage() for rec in caplog.records) def test_post_tool_watchdog_resets_on_further_activity(self): """A tool completion followed by an agent message should NOT trip From 54623b0dbef95aeb89a9bfe7c9c74e8d2ec72725 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:47:16 -0700 Subject: [PATCH 2/2] fix(codex): drop the stale 'watchdog tripped' retirement cause and re-scope the reset test The post-tool quiet timer no longer retires the session (it only warns), so the codex_runtime retirement comment listed a cause that cannot happen and the retained test's name/docstring still promised a watchdog that cannot trip. Comment and test wording now describe the warning semantics; assertions unchanged. --- agent/codex_runtime.py | 3 ++- .../agent/transports/test_codex_app_server_session.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index ca3092ed6b99d..28254a50f3e91 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -490,7 +490,8 @@ def run_codex_app_server_turn(agent, *, user_message: str, original_user_message final_response=f"Codex app-server turn failed: {exc}. Fall back to default runtime with `/codex-runtime auto`.", ) interrupt = _consume_user_interrupt(agent, turn.interrupted) - # Wedged client (deadline blown, watchdog tripped, OAuth refresh died, subprocess exited): retire it. + # Wedged client (turn deadline blown, OAuth refresh died, subprocess exited): retire it. Post-tool + # silence alone no longer retires — it only logs a warning (#112928). if getattr(turn, "should_retire", False): logger.warning("codex app-server session retired (turn error: %s)", turn.error) _close_codex_session(agent) diff --git a/tests/agent/transports/test_codex_app_server_session.py b/tests/agent/transports/test_codex_app_server_session.py index da1e318f17749..65b2d1fe190ee 100644 --- a/tests/agent/transports/test_codex_app_server_session.py +++ b/tests/agent/transports/test_codex_app_server_session.py @@ -791,9 +791,9 @@ def test_post_tool_silence_warns_but_does_not_retire_a_healthy_turn(self, caplog assert not any(method == "turn/interrupt" for method, _ in client.requests) assert any("no events for" in rec.getMessage() for rec in caplog.records) - def test_post_tool_watchdog_resets_on_further_activity(self): - """A tool completion followed by an agent message should NOT trip - the watchdog — further activity = codex still alive.""" + def test_post_tool_activity_clears_the_quiet_timer_and_never_retires(self): + """A tool completion followed by an agent message completes normally: further activity clears + the post-tool quiet timer, and even when it expires it only warns, never retires.""" client = FakeClient() client.queue_notification( "item/completed", @@ -805,7 +805,7 @@ def test_post_tool_watchdog_resets_on_further_activity(self): }, threadId="t", turnId="tu1", ) - # Non-tool activity immediately after — resets watchdog. + # Non-tool activity immediately after — clears the quiet timer. client.queue_notification( "item/completed", item={"type": "agentMessage", "id": "m1", "text": "tool finished"}, @@ -821,7 +821,7 @@ def test_post_tool_watchdog_resets_on_further_activity(self): notification_poll_timeout=0.01, post_tool_quiet_timeout=0.05, ) - # Tool ran, then text reset the watchdog, then turn/completed. + # Tool ran, then text cleared the quiet timer, then turn/completed. # Should NOT be a retirement case. assert r.tool_iterations == 1 assert r.final_text == "tool finished"