From 47bad20f5a0f1775751b5d7e0ae5d082dc5cb4f1 Mon Sep 17 00:00:00 2001 From: webtecnica <75556242+webtecnica@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:10:25 -0300 Subject: [PATCH] fix(agent): prevent auto retry loop from slow LLM backend prefilling large contexts (#69424) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-pronged fix for the stale-stream detector killing connections before a slow local/cloud model finishes prompt prefill: 1. Apply context-size scaling to local endpoints too The local-endpoint stale-timeout branch (default 900s) skipped the context-token scaling that the cloud path applied, so a 900s flat ceiling could still fire before a 122B model finishes prefilling 140K+ tokens. Move scaling out of the branch so both local and cloud paths get proportional timeouts: - >200K tokens → 1800s (30 min) - >100K tokens → 1200s (20 min) - >50K tokens → 600s (10 min) 2. Add stale-streak backoff After 2+ consecutive stale kills, apply a progressive multiplier (1× → 2.5× → 4× … up to 10×) to the stale timeout so each retry waits longer, eventually outlasting the prefill and breaking the infinite retry loop. Resets on successful response. 3. Raise the non-streaming stale timeout tiers consistently The non-streaming path () and Bedrock path () now share the same increased floors for consistency. Closes #69424. --- agent/chat_completion_helpers.py | 77 +++++++++++++------- run_agent.py | 6 +- tests/agent/test_non_stream_stale_timeout.py | 8 +- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index b2e5c8653a48..36f90d0b77a0 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -282,10 +282,12 @@ def _derive_stream_stale_timeout(agent, api_kwargs: dict) -> float: else: _base = env_float("HERMES_STREAM_STALE_TIMEOUT", 180.0) _est_tokens = estimate_request_context_tokens(api_kwargs) - if _est_tokens > 100_000: - _timeout = max(_base, 300.0) + if _est_tokens > 200_000: + _timeout = max(_base, 1800.0) + elif _est_tokens > 100_000: + _timeout = max(_base, 1200.0) elif _est_tokens > 50_000: - _timeout = max(_base, 240.0) + _timeout = max(_base, 600.0) else: _timeout = _base from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor @@ -3588,29 +3590,52 @@ def _call(): agent.base_url, _stream_stale_timeout, ) else: - # Scale the stale timeout for large contexts: slow models (like Opus) - # can legitimately think for minutes before producing the first token - # when the context is large. Without this, the stale detector kills - # healthy connections during the model's thinking phase, producing - # spurious RemoteProtocolError ("peer closed connection"). - _est_tokens = estimate_request_context_tokens(api_kwargs) - if _est_tokens > 100_000: - _stream_stale_timeout = max(_stream_stale_timeout_base, 300.0) - elif _est_tokens > 50_000: - _stream_stale_timeout = max(_stream_stale_timeout_base, 240.0) - else: - _stream_stale_timeout = _stream_stale_timeout_base - # Reasoning-model floor: known reasoning models (Nemotron 3 Ultra, - # OpenAI o1/o3, Anthropic Opus 4.x thinking, DeepSeek R1, Qwen QwQ, - # xAI Grok reasoning, etc.) routinely exceed the default 180s chat- - # model threshold during their thinking phase. The cloud gateway - # upstream kills the socket first, surfacing as BrokenPipeError. - # Raises the floor only — never overrides explicit user config - # (handled by get_provider_stale_timeout above). - from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor - _reasoning_floor = get_reasoning_stale_timeout_floor(api_kwargs.get("model")) - if _reasoning_floor is not None: - _stream_stale_timeout = max(_stream_stale_timeout, _reasoning_floor) + _stream_stale_timeout = _stream_stale_timeout_base + + # ── Context-size scaling ──────────────────────────────────────────── + # Slow models (Opus, Qwen 3.5 122B, local 120B+ GGUF) can take many + # *minutes* of prefill/thinking before producing the first token when + # the context is large. Scale the stale timeout by estimated context + # size so the detector doesn't kill healthy connections during the + # model's thinking/prefill phase. Applied to both local and cloud paths + # (local defaults to 900s but gets a further bump for extreme contexts). + _est_tokens = estimate_request_context_tokens(api_kwargs) + if _est_tokens > 200_000: + _stream_stale_timeout = max(_stream_stale_timeout, 1800.0) + elif _est_tokens > 100_000: + _stream_stale_timeout = max(_stream_stale_timeout, 1200.0) + elif _est_tokens > 50_000: + _stream_stale_timeout = max(_stream_stale_timeout, 600.0) + elif _est_tokens > 10_000: + _stream_stale_timeout = max(_stream_stale_timeout, _stream_stale_timeout_base) + + # Reasoning-model floor: known reasoning models (Nemotron 3 Ultra, + # OpenAI o1/o3, Anthropic Opus 4.x thinking, DeepSeek R1, Qwen QwQ, + # xAI Grok reasoning, etc.) routinely exceed the default 180s chat- + # model threshold during their thinking phase. The cloud gateway + # upstream kills the socket first, surfacing as BrokenPipeError. + # Raises the floor only — never overrides explicit user config + # (handled by get_provider_stale_timeout above). + from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor + _reasoning_floor = get_reasoning_stale_timeout_floor(api_kwargs.get("model")) + if _reasoning_floor is not None: + _stream_stale_timeout = max(_stream_stale_timeout, _reasoning_floor) + + # ── Stale-streak backoff (#69424) ─────────────────────────────────── + # After consecutive stale kills in the same session, the retry loop + # restarts the same large-context request from scratch, hitting the + # same short timeout each time → infinite retry loop. Apply a + # progressive multiplier so each retry waits longer, eventually + # outlasting the prefill. Resets on success (see _reset_stale_streak). + _streak = _stale_streak(agent) + if _streak >= 2: + _multiplier = min(1.0 + (_streak - 1) * 1.5, 10.0) + _previous = _stream_stale_timeout + _stream_stale_timeout = _stream_stale_timeout * _multiplier + logger.info( + "Stale-streak %s — bumped stale timeout from %.0fs to %.0fs", + _streak, _previous, _stream_stale_timeout, + ) t = threading.Thread(target=_call, daemon=True) t.start() diff --git a/run_agent.py b/run_agent.py index 7a37b9f3554b..b229c01a6b09 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1331,10 +1331,12 @@ def _compute_non_stream_stale_timeout(self, api_payload: Any) -> float: from agent.chat_completion_helpers import estimate_request_context_tokens est_tokens = estimate_request_context_tokens(api_payload) + if est_tokens > 200_000: + return max(stale_base, 3600.0) if est_tokens > 100_000: - return max(stale_base, 240.0) + return max(stale_base, 1200.0) if est_tokens > 50_000: - return max(stale_base, 150.0) + return max(stale_base, 600.0) return stale_base def _codex_silent_hang_hint(self, model: Optional[str] = None) -> Optional[str]: diff --git a/tests/agent/test_non_stream_stale_timeout.py b/tests/agent/test_non_stream_stale_timeout.py index 25a74f31c306..41248250a4b4 100644 --- a/tests/agent/test_non_stream_stale_timeout.py +++ b/tests/agent/test_non_stream_stale_timeout.py @@ -135,12 +135,12 @@ def test_long_codex_request_bumps_to_50k_tier(monkeypatch, tmp_path): agent = _make_agent(tmp_path) payload = {"model": "gpt-5.5", "input": "x" * 240_000, "instructions": ""} timeout = agent._compute_non_stream_stale_timeout(payload) - assert timeout >= 150.0 - assert timeout < 240.0 + assert timeout >= 600.0 + assert timeout < 1200.0 def test_very_long_codex_request_bumps_to_100k_tier(monkeypatch, tmp_path): - """Codex payload > 100k tokens -> at least 240s.""" + """Codex payload > 100k tokens -> at least 1200s.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) (tmp_path / ".env").write_text("", encoding="utf-8") monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False) @@ -148,7 +148,7 @@ def test_very_long_codex_request_bumps_to_100k_tier(monkeypatch, tmp_path): agent = _make_agent(tmp_path) payload = {"model": "gpt-5.5", "input": "x" * 500_000, "instructions": ""} - assert agent._compute_non_stream_stale_timeout(payload) >= 240.0 + assert agent._compute_non_stream_stale_timeout(payload) >= 1200.0 def test_chat_completions_long_messages_bumps_tier(monkeypatch, tmp_path):