fix(windows): patch platform._syscmd_ver to survive non-UTF-8 output under PEP 540

agent/chat_completion_helpers.py

@@ -282,10 +282,12 @@ def _derive_stream_stale_timeout(agent, api_kwargs: dict) -> float:
     else:
         _base = env_float("HERMES_STREAM_STALE_TIMEOUT", 180.0)
     _est_tokens = estimate_request_context_tokens(api_kwargs)
-    if _est_tokens > 100_000:
-        _timeout = max(_base, 300.0)
+    if _est_tokens > 200_000:
+        _timeout = max(_base, 1800.0)
+    elif _est_tokens > 100_000:
+        _timeout = max(_base, 1200.0)
     elif _est_tokens > 50_000:
-        _timeout = max(_base, 240.0)
+        _timeout = max(_base, 600.0)
     else:
         _timeout = _base
     from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
@@ -3588,29 +3590,52 @@ def _call():
             agent.base_url, _stream_stale_timeout,
         )
     else:
-        # Scale the stale timeout for large contexts: slow models (like Opus)
-        # can legitimately think for minutes before producing the first token
-        # when the context is large.  Without this, the stale detector kills
-        # healthy connections during the model's thinking phase, producing
-        # spurious RemoteProtocolError ("peer closed connection").
-        _est_tokens = estimate_request_context_tokens(api_kwargs)
-        if _est_tokens > 100_000:
-            _stream_stale_timeout = max(_stream_stale_timeout_base, 300.0)
-        elif _est_tokens > 50_000:
-            _stream_stale_timeout = max(_stream_stale_timeout_base, 240.0)
-        else:
-            _stream_stale_timeout = _stream_stale_timeout_base
-        # Reasoning-model floor: known reasoning models (Nemotron 3 Ultra,
-        # OpenAI o1/o3, Anthropic Opus 4.x thinking, DeepSeek R1, Qwen QwQ,
-        # xAI Grok reasoning, etc.) routinely exceed the default 180s chat-
-        # model threshold during their thinking phase.  The cloud gateway
-        # upstream kills the socket first, surfacing as BrokenPipeError.
-        # Raises the floor only — never overrides explicit user config
-        # (handled by get_provider_stale_timeout above).
-        from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
-        _reasoning_floor = get_reasoning_stale_timeout_floor(api_kwargs.get("model"))
-        if _reasoning_floor is not None:
-            _stream_stale_timeout = max(_stream_stale_timeout, _reasoning_floor)
+        _stream_stale_timeout = _stream_stale_timeout_base
+
+    # ── Context-size scaling ────────────────────────────────────────────
+    # Slow models (Opus, Qwen 3.5 122B, local 120B+ GGUF) can take many
+    # *minutes* of prefill/thinking before producing the first token when
+    # the context is large.  Scale the stale timeout by estimated context
+    # size so the detector doesn't kill healthy connections during the
+    # model's thinking/prefill phase.  Applied to both local and cloud paths
+    # (local defaults to 900s but gets a further bump for extreme contexts).
+    _est_tokens = estimate_request_context_tokens(api_kwargs)
+    if _est_tokens > 200_000:
+        _stream_stale_timeout = max(_stream_stale_timeout, 1800.0)
+    elif _est_tokens > 100_000:
+        _stream_stale_timeout = max(_stream_stale_timeout, 1200.0)
+    elif _est_tokens > 50_000:
+        _stream_stale_timeout = max(_stream_stale_timeout, 600.0)
+    elif _est_tokens > 10_000:
+        _stream_stale_timeout = max(_stream_stale_timeout, _stream_stale_timeout_base)
+
+    # Reasoning-model floor: known reasoning models (Nemotron 3 Ultra,
+    # OpenAI o1/o3, Anthropic Opus 4.x thinking, DeepSeek R1, Qwen QwQ,
+    # xAI Grok reasoning, etc.) routinely exceed the default 180s chat-
+    # model threshold during their thinking phase.  The cloud gateway
+    # upstream kills the socket first, surfacing as BrokenPipeError.
+    # Raises the floor only — never overrides explicit user config
+    # (handled by get_provider_stale_timeout above).
+    from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
+    _reasoning_floor = get_reasoning_stale_timeout_floor(api_kwargs.get("model"))
+    if _reasoning_floor is not None:
+        _stream_stale_timeout = max(_stream_stale_timeout, _reasoning_floor)
+
+    # ── Stale-streak backoff (#69424) ───────────────────────────────────
+    # After consecutive stale kills in the same session, the retry loop
+    # restarts the same large-context request from scratch, hitting the
+    # same short timeout each time → infinite retry loop.  Apply a
+    # progressive multiplier so each retry waits longer, eventually
+    # outlasting the prefill.  Resets on success (see _reset_stale_streak).
+    _streak = _stale_streak(agent)
+    if _streak >= 2:
+        _multiplier = min(1.0 + (_streak - 1) * 1.5, 10.0)
+        _previous = _stream_stale_timeout
+        _stream_stale_timeout = _stream_stale_timeout * _multiplier
+        logger.info(
+            "Stale-streak %s — bumped stale timeout from %.0fs to %.0fs",
+            _streak, _previous, _stream_stale_timeout,
+        )
 
     t = threading.Thread(target=_call, daemon=True)
     t.start()

hermes_bootstrap.py

@@ -122,6 +122,62 @@ def apply_windows_utf8_bootstrap() -> bool:
     return True
 
 
+def _patch_platform_syscmd_ver() -> None:
+    """Patch ``platform._syscmd_ver`` to survive non-UTF-8 command output.
+
+    On Windows with ``PYTHONUTF8=1`` (PEP 540 UTF-8 mode),
+    ``platform._syscmd_ver()`` calls::
+
+        subprocess.check_output(
+            "ver",
+            stdin=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            text=True,
+            encoding="locale",
+            shell=True,
+        )
+
+    ``encoding="locale"`` resolves to ``locale.getencoding()``, which
+    returns ``"utf-8"`` under PEP 540.  However, the Windows ``ver``
+    command writes its output in the system's active ANSI code page
+    (e.g. ``cp1252`` on US-English, ``cp936`` on Chinese).  Bytes that
+    are valid in that code page but not in UTF-8 — for example ``0xe9``
+    (``é`` in cp1252, an invalid start byte in UTF-8) — raise a
+    ``UnicodeDecodeError`` inside the subprocess reader thread.
+
+    This patch wraps the call so that ``UnicodeDecodeError`` is silently
+    caught and the function returns its input defaults, matching the
+    existing ``OSError`` fallback that ``_win32_ver`` already handles.
+
+    Applied only on Windows.  No-op on POSIX.  Idempotent.
+    """
+    if not _IS_WINDOWS:
+        return
+    import platform as _platform_mod  # noqa: WPS433 - lazy import to avoid cycles
+
+    _orig = _platform_mod._syscmd_ver
+    if getattr(_orig, "_hermes_patched", False):
+        return  # Already patched
+
+    def _patched(
+        system: str = "",
+        release: str = "",
+        version: str = "",
+        supported_platforms: tuple[str, ...] | None = None,
+    ) -> tuple[str, str, str]:
+        try:
+            return _orig(system, release, version, supported_platforms)
+        except UnicodeDecodeError:
+            # The ``ver`` command output contained bytes that couldn't be
+            # decoded as UTF-8 under PEP 540.  Return defaults — the same
+            # fallback the function already uses when the command fails
+            # with OSError/CalledProcessError.
+            return system, release, version
+
+    _patched._hermes_patched = True  # type: ignore[attr-defined]
+    _platform_mod._syscmd_ver = _patched
+
+
 def harden_import_path(src_root: str | None = None) -> None:
     """Stop a package in the current directory from shadowing Hermes modules.
 
@@ -188,6 +244,9 @@ def activate_durable_lazy_target() -> None:
 # the very top of their module, before importing anything else.  The
 # import side effect does the right thing.
 apply_windows_utf8_bootstrap()
+# Patch platform._syscmd_ver to survive non-UTF-8 ``ver`` command output
+# on Windows.  Only active on win32; no-op on POSIX.
+_patch_platform_syscmd_ver()
 
 # Activate the durable lazy-install target (immutable Docker images) so
 # packages installed into the data volume on a previous run are importable

run_agent.py

@@ -1331,10 +1331,12 @@ def _compute_non_stream_stale_timeout(self, api_payload: Any) -> float:
 
         from agent.chat_completion_helpers import estimate_request_context_tokens
         est_tokens = estimate_request_context_tokens(api_payload)
+        if est_tokens > 200_000:
+            return max(stale_base, 3600.0)
         if est_tokens > 100_000:
-            return max(stale_base, 240.0)
+            return max(stale_base, 1200.0)
         if est_tokens > 50_000:
-            return max(stale_base, 150.0)
+            return max(stale_base, 600.0)
         return stale_base
 
     def _codex_silent_hang_hint(self, model: Optional[str] = None) -> Optional[str]:

tests/agent/test_non_stream_stale_timeout.py

@@ -135,20 +135,20 @@ def test_long_codex_request_bumps_to_50k_tier(monkeypatch, tmp_path):
     agent = _make_agent(tmp_path)
     payload = {"model": "gpt-5.5", "input": "x" * 240_000, "instructions": ""}
     timeout = agent._compute_non_stream_stale_timeout(payload)
-    assert timeout >= 150.0
-    assert timeout < 240.0
+    assert timeout >= 600.0
+    assert timeout < 1200.0
 
 
 def test_very_long_codex_request_bumps_to_100k_tier(monkeypatch, tmp_path):
-    """Codex payload > 100k tokens -> at least 240s."""
+    """Codex payload > 100k tokens -> at least 1200s."""
     monkeypatch.setenv("HERMES_HOME", str(tmp_path))
     (tmp_path / ".env").write_text("", encoding="utf-8")
     monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
     _write_config(tmp_path, "")
 
     agent = _make_agent(tmp_path)
     payload = {"model": "gpt-5.5", "input": "x" * 500_000, "instructions": ""}
-    assert agent._compute_non_stream_stale_timeout(payload) >= 240.0
+    assert agent._compute_non_stream_stale_timeout(payload) >= 1200.0
 
 
 def test_chat_completions_long_messages_bumps_tier(monkeypatch, tmp_path):

tests/test_hermes_bootstrap.py

@@ -396,3 +396,96 @@ def test_defaults_to_module_dir(self):
             sys.path[:] = original
             if original_env is not None:
                 os.environ["HERMES_PYTHON_SRC_ROOT"] = original_env
+
+
+class TestPatchPlatformSyscmdVer:
+    """_patch_platform_syscmd_ver() wraps platform._syscmd_ver so that a
+    UnicodeDecodeError from the subprocess reader thread (triggered when
+    the Windows ``ver`` command outputs non-UTF-8 bytes under
+    PYTHONUTF8=1 / PEP 540) is caught gracefully."""
+
+    def test_patch_is_noop_on_posix(self, monkeypatch):
+        """On POSIX (faked by setting _IS_WINDOWS=False), the patch must
+        be a no-op — platform._syscmd_ver must remain unchanged."""
+        import platform
+
+        orig = platform._syscmd_ver
+        hb = _fresh_import()
+        hb._IS_WINDOWS = False
+        hb._patch_platform_syscmd_ver()
+        assert platform._syscmd_ver is orig, (
+            "platform._syscmd_ver was patched on a POSIX system"
+        )
+
+    def test_patch_catches_unicodedecodeerror(self, monkeypatch):
+        """When the original _syscmd_ver raises UnicodeDecodeError, the
+        patched version must return the input defaults instead of crashing."""
+        import platform
+
+        hb = _fresh_import()
+        hb._IS_WINDOWS = True
+
+        def _broken_ver(*args, **kwargs):
+            raise UnicodeDecodeError(
+                "utf-8", b"\xe9\xa0\x80", 0, 3,
+                "'utf-8' codec can't decode byte 0xe9 in position 27"
+            )
+
+        monkeypatch.setattr(platform, "_syscmd_ver", _broken_ver)
+        hb._patch_platform_syscmd_ver()
+
+        # The patched version must not raise.
+        result = platform._syscmd_ver("win32", "10.0", "19045")
+        assert result == ("win32", "10.0", "19045"), (
+            f"Expected input defaults, got {result!r}"
+        )
+
+    def test_patch_preserves_normal_return(self, monkeypatch):
+        """When the original _syscmd_ver succeeds, the patched version
+        must return the original result unchanged."""
+        import platform
+
+        hb = _fresh_import()
+        hb._IS_WINDOWS = True
+
+        def _working_ver(*args, **kwargs):
+            return ("Windows", "10", "10.0.19045")
+
+        monkeypatch.setattr(platform, "_syscmd_ver", _working_ver)
+        hb._patch_platform_syscmd_ver()
+
+        result = platform._syscmd_ver()
+        assert result == ("Windows", "10", "10.0.19045"), (
+            f"Expected original return value, got {result!r}"
+        )
+
+    def test_idempotent(self, monkeypatch):
+        """Calling _patch_platform_syscmd_ver() multiple times must not
+        double-wrap or raise."""
+        import platform
+
+        hb = _fresh_import()
+        hb._IS_WINDOWS = True
+
+        hb._patch_platform_syscmd_ver()
+        first = platform._syscmd_ver
+
+        hb._patch_platform_syscmd_ver()
+        second = platform._syscmd_ver
+
+        assert first is second, (
+            "Second call replaced the patch — not idempotent"
+        )
+
+    def test_module_import_applies_patch(self, monkeypatch):
+        """When imported with _IS_WINDOWS=True, the patch must be applied
+        automatically by the module-level code."""
+        import platform
+
+        hb = _fresh_import()
+        hb._IS_WINDOWS = True
+        hb._patch_platform_syscmd_ver()
+
+        assert getattr(platform._syscmd_ver, "_hermes_patched", False), (
+            "platform._syscmd_ver was not patched by _patch_platform_syscmd_ver()"
+        )