fix(runtime): report unverified Agent updates without automatic restart

api/agent_runtime.py

@@ -8,20 +8,35 @@
 
 from __future__ import annotations
 
+import errno
+import math
+import os
 from pathlib import Path
 import sys
 import subprocess
 import threading
+import time
 
 # Retain the discovered path as a diagnostic/test-visible compatibility value;
 # runtime identity is deliberately captured from the loaded module below.
-from api.config import _AGENT_DIR  # noqa: F401
+from api.config import (
+    PYTHON_EXE,
+    _AGENT_DIR,  # noqa: F401
+    _DEFAULT_STATE_HOME,
+)
 from api.subprocess_utils import windows_hide_flags
 
-_RESTART_MESSAGE = (
+_RESTART_REQUIRED_MESSAGE = (
     "Hermes Agent was updated while Hermes WebUI was running. "
-    "Restart Hermes WebUI before retrying this action."
+    "WebUI cannot verify that the Agent update completed safely. "
+    "Check the Agent update outcome and environment first. "
+    "Restart Hermes WebUI manually before retrying this action."
 )
+_AGENT_UPDATE_MARKER = ".hermes-update-in-progress"
+_AGENT_RECOVERY_MARKERS = (".update-incomplete", ".lazy-refresh-incomplete")
+_AGENT_UPDATE_MAX_AGE_SECONDS = 20 * 60
+_HERMES_HOME = Path(_DEFAULT_STATE_HOME)
+_AGENT_PYTHON = Path(PYTHON_EXE).expanduser() if PYTHON_EXE else None
 
 
 def _read_agent_revision(
@@ -100,6 +115,180 @@ def _read_agent_revision(
 class AgentRuntimeChangedError(RuntimeError):
     """Raised when the loaded Agent runtime no longer matches its source tree."""
 
+    def __init__(
+        self,
+        message: str,
+        *,
+        agent_update_state: str | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.agent_update_state = agent_update_state
+
+
+def agent_runtime_stale_payload(exc: AgentRuntimeChangedError) -> dict:
+    """Return the shared retry response for every stale-runtime entry point."""
+    payload = {
+        "error": str(exc),
+        "type": "agent_runtime_stale",
+        "retryable": True,
+        "restart_scheduled": False,
+    }
+    if exc.agent_update_state is not None:
+        payload["agent_update_state"] = exc.agent_update_state
+    return payload
+
+
+def _pid_is_alive(pid: int) -> bool | None:
+    """Return PID liveness, or ``None`` when the platform cannot confirm it."""
+    if pid <= 0:
+        return False
+    if pid.bit_length() > 32:
+        return None
+    if sys.platform == "win32":
+        try:
+            import ctypes
+            from ctypes import wintypes
+
+            kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+            kernel32.OpenProcess.argtypes = (
+                wintypes.DWORD,
+                wintypes.BOOL,
+                wintypes.DWORD,
+            )
+            kernel32.OpenProcess.restype = wintypes.HANDLE
+            kernel32.GetExitCodeProcess.argtypes = (
+                wintypes.HANDLE,
+                ctypes.POINTER(wintypes.DWORD),
+            )
+            kernel32.GetExitCodeProcess.restype = wintypes.BOOL
+            kernel32.CloseHandle.argtypes = (wintypes.HANDLE,)
+            kernel32.CloseHandle.restype = wintypes.BOOL
+            handle = kernel32.OpenProcess(0x1000, False, pid)
+            if not handle:
+                error = ctypes.get_last_error()
+                if error == 5:  # ERROR_ACCESS_DENIED still proves the PID exists.
+                    return True
+                if error == 87:  # ERROR_INVALID_PARAMETER for a missing PID.
+                    return False
+                return None
+            try:
+                exit_code = wintypes.DWORD()
+                if not kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)):
+                    return None
+                return exit_code.value == 259  # STILL_ACTIVE
+            finally:
+                kernel32.CloseHandle(handle)
+        except (AttributeError, OSError, TypeError, ValueError):
+            return None
+
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except (OverflowError, ValueError):
+        return None
+    except OSError as exc:
+        if exc.errno == errno.ESRCH:
+            return False
+        if exc.errno == errno.EPERM:
+            return True
+        return None
+    return True
+
+
+def _read_live_agent_update(marker: Path) -> str:
+    """Classify the shared Agent update marker without changing Agent state."""
+    try:
+        raw = marker.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        try:
+            marker.lstat()
+        except FileNotFoundError:
+            return "absent"
+        except OSError:
+            return "unknown"
+        return "unknown"
+    except (OSError, UnicodeError):
+        return "unknown"
+
+    lines = raw.splitlines()
+    try:
+        pid = int(lines[0].strip())
+        started_at = float(lines[1].strip())
+    except (IndexError, TypeError, ValueError):
+        return "unknown"
+    if pid <= 0 or not math.isfinite(started_at):
+        return "unknown"
+
+    age_seconds = time.time() - started_at
+    if age_seconds < 0:
+        return "unknown"
+    if age_seconds > _AGENT_UPDATE_MAX_AGE_SECONDS:
+        return "stale"
+    alive = _pid_is_alive(pid)
+    if alive is None:
+        return "unknown"
+    return "active" if alive else "stale"
+
+
+def _marker_presence(marker: Path) -> str:
+    """Return ``present``, ``absent``, or ``unknown`` for a recovery marker."""
+    try:
+        marker.lstat()
+    except FileNotFoundError:
+        return "absent"
+    except OSError:
+        return "unknown"
+    return "present"
+
+
+def _agent_install_roots() -> tuple[Path, ...]:
+    """Return portable roots that can own the Agent's venv recovery markers."""
+    candidates: list[Path] = []
+    if _AGENT_SOURCE_DIR is not None:
+        candidates.append(_AGENT_SOURCE_DIR)
+    if _AGENT_PYTHON is not None:
+        # A venv Python is commonly a symlink to a shared interpreter. Keep the
+        # configured venv path so its installation's recovery markers are read.
+        python_path = _AGENT_PYTHON
+        if python_path.parent.name.lower() in {"bin", "scripts"}:
+            venv_dir = python_path.parent.parent
+            if venv_dir.name.lower() in {"venv", ".venv"}:
+                candidates.append(venv_dir.parent)
+
+    roots: list[Path] = []
+    seen: set[str] = set()
+    for candidate in candidates:
+        key = os.path.normcase(os.path.abspath(str(candidate)))
+        if key not in seen:
+            seen.add(key)
+            roots.append(candidate)
+    return tuple(roots)
+
+
+def _agent_update_transaction_state() -> str:
+    """Report marker diagnostics, never proof of successful completion.
+
+    The Agent removes its active marker on failed/interrupted exits too. Neither
+    its absence nor a stale PID proves the checkout or environment is healthy.
+    """
+    live_state = _read_live_agent_update(_HERMES_HOME / _AGENT_UPDATE_MARKER)
+    if live_state == "unknown":
+        return "unknown"
+
+    recovery_present = False
+    for root in _agent_install_roots():
+        for marker_name in _AGENT_RECOVERY_MARKERS:
+            presence = _marker_presence(root / marker_name)
+            if presence == "unknown":
+                return "unknown"
+            recovery_present = recovery_present or presence == "present"
+    if recovery_present:
+        return "incomplete"
+    return "unverified" if live_state == "absent" else live_state
+
 
 def _loaded_agent_source_identity() -> tuple[Path, Path] | None:
     """Return the source directory and file that supplied ``run_agent``."""
@@ -140,7 +329,14 @@ def ensure_agent_runtime_current() -> None:
         _read_agent_revision(_AGENT_SOURCE_DIR, module_path=_AGENT_MODULE_PATH)
         != _AGENT_REVISION
     ):
-        raise AgentRuntimeChangedError(_RESTART_MESSAGE)
+        # Automatic restart needs an Agent-owned success receipt bound to this
+        # transaction, final revision and healthy environment, plus an atomic
+        # handoff excluding mutations across replacement. Marker polling and a
+        # final revision read supply neither contract. Keep this path manual.
+        raise AgentRuntimeChangedError(
+            _RESTART_REQUIRED_MESSAGE,
+            agent_update_state=_agent_update_transaction_state(),
+        )
 
 
 def require_ai_agent_class():

api/routes.py

@@ -36,6 +36,7 @@
 from urllib.request import HTTPRedirectHandler, HTTPSHandler, ProxyHandler, Request, build_opener
 from api.agent_runtime import (
     AgentRuntimeChangedError,
+    agent_runtime_stale_payload,
     ensure_agent_runtime_current,
     require_ai_agent_class,
 )
@@ -23017,11 +23018,7 @@ def _agent_runtime_barrier_response(
     try:
         ensure_agent_runtime_current()
     except AgentRuntimeChangedError as exc:
-        return {
-            "error": str(exc),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }
+        return agent_runtime_stale_payload(exc)
     return None
 
 
@@ -25132,11 +25129,7 @@ def _handle_git_commit_message(handler, body):
     except GitWorkspaceError as e:
         return _git_bad(handler, e)
     except AgentRuntimeChangedError as e:
-        return j(handler, {
-            "error": str(e),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }, status=409)
+        return j(handler, agent_runtime_stale_payload(e), status=409)
     except Exception as e:
         logger.exception("git commit message generation failed")
         return bad(handler, _sanitize_error(e), 500)
@@ -25169,11 +25162,7 @@ def _handle_git_commit_message_selected(handler, body):
     except GitWorkspaceError as e:
         return _git_bad(handler, e)
     except AgentRuntimeChangedError as e:
-        return j(handler, {
-            "error": str(e),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }, status=409)
+        return j(handler, agent_runtime_stale_payload(e), status=409)
     except Exception as e:
         logger.exception("selected git commit message generation failed")
         return bad(handler, _sanitize_error(e), 500)
@@ -26730,6 +26719,10 @@ def _manual_compression_status_payload(job):
             payload["type"] = job["error_type"]
         if job.get("retryable") is not None:
             payload["retryable"] = bool(job["retryable"])
+        if job.get("restart_scheduled") is not None:
+            payload["restart_scheduled"] = bool(job["restart_scheduled"])
+        if job.get("agent_update_state") is not None:
+            payload["agent_update_state"] = job["agent_update_state"]
     elif status == "cancelled":
         payload["ok"] = False
         payload["error"] = job.get("error") or "Compression cancelled"
@@ -26766,6 +26759,8 @@ def _run_manual_compression_job(sid, body):
                         "error_status": status,
                         "error_type": (payload or {}).get("type"),
                         "retryable": (payload or {}).get("retryable"),
+                        "restart_scheduled": (payload or {}).get("restart_scheduled"),
+                        "agent_update_state": (payload or {}).get("agent_update_state"),
                         "updated_at": now,
                     }
                 )
@@ -26779,16 +26774,19 @@ def _run_manual_compression_job(sid, body):
                 )
     except AgentRuntimeChangedError as exc:
         logger.warning("Manual compression worker found stale Agent runtime for session %s", sid)
+        stale_payload = agent_runtime_stale_payload(exc)
         with _MANUAL_COMPRESSION_JOBS_LOCK:
             job = _MANUAL_COMPRESSION_JOBS.get(sid)
             if job:
                 job.update(
                     {
                         "status": "error",
-                        "error": str(exc),
+                        "error": stale_payload["error"],
                         "error_status": 409,
-                        "error_type": "agent_runtime_stale",
-                        "retryable": True,
+                        "error_type": stale_payload["type"],
+                        "retryable": stale_payload["retryable"],
+                        "restart_scheduled": stale_payload.get("restart_scheduled"),
+                        "agent_update_state": stale_payload.get("agent_update_state"),
                         "updated_at": time.time(),
                     }
                 )
@@ -26846,11 +26844,7 @@ def _handle_session_compress_start(handler, body):
     except AgentRuntimeChangedError as exc:
         return j(
             handler,
-            {
-                "error": str(exc),
-                "type": "agent_runtime_stale",
-                "retryable": True,
-            },
+            agent_runtime_stale_payload(exc),
             status=409,
         )
 
@@ -27214,11 +27208,7 @@ def _summarize_manual_compression(
             },
         )
     except AgentRuntimeChangedError as e:
-        return j(handler, {
-            "error": str(e),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }, status=409)
+        return j(handler, agent_runtime_stale_payload(e), status=409)
     except Exception as e:
         logger.warning("Manual session compression failed: %s", e)
         return bad(handler, f"Compression failed: {_sanitize_error(e)}")
@@ -27893,11 +27883,7 @@ def _agent_text_completion(agent, system_prompt, user_text, max_tokens=700):
             "fallback": fallback,
         })
     except AgentRuntimeChangedError as e:
-        return j(handler, {
-            "error": str(e),
-            "type": "agent_runtime_stale",
-            "retryable": True,
-        }, status=409)
+        return j(handler, agent_runtime_stale_payload(e), status=409)
     except api_config.AmbiguousCustomProviderError as e:
         # A custom-provider slug collision is a user-fixable misconfiguration,
         # not a transient summary failure. Return 400 with the actionable rename