fix(gateway): route the post-update restart handoff through a job-surviving owner

hermes_cli/_subprocess_compat.py

@@ -24,6 +24,7 @@
     "suppress_platform_ver_console",
     "windows_detach_flags",
     "windows_detach_flags_without_breakaway",
+    "process_is_in_job",
     "windows_hide_flags",
     "windows_detach_popen_kwargs",
     "bounded_git_probe",
@@ -208,6 +209,26 @@ def windows_detach_flags_without_breakaway() -> int:
     return _CREATE_NEW_PROCESS_GROUP | _CREATE_NO_WINDOW
 
 
+def process_is_in_job() -> bool:
+    """Whether this process belongs to a Windows Job Object; false on other platforms."""
+    if not IS_WINDOWS:
+        return False
+
+    import ctypes
+    from ctypes import wintypes
+
+    kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+    get_current_process = kernel32.GetCurrentProcess
+    get_current_process.restype = wintypes.HANDLE
+    is_process_in_job = kernel32.IsProcessInJob
+    is_process_in_job.argtypes = [wintypes.HANDLE, wintypes.HANDLE, ctypes.POINTER(wintypes.BOOL)]
+    is_process_in_job.restype = wintypes.BOOL
+    in_job = wintypes.BOOL()
+    if not is_process_in_job(get_current_process(), None, ctypes.byref(in_job)):
+        raise ctypes.WinError(ctypes.get_last_error())
+    return bool(in_job.value)
+
+
 def windows_hide_flags() -> int:
     """Win32 creationflags hiding the child's console without detaching it; 0 elsewhere.
 

hermes_cli/gateway.py

@@ -1122,31 +1122,33 @@ def _spawn_gateway_restart_watcher(old_pid: int, run_argv: list[str], *, host: b
         except Exception:
             pass
 
-        # Platform-appropriate detach for the respawned gateway: POSIX start_new_session (setsid);
-        # Windows needs explicit creationflags. CREATE_BREAKAWAY_FROM_JOB is critical: the watcher may
-        # itself sit inside a job object (Electron/Tauri parent) and without breakaway the respawned
-        # gateway dies when that job tears down. See _subprocess_compat.windows_detach_flags().
+        # A breakaway flag can be accepted while a nested Job Object still owns this watcher.
+        # Delegate from any job to Task Scheduler, whose service outlives the updater's job.
         _popen_kwargs = {{"stdout": _stdio_target, "stderr": _stdio_target}}
-        # Anchor at the stable working dir and overlay the env (VIRTUAL_ENV / PYTHONPATH /
-        # HERMES_HOME) the windowless base interpreter needs to import hermes_cli. Empty on POSIX.
         if _respawn_cwd:
             _popen_kwargs["cwd"] = _respawn_cwd
         _base_env = {{**os.environ, **_respawn_env_overlay}}
         try:
             if sys.platform == "win32":
-                try:
-                    _popen_kwargs["creationflags"] = windows_detach_flags()
-                    # Stamp the breakaway state exactly like gateway_windows._spawn_detached so the
-                    # respawned gateway's exit-diag / lifecycle records show whether it escaped the
-                    # parent Job Object (a job-teardown kill is otherwise indistinguishable).
-                    _popen_kwargs["env"] = {{**_base_env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: "1"}}
-                    subprocess.Popen(cmd, **_popen_kwargs)
-                except OSError:
-                    # CREATE_BREAKAWAY_FROM_JOB is rejected with ERROR_ACCESS_DENIED when the parent's
-                    # job object refuses breakaway; retry without it (mirrors _spawn_detached).
-                    _popen_kwargs["creationflags"] = windows_detach_flags_without_breakaway()
-                    _popen_kwargs["env"] = {{**_base_env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: "0"}}
-                    subprocess.Popen(cmd, **_popen_kwargs)
+                from hermes_cli._subprocess_compat import process_is_in_job
+                if process_is_in_job():
+                    from hermes_cli.gateway_windows import _spawn_gateway_via_transient_task
+                    try:
+                        _spawn_gateway_via_transient_task(cmd, _respawn_cwd, _respawn_env_overlay)
+                    except Exception as _error:
+                        if _stdio_fh is not None:
+                            _stdio_fh.write(("[watcher] Job Object handoff failed: " + str(_error) + "\\n").encode("utf-8", "replace"))
+                        raise
+                else:
+                    try:
+                        _popen_kwargs["creationflags"] = windows_detach_flags()
+                        _popen_kwargs["env"] = {{**_base_env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: "1"}}
+                        subprocess.Popen(cmd, **_popen_kwargs)
+                    except OSError:
+                        # With no enclosing job, retrying without breakaway cannot be reaped by it.
+                        _popen_kwargs["creationflags"] = windows_detach_flags_without_breakaway()
+                        _popen_kwargs["env"] = {{**_base_env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: "0"}}
+                        subprocess.Popen(cmd, **_popen_kwargs)
             else:
                 if _respawn_env_overlay:
                     _popen_kwargs["env"] = _base_env

hermes_cli/gateway_windows.py

@@ -17,6 +17,7 @@
 import shutil
 import subprocess
 import sys
+import tempfile
 import time
 import uuid
 from datetime import datetime, timezone
@@ -180,6 +181,15 @@ def _quote_vbs_string(value: str) -> str:
     return '"' + value.replace('"', '""') + '"'
 
 
+def _quote_vbs_command_line(value: str) -> str:
+    """VBScript expression preserving command-line newlines as ``vbCrLf``/``vbLf`` values."""
+    parts = re.split(r"(\r\n|\r|\n)", value)
+    return " & ".join(
+        {"\r\n": "vbCrLf", "\r": "vbCr", "\n": "vbLf"}.get(part, _quote_vbs_string(part))
+        for part in parts
+    )
+
+
 # ── schtasks.exe wrapper
 
 def _exec_schtasks(args: list[str]) -> tuple[int, str, str]:
@@ -595,6 +605,115 @@ def _install_scheduled_task(task_name: str, script_path: Path) -> tuple[bool, st
     return (False, f"schtasks /Create failed (code {last_code}): {last_err.strip()}")
 
 
+def _build_update_handoff_vbs(
+    argv: list[str], working_dir: str, env_overlay: dict[str, str], started_path: Path,
+) -> str:
+    """Render a one-shot WScript launcher for an exact post-update gateway argv."""
+    quote = _quote_vbs_string
+    lines = [
+        "Option Explicit",
+        "Dim sh, env, fso, marker",
+        'Set sh = CreateObject("WScript.Shell")',
+        'Set env = sh.Environment("PROCESS")',
+        *[f"env.Item({quote(key)}) = {quote(value)}" for key, value in env_overlay.items()],
+        # Task Scheduler, not the gateway process, owns this handoff, so no Popen breakaway stamp applies.
+        f"env.Item({quote(_WINDOWS_GATEWAY_BREAKAWAY_ENV)}) = {quote('')}",
+    ]
+    if working_dir:
+        lines.append(f"sh.CurrentDirectory = {quote(working_dir)}")
+    lines.extend([
+        f"sh.Run {quote(subprocess.list2cmdline(argv))}, 0, False",
+        'Set fso = CreateObject("Scripting.FileSystemObject")',
+        f"Set marker = fso.CreateTextFile({quote(str(started_path))}, True)",
+        'marker.WriteLine "started"',
+        "marker.Close",
+    ])
+    return "\r\n".join(lines) + "\r\n"
+
+
+def _build_update_handoff_task_xml(launcher_path: Path, user: str | None) -> str:
+    """An on-demand-only task: no logon trigger or persistent autostart side effect."""
+    user_id = f"<UserId>{escape(user)}</UserId>" if user else ""
+    arguments = escape(f'//B //Nologo "{launcher_path}"')
+    return f'''<?xml version="1.0" encoding="UTF-16"?>
+<Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
+  <RegistrationInfo><Description>Hermes gateway post-update handoff</Description></RegistrationInfo>
+  <Triggers />
+  <Principals><Principal id="Author">{user_id}
+    <LogonType>InteractiveToken</LogonType><RunLevel>LeastPrivilege</RunLevel>
+  </Principal></Principals>
+  <Settings><AllowStartOnDemand>true</AllowStartOnDemand><Enabled>true</Enabled><Hidden>true</Hidden>
+    <ExecutionTimeLimit>PT0S</ExecutionTimeLimit>
+  </Settings>
+  <Actions Context="Author"><Exec><Command>wscript.exe</Command><Arguments>{arguments}</Arguments></Exec></Actions>
+</Task>
+'''
+
+
+def _spawn_gateway_via_transient_task(
+    argv: list[str], working_dir: str, env_overlay: dict[str, str], *, timeout_s: float = 30.0,
+) -> None:
+    """Hand a gateway start to Task Scheduler when this watcher belongs to a Job Object.
+
+    Task Scheduler starts the launcher outside the updater's Job Object. The temporary task has no
+    trigger, uses the exact respawn argv, and is removed after WScript confirms the command was
+    launched. Environment values are limited to the non-secret runtime overlay; credentials remain
+    profile-scoped in HERMES_HOME rather than being written into a task or launcher file.
+    """
+    _assert_windows()
+    if not argv:
+        raise ValueError("gateway handoff requires a command")
+
+    task_name = f"Hermes_Update_Handoff_{os.getpid()}_{uuid.uuid4().hex[:12]}"
+    task_dir = Path(tempfile.mkdtemp(prefix="hermes-gateway-handoff-"))
+    launcher_path = task_dir / "gateway.vbs"
+    xml_path = task_dir / "task.xml"
+    started_path = task_dir / "started.marker"
+    registered = False
+    try:
+        launcher_path.write_text(
+            _build_update_handoff_vbs(argv, working_dir, env_overlay, started_path),
+            encoding="utf-16",
+            newline="",
+        )
+        user = _resolve_task_user()
+        xml_path.write_text(
+            _build_update_handoff_task_xml(launcher_path, user), encoding="utf-16", newline=""
+        )
+        base = ["/Create", "/F", "/TN", task_name, "/XML", str(xml_path)]
+        variants = [[*base, "/RU", user, "/NP", "/IT"], base] if user else [base]
+        last_detail = ""
+        for args in variants:
+            code, out, err = _exec_schtasks(args)
+            if code == 0:
+                registered = True
+                break
+            last_detail = (err or out or f"exit code {code}").strip()
+        if not registered:
+            raise RuntimeError(f"could not register temporary gateway handoff task: {last_detail}")
+
+        code, out, err = _exec_schtasks(["/Run", "/TN", task_name])
+        if code != 0:
+            raise RuntimeError(f"could not run temporary gateway handoff task: {(err or out).strip()}")
+
+        deadline = time.monotonic() + timeout_s
+        while time.monotonic() < deadline:
+            if started_path.is_file():
+                return
+            time.sleep(0.05)
+        raise RuntimeError(f"temporary gateway handoff task did not launch within {timeout_s}s")
+    finally:
+        if registered:
+            code, out, err = _exec_schtasks(["/Delete", "/F", "/TN", task_name])
+            if code != 0:
+                logger.warning(
+                    "Could not remove temporary gateway handoff task %s: %s",
+                    task_name,
+                    (err or out).strip(),
+                )
+        shutil.rmtree(task_dir, ignore_errors=True)
+
+
 def _install_startup_entry(script_path: Path) -> Path:
     """Write the Startup-folder fallback launcher. Returns its path."""
     entry = get_startup_entry_path()

tests/hermes_cli/test_gateway_restart_job_handoff.py

@@ -0,0 +1,111 @@
+"""A post-update gateway respawn must outlive its updater's Windows Job Object."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import psutil
+import pytest
+
+pytestmark = pytest.mark.platforms("windows")
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def _alive(pid: int) -> bool:
+    try:
+        process = psutil.Process(pid)
+        return process.is_running() and process.status() != psutil.STATUS_ZOMBIE
+    except psutil.NoSuchProcess:
+        return False
+
+
+def _wait_for(predicate, timeout: float = 25.0) -> bool:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if predicate():
+            return True
+        time.sleep(0.1)
+    return bool(predicate())
+
+
+def test_restart_watcher_child_survives_updater_job_teardown(tmp_path, _isolate_hermes_home):
+    """Exercise the production watcher inside a real non-breakaway Job Object."""
+    from hermes_cli.local_runtime.processes import _WindowsJob
+
+    home = tmp_path / "hermes-home"
+    home.mkdir()
+    child_script = tmp_path / "respawned_gateway.py"
+    child_pid_file = tmp_path / "gateway.pid"
+    go_file = tmp_path / "go.marker"
+    child_script.write_text(
+        "import pathlib, sys, time\n"
+        "pathlib.Path(sys.argv[1]).write_text(str(__import__('os').getpid()), encoding='utf-8')\n"
+        "time.sleep(300)\n",
+        encoding="utf-8",
+    )
+    driver = (
+        "import pathlib, sys, time\n"
+        f"go = pathlib.Path({str(go_file)!r})\n"
+        "deadline = time.monotonic() + 30\n"
+        "while not go.exists():\n"
+        "    if time.monotonic() > deadline: raise SystemExit('assignment handshake timed out')\n"
+        "    time.sleep(0.05)\n"
+        "from hermes_cli._subprocess_compat import process_is_in_job\n"
+        "if not process_is_in_job(): raise SystemExit('driver was not assigned to the real Job Object')\n"
+        "from hermes_cli.gateway import _spawn_gateway_restart_watcher\n"
+        f"ok = _spawn_gateway_restart_watcher(2147483647, [sys.executable, {str(child_script)!r}, {str(child_pid_file)!r}], host=False)\n"
+        "if not ok: raise SystemExit('production restart watcher did not spawn')\n"
+        f"pid_file = pathlib.Path({str(child_pid_file)!r})\n"
+        "deadline = time.monotonic() + 30\n"
+        "while not pid_file.exists():\n"
+        "    if time.monotonic() > deadline: raise SystemExit('watcher did not start its child')\n"
+        "    time.sleep(0.05)\n"
+        "print(pid_file.read_text(encoding='utf-8'), flush=True)\n"
+    )
+    env = os.environ.copy()
+    env["HERMES_HOME"] = str(home)
+    process = subprocess.Popen(
+        [sys.executable, "-c", driver],
+        cwd=str(_REPO_ROOT),
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    job = _WindowsJob()
+    gateway_pid = None
+    try:
+        job.assign(process)
+        go_file.write_text("go", encoding="utf-8")
+        stdout, stderr = process.communicate(timeout=45)
+        assert process.returncode == 0, f"watcher driver failed: {stderr}\n{stdout}"
+        gateway_pid = int(stdout.strip())
+        assert _wait_for(lambda: _alive(gateway_pid)), "respawned gateway never became live"
+
+        # Closing the last real Job Object handle kills every contained process.
+        job.close()
+        job = None
+        if not _wait_for(lambda: _alive(gateway_pid), timeout=3):
+            pytest.fail(
+                "respawned gateway died when the updater's non-breakaway Job Object closed",
+                pytrace=False,
+            )
+    finally:
+        go_file.unlink(missing_ok=True)
+        if job is not None:
+            job.close()
+        if process.poll() is None:
+            process.kill()
+            process.wait(timeout=10)
+        if gateway_pid is not None and _alive(gateway_pid):
+            try:
+                process = psutil.Process(gateway_pid)
+                process.kill()
+                process.wait(timeout=10)
+            except psutil.NoSuchProcess:
+                pass

tests/home_io_guard.py

@@ -9,10 +9,15 @@
 import shutil
 import sqlite3
 import sys
+import sysconfig
 import threading
 
 _INTERPRETER_PREFIXES = tuple({
     Path(p).resolve() for p in (sys.prefix, sys.base_prefix, sys.exec_prefix, sys.base_exec_prefix)
+} | {
+    # Python may keep stdlib source under a stable ABI directory whose name omits the patch
+    # version, while sys.base_prefix points at the versioned runtime directory.
+    Path(sysconfig.get_path("stdlib")).resolve(),
 } | {
     # A PM-activated developer shell runs sys.prefix's python against a dependency generation
     # whose site-packages sits under the (real) Hermes home; third-party imports from it are the