From c1fb7471832c84a8d01d30e14963556ed90bc48e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 10:05:17 -0700 Subject: [PATCH 01/63] fix(update): make the Windows gateway pause durable and never resume onto a torn tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The set of gateways/services `hermes update` stops on Windows lived only in the updater's memory plus an atexit resume, so a killed updater (console closed, taskkill, power loss) left them stopped with nothing on disk saying so, and the atexit/foreground resume could restart them onto a half-written checkout or the pre-sync dependencies. - hermes_cli/update_pause_record.py: the paused set is written under the root Hermes home BEFORE the first stop, owned by the updater (pid + creation time in the update-marker ct format), mirrored after every resume attempt and removed (compare-and-delete) only once the resume is done. - Every in-process resume is gated on a whole tree: no interrupted-pull marker, no tracked change beyond those present at pause time, and HEAD still at the pre-update commit or dependencies current for the moved HEAD. Otherwise the record stays for recovery. - Start-of-run recovery (hermes_cli/main.py, beside the fleet-restart hint) claims an orphaned record (dead owner, no live update) by rename and resumes it; a later `hermes update` adopts an orphaned record into its own pause instead. - update_lock: a claim is never adopted across a Hermes gateway/serve/dashboard host in the process ancestry, so an update started from a gateway the first update relaunched cannot run under that update's claim (cli §7 V9). (cherry picked from commit fc0a9c3a53247a8779b5bd9f40b678a22a173521) (cherry picked from commit 1c0292b5763ad82678309561da7a1eeec3fa0581) --- hermes_cli/main.py | 2 + hermes_cli/update_cmd_windows.py | 77 +++++++-- hermes_cli/update_lock.py | 37 +++- hermes_cli/update_pause_record.py | 279 ++++++++++++++++++++++++++++++ 4 files changed, 384 insertions(+), 11 deletions(-) create mode 100644 hermes_cli/update_pause_record.py diff --git a/hermes_cli/main.py b/hermes_cli/main.py index f100a2a2150fa..b9e14dde54f0e 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -3562,6 +3562,8 @@ def main(): _warn_pending_fleet_restart_on_startup() except Exception: pass + from hermes_cli.update_pause_record import recover as _recover_paused_gateways + _recover_paused_gateways() # a killed `hermes update` left gateways paused (Windows) if _first_positional_argv() != "update": from hermes_cli.boot_bootstrap import maybe_run_boot_bootstrap diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 21b490525a1da..bcef081c67c06 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -920,6 +920,10 @@ def _pause_windows_gateways_for_update() -> dict | None: with _abort_on_error("Could not prepare Windows gateway pause for update"): from gateway.status import get_process_start_time, terminate_pid from hermes_cli.gateway import _capture_gateway_argv + from hermes_cli import update_pause_record as pause_record + # A previous update killed mid-pause left its set stopped: bring it back with this run's resume. + orphan = pause_record.orphaned_record() + adopted = pause_record.drop_never_stopped(dict(orphan["token"])) if orphan else None profile_processes, service_gateways, service_gateway_pids, running_pids = _discover_windows_gateways() if not running_pids: token = _windows_cold_start_plan() @@ -927,22 +931,44 @@ def _pause_windows_gateways_for_update() -> dict | None: # (clean exit, autostart not installed): give them a token to ride on. probe = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []} _record_attested_cold_start_profiles(probe, set()) - return probe if probe.get("cold_start_profiles") else token - profiles, mapped_pids, socket_acks = _request_socket_pauses(running_pids, profile_processes, service_gateway_pids) - # Resolve venv-side launchers BEFORE draining: a dead worker's parent cannot be recovered (NoSuchProcess). - # The launcher keeps ``.pyd`` mapped and would trip the venv-holder guard; it is killed with the survivors. - launcher_pids = _m()._venv_launcher_ancestors(mapped_pids) - print("→ Stopping Windows gateway process(es) before updating Hermes...") - drain_timeout = _gateway_drain_timeout(socket_acks) - survivors = _m()._wait_for_windows_update_gateway_exit(mapped_pids, timeout=drain_timeout) + token = probe if probe.get("cold_start_profiles") else token + if adopted is None: + return pause_record.stamp_tree(token) if token is not None else None + token = pause_record.stamp_tree(pause_record.merge_into(token, adopted)) + with _abort_on_error("Could not record the adopted paused-gateway set"): + pause_record.write(token, owner=pause_record.identity()) + return token unmapped_pids = [pid for pid in running_pids if pid not in profile_processes and pid not in service_gateway_pids] - # Snapshot unmapped gateways' argv *before* force-killing so resume can replay it. + # Snapshot unmapped gateways' argv *before* anything stops so resume (or a later launch) can replay it. # Unmapped = no profile->PID-file mapping (e.g. Scheduled Task ``pythonw.exe -m ...``). unmapped = [ {"pid": int(pid), "argv": _try_call(lambda p=int(pid): _capture_gateway_argv(p), "Could not capture argv for unmapped gateway %s: %s", int(pid))} for pid in unmapped_pids ] + # Durable BEFORE the first stop: a killed updater must leave a record the next launch can resume. + intended = { + "resume_needed": True, "unmapped_pids": unmapped_pids, "unmapped": unmapped, + "profiles": {str(profile_processes[pid].profile): int(pid) for pid in running_pids + if pid in profile_processes and pid not in service_gateway_pids}, + "identities": {str(pid): pause_record.identity(pid)["ct"] for pid in running_pids}, + } + if service_gateways: + names = [str(s.name) for s in service_gateways] + intended.update(services=names, expected_services=list(names), restarted_services=[], + service_profiles={str(s.name): str(s.profile) for s in service_gateways}) + if adopted is not None: + pause_record.merge_into(intended, adopted) + pause_record.stamp_tree(intended) + with _abort_on_error("Could not record the gateways this update pauses"): + pause_record.write(intended, owner=pause_record.identity()) + profiles, mapped_pids, socket_acks = _request_socket_pauses(running_pids, profile_processes, service_gateway_pids) + # Resolve venv-side launchers BEFORE draining: a dead worker's parent cannot be recovered (NoSuchProcess). + # The launcher keeps ``.pyd`` mapped and would trip the venv-holder guard; it is killed with the survivors. + launcher_pids = _m()._venv_launcher_ancestors(mapped_pids) + print("→ Stopping Windows gateway process(es) before updating Hermes...") + drain_timeout = _gateway_drain_timeout(socket_acks) + survivors = _m()._wait_for_windows_update_gateway_exit(mapped_pids, timeout=drain_timeout) # Tree-kill survivors, unmapped gateways, and pre-drain launchers; a launcher # already gone with its worker raises ProcessLookupError and is skipped. force_killed = [] @@ -963,7 +989,17 @@ def _pause_windows_gateways_for_update() -> dict | None: # socket pause (absent from ``profiles``) but the SCM restart brings them back, not a cold-start. running_profiles = set(profiles) | {str(p.profile) for p in profile_processes.values()} | {str(s.profile) for s in service_gateways} _record_attested_cold_start_profiles(token, running_profiles) - return _pause_windows_gateway_services(service_gateways, token, profiles, unmapped) + try: + token = _pause_windows_gateway_services(service_gateways, token, profiles, unmapped) + except RuntimeError as exc: + if "rollback failures" not in str(exc): # everything stopped was restarted in-line + pause_record.discharge(intended) + raise + if adopted is not None: + pause_record.merge_into(token, adopted) + token.update({key: intended[key] for key in ("pause_id", "pre_sha", "dirty_at_pause", "identities") if key in intended}) + pause_record.sync(token) + return token def _record_attested_cold_start_profiles(token: dict, running_profiles: set) -> None: @@ -1299,6 +1335,27 @@ def _resume_windows_gateways_after_update(token: dict | None) -> None: if not _m()._is_windows(): token["resume_needed"] = False return + from hermes_cli import update_pause_record as pause_record + if token.get("pause_id"): + whole, why = pause_record.tree_is_whole(token) + if not whole: + # Starting gateways now would boot them on a half-written tree or the pre-sync + # dependencies. The durable record keeps the obligation for the next launch, which + # runs after the interrupted-pull restore and the dependency sync. + print(f" ⚠ Paused gateway(s) left stopped: {why}.\n" + " The next `hermes` command (or `hermes update`) restarts them.") + token["resume_needed"] = False + token["resume_deferred"] = why + pause_record.sync(token) + return + try: + _resume_paused_set(token) + finally: + pause_record.sync(token) + + +def _resume_paused_set(token: dict) -> None: + from hermes_cli.update_cmd import _m # Regenerate launcher scripts before respawning so a legacy pythonw-era # autostart entry comes back on the current design at next login too. _m()._refresh_windows_gateway_launchers() diff --git a/hermes_cli/update_lock.py b/hermes_cli/update_lock.py index 90b2c0d71bfd0..dd59b6eb929dd 100644 --- a/hermes_cli/update_lock.py +++ b/hermes_cli/update_lock.py @@ -480,6 +480,41 @@ def _is_ancestor_pid(pid: int) -> bool: return False +def _is_runtime_host(cmdline: list[str]) -> bool: + """A long-lived Hermes host (``gateway run`` / ``serve`` / ``dashboard``), by the canonical + command-line matchers (profile flags, ``hermes_cli/main.py`` paths, inline bootstraps).""" + from gateway.status import looks_like_gateway_command_line + from hermes_cli.update_cmd_windows import _hermes_holder_subcommand + line = " ".join(cmdline) + return looks_like_gateway_command_line(line) or _hermes_holder_subcommand(line) in ("serve", "dashboard") + + +def _runtime_host_below(holder_pid: int) -> bool: + """True when a Hermes gateway/serve/dashboard sits between us and *holder_pid* (or anywhere + above us when the holder is not reached). + + Such a host is relaunched BY an update and outlives its stages; a ``hermes update`` its agent + or ``/update`` starts is an independent update that must not run under the first one's claim + (cli §7 V9). Unreadable command lines count as not-a-host (the legacy adoption stands). + """ + try: + import psutil + except ImportError: + return False + try: + proc = psutil.Process().parent() + for _ in range(_MAX_ANCESTRY_DEPTH): + if proc is None or proc.pid == holder_pid: + return False + with suppress(psutil.Error): + if _is_runtime_host(proc.cmdline()): + return True + proc = proc.parent() + except psutil.Error: + return False + return False + + # --- the marker -------------------------------------------------------------------------- @@ -1490,7 +1525,7 @@ def _adopt_or_refuse(self, existing: _Marker) -> bool: """C1 rule 4: a LIVE claim by us, an ancestor or the hand-off partner is run under. Called inside the marker mutex.""" partners = _live_partners(existing) - if not any(self._is_partner(p) for p in partners): + if not any(self._is_partner(p) and (p == os.getpid() or not _runtime_host_below(p)) for p in partners): self.holder = UpdateHolder(pid=partners[0], age_seconds=existing.age() if existing.started_at else 0.0) return False own = _identity_line() diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py new file mode 100644 index 0000000000000..8fd60dfeb6d1e --- /dev/null +++ b/hermes_cli/update_pause_record.py @@ -0,0 +1,279 @@ +"""Durable record of the gateways and services ``hermes update`` paused on Windows. + +The pause token used to live only in the updater's memory (plus an ``atexit`` resume), so a +killed updater (console closed, ``taskkill``, power loss) left every paused gateway and SCM +service stopped with nothing on disk saying so. The record is written under the ROOT Hermes +home BEFORE anything is stopped, names its owner (pid + creation time, the update-marker +identity format), is rewritten as the token changes, and is removed only after a verified +resume. A later launch whose owner check finds the updater dead (and no update live) resumes it. + +Resuming is gated on a whole tree: no interrupted-pull marker, no tracked change beyond the ones +present at pause time, and HEAD still at the pre-update commit or dependencies current for the +moved HEAD. Otherwise the record stays for the next launch, which runs after the interrupted-pull +restore and the dependency sync. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import uuid +from contextlib import suppress +from pathlib import Path + +RECORD_NAME = ".hermes-update-paused-gateways.json" +# Same tolerance as the update marker's creation-time identity (C1 rule 3). +_CT_TOLERANCE_S = 2.0 + + +def record_path() -> Path: + from hermes_constants import get_default_hermes_root + return get_default_hermes_root() / RECORD_NAME + + +def install_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def _create_time(pid: int) -> float | None: + from hermes_cli.process_identity import _process_create_time + return _process_create_time(pid) + + +def identity(pid: int | None = None) -> dict: + """``{"pid", "ct"}`` with ``ct`` spelled like the update marker's line 3 (``ct:``).""" + pid = os.getpid() if pid is None else int(pid) + created = _create_time(pid) + return {"pid": pid, "ct": None if created is None else f"ct:{created:.3f}"} + + +def identity_is_live(ident: dict | None) -> bool: + """Alive (not a zombie) and, when a creation time was recorded, the same incarnation.""" + from hermes_cli._early_recovery import _pid_is_running + try: + pid = int((ident or {}).get("pid") or 0) + except (TypeError, ValueError): + return False + if pid <= 0 or not _pid_is_running(pid): + return False + recorded = str((ident or {}).get("ct") or "") + if not recorded.startswith("ct:"): + return True + actual = _create_time(pid) + if actual is None: + return True + try: + return abs(float(recorded[3:]) - actual) <= _CT_TOLERANCE_S + except ValueError: + return True + + +def _git(root: Path, *args: str) -> subprocess.CompletedProcess | None: + try: + return subprocess.run( + ["git", "-C", str(root), *args], capture_output=True, text=True, encoding="utf-8", + errors="replace", stdin=subprocess.DEVNULL, timeout=30, check=False) + except (OSError, subprocess.SubprocessError): + return None + + +def head_sha(root: Path) -> str | None: + result = _git(root, "rev-parse", "--verify", "HEAD") + if result is None or result.returncode != 0: + return None + return result.stdout.strip() or None + + +def tracked_changes(root: Path) -> list[str] | None: + """Tracked paths that differ from HEAD (index or worktree); ``None`` when git cannot say.""" + result = _git(root, "status", "--porcelain=v1", "-z", "--untracked-files=no") + if result is None or result.returncode != 0: + return None + paths = set() + entries = result.stdout.split("\0") + index = 0 + while index < len(entries): + entry = entries[index] + index += 1 + if len(entry) < 4: + continue + paths.add(entry[3:]) + if entry[0] in "RC": # rename/copy: the source path follows as its own entry + index += 1 + return sorted(paths) + + +def stamp_tree(token: dict, root: Path | None = None) -> dict: + """Record what "the tree before this update" was, for the whole-tree gate.""" + root = root or install_root() + if (root / ".git").exists(): + token.setdefault("pre_sha", head_sha(root)) + token.setdefault("dirty_at_pause", tracked_changes(root)) + token.setdefault("pause_id", uuid.uuid4().hex) + return token + + +def _venv_is_current(root: Path) -> bool: + try: + import pm + return bool(pm.venv_is_current(project_root=root)) + except Exception: + return False + + +def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]: + """May paused gateways start on this checkout now? ``(verdict, reason when not)``.""" + from hermes_cli._early_recovery import interrupted_pull_marker + root = root or install_root() + if (root / ".git").exists(): + if interrupted_pull_marker(root).exists(): + return False, "the checkout is mid-pull (interrupted-pull marker present)" + changes = tracked_changes(root) + if changes is None: + return False, "git cannot read the checkout state" + unexpected = sorted(set(changes) - set(token.get("dirty_at_pause") or [])) + if unexpected: + return False, f"the checkout has {len(unexpected)} file(s) git left half-written (e.g. {unexpected[0]})" + head = head_sha(root) + if head is None: + return False, "the checkout HEAD is unreadable" + if head == token.get("pre_sha"): + return True, "" + if not _venv_is_current(root): + return False, "dependencies are not current for the updated code yet" + return True, "" + + +def _atomic_write(path: Path, body: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(body, fh, indent=1, sort_keys=True) + fh.flush() + os.fsync(fh.fileno()) + os.replace(tmp, path) + + +def read(path: Path | None = None) -> dict | None: + path = path or record_path() + try: + body = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, ValueError): + return None + return body if isinstance(body, dict) and isinstance(body.get("token"), dict) else None + + +def write(token: dict, *, owner: dict | None = None, path: Path | None = None) -> None: + """Persist *token*; the owner of an existing record for the same pause is kept.""" + path = path or record_path() + existing = read(path) + if owner is None: + if existing is not None and existing["token"].get("pause_id") == token.get("pause_id"): + owner = existing["owner"] + else: + owner = identity() + _atomic_write(path, {"schema": 1, "owner": owner, "install_root": str(install_root()), "token": token}) + + +def discharge(token: dict, path: Path | None = None) -> None: + """Compare-and-delete: remove the record only while it still names this pause.""" + path = path or record_path() + existing = read(path) + if existing is not None and existing["token"].get("pause_id") == token.get("pause_id"): + with suppress(OSError): + path.unlink() + + +def sync(token: dict) -> None: + """Mirror the token after a resume attempt: done → delete; anything still owed → rewrite.""" + if not token.get("pause_id"): + return + try: + if token.get("resume_needed") or token.get("resume_deferred"): + owed = {**token, "resume_needed": True} + owed.pop("resume_deferred", None) + write(owed) + else: + discharge(token) + except OSError as exc: + print(f" ⚠ Could not update the paused-gateway record {record_path()}: {exc}") + + +def _live_update_elsewhere() -> bool: + # TODO(LP-LOCK): switch to update_lock.update_in_progress(install_root) once it lands; the + # marker read below is the existing path and misses an orphaned completion child. + from hermes_cli.update_lock import read_live_update + holder = read_live_update() + return holder is not None and holder.pid != os.getpid() + + +def orphaned_record(path: Path | None = None) -> dict | None: + """The record when its owner is dead, it belongs to this install and no other update is live.""" + body = read(path) + if body is None or body.get("install_root") != str(install_root()): + return None + if identity_is_live(body.get("owner")) or _live_update_elsewhere(): + return None + return body + + +def drop_never_stopped(token: dict) -> dict: + """Entries whose recorded process is still the same live incarnation were never stopped.""" + alive = {pid for pid, ct in (token.get("identities") or {}).items() + if identity_is_live({"pid": pid, "ct": ct})} + token["profiles"] = {p: pid for p, pid in (token.get("profiles") or {}).items() if str(pid) not in alive} + token["unmapped"] = [u for u in (token.get("unmapped") or []) if str(u.get("pid")) not in alive] + return token + + +def merge_into(token: dict | None, adopted: dict) -> dict: + """Fold an orphaned pause into this update's token so its resume brings both sets back.""" + token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []} + token["resume_needed"] = True + profiles = token.setdefault("profiles", {}) + for name, pid in (adopted.get("profiles") or {}).items(): + profiles.setdefault(name, pid) + token.setdefault("unmapped", []).extend(adopted.get("unmapped") or []) + services = token.setdefault("services", []) + services.extend(s for s in adopted.get("services") or [] if s not in services) + if services: + token.setdefault("expected_services", []).extend(s for s in services if s not in token["expected_services"]) + token.setdefault("restarted_services", []) + token.setdefault("service_profiles", {}).update(adopted.get("service_profiles") or {}) + return token + + +def recover(argv: list[str] | None = None) -> None: + """Start-of-run recovery: resume an orphaned pause. Never raises.""" + from hermes_cli._parser import command_argv + command = command_argv(list(sys.argv[1:] if argv is None else argv)) + if command[:1] == ["update"] or command[:2] == ["gateway", "run"]: + return # the update adopts it itself; a booting gateway must not block on its siblings + try: + body = orphaned_record() + if body is None: + return + path = record_path() + claim = path.with_name(f"{path.name}.{os.getpid()}.claim") + try: + os.rename(path, claim) # exactly one concurrent launch wins the record + except OSError: + return + token = drop_never_stopped(dict(body["token"])) + token["resume_needed"] = True + print("→ Restarting gateway(s) paused by an interrupted `hermes update`...", file=sys.stderr) + try: + # Its own finally mirrors the outcome: whatever is still owed is re-recorded + # under this process, so the next launch retries once we have exited. + from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update + _resume_windows_gateways_after_update(token) + except Exception as exc: + print(f" ⚠ Could not restart every paused gateway: {exc}. Run `hermes update` or " + "`hermes gateway start`.", file=sys.stderr) + finally: + with suppress(OSError): + claim.unlink() + except Exception as exc: # never brick a launch on recovery + print(f" ⚠ Paused-gateway recovery skipped: {exc}", file=sys.stderr) From dad6c01d9507f23ad93ab5f2ed716ab6be3baf30 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 10:06:14 -0700 Subject: [PATCH 02/63] =?UTF-8?q?test(update):=20durable=20Windows=20pause?= =?UTF-8?q?=20=E2=80=94=20record=20ownership,=20whole-tree=20gate,=20claim?= =?UTF-8?q?=20adoption;=20wine2e=20cells?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit fe97d553c4ee91916a617bb1f1c1eefeed0e6bf0) (cherry picked from commit f2adbee6548fdc4b96164718ae04ae10582d2ec6) --- .../core/windows_update/test_pause_durable.py | 153 ++++++++++++++++++ tests/hermes_cli/test_update_pause_record.py | 108 +++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 tests/e2e/core/windows_update/test_pause_durable.py create mode 100644 tests/hermes_cli/test_update_pause_record.py diff --git a/tests/e2e/core/windows_update/test_pause_durable.py b/tests/e2e/core/windows_update/test_pause_durable.py new file mode 100644 index 0000000000000..2aed7efeb786b --- /dev/null +++ b/tests/e2e/core/windows_update/test_pause_durable.py @@ -0,0 +1,153 @@ +"""A killed or failed ``hermes update`` never strands — or wrongly starts — a paused gateway. + +Failure class: Windows pause durability. ``hermes update`` stops every running gateway before it +touches the checkout. Cell (a): the updater is killed (``taskkill /F /T`` — console closed, Desktop +kill) right after the pause; the next plain ``hermes`` command must bring the gateway back. +Cell (b): the pull fails half-way (a held file makes git's fast-forward die after writing part of +the tree); the gateway must NOT be started onto that torn tree. +""" + +from __future__ import annotations + +import subprocess +import threading +import time + +import psutil +import pytest + +from tests.e2e.core.windows_update._machine import ( + REQUIRES_OPT_IN, + fail_with, + harness_git, + new_machine, +) +from tests.fakes.fake_llm_provider import FakeLLMServer + +pytestmark = [pytest.mark.platforms("windows"), pytest.mark.integration, + pytest.mark.live_system_guard_bypass, REQUIRES_OPT_IN] + +_PAUSED = "Paused gateway profile" +_RECORD = ".hermes-update-paused-gateways.json" +_EARLY, _HELD = "AGENTS.md", "website/package.json" # checkout order: the early file is written first + + +def _alive(pid: int) -> bool: + try: + return psutil.Process(pid).is_running() and psutil.Process(pid).status() != psutil.STATUS_ZOMBIE + except psutil.Error: + return False + + +def _running_gateway(machine, not_pid: int, timeout: float) -> dict | None: + try: + return machine.wait_gateway_running(not_pid=not_pid, timeout=timeout) + except AssertionError: + return None + + +def _update_killed_after_pause(machine) -> tuple[bool, str]: + """Start ``hermes update`` and ``taskkill /F /T`` it the moment it reports the pause.""" + log = machine.logs / "update-killed.log" + proc = subprocess.Popen([str(machine.hermes_exe), "update", "--yes"], cwd=machine.profile, + env=machine.env(), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + seen = threading.Event() + lines: list[str] = [] + assert proc.stdout is not None + + def pump() -> None: + with log.open("w", encoding="utf-8") as fh: + for raw in iter(proc.stdout.readline, b""): + text = raw.decode("utf-8", "replace").rstrip() + lines.append(text) + fh.write(text + "\n") + fh.flush() + if _PAUSED in text: + seen.set() + + threading.Thread(target=pump, daemon=True).start() + deadline = time.monotonic() + 600 + while not seen.is_set() and proc.poll() is None and time.monotonic() < deadline: + time.sleep(0.05) + subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60) + proc.wait(timeout=60) + return seen.is_set(), "\n".join(lines[-40:]) + + +def _mint_two_file_change(machine) -> str: + """A target commit on top of the installed HEAD that rewrites two existing tracked files.""" + base = machine.installed_head() + index = machine.root / "torn.index" + env = {"GIT_INDEX_FILE": str(index), "GIT_AUTHOR_NAME": "Hermes E2E", "GIT_AUTHOR_EMAIL": "e2e@hermes.invalid", + "GIT_COMMITTER_NAME": "Hermes E2E", "GIT_COMMITTER_EMAIL": "e2e@hermes.invalid"} + harness_git("-C", str(machine.serve), "read-tree", base, env=env) + for path in (_EARLY, _HELD): + src = machine.root / ("torn-" + path.replace("/", "_")) + src.write_bytes((machine.install_dir / path).read_bytes() + b"\n") + blob = harness_git("-C", str(machine.serve), "hash-object", "-w", "--no-filters", str(src)) + harness_git("-C", str(machine.serve), "update-index", "--cacheinfo", f"100644,{blob},{path}", env=env) + tree = harness_git("-C", str(machine.serve), "write-tree", env=env) + index.unlink(missing_ok=True) + target = harness_git("-C", str(machine.serve), "commit-tree", tree, "-p", base, "-m", "e2e: torn target", env=env) + harness_git("-C", str(machine.serve), "update-ref", "refs/heads/main", target) + return base + + +@pytest.fixture(scope="module") +def journey(tmp_path_factory): + out: dict = {} + with FakeLLMServer() as srv: + machine = new_machine(tmp_path_factory.mktemp("pd"), srv.base_url, label="pd", system_git=True) + out["machine"] = machine + try: + install = machine.install() + assert install.returncode == 0, fail_with(machine, f"install.ps1 exited {install.returncode}", install) + with machine.gateway_phase(): + # (a) kill the updater right after it paused a real gateway + machine.spawn_gateway() + old = int(machine.wait_gateway_running().get("pid") or 0) + machine.advance() + out["a_paused"], out["a_tail"] = _update_killed_after_pause(machine) + out["a_old_stopped"] = not _alive(old) + out["a_record"] = (machine.hermes_home / _RECORD).is_file() + out["a_launch"] = machine.hermes("gateway", "status", label="next-launch") + out["a_after"] = _running_gateway(machine, old, 120) + machine.kill_owned() + + # (b) the pull fails with a torn tree + machine.spawn_gateway() + old = int(machine.wait_gateway_running().get("pid") or 0) + out["b_pre"] = _mint_two_file_change(machine) + with (machine.install_dir / _HELD).open("rb"): # no FILE_SHARE_DELETE: git cannot replace it + out["b_update"] = machine.hermes("update", "--yes", label="update-torn", timeout=900) + out["b_head"] = machine.installed_head() + out["b_status"] = harness_git("-C", str(machine.install_dir), "status", "--porcelain", + "--untracked-files=no") + out["b_after"] = _running_gateway(machine, old, 60) + out["b_record"] = (machine.hermes_home / _RECORD).is_file() + machine.kill_owned() + yield out + finally: + machine.teardown() + + +def test_killed_updater_gateway_resumed_by_next_launch(journey) -> None: + m = journey["machine"] + assert journey["a_paused"] and journey["a_old_stopped"], fail_with( + m, f"premise: the update never paused the gateway before the kill\n{journey['a_tail']}") + assert journey["a_after"] is not None, fail_with( + m, "after `taskkill /F` of an updater that paused the gateway, the next `hermes` launch left it " + f"stopped (pause record present after kill: {journey['a_record']})", journey["a_launch"]) + + +def test_failed_pull_does_not_start_gateway_on_torn_tree(journey) -> None: + m, run = journey["machine"], journey["b_update"] + torn = journey["b_head"] == journey["b_pre"] and _EARLY in journey["b_status"] + assert run.returncode != 0 and torn, fail_with( + m, f"premise: the held file did not tear the pull (rc={run.returncode}, HEAD={journey['b_head']}, " + f"pre={journey['b_pre']}, status={journey['b_status']!r})", run) + assert journey["b_after"] is None, fail_with( + m, f"the gateway was restarted onto a torn checkout ({journey['b_status']!r}) after the failed update " + f"(pid {journey['b_after'].get('pid') if journey['b_after'] else None})", run) + assert journey["b_record"], fail_with(m, "the paused set was not kept for recovery after the torn pull", run) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py new file mode 100644 index 0000000000000..a60d2df7163b2 --- /dev/null +++ b/tests/hermes_cli/test_update_pause_record.py @@ -0,0 +1,108 @@ +"""Paused-gateway record and update-claim adoption, with real processes and a real checkout.""" + +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import textwrap +import time +from pathlib import Path + +import pytest + +from hermes_cli import update_pause_record as pause_record + +REPO = Path(__file__).resolve().parents[2] +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals; Windows cells live in wine2e") + + +def _child(code: str, *argv: str, env: dict | None = None) -> subprocess.Popen: + return subprocess.Popen([sys.executable, "-c", textwrap.dedent(code), *argv], cwd=REPO, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, text=True, env={**os.environ, "PYTHONPATH": str(REPO), **(env or {})}) + + +def test_record_written_by_a_killed_updater_is_orphaned_only_after_its_death(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + owner = _child(""" + import sys, time + from hermes_cli import update_pause_record as r + r.write(r.stamp_tree({"resume_needed": True, "profiles": {"default": 4242}}), owner=r.identity()) + print("written", flush=True) + time.sleep(120) + """, env={"HERMES_HOME": str(tmp_path)}) + try: + assert owner.stdout.readline().strip() == "written" + body = pause_record.read() + assert body["owner"]["pid"] == owner.pid and body["owner"]["ct"].startswith("ct:") + assert body["token"]["profiles"] == {"default": 4242} + assert pause_record.orphaned_record() is None # live owner: its own resume owns the set + finally: + owner.send_signal(signal.SIGKILL) # windows-footgun: ok — module skips on Windows + owner.wait(timeout=10) + orphan = pause_record.orphaned_record() + assert orphan is not None and orphan["token"]["profiles"] == {"default": 4242} + + +def _git(root: Path, *args: str) -> str: + return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True, encoding="utf-8", + env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}).stdout.strip() + + +def test_resume_waits_for_a_whole_tree(tmp_path): + root = tmp_path / "checkout" + root.mkdir() + _git(root, "init", "-q") + for name in ("a.py", "b.py", "local.txt"): + (root / name).write_text("v1\n", encoding="utf-8") + _git(root, "add", ".") + _git(root, "commit", "-qm", "v1") + (root / "local.txt").write_text("user edit\n", encoding="utf-8") # dirty before the update: not git's doing + token = pause_record.stamp_tree({"resume_needed": True}, root) + assert pause_record.tree_is_whole(token, root) == (True, "") + + (root / "a.py").write_text("v2\n", encoding="utf-8") # git wrote a.py, then died before b.py and HEAD + whole, why = pause_record.tree_is_whole(token, root) + assert not whole and "a.py" in why + + (root / "a.py").write_text("v1\n", encoding="utf-8") + marker = root / ".git" / "hermes-update-pull" + marker.write_text("pid\n", encoding="utf-8") + assert pause_record.tree_is_whole(token, root)[0] is False + marker.unlink() + + _git(root, "commit", "-qam", "v2") # HEAD moved: only a dependency sync for it makes it whole + whole, why = pause_record.tree_is_whole(token, root) + assert not whole and "dependencies" in why + + +_HOLDER = """ + import subprocess, sys, time + from hermes_cli.update_lock import UpdateLock + lock = UpdateLock() + assert lock.acquire() and lock.acquired + host = subprocess.Popen([sys.executable, sys.argv[1], "hermes_cli.main", *sys.argv[2:]], stdout=subprocess.PIPE, text=True) + print(host.stdout.read().strip(), flush=True) + host.wait() +""" +# A stand-in for the host the update relaunched: its argv names the host command, and the +# ``hermes update`` its agent starts is its child. +_HOST = """ +import subprocess, sys +probe = "from hermes_cli.update_lock import UpdateLock; l = UpdateLock(); print(l.acquire(), l.holder is not None)" +print(subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True).stdout.strip()) +""" + + +# The intermediate is a stand-in script, not a real gateway/update: nothing touches the checkout. +@pytest.mark.live_system_guard_bypass +@pytest.mark.parametrize("host_argv, adopts", [(("gateway", "run"), False), (("status",), True)]) +def test_update_started_from_a_relaunched_gateway_does_not_share_the_claim(tmp_path, host_argv, adopts): + script = tmp_path / "host.py" + script.write_text(_HOST, encoding="utf-8") + holder = _child(_HOLDER, str(script), *host_argv, env={"HERMES_HOME": str(tmp_path)}) + out, _ = holder.communicate(timeout=60) + assert holder.returncode == 0, out + assert out.strip() == ("True False" if adopts else "False True"), out From 38c8a00270573224a9c1359835071a5366115afd Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:38:53 -0700 Subject: [PATCH 03/63] fix(update): pause record survives killed recoveries and partner-claimed updates - A recovering launch killed mid-resume (taskkill, console close) left only its .claim file, which nothing read: the paused gateways were never restarted. Claims now carry the claimer's identity; a claim whose claimer is dead is re-adopted like a dead owner's record. A launch that still owes part of the set hands its claim back unowned so the next launch retries at once. - Under a Desktop/Tauri-driven update the orchestrator holds the marker, so the update saw its own partner as "another update", never adopted an orphaned pause record and then overwrote it. The update tree holding the checkout lock is no longer "elsewhere" (update_lock.update_in_progress for everyone else), and write() merges another pause's orphaned set instead of overwriting it (refuses when that pause's owner is live). - Runtime-host detection (gateway run / serve / dashboard in the ancestry) uses the canonical command-line matchers, so `--profile X gateway run` and the hermes_cli/main.py path form are recognised; the rule sits in _adopt_or_refuse's partner check. - Whole-tree gate: at a moved HEAD only the interrupted-pull marker and current dependencies count; a tracked file a build rewrote no longer keeps gateways stopped forever. At the pre-update HEAD a new tracked change is still git's torn checkout. - _pause_windows_gateways_for_update split into record / cold-start / stop phases. (cherry picked from commit a56eaa3a0f65c44f45600a9b2868859d98cc382e) (cherry picked from commit a72664b05f5f789c0b238ee1fcd1e9e087851182) --- hermes_cli/update_cmd_windows.py | 80 ++++++------ hermes_cli/update_lock.py | 9 +- hermes_cli/update_pause_record.py | 206 ++++++++++++++++++++++-------- 3 files changed, 203 insertions(+), 92 deletions(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index bcef081c67c06..2c36161482521 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -913,31 +913,19 @@ def _pause_windows_gateways_for_update() -> dict | None: """Stop running Windows gateways before mutating the checkout or venv. Scheduled/startup gateways run via pythonw.exe, invisible to the hermes.exe instance guard, yet keep files - locked during ``git``/``uv``. Stop only PIDs the gateway discovery code identifies.""" + locked during ``git``/``uv``. Stop only PIDs the gateway discovery code identifies. The paused set is on + disk (``update_pause_record``) before the first stop, with any set a killed earlier update orphaned.""" from hermes_cli.update_cmd import _m if not _m()._is_windows(): return None with _abort_on_error("Could not prepare Windows gateway pause for update"): - from gateway.status import get_process_start_time, terminate_pid + import gateway.status # noqa: F401 — fail before the first stop, not half-way through it from hermes_cli.gateway import _capture_gateway_argv from hermes_cli import update_pause_record as pause_record - # A previous update killed mid-pause left its set stopped: bring it back with this run's resume. - orphan = pause_record.orphaned_record() - adopted = pause_record.drop_never_stopped(dict(orphan["token"])) if orphan else None + adopted, claims = pause_record.adopt_orphans() profile_processes, service_gateways, service_gateway_pids, running_pids = _discover_windows_gateways() if not running_pids: - token = _windows_cold_start_plan() - # Other profiles may hold a dead attestation even when the active profile owes nothing - # (clean exit, autostart not installed): give them a token to ride on. - probe = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []} - _record_attested_cold_start_profiles(probe, set()) - token = probe if probe.get("cold_start_profiles") else token - if adopted is None: - return pause_record.stamp_tree(token) if token is not None else None - token = pause_record.stamp_tree(pause_record.merge_into(token, adopted)) - with _abort_on_error("Could not record the adopted paused-gateway set"): - pause_record.write(token, owner=pause_record.identity()) - return token + return _cold_start_pause_token(adopted, claims) unmapped_pids = [pid for pid in running_pids if pid not in profile_processes and pid not in service_gateway_pids] # Snapshot unmapped gateways' argv *before* anything stops so resume (or a later launch) can replay it. # Unmapped = no profile->PID-file mapping (e.g. Scheduled Task ``pythonw.exe -m ...``). @@ -946,7 +934,6 @@ def _pause_windows_gateways_for_update() -> dict | None: "Could not capture argv for unmapped gateway %s: %s", int(pid))} for pid in unmapped_pids ] - # Durable BEFORE the first stop: a killed updater must leave a record the next launch can resume. intended = { "resume_needed": True, "unmapped_pids": unmapped_pids, "unmapped": unmapped, "profiles": {str(profile_processes[pid].profile): int(pid) for pid in running_pids @@ -957,11 +944,43 @@ def _pause_windows_gateways_for_update() -> dict | None: names = [str(s.name) for s in service_gateways] intended.update(services=names, expected_services=list(names), restarted_services=[], service_profiles={str(s.name): str(s.profile) for s in service_gateways}) - if adopted is not None: - pause_record.merge_into(intended, adopted) - pause_record.stamp_tree(intended) with _abort_on_error("Could not record the gateways this update pauses"): - pause_record.write(intended, owner=pause_record.identity()) + pause_record.record_pause(intended, adopted, claims) + profiles = _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped) + token = {"resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped} + # Every profile with ANY live gateway at discovery counts as running: service-supervised ones skip the + # socket pause (absent from ``profiles``) but the SCM restart brings them back, not a cold-start. + running_profiles = set(profiles) | {str(p.profile) for p in profile_processes.values()} | {str(s.profile) for s in service_gateways} + _record_attested_cold_start_profiles(token, running_profiles) + try: + token = _pause_windows_gateway_services(service_gateways, token, profiles, unmapped) + except RuntimeError as exc: + if "rollback failures" not in str(exc): # everything stopped was restarted in-line + pause_record.discharge(intended) + raise + return pause_record.finish_pause(token, intended, adopted) + + +def _cold_start_pause_token(adopted: dict | None, claims: list) -> dict | None: + """No gateway runs: the cold-start plan, plus any orphaned set adopted from a killed update.""" + from hermes_cli import update_pause_record as pause_record + token = _windows_cold_start_plan() + # Other profiles may hold a dead attestation even when the active profile owes nothing + # (clean exit, autostart not installed): give them a token to ride on. + probe = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []} + _record_attested_cold_start_profiles(probe, set()) + token = probe if probe.get("cold_start_profiles") else token + if adopted is None: + return pause_record.stamp_tree(token) if token is not None else None + token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []} + with _abort_on_error("Could not record the adopted paused-gateway set"): + return pause_record.record_pause(token, adopted, claims) + + +def _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped) -> dict: + """Socket-pause the mapped gateways, drain, then tree-kill survivors; the paused ``profiles``.""" + from gateway.status import get_process_start_time, terminate_pid + from hermes_cli.update_cmd import _m profiles, mapped_pids, socket_acks = _request_socket_pauses(running_pids, profile_processes, service_gateway_pids) # Resolve venv-side launchers BEFORE draining: a dead worker's parent cannot be recovered (NoSuchProcess). # The launcher keeps ``.pyd`` mapped and would trip the venv-holder guard; it is killed with the survivors. @@ -984,22 +1003,7 @@ def _pause_windows_gateways_for_update() -> dict | None: print(f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping") if any(not u.get("argv") for u in unmapped): # no recoverable cmdline (psutil missing, denied, gone) print(" Restart manually after update: hermes gateway run") - token = {"resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped} - # Every profile with ANY live gateway at discovery counts as running: service-supervised ones skip the - # socket pause (absent from ``profiles``) but the SCM restart brings them back, not a cold-start. - running_profiles = set(profiles) | {str(p.profile) for p in profile_processes.values()} | {str(s.profile) for s in service_gateways} - _record_attested_cold_start_profiles(token, running_profiles) - try: - token = _pause_windows_gateway_services(service_gateways, token, profiles, unmapped) - except RuntimeError as exc: - if "rollback failures" not in str(exc): # everything stopped was restarted in-line - pause_record.discharge(intended) - raise - if adopted is not None: - pause_record.merge_into(token, adopted) - token.update({key: intended[key] for key in ("pause_id", "pre_sha", "dirty_at_pause", "identities") if key in intended}) - pause_record.sync(token) - return token + return profiles def _record_attested_cold_start_profiles(token: dict, running_profiles: set) -> None: diff --git a/hermes_cli/update_lock.py b/hermes_cli/update_lock.py index dd59b6eb929dd..725e2ab13ae6c 100644 --- a/hermes_cli/update_lock.py +++ b/hermes_cli/update_lock.py @@ -1305,6 +1305,11 @@ class _Extended(ctypes.Structure): return job +def holds_checkout_lock(install_root: Path | str | None = None) -> bool: + """True when this process holds (or joined) the checkout lock: it IS the running update.""" + return _HELD is not None and os.path.realpath(_HELD["path"]) == os.path.realpath(checkout_lock_path(install_root)) + + def update_in_progress(install_root: Path | str | None = None) -> bool: """True while an update owns this install: a LIVE marker or a held checkout lock.""" return read_live_update(install_root=install_root) is not None or checkout_lock_held(install_root) @@ -1322,9 +1327,9 @@ def checkout_lock_held(install_root: Path | str | None = None) -> bool: launch the others would park. A lock call that fails outright (ENOLCK on NFS without lockd, EOPNOTSUPP on some SMB shares) means nothing can hold it: free, so the interrupted-pull repair runs unguarded there as designed.""" - path = checkout_lock_path(install_root) - if _HELD is not None and _HELD["path"] == str(path): + if holds_checkout_lock(install_root): return True + path = checkout_lock_path(install_root) try: fd = os.open(path, os.O_RDONLY | getattr(os, "O_BINARY", 0)) except FileNotFoundError: diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 8fd60dfeb6d1e..3f0e34803418f 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -7,14 +7,19 @@ identity format), is rewritten as the token changes, and is removed only after a verified resume. A later launch whose owner check finds the updater dead (and no update live) resumes it. -Resuming is gated on a whole tree: no interrupted-pull marker, no tracked change beyond the ones -present at pause time, and HEAD still at the pre-update commit or dependencies current for the -moved HEAD. Otherwise the record stays for the next launch, which runs after the interrupted-pull -restore and the dependency sync. +Resuming is gated on a whole tree: no interrupted-pull marker; at the pre-update HEAD, no tracked +change beyond the ones present at pause time (git died before moving HEAD); at a moved HEAD, +dependencies current for it (a build step may rewrite tracked files there). Otherwise the record +stays for the next launch, which runs after the interrupted-pull restore and the dependency sync. + +A recovering launch claims the record by renaming it to ``...claim`` and names +itself inside; a claim whose claimer died (killed mid-resume) is re-adopted like a dead owner's +record. ``write`` never overwrites another pause: an orphan's set is merged in, a live one refuses. """ from __future__ import annotations +import glob import json import os import subprocess @@ -130,16 +135,17 @@ def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]: if (root / ".git").exists(): if interrupted_pull_marker(root).exists(): return False, "the checkout is mid-pull (interrupted-pull marker present)" - changes = tracked_changes(root) - if changes is None: - return False, "git cannot read the checkout state" - unexpected = sorted(set(changes) - set(token.get("dirty_at_pause") or [])) - if unexpected: - return False, f"the checkout has {len(unexpected)} file(s) git left half-written (e.g. {unexpected[0]})" head = head_sha(root) if head is None: return False, "the checkout HEAD is unreadable" if head == token.get("pre_sha"): + # HEAD never moved: a new tracked change is git's half-written checkout. + changes = tracked_changes(root) + if changes is None: + return False, "git cannot read the checkout state" + unexpected = sorted(set(changes) - set(token.get("dirty_at_pause") or [])) + if unexpected: + return False, f"the checkout has {len(unexpected)} file(s) git left half-written (e.g. {unexpected[0]})" return True, "" if not _venv_is_current(root): return False, "dependencies are not current for the updated code yet" @@ -165,15 +171,25 @@ def read(path: Path | None = None) -> dict | None: return body if isinstance(body, dict) and isinstance(body.get("token"), dict) else None +UNOWNED = {"pid": 0, "ct": None} + + +class RecordConflict(OSError): + """The record names another pause whose owner is still live.""" + + def write(token: dict, *, owner: dict | None = None, path: Path | None = None) -> None: - """Persist *token*; the owner of an existing record for the same pause is kept.""" + """Persist *token*. The owner of the same pause is kept; another pause's orphaned set is merged + into *token* (in place, so this run's resume brings it back); a live one refuses.""" path = path or record_path() existing = read(path) + same = existing is not None and existing["token"].get("pause_id") == token.get("pause_id") + if existing is not None and not same: + if identity_is_live(existing.get("owner")): + raise RecordConflict(f"{path} holds gateways paused by live process {existing['owner'].get('pid')}") + merge_into(token, drop_never_stopped(dict(existing["token"]))) if owner is None: - if existing is not None and existing["token"].get("pause_id") == token.get("pause_id"): - owner = existing["owner"] - else: - owner = identity() + owner = existing["owner"] if same else identity() _atomic_write(path, {"schema": 1, "owner": owner, "install_root": str(install_root()), "token": token}) @@ -188,8 +204,8 @@ def discharge(token: dict, path: Path | None = None) -> None: def sync(token: dict) -> None: """Mirror the token after a resume attempt: done → delete; anything still owed → rewrite.""" - if not token.get("pause_id"): - return + if not token.get("pause_id") or token.get("recovery"): + return # a recovering launch keeps what it owes in its own claim (_resume_claimed) try: if token.get("resume_needed") or token.get("resume_deferred"): owed = {**token, "resume_needed": True} @@ -202,21 +218,103 @@ def sync(token: dict) -> None: def _live_update_elsewhere() -> bool: - # TODO(LP-LOCK): switch to update_lock.update_in_progress(install_root) once it lands; the - # marker read below is the existing path and misses an orphaned completion child. - from hermes_cli.update_lock import read_live_update - holder = read_live_update() - return holder is not None and holder.pid != os.getpid() + """A live update other than the one this process belongs to. The update tree holding the + checkout lock (this process plus the partner whose marker it adopted) is not "elsewhere".""" + from hermes_cli import update_lock + root = install_root() + return not update_lock.holds_checkout_lock(root) and update_lock.update_in_progress(root) -def orphaned_record(path: Path | None = None) -> dict | None: - """The record when its owner is dead, it belongs to this install and no other update is live.""" +def _claims(path: Path) -> list[Path]: + return sorted(path.parent.glob(f"{glob.escape(path.name)}.*.claim")) + + +def _claimer(claim: Path, body: dict) -> dict: + """Who holds a claim: the identity it wrote, else (killed before writing it) its name's pid.""" + if isinstance(body.get("claimer"), dict): + return body["claimer"] + pid = claim.name[len(RECORD_NAME) + 1:].split(".", 1)[0] + return {"pid": int(pid) if pid.isdigit() else 0, "ct": None} + + +def orphans(path: Path | None = None) -> list[tuple[Path, dict]]: + """``(file, body)`` for this install's record whose owner is dead and every claim whose claimer + is dead — empty while another update is live (it adopts them itself).""" + path = path or record_path() + found = [] body = read(path) - if body is None or body.get("install_root") != str(install_root()): + if body is not None and not identity_is_live(body.get("owner")): + found.append((path, body)) + for claim in _claims(path): + claimed = read(claim) + if claimed is not None and not identity_is_live(_claimer(claim, claimed)): + found.append((claim, claimed)) + found = [(src, b) for src, b in found if b.get("install_root") == str(install_root())] + if not found or _live_update_elsewhere(): + return [] + return found + + +def orphaned_record(path: Path | None = None) -> dict | None: + found = orphans(path) + return found[0][1] if found else None + + +def claim(src: Path) -> tuple[Path, dict] | None: + """Take *src* (an orphaned record or a dead claimer's claim) for this process; ``None`` when a + concurrent launch won it. The rename is the arbiter; the claimer line names our incarnation.""" + path = record_path() + mine = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.claim") + try: + os.rename(src, mine) + except OSError: return None - if identity_is_live(body.get("owner")) or _live_update_elsewhere(): + body = read(mine) + if body is None: + with suppress(OSError): + mine.unlink() return None - return body + body["claimer"] = identity() + _atomic_write(mine, body) + return mine, body + + +def adopt_orphans() -> tuple[dict | None, list[Path]]: + """For ``hermes update``: claim every orphaned pause and merge their sets. The caller records the + merged set durably, then :func:`release_claims` — a crash in between leaves claims to re-adopt.""" + adopted, claims = None, [] + for src, _body in orphans(): + won = claim(src) + if won is None: + continue + claims.append(won[0]) + adopted = merge_into(adopted, drop_never_stopped(dict(won[1]["token"]))) + return adopted, claims + + +def release_claims(claims: list[Path]) -> None: + for path in claims: + with suppress(OSError): + path.unlink() + + +def record_pause(token: dict, adopted: dict | None, claims: list[Path]) -> dict: + """Merge the adopted set, stamp the tree and persist under this process BEFORE the first stop.""" + if adopted is not None: + merge_into(token, adopted) + stamp_tree(token) + write(token, owner=identity()) + release_claims(claims) + return token + + +def finish_pause(token: dict, intended: dict, adopted: dict | None) -> dict: + """Carry the recorded pause identity onto the final token and mirror it to disk.""" + if adopted is not None: + merge_into(token, adopted) + token.update({key: intended[key] for key in ("pause_id", "pre_sha", "dirty_at_pause", "identities") if key in intended}) + sync(token) + return token def drop_never_stopped(token: dict) -> dict: @@ -235,7 +333,8 @@ def merge_into(token: dict | None, adopted: dict) -> dict: profiles = token.setdefault("profiles", {}) for name, pid in (adopted.get("profiles") or {}).items(): profiles.setdefault(name, pid) - token.setdefault("unmapped", []).extend(adopted.get("unmapped") or []) + unmapped = token.setdefault("unmapped", []) + unmapped.extend(u for u in adopted.get("unmapped") or [] if u not in unmapped) services = token.setdefault("services", []) services.extend(s for s in adopted.get("services") or [] if s not in services) if services: @@ -245,35 +344,38 @@ def merge_into(token: dict | None, adopted: dict) -> dict: return token +def _resume_claimed(claim_path: Path, body: dict) -> None: + token = drop_never_stopped(dict(body["token"])) + token.update(resume_needed=True, recovery=True) + print("→ Restarting gateway(s) paused by an interrupted `hermes update`...", file=sys.stderr) + try: + from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update + _resume_windows_gateways_after_update(token) + except Exception as exc: + print(f" ⚠ Could not restart every paused gateway: {exc}. Run `hermes update` or " + "`hermes gateway start`.", file=sys.stderr) + finally: + with suppress(OSError): + if token.get("resume_needed") or token.get("resume_deferred"): + # Still owed: hand the claim back unowned so the next launch retries at once — this + # launch may live for hours (a chat). A failed rewrite leaves our claimer line, which + # turns dead (re-adoptable) when we exit. + owed = {key: value for key, value in token.items() if key not in ("recovery", "resume_deferred")} + _atomic_write(claim_path, {**body, "token": {**owed, "resume_needed": True}, "claimer": UNOWNED}) + else: + claim_path.unlink() + + def recover(argv: list[str] | None = None) -> None: - """Start-of-run recovery: resume an orphaned pause. Never raises.""" + """Start-of-run recovery: resume every orphaned pause. Never raises.""" from hermes_cli._parser import command_argv command = command_argv(list(sys.argv[1:] if argv is None else argv)) if command[:1] == ["update"] or command[:2] == ["gateway", "run"]: return # the update adopts it itself; a booting gateway must not block on its siblings try: - body = orphaned_record() - if body is None: - return - path = record_path() - claim = path.with_name(f"{path.name}.{os.getpid()}.claim") - try: - os.rename(path, claim) # exactly one concurrent launch wins the record - except OSError: - return - token = drop_never_stopped(dict(body["token"])) - token["resume_needed"] = True - print("→ Restarting gateway(s) paused by an interrupted `hermes update`...", file=sys.stderr) - try: - # Its own finally mirrors the outcome: whatever is still owed is re-recorded - # under this process, so the next launch retries once we have exited. - from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update - _resume_windows_gateways_after_update(token) - except Exception as exc: - print(f" ⚠ Could not restart every paused gateway: {exc}. Run `hermes update` or " - "`hermes gateway start`.", file=sys.stderr) - finally: - with suppress(OSError): - claim.unlink() + for src, _body in orphans(): + won = claim(src) + if won is not None: + _resume_claimed(*won) except Exception as exc: # never brick a launch on recovery print(f" ⚠ Paused-gateway recovery skipped: {exc}", file=sys.stderr) From 6ef70c6f60e10470edf7c6e7ecb4e6aeb9907fe2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:38:53 -0700 Subject: [PATCH 04/63] test(update): killed recovery re-adopted, partner-claimed update merges an orphan, profile-form hosts - unit: a launch SIGKILLed mid-recovery leaves the set to the next launch (and it is resumed exactly once); runtime-host refusal for --profile / -p / hermes_cli/main.py / dashboard argv; a committed update with a rewritten tracked file is judged on dependencies only. - e2e (real clone checkout): an update child running under an orchestrator's marker adopts and merges an orphaned pause record. (cherry picked from commit 67f2dd2c4cbeb0719b76d965d7e5270895c2663a) (cherry picked from commit 33820b493a5bf4c9cd629da2e9d3511021bac5fb) --- .../hosts/test_hostile_pause_record.py | 77 +++++++++++++++++++ tests/hermes_cli/test_update_pause_record.py | 64 ++++++++++++++- 2 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py diff --git a/tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py b/tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py new file mode 100644 index 0000000000000..a3e461cf6772f --- /dev/null +++ b/tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py @@ -0,0 +1,77 @@ +"""The paused-gateway record survives an update that runs under a partner's claim. + +A real checkout (a clone of the commit under test) and real processes: a ``hermes update`` +stand-in that recorded a paused set is SIGKILLed; the next update runs the way Desktop/Tauri +drives it — the orchestrator holds the marker and the ``hermes update`` child takes the checkout +lock and adopts the orchestrator's claim. That update must adopt the orphaned set and fold it into +its own record, never treat its own partner as "another update" and overwrite the record. +""" + +from __future__ import annotations + +import json +import signal +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from tests.e2e.core.upgrade import _helpers as H +from tests.e2e.core.upgrade import _install_helpers as I + +pytestmark = [pytest.mark.platforms("linux"), pytest.mark.live_system_guard_bypass] + +_ORPHAN = """ +import time +from hermes_cli import update_pause_record as r +r.write(r.stamp_tree({"resume_needed": True, "profiles": {"orphan": 4242}}), owner=r.identity()) +print("written", flush=True) +time.sleep(120) +""" +# The orchestrator (Tauri ``hermes-setup --update``) claims the marker, then runs the update child. +_ORCHESTRATOR = """ +import subprocess, sys +from hermes_cli.update_lock import UpdateLock +lock = UpdateLock() +assert lock.acquire() and lock.acquired +sys.exit(subprocess.call([sys.executable, "-c", sys.argv[1], sys.argv[2]])) +""" +_UPDATE_CHILD = """ +import json, sys +from hermes_cli import update_pause_record as r +from hermes_cli.update_lock import UpdateLock +lock = UpdateLock(install_root=sys.argv[1]) +adopted_claim = lock.acquire() and not lock.acquired +saw_orphan = r.orphaned_record() is not None +r.write(r.stamp_tree({"resume_needed": True, "profiles": {"mine": 1}}), owner=r.identity()) +print(json.dumps({"adopted_claim": adopted_claim, "saw_orphan": saw_orphan, + "recorded": sorted(r.read()["token"]["profiles"])})) +lock.release() +""" + + +def _py(code: str, *args: str, env: dict, cwd: Path) -> subprocess.Popen: + return subprocess.Popen([sys.executable, "-c", textwrap.dedent(code), *args], env=env, cwd=cwd, + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + +def test_update_under_a_partner_claim_merges_an_orphaned_pause(tmp_path): + origin = I.make_origin(tmp_path, I.head_sha()) + checkout = tmp_path / "checkout" + I.git("clone", "-q", "--shared", "-b", "main", str(origin), str(checkout), cwd=tmp_path) + env = H.isolated_env(tmp_path / "env", pythonpath=checkout) + + owner = _py(_ORPHAN, env=env, cwd=checkout) + assert owner.stdout.readline().strip() == "written", owner.stderr.read() + owner.send_signal(signal.SIGKILL) # windows-footgun: ok — linux-only cell + owner.wait(timeout=10) + + run = _py(_ORCHESTRATOR, textwrap.dedent(_UPDATE_CHILD), str(checkout), env=env, cwd=checkout) + out, err = run.communicate(timeout=120) + assert run.returncode == 0, f"{out}\n{err}" + result = json.loads(out.strip().splitlines()[-1]) + assert result["adopted_claim"], f"premise: the update child did not run under the orchestrator's claim: {result}" + assert result["saw_orphan"], "the update treated its own orchestrator as another update and skipped the orphan" + assert result["recorded"] == ["mine", "orphan"], f"the orphaned paused set was overwritten: {result}" diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index a60d2df7163b2..1b284c083cd20 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -74,8 +74,60 @@ def test_resume_waits_for_a_whole_tree(tmp_path): marker.unlink() _git(root, "commit", "-qam", "v2") # HEAD moved: only a dependency sync for it makes it whole + (root / "b.py").write_text("rewritten by a build step\n", encoding="utf-8") whole, why = pause_record.tree_is_whole(token, root) - assert not whole and "dependencies" in why + # A committed update is judged on its dependencies alone; a tracked file the build rewrote + # must not keep the gateways stopped on every later launch. + assert not whole and "dependencies" in why, why + + +def _orphan(tmp_path: Path, profiles: dict) -> None: + """A record whose owner — a real ``hermes update`` stand-in — was SIGKILLed after writing it.""" + owner = _child(""" + import time + from hermes_cli import update_pause_record as r + r.write(r.stamp_tree({"resume_needed": True, "profiles": %r}), owner=r.identity()) + print("written", flush=True) + time.sleep(120) + """ % profiles, env={"HERMES_HOME": str(tmp_path)}) + assert owner.stdout.readline().strip() == "written" + owner.send_signal(signal.SIGKILL) # windows-footgun: ok — module skips on Windows + owner.wait(timeout=10) + + +# The Windows resume cannot run here: the stand-in for it prints the set it was handed and +# (mode "hang") parks like a resume waiting on relaunch verification. +_RECOVER = """ + import sys, time + import hermes_cli.update_cmd_windows as w + def resume(token): + print("resume", sorted(token.get("profiles") or {}), flush=True) + if sys.argv[1] == "hang": + time.sleep(120) + token["resume_needed"] = False + w._resume_windows_gateways_after_update = resume + from hermes_cli import update_pause_record as r + r.recover(["status"]) + print("done", flush=True) +""" + + +@pytest.mark.live_system_guard_bypass +def test_a_launch_killed_mid_recovery_leaves_the_set_to_the_next_launch(tmp_path): + _orphan(tmp_path, {"default": 4242}) + env = {"HERMES_HOME": str(tmp_path)} + first = _child(_RECOVER, "hang", env=env) + try: + assert first.stdout.readline().strip() == "resume ['default']" + finally: + first.send_signal(signal.SIGKILL) # windows-footgun: ok — taskkill / console close mid-resume + first.wait(timeout=10) + second = _child(_RECOVER, "ok", env=env) + out, _ = second.communicate(timeout=60) + assert out.splitlines()[:1] == ["resume ['default']"], out + third = _child(_RECOVER, "ok", env=env) + out, _ = third.communicate(timeout=60) + assert out.strip() == "done", f"a resumed set was resumed again: {out}" _HOLDER = """ @@ -83,7 +135,7 @@ def test_resume_waits_for_a_whole_tree(tmp_path): from hermes_cli.update_lock import UpdateLock lock = UpdateLock() assert lock.acquire() and lock.acquired - host = subprocess.Popen([sys.executable, sys.argv[1], "hermes_cli.main", *sys.argv[2:]], stdout=subprocess.PIPE, text=True) + host = subprocess.Popen([sys.executable, *sys.argv[1:]], stdout=subprocess.PIPE, text=True) print(host.stdout.read().strip(), flush=True) host.wait() """ @@ -98,7 +150,13 @@ def test_resume_waits_for_a_whole_tree(tmp_path): # The intermediate is a stand-in script, not a real gateway/update: nothing touches the checkout. @pytest.mark.live_system_guard_bypass -@pytest.mark.parametrize("host_argv, adopts", [(("gateway", "run"), False), (("status",), True)]) +@pytest.mark.parametrize("host_argv, adopts", [ + (("hermes_cli.main", "gateway", "run"), False), + (("hermes_cli.main", "--profile", "work", "gateway", "run"), False), + (("/opt/hermes/hermes_cli/main.py", "-p", "work", "gateway", "run"), False), + (("-m", "hermes_cli.main", "--profile", "work", "dashboard"), False), + (("hermes_cli.main", "status"), True), +]) def test_update_started_from_a_relaunched_gateway_does_not_share_the_claim(tmp_path, host_argv, adopts): script = tmp_path / "host.py" script.write_text(_HOST, encoding="utf-8") From 7f4d3465567c9f771b6883599fdb3db4a21e9ef2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:40:33 -0700 Subject: [PATCH 05/63] =?UTF-8?q?test(update):=20wine2e=20cell=20=E2=80=94?= =?UTF-8?q?=20a=20successful=20update=20restarts=20the=20paused=20gateway?= =?UTF-8?q?=20and=20it=20outlives=20the=20updater?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cell review r1 found missing (m5/B1): after a normal Windows update the resumed gateway must still be alive 30 s after `hermes update` exits, and the record must be discharged. (cherry picked from commit 59319ec14371e88c121c98dc404a61da4ef40bac) (cherry picked from commit b3f98041c4741c60fc9dd318ed617323fd2e4ff1) --- .../core/windows_update/test_pause_durable.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/e2e/core/windows_update/test_pause_durable.py b/tests/e2e/core/windows_update/test_pause_durable.py index 2aed7efeb786b..53e91b78fe233 100644 --- a/tests/e2e/core/windows_update/test_pause_durable.py +++ b/tests/e2e/core/windows_update/test_pause_durable.py @@ -3,8 +3,9 @@ Failure class: Windows pause durability. ``hermes update`` stops every running gateway before it touches the checkout. Cell (a): the updater is killed (``taskkill /F /T`` — console closed, Desktop kill) right after the pause; the next plain ``hermes`` command must bring the gateway back. -Cell (b): the pull fails half-way (a held file makes git's fast-forward die after writing part of -the tree); the gateway must NOT be started onto that torn tree. +Cell (c): a successful update brings the paused gateway back and it outlives the updater (the +record is discharged). Cell (b): the pull fails half-way (a held file makes git's fast-forward die +after writing part of the tree); the gateway must NOT be started onto that torn tree. """ from __future__ import annotations @@ -115,6 +116,19 @@ def journey(tmp_path_factory): out["a_after"] = _running_gateway(machine, old, 120) machine.kill_owned() + # (c) a successful update: the gateway is back and still alive after the updater exits + machine.spawn_gateway() + old = int(machine.wait_gateway_running().get("pid") or 0) + out["c_pre"] = machine.installed_head() + out["c_update"] = machine.hermes("update", "--yes", label="update-ok", timeout=900) + out["c_head"] = machine.installed_head() + after = _running_gateway(machine, old, 120) + time.sleep(30) + out["c_after"] = after + out["c_alive_30s"] = bool(after) and _alive(int(after.get("pid") or 0)) + out["c_record"] = (machine.hermes_home / _RECORD).is_file() + machine.kill_owned() + # (b) the pull fails with a torn tree machine.spawn_gateway() old = int(machine.wait_gateway_running().get("pid") or 0) @@ -141,6 +155,16 @@ def test_killed_updater_gateway_resumed_by_next_launch(journey) -> None: f"stopped (pause record present after kill: {journey['a_record']})", journey["a_launch"]) +def test_successful_update_gateway_outlives_the_updater(journey) -> None: + m, run = journey["machine"], journey["c_update"] + assert run.returncode == 0 and journey["c_head"] != journey["c_pre"], fail_with( + m, f"premise: the update did not complete (rc={run.returncode}, HEAD {journey['c_pre']} -> {journey['c_head']})", run) + assert journey["c_after"] is not None, fail_with(m, "the paused gateway was not restarted by a successful update", run) + assert journey["c_alive_30s"], fail_with( + m, f"the restarted gateway (pid {journey['c_after'].get('pid')}) died after `hermes update` exited", run) + assert not journey["c_record"], fail_with(m, "a fully resumed pause left its record behind", run) + + def test_failed_pull_does_not_start_gateway_on_torn_tree(journey) -> None: m, run = journey["machine"], journey["b_update"] torn = journey["b_head"] == journey["b_pre"] and _EARLY in journey["b_status"] From 10a0357e394d8441d0ef2adc2920bc8e3658172c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:45:57 -0700 Subject: [PATCH 06/63] fix(update): a rolled-back service pause keeps the adopted orphan set owed When the service pause fails and everything this run stopped is restarted in-line, the record used to be discharged outright, dropping a set adopted from an earlier killed update. Only this run's part is discharged now; the adopted set goes back on disk unowned for the next launch. (cherry picked from commit 8f82322877ed852b16739716c2712b4c7ec7530f) (cherry picked from commit 41f91c428c79b11ef14f1a44694d0e861c07d656) --- hermes_cli/update_cmd_windows.py | 2 +- hermes_cli/update_pause_record.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 2c36161482521..7e8a9fdc126dd 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -956,7 +956,7 @@ def _pause_windows_gateways_for_update() -> dict | None: token = _pause_windows_gateway_services(service_gateways, token, profiles, unmapped) except RuntimeError as exc: if "rollback failures" not in str(exc): # everything stopped was restarted in-line - pause_record.discharge(intended) + pause_record.abandon_pause(intended, adopted) raise return pause_record.finish_pause(token, intended, adopted) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 3f0e34803418f..07ed56a2dc503 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -317,6 +317,16 @@ def finish_pause(token: dict, intended: dict, adopted: dict | None) -> dict: return token +def abandon_pause(intended: dict, adopted: dict | None) -> None: + """This run's own stops were rolled back in-line: only an adopted orphan set is still owed, + and it goes back on disk unowned for the next launch.""" + if adopted is None: + discharge(intended) + return + carried = {key: intended[key] for key in ("pause_id", "pre_sha", "dirty_at_pause") if key in intended} + write({**adopted, **carried}, owner=UNOWNED) + + def drop_never_stopped(token: dict) -> dict: """Entries whose recorded process is still the same live incarnation were never stopped.""" alive = {pid for pid, ct in (token.get("identities") or {}).items() From 1adf9dc62371eb38f705f4307e09d68b2259ef1c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:07:51 -0700 Subject: [PATCH 07/63] test(update): read the orphan verdict from a fresh process The liveness probe reads the checkout git dir; in-process it trips the home I/O guard when the checkout lives under the hermes home. (cherry picked from commit e55b619a9d59357e6e9f6390edbfc539dfc7f303) (cherry picked from commit 4d393edef07fccb33fcd1749dcc5f249698d2d7a) --- tests/hermes_cli/test_update_pause_record.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 1b284c083cd20..9888467280af9 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import signal import subprocess @@ -23,6 +24,18 @@ def _child(code: str, *argv: str, env: dict | None = None) -> subprocess.Popen: stdout=subprocess.PIPE, text=True, env={**os.environ, "PYTHONPATH": str(REPO), **(env or {})}) +def _orphaned_profiles(home: Path) -> dict | None: + """``orphaned_record()`` read by a fresh process (the liveness probe reads the checkout's git dir).""" + probe = _child(""" + import json + from hermes_cli import update_pause_record as r + body = r.orphaned_record() + print(json.dumps(None if body is None else body["token"]["profiles"])) + """, env={"HERMES_HOME": str(home)}) + out, _ = probe.communicate(timeout=60) + return json.loads(out.strip().splitlines()[-1]) + + def test_record_written_by_a_killed_updater_is_orphaned_only_after_its_death(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) owner = _child(""" @@ -37,12 +50,11 @@ def test_record_written_by_a_killed_updater_is_orphaned_only_after_its_death(tmp body = pause_record.read() assert body["owner"]["pid"] == owner.pid and body["owner"]["ct"].startswith("ct:") assert body["token"]["profiles"] == {"default": 4242} - assert pause_record.orphaned_record() is None # live owner: its own resume owns the set + assert _orphaned_profiles(tmp_path) is None # live owner: its own resume owns the set finally: owner.send_signal(signal.SIGKILL) # windows-footgun: ok — module skips on Windows owner.wait(timeout=10) - orphan = pause_record.orphaned_record() - assert orphan is not None and orphan["token"]["profiles"] == {"default": 4242} + assert _orphaned_profiles(tmp_path) == {"default": 4242} def _git(root: Path, *args: str) -> str: From 5795af57b5636955e9d948d3a1a190299786171e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:27:41 -0700 Subject: [PATCH 08/63] test(update): pause-record children lock a private checkout file The children in test_update_pause_record ran against this repo's real checkout lock, which lives in the git common dir every parallel test file (and, from a linked worktree, the live install's `hermes update`) shares. When another process held it, recovery correctly read "another update is live" and deferred, so the first launch printed `done` instead of resuming (the CI flake on test_a_launch_killed_mid_recovery_leaves_the_set_to_the_next_launch). A sitecustomize shim on the children's PYTHONPATH points only this repo's lock at a per-test file, the subprocess twin of the conftest fixture that does the same in-process. Probe: with that private lock held by another process the first launch prints ['done']; free, it prints the resume. (cherry picked from commit 654de7caf4cabcfb155613f98d8d0015204cdaab) (cherry picked from commit ea24778a1574112e5eec3922e2c4760e5d2e96fe) --- tests/hermes_cli/test_update_pause_record.py | 25 ++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 9888467280af9..2e5601fff26fa 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -19,9 +19,30 @@ pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals; Windows cells live in wine2e") -def _child(code: str, *argv: str, env: dict | None = None) -> subprocess.Popen: +# Every child (and its children) locks a private file for THIS repo's checkout, like the +# conftest fixture does in-process: the real one lives in the git common dir that parallel test +# files and, from a linked worktree, the live install's `hermes update` share. A held one reads +# as "another update is live" and recovery correctly defers to it. +_PRIVATE_CHECKOUT_LOCK = """ +import os +from pathlib import Path +from hermes_cli import update_lock as _lock +_repo, _real = Path(_lock.__file__).resolve().parents[1], _lock.checkout_lock_path +def checkout_lock_path(install_root=None): + root = Path(install_root) if install_root else _repo + return Path(os.environ["HERMES_TEST_CHECKOUT_LOCK"]) if root.resolve() == _repo else _real(install_root) +_lock.checkout_lock_path = checkout_lock_path +""" + + +def _child(code: str, *argv: str, env: dict) -> subprocess.Popen: + shim = Path(env["HERMES_HOME"]) / "checkout-lock-shim" + shim.mkdir(exist_ok=True) + (shim / "sitecustomize.py").write_text(_PRIVATE_CHECKOUT_LOCK, encoding="utf-8") + env = {**os.environ, "PYTHONPATH": os.pathsep.join((str(shim), str(REPO))), + "HERMES_TEST_CHECKOUT_LOCK": str(shim / "hermes-update.lock"), **env} return subprocess.Popen([sys.executable, "-c", textwrap.dedent(code), *argv], cwd=REPO, stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, text=True, env={**os.environ, "PYTHONPATH": str(REPO), **(env or {})}) + stdout=subprocess.PIPE, text=True, env=env) def _orphaned_profiles(home: Path) -> dict | None: From fb6d37d70dca179bb435c0cd9d2e54b6832a2771 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:12:20 -0700 Subject: [PATCH 09/63] test(update): torn-pull cell holds on a base that restores the pre-update tree The held-file cell asserted that a failed fast-forward leaves git's half-written tree behind. With the commit-point lane (#132361) the updater puts HEAD/index/files back to the pre-update commit itself, so that tree never exists and the premise failed. The premise now proves the pull failed on the held file (rc != 0 + git's unlink error). Then exactly one of two end states is accepted: - torn (HEAD == pre, git's early file dirty): gateway stays stopped, record kept; - restored verbatim (HEAD == pre, no tracked dirt, no pull marker, restore reported): the paused gateway runs again on the old code, outlives the updater, record discharged. Anything else (HEAD moved, restore claimed over a dirty tree) fails the cell. (cherry picked from commit 0b5c6b9a041bc0c389673652e98a02396dd6c4d0) (cherry picked from commit d0b04565d9633960133ba3b4f1483268e5ad51cf) --- .../core/windows_update/test_pause_durable.py | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/e2e/core/windows_update/test_pause_durable.py b/tests/e2e/core/windows_update/test_pause_durable.py index 53e91b78fe233..eb96689b38670 100644 --- a/tests/e2e/core/windows_update/test_pause_durable.py +++ b/tests/e2e/core/windows_update/test_pause_durable.py @@ -5,7 +5,10 @@ kill) right after the pause; the next plain ``hermes`` command must bring the gateway back. Cell (c): a successful update brings the paused gateway back and it outlives the updater (the record is discharged). Cell (b): the pull fails half-way (a held file makes git's fast-forward die -after writing part of the tree); the gateway must NOT be started onto that torn tree. +after writing part of the tree). Either the tree stays torn (no in-updater restore): the gateway +must NOT be started onto it and the record keeps the obligation; or the updater puts the +pre-update tree back verbatim: the paused gateway must come back on that old code and the record is +discharged. Any other end state (HEAD moved, restore claimed but tree dirty) fails the cell. """ from __future__ import annotations @@ -13,6 +16,7 @@ import subprocess import threading import time +from pathlib import Path import psutil import pytest @@ -31,6 +35,8 @@ _PAUSED = "Paused gateway profile" _RECORD = ".hermes-update-paused-gateways.json" _EARLY, _HELD = "AGENTS.md", "website/package.json" # checkout order: the early file is written first +_PULL_MARKER = "hermes-update-pull" # hermes_cli._early_recovery.INTERRUPTED_PULL_MARKER, in the git dir +_RESTORED = "Checkout restored" # the updater's own in-place restore of git's half-written files def _alive(pid: int) -> bool: @@ -138,7 +144,12 @@ def journey(tmp_path_factory): out["b_head"] = machine.installed_head() out["b_status"] = harness_git("-C", str(machine.install_dir), "status", "--porcelain", "--untracked-files=no") - out["b_after"] = _running_gateway(machine, old, 60) + git_dir = harness_git("-C", str(machine.install_dir), "rev-parse", "--absolute-git-dir") + out["b_marker"] = (Path(git_dir) / _PULL_MARKER).exists() + after = _running_gateway(machine, old, 60) + time.sleep(15 if after else 0) + out["b_after"] = after + out["b_alive_15s"] = bool(after) and _alive(int(after.get("pid") or 0)) out["b_record"] = (machine.hermes_home / _RECORD).is_file() machine.kill_owned() yield out @@ -167,11 +178,26 @@ def test_successful_update_gateway_outlives_the_updater(journey) -> None: def test_failed_pull_does_not_start_gateway_on_torn_tree(journey) -> None: m, run = journey["machine"], journey["b_update"] - torn = journey["b_head"] == journey["b_pre"] and _EARLY in journey["b_status"] - assert run.returncode != 0 and torn, fail_with( - m, f"premise: the held file did not tear the pull (rc={run.returncode}, HEAD={journey['b_head']}, " - f"pre={journey['b_pre']}, status={journey['b_status']!r})", run) - assert journey["b_after"] is None, fail_with( - m, f"the gateway was restarted onto a torn checkout ({journey['b_status']!r}) after the failed update " - f"(pid {journey['b_after'].get('pid') if journey['b_after'] else None})", run) - assert journey["b_record"], fail_with(m, "the paused set was not kept for recovery after the torn pull", run) + out = (run.stdout or "") + (run.stderr or "") + head, pre, status = journey["b_head"], journey["b_pre"], journey["b_status"] + state = f"rc={run.returncode}, HEAD={head}, pre={pre}, status={status!r}, marker={journey['b_marker']}" + # Premise on every base: git's fast-forward really died on the held file. + assert run.returncode != 0 and "unable to unlink old" in out and _HELD in out, fail_with( + m, f"premise: the held file did not fail the pull ({state})", run) + torn = head == pre and _EARLY in status + restored = head == pre and not status and not journey["b_marker"] and _RESTORED in out + assert torn or restored, fail_with( + m, f"the failed pull left neither the torn tree git wrote nor the verbatim pre-update tree ({state})", run) + pid = journey["b_after"].get("pid") if journey["b_after"] else None + if torn: + assert journey["b_after"] is None, fail_with( + m, f"the gateway was restarted onto a torn checkout ({status!r}) after the failed update (pid {pid})", run) + assert journey["b_record"], fail_with(m, "the paused set was not kept for recovery after the torn pull", run) + return + # The updater put the pre-update tree back (HEAD, index and every file at ``pre``, no pull marker): + # the paused gateway must not be lost to a failed update — it runs again on the old code. + assert journey["b_after"] is not None, fail_with( + m, f"the updater restored the pre-update tree but left the paused gateway stopped ({state})", run) + assert journey["b_alive_15s"], fail_with( + m, f"the gateway restarted on the restored tree (pid {pid}) died after `hermes update` exited", run) + assert not journey["b_record"], fail_with(m, "a fully resumed pause left its record behind", run) From 3762c71523695bc985ef16069c516cf902456c17 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:14:54 -0700 Subject: [PATCH 10/63] fix(update): conserve Windows restart debt across pause custody transfers (R7, R13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Every record/claim mutation decides read -> judge -> mutate under a kernel lock on a never-deleted sidecar (.hermes-update-paused-gateways.lock, A7). A claim is published with the claimer identity already inside (atomic replace), then its source is retired: no window where a renamed claim still names the dead claimer. - Stable obligation ids: a claim keeps its pause_id; an update folding orphaned sets lists their ids under `absorbed`. A crash between publish and retire leaves a copy that readers skip and the next mutator retires — a set is never resumed twice. - Stop acknowledgment: the update records `stop_requested` right before its first stop request. A live entry never asked is dropped (still serving); a live entry asked to stop is draining and keeps its restart debt until it has exited (the claim goes back unowned meanwhile). - One record per checkout (file name keyed by install root): another checkout sharing the home never imports, relabels or certifies this checkout's paused set. - Resume retires each runtime on its own readiness (profile home / unmapped argv), and a failed SCM service or profile never keeps the independent ones stopped. (cherry picked from commit 4301239b4ae702cf619cc90ddd3bb605159db94e) (cherry picked from commit c54cb419ae943694883ab829f274b168772e13ca) --- hermes_cli/update_cmd_windows.py | 165 +++++---- hermes_cli/update_pause_record.py | 345 ++++++++++++++----- tests/hermes_cli/test_update_pause_record.py | 194 +++++++++++ 3 files changed, 549 insertions(+), 155 deletions(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 7e8a9fdc126dd..e2c3fc1ab5822 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -946,6 +946,9 @@ def _pause_windows_gateways_for_update() -> dict | None: service_profiles={str(s.name): str(s.profile) for s in service_gateways}) with _abort_on_error("Could not record the gateways this update pauses"): pause_record.record_pause(intended, adopted, claims) + # A second write, right before the first request: a crash before it leaves every live entry + # "never asked" (still serving, dropped); after it, a live one is draining and stays owed. + pause_record.mark_stop_requested(intended, running_pids) profiles = _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped) token = {"resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped} # Every profile with ANY live gateway at discovery counts as running: service-supervised ones skip the @@ -1226,49 +1229,30 @@ def _resume_windows_services(token: dict) -> None: print("\n ✓ Restarted Windows gateway service(s): " + ", ".join(restarted_services)) -def _relaunch_paused_gateways(token: dict, profiles: dict, unmapped: list) -> tuple[list[str], int]: - """Relaunch profile gateways and replay unmapped argv; ``(relaunched_profiles, unmapped_count)``. - - Failed relaunches stay on the token (and off ``relaunched_profiles``) so plan-vs-execution - reconciliation still surfaces them — Windows has no watcher to recover them.""" +def _relaunch_paused_gateways(profiles: dict, unmapped: list) -> tuple[dict, list]: + """Spawn the restart watcher for each paused profile gateway and unmapped argv; ``(launched + profiles, launched unmapped entries)``. Nothing leaves the token here: a watcher only proves a + relaunch was scheduled, so every entry stays owed until ITS gateway is verified ready.""" with _abort_on_error("Could not load Windows gateway restart helper"): from hermes_cli.gateway import launch_detached_gateway_restart_by_cmdline, launch_detached_profile_gateway_restart # An exception from a launch (incl. bad pid/argv coercion) logs at debug and reads as a failed relaunch. - relaunched = [] - failed_profiles = {} + launched = {} for profile, old_pid in sorted(profiles.items()): if _try_call(lambda p=profile, o=old_pid: launch_detached_profile_gateway_restart(str(p), int(o)), "Could not restart Windows gateway profile %s after update: %s", profile): - relaunched.append(str(profile)) - else: - failed_profiles[str(profile)] = int(old_pid) - # Surface the outcome on the token (#91277 Phase 2 plan-vs-execution reconciliation): the git-based - # update path's fleet reconciliation cross-checks every planned runtime against restarted_services / - # relaunched_profiles / externally_supervised_profiles / killed_pids — bookkeeping this Windows-specific - # pause/resume never fed, so a correctly-paused-and-relaunched Windows gateway was reported - # "unaccounted" (loud warning + exit 1) even though the restart succeeded. The caller merges this into - # the shared relaunched_profiles list before reconciliation runs. A profile whose relaunch genuinely - # failed is deliberately left off this list — it must still surface as unaccounted so the user is told - # to restart it manually (Windows has no watcher to recover a failed relaunch). - token["relaunched_profiles"] = relaunched - unmapped_relaunched = 0 - failed_unmapped = [] + launched[str(profile)] = int(old_pid) + launched_unmapped = [] for entry in unmapped: argv, old_pid = entry.get("argv"), entry.get("pid") if argv and old_pid and _try_call(lambda o=old_pid, a=argv: launch_detached_gateway_restart_by_cmdline(int(o), list(a)), "Could not restart unmapped Windows gateway (pid %s) after update: %s", old_pid): - unmapped_relaunched += 1 - else: - failed_unmapped.append(entry) - token["profiles"] = failed_profiles - token["unmapped"] = failed_unmapped - if failed_profiles or failed_unmapped: - raise RuntimeError("Could not restart every paused Windows gateway") - return relaunched, unmapped_relaunched + launched_unmapped.append(entry) + return launched, launched_unmapped _RELAUNCH_VERIFY_TIMEOUT_S = 30.0 +_PER_TARGET_FLOOR_S = 6.0 def _pending_relaunch_pids(profiles: dict, unmapped: list, pid_exists) -> list[int]: @@ -1297,30 +1281,75 @@ def _relaunch_verify_timeout_s(profiles: dict, unmapped: list, pid_exists) -> fl return float(GATEWAY_RESTART_WATCHER_TIMEOUT_S) + _RELAUNCH_VERIFY_TIMEOUT_S +def _unmapped_ready_filter(entry: dict, taken: set): + """``pid_filter`` matching the gateway an unmapped entry's argv replay started: same arguments + after the (normalized) interpreter, not the old pid, not one another entry already vouched for.""" + import psutil + tail, old = list(entry.get("argv") or [])[1:], int(entry.get("pid") or 0) + + def matches(pid: int) -> bool: + if pid == old or pid in taken: + return False + try: + return list(psutil.Process(pid).cmdline() or [])[1:] == tail + except Exception: + return False + return lambda pids: [pid for pid in _owned_gateway_pids(pids) if matches(int(pid))] + + def _verify_relaunched_gateways_alive(token: dict, profiles: dict, unmapped: list) -> None: - """Gate success on the shared liveness poll: a truthy launch only proves the watcher was created. + """Retire each relaunched runtime on ITS OWN readiness: a profile on a stable gateway in its + profile home, an unmapped entry on a stable gateway running its argv. A watcher only proves a + relaunch was scheduled, and another target's readiness proves nothing about this one. - A parent Job Object denying CREATE_BREAKAWAY_FROM_JOB can kill the gateway on updater teardown; - ``all_profiles=True`` covers the fleet. Vouched PIDs are persisted so a death AFTER updater exit - is reported by the next CLI invocation (best-effort).""" + A parent Job Object denying CREATE_BREAKAWAY_FROM_JOB can kill the gateway on updater teardown. + Vouched PIDs are persisted so a death AFTER updater exit is reported by the next CLI invocation.""" with _abort_on_error("Could not load Windows gateway liveness helpers"): from gateway.status import _pid_exists from hermes_cli import gateway_windows - timeout_s = _relaunch_verify_timeout_s(profiles, unmapped, _pid_exists) - ready_pids = gateway_windows._wait_for_gateway_ready( - timeout_s=timeout_s, all_profiles=True, pid_filter=_owned_gateway_pids - ) - if not ready_pids: - token["profiles"] = dict(profiles) - token["unmapped"] = list(unmapped) + from hermes_cli.profiles import get_profile_dir + deadline = _time.monotonic() + _relaunch_verify_timeout_s(profiles, unmapped, _pid_exists) + + def budget() -> float: + return max(deadline - _time.monotonic(), _PER_TARGET_FLOOR_S) + + ready_pids: list[int] = [] + ready_profiles = [] + for name, old_pid in sorted(profiles.items()): + pids = _try_call(lambda n=name, o=old_pid: gateway_windows._wait_for_gateway_ready( + timeout_s=budget(), home=Path(get_profile_dir(n)), pid_filter=lambda ps, o=o: [p for p in ps if int(p) != int(o)]), + "Could not verify Windows gateway profile %s after update: %s", name, default=[]) + if pids: + ready_profiles.append(name) + ready_pids.extend(pids) + ready_unmapped = [] + for entry in unmapped: + pids = _try_call(lambda e=entry: gateway_windows._wait_for_gateway_ready( + timeout_s=budget(), all_profiles=True, pid_filter=_unmapped_ready_filter(e, set(ready_pids))), + "Could not verify unmapped Windows gateway (pid %s) after update: %s", entry.get("pid"), default=[]) + if pids: + ready_unmapped.append(entry) + ready_pids.extend(pids[:1]) + # Fleet reconciliation (#91277) cross-checks every planned runtime against relaunched_profiles: + # only a verified one is listed, so an unready one still surfaces as unaccounted. + token["relaunched_profiles"] = ready_profiles + token["profiles"] = {n: pid for n, pid in (token.get("profiles") or {}).items() if n not in ready_profiles} + token["unmapped"] = [u for u in (token.get("unmapped") or []) if u not in ready_unmapped] + if ready_pids: + with suppress(Exception): + gateway_windows._write_start_attestation(sorted(set(ready_pids)), "post-update relaunch") + if ready_profiles: + print(f"\n ✓ Restarting Windows gateway profile(s): {', '.join(ready_profiles)}") + if ready_unmapped: + print(f" ✓ Restarting {len(ready_unmapped)} unmapped Windows gateway process(es)") + unready = sorted(set(profiles) - set(ready_profiles)) + [f"pid {u.get('pid')}" for u in unmapped if u not in ready_unmapped] + if unready: print( - "\n ⚠ Windows gateway restart could not be verified — no stable gateway process appeared after relaunch.\n" + "\n ⚠ Windows gateway restart could not be verified for: " + ", ".join(unready) + "\n" " (The respawned gateway may have been killed by a parent Job Object during updater teardown, #48820.)\n" " Recover with: hermes gateway restart" ) - raise RuntimeError("Windows gateway relaunch after update was not verified alive") - with suppress(Exception): - gateway_windows._write_start_attestation(ready_pids, "post-update relaunch") + raise RuntimeError("Windows gateway relaunch after update was not verified alive: " + ", ".join(unready)) def _resume_windows_gateways_after_update(token: dict | None) -> None: @@ -1359,33 +1388,45 @@ def _resume_windows_gateways_after_update(token: dict | None) -> None: def _resume_paused_set(token: dict) -> None: + """Bring every paused runtime back; each one is retired from *token* only on its own verified + start, and one failure (an SCM service, a profile) never keeps the others stopped.""" from hermes_cli.update_cmd import _m # Regenerate launcher scripts before respawning so a legacy pythonw-era # autostart entry comes back on the current design at next login too. _m()._refresh_windows_gateway_launchers() - _resume_windows_services(token) - profiles = token.get("profiles") or {} - unmapped = token.get("unmapped") or [] - if not profiles and not any(u.get("argv") for u in unmapped): - if token.get("cold_start_if_installed"): - # Before the per-profile spawns: this guard is fleet-wide (any live gateway ⇒ done). + failures: list[str] = [] + + def attempt(step) -> None: + try: + step() + except Exception as exc: + failures.append(str(exc)) + + attempt(lambda: _resume_windows_services(token)) + profiles = dict(token.get("profiles") or {}) + unmapped = list(token.get("unmapped") or []) + if profiles or any(u.get("argv") for u in unmapped): + launched, launched_unmapped = _relaunch_paused_gateways(profiles, unmapped) + if launched or launched_unmapped: + attempt(lambda: _verify_relaunched_gateways_alive(token, launched, launched_unmapped)) + else: + token["relaunched_profiles"] = [] + not_launched = sorted(set(profiles) - set(launched)) + [ + f"pid {u.get('pid')}" for u in unmapped if u not in launched_unmapped] + if not_launched: + failures.append("Could not restart every paused Windows gateway: " + ", ".join(not_launched)) + elif token.get("cold_start_if_installed"): + # Before the per-profile spawns: this guard is fleet-wide (any live gateway ⇒ done). + def cold_start() -> None: if not _m()._cold_start_windows_gateway_after_update(token): raise RuntimeError("Windows gateway cold-start was not verified") token["cold_start_if_installed"] = False - _cold_start_attested_profiles(token) - token["resume_needed"] = False - return - relaunched, unmapped_relaunched = _relaunch_paused_gateways(token, profiles, unmapped) - if relaunched or unmapped_relaunched: - _verify_relaunched_gateways_alive(token, profiles, unmapped) - if relaunched: - print(f"\n ✓ Restarting Windows gateway profile(s): {', '.join(relaunched)}") - if unmapped_relaunched: - lead = "" if relaunched else "\n" - print(f"{lead} ✓ Restarting {unmapped_relaunched} unmapped Windows gateway process(es)") + attempt(cold_start) # After the paused profiles are back: a dead-attested sibling that fails to cold-start must not # keep the profiles that WERE running from being relaunched. - _cold_start_attested_profiles(token) + attempt(lambda: _cold_start_attested_profiles(token)) + if failures: + raise RuntimeError("; ".join(failures)) token["resume_needed"] = False diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 07ed56a2dc503..5e2ee0c196ed1 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -7,41 +7,58 @@ identity format), is rewritten as the token changes, and is removed only after a verified resume. A later launch whose owner check finds the updater dead (and no update live) resumes it. +One record per checkout (the file name carries a key of the install root): a checkout sharing +the home never imports, relabels or certifies another checkout's paused set — its tree gate +says nothing about the other tree. + Resuming is gated on a whole tree: no interrupted-pull marker; at the pre-update HEAD, no tracked change beyond the ones present at pause time (git died before moving HEAD); at a moved HEAD, dependencies current for it (a build step may rewrite tracked files there). Otherwise the record stays for the next launch, which runs after the interrupted-pull restore and the dependency sync. -A recovering launch claims the record by renaming it to ``...claim`` and names -itself inside; a claim whose claimer died (killed mid-resume) is re-adopted like a dead owner's -record. ``write`` never overwrites another pause: an orphan's set is merged in, a live one refuses. +Custody: every mutation (write, claim, retire, discharge) happens while holding a kernel lock on +``/.hermes-update-paused-gateways.lock`` (flock / msvcrt, released by the kernel when +the holder dies; the file is never deleted), and the read → judge → mutate decision is made +inside that hold. A recovering launch claims a record by publishing ``...claim`` +with its own identity already inside (atomic replace), then retires the source; a crash between the +two leaves two files with ONE obligation id (``pause_id``), and an update that folds orphaned sets +into its own record lists their ids under ``absorbed`` before retiring them. Readers treat a file +whose id another file carries as a copy, never as a second obligation, and the next mutator +retires it. """ from __future__ import annotations import glob +import hashlib import json import os import subprocess import sys +import time import uuid -from contextlib import suppress +from contextlib import contextmanager, suppress from pathlib import Path -RECORD_NAME = ".hermes-update-paused-gateways.json" -# Same tolerance as the update marker's creation-time identity (C1 rule 3). -_CT_TOLERANCE_S = 2.0 - - -def record_path() -> Path: - from hermes_constants import get_default_hermes_root - return get_default_hermes_root() / RECORD_NAME +RECORD_STEM = ".hermes-update-paused-gateways" +MUTEX_NAME = RECORD_STEM + ".lock" +_MUTEX_WAIT_S = 10.0 +_mutex_depth = 0 def install_root() -> Path: return Path(__file__).resolve().parents[1] +def _install_key(root: Path | None = None) -> str: + return hashlib.sha256(os.path.normcase(str(root or install_root())).encode("utf-8")).hexdigest()[:12] + + +def record_path() -> Path: + from hermes_constants import get_default_hermes_root + return get_default_hermes_root() / f"{RECORD_STEM}.{_install_key()}.json" + + def _create_time(pid: int) -> float | None: from hermes_cli.process_identity import _process_create_time return _process_create_time(pid) @@ -55,24 +72,57 @@ def identity(pid: int | None = None) -> dict: def identity_is_live(ident: dict | None) -> bool: - """Alive (not a zombie) and, when a creation time was recorded, the same incarnation.""" - from hermes_cli._early_recovery import _pid_is_running + """The update marker's incarnation rule (``update_lock``), applied to a recorded identity.""" + from hermes_cli import update_lock try: pid = int((ident or {}).get("pid") or 0) except (TypeError, ValueError): return False - if pid <= 0 or not _pid_is_running(pid): - return False recorded = str((ident or {}).get("ct") or "") - if not recorded.startswith("ct:"): - return True - actual = _create_time(pid) - if actual is None: - return True try: - return abs(float(recorded[3:]) - actual) <= _CT_TOLERANCE_S + created = float(recorded[3:]) if recorded.startswith("ct:") else None except ValueError: - return True + created = None + return pid > 0 and update_lock._identity_live(pid, created, 0.0) + + +class RecordConflict(OSError): + """The record names another pause whose owner is still live.""" + + +class RecordBusy(OSError): + """Another process held the record mutex past the bounded wait.""" + + +@contextmanager +def _mutex(): + """Exclusive kernel lock on the record directory's sidecar (A7). Re-entrant in-process.""" + global _mutex_depth + if _mutex_depth: + _mutex_depth += 1 + try: + yield + finally: + _mutex_depth -= 1 + return + from hermes_cli import update_lock + path = record_path().with_name(MUTEX_NAME) + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o644) + try: + deadline = time.monotonic() + _MUTEX_WAIT_S + while not update_lock._try_lock(fd): + if time.monotonic() > deadline: + raise RecordBusy(f"{path} is held by another process") + time.sleep(0.05) + _mutex_depth = 1 + try: + yield + finally: + _mutex_depth = 0 + update_lock._unlock(fd) + finally: + os.close(fd) def _git(root: Path, *args: str) -> subprocess.CompletedProcess | None: @@ -174,32 +224,39 @@ def read(path: Path | None = None) -> dict | None: UNOWNED = {"pid": 0, "ct": None} -class RecordConflict(OSError): - """The record names another pause whose owner is still live.""" - - def write(token: dict, *, owner: dict | None = None, path: Path | None = None) -> None: """Persist *token*. The owner of the same pause is kept; another pause's orphaned set is merged into *token* (in place, so this run's resume brings it back); a live one refuses.""" path = path or record_path() - existing = read(path) - same = existing is not None and existing["token"].get("pause_id") == token.get("pause_id") - if existing is not None and not same: - if identity_is_live(existing.get("owner")): - raise RecordConflict(f"{path} holds gateways paused by live process {existing['owner'].get('pid')}") - merge_into(token, drop_never_stopped(dict(existing["token"]))) - if owner is None: - owner = existing["owner"] if same else identity() - _atomic_write(path, {"schema": 1, "owner": owner, "install_root": str(install_root()), "token": token}) + with _mutex(): + existing = read(path) + same = existing is not None and existing["token"].get("pause_id") == token.get("pause_id") + if existing is not None and not same: + if existing.get("install_root") != str(install_root()): + raise RecordConflict(f"{path} holds gateways paused for another checkout ({existing.get('install_root')})") + if identity_is_live(existing.get("owner")): + raise RecordConflict(f"{path} holds gateways paused by live process {existing['owner'].get('pid')}") + merge_into(token, drop_never_stopped(dict(existing["token"]))) + if owner is None: + owner = existing["owner"] if same and existing is not None else identity() + _atomic_write(path, {"schema": 1, "owner": owner, "install_root": str(install_root()), "token": token}) + + +def mark_stop_requested(token: dict, pids) -> None: + """Persist, BEFORE the first stop request, that these processes are being asked to stop: one + that acknowledged and is still draining keeps its restart obligation after a crash.""" + token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *(str(int(p)) for p in pids)}) + write(token, owner=identity()) def discharge(token: dict, path: Path | None = None) -> None: - """Compare-and-delete: remove the record only while it still names this pause.""" + """Remove the record only while it still names this pause (decided under the mutex).""" path = path or record_path() - existing = read(path) - if existing is not None and existing["token"].get("pause_id") == token.get("pause_id"): - with suppress(OSError): - path.unlink() + with _mutex(): + existing = read(path) + if existing is not None and existing["token"].get("pause_id") == token.get("pause_id"): + with suppress(OSError): + path.unlink() def sync(token: dict) -> None: @@ -229,27 +286,57 @@ def _claims(path: Path) -> list[Path]: return sorted(path.parent.glob(f"{glob.escape(path.name)}.*.claim")) -def _claimer(claim: Path, body: dict) -> dict: - """Who holds a claim: the identity it wrote, else (killed before writing it) its name's pid.""" - if isinstance(body.get("claimer"), dict): - return body["claimer"] - pid = claim.name[len(RECORD_NAME) + 1:].split(".", 1)[0] - return {"pid": int(pid) if pid.isdigit() else 0, "ct": None} +def _holder(src: Path, body: dict) -> dict: + """Who holds *src*: a record's owner, a claim's claimer (published with the claim itself).""" + held_by = body.get("owner") if src.suffix == ".json" else body.get("claimer") + return held_by if isinstance(held_by, dict) else UNOWNED + + +def _files(path: Path) -> list[tuple[Path, dict]]: + """This checkout's record and claims, readable ones only.""" + found = [] + for src in (path, *_claims(path)): + body = read(src) + if body is not None and body.get("install_root") == str(install_root()): + found.append((src, body)) + return found + + +def _redundant(found: list[tuple[Path, dict]], held: set[Path]) -> set[Path]: + """Files that only copy an obligation another file carries: a source an update absorbed but + died before retiring, or one of two same-id files a claim transfer left. Never a held file.""" + carriers: dict[str, Path] = {} + for src, body in found: + for oid in body["token"].get("absorbed") or []: + carriers.setdefault(str(oid), src) + redundant: set[Path] = set() + by_id: dict[str, list[Path]] = {} + for src, body in found: + oid = str(body["token"].get("pause_id") or "") + if not oid: + continue + if oid in carriers and carriers[oid] != src: + redundant.add(src) + else: + by_id.setdefault(oid, []).append(src) + for copies in by_id.values(): + keep = next((s for s in copies if s in held), copies[0]) + redundant.update(s for s in copies if s != keep) + return redundant - held + + +def _survey(path: Path) -> tuple[list[tuple[Path, dict]], set[Path], set[Path]]: + found = _files(path) + held = {src for src, body in found if identity_is_live(_holder(src, body))} + return found, held, _redundant(found, held) def orphans(path: Path | None = None) -> list[tuple[Path, dict]]: - """``(file, body)`` for this install's record whose owner is dead and every claim whose claimer - is dead — empty while another update is live (it adopts them itself).""" + """``(file, body)`` for every obligation of this checkout whose holder is dead — one file per + obligation id; empty while another update is live (it adopts them itself).""" path = path or record_path() - found = [] - body = read(path) - if body is not None and not identity_is_live(body.get("owner")): - found.append((path, body)) - for claim in _claims(path): - claimed = read(claim) - if claimed is not None and not identity_is_live(_claimer(claim, claimed)): - found.append((claim, claimed)) - found = [(src, b) for src, b in found if b.get("install_root") == str(install_root())] + found, held, redundant = _survey(path) + found = [(src, body) for src, body in found if src not in held and src not in redundant] if not found or _live_update_elsewhere(): return [] return found @@ -260,28 +347,45 @@ def orphaned_record(path: Path | None = None) -> dict | None: return found[0][1] if found else None +def retire_redundant(path: Path | None = None) -> None: + """Delete copies of obligations another file already carries (crash leftovers of a transfer).""" + path = path or record_path() + if not path.exists() and not _claims(path): + return + with suppress(RecordBusy), _mutex(): + _found, _held, redundant = _survey(path) + for src in redundant: + with suppress(OSError): + src.unlink() + + def claim(src: Path) -> tuple[Path, dict] | None: """Take *src* (an orphaned record or a dead claimer's claim) for this process; ``None`` when a - concurrent launch won it. The rename is the arbiter; the claimer line names our incarnation.""" + concurrent launch won it or holds the mutex. The claim is published with our identity inside, + then the source is retired; a crash between leaves a same-id copy :func:`_redundant` drops.""" path = record_path() mine = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.claim") try: - os.rename(src, mine) + with _mutex(): + body = read(src) + if body is None or body.get("install_root") != str(install_root()): + return None + _found, held, redundant = _survey(path) + if src in held or src in redundant or src not in {s for s, _ in _found}: + return None + body["claimer"] = identity() + _atomic_write(mine, body) + with suppress(OSError): + src.unlink() except OSError: return None - body = read(mine) - if body is None: - with suppress(OSError): - mine.unlink() - return None - body["claimer"] = identity() - _atomic_write(mine, body) return mine, body def adopt_orphans() -> tuple[dict | None, list[Path]]: """For ``hermes update``: claim every orphaned pause and merge their sets. The caller records the - merged set durably, then :func:`release_claims` — a crash in between leaves claims to re-adopt.""" + merged set durably (its ``absorbed`` ids name the claims), then :func:`release_claims`.""" + retire_redundant() adopted, claims = None, [] for src, _body in orphans(): won = claim(src) @@ -293,26 +397,32 @@ def adopt_orphans() -> tuple[dict | None, list[Path]]: def release_claims(claims: list[Path]) -> None: - for path in claims: - with suppress(OSError): - path.unlink() + with suppress(RecordBusy), _mutex(): + for path in claims: + with suppress(OSError): + path.unlink() def record_pause(token: dict, adopted: dict | None, claims: list[Path]) -> dict: - """Merge the adopted set, stamp the tree and persist under this process BEFORE the first stop.""" + """Merge the adopted set, stamp the tree and persist under this process BEFORE the first stop. + Publish then retire: a crash between leaves claims whose ids the record lists as absorbed.""" if adopted is not None: merge_into(token, adopted) stamp_tree(token) + token.setdefault("stop_requested", []) write(token, owner=identity()) release_claims(claims) return token +_CARRIED = ("pause_id", "pre_sha", "dirty_at_pause", "identities", "stop_requested", "absorbed") + + def finish_pause(token: dict, intended: dict, adopted: dict | None) -> dict: """Carry the recorded pause identity onto the final token and mirror it to disk.""" if adopted is not None: merge_into(token, adopted) - token.update({key: intended[key] for key in ("pause_id", "pre_sha", "dirty_at_pause", "identities") if key in intended}) + token.update({key: intended[key] for key in _CARRIED if key in intended}) sync(token) return token @@ -323,28 +433,60 @@ def abandon_pause(intended: dict, adopted: dict | None) -> None: if adopted is None: discharge(intended) return - carried = {key: intended[key] for key in ("pause_id", "pre_sha", "dirty_at_pause") if key in intended} + carried = {key: intended[key] for key in ("pause_id", "pre_sha", "dirty_at_pause", "absorbed") if key in intended} write({**adopted, **carried}, owner=UNOWNED) -def drop_never_stopped(token: dict) -> dict: - """Entries whose recorded process is still the same live incarnation were never stopped.""" - alive = {pid for pid, ct in (token.get("identities") or {}).items() - if identity_is_live({"pid": pid, "ct": ct})} - token["profiles"] = {p: pid for p, pid in (token.get("profiles") or {}).items() if str(pid) not in alive} - token["unmapped"] = [u for u in (token.get("unmapped") or []) if str(u.get("pid")) not in alive] +def _live_pids(token: dict) -> set[str]: + return {str(pid) for pid, ct in (token.get("identities") or {}).items() + if identity_is_live({"pid": pid, "ct": ct})} + + +def _without(token: dict, pids: set[str]) -> dict: + token["profiles"] = {p: pid for p, pid in (token.get("profiles") or {}).items() if str(pid) not in pids} + token["unmapped"] = [u for u in (token.get("unmapped") or []) if str(u.get("pid")) not in pids] return token +def drop_never_stopped(token: dict) -> dict: + """Drop entries whose recorded process is the same live incarnation AND was never asked to stop + (the update died before its first stop request). One it asked to stop may be draining: it keeps + its restart debt (:func:`split_draining`). No stop record at all: nothing can be told apart.""" + if token.get("stop_requested") is None: + return token + return _without(token, _live_pids(token) - {str(p) for p in token["stop_requested"]}) + + +def split_draining(token: dict) -> dict: + """Remove and return ``{"profiles", "unmapped"}`` whose recorded process is still running: it was + asked to stop and has not exited yet, so it can only be restarted once it has.""" + live = _live_pids(token) + draining = {"profiles": {p: pid for p, pid in (token.get("profiles") or {}).items() if str(pid) in live}, + "unmapped": [u for u in (token.get("unmapped") or []) if str(u.get("pid")) in live]} + _without(token, live) + return draining + + def merge_into(token: dict | None, adopted: dict) -> dict: - """Fold an orphaned pause into this update's token so its resume brings both sets back.""" + """Fold an orphaned pause into this update's token so its resume brings both sets back; its + obligation id (and every id it absorbed) is recorded so a leftover copy is never resumed twice.""" token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []} token["resume_needed"] = True + absorbed = token.setdefault("absorbed", []) + absorbed.extend(i for i in [adopted.get("pause_id"), *(adopted.get("absorbed") or [])] if i and i not in absorbed) profiles = token.setdefault("profiles", {}) for name, pid in (adopted.get("profiles") or {}).items(): profiles.setdefault(name, pid) unmapped = token.setdefault("unmapped", []) unmapped.extend(u for u in adopted.get("unmapped") or [] if u not in unmapped) + identities = token.setdefault("identities", {}) + for pid, ct in (adopted.get("identities") or {}).items(): + identities.setdefault(pid, ct) + if "stop_requested" in adopted or "stop_requested" in token: + requested = adopted.get("stop_requested") + if requested is None: # a set with no stop record keeps every live entry: all count as asked + requested = list(adopted.get("identities") or {}) + token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *map(str, requested)}) services = token.setdefault("services", []) services.extend(s for s in adopted.get("services") or [] if s not in services) if services: @@ -354,26 +496,42 @@ def merge_into(token: dict | None, adopted: dict) -> dict: return token +def _has_work(token: dict) -> bool: + return bool(token.get("profiles") or any(u.get("argv") for u in token.get("unmapped") or []) + or token.get("services") or token.get("cold_start_if_installed") or token.get("cold_start_profiles")) + + def _resume_claimed(claim_path: Path, body: dict) -> None: token = drop_never_stopped(dict(body["token"])) + draining = split_draining(token) token.update(resume_needed=True, recovery=True) - print("→ Restarting gateway(s) paused by an interrupted `hermes update`...", file=sys.stderr) try: - from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update - _resume_windows_gateways_after_update(token) + if _has_work(token): + print("→ Restarting gateway(s) paused by an interrupted `hermes update`...", file=sys.stderr) + from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update + _resume_windows_gateways_after_update(token) + else: + token["resume_needed"] = False except Exception as exc: print(f" ⚠ Could not restart every paused gateway: {exc}. Run `hermes update` or " "`hermes gateway start`.", file=sys.stderr) finally: - with suppress(OSError): - if token.get("resume_needed") or token.get("resume_deferred"): - # Still owed: hand the claim back unowned so the next launch retries at once — this - # launch may live for hours (a chat). A failed rewrite leaves our claimer line, which - # turns dead (re-adoptable) when we exit. - owed = {key: value for key, value in token.items() if key not in ("recovery", "resume_deferred")} - _atomic_write(claim_path, {**body, "token": {**owed, "resume_needed": True}, "claimer": UNOWNED}) - else: - claim_path.unlink() + _hand_back(claim_path, body, token, draining) + + +def _hand_back(claim_path: Path, body: dict, token: dict, draining: dict) -> None: + """Retire the claim when nothing is owed; else hand it back unowned so the next launch retries + at once — this launch may live for hours (a chat). A draining process stays owed until it has + exited and been restarted. A failed rewrite leaves our claimer line, dead once we exit.""" + token["profiles"] = {**(token.get("profiles") or {}), **draining["profiles"]} + token["unmapped"] = [*(token.get("unmapped") or []), *draining["unmapped"]] + owed = bool(token.get("resume_needed") or token.get("resume_deferred") or draining["profiles"] or draining["unmapped"]) + with suppress(OSError), _mutex(): + if owed: + kept = {key: value for key, value in token.items() if key not in ("recovery", "resume_deferred")} + _atomic_write(claim_path, {**body, "token": {**kept, "resume_needed": True}, "claimer": UNOWNED}) + else: + claim_path.unlink() def recover(argv: list[str] | None = None) -> None: @@ -383,6 +541,7 @@ def recover(argv: list[str] | None = None) -> None: if command[:1] == ["update"] or command[:2] == ["gateway", "run"]: return # the update adopts it itself; a booting gateway must not block on its siblings try: + retire_redundant() for src, _body in orphans(): won = claim(src) if won is not None: diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 2e5601fff26fa..2d17fbd563035 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -197,3 +197,197 @@ def test_update_started_from_a_relaunched_gateway_does_not_share_the_claim(tmp_p out, _ = holder.communicate(timeout=60) assert holder.returncode == 0, out assert out.strip() == ("True False" if adopts else "False True"), out + + +# --- R7: restart debt is conserved across custody transfers ------------------------------------ +# A child stops ITSELF at the named line of the real module (settrace), so a kill or a rival lands +# exactly on the transfer boundary; nothing in the module under test is replaced. +_AT_LINE = """ + import inspect, json, os, sys, time + from pathlib import Path + from hermes_cli import update_pause_record as r + def stop_at(fn, text, action, flag=None): + lines, start = inspect.getsourcelines(fn) + line = start + next(i for i, t in enumerate(lines) if text in t) + def trace(frame, event, arg): + if event == "line" and frame.f_code is fn.__code__ and frame.f_lineno == line: + if action == "kill": + os._exit(71) + print("paused", flush=True) + while not Path(flag).exists(): + time.sleep(0.01) + return trace + sys.settrace(trace) +""" +_RESUMES = """ + import hermes_cli.update_cmd_windows as w + def resume(token): + print("resume", sorted(token.get("profiles") or {}), flush=True) + token["resume_needed"] = False + w._resume_windows_gateways_after_update = resume + r.recover(["status"]) + print("done", flush=True) +""" + + +def _launches(tmp_path: Path, n: int) -> list[list[str]]: + """What each of *n* successive fresh launches resumed.""" + seen = [] + for _ in range(n): + out, _ = _child(_AT_LINE + _RESUMES, env={"HERMES_HOME": str(tmp_path)}).communicate(timeout=60) + seen.append([line for line in out.splitlines() if line.startswith("resume")]) + return seen + + +def _record_files(tmp_path: Path) -> list[str]: + return sorted(p.name for p in tmp_path.iterdir() if p.name.startswith(pause_record.RECORD_STEM) + and p.suffix in (".json", ".claim")) + + +@pytest.mark.live_system_guard_bypass +def test_a_claim_in_transfer_cannot_be_taken_by_a_second_launch(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(pause_record, "_MUTEX_WAIT_S", 0.5) + _orphan(tmp_path, {"default": 4242}) + src = pause_record.record_path() + flag = tmp_path / "go" + first = _child(_AT_LINE + """ + stop_at(r.claim, "src.unlink()", "pause", sys.argv[2]) + won = r.claim(Path(sys.argv[1])) + sys.settrace(None) + print(json.dumps(won and str(won[0])), flush=True) + """, str(src), str(flag), env={"HERMES_HOME": str(tmp_path)}) + try: + assert first.stdout.readline().strip() == "paused" + rivals = [pause_record.claim(p) for p in (src, *pause_record._claims(src))] + assert rivals == [None] * len(rivals), "a second launch took a claim whose transfer was in flight" + finally: + flag.touch() + won = json.loads(first.stdout.readline()) + first.wait(timeout=30) + assert _record_files(tmp_path) == [Path(won).name], "the paused set is now carried by two files" + + +@pytest.mark.live_system_guard_bypass +def test_a_launch_killed_between_claim_and_retire_resumes_the_set_once(tmp_path): + _orphan(tmp_path, {"default": 4242}) + killed = _child(_AT_LINE + """ + stop_at(r.claim, "src.unlink()", "kill") + r.recover(["status"]) + """, env={"HERMES_HOME": str(tmp_path)}) + killed.wait(timeout=60) + assert killed.returncode == 71 and len(_record_files(tmp_path)) == 2, "premise: killed after publishing the claim" + assert _launches(tmp_path, 2) == [["resume ['default']"], []] + assert _record_files(tmp_path) == [] + + +@pytest.mark.live_system_guard_bypass +def test_an_update_killed_after_publishing_its_record_never_restarts_a_set_twice(tmp_path): + _orphan(tmp_path, {"default": 4242}) + killed = _child(_AT_LINE + """ + adopted, claims = r.adopt_orphans() + stop_at(r.record_pause, "release_claims(claims)", "kill") + r.record_pause({"resume_needed": True, "profiles": {"beta": 99}}, adopted, claims) + """, env={"HERMES_HOME": str(tmp_path)}) + killed.wait(timeout=60) + assert killed.returncode == 71 and len(_record_files(tmp_path)) == 2, "premise: killed before retiring the claim" + assert _launches(tmp_path, 2) == [["resume ['beta', 'default']"], []] + assert _record_files(tmp_path) == [] + + +_DRAINING = """ + import os, signal, sys, time + from pathlib import Path + flag = Path(sys.argv[1]) + def stop(*_): + print("stopping", flush=True) # acknowledged; drains until the flag appears + while not flag.exists(): + time.sleep(0.01) + sys.exit(0) + signal.signal(signal.SIGTERM, stop) + print("up", flush=True) + while True: + time.sleep(0.1) +""" + + +@pytest.mark.live_system_guard_bypass +def test_a_gateway_draining_after_the_stop_request_keeps_its_restart_debt(tmp_path): + flag = tmp_path / "drained" + gateway = _child(_DRAINING, str(flag), env={"HERMES_HOME": str(tmp_path)}) + assert gateway.stdout.readline().strip() == "up" + updater = _child(_AT_LINE + """ + pid = int(sys.argv[1]) + token = r.record_pause({"resume_needed": True, "profiles": {"default": pid}, + "identities": {str(pid): r.identity(pid)["ct"]}}, None, []) + r.mark_stop_requested(token, [pid]) + os.kill(pid, 15) + print("asked", flush=True) + time.sleep(120) + """, str(gateway.pid), env={"HERMES_HOME": str(tmp_path)}) + try: + assert updater.stdout.readline().strip() == "asked" + assert gateway.stdout.readline().strip() == "stopping" + finally: + updater.send_signal(signal.SIGKILL) # windows-footgun: ok — module skips on Windows + updater.wait(timeout=10) + try: + assert gateway.poll() is None, "premise: the gateway is still draining" + assert _launches(tmp_path, 1) == [[]], "a draining gateway was restarted before it exited" + assert len(_record_files(tmp_path)) == 1, "a draining gateway's restart debt was dropped" + finally: + flag.touch() + gateway.wait(timeout=10) + assert _launches(tmp_path, 2) == [["resume ['default']"], []] + + +@pytest.mark.live_system_guard_bypass +def test_a_gateway_never_asked_to_stop_is_not_restarted(tmp_path): + gateway = _child(_DRAINING, str(tmp_path / "unused"), env={"HERMES_HOME": str(tmp_path)}) + assert gateway.stdout.readline().strip() == "up" + updater = _child(_AT_LINE + """ + pid = int(sys.argv[1]) + r.record_pause({"resume_needed": True, "profiles": {"default": pid}, + "identities": {str(pid): r.identity(pid)["ct"]}}, None, []) + print("recorded", flush=True) + time.sleep(120) + """, str(gateway.pid), env={"HERMES_HOME": str(tmp_path)}) + try: + assert updater.stdout.readline().strip() == "recorded" + updater.send_signal(signal.SIGKILL) # windows-footgun: ok — killed before its first stop request + updater.wait(timeout=10) + assert _launches(tmp_path, 1) == [[]] + assert _record_files(tmp_path) == [], "a still-serving gateway's entry stayed owed" + finally: + gateway.kill() + gateway.wait(timeout=10) + + +# --- R13: a checkout sharing the home never takes another checkout's paused set -------------- +def _checkout_copy(root: Path) -> Path: + """A second checkout: the real module file under another install root.""" + (root / "hermes_cli").mkdir(parents=True) + (root / "hermes_cli" / "update_pause_record.py").write_bytes(Path(pause_record.__file__).read_bytes()) + _git(root, "init", "-q") + _git(root, "add", ".") + _git(root, "commit", "-qm", "b") + return root + + +@pytest.mark.live_system_guard_bypass +def test_another_checkouts_writer_never_imports_or_relabels_this_checkouts_debt(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + _orphan(tmp_path, {"alpha": 4242}) + other = _checkout_copy(tmp_path / "other") + writer = _child(""" + import importlib.util, json, sys + spec = importlib.util.spec_from_file_location("other_pause", sys.argv[1]) + b = importlib.util.module_from_spec(spec); spec.loader.exec_module(b) + b.write(b.stamp_tree({"resume_needed": True, "profiles": {"beta": 1}}), owner=b.UNOWNED) + print(json.dumps([len(b.orphans()), b.read()]), flush=True) + """, str(other / "hermes_cli" / "update_pause_record.py"), env={"HERMES_HOME": str(tmp_path)}) + out, _ = writer.communicate(timeout=60) + seen, theirs = json.loads(out.strip().splitlines()[-1]) + assert seen == 1 and sorted(theirs["token"]["profiles"]) == ["beta"], f"imported this checkout's set: {theirs}" + ours = pause_record.read() + assert ours["install_root"] == str(REPO) and sorted(ours["token"]["profiles"]) == ["alpha"], ours From ccec6b6f34a6e56a82bccb158c34bafe9c82ce49 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:19:44 -0700 Subject: [PATCH 11/63] test(e2e): native Windows custody cells for the pause record (claim race, publish crash, draining, per-target readiness) Every cell plants its schedule with the installed pause-record module (a driver stops itself on the named source line) and recovers through a real hermes.exe launch that really restarts the real gateway. test_pause_durable reads the checkout-keyed record name. (cherry picked from commit f25dbc85cb23b17a9427218d367385d06958ad19) (cherry picked from commit 488a7f25f0d58ad27df709858bbb37ed49943a9c) --- .../core/windows_update/test_pause_custody.py | 291 ++++++++++++++++++ .../core/windows_update/test_pause_durable.py | 13 +- 2 files changed, 300 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/core/windows_update/test_pause_custody.py diff --git a/tests/e2e/core/windows_update/test_pause_custody.py b/tests/e2e/core/windows_update/test_pause_custody.py new file mode 100644 index 0000000000000..13afdd3151256 --- /dev/null +++ b/tests/e2e/core/windows_update/test_pause_custody.py @@ -0,0 +1,291 @@ +"""Restart debt survives every custody transfer of a Windows update pause (review R7). + +Failure class: the durable pause record changes hands — a recovering launch claims it, an update +folds an orphaned set into its own record — and a crash or a rival at the transfer boundary must +neither lose nor double a paused gateway. Each cell plants the exact schedule with the INSTALLED +code: a driver (the install's venv python) runs the real pause-record module and stops itself on +the named source line; every recovery is a real ``hermes gateway status`` launch through the +published ``hermes.exe`` that really restarts the real gateway. + +* claim race: a launch is paused inside its claim, between taking the set and naming itself in it + ("after rename, before identity write" on the old design); a second launch runs meanwhile. The + paused gateway is restarted exactly once and no record survives. +* publish crash: an update dies after publishing the merged record, before retiring the claim it + absorbed. The next launch restarts the gateway once; nothing is left behind. +* draining: the record says the update asked the gateway to stop and the gateway is still running + (draining). A launch must keep that debt while it runs, and the first launch after it exited + must start it again. +* readiness: a launch owes a gateway that comes back, one that never becomes ready, and an SCM + service that cannot start. The ready one runs; only the other two stay owed. +""" + +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path + +import psutil +import pytest + +from tests.e2e.core.windows_update._machine import REQUIRES_OPT_IN, fail_with, new_machine +from tests.fakes.fake_llm_provider import FakeLLMServer + +pytestmark = [pytest.mark.platforms("windows"), pytest.mark.integration, + pytest.mark.live_system_guard_bypass, REQUIRES_OPT_IN] + +_STEM = ".hermes-update-paused-gateways" +_RESTARTING = "Restarting gateway(s) paused by an interrupted" +_MISSING_SERVICE = "hermes-e2e-no-such-gateway-service" + +# Every driver: the installed module, and a hook that stops on one line of it (kill or park). +_DRIVER = """ +import inspect, json, os, sys, time +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from hermes_cli import update_pause_record as r + +def stop_at(fn, texts, action, flag=None): + lines, start = inspect.getsourcelines(fn) + line = next(start + i for text in texts for i, t in enumerate(lines) if text in t) + def trace(frame, event, arg): + if event == "line" and frame.f_code is fn.__code__ and frame.f_lineno == line: + if action == "kill": + os._exit(71) + print("paused", flush=True) + while not Path(flag).exists(): + time.sleep(0.05) + return trace + sys.settrace(trace) + +def orphan(profiles, **extra): + # The record of an updater that died after recording (owner unowned = dead). + r.write(r.stamp_tree({"resume_needed": True, "profiles": profiles, **extra}), owner=r.UNOWNED) +""" +# Claim boundary: the new design publishes the claim (identity inside) then retires the source; +# the old one renamed the source first and wrote its identity after. +_CLAIM_BOUNDARY = '("src.unlink()", \'body["claimer"] = identity()\')' + + +def _python(machine) -> Path: + found = sorted(machine.hermes_home.glob("installs/*/environments/*/venv/Scripts/python.exe")) + assert found, fail_with(machine, "harness: the install has no venv python") + return found[0] + + +def _driver(machine, body: str, *args: str) -> subprocess.Popen: + proc = subprocess.Popen([str(_python(machine)), "-c", _DRIVER + body, str(machine.install_dir), *args], + cwd=machine.profile, env=machine.env(), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags=subprocess.CREATE_NO_WINDOW) + machine._spawned.append(proc) + return proc + + +def _run_driver(machine, body: str, *args: str, timeout: float = 300) -> tuple[int, str]: + proc = _driver(machine, body, *args) + out, _ = proc.communicate(timeout=timeout) + return proc.returncode, out.decode("utf-8", "replace") + + +def _records(machine) -> list[Path]: + return sorted(p for p in machine.hermes_home.glob(_STEM + "*") if p.suffix in (".json", ".claim")) + + +def _owed(machine) -> list[dict]: + out = [] + for path in _records(machine): + try: + out.append(json.loads(path.read_text(encoding="utf-8-sig"))["token"]) + except (OSError, ValueError, KeyError): + continue + return out + + +def _clear(machine) -> None: + machine.kill_owned() + for path in _records(machine): + path.unlink(missing_ok=True) + + +def _running(machine, not_pid: int, timeout: float) -> int | None: + try: + return int(machine.wait_gateway_running(not_pid=not_pid, timeout=timeout).get("pid") or 0) or None + except AssertionError: + return None + + +def _paused_gateway(machine) -> int: + """A real gateway, then stopped hard: the state a pause left it in. Its (dead) pid.""" + machine.spawn_gateway() + pid = int(machine.wait_gateway_running().get("pid") or 0) + machine.kill_owned() + return pid + + +def _launch(machine, label: str): + return machine.hermes("gateway", "status", label=label, timeout=600) + + +def _claim_race(machine, out: dict) -> None: + dead = _paused_gateway(machine) + # A claim handed back unowned (a launch that could not finish the resume): any launch may take it. + out["race_seed"] = _run_driver(machine, f""" +orphan({{"default": {dead}}}) +won = r.claim(r.record_path()) +r._atomic_write(won[0], {{**won[1], "claimer": r.UNOWNED}}) +""") + flag = machine.root / "claim-race-go" + first = _driver(machine, f""" +stop_at(r.claim, {_CLAIM_BOUNDARY}, "park", sys.argv[2]) +r.recover(["status"]) +""", str(flag)) + assert first.stdout is not None + out["race_parked"] = first.stdout.readline().decode("utf-8", "replace").strip() + out["race_second"] = _launch(machine, "claim-race-second") + flag.touch() + rest, _ = first.communicate(timeout=600) + out["race_first"] = out["race_parked"] + "\n" + rest.decode("utf-8", "replace") + out["race_running"] = _running(machine, dead, 120) + out["race_owed"] = _owed(machine) + _clear(machine) + + +def _publish_crash(machine, out: dict) -> None: + dead = _paused_gateway(machine) + out["publish_seed"] = _run_driver(machine, f'orphan({{"default": {dead}}})') + out["publish_kill"] = _run_driver(machine, """ +adopted, claims = r.adopt_orphans() +stop_at(r.record_pause, ("release_claims(claims)",), "kill") +r.record_pause({"resume_needed": True, "profiles": {}}, adopted, claims) +""") + out["publish_files"] = [p.name for p in _records(machine)] + out["publish_launch"] = _launch(machine, "after-publish-crash") + out["publish_running"] = _running(machine, dead, 120) + out["publish_again"] = _launch(machine, "after-publish-crash-2") + out["publish_owed"] = _owed(machine) + _clear(machine) + + +def _draining(machine, out: dict) -> None: + # The real gateway keeps running: it was asked to stop and has not exited yet (draining). + machine.spawn_gateway() + live = int(machine.wait_gateway_running().get("pid") or 0) + updater = _driver(machine, f""" +pid = {live} +token = {{"resume_needed": True, "profiles": {{"default": pid}}, "identities": {{str(pid): r.identity(pid)["ct"]}}}} +if hasattr(r, "mark_stop_requested"): + r.mark_stop_requested(r.record_pause(token, None, []), [pid]) +else: # the old design has no stop record: write the same fact it would have needed + r.write(r.stamp_tree({{**token, "stop_requested": [str(pid)]}}), owner=r.identity()) +print("asked", flush=True) +time.sleep(600) +""") + assert updater.stdout is not None + out["drain_asked"] = updater.stdout.readline().decode("utf-8", "replace").strip() + subprocess.run(["taskkill", "/PID", str(updater.pid), "/T", "/F"], capture_output=True, timeout=60) + updater.wait(timeout=60) + out["drain_live_before"] = psutil.pid_exists(live) + out["drain_launch_while"] = _launch(machine, "while-draining") + out["drain_owed_while"] = _owed(machine) + subprocess.run(["taskkill", "/PID", str(live), "/T", "/F"], capture_output=True, timeout=60) + out["drain_launch_after"] = _launch(machine, "after-drained") + out["drain_running_after"] = _running(machine, live, 120) + out["drain_owed_after"] = _owed(machine) + _clear(machine) + + +def _readiness(machine, out: dict) -> None: + dead = _paused_gateway(machine) + out["ready_seed"] = _run_driver(machine, f""" +orphan({{"default": {dead}, "ghost": {dead}}}, services=["{_MISSING_SERVICE}"], + expected_services=["{_MISSING_SERVICE}"], restarted_services=[], + service_profiles={{"{_MISSING_SERVICE}": "svc"}}) +""") + out["ready_launch"] = _launch(machine, "readiness") + out["ready_running"] = _running(machine, dead, 120) + out["ready_owed"] = _owed(machine) + _clear(machine) + + +@pytest.fixture(scope="module") +def journey(tmp_path_factory): + out: dict = {} + with FakeLLMServer() as srv: + machine = new_machine(tmp_path_factory.mktemp("pc"), srv.base_url, label="pc", system_git=True) + out["machine"] = machine + try: + install = machine.install() + assert install.returncode == 0, fail_with(machine, f"install.ps1 exited {install.returncode}", install) + with machine.gateway_phase(): + for cell in (_claim_race, _publish_crash, _draining, _readiness): + started = time.monotonic() + try: + cell(machine, out) + except Exception as exc: # one broken cell must not hide the others' verdicts + out[f"{cell.__name__}_error"] = f"{type(exc).__name__}: {exc}" + _clear(machine) + machine.timings.append((cell.__name__, round(time.monotonic() - started, 1))) + yield out + finally: + machine.teardown() + + +def _no_error(journey, cell: str) -> None: + error = journey.get(f"_{cell}_error") + assert error is None, fail_with(journey["machine"], f"harness: cell {cell} broke: {error}") + + +def _restarts(text: str) -> int: + return text.count(_RESTARTING) + + +def test_a_claim_in_transfer_is_restarted_exactly_once(journey) -> None: + _no_error(journey, "claim_race") + m, second = journey["machine"], journey["race_second"] + assert journey["race_parked"] == "paused", fail_with(m, f"premise: the first launch never reached the boundary: {journey['race_first']}") + restarts = _restarts(journey["race_first"]) + _restarts(second.stdout) + assert restarts == 1, fail_with( + m, f"a set claimed while a second launch ran was restarted {restarts} times\n--- first ---\n" + f"{journey['race_first']}", second) + assert journey["race_running"], fail_with(m, "the paused gateway did not come back", second) + assert journey["race_owed"] == [], fail_with(m, f"a restarted set is still on disk: {journey['race_owed']}", second) + + +def test_an_update_killed_after_publishing_restarts_the_set_once(journey) -> None: + _no_error(journey, "publish_crash") + m, run = journey["machine"], journey["publish_launch"] + rc, text = journey["publish_kill"] + assert rc == 71 and len(journey["publish_files"]) == 2, fail_with( + m, f"premise: the update was not killed between publish and retire (rc={rc}, files={journey['publish_files']})\n{text}") + assert _restarts(run.stdout) == 1, fail_with( + m, f"one paused set was restarted {_restarts(run.stdout)} times from two copies", run) + assert journey["publish_running"], fail_with(m, "the paused gateway did not come back", run) + assert _restarts(journey["publish_again"].stdout) == 0 and journey["publish_owed"] == [], fail_with( + m, f"a restarted set was owed again: {journey['publish_owed']}", journey["publish_again"]) + + +def test_a_draining_gateway_keeps_its_restart_debt_until_it_exits(journey) -> None: + _no_error(journey, "draining") + m, during, after = journey["machine"], journey["drain_launch_while"], journey["drain_launch_after"] + assert journey["drain_asked"] == "asked" and journey["drain_live_before"], fail_with( + m, f"premise: no live gateway with a recorded stop request ({journey['drain_asked']!r})") + owed = [sorted(t.get("profiles") or {}) for t in journey["drain_owed_while"]] + assert owed == [["default"]], fail_with( + m, f"a gateway asked to stop and still running lost its restart debt (owed while draining: {owed})", during) + assert journey["drain_running_after"], fail_with( + m, "after the draining gateway exited, the next launch did not start it again", after) + assert journey["drain_owed_after"] == [], fail_with(m, f"debt left after the restart: {journey['drain_owed_after']}", after) + + +def test_each_runtime_is_retired_only_on_its_own_readiness(journey) -> None: + _no_error(journey, "readiness") + m, run = journey["machine"], journey["ready_launch"] + assert journey["ready_running"], fail_with( + m, "a gateway that could come back stayed stopped behind a failed service/profile", run) + owed = journey["ready_owed"] + assert len(owed) == 1, fail_with(m, f"the unready runtimes are not owed any more: {owed}", run) + assert sorted(owed[0].get("profiles") or {}) == ["ghost"], fail_with( + m, f"retired on another target's readiness (owed profiles {owed[0].get('profiles')})", run) + assert owed[0].get("services") == [_MISSING_SERVICE], fail_with( + m, f"the service that did not start is not owed (services {owed[0].get('services')})", run) diff --git a/tests/e2e/core/windows_update/test_pause_durable.py b/tests/e2e/core/windows_update/test_pause_durable.py index eb96689b38670..15537a3e97507 100644 --- a/tests/e2e/core/windows_update/test_pause_durable.py +++ b/tests/e2e/core/windows_update/test_pause_durable.py @@ -33,7 +33,7 @@ pytest.mark.live_system_guard_bypass, REQUIRES_OPT_IN] _PAUSED = "Paused gateway profile" -_RECORD = ".hermes-update-paused-gateways.json" +_RECORD_STEM = ".hermes-update-paused-gateways" # ..json, plus ...claim _EARLY, _HELD = "AGENTS.md", "website/package.json" # checkout order: the early file is written first _PULL_MARKER = "hermes-update-pull" # hermes_cli._early_recovery.INTERRUPTED_PULL_MARKER, in the git dir _RESTORED = "Checkout restored" # the updater's own in-place restore of git's half-written files @@ -46,6 +46,11 @@ def _alive(pid: int) -> bool: return False +def _record_held(machine) -> bool: + """The paused set is still on disk (the record or a recovering launch's claim).""" + return any(p.suffix in (".json", ".claim") for p in machine.hermes_home.glob(_RECORD_STEM + "*")) + + def _running_gateway(machine, not_pid: int, timeout: float) -> dict | None: try: return machine.wait_gateway_running(not_pid=not_pid, timeout=timeout) @@ -117,7 +122,7 @@ def journey(tmp_path_factory): machine.advance() out["a_paused"], out["a_tail"] = _update_killed_after_pause(machine) out["a_old_stopped"] = not _alive(old) - out["a_record"] = (machine.hermes_home / _RECORD).is_file() + out["a_record"] = _record_held(machine) out["a_launch"] = machine.hermes("gateway", "status", label="next-launch") out["a_after"] = _running_gateway(machine, old, 120) machine.kill_owned() @@ -132,7 +137,7 @@ def journey(tmp_path_factory): time.sleep(30) out["c_after"] = after out["c_alive_30s"] = bool(after) and _alive(int(after.get("pid") or 0)) - out["c_record"] = (machine.hermes_home / _RECORD).is_file() + out["c_record"] = _record_held(machine) machine.kill_owned() # (b) the pull fails with a torn tree @@ -150,7 +155,7 @@ def journey(tmp_path_factory): time.sleep(15 if after else 0) out["b_after"] = after out["b_alive_15s"] = bool(after) and _alive(int(after.get("pid") or 0)) - out["b_record"] = (machine.hermes_home / _RECORD).is_file() + out["b_record"] = _record_held(machine) machine.kill_owned() yield out finally: From b4f853d9f274af5fcade73911c1f1d348278e3f1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:20:23 -0700 Subject: [PATCH 12/63] test(update): the pause reader applies the marker's own-pid incarnation rule (R12) (cherry picked from commit eab2a97ab97e6f91912f2cee2395fa0afcdfaead) (cherry picked from commit 2824ef49061f6ba13c713c9724d17b7c2baf30ed) --- tests/hermes_cli/test_update_pause_record.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 2d17fbd563035..f637a3ce6f1fc 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -391,3 +391,13 @@ def test_another_checkouts_writer_never_imports_or_relabels_this_checkouts_debt( assert seen == 1 and sorted(theirs["token"]["profiles"]) == ["beta"], f"imported this checkout's set: {theirs}" ours = pause_record.read() assert ours["install_root"] == str(REPO) and sorted(ours["token"]["profiles"]) == ["alpha"], ours + + +# --- R12: the pause reader judges incarnations by the update marker's rule --------------------- +def test_our_own_pid_is_ours_only_at_our_exact_creation_time(): + """A record left by a killed update whose pid this launch now has (fresh pid namespace) is dead.""" + me = pause_record.identity() + assert me["ct"].startswith("ct:") and pause_record.identity_is_live(me) + created = float(me["ct"][3:]) + reused = {"pid": os.getpid(), "ct": f"ct:{created - 0.5:.3f}"} # inside the cross-writer skew + assert not pause_record.identity_is_live(reused), "a killed update's record became ours by pid reuse" From 24e5950e342270b053677bb476d8893ccff1c9dd Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:01:48 -0700 Subject: [PATCH 13/63] test(e2e): custody cells read a fresh gateway, log every driver, trace only the boundary function (cherry picked from commit 1a7ed05ffe1de97d88a5c0bbd0d2370f5440955e) (cherry picked from commit 0b1f697d5212e9eedaa52772569fb9f80aa165c9) --- .../core/windows_update/test_pause_custody.py | 128 ++++++++++++------ tests/hermes_cli/test_update_pause_record.py | 6 +- 2 files changed, 92 insertions(+), 42 deletions(-) diff --git a/tests/e2e/core/windows_update/test_pause_custody.py b/tests/e2e/core/windows_update/test_pause_custody.py index 13afdd3151256..211662c4503e5 100644 --- a/tests/e2e/core/windows_update/test_pause_custody.py +++ b/tests/e2e/core/windows_update/test_pause_custody.py @@ -49,8 +49,10 @@ def stop_at(fn, texts, action, flag=None): lines, start = inspect.getsourcelines(fn) line = next(start + i for text in texts for i, t in enumerate(lines) if text in t) - def trace(frame, event, arg): - if event == "line" and frame.f_code is fn.__code__ and frame.f_lineno == line: + def trace(frame, event, arg): # traces only fn's own frames: the rest runs at full speed + if frame.f_code is not fn.__code__: + return None + if event == "line" and frame.f_lineno == line: if action == "kill": os._exit(71) print("paused", flush=True) @@ -74,18 +76,41 @@ def _python(machine) -> Path: return found[0] -def _driver(machine, body: str, *args: str) -> subprocess.Popen: - proc = subprocess.Popen([str(_python(machine)), "-c", _DRIVER + body, str(machine.install_dir), *args], - cwd=machine.profile, env=machine.env(), stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags=subprocess.CREATE_NO_WINDOW) +def _driver(machine, name: str, body: str, *args: str) -> tuple[subprocess.Popen, Path]: + """The install's python running *body*; its output goes to a transcript (as a launch's would).""" + machine._seq += 1 + log = machine.logs / f"{machine._seq:02d}-driver-{name}.log" + flags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW + with log.open("wb") as fh: + proc = subprocess.Popen([str(_python(machine)), "-u", "-c", _DRIVER + body, str(machine.install_dir), *args], + cwd=machine.profile, env=machine.env(), stdin=subprocess.DEVNULL, stdout=fh, + stderr=subprocess.STDOUT, creationflags=flags) machine._spawned.append(proc) - return proc + return proc, log -def _run_driver(machine, body: str, *args: str, timeout: float = 300) -> tuple[int, str]: - proc = _driver(machine, body, *args) - out, _ = proc.communicate(timeout=timeout) - return proc.returncode, out.decode("utf-8", "replace") +def _text(log: Path) -> str: + try: + return log.read_text(encoding="utf-8-sig", errors="replace") + except OSError: + return "" + + +def _await_line(proc: subprocess.Popen, log: Path, line: str, timeout: float = 120) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if line in _text(log).splitlines(): + return True + if proc.poll() is not None: + return line in _text(log).splitlines() + time.sleep(0.2) + return False + + +def _run_driver(machine, name: str, body: str, *args: str, timeout: float = 300) -> tuple[int, str]: + proc, log = _driver(machine, name, body, *args) + proc.wait(timeout=timeout) + return proc.returncode, _text(log) def _records(machine) -> list[Path]: @@ -104,8 +129,14 @@ def _owed(machine) -> list[dict]: def _clear(machine) -> None: machine.kill_owned() + deadline = time.monotonic() + 60 + while machine.owned_processes() and time.monotonic() < deadline: + time.sleep(0.5) for path in _records(machine): path.unlink(missing_ok=True) + # A hard-killed gateway leaves its last ``running`` state behind: the next cell's gateway must + # be the one it reads, never a dying pid from this one. + (machine.hermes_home / "gateway_state.json").unlink(missing_ok=True) def _running(machine, not_pid: int, timeout: float) -> int | None: @@ -115,11 +146,19 @@ def _running(machine, not_pid: int, timeout: float) -> int | None: return None +def _fresh_gateway(machine) -> int: + """A real gateway started now (``hermes gateway run``); its pid once it reports running.""" + (machine.hermes_home / "gateway_state.json").unlink(missing_ok=True) + proc = machine.spawn_gateway() + pid = int(machine.wait_gateway_running().get("pid") or 0) + assert pid and psutil.pid_exists(pid) and proc.poll() is None, fail_with(machine, f"harness: gateway {pid} not up") + return pid + + def _paused_gateway(machine) -> int: """A real gateway, then stopped hard: the state a pause left it in. Its (dead) pid.""" - machine.spawn_gateway() - pid = int(machine.wait_gateway_running().get("pid") or 0) - machine.kill_owned() + pid = _fresh_gateway(machine) + _clear(machine) return pid @@ -130,38 +169,41 @@ def _launch(machine, label: str): def _claim_race(machine, out: dict) -> None: dead = _paused_gateway(machine) # A claim handed back unowned (a launch that could not finish the resume): any launch may take it. - out["race_seed"] = _run_driver(machine, f""" + out["race_seed"] = _run_driver(machine, "race-seed", f""" orphan({{"default": {dead}}}) won = r.claim(r.record_path()) r._atomic_write(won[0], {{**won[1], "claimer": r.UNOWNED}}) """) flag = machine.root / "claim-race-go" - first = _driver(machine, f""" + first, log = _driver(machine, "race-first", f""" +import psutil +print("paused pid {dead} alive before recovery:", psutil.pid_exists({dead}), flush=True) stop_at(r.claim, {_CLAIM_BOUNDARY}, "park", sys.argv[2]) r.recover(["status"]) +sys.settrace(None) +print("left on disk:", sorted(p.name for p in r.record_path().parent.glob(r.RECORD_STEM + "*")), flush=True) """, str(flag)) - assert first.stdout is not None - out["race_parked"] = first.stdout.readline().decode("utf-8", "replace").strip() + out["race_parked"] = _await_line(first, log, "paused") out["race_second"] = _launch(machine, "claim-race-second") flag.touch() - rest, _ = first.communicate(timeout=600) - out["race_first"] = out["race_parked"] + "\n" + rest.decode("utf-8", "replace") - out["race_running"] = _running(machine, dead, 120) + first.wait(timeout=600) + out["race_first"] = _text(log) + out["race_running"] = _running(machine, dead, 150) out["race_owed"] = _owed(machine) _clear(machine) def _publish_crash(machine, out: dict) -> None: dead = _paused_gateway(machine) - out["publish_seed"] = _run_driver(machine, f'orphan({{"default": {dead}}})') - out["publish_kill"] = _run_driver(machine, """ + out["publish_seed"] = _run_driver(machine, "publish-seed", f'orphan({{"default": {dead}}})') + out["publish_kill"] = _run_driver(machine, "publish-kill", """ adopted, claims = r.adopt_orphans() stop_at(r.record_pause, ("release_claims(claims)",), "kill") r.record_pause({"resume_needed": True, "profiles": {}}, adopted, claims) """) out["publish_files"] = [p.name for p in _records(machine)] out["publish_launch"] = _launch(machine, "after-publish-crash") - out["publish_running"] = _running(machine, dead, 120) + out["publish_running"] = _running(machine, dead, 150) out["publish_again"] = _launch(machine, "after-publish-crash-2") out["publish_owed"] = _owed(machine) _clear(machine) @@ -169,9 +211,8 @@ def _publish_crash(machine, out: dict) -> None: def _draining(machine, out: dict) -> None: # The real gateway keeps running: it was asked to stop and has not exited yet (draining). - machine.spawn_gateway() - live = int(machine.wait_gateway_running().get("pid") or 0) - updater = _driver(machine, f""" + live = _fresh_gateway(machine) + updater, log = _driver(machine, "drain-updater", f""" pid = {live} token = {{"resume_needed": True, "profiles": {{"default": pid}}, "identities": {{str(pid): r.identity(pid)["ct"]}}}} if hasattr(r, "mark_stop_requested"): @@ -181,29 +222,34 @@ def _draining(machine, out: dict) -> None: print("asked", flush=True) time.sleep(600) """) - assert updater.stdout is not None - out["drain_asked"] = updater.stdout.readline().decode("utf-8", "replace").strip() + out["drain_asked"] = _await_line(updater, log, "asked") + # The updater dies with the stop request on disk (its venv launcher and the interpreter under it). subprocess.run(["taskkill", "/PID", str(updater.pid), "/T", "/F"], capture_output=True, timeout=60) updater.wait(timeout=60) + out["drain_updater"] = _text(log) out["drain_live_before"] = psutil.pid_exists(live) out["drain_launch_while"] = _launch(machine, "while-draining") + out["drain_live_after_launch"] = psutil.pid_exists(live) out["drain_owed_while"] = _owed(machine) - subprocess.run(["taskkill", "/PID", str(live), "/T", "/F"], capture_output=True, timeout=60) + subprocess.run(["taskkill", "/PID", str(live), "/F"], capture_output=True, timeout=60) + deadline = time.monotonic() + 30 + while psutil.pid_exists(live) and time.monotonic() < deadline: + time.sleep(0.2) out["drain_launch_after"] = _launch(machine, "after-drained") - out["drain_running_after"] = _running(machine, live, 120) + out["drain_running_after"] = _running(machine, live, 150) out["drain_owed_after"] = _owed(machine) _clear(machine) def _readiness(machine, out: dict) -> None: dead = _paused_gateway(machine) - out["ready_seed"] = _run_driver(machine, f""" + out["ready_seed"] = _run_driver(machine, "ready-seed", f""" orphan({{"default": {dead}, "ghost": {dead}}}, services=["{_MISSING_SERVICE}"], expected_services=["{_MISSING_SERVICE}"], restarted_services=[], service_profiles={{"{_MISSING_SERVICE}": "svc"}}) """) out["ready_launch"] = _launch(machine, "readiness") - out["ready_running"] = _running(machine, dead, 120) + out["ready_running"] = _running(machine, dead, 150) out["ready_owed"] = _owed(machine) _clear(machine) @@ -243,13 +289,13 @@ def _restarts(text: str) -> int: def test_a_claim_in_transfer_is_restarted_exactly_once(journey) -> None: _no_error(journey, "claim_race") m, second = journey["machine"], journey["race_second"] - assert journey["race_parked"] == "paused", fail_with(m, f"premise: the first launch never reached the boundary: {journey['race_first']}") + first = f"--- first launch (driver) ---\n{journey['race_first']}" + assert journey["race_parked"], fail_with(m, f"premise: the first launch never reached the boundary\n{first}") restarts = _restarts(journey["race_first"]) + _restarts(second.stdout) assert restarts == 1, fail_with( - m, f"a set claimed while a second launch ran was restarted {restarts} times\n--- first ---\n" - f"{journey['race_first']}", second) - assert journey["race_running"], fail_with(m, "the paused gateway did not come back", second) - assert journey["race_owed"] == [], fail_with(m, f"a restarted set is still on disk: {journey['race_owed']}", second) + m, f"a set claimed while a second launch ran was restarted {restarts} times\n{first}", second) + assert journey["race_running"], fail_with(m, f"the paused gateway did not come back\n{first}", second) + assert journey["race_owed"] == [], fail_with(m, f"a restarted set is still on disk: {journey['race_owed']}\n{first}", second) def test_an_update_killed_after_publishing_restarts_the_set_once(journey) -> None: @@ -268,8 +314,10 @@ def test_an_update_killed_after_publishing_restarts_the_set_once(journey) -> Non def test_a_draining_gateway_keeps_its_restart_debt_until_it_exits(journey) -> None: _no_error(journey, "draining") m, during, after = journey["machine"], journey["drain_launch_while"], journey["drain_launch_after"] - assert journey["drain_asked"] == "asked" and journey["drain_live_before"], fail_with( - m, f"premise: no live gateway with a recorded stop request ({journey['drain_asked']!r})") + assert journey["drain_asked"] and journey["drain_live_before"], fail_with( + m, f"premise: no live gateway with a recorded stop request\n{journey['drain_updater']}") + assert journey["drain_live_after_launch"] and _restarts(during.stdout) == 0, fail_with( + m, "a launch restarted (replaced) a gateway that was still draining", during) owed = [sorted(t.get("profiles") or {}) for t in journey["drain_owed_while"]] assert owed == [["default"]], fail_with( m, f"a gateway asked to stop and still running lost its restart debt (owed while draining: {owed})", during) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index f637a3ce6f1fc..7a62e3d28da30 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -209,8 +209,10 @@ def test_update_started_from_a_relaunched_gateway_does_not_share_the_claim(tmp_p def stop_at(fn, text, action, flag=None): lines, start = inspect.getsourcelines(fn) line = start + next(i for i, t in enumerate(lines) if text in t) - def trace(frame, event, arg): - if event == "line" and frame.f_code is fn.__code__ and frame.f_lineno == line: + def trace(frame, event, arg): # traces only fn's own frames: the rest runs at full speed + if frame.f_code is not fn.__code__: + return None + if event == "line" and frame.f_lineno == line: if action == "kill": os._exit(71) print("paused", flush=True) From 5c8c08c4220ff3d96d4962c95ed48d39f65c6fe8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:10:45 -0700 Subject: [PATCH 14/63] fix(update): the pause reader calls update_lock.incarnation_live, the one incarnation rule (R12) (cherry picked from commit 4a31b3bc0c7ffa6fe9ddafb22cbc7223d80de3b7) (cherry picked from commit 0681528e2a2847c8da909b3d9d8566c8689677ca) --- hermes_cli/update_pause_record.py | 18 +++++++----------- tests/hermes_cli/test_update_pause_record.py | 4 ++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 5e2ee0c196ed1..422f10fe02da5 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -72,18 +72,14 @@ def identity(pid: int | None = None) -> dict: def identity_is_live(ident: dict | None) -> bool: - """The update marker's incarnation rule (``update_lock``), applied to a recorded identity.""" + """The update marker's incarnation rule (``update_lock.incarnation_live``) for a recorded identity. + + Unprovable (alive, no readable creation time) counts as live: a record is never taken from, nor + a paused process restarted over, a process that may still be the one recorded. + """ from hermes_cli import update_lock - try: - pid = int((ident or {}).get("pid") or 0) - except (TypeError, ValueError): - return False - recorded = str((ident or {}).get("ct") or "") - try: - created = float(recorded[3:]) if recorded.startswith("ct:") else None - except ValueError: - created = None - return pid > 0 and update_lock._identity_live(pid, created, 0.0) + ident = ident or {} + return update_lock.incarnation_live(ident.get("pid") or 0, ident.get("ct") or None) is not False class RecordConflict(OSError): diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 7a62e3d28da30..dd65e83504881 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -403,3 +403,7 @@ def test_our_own_pid_is_ours_only_at_our_exact_creation_time(): created = float(me["ct"][3:]) reused = {"pid": os.getpid(), "ct": f"ct:{created - 0.5:.3f}"} # inside the cross-writer skew assert not pause_record.identity_is_live(reused), "a killed update's record became ours by pid reuse" + + +def test_a_record_naming_our_pid_without_a_creation_time_is_a_previous_incarnation(): + assert not pause_record.identity_is_live({"pid": os.getpid(), "ct": None}) From d4cc23d2e640345ef4594729a09e9ff9885736a7 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:19:04 -0700 Subject: [PATCH 15/63] test(e2e): custody drivers boot through hermes_bootstrap like a launch (cherry picked from commit dd30f9fe3cbdebedd9772137daf8b0ace43953a3) (cherry picked from commit 3a12ed34130d851a253b3b5c3787583cb5048620) --- tests/e2e/core/windows_update/test_pause_custody.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/core/windows_update/test_pause_custody.py b/tests/e2e/core/windows_update/test_pause_custody.py index 211662c4503e5..05d7cc79f35d6 100644 --- a/tests/e2e/core/windows_update/test_pause_custody.py +++ b/tests/e2e/core/windows_update/test_pause_custody.py @@ -39,11 +39,14 @@ _RESTARTING = "Restarting gateway(s) paused by an interrupted" _MISSING_SERVICE = "hermes-e2e-no-such-gateway-service" -# Every driver: the installed module, and a hook that stops on one line of it (kill or park). +# Every driver: the installed module behind the same bootstrap a launch runs (a resume imports +# hermes_cli.main, whose bootstrap would otherwise re-enter this script mid-resume), and a hook that +# stops on one line of it (kill or park). _DRIVER = """ import inspect, json, os, sys, time from pathlib import Path sys.path.insert(0, sys.argv[1]) +import hermes_bootstrap # as every launch does: re-enters the managed interpreter with its dependencies from hermes_cli import update_pause_record as r def stop_at(fn, texts, action, flag=None): From 402005bf07a4ce7270c763e8c7f0e3dd65bf7a25 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:53:34 -0700 Subject: [PATCH 16/63] test(e2e): custody cells pause the gateway with a clean stop, as an update does (cherry picked from commit 78842c6b59cc35e0e7825612e6bdddf362ad8017) (cherry picked from commit ce7b7d362016896d09bc384ecb41907b7904228c) --- tests/e2e/core/windows_update/test_pause_custody.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/e2e/core/windows_update/test_pause_custody.py b/tests/e2e/core/windows_update/test_pause_custody.py index 05d7cc79f35d6..c6bf274ed06ba 100644 --- a/tests/e2e/core/windows_update/test_pause_custody.py +++ b/tests/e2e/core/windows_update/test_pause_custody.py @@ -159,8 +159,13 @@ def _fresh_gateway(machine) -> int: def _paused_gateway(machine) -> int: - """A real gateway, then stopped hard: the state a pause left it in. Its (dead) pid.""" + """A real gateway, stopped the way a pause stops it (a clean ``hermes gateway stop``, not a + hard kill whose stale lock state slows the next ``--replace``). Its (now dead) pid.""" pid = _fresh_gateway(machine) + machine.hermes("gateway", "stop", label="pause-stop", timeout=180) + deadline = time.monotonic() + 60 + while psutil.pid_exists(pid) and time.monotonic() < deadline: + time.sleep(0.2) _clear(machine) return pid From a49e65695b035e60a0f422f4848fa803afe261e5 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:55:22 -0700 Subject: [PATCH 17/63] test(e2e): custody journey turns the respawn-storm breaker off (cells restart the gateway on purpose) (cherry picked from commit 93bbd366f3da731b75326a086e4cf62946d8fd22) (cherry picked from commit fd24b22b1b8877f41f472db3c850049228558839) --- tests/e2e/core/windows_update/test_pause_custody.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/e2e/core/windows_update/test_pause_custody.py b/tests/e2e/core/windows_update/test_pause_custody.py index c6bf274ed06ba..0a6a0a89e5f8e 100644 --- a/tests/e2e/core/windows_update/test_pause_custody.py +++ b/tests/e2e/core/windows_update/test_pause_custody.py @@ -271,6 +271,11 @@ def journey(tmp_path_factory): try: install = machine.install() assert install.returncode == 0, fail_with(machine, f"install.ps1 exited {install.returncode}", install) + # The cells start and stop the gateway ~10 times in a few minutes on purpose; the respawn-storm + # breaker (5 starts / 120 s, then a 40 s sleep before boot) would otherwise outlast a resume's + # 30 s readiness window and make "not verified" a property of the harness, not the code. + off = machine.hermes("config", "set", "gateway.respawn_storm.max_starts", "0", label="storm-breaker-off") + assert off.returncode == 0, fail_with(machine, "harness: could not disable the respawn-storm breaker", off) with machine.gateway_phase(): for cell in (_claim_race, _publish_crash, _draining, _readiness): started = time.monotonic() From 476c33f6d707a1d551583efc016795b380ad23d5 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:55:34 -0700 Subject: [PATCH 18/63] fix(update): a pause owes a live gateway only once its stop request was issued (N1) mark_stop_requested recorded intent for every running gateway before the first request; an updater killed in between left a never-asked, still-serving gateway held as 'draining' forever, and restarted it after a later user stop. Now the intent also names where each request goes (planned-stop marker) and the stopper; mark_stop_sent records each request once issued (marker on disk, else a positive socket ACK), before the socket call. A live entry is owed only when it was recorded sent or this update's own marker still names it (a user's stop marker is not evidence); the evidence is resolved once into stop_sent so a merge or a hand-back carries it. Force-killed pids need no record (dead = owed). (cherry picked from commit e199442e5690614065e3013f06ad54b18584811d) (cherry picked from commit f2af22c6e8a45a7712046107b10c17fc9ad9d690) --- hermes_cli/update_cmd_windows.py | 39 ++++++++---- hermes_cli/update_pause_record.py | 63 ++++++++++++++++--- tests/hermes_cli/test_update_pause_record.py | 57 +++++++++++++++++ ...ws_gateway_cold_start_desktop_lifecycle.py | 4 +- 4 files changed, 140 insertions(+), 23 deletions(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index e2c3fc1ab5822..29293d224ab65 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -40,13 +40,17 @@ def _abort_on_error(prefix: str): raise RuntimeError(f"{prefix}: {exc}") from exc +def _planned_stop_marker_path(profile_path: Path) -> Path: + return Path(profile_path) / ".gateway-planned-stop.json" + + def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool: """Write a planned-stop marker into a specific profile home.""" try: from gateway.status import _get_process_start_time from utils import atomic_json_write atomic_json_write( - Path(profile_path) / ".gateway-planned-stop.json", + _planned_stop_marker_path(profile_path), {"target_pid": pid, "target_start_time": _get_process_start_time(pid), "stopper_pid": os.getpid(), "written_at": datetime.now(timezone.utc).isoformat()}, indent=None, separators=(",", ":"), @@ -863,11 +867,13 @@ def _discover_windows_gateways(): return profile_processes, service_gateways, service_gateway_pids, running_pids -def _request_socket_pauses(running_pids, profile_processes, service_gateway_pids): +def _request_socket_pauses(running_pids, profile_processes, service_gateway_pids, on_request=None): """Marker + socket-first pause for every profile-mapped gateway; ``(profiles, mapped_pids, socket_acks)``. Socket ACK = the gateway drains and exits by its own graceful path. No answer (older - gateway) -> the marker poll / force-kill ladder in the caller.""" + gateway) -> the marker poll / force-kill ladder in the caller. ``on_request(pid)`` runs once the + request is issued (marker on disk, else a positive socket ACK) and before the socket call: from + then on a crash leaves that gateway owed while it drains, never before.""" profiles: dict[str, int] = {} mapped_pids = [] socket_acks: list[dict] = [] @@ -877,7 +883,9 @@ def _request_socket_pauses(running_pids, profile_processes, service_gateway_pids continue profiles[str(proc.profile)] = int(pid) mapped_pids.append(int(pid)) - _write_update_planned_stop_marker(Path(proc.path), int(pid)) + requested = _write_update_planned_stop_marker(Path(proc.path), int(pid)) + if requested and on_request is not None: + on_request(int(pid)) try: # Socket-first pause (#92091 step 2): ask the gateway to drain and exit itself instead of # relying on the marker poll + force-kill ladder. A positive ACK means the gateway is running @@ -888,6 +896,8 @@ def _request_socket_pauses(running_pids, profile_processes, service_gateway_pids ack = pause_gateway_for_update(Path(proc.path)) if ack and (ack.get("pausing") or ack.get("already_stopping")): socket_acks.append(ack) + if not requested and on_request is not None: + on_request(int(pid)) except Exception as exc: logger.debug("Socket pause unavailable for gateway %s: %s", pid, exc) return profiles, mapped_pids, socket_acks @@ -946,10 +956,14 @@ def _pause_windows_gateways_for_update() -> dict | None: service_profiles={str(s.name): str(s.profile) for s in service_gateways}) with _abort_on_error("Could not record the gateways this update pauses"): pause_record.record_pause(intended, adopted, claims) - # A second write, right before the first request: a crash before it leaves every live entry - # "never asked" (still serving, dropped); after it, a live one is draining and stays owed. - pause_record.mark_stop_requested(intended, running_pids) - profiles = _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped) + # The intent (and where each request goes), right before the first request. A live entry is owed + # only once its request was issued (mark_stop_sent, or this update's marker still on disk): a + # crash before that leaves it "never asked" (still serving, dropped), after it, draining (owed). + pause_record.mark_stop_requested(intended, running_pids, markers={ + pid: _planned_stop_marker_path(Path(profile_processes[pid].path)) for pid in running_pids + if pid in profile_processes and pid not in service_gateway_pids}) + profiles = _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped, + on_request=lambda pid: pause_record.mark_stop_sent(intended, pid)) token = {"resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped} # Every profile with ANY live gateway at discovery counts as running: service-supervised ones skip the # socket pause (absent from ``profiles``) but the SCM restart brings them back, not a cold-start. @@ -980,11 +994,14 @@ def _cold_start_pause_token(adopted: dict | None, claims: list) -> dict | None: return pause_record.record_pause(token, adopted, claims) -def _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped) -> dict: - """Socket-pause the mapped gateways, drain, then tree-kill survivors; the paused ``profiles``.""" +def _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped, + on_request=None) -> dict: + """Socket-pause the mapped gateways, drain, then tree-kill survivors; the paused ``profiles``. + A force-killed process needs no stop record: dead, it is owed; alive, it was never stopped.""" from gateway.status import get_process_start_time, terminate_pid from hermes_cli.update_cmd import _m - profiles, mapped_pids, socket_acks = _request_socket_pauses(running_pids, profile_processes, service_gateway_pids) + profiles, mapped_pids, socket_acks = _request_socket_pauses( + running_pids, profile_processes, service_gateway_pids, on_request=on_request) # Resolve venv-side launchers BEFORE draining: a dead worker's parent cannot be recovered (NoSuchProcess). # The launcher keeps ``.pyd`` mapped and would trip the venv-holder guard; it is killed with the survivors. launcher_pids = _m()._venv_launcher_ancestors(mapped_pids) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 422f10fe02da5..471e1f60385a1 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -238,13 +238,29 @@ def write(token: dict, *, owner: dict | None = None, path: Path | None = None) - _atomic_write(path, {"schema": 1, "owner": owner, "install_root": str(install_root()), "token": token}) -def mark_stop_requested(token: dict, pids) -> None: - """Persist, BEFORE the first stop request, that these processes are being asked to stop: one - that acknowledged and is still draining keeps its restart obligation after a crash.""" +def mark_stop_requested(token: dict, pids, markers: dict | None = None) -> None: + """Persist, BEFORE the first stop request, the stops this update intends and where each mapped + gateway's request (its planned-stop marker) will be written. Intent is not a request: a live + process counts as asked only once :func:`mark_stop_sent` recorded it, or while the request this + update wrote is still on disk (:func:`_asked`).""" token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *(str(int(p)) for p in pids)}) + token["stop_sent"] = sorted(map(str, token.get("stop_sent") or [])) + token["stopper_pid"] = os.getpid() + token["stop_markers"] = {**(token.get("stop_markers") or {}), **{str(int(p)): str(m) for p, m in (markers or {}).items()}} write(token, owner=identity()) +def mark_stop_sent(token: dict, pid) -> None: + """Persist that *pid*'s stop request was issued (its planned-stop marker is on disk): from here a + live *pid* is draining and keeps its restart debt until it exits. Best effort: the marker itself + is the evidence until this lands, and the token carries it to the next record write.""" + token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), str(int(pid))}) + try: + write(token, owner=identity()) + except OSError as exc: + print(f" ⚠ Could not record the stop request for gateway {pid}: {exc}") + + def discharge(token: dict, path: Path | None = None) -> None: """Remove the record only while it still names this pause (decided under the mutex).""" path = path or record_path() @@ -411,7 +427,8 @@ def record_pause(token: dict, adopted: dict | None, claims: list[Path]) -> dict: return token -_CARRIED = ("pause_id", "pre_sha", "dirty_at_pause", "identities", "stop_requested", "absorbed") +_CARRIED = ("pause_id", "pre_sha", "dirty_at_pause", "identities", "stop_requested", "stop_sent", "stopper_pid", + "stop_markers", "absorbed") def finish_pause(token: dict, intended: dict, adopted: dict | None) -> dict: @@ -444,13 +461,38 @@ def _without(token: dict, pids: set[str]) -> dict: return token +def _request_on_disk(token: dict, pid: str) -> bool: + """The planned-stop marker this update's stopper wrote for *pid* is still on disk (the gateway's + watcher has not consumed it yet): the request was issued even if the updater died before + :func:`mark_stop_sent`. A marker naming another stopper (a user's ``hermes gateway stop``) is not.""" + path = (token.get("stop_markers") or {}).get(str(pid)) + if not path: + return False + try: + marker = json.loads(Path(path).read_text(encoding="utf-8-sig")) + return int(marker["target_pid"]) == int(pid) and int(marker["stopper_pid"]) == int(token["stopper_pid"]) + except (OSError, ValueError, TypeError, KeyError): + return False + + +def _asked(token: dict) -> set[str]: + """Pids this update actually asked to stop: recorded as sent, or whose request is still on disk.""" + sent = {str(p) for p in token.get("stop_sent") or []} + return sent | {str(p) for p in token.get("stop_requested") or [] if str(p) not in sent and _request_on_disk(token, str(p))} + + def drop_never_stopped(token: dict) -> dict: - """Drop entries whose recorded process is the same live incarnation AND was never asked to stop - (the update died before its first stop request). One it asked to stop may be draining: it keeps - its restart debt (:func:`split_draining`). No stop record at all: nothing can be told apart.""" + """Drop entries whose recorded process is the same live incarnation AND was never actually asked + to stop (the update died before issuing its request, even with the intent on disk): it is still + serving, and a later exit — a user's ``hermes gateway stop`` included — is not this update's to + undo. One it asked may be draining: it keeps its restart debt (:func:`split_draining`). The + evidence is resolved here, once, into ``stop_sent`` (a merge carries it under another stopper). + No stop record at all (a set from before stop tracking): nothing can be told apart.""" if token.get("stop_requested") is None: return token - return _without(token, _live_pids(token) - {str(p) for p in token["stop_requested"]}) + asked = _asked(token) + token["stop_sent"] = sorted(asked) + return _without(token, _live_pids(token) - asked) def split_draining(token: dict) -> dict: @@ -479,10 +521,11 @@ def merge_into(token: dict | None, adopted: dict) -> dict: for pid, ct in (adopted.get("identities") or {}).items(): identities.setdefault(pid, ct) if "stop_requested" in adopted or "stop_requested" in token: - requested = adopted.get("stop_requested") + requested, sent = adopted.get("stop_requested"), adopted.get("stop_sent") if requested is None: # a set with no stop record keeps every live entry: all count as asked - requested = list(adopted.get("identities") or {}) + requested = sent = list(adopted.get("identities") or {}) token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *map(str, requested)}) + token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), *map(str, sent or [])}) services = token.setdefault("services", []) services.extend(s for s in adopted.get("services") or [] if s not in services) if services: diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index dd65e83504881..336acc9cec1a1 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -323,6 +323,7 @@ def test_a_gateway_draining_after_the_stop_request_keeps_its_restart_debt(tmp_pa token = r.record_pause({"resume_needed": True, "profiles": {"default": pid}, "identities": {str(pid): r.identity(pid)["ct"]}}, None, []) r.mark_stop_requested(token, [pid]) + r.mark_stop_sent(token, pid) os.kill(pid, 15) print("asked", flush=True) time.sleep(120) @@ -365,6 +366,62 @@ def test_a_gateway_never_asked_to_stop_is_not_restarted(tmp_path): gateway.wait(timeout=10) +@pytest.mark.live_system_guard_bypass +def test_a_stop_intended_but_never_issued_leaves_a_serving_gateway_alone_and_a_later_user_stop_final(tmp_path): + """The updater recorded its intent, then died before the request (N1): the gateway was never + asked, keeps serving and owes nothing; when the user stops it later, no launch restarts it.""" + gateway = _child(_DRAINING, str(tmp_path / "unused"), env={"HERMES_HOME": str(tmp_path)}) + assert gateway.stdout.readline().strip() == "up" + updater = _child(_AT_LINE + """ + pid = int(sys.argv[1]) + token = r.record_pause({"resume_needed": True, "profiles": {"default": pid}, + "identities": {str(pid): r.identity(pid)["ct"]}}, None, []) + r.mark_stop_requested(token, [pid]) + print("intended", flush=True) + time.sleep(120) + """, str(gateway.pid), env={"HERMES_HOME": str(tmp_path)}) + try: + assert updater.stdout.readline().strip() == "intended" + updater.send_signal(signal.SIGKILL) # windows-footgun: ok — killed between the intent and the request + updater.wait(timeout=10) + assert _launches(tmp_path, 1) == [[]] + assert _record_files(tmp_path) == [], "a gateway never asked to stop is held as draining" + finally: + gateway.kill() # the user's stop, after the update's obligation was retired + gateway.wait(timeout=10) + assert _launches(tmp_path, 1) == [[]], "a gateway the user stopped was restarted for an update that never stopped it" + + +@pytest.mark.live_system_guard_bypass +def test_a_request_on_disk_is_owed_even_before_it_is_recorded_sent(tmp_path): + """Killed after writing the planned-stop marker (the request), before recording it sent: the + gateway acts on it, so it is owed; a marker the USER's stop wrote is no evidence.""" + marker = tmp_path / "profile" / ".gateway-planned-stop.json" + marker.parent.mkdir() + gateway = _child(_DRAINING, str(tmp_path / "unused"), env={"HERMES_HOME": str(tmp_path)}) + assert gateway.stdout.readline().strip() == "up" + updater = _child(_AT_LINE + """ + pid, marker = int(sys.argv[1]), Path(sys.argv[2]) + token = r.record_pause({"resume_needed": True, "profiles": {"default": pid}, + "identities": {str(pid): r.identity(pid)["ct"]}}, None, []) + r.mark_stop_requested(token, [pid], markers={pid: marker}) + marker.write_text(json.dumps({"target_pid": pid, "stopper_pid": os.getpid()}), encoding="utf-8") + print("requested", flush=True) + time.sleep(120) + """, str(gateway.pid), str(marker), env={"HERMES_HOME": str(tmp_path)}) + try: + assert updater.stdout.readline().strip() == "requested" + updater.send_signal(signal.SIGKILL) # windows-footgun: ok — killed before mark_stop_sent + updater.wait(timeout=10) + assert _launches(tmp_path, 1) == [[]] and len(_record_files(tmp_path)) == 1, "an issued request lost its debt" + marker.write_text(json.dumps({"target_pid": gateway.pid, "stopper_pid": 1}), encoding="utf-8") # a user's stop replaced it + assert _launches(tmp_path, 1) == [[]] and len(_record_files(tmp_path)) == 1, "resolved evidence was re-judged" + finally: + gateway.kill() + gateway.wait(timeout=10) + assert _launches(tmp_path, 2) == [["resume ['default']"], []] + + # --- R13: a checkout sharing the home never takes another checkout's paused set -------------- def _checkout_copy(root: Path) -> Path: """A second checkout: the real module file under another install root.""" diff --git a/tests/hermes_cli/test_windows_gateway_cold_start_desktop_lifecycle.py b/tests/hermes_cli/test_windows_gateway_cold_start_desktop_lifecycle.py index 64fc4367961de..7d30978192c90 100644 --- a/tests/hermes_cli/test_windows_gateway_cold_start_desktop_lifecycle.py +++ b/tests/hermes_cli/test_windows_gateway_cold_start_desktop_lifecycle.py @@ -104,7 +104,7 @@ def _running_beta_pause_fixture(monkeypatch, tmp_path): monkeypatch.setattr(update_cmd_windows, "_desktop_owns_gateway_lifecycle", lambda: True) beta = SimpleNamespace(pid=777, profile="beta") monkeypatch.setattr(update_cmd_windows, "_discover_windows_gateways", lambda: ({777: beta}, [], set(), [777])) - monkeypatch.setattr(update_cmd_windows, "_request_socket_pauses", lambda *a: ({"beta": 777}, [777], [])) + monkeypatch.setattr(update_cmd_windows, "_request_socket_pauses", lambda *a, **k: ({"beta": 777}, [777], [])) monkeypatch.setattr(cli_main, "_venv_launcher_ancestors", lambda pids: []) monkeypatch.setattr(cli_main, "_wait_for_windows_update_gateway_exit", lambda pids, timeout: set()) monkeypatch.setattr(profiles_mod, "get_active_profile_name", lambda: "default") @@ -194,7 +194,7 @@ def test_service_supervised_running_profile_is_not_cold_started(monkeypatch, tmp service = SimpleNamespace(name="HermesGw-beta", profile="beta", service_pid=800, gateway_pid=900, descendant_identities=(), service_create_time=1.0, gateway_create_time=2.0) monkeypatch.setattr(update_cmd_windows, "_discover_windows_gateways", lambda: ({900: svc_proc}, [service], {900}, [900])) - monkeypatch.setattr(update_cmd_windows, "_request_socket_pauses", lambda *a: ({}, [], [])) + monkeypatch.setattr(update_cmd_windows, "_request_socket_pauses", lambda *a, **k: ({}, [], [])) monkeypatch.setattr(update_cmd, "_stop_windows_gateway_service", lambda *a, **k: None) gateway_windows._write_start_attestation([900], "direct spawn (PID 900)", home=homes["beta"]) From 492cbfe72934667e886ceebc9d803970c5110061 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:55:34 -0700 Subject: [PATCH 19/63] test(e2e): native cells for the real pause killed before and right after its stop request (N1) (cherry picked from commit b1d8cadaee3744cba52d334d04b53ef81aef625a) (cherry picked from commit 7c637e180f438101b2a0d1b87bddd1732c4b1add) --- .../core/windows_update/test_pause_custody.py | 91 ++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/tests/e2e/core/windows_update/test_pause_custody.py b/tests/e2e/core/windows_update/test_pause_custody.py index 0a6a0a89e5f8e..4fc8b36e64f8d 100644 --- a/tests/e2e/core/windows_update/test_pause_custody.py +++ b/tests/e2e/core/windows_update/test_pause_custody.py @@ -17,6 +17,10 @@ must start it again. * readiness: a launch owes a gateway that comes back, one that never becomes ready, and an SCM service that cannot start. The ready one runs; only the other two stay owed. +* never asked: the REAL pause dies after recording its intent to stop the gateway, before asking it. + The gateway keeps serving and owes nothing; when the user stops it later, no launch restarts it. +* asked: the REAL pause dies right after issuing the request (planned-stop marker on disk), before + the socket call. The gateway drains and exits on its own; the next launch restarts it once. """ from __future__ import annotations @@ -224,7 +228,10 @@ def _draining(machine, out: dict) -> None: pid = {live} token = {{"resume_needed": True, "profiles": {{"default": pid}}, "identities": {{str(pid): r.identity(pid)["ct"]}}}} if hasattr(r, "mark_stop_requested"): - r.mark_stop_requested(r.record_pause(token, None, []), [pid]) + token = r.record_pause(token, None, []) + r.mark_stop_requested(token, [pid]) + if hasattr(r, "mark_stop_sent"): # the request went out; the gateway has not exited yet + r.mark_stop_sent(token, pid) else: # the old design has no stop record: write the same fact it would have needed r.write(r.stamp_tree({{**token, "stop_requested": [str(pid)]}}), owner=r.identity()) print("asked", flush=True) @@ -262,6 +269,53 @@ def _readiness(machine, out: dict) -> None: _clear(machine) +def _gone(pid: int, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while psutil.pid_exists(pid) and time.monotonic() < deadline: + time.sleep(0.2) + return not psutil.pid_exists(pid) + + +# The real pause, killed on one line of the installed update_cmd_windows (texts present before and +# after round 6, so the same cell runs on the red branch). +_PAUSE = """ +from hermes_cli import update_cmd_windows as w +stop_at(getattr(w, sys.argv[2]), (sys.argv[3],), "kill") +w._pause_windows_gateways_for_update() +""" + + +def _never_asked(machine, out: dict) -> None: + live = _fresh_gateway(machine) + out["never_kill"] = _run_driver(machine, "never-asked-update", _PAUSE, "_pause_windows_gateways_for_update", + "profiles = _stop_windows_gateways(") + out["never_intent"] = _owed(machine) + out["never_live_before"] = psutil.pid_exists(live) + out["never_launch_while"] = _launch(machine, "never-asked-while-serving") + out["never_live_after_launch"] = psutil.pid_exists(live) + out["never_owed_while"] = _owed(machine) + # The user's deliberate stop, after the update's obligation was retired. + out["never_user_stop"] = machine.hermes("gateway", "stop", label="never-asked-user-stop", timeout=180) + out["never_gone"] = _gone(live, 60) + out["never_launch_after"] = _launch(machine, "never-asked-after-user-stop") + out["never_running_after"] = _running(machine, live, 30) + out["never_owed_after"] = _owed(machine) + _clear(machine) + + +def _asked(machine, out: dict) -> None: + live = _fresh_gateway(machine) + out["asked_kill"] = _run_driver(machine, "asked-update", _PAUSE, "_request_socket_pauses", + "ack = pause_gateway_for_update(Path(proc.path))") + out["asked_owed_at_death"] = _owed(machine) + out["asked_gone"] = _gone(live, 120) # the gateway acts on the request on disk by itself + out["asked_launch"] = _launch(machine, "asked-after-exit") + out["asked_running"] = _running(machine, live, 150) + out["asked_again"] = _launch(machine, "asked-again") + out["asked_owed"] = _owed(machine) + _clear(machine) + + @pytest.fixture(scope="module") def journey(tmp_path_factory): out: dict = {} @@ -277,7 +331,7 @@ def journey(tmp_path_factory): off = machine.hermes("config", "set", "gateway.respawn_storm.max_starts", "0", label="storm-breaker-off") assert off.returncode == 0, fail_with(machine, "harness: could not disable the respawn-storm breaker", off) with machine.gateway_phase(): - for cell in (_claim_race, _publish_crash, _draining, _readiness): + for cell in (_claim_race, _publish_crash, _draining, _readiness, _never_asked, _asked): started = time.monotonic() try: cell(machine, out) @@ -350,3 +404,36 @@ def test_each_runtime_is_retired_only_on_its_own_readiness(journey) -> None: m, f"retired on another target's readiness (owed profiles {owed[0].get('profiles')})", run) assert owed[0].get("services") == [_MISSING_SERVICE], fail_with( m, f"the service that did not start is not owed (services {owed[0].get('services')})", run) + + +def test_a_gateway_the_update_never_asked_is_left_serving_and_a_user_stop_is_final(journey) -> None: + _no_error(journey, "never_asked") + m, during, after = journey["machine"], journey["never_launch_while"], journey["never_launch_after"] + rc, text = journey["never_kill"] + assert rc == 71 and len(journey["never_intent"]) == 1 and journey["never_live_before"], fail_with( + m, f"premise: the pause did not die between its intent and its request (rc={rc}, " + f"records={journey['never_intent']})\n{text}") + assert journey["never_live_after_launch"] and _restarts(during.stdout) == 0, fail_with( + m, "a launch restarted (replaced) a gateway the update never asked to stop", during) + assert journey["never_owed_while"] == [], fail_with( + m, f"a gateway never asked to stop is held as draining (owed while it serves: {journey['never_owed_while']})", + during) + assert journey["never_gone"], fail_with(m, "premise: the user's `hermes gateway stop` did not stop it", + journey["never_user_stop"]) + assert _restarts(after.stdout) == 0 and journey["never_running_after"] is None, fail_with( + m, "a gateway the user stopped was restarted for an update that never stopped it", after) + assert journey["never_owed_after"] == [], fail_with(m, f"debt left: {journey['never_owed_after']}", after) + + +def test_a_gateway_the_update_asked_is_restarted_once_after_it_exits(journey) -> None: + _no_error(journey, "asked") + m, run = journey["machine"], journey["asked_launch"] + rc, text = journey["asked_kill"] + assert rc == 71 and len(journey["asked_owed_at_death"]) == 1, fail_with( + m, f"premise: the pause did not die right after issuing its request (rc={rc}, " + f"records={journey['asked_owed_at_death']})\n{text}") + assert journey["asked_gone"], fail_with(m, "premise: the asked gateway never acted on the request on disk", run) + assert _restarts(run.stdout) == 1 and journey["asked_running"], fail_with( + m, "a gateway the update asked to stop was not started again after it exited", run) + assert _restarts(journey["asked_again"].stdout) == 0 and journey["asked_owed"] == [], fail_with( + m, f"a restarted gateway was owed again: {journey['asked_owed']}", journey["asked_again"]) From 8e39ffb3913482c5f7e6d033c8b73702b405ff57 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:22:20 -0700 Subject: [PATCH 20/63] fix(update): planned-stop marker paths are best effort (a process without a profile home has none) (cherry picked from commit 6b44b5ee73379e2959c09080f34eb5f8c2e97e9d) (cherry picked from commit cdf40f8a8ebef1a33488ecc3c6f19d95c14a6f63) --- hermes_cli/update_cmd_windows.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 29293d224ab65..d4b2a017c642a 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -44,6 +44,16 @@ def _planned_stop_marker_path(profile_path: Path) -> Path: return Path(profile_path) / ".gateway-planned-stop.json" +def _planned_stop_markers(running_pids, profile_processes, service_gateway_pids) -> dict: + """``{pid: marker path}`` for each gateway the socket pause will ask (profile-mapped, not SCM).""" + markers = {} + for pid in running_pids: + home = None if pid in service_gateway_pids else getattr(profile_processes.get(pid), "path", None) + if home: + markers[pid] = _planned_stop_marker_path(Path(home)) + return markers + + def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool: """Write a planned-stop marker into a specific profile home.""" try: @@ -959,9 +969,8 @@ def _pause_windows_gateways_for_update() -> dict | None: # The intent (and where each request goes), right before the first request. A live entry is owed # only once its request was issued (mark_stop_sent, or this update's marker still on disk): a # crash before that leaves it "never asked" (still serving, dropped), after it, draining (owed). - pause_record.mark_stop_requested(intended, running_pids, markers={ - pid: _planned_stop_marker_path(Path(profile_processes[pid].path)) for pid in running_pids - if pid in profile_processes and pid not in service_gateway_pids}) + pause_record.mark_stop_requested( + intended, running_pids, markers=_planned_stop_markers(running_pids, profile_processes, service_gateway_pids)) profiles = _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped, on_request=lambda pid: pause_record.mark_stop_sent(intended, pid)) token = {"resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped} From b113b62b33c8f0e6d1cdc94087caedc0b461af8b Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:48:48 -0700 Subject: [PATCH 21/63] fix(update): preserve pause debt across marker consumption and adoption Checkpoint validated stop consumption before unlinking requests and reread claimed tokens under custody. Reject stale and mismatched marker evidence, and retain cold-start generations when orphan pauses are adopted. Real-process regressions cover producer death, claim/consume ordering, failed checkpoint writes, marker validation, and successive orphan adoption. (cherry picked from commit 2f4fd37ebd61a2a6676763e62fa812acddb98a1b) (cherry picked from commit 4e16ee29bbe0e681f6692006f2b44b95c4e08dc7) --- gateway/status.py | 15 +- hermes_cli/update_pause_record.py | 52 +++- .../test_update_pause_boundaries.py | 224 ++++++++++++++++++ tests/hermes_cli/test_update_pause_record.py | 3 +- 4 files changed, 282 insertions(+), 12 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_boundaries.py diff --git a/gateway/status.py b/gateway/status.py index 51d01e9e706c3..14025c2b4fa41 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -1814,7 +1814,7 @@ def _pid_marker_names_self(target_pid: int, target_start_time: Any) -> bool: return None in (target_start_time, our_start_time) or target_start_time == our_start_time -def _consume_pid_marker_for_self(path: Path, *, ttl_s: int) -> bool: +def _consume_pid_marker_for_self(path: Path, *, ttl_s: int, on_consume=None) -> bool: parsed = _read_live_pid_marker(path, ttl_s) if parsed is None: return False @@ -1833,6 +1833,8 @@ def _consume_pid_marker_for_self(path: Path, *, ttl_s: int) -> bool: if replacer_home is not None and not _same_hermes_home(replacer_home, our_home): return False matches = _pid_marker_names_self(target_pid, target_start_time) + if matches and on_consume is not None: + on_consume(path, record) _unlink_quietly(path) return matches @@ -2071,9 +2073,14 @@ def write_planned_stop_marker(target_pid: int) -> bool: def consume_planned_stop_marker_for_self() -> bool: """Return True when the current process is being intentionally stopped.""" - return _consume_pid_marker_for_self( - _get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S - ) + from hermes_cli import update_pause_record + # Recovery must see either the request or its checkpoint, never the gap + # between validating/consuming the marker and scheduling asynchronous stop. + with update_pause_record._mutex(): + return _consume_pid_marker_for_self( + _get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S, + on_consume=update_pause_record.mark_stop_consumed, + ) def planned_stop_marker_targets_self() -> bool: diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 471e1f60385a1..287fe8059df84 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -241,8 +241,8 @@ def write(token: dict, *, owner: dict | None = None, path: Path | None = None) - def mark_stop_requested(token: dict, pids, markers: dict | None = None) -> None: """Persist, BEFORE the first stop request, the stops this update intends and where each mapped gateway's request (its planned-stop marker) will be written. Intent is not a request: a live - process counts as asked only once :func:`mark_stop_sent` recorded it, or while the request this - update wrote is still on disk (:func:`_asked`).""" + process counts as asked only once the producer or consumer checkpoints it, or while the + valid request this update wrote is still on disk (:func:`_asked`).""" token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *(str(int(p)) for p in pids)}) token["stop_sent"] = sorted(map(str, token.get("stop_sent") or [])) token["stopper_pid"] = os.getpid() @@ -253,7 +253,7 @@ def mark_stop_requested(token: dict, pids, markers: dict | None = None) -> None: def mark_stop_sent(token: dict, pid) -> None: """Persist that *pid*'s stop request was issued (its planned-stop marker is on disk): from here a live *pid* is draining and keeps its restart debt until it exits. Best effort: the marker itself - is the evidence until this lands, and the token carries it to the next record write.""" + is evidence until this lands; its consumer checkpoints before unlinking it.""" token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), str(int(pid))}) try: write(token, owner=identity()) @@ -261,6 +261,25 @@ def mark_stop_sent(token: dict, pid) -> None: print(f" ⚠ Could not record the stop request for gateway {pid}: {exc}") +def mark_stop_consumed(path: Path, marker: dict) -> None: + """Checkpoint a validated gateway consume BEFORE its marker disappears. + + Called under the pause mutex by the consumer. Preserve the holder's identity; + this is evidence of a stop, not a transfer of the updater's custody. + """ + from gateway.status import _same_hermes_home + pid = str(marker["target_pid"]) + for src, body in _files(record_path()): + token = body["token"] + recorded_path = (token.get("stop_markers") or {}).get(pid) + if (str(token.get("stopper_pid")) != str(marker.get("stopper_pid")) + or pid not in (token.get("stop_requested") or []) + or not recorded_path or not _same_hermes_home(recorded_path, path)): + continue + token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), pid}) + _atomic_write(src, body) # Failure leaves the request intact for recovery. + + def discharge(token: dict, path: Path | None = None) -> None: """Remove the record only while it still names this pause (decided under the mutex).""" path = path or record_path() @@ -404,7 +423,9 @@ def adopt_orphans() -> tuple[dict | None, list[Path]]: if won is None: continue claims.append(won[0]) - adopted = merge_into(adopted, drop_never_stopped(dict(won[1]["token"]))) + with _mutex(): + body = read(won[0]) or won[1] + adopted = merge_into(adopted, drop_never_stopped(dict(body["token"]))) return adopted, claims @@ -468,9 +489,16 @@ def _request_on_disk(token: dict, pid: str) -> bool: path = (token.get("stop_markers") or {}).get(str(pid)) if not path: return False + from gateway.status import _marker_is_stale, _PLANNED_STOP_MARKER_TTL_S, get_process_start_time try: marker = json.loads(Path(path).read_text(encoding="utf-8-sig")) - return int(marker["target_pid"]) == int(pid) and int(marker["stopper_pid"]) == int(token["stopper_pid"]) + if (int(marker["target_pid"]) != int(pid) + or int(marker["stopper_pid"]) != int(token["stopper_pid"]) + or _marker_is_stale(marker.get("written_at") or "", _PLANNED_STOP_MARKER_TTL_S)): + return False + expected, actual = marker.get("target_start_time"), get_process_start_time(int(pid)) + # Match the consumer's optional birth fingerprint, including unavailable clocks. + return None in (expected, actual) or expected == actual except (OSError, ValueError, TypeError, KeyError): return False @@ -526,6 +554,13 @@ def merge_into(token: dict | None, adopted: dict) -> dict: requested = sent = list(adopted.get("identities") or {}) token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *map(str, requested)}) token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), *map(str, sent or [])}) + cold = token.setdefault("cold_start_profiles", {}) + for name, generation in (adopted.get("cold_start_profiles") or {}).items(): + cold.setdefault(name, generation) + if adopted.get("cold_start_if_installed"): + token["cold_start_if_installed"] = True + if "attested_generation" in adopted: + token.setdefault("attested_generation", adopted["attested_generation"]) services = token.setdefault("services", []) services.extend(s for s in adopted.get("services") or [] if s not in services) if services: @@ -541,8 +576,11 @@ def _has_work(token: dict) -> bool: def _resume_claimed(claim_path: Path, body: dict) -> None: - token = drop_never_stopped(dict(body["token"])) - draining = split_draining(token) + with _mutex(): + # A live consumer may have checkpointed after claim() returned its snapshot. + body = read(claim_path) or body + token = drop_never_stopped(dict(body["token"])) + draining = split_draining(token) token.update(resume_needed=True, recovery=True) try: if _has_work(token): diff --git a/tests/hermes_cli/test_update_pause_boundaries.py b/tests/hermes_cli/test_update_pause_boundaries.py new file mode 100644 index 0000000000000..d86b631c01baa --- /dev/null +++ b/tests/hermes_cli/test_update_pause_boundaries.py @@ -0,0 +1,224 @@ +"""Real-process pause/consume/recovery contracts, not native Windows SCM evidence.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import textwrap +import time + +import pytest + +REPO = Path(__file__).resolve().parents[2] + + +def _wait(path, process): + deadline = time.monotonic() + 30 + while not path.exists(): + assert process.poll() is None, process.communicate() + assert time.monotonic() < deadline, str(path) + time.sleep(0.01) + + +@pytest.fixture +def cell(tmp_path): + # An unchanged copy owns its private checkout lock/root; never the live install's. + root = tmp_path / "checkout" + (root / "hermes_cli").mkdir(parents=True) + shutil.copyfile(REPO / "hermes_cli/update_pause_record.py", root / "hermes_cli/update_pause_record.py") + home = tmp_path / "home" + (home / ".hermes").mkdir(parents=True) + env = {**os.environ, "HOME": str(home), "USERPROFILE": str(home), + "HERMES_HOME": str(home / ".hermes"), "PYTHONPATH": str(REPO), + "PYTHONDONTWRITEBYTECODE": "1"} + children = [] + + def launch(code, *args): + script = tmp_path / f"child-{len(children)}.py" + script.write_text(textwrap.dedent(''' + import importlib.util, json, os, sys, time + from pathlib import Path + root = Path(sys.argv[1]) + spec = importlib.util.spec_from_file_location("hermes_cli.update_pause_record", root / "hermes_cli/update_pause_record.py") + r = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = r + spec.loader.exec_module(r) + ''') + textwrap.dedent(code), encoding="utf-8") + proc = subprocess.Popen([sys.executable, str(script), str(root), *map(str, args)], + cwd=tmp_path, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + children.append(proc) + return proc + + yield tmp_path, home / ".hermes", launch + for proc in children: + if proc.poll() is None: + proc.kill() + proc.communicate(timeout=10) + + +def _result(proc): + out, err = proc.communicate(timeout=30) + assert proc.returncode == 0, (out, err) + return json.loads(out.splitlines()[-1]) + + +@pytest.mark.parametrize("claim_first", [False, True]) +def test_consumption_checkpoints_debt_before_unlink_and_producer_death(cell, claim_first): + base, home, launch = cell + consumer = launch(''' + from gateway.status import consume_planned_stop_marker_for_self + Path(sys.argv[2]).write_text(str(os.getpid()), encoding="utf-8") + while not Path(sys.argv[3]).exists(): time.sleep(.01) + accepted = consume_planned_stop_marker_for_self() + Path(sys.argv[4]).write_text(json.dumps(accepted), encoding="utf-8") + while True: time.sleep(.05) + ''', base / "ready", base / "consume", base / "consumed") + _wait(base / "ready", consumer) + owner = launch(''' + from hermes_cli import update_cmd_windows as w + from types import SimpleNamespace + pid = int(sys.argv[2]) + home = Path(os.environ["HERMES_HOME"]) + token = r.record_pause({"resume_needed": True, "profiles": {"default": pid}, + "identities": {str(pid): r.identity(pid)["ct"]}}, None, []) + # Discovery and the gateway can spell the same profile path differently. + r.mark_stop_requested(token, [pid], {pid: home / ".." / ".hermes" / ".gateway-planned-stop.json"}) + # The real callback is the first operation after marker publication. Hold it + # before its checkpoint while another interpreter runs the actual consumer. + def on_request(pid): + Path(sys.argv[3]).touch() + while True: time.sleep(.05) + w._request_socket_pauses([pid], {pid: SimpleNamespace(profile="default", path=home)}, + set(), on_request=on_request) + ''', consumer.pid, base / "published") + _wait(base / "published", owner) + recovery = None + if claim_first: + owner.kill() + owner.communicate(timeout=10) + recovery = launch(''' + won = r.claim(r.record_path()) + assert won is not None + Path(sys.argv[2]).touch() + while not Path(sys.argv[3]).exists(): time.sleep(.01) + r._resume_claimed(*won) + print(json.dumps([body["token"] for _, body in r.orphans()])) + ''', base / "claimed", base / "recover") + _wait(base / "claimed", recovery) + (base / "consume").touch() + _wait(base / "consumed", consumer) + assert json.loads((base / "consumed").read_text(encoding="utf-8-sig")) is True + assert not (home / ".gateway-planned-stop.json").exists() + if owner.poll() is None: + owner.kill() + owner.communicate(timeout=10) + assert consumer.poll() is None + if recovery is not None: + (base / "recover").touch() + else: + recovery = launch(''' + r.recover(["status"]) + print(json.dumps([body["token"] for _, body in r.orphans()])) + ''') + after = _result(recovery) + assert len(after) == 1, "consumed request lost all durable restart debt" + assert after[0]["profiles"] == {"default": consumer.pid} + assert after[0]["stop_sent"] == [str(consumer.pid)] + + +def test_failed_consumer_checkpoint_keeps_request_and_user_stop_is_not_update_debt(cell): + _, _, launch = cell + result = _result(launch(''' + from gateway import status + from hermes_cli.update_cmd_windows import _write_update_planned_stop_marker + pid = os.getpid() + home = Path(os.environ["HERMES_HOME"]) + path = home / ".gateway-planned-stop.json" + token = r.record_pause({"resume_needed": True, "profiles": {"default": pid}, + "identities": {str(pid): r.identity(pid)["ct"]}}, None, []) + r.mark_stop_requested(token, [pid], {pid: path}) + assert _write_update_planned_stop_marker(home, pid) + # An actual directory at the atomic writer's temp path rejects publication. + obstruction = r.record_path().with_name(f"{r.record_path().name}.{pid}.tmp") + obstruction.mkdir() + try: + status.consume_planned_stop_marker_for_self() + except OSError: + pass + else: + raise AssertionError("consume succeeded without its durable checkpoint") + retained = path.exists() and r.read()["token"]["stop_sent"] == [] + obstruction.rmdir() + marker = json.loads(path.read_text(encoding="utf-8-sig")) + path.write_text(json.dumps({**marker, "stopper_pid": pid + 1}), encoding="utf-8") + user_consumed = status.consume_planned_stop_marker_for_self() + user_debt = r.read()["token"]["stop_sent"] + assert _write_update_planned_stop_marker(home, pid) + accepted = status.consume_planned_stop_marker_for_self() + print(json.dumps({"retained": retained, "user_consumed": user_consumed, "user_debt": user_debt, + "accepted": accepted, "sent": r.read()["token"]["stop_sent"], "pid": pid})) + ''')) + assert result == {"retained": True, "user_consumed": True, "user_debt": [], + "accepted": True, "sent": [str(result["pid"])], "pid": result["pid"]} + + +def test_recovery_only_accepts_markers_the_live_consumer_accepts(cell): + _, _, launch = cell + result = _result(launch(''' + from datetime import datetime, timedelta, timezone + from gateway import status + pid = os.getpid() + home = Path(os.environ["HERMES_HOME"]) + marker_path = home / ".gateway-planned-stop.json" + token = {"profiles": {"default": pid}, "identities": {str(pid): r.identity(pid)["ct"]}, + "stopper_pid": pid, "stop_requested": [str(pid)], "stop_markers": {str(pid): str(marker_path)}} + current = status.get_process_start_time(pid) + assert current is not None + fresh = {"target_pid": pid, "stopper_pid": pid, "target_start_time": current, + "written_at": datetime.now(timezone.utc).isoformat()} + cases = {"valid": fresh, + "expired": {**fresh, "written_at": (datetime.now(timezone.utc) - timedelta(seconds=120)).isoformat()}, + "wrong_incarnation": {**fresh, "target_start_time": current + 1}, + "malformed": {"target_pid": pid, "stopper_pid": pid}} + out = {} + for name, marker in cases.items(): + marker_path.write_text(json.dumps(marker), encoding="utf-8") + kept = r.drop_never_stopped(dict(token)) + consumed = status.consume_planned_stop_marker_for_self() + out[name] = {"kept": bool(kept["profiles"]), "consumed": consumed} + print(json.dumps(out)) + ''')) + assert result["valid"] == {"kept": True, "consumed": True} + for name in ("expired", "wrong_incarnation", "malformed"): + assert result[name] == {"kept": False, "consumed": False}, (name, result) + + +def test_orphan_adoption_preserves_cold_start_generation_and_current_precedence(cell): + _, _, launch = cell + _result(launch(''' + token = {"resume_needed": True, "profiles": {}, "cold_start_if_installed": True, + "attested_generation": "old-active", "cold_start_profiles": {"beta": "old-beta", "gamma": "old-gamma"}} + r.write(r.stamp_tree(token), owner=r.UNOWNED) + print(json.dumps(r.read()["token"])) + ''')) + adopted = _result(launch(''' + adopted, claims = r.adopt_orphans() + token = {"resume_needed": True, "profiles": {}, "cold_start_profiles": {"beta": "current-beta"}} + r.record_pause(token, adopted, claims) + print(json.dumps(r.read()["token"])) + ''')) + assert adopted.get("cold_start_if_installed") is True + assert adopted.get("attested_generation") == "old-active" + assert adopted.get("cold_start_profiles") == {"beta": "current-beta", "gamma": "old-gamma"} + again = _result(launch(''' + adopted, claims = r.adopt_orphans() + r.record_pause({"resume_needed": True, "profiles": {}, "cold_start_if_installed": True, + "attested_generation": "current-active"}, adopted, claims) + print(json.dumps(r.read()["token"])) + ''')) + assert again["attested_generation"] == "current-active" + assert again["cold_start_profiles"] == adopted["cold_start_profiles"] diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 336acc9cec1a1..b0bb17cb8fe3c 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -405,7 +405,8 @@ def test_a_request_on_disk_is_owed_even_before_it_is_recorded_sent(tmp_path): token = r.record_pause({"resume_needed": True, "profiles": {"default": pid}, "identities": {str(pid): r.identity(pid)["ct"]}}, None, []) r.mark_stop_requested(token, [pid], markers={pid: marker}) - marker.write_text(json.dumps({"target_pid": pid, "stopper_pid": os.getpid()}), encoding="utf-8") + from hermes_cli.update_cmd_windows import _write_update_planned_stop_marker + assert _write_update_planned_stop_marker(marker.parent, pid) print("requested", flush=True) time.sleep(120) """, str(gateway.pid), str(marker), env={"HERMES_HOME": str(tmp_path)}) From cf50c3f2d1153f8dfcbee6a77018a94f06127dac Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:24:16 -0700 Subject: [PATCH 22/63] fix(update): a completed pause obligation never runs again from a copy Windows kept Completion (a recovering launch's finished claim, an update's discharged record) now lists the obligation's ids (and every id it absorbed) on a durable retired list before unlinking its files. A reader holding a file without FILE_SHARE_DELETE makes the unlink fail; that leftover copy is now redundant by id instead of an orphan whose stale set the next launch resumes again. Same-id copies keep the furthest-progressed one (a claim's rev), so a partly resumed claim is not shadowed by the stale source. The list is pruned once no readable file carries an id. Native windows tests hold the file open from a real second process. (cherry picked from commit 7f5b98a0e10ce2f578ae576481b51dc8029f4e99) (cherry picked from commit 4fba720f1e9879f93b9d17d4a5c6bb4980ffe2ae) --- hermes_cli/update_pause_record.py | 69 +++++++++-- .../test_update_pause_debt_windows_live.py | 111 ++++++++++++++++++ 2 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_debt_windows_live.py diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 287fe8059df84..a71e8975c57d9 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -24,7 +24,9 @@ two leaves two files with ONE obligation id (``pause_id``), and an update that folds orphaned sets into its own record lists their ids under ``absorbed`` before retiring them. Readers treat a file whose id another file carries as a copy, never as a second obligation, and the next mutator -retires it. +retires it. Completing an obligation first adds its ids to ``.retired`` (atomic), then +unlinks its files: an unlink Windows refuses (a reader holding the file without +FILE_SHARE_DELETE) leaves a copy that is redundant by id, never an obligation that executes again. """ from __future__ import annotations @@ -286,8 +288,7 @@ def discharge(token: dict, path: Path | None = None) -> None: with _mutex(): existing = read(path) if existing is not None and existing["token"].get("pause_id") == token.get("pause_id"): - with suppress(OSError): - path.unlink() + _retire(path, [(path, existing)]) def sync(token: dict) -> None: @@ -317,6 +318,31 @@ def _claims(path: Path) -> list[Path]: return sorted(path.parent.glob(f"{glob.escape(path.name)}.*.claim")) +def _retired_path(path: Path) -> Path: + return path.with_suffix(".retired") + + +def _retired(path: Path) -> set[str]: + try: + ids = json.loads(_retired_path(path).read_text(encoding="utf-8-sig")).get("ids") + except (OSError, ValueError, AttributeError): + return set() + return {str(i) for i in ids} if isinstance(ids, list) else set() + + +def _retire(path: Path, carriers: list[tuple[Path, dict]]) -> None: + """Complete the obligations *carriers* hold (call under the mutex): their ids (and every id + they absorbed) go on the durable retired list FIRST; a failed write raises and deletes + nothing. Then the files go; one that cannot be deleted is a redundant copy from here on.""" + ids = {str(i) for _src, body in carriers + for i in [body["token"].get("pause_id"), *(body["token"].get("absorbed") or [])] if i} + if not ids <= _retired(path): + _atomic_write(_retired_path(path), {"schema": 1, "ids": sorted(_retired(path) | ids)}) + for src, _body in carriers: + with suppress(OSError): + src.unlink() + + def _holder(src: Path, body: dict) -> dict: """Who holds *src*: a record's owner, a claim's claimer (published with the claim itself).""" held_by = body.get("owner") if src.suffix == ".json" else body.get("claimer") @@ -333,9 +359,10 @@ def _files(path: Path) -> list[tuple[Path, dict]]: return found -def _redundant(found: list[tuple[Path, dict]], held: set[Path]) -> set[Path]: - """Files that only copy an obligation another file carries: a source an update absorbed but - died before retiring, or one of two same-id files a claim transfer left. Never a held file.""" +def _redundant(found: list[tuple[Path, dict]], held: set[Path], retired: set[str] = frozenset()) -> set[Path]: + """Files that only copy an obligation another file carries or that was completed: a source an + update absorbed but died before retiring, one of two same-id files a claim transfer left (the + furthest-progressed copy is kept), a retired id's leftover. Never a held file.""" carriers: dict[str, Path] = {} for src, body in found: for oid in body["token"].get("absorbed") or []: @@ -346,12 +373,13 @@ def _redundant(found: list[tuple[Path, dict]], held: set[Path]) -> set[Path]: oid = str(body["token"].get("pause_id") or "") if not oid: continue - if oid in carriers and carriers[oid] != src: + if oid in retired or (oid in carriers and carriers[oid] != src): redundant.add(src) else: by_id.setdefault(oid, []).append(src) + rev = {src: int(body.get("rev") or 0) for src, body in found} for copies in by_id.values(): - keep = next((s for s in copies if s in held), copies[0]) + keep = next((s for s in copies if s in held), max(copies, key=lambda s: rev[s])) redundant.update(s for s in copies if s != keep) return redundant - held @@ -359,7 +387,7 @@ def _redundant(found: list[tuple[Path, dict]], held: set[Path]) -> set[Path]: def _survey(path: Path) -> tuple[list[tuple[Path, dict]], set[Path], set[Path]]: found = _files(path) held = {src for src, body in found if identity_is_live(_holder(src, body))} - return found, held, _redundant(found, held) + return found, held, _redundant(found, held, _retired(path)) def orphans(path: Path | None = None) -> list[tuple[Path, dict]]: @@ -381,13 +409,31 @@ def orphaned_record(path: Path | None = None) -> dict | None: def retire_redundant(path: Path | None = None) -> None: """Delete copies of obligations another file already carries (crash leftovers of a transfer).""" path = path or record_path() - if not path.exists() and not _claims(path): + if not path.exists() and not _claims(path) and not _retired_path(path).exists(): return with suppress(RecordBusy), _mutex(): _found, _held, redundant = _survey(path) for src in redundant: with suppress(OSError): src.unlink() + _prune_retired(path) + + +def _prune_retired(path: Path) -> None: + """Forget retired ids no file carries any more — only when every candidate file was readable + (one Windows would not let us read may still carry a retired id).""" + retired = _retired(path) + candidates = [p for p in (path, *_claims(path)) if p.exists()] + bodies = [read(p) for p in candidates] + if not retired or None in bodies: + return + carried = {str(b["token"].get("pause_id")) for b in bodies} + keep = retired & carried + with suppress(OSError): + if keep: + _atomic_write(_retired_path(path), {"schema": 1, "ids": sorted(keep)}) + else: + _retired_path(path).unlink() def claim(src: Path) -> tuple[Path, dict] | None: @@ -405,6 +451,7 @@ def claim(src: Path) -> tuple[Path, dict] | None: if src in held or src in redundant or src not in {s for s, _ in _found}: return None body["claimer"] = identity() + body["rev"] = int(body.get("rev") or 0) + 1 # a copy left behind is never the newer one _atomic_write(mine, body) with suppress(OSError): src.unlink() @@ -608,7 +655,7 @@ def _hand_back(claim_path: Path, body: dict, token: dict, draining: dict) -> Non kept = {key: value for key, value in token.items() if key not in ("recovery", "resume_deferred")} _atomic_write(claim_path, {**body, "token": {**kept, "resume_needed": True}, "claimer": UNOWNED}) else: - claim_path.unlink() + _retire(record_path(), [(claim_path, read(claim_path) or body)]) def recover(argv: list[str] | None = None) -> None: diff --git a/tests/hermes_cli/test_update_pause_debt_windows_live.py b/tests/hermes_cli/test_update_pause_debt_windows_live.py new file mode 100644 index 0000000000000..086f868a05422 --- /dev/null +++ b/tests/hermes_cli/test_update_pause_debt_windows_live.py @@ -0,0 +1,111 @@ +"""Native Windows restart-debt boundaries of the update pause (#132338 R8): real files held the way +Windows readers hold them, a real SCM service, real processes and taskkill. Nothing here is mocked. + +- A completed obligation stays completed when Windows refuses to delete one of its copies + (a reader without FILE_SHARE_DELETE): the copy must never execute again. +- An SCM service reporting ``running`` retires its debt only once the gateway it supervises is ready. +- The pause's force-kill is authorized by the birth discovered before the drain, not one re-read + after it (a PID reused during the drain reads its replacement's own birth). +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +import uuid +from pathlib import Path + +import pytest + +from hermes_cli import update_pause_record as pause_record + +REPO = Path(__file__).resolve().parents[2] +pytestmark = [pytest.mark.platforms("windows"), pytest.mark.live_system_guard_bypass] + + +def _hold(path: Path) -> subprocess.Popen: + """Another process reading *path* with a plain ``open`` (shares read/write, not delete).""" + proc = subprocess.Popen( + [sys.executable, "-c", "import sys; f = open(sys.argv[1], 'rb'); print('held', flush=True); sys.stdin.read()", + str(path)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, encoding="utf-8") + assert proc.stdout.readline().strip() == "held" + return proc + + +def _release(proc: subprocess.Popen) -> None: + proc.stdin.close() + proc.wait(timeout=30) + + +def _record_files(home: Path) -> list[str]: + return sorted(p.name for p in home.glob(pause_record.RECORD_STEM + "*") if p.suffix in (".json", ".claim")) + + +def _orphan(profiles: dict) -> dict: + token = {"pause_id": uuid.uuid4().hex, "resume_needed": True, "profiles": profiles} + pause_record.write(token, owner=pause_record.UNOWNED) + return token + + +_UPDATER = """ +import sys +from hermes_cli import update_pause_record as r +token = {"pause_id": sys.argv[1], "resume_needed": True, "profiles": {"default": 4242}} +r.write(token, owner=r.identity()) +print("written", flush=True) +sys.stdin.readline() +r.discharge(token) +print("discharged", flush=True) +""" + + +@pytest.mark.parametrize("carrier", ["claim", "record"]) +def test_a_completed_obligation_never_runs_again_from_a_copy_windows_kept(tmp_path, monkeypatch, carrier): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + src = pause_record.record_path() + if carrier == "claim": # a recovering launch claims an orphan and finishes its resume + _orphan({"default": 4242, "work": 4343}) + holder = _hold(src) + try: + won = pause_record.claim(src) + assert won is not None and src.exists(), "premise: Windows refused to delete the claimed source" + pause_record._hand_back(won[0], won[1], {**won[1]["token"], "resume_needed": False, "profiles": {}}, + {"profiles": {}, "unmapped": []}) + assert pause_record.orphans() == [], "the completed set is owed again from the copy Windows kept" + finally: + _release(holder) + else: # the update that paused them resumes them all and discharges its record, then exits + updater = subprocess.Popen([sys.executable, "-c", textwrap.dedent(_UPDATER), uuid.uuid4().hex], + cwd=REPO, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, encoding="utf-8", + env={**os.environ, "HERMES_HOME": str(tmp_path), "PYTHONPATH": str(REPO)}) + assert updater.stdout.readline().strip() == "written" + holder = _hold(src) + try: + out, _ = updater.communicate("go\n", timeout=60) + assert "discharged" in out and src.exists(), "premise: Windows refused to delete the discharged record" + assert pause_record.orphans() == [], "a dead updater's discharged record is owed again" + finally: + _release(holder) + pause_record.retire_redundant() # the next launch, once nothing holds the copy + assert _record_files(tmp_path) == [] + assert pause_record.orphans() == [] + assert not src.with_suffix(".retired").exists(), "the retired list outlived every copy it guarded" + + +def test_a_partly_resumed_claim_is_what_the_next_launch_resumes_not_the_copy_windows_kept(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + src = pause_record.record_path() + _orphan({"default": 4242, "work": 4343}) + holder = _hold(src) + try: + won = pause_record.claim(src) + assert won is not None and src.exists(), "premise: Windows refused to delete the claimed source" + # default came back; work did not: the claim goes back unowned still owing work only. + pause_record._hand_back(won[0], won[1], {**won[1]["token"], "profiles": {"work": 4343}}, + {"profiles": {}, "unmapped": []}) + owed = [body["token"]["profiles"] for _src, body in pause_record.orphans()] + finally: + _release(holder) + assert owed == [{"work": 4343}], "the next launch would restart a gateway this launch already restarted" From aa9023c2e263b77bfca9f11362660337923ead8f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:24:30 -0700 Subject: [PATCH 23/63] fix(update): an SCM service retires its restart debt only once its gateway is ready SCM 'running' proves the service wrapper started, not that the gateway it supervises came up: a service-only pause token was discharged (and its durable record deleted) while the gateway was absent or crashed, losing the retry. Each restarted service now leaves the token only after a stable gateway in its profile home, descended from the service's current process, passes the same readiness poll relaunched profiles use; otherwise it stays owed. Native windows test: a real compiled service wrapper under the SCM, absent gateway (stays owed) and a real 'hermes gateway run' (retired). (cherry picked from commit d3d8349c82122773017a0fdb645057831557feaa) (cherry picked from commit ea0758b6ba58b489843e33f2201393efb917535e) --- hermes_cli/update_cmd_windows.py | 32 ++++++- .../test_update_pause_debt_windows_live.py | 93 +++++++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index d4b2a017c642a..8ef69c9e23c90 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -1229,8 +1229,36 @@ def _refresh_bootstrap_cache_scripts(branch: str = "main") -> None: print(" ✓ Refreshed installer bootstrap-cache script(s): " + ", ".join(sorted(refreshed))) +_SERVICE_READY_TIMEOUT_S = 60.0 + + +def _service_gateway_ready(name: str, profile: str | None) -> list[int]: + """The stable gateway *name*'s service process supervises (in *profile*'s home when known): SCM + ``running`` proves only the wrapper started, not that the gateway it hosts came up.""" + from hermes_cli import gateway_windows + from hermes_cli.profiles import get_profile_dir + psutil, service = _win_service(name) + + def under_service(pids): + try: + service_pid = int(service.pid() or 0) + except Exception: + return [] + owned = [] + for pid in pids: + with suppress(Exception): + if service_pid and service_pid in {int(p.pid) for p in psutil.Process(int(pid)).parents()}: + owned.append(int(pid)) + return owned + + home = Path(get_profile_dir(profile)) if profile else None + return gateway_windows._wait_for_gateway_ready( + timeout_s=_SERVICE_READY_TIMEOUT_S, home=home, all_profiles=home is None, pid_filter=under_service) + + def _resume_windows_services(token: dict) -> None: - """Restart the SCM services recorded on *token*; failed ones stay on the token so a retry sees them.""" + """Restart the SCM services recorded on *token*; one leaves the token only once the gateway it + supervises is ready. Failed ones stay on the token so a retry sees them.""" from hermes_cli.update_cmd import _start_windows_gateway_service services = list(token.get("services") or []) token.setdefault("expected_services", list(services)) @@ -1240,6 +1268,8 @@ def _resume_windows_services(token: dict) -> None: for service_name in map(str, services): try: _start_windows_gateway_service(service_name) + if not _service_gateway_ready(service_name, (token.get("service_profiles") or {}).get(service_name)): + raise RuntimeError(f"Windows service {service_name} is running but its gateway did not become ready") restarted_services.append(service_name) if service_name not in verified_restarts: verified_restarts.append(service_name) diff --git a/tests/hermes_cli/test_update_pause_debt_windows_live.py b/tests/hermes_cli/test_update_pause_debt_windows_live.py index 086f868a05422..95507c05f9444 100644 --- a/tests/hermes_cli/test_update_pause_debt_windows_live.py +++ b/tests/hermes_cli/test_update_pause_debt_windows_live.py @@ -15,8 +15,10 @@ import sys import textwrap import uuid +from contextlib import suppress from pathlib import Path +import psutil import pytest from hermes_cli import update_pause_record as pause_record @@ -109,3 +111,94 @@ def test_a_partly_resumed_claim_is_what_the_next_launch_resumes_not_the_copy_win finally: _release(holder) assert owed == [{"work": 4343}], "the next launch would restart a gateway this launch already restarted" + + +# --- SCM: a running wrapper is not a ready gateway --------------------------------------------- + +_WRAPPER_CS = r""" +using System; +using System.Diagnostics; +using System.IO; +using System.ServiceProcess; + +public class Wrapper : ServiceBase { + Process child; + protected override void OnStart(string[] args) { + string[] spec = File.ReadAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "child.txt")); + ProcessStartInfo psi = new ProcessStartInfo(spec[0], spec[1]); + psi.UseShellExecute = false; + psi.CreateNoWindow = true; + psi.WorkingDirectory = spec[2]; + for (int i = 3; i < spec.Length; i++) { + int eq = spec[i].IndexOf('='); + if (eq > 0) psi.EnvironmentVariables[spec[i].Substring(0, eq)] = spec[i].Substring(eq + 1); + } + child = Process.Start(psi); + } + public static void Main() { ServiceBase.Run(new Wrapper()); } +} +""" + + +@pytest.fixture(scope="module") +def service_wrapper(tmp_path_factory) -> Path: + """A real SCM service executable that starts one configured child process, like a service wrapper.""" + windir = Path(os.environ.get("WINDIR", r"C:\Windows")) / "Microsoft.NET" + csc = next((c for c in sorted(windir.glob("Framework*/v4.*/csc.exe"), reverse=True)), None) + assert csc is not None, f"no .NET Framework C# compiler under {windir}" + out = tmp_path_factory.mktemp("scm-wrapper") + (out / "wrapper.cs").write_text(_WRAPPER_CS, encoding="utf-8") + subprocess.run([str(csc), "/nologo", "/target:exe", "/reference:System.ServiceProcess.dll", + f"/out:{out / 'wrapper.exe'}", str(out / "wrapper.cs")], check=True, capture_output=True) + return out / "wrapper.exe" + + +def _sc(*args: str) -> subprocess.CompletedProcess: + return subprocess.run(["sc.exe", *args], capture_output=True, text=True, encoding="utf-8", errors="replace", + check=False) + + +@pytest.mark.parametrize("gateway", ["absent", "ready"]) +def test_a_running_service_owes_its_gateway_until_that_gateway_is_ready(tmp_path, monkeypatch, service_wrapper, gateway): + from hermes_cli import update_cmd_windows as w + + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + bin_dir = tmp_path / "svc" + bin_dir.mkdir() + exe = bin_dir / "wrapper.exe" + exe.write_bytes(service_wrapper.read_bytes()) + child = ([sys.executable, "-c \"import time; time.sleep(900)\""] if gateway == "absent" + else [sys.executable, "-m hermes_cli.main gateway run"]) + (bin_dir / "child.txt").write_text("\n".join([*child, str(REPO), f"HERMES_HOME={home}", f"PYTHONPATH={REPO}", + "PYTHONIOENCODING=utf-8"]) + "\n", encoding="utf-8") + name = f"hermes-r8-debt-{gateway}-{os.getpid()}" + created = _sc("create", name, "binPath=", f'"{exe}"', "start=", "demand") + assert created.returncode == 0, created.stdout + created.stderr + if gateway == "absent": # the budget only bounds how long the miss takes to report + monkeypatch.setattr(w, "_SERVICE_READY_TIMEOUT_S", 20.0, raising=False) + token = {"services": [name], "expected_services": [name], "restarted_services": [], "service_profiles": {name: "default"}} + try: + if gateway == "absent": + with pytest.raises(RuntimeError): + w._resume_windows_services(token) + assert token["services"] == [name], "SCM 'running' retired the debt of a gateway that never started" + else: + try: + w._resume_windows_services(token) + except RuntimeError as exc: + pytest.fail(f"{exc}\n{_logs(home)}") + assert token["services"] == [] and token["restarted_services"] == [name] + finally: + pid = 0 + with suppress(Exception): + pid = int(psutil.win_service_get(name).pid() or 0) # type: ignore[attr-defined] + if pid: + subprocess.run(["taskkill", "/T", "/F", "/PID", str(pid)], capture_output=True, check=False) + _sc("delete", name) + + +def _logs(home: Path) -> str: + return "\n".join(f"--- {p}\n{p.read_text(encoding='utf-8-sig', errors='replace')[-3000:]}" + for p in sorted((home / "logs").glob("*.log"))) From f3c44e790499c7a2c2bd9dc50500aef7d9233182 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:24:30 -0700 Subject: [PATCH 24/63] fix(update): the pause force-kills only the processes it discovered The force-kill after the drain passed get_process_start_time(pid) read at kill time as the expected identity, so a PID that exited during the drain and was reused validated against its replacement's own birth. Births are now captured at discovery (a mapped gateway's strict PID-file identity first, the rest right after discovery, launchers before the first request) and the kill is authorized by those; a PID with no discovered birth is still refused. Native windows test drives _stop_windows_gateways against a real survivor whose discovered birth is its own (killed) or an earlier process's (spared). (cherry picked from commit 9e01b731511996da4f8300d0c147a1ce4859742e) (cherry picked from commit 1301767087185ca81bde9cec2eea7684c1c03479) --- hermes_cli/update_cmd_windows.py | 29 +++++++++++----- .../test_update_pause_debt_windows_live.py | 33 +++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 8ef69c9e23c90..2e0c1dd7714de 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -954,6 +954,8 @@ def _pause_windows_gateways_for_update() -> dict | None: "Could not capture argv for unmapped gateway %s: %s", int(pid))} for pid in unmapped_pids ] + from gateway.status import get_process_start_time + born = {int(pid): get_process_start_time(int(pid)) for pid in running_pids} # the processes discovered intended = { "resume_needed": True, "unmapped_pids": unmapped_pids, "unmapped": unmapped, "profiles": {str(profile_processes[pid].profile): int(pid) for pid in running_pids @@ -972,7 +974,7 @@ def _pause_windows_gateways_for_update() -> dict | None: pause_record.mark_stop_requested( intended, running_pids, markers=_planned_stop_markers(running_pids, profile_processes, service_gateway_pids)) profiles = _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped, - on_request=lambda pid: pause_record.mark_stop_sent(intended, pid)) + on_request=lambda pid: pause_record.mark_stop_sent(intended, pid), born=born) token = {"resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped} # Every profile with ANY live gateway at discovery counts as running: service-supervised ones skip the # socket pause (absent from ``profiles``) but the SCM restart brings them back, not a cold-start. @@ -1004,25 +1006,34 @@ def _cold_start_pause_token(adopted: dict | None, claims: list) -> dict | None: def _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped, - on_request=None) -> dict: + on_request=None, born=None) -> dict: """Socket-pause the mapped gateways, drain, then tree-kill survivors; the paused ``profiles``. - A force-killed process needs no stop record: dead, it is owed; alive, it was never stopped.""" + A force-killed process needs no stop record: dead, it is owed; alive, it was never stopped. + + The kill is authorized by each process's birth as discovered (*born*, a mapped gateway's + strict PID-file identity first), never re-read after the drain: a PID that exited and was + reused meanwhile reads its replacement's own birth. No discovered birth refuses the kill.""" from gateway.status import get_process_start_time, terminate_pid from hermes_cli.update_cmd import _m + mapped = [int(p) for p in running_pids if int(p) not in service_gateway_pids and int(p) in profile_processes] + born = {**{int(k): v for k, v in (born or {}).items()}, + **{p: int(round(float(profile_processes[p].create_time) * 100)) for p in mapped + if float(getattr(profile_processes[p], "create_time", 0) or 0) > 0}} + # Resolve venv-side launchers before any request: the launcher keeps ``.pyd`` mapped and would trip the + # venv-holder guard, so it is killed with the survivors, under the identity it has now. + launcher_pids = _m()._venv_launcher_ancestors(mapped) + born.update({int(p): get_process_start_time(int(p)) for p in launcher_pids if int(p) not in born}) profiles, mapped_pids, socket_acks = _request_socket_pauses( running_pids, profile_processes, service_gateway_pids, on_request=on_request) - # Resolve venv-side launchers BEFORE draining: a dead worker's parent cannot be recovered (NoSuchProcess). - # The launcher keeps ``.pyd`` mapped and would trip the venv-holder guard; it is killed with the survivors. - launcher_pids = _m()._venv_launcher_ancestors(mapped_pids) print("→ Stopping Windows gateway process(es) before updating Hermes...") drain_timeout = _gateway_drain_timeout(socket_acks) survivors = _m()._wait_for_windows_update_gateway_exit(mapped_pids, timeout=drain_timeout) - # Tree-kill survivors, unmapped gateways, and pre-drain launchers; a launcher - # already gone with its worker raises ProcessLookupError and is skipped. + # Tree-kill survivors, unmapped gateways, and pre-request launchers; one already gone raises + # ProcessLookupError, one whose birth differs (or was never read) is refused, and both are skipped. force_killed = [] for pid in sorted(set(survivors).union(unmapped_pids).union(launcher_pids)): with suppress(ProcessLookupError, PermissionError, OSError): - terminate_pid(int(pid), force=True, expected_start_time=get_process_start_time(int(pid))) + terminate_pid(int(pid), force=True, expected_start_time=born.get(int(pid))) force_killed.append(int(pid)) if profiles: print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}") diff --git a/tests/hermes_cli/test_update_pause_debt_windows_live.py b/tests/hermes_cli/test_update_pause_debt_windows_live.py index 95507c05f9444..875952d70196b 100644 --- a/tests/hermes_cli/test_update_pause_debt_windows_live.py +++ b/tests/hermes_cli/test_update_pause_debt_windows_live.py @@ -14,6 +14,7 @@ import subprocess import sys import textwrap +import time import uuid from contextlib import suppress from pathlib import Path @@ -202,3 +203,35 @@ def test_a_running_service_owes_its_gateway_until_that_gateway_is_ready(tmp_path def _logs(home: Path) -> str: return "\n".join(f"--- {p}\n{p.read_text(encoding='utf-8-sig', errors='replace')[-3000:]}" for p in sorted((home / "logs").glob("*.log"))) + + +# --- force-kill identity ------------------------------------------------------------------------ + +@pytest.mark.parametrize("discovered", ["this process", "an earlier process at this pid"]) +def test_a_pause_force_kills_only_the_process_it_discovered(tmp_path, monkeypatch, discovered): + from hermes_cli import update_cmd_windows as w + from hermes_cli.gateway import ProfileGatewayProcess + + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setenv("HERMES_RESTART_DRAIN_TIMEOUT", "1") + # Never answers its pause, so it survives the drain and reaches the force-kill. + victim = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"], stdin=subprocess.DEVNULL) + try: + born = psutil.Process(victim.pid).create_time() + # The gateway discovered at this PID: this very process, or one born earlier that exited + # during the drain while Windows handed its PID to the process running now. + seen = born if discovered == "this process" else born - 5.0 + found = ProfileGatewayProcess(profile="r8kill", path=home, pid=victim.pid, create_time=seen) + w._stop_windows_gateways([victim.pid], {victim.pid: found}, set(), [], []) + deadline = time.monotonic() + 15 + while victim.poll() is None and time.monotonic() < deadline: + time.sleep(0.1) + if discovered == "this process": + assert victim.poll() is not None, "the discovered gateway survived its force-kill" + else: + assert victim.poll() is None, "force-kill took a process born after discovery (a reused PID)" + finally: + subprocess.run(["taskkill", "/T", "/F", "/PID", str(victim.pid)], capture_output=True, check=False) + victim.wait(timeout=30) From 834ed496b432ca604fa8f87fe8e2c351e5a90f3e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:42:56 -0700 Subject: [PATCH 25/63] fix(update): an unreadable retired list never makes a completed pause owed again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R8 re-review M2. `_retired()` read any OSError/ValueError as "nothing retired": while the list was unreadable (a Windows sharing violation, an AV scanner) a completed obligation came back into `orphans()`, `claim()` took it, and a retirement made in that window rewrote the list with only its own ids, erasing the history for good. - Only an absent list is empty. One that exists but cannot be read (or is malformed) raises `RetiredUnknown`: `orphans()` reports it (recovery prints the skip), `claim()` takes nothing, `_retire()` raises before rewriting the list or deleting a carrier. - `orphans()` surveys under the pause mutex, so it no longer reads the list while a mutator replaces it (Windows opens without FILE_SHARE_DELETE). - `hermes update` aborts before its first stop with a clear message when it cannot read what an earlier update paused, instead of adopting from an unknown list. - `_mutex()` takes an optional wait (used by the planned-stop consumer in a later commit). Test: test_an_unreadable_retired_list_claims_nothing_and_forgets_nothing (real file, mode 000 as the stand-in for a Windows read refusal) — RED on 13017670871, GREEN here. (cherry picked from commit 08ce58ae672457def25deb5e7f775f5166fa7eaf) --- hermes_cli/update_cmd_windows.py | 3 +- hermes_cli/update_pause_record.py | 39 +++++++++++++++----- tests/hermes_cli/test_update_pause_record.py | 36 ++++++++++++++++++ 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 2e0c1dd7714de..7ff26be200236 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -942,7 +942,8 @@ def _pause_windows_gateways_for_update() -> dict | None: import gateway.status # noqa: F401 — fail before the first stop, not half-way through it from hermes_cli.gateway import _capture_gateway_argv from hermes_cli import update_pause_record as pause_record - adopted, claims = pause_record.adopt_orphans() + with _abort_on_error("Could not read the gateways an earlier update paused"): + adopted, claims = pause_record.adopt_orphans() profile_processes, service_gateways, service_gateway_pids, running_pids = _discover_windows_gateways() if not running_pids: return _cold_start_pause_token(adopted, claims) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index a71e8975c57d9..fc08695cbb807 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -27,6 +27,8 @@ retires it. Completing an obligation first adds its ids to ``.retired`` (atomic), then unlinks its files: an unlink Windows refuses (a reader holding the file without FILE_SHARE_DELETE) leaves a copy that is redundant by id, never an obligation that executes again. +The list is read only under the mutex, and only its absence means "nothing retired": a list that +cannot be read means unknown, so nothing is claimed and the list is never rewritten from that read. """ from __future__ import annotations @@ -92,8 +94,12 @@ class RecordBusy(OSError): """Another process held the record mutex past the bounded wait.""" +class RetiredUnknown(OSError): + """The retired-id list exists but cannot be read: which obligations are complete is unknown.""" + + @contextmanager -def _mutex(): +def _mutex(wait_s: float = _MUTEX_WAIT_S): """Exclusive kernel lock on the record directory's sidecar (A7). Re-entrant in-process.""" global _mutex_depth if _mutex_depth: @@ -108,7 +114,7 @@ def _mutex(): path.parent.mkdir(parents=True, exist_ok=True) fd = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o644) try: - deadline = time.monotonic() + _MUTEX_WAIT_S + deadline = time.monotonic() + wait_s while not update_lock._try_lock(fd): if time.monotonic() > deadline: raise RecordBusy(f"{path} is held by another process") @@ -323,21 +329,30 @@ def _retired_path(path: Path) -> Path: def _retired(path: Path) -> set[str]: + """The completed obligation ids (call under the mutex). Only an absent list is empty: one that + exists but cannot be read raises :class:`RetiredUnknown`, so a completed obligation is never + claimed again and the list is never rewritten from a read that failed.""" + target = _retired_path(path) try: - ids = json.loads(_retired_path(path).read_text(encoding="utf-8-sig")).get("ids") - except (OSError, ValueError, AttributeError): + ids = json.loads(target.read_text(encoding="utf-8-sig")).get("ids") + except FileNotFoundError: return set() - return {str(i) for i in ids} if isinstance(ids, list) else set() + except (OSError, ValueError, AttributeError) as exc: + raise RetiredUnknown(f"cannot read the retired paused-gateway list {target}: {exc}") from exc + if not isinstance(ids, list): + raise RetiredUnknown(f"the retired paused-gateway list {target} is malformed") + return {str(i) for i in ids} def _retire(path: Path, carriers: list[tuple[Path, dict]]) -> None: """Complete the obligations *carriers* hold (call under the mutex): their ids (and every id - they absorbed) go on the durable retired list FIRST; a failed write raises and deletes + they absorbed) go on the durable retired list FIRST; a failed read or write raises and deletes nothing. Then the files go; one that cannot be deleted is a redundant copy from here on.""" ids = {str(i) for _src, body in carriers for i in [body["token"].get("pause_id"), *(body["token"].get("absorbed") or [])] if i} - if not ids <= _retired(path): - _atomic_write(_retired_path(path), {"schema": 1, "ids": sorted(_retired(path) | ids)}) + retired = _retired(path) + if not ids <= retired: + _atomic_write(_retired_path(path), {"schema": 1, "ids": sorted(retired | ids)}) for src, _body in carriers: with suppress(OSError): src.unlink() @@ -392,9 +407,13 @@ def _survey(path: Path) -> tuple[list[tuple[Path, dict]], set[Path], set[Path]]: def orphans(path: Path | None = None) -> list[tuple[Path, dict]]: """``(file, body)`` for every obligation of this checkout whose holder is dead — one file per - obligation id; empty while another update is live (it adopts them itself).""" + obligation id; empty while another update is live (it adopts them itself). Surveyed under the + mutex (a mutator replaces the retired list meanwhile); raises when the list cannot be read.""" path = path or record_path() - found, held, redundant = _survey(path) + if not path.exists() and not _claims(path): + return [] + with _mutex(): + found, held, redundant = _survey(path) found = [(src, body) for src, body in found if src not in held and src not in redundant] if not found or _live_update_elsewhere(): return [] diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index b0bb17cb8fe3c..937fd39bde963 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -465,3 +465,39 @@ def test_our_own_pid_is_ours_only_at_our_exact_creation_time(): def test_a_record_naming_our_pid_without_a_creation_time_is_a_previous_incarnation(): assert not pause_record.identity_is_live({"pid": os.getpid(), "ct": None}) + + +# --- R9: the retired list never fails open ------------------------------------------------------ +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root reads a mode-000 file") +def test_an_unreadable_retired_list_claims_nothing_and_forgets_nothing(tmp_path, monkeypatch): + """Mode 000 on the real list stands in for a Windows read refusal (a sharing violation, an AV + scanner): which obligations are complete is then unknown, never "none".""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + src = pause_record.record_path() + retired = src.with_suffix(".retired") + done = "a" * 32 # completed earlier; Windows kept its record file + pause_record.write({"pause_id": done, "resume_needed": True, "profiles": {"default": 4242}}, + owner=pause_record.UNOWNED) + pause_record._atomic_write(retired, {"schema": 1, "ids": [done]}) + assert pause_record.orphans() == [] + other = src.with_name(f"{src.name}.999.deadbeef.claim") # an obligation completing meanwhile + pause_record._atomic_write(other, {"install_root": str(REPO), "claimer": pause_record.UNOWNED, + "token": {"pause_id": "b" * 32}}) + + def owed() -> list[str]: + try: + return [body["token"]["pause_id"] for _src, body in pause_record.orphans()] + except OSError: # unknown: recovery reports it and claims nothing + return [] + + retired.chmod(0) + try: + assert done not in owed(), "a completed obligation is owed again while its list is unreadable" + assert pause_record.claim(src) is None, "a completed obligation was claimed again" + with pytest.raises(OSError), pause_record._mutex(): + pause_record._retire(src, [(other, json.loads(other.read_text(encoding="utf-8-sig")))]) + assert other.exists(), "a retirement that could not be recorded deleted its carrier" + finally: + retired.chmod(0o644) + assert json.loads(retired.read_text(encoding="utf-8-sig"))["ids"] == [done], "the history was rewritten" + assert done not in owed() From 2cbfd69e545e9477120de8256164d0d05d0aa666 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:42:56 -0700 Subject: [PATCH 26/63] fix(update): a launch recovering an unready SCM service no longer stalls on it R8 re-review M3. Since D4b a running service stays owed until its gateway is ready, and `update_pause_record.recover()` runs before every non-update command: each launch re-claimed the debt and waited the full 60 s readiness budget (+2 s confirm), with no backoff, for as long as the service's gateway stayed unhealthy. - A recovering launch (`token["recovery"]`) probes readiness for 5 s; the 60 s wait stays with `hermes update`. - A service found running but unready records `service_retry` {attempts, next_at} on the token (60 s, doubling, capped at 1 h). A recovering launch inside that window skips a still-running service at once and keeps it owed; a stopped one is still started. Success clears the entry. Test (native Windows, real SCM service whose gateway never comes up, two real recovering launches): test_a_launch_recovering_an_unready_service_keeps_the_debt_without_stalling. (cherry picked from commit 823c214534e5f8d301097c063cfc79ebbfddc13c) --- hermes_cli/update_cmd_windows.py | 43 ++++++++++++++-- .../test_update_pause_debt_windows_live.py | 51 +++++++++++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 7ff26be200236..647f6b40f9820 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -1242,9 +1242,15 @@ def _refresh_bootstrap_cache_scripts(branch: str = "main") -> None: _SERVICE_READY_TIMEOUT_S = 60.0 +# A launch recovering an orphaned pause (``update_pause_record.recover``, before every command) +# only probes briefly, and a service whose gateway stays unready is re-probed with backoff; the +# full wait belongs to ``hermes update`` itself. +_SERVICE_READY_PROBE_S = 5.0 +_SERVICE_RETRY_BASE_S = 60.0 +_SERVICE_RETRY_MAX_S = 3600.0 -def _service_gateway_ready(name: str, profile: str | None) -> list[int]: +def _service_gateway_ready(name: str, profile: str | None, timeout_s: float | None = None) -> list[int]: """The stable gateway *name*'s service process supervises (in *profile*'s home when known): SCM ``running`` proves only the wrapper started, not that the gateway it hosts came up.""" from hermes_cli import gateway_windows @@ -1265,23 +1271,48 @@ def under_service(pids): home = Path(get_profile_dir(profile)) if profile else None return gateway_windows._wait_for_gateway_ready( - timeout_s=_SERVICE_READY_TIMEOUT_S, home=home, all_profiles=home is None, pid_filter=under_service) + timeout_s=_SERVICE_READY_TIMEOUT_S if timeout_s is None else timeout_s, home=home, + all_profiles=home is None, pid_filter=under_service) + + +def _service_running(name: str) -> bool: + try: + return _win_service(name)[1].status() == "running" + except Exception: + return False def _resume_windows_services(token: dict) -> None: """Restart the SCM services recorded on *token*; one leaves the token only once the gateway it - supervises is ready. Failed ones stay on the token so a retry sees them.""" + supervises is ready. Failed ones stay on the token so a retry sees them. + + A recovering launch (``token["recovery"]``) probes readiness briefly, and a running service + whose gateway was not ready is not re-probed before its backoff (``service_retry``) elapses: + the debt stays owed without stalling every command.""" from hermes_cli.update_cmd import _start_windows_gateway_service services = list(token.get("services") or []) token.setdefault("expected_services", list(services)) verified_restarts = list(token.get("restarted_services") or []) + retry = {name: state for name, state in (token.get("service_retry") or {}).items() if name in services} + recovering = bool(token.get("recovery")) restarted_services = [] failed_services = [] for service_name in map(str, services): + backoff = retry.get(service_name) or {} try: + if recovering and _time.time() < float(backoff.get("next_at") or 0) and _service_running(service_name): + failed_services.append(service_name) + print(f" ⚠ Windows gateway service {service_name}: its gateway was not ready; " + f"next check after {int(float(backoff['next_at']) - _time.time())}s") + continue _start_windows_gateway_service(service_name) - if not _service_gateway_ready(service_name, (token.get("service_profiles") or {}).get(service_name)): + if not _service_gateway_ready(service_name, (token.get("service_profiles") or {}).get(service_name), + _SERVICE_READY_PROBE_S if recovering else None): + attempts = int(backoff.get("attempts") or 0) + 1 + retry[service_name] = {"attempts": attempts, "next_at": _time.time() + min( + _SERVICE_RETRY_BASE_S * 2 ** min(attempts - 1, 6), _SERVICE_RETRY_MAX_S)} raise RuntimeError(f"Windows service {service_name} is running but its gateway did not become ready") + retry.pop(service_name, None) restarted_services.append(service_name) if service_name not in verified_restarts: verified_restarts.append(service_name) @@ -1291,6 +1322,10 @@ def _resume_windows_services(token: dict) -> None: failed_services.append(service_name) token["restarted_services"] = verified_restarts token["services"] = failed_services + if retry: + token["service_retry"] = retry + else: + token.pop("service_retry", None) if failed_services: raise RuntimeError("Could not restart Windows gateway service(s): " + ", ".join(failed_services)) if restarted_services: diff --git a/tests/hermes_cli/test_update_pause_debt_windows_live.py b/tests/hermes_cli/test_update_pause_debt_windows_live.py index 875952d70196b..3e39da19ccbdc 100644 --- a/tests/hermes_cli/test_update_pause_debt_windows_live.py +++ b/tests/hermes_cli/test_update_pause_debt_windows_live.py @@ -10,6 +10,7 @@ from __future__ import annotations +import json import os import subprocess import sys @@ -200,6 +201,56 @@ def test_a_running_service_owes_its_gateway_until_that_gateway_is_ready(tmp_path _sc("delete", name) +_RECOVERING_LAUNCH = """ +import json, time +from hermes_cli import update_pause_record as r +started = time.monotonic() +r.recover(["status"]) +print(json.dumps({"seconds": time.monotonic() - started, + "owed": [body["token"].get("services") for _src, body in r.orphans()]})) +""" + + +def test_a_launch_recovering_an_unready_service_keeps_the_debt_without_stalling(tmp_path, monkeypatch, service_wrapper): + """Two CLI launches in a row recover an orphaned pause whose SCM service runs but whose gateway + never comes up: the debt stays owed, and neither launch waits out the update's full readiness budget.""" + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + bin_dir = tmp_path / "svc" + bin_dir.mkdir() + exe = bin_dir / "wrapper.exe" + exe.write_bytes(service_wrapper.read_bytes()) + (bin_dir / "child.txt").write_text("\n".join([sys.executable, "-c \"import time; time.sleep(900)\"", str(REPO)]) + "\n", + encoding="utf-8") + name = f"hermes-r9-recover-{os.getpid()}" + created = _sc("create", name, "binPath=", f'"{exe}"', "start=", "demand") + assert created.returncode == 0, created.stdout + created.stderr + # The record a killed update left; the tree gate passes (HEAD and tracked changes as stamped). + pause_record.write(pause_record.stamp_tree({ + "resume_needed": True, "services": [name], "expected_services": [name], "restarted_services": [], + "service_profiles": {name: "default"}}), owner=pause_record.UNOWNED) + try: + launches = [] + for _ in range(2): + done = subprocess.run([sys.executable, "-c", _RECOVERING_LAUNCH], cwd=REPO, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=300, + env={**os.environ, "HERMES_HOME": str(home), "PYTHONPATH": str(REPO)}) + assert done.returncode == 0, done.stdout + done.stderr + launches.append(json.loads(done.stdout.strip().splitlines()[-1])) + finally: + pid = 0 + with suppress(Exception): + pid = int(psutil.win_service_get(name).pid() or 0) # type: ignore[attr-defined] + if pid: + subprocess.run(["taskkill", "/T", "/F", "/PID", str(pid)], capture_output=True, check=False) + _sc("delete", name) + assert [launch["owed"] for launch in launches] == [[[name]], [[name]]], f"the unready service's debt was lost: {launches}" + first, second = (launch["seconds"] for launch in launches) + assert first < 40, f"the first launch stalled {first:.0f}s on the service's readiness" + assert second < 20, f"the next launch stalled {second:.0f}s on a service it had just found unready" + + def _logs(home: Path) -> str: return "\n".join(f"--- {p}\n{p.read_text(encoding='utf-8-sig', errors='replace')[-3000:]}" for p in sorted((home / "logs").glob("*.log"))) From e2ef545a14f5d66993c01ddf3ecd4387cd26d969 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:42:56 -0700 Subject: [PATCH 27/63] fix(gateway): the pause bookkeeping never turns a planned stop into an unplanned one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R8 re-review m5 (first bullet). `consume_planned_stop_marker_for_self` waited up to 10 s on the pause mutex inside the shutdown path and raised when the consumer checkpoint could not be written, so a planned stop was classified as an unexpected signal (on Windows during a pause a supervisor could then respawn the gateway while the update holds the venv). - The mutex wait here is 2 s. - A busy mutex or a failed checkpoint classifies the stop without consuming it: the request stays on disk, which is the evidence recovery reads in place of the checkpoint (it expires by TTL). Tests: test_a_busy_pause_mutex_never_turns_a_planned_stop_into_an_unplanned_one (new) and test_failed_consumer_checkpoint_keeps_request_and_user_stop_is_not_update_debt (now asserts the stop is planned while the request is kept) — both RED on 13017670871, GREEN here. (cherry picked from commit 7145a469d5916bc2cddb974401a509ebb656f4ba) --- gateway/status.py | 24 +++++++--- .../test_update_pause_boundaries.py | 45 +++++++++++++++---- 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/gateway/status.py b/gateway/status.py index 14025c2b4fa41..8d1084df4fbe2 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -1814,7 +1814,7 @@ def _pid_marker_names_self(target_pid: int, target_start_time: Any) -> bool: return None in (target_start_time, our_start_time) or target_start_time == our_start_time -def _consume_pid_marker_for_self(path: Path, *, ttl_s: int, on_consume=None) -> bool: +def _consume_pid_marker_for_self(path: Path, *, ttl_s: int, on_consume=None, keep: bool = False) -> bool: parsed = _read_live_pid_marker(path, ttl_s) if parsed is None: return False @@ -1835,7 +1835,8 @@ def _consume_pid_marker_for_self(path: Path, *, ttl_s: int, on_consume=None) -> matches = _pid_marker_names_self(target_pid, target_start_time) if matches and on_consume is not None: on_consume(path, record) - _unlink_quietly(path) + if not keep: + _unlink_quietly(path) return matches @@ -2071,16 +2072,27 @@ def write_planned_stop_marker(target_pid: int) -> bool: }) +_PLANNED_STOP_MUTEX_WAIT_S = 2.0 + + def consume_planned_stop_marker_for_self() -> bool: """Return True when the current process is being intentionally stopped.""" from hermes_cli import update_pause_record # Recovery must see either the request or its checkpoint, never the gap # between validating/consuming the marker and scheduling asynchronous stop. - with update_pause_record._mutex(): + try: + with update_pause_record._mutex(_PLANNED_STOP_MUTEX_WAIT_S): + return _consume_pid_marker_for_self( + _get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S, + on_consume=update_pause_record.mark_stop_consumed, + ) + except OSError as exc: + # The pause bookkeeping (a busy mutex, a checkpoint that cannot be written) never decides + # whether this stop was planned: classify without consuming, so the request stays on + # disk as the evidence recovery reads in place of the checkpoint (it expires by TTL). + logger.warning("Planned-stop checkpoint skipped (%s); the stop request stays on disk", exc) return _consume_pid_marker_for_self( - _get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S, - on_consume=update_pause_record.mark_stop_consumed, - ) + _get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S, keep=True) def planned_stop_marker_targets_self() -> bool: diff --git a/tests/hermes_cli/test_update_pause_boundaries.py b/tests/hermes_cli/test_update_pause_boundaries.py index d86b631c01baa..0da9e794c36f5 100644 --- a/tests/hermes_cli/test_update_pause_boundaries.py +++ b/tests/hermes_cli/test_update_pause_boundaries.py @@ -145,12 +145,11 @@ def test_failed_consumer_checkpoint_keeps_request_and_user_stop_is_not_update_de # An actual directory at the atomic writer's temp path rejects publication. obstruction = r.record_path().with_name(f"{r.record_path().name}.{pid}.tmp") obstruction.mkdir() + # The checkpoint cannot land: the stop is still planned, and the request stays on disk. try: - status.consume_planned_stop_marker_for_self() - except OSError: - pass - else: - raise AssertionError("consume succeeded without its durable checkpoint") + planned = status.consume_planned_stop_marker_for_self() + except OSError as exc: + planned = type(exc).__name__ retained = path.exists() and r.read()["token"]["stop_sent"] == [] obstruction.rmdir() marker = json.loads(path.read_text(encoding="utf-8-sig")) @@ -159,13 +158,43 @@ def test_failed_consumer_checkpoint_keeps_request_and_user_stop_is_not_update_de user_debt = r.read()["token"]["stop_sent"] assert _write_update_planned_stop_marker(home, pid) accepted = status.consume_planned_stop_marker_for_self() - print(json.dumps({"retained": retained, "user_consumed": user_consumed, "user_debt": user_debt, - "accepted": accepted, "sent": r.read()["token"]["stop_sent"], "pid": pid})) + print(json.dumps({"planned": planned, "retained": retained, "user_consumed": user_consumed, + "user_debt": user_debt, "accepted": accepted, "sent": r.read()["token"]["stop_sent"], + "pid": pid})) ''')) - assert result == {"retained": True, "user_consumed": True, "user_debt": [], + assert result == {"planned": True, "retained": True, "user_consumed": True, "user_debt": [], "accepted": True, "sent": [str(result["pid"])], "pid": result["pid"]} +def test_a_busy_pause_mutex_never_turns_a_planned_stop_into_an_unplanned_one(cell): + base, _, launch = cell + holder = launch(''' + with r._mutex(): + Path(sys.argv[2]).touch() + while not Path(sys.argv[3]).exists(): time.sleep(.05) + ''', base / "held", base / "release") + _wait(base / "held", holder) + try: + result = _result(launch(''' + from gateway import status + from hermes_cli.update_cmd_windows import _write_update_planned_stop_marker + pid = os.getpid() + home = Path(os.environ["HERMES_HOME"]) + assert _write_update_planned_stop_marker(home, pid) + started = time.monotonic() + try: + planned = status.consume_planned_stop_marker_for_self() + except OSError as exc: + planned = type(exc).__name__ + print(json.dumps({"planned": planned, "seconds": time.monotonic() - started, + "retained": (home / ".gateway-planned-stop.json").exists()})) + ''')) + finally: + (base / "release").touch() + assert result["planned"] is True and result["retained"], result + assert result["seconds"] < 5, f"the gateway's shutdown blocked {result['seconds']:.1f}s on the pause mutex" + + def test_recovery_only_accepts_markers_the_live_consumer_accepts(cell): _, _, launch = cell result = _result(launch(''' From cdd58de5651601fca628167098da8a7619ad1f6e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:04:29 -0700 Subject: [PATCH 28/63] fix(update): adopting an orphaned pause keeps its pre-update tree evidence (review F2-adoption-restamps-baseline) What: merge_into carries the adopted pause's pre_sha/dirty_at_pause (setdefault), so record_pause's stamp_tree keeps the oldest baseline instead of stamping the current tree. Why: re-stamping certified a tree the killed update left half-written as "dirty before the pause", letting the adopting update's resume start gateways on a torn checkout. Red: pre-fix update_pause_record.py -> test fails "adoption certified the torn tree as the pre-update baseline". Green: test_an_update_adopting_an_orphan_never_certifies_the_tree_it_left_torn passes. --- hermes_cli/update_pause_record.py | 5 +++++ tests/hermes_cli/test_update_pause_record.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index fc08695cbb807..03c6edf250df1 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -604,6 +604,11 @@ def merge_into(token: dict | None, adopted: dict) -> dict: obligation id (and every id it absorbed) is recorded so a leftover copy is never resumed twice.""" token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []} token["resume_needed"] = True + # The oldest tree evidence wins: stamping the current tree instead would certify whatever a + # killed update left half-written as "dirty before the pause". + for key in ("pre_sha", "dirty_at_pause"): + if key in adopted: + token.setdefault(key, adopted[key]) absorbed = token.setdefault("absorbed", []) absorbed.extend(i for i in [adopted.get("pause_id"), *(adopted.get("absorbed") or [])] if i and i not in absorbed) profiles = token.setdefault("profiles", {}) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 937fd39bde963..05f2fd3b25ed9 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -114,6 +114,26 @@ def test_resume_waits_for_a_whole_tree(tmp_path): assert not whole and "dependencies" in why, why +def test_an_update_adopting_an_orphan_never_certifies_the_tree_it_left_torn(tmp_path, monkeypatch): + root = tmp_path / "checkout" + root.mkdir() + _git(root, "init", "-q") + (root / "a.py").write_text("v1\n", encoding="utf-8") + _git(root, "add", ".") + _git(root, "commit", "-qm", "v1") + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home")) + monkeypatch.setattr(pause_record, "install_root", lambda: root) + pause_record.write(pause_record.stamp_tree({"resume_needed": True, "profiles": {"default": 4242}}), + owner=pause_record.UNOWNED) + (root / "a.py").write_text("half-written\n", encoding="utf-8") # the killed update's git died mid-checkout + assert not pause_record.tree_is_whole(pause_record.read()["token"], root)[0], "premise: the orphan's gate refuses" + + adopted, claims = pause_record.adopt_orphans() + token = pause_record.record_pause({"resume_needed": True, "profiles": {"beta": 99}}, adopted, claims) + whole, why = pause_record.tree_is_whole(token, root) + assert not whole and "a.py" in why, "adoption certified the torn tree as the pre-update baseline" + + def _orphan(tmp_path: Path, profiles: dict) -> None: """A record whose owner — a real ``hermes update`` stand-in — was SIGKILLed after writing it.""" owner = _child(""" From 36456682c124d85efc6169e11b6b035971115892 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:07:29 -0700 Subject: [PATCH 29/63] fix(update): startup recovery resumes gateways only while holding the checkout lock (review F5-recovery-start-vs-checkout-mutation) What: _resume_claimed takes the checkout lock (bounded contention wait) around the tree gate and the verified gateway start; when an update already took the checkout after our claim, the claim is handed back unowned and untouched, so that update or the next launch owns it. Why: recovery judged the tree and launched gateways with no checkout custody, so a concurrent `hermes update` could mutate the checkout under freshly started gateways (the updater skips a live claim and may discover no running gateway to pause). Red: pre-fix update_pause_record.py -> "gateways restarted while an update owned the checkout: resume ['default']". Green: test_recovery_never_starts_gateways_under_an_update_that_took_the_checkout passes (set resumed by the next launch). --- hermes_cli/update_pause_record.py | 15 ++++++++++ tests/hermes_cli/test_update_pause_record.py | 30 ++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 03c6edf250df1..c5a22b68fb04e 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -647,6 +647,21 @@ def _has_work(token: dict) -> bool: def _resume_claimed(claim_path: Path, body: dict) -> None: + """Resume under the checkout lock, so no update mutates the tree between the gate and the + verified start; an update that took the checkout after our claim gets the set back untouched.""" + from hermes_cli.update_lock import UpdateLock + fence = UpdateLock() + if not fence.acquire_checkout(install_root()): + with suppress(OSError), _mutex(): + _atomic_write(claim_path, {**(read(claim_path) or body), "claimer": UNOWNED}) + return + try: + _resume_fenced(claim_path, body) + finally: + fence.release() + + +def _resume_fenced(claim_path: Path, body: dict) -> None: with _mutex(): # A live consumer may have checkpointed after claim() returned its snapshot. body = read(claim_path) or body diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 05f2fd3b25ed9..4df57760cbf25 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -183,6 +183,36 @@ def test_a_launch_killed_mid_recovery_leaves_the_set_to_the_next_launch(tmp_path assert out.strip() == "done", f"a resumed set was resumed again: {out}" +# An update takes the checkout right after this launch claimed the set (the race window). +_UPDATE_AFTER_CLAIM = """ + import subprocess, sys + from hermes_cli import update_pause_record as r + hold = ("import time; from hermes_cli import update_lock as l; from hermes_cli.update_pause_record import install_root;" + "print(l.UpdateLock().acquire_checkout(install_root()), flush=True); time.sleep(120)") + real_claim, updates = r.claim, [] + def claim(src): + won = real_claim(src) + updates.append(subprocess.Popen([sys.executable, "-c", hold], stdout=subprocess.PIPE, text=True)) + assert updates[-1].stdout.readline().strip() == "True" + return won + r.claim = claim +""" + _RECOVER + """ + for update in updates: + update.kill() + update.wait() +""" + + +@pytest.mark.live_system_guard_bypass +def test_recovery_never_starts_gateways_under_an_update_that_took_the_checkout(tmp_path): + _orphan(tmp_path, {"default": 4242}) + env = {"HERMES_HOME": str(tmp_path)} + out, _ = _child(_UPDATE_AFTER_CLAIM, "ok", env=env).communicate(timeout=60) + assert out.strip() == "done", f"gateways restarted while an update owned the checkout: {out}" + out, _ = _child(_RECOVER, "ok", env=env).communicate(timeout=60) + assert out.splitlines()[:1] == ["resume ['default']"], f"the handed-back set was stranded: {out}" + + _HOLDER = """ import subprocess, sys, time from hermes_cli.update_lock import UpdateLock From 3a4cc4331a255dfc706ab0d8e7ae3950eb812dea Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:10:01 -0700 Subject: [PATCH 30/63] fix(update): an argv value equal to "update" no longer suppresses paused-gateway recovery (review R5405535944-4-raw-update-argv) What: main() calls update_pause_record.recover() unconditionally instead of inside the raw `"update" not in sys.argv[1:]` block; recover() already skips the parsed `update`/`gateway run`. Why: `hermes --resume update ...` (a session named "update") left gateways a killed update paused stopped, though no update was running. Red: pre-fix main.py -> [--resume update --version] leaves the orphaned record on disk. Green: test_startup_recovery_follows_the_parsed_command_not_raw_argv passes both cases (`update --help` still defers). --- hermes_cli/main.py | 6 ++++-- tests/hermes_cli/test_update_pause_record.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index b9e14dde54f0e..dcb4ee27f8b6f 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -3562,8 +3562,10 @@ def main(): _warn_pending_fleet_restart_on_startup() except Exception: pass - from hermes_cli.update_pause_record import recover as _recover_paused_gateways - _recover_paused_gateways() # a killed `hermes update` left gateways paused (Windows) + from hermes_cli.update_pause_record import recover as _recover_paused_gateways + # Unconditional: recover() itself skips the PARSED `update`/`gateway run` commands; a raw argv + # value equal to "update" (`--resume update`) is no reason to strand paused gateways. + _recover_paused_gateways() # a killed `hermes update` left gateways paused (Windows) if _first_positional_argv() != "update": from hermes_cli.boot_bootstrap import maybe_run_boot_bootstrap diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 4df57760cbf25..4a31f40b38fad 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -213,6 +213,26 @@ def test_recovery_never_starts_gateways_under_an_update_that_took_the_checkout(t assert out.splitlines()[:1] == ["resume ['default']"], f"the handed-back set was stranded: {out}" +@pytest.mark.live_system_guard_bypass +@pytest.mark.parametrize("argv, recovers", [ + (["--resume", "update", "--version"], True), # "update" is a session name here, not the command + (["update", "--help"], False), # the update adopts the set itself +]) +def test_startup_recovery_follows_the_parsed_command_not_raw_argv(tmp_path, argv, recovers): + _orphan(tmp_path, {}) # nothing to start: a recovering launch just retires it + cli = _child(""" + import sys + from hermes_cli.main import main + sys.argv = ["hermes", *sys.argv[1:]] + try: + main() + except SystemExit: + pass + """, *argv, env={"HERMES_HOME": str(tmp_path), "HOME": str(tmp_path / "home")}) + out, _ = cli.communicate(timeout=120) + assert (_record_files(tmp_path) == []) is recovers, out + + _HOLDER = """ import subprocess, sys, time from hermes_cli.update_lock import UpdateLock From 1592f09843c1a879b1ba9e2fffd8554a6d387b6b Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:18:51 -0700 Subject: [PATCH 31/63] refactor(gateway): move inline-source argv parsing out of gateway/status.py gateway/status.py grew past main's size (code-health FILE_LINES). Move the cohesive `python -c ` parsing group -- inline_source_flag_index, command_line_runs_inline_source, the Hermes bootstrap patterns, _bootstrap_entry and inline_bootstrap_argv -- verbatim into the new sibling gateway/status_inline_source.py (no imports of the facade, so no cycle). status.py imports the three names it uses; the one external caller (hermes_cli/update_cmd_windows.py) now imports from the sibling. No shim, no docs named the symbols. Behaviour unchanged. --- gateway/status.py | 128 ++----------------------------- gateway/status_inline_source.py | 132 ++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 123 deletions(-) create mode 100644 gateway/status_inline_source.py diff --git a/gateway/status.py b/gateway/status.py index 8d1084df4fbe2..3fe98e8b662f4 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -21,6 +21,11 @@ from pathlib import Path from typing import Any, Callable, NamedTuple, Optional +from gateway.status_inline_source import ( + command_line_runs_inline_source, + inline_bootstrap_argv, + inline_source_flag_index, +) from hermes_constants import _get_platform_default_hermes_home, get_hermes_home, get_process_hermes_home from hermes_cli._subprocess_compat import pid_exists_stdlib from utils import atomic_json_write @@ -529,129 +534,6 @@ def _read_process_cmdline(pid: int) -> Optional[str]: return None - -def inline_source_flag_index(tokens: list[str]) -> int | None: - """Index of the ``-c`` token when *tokens* is an interpreter running INLINE SOURCE, else None. - - Everything after ``-c`` is data the inline program receives, not this process's own identity. - The detached gateway restart watcher (``gateway._spawn_gateway_restart_watcher``) is spawned as - ``python -c -m hermes_cli.main gateway run``: its trailing - argv is the command the watcher will LATER spawn, so every argv matcher used to read it as a - live gateway. See #107002 and the "never infer process identity from argv substrings" rule. - - Only interpreter options may precede ``-c``; the first non-option token ends the option block - (``python -m hermes_cli.main …`` therefore never matches). - - The walk is VALUE-AWARE: ``-X``/``-W``/``-Q`` and ``--check-hash-based-pycs``/``--jit`` take a - SEPARATE operand, so a naive "first non-option token ends the block" walk mistakes that operand - for the end of the block and never reaches the ``-c`` behind it (``python -X utf8 -c …`` - was still read as a live gateway). The operand sets are the canonical ones in - ``hermes_state_holders``, not a second hand-rolled copy. - - *tokens* must be CASE-PRESERVING: the operand-taking ``-Q``/``-W``/``-X`` differ from the - operand-less ``-q``/``-b``, so a lowercased argv would skip the token after a plain ``-q``. - """ - from hermes_state_holders import ( - _PYTHON_LONG_OPTIONS_WITH_OPERANDS, - _PYTHON_SHORT_OPTIONS_WITH_OPERANDS, - ) - - index = 1 - while index < len(tokens): - token = tokens[index] - if token == "--": - return None - if token in _PYTHON_LONG_OPTIONS_WITH_OPERANDS: - index += 2 # the next token is this option's operand, not the end of the option block - continue - if token.startswith("--"): - index += 1 # ``--opt=value`` and operand-less long options - continue - if not token.startswith("-") or token == "-": - return None - # Clustered short options (``-uc``, ``-IsB``). An operand-taking letter consumes the rest of - # the cluster as its attached value, or the following token when the cluster ends there -- - # so ``-Xc`` is ``-X c``, NOT an inline-source ``-c``. - cluster = token[1:] - for position, letter in enumerate(cluster): - if letter == "c": - return index - if letter in _PYTHON_SHORT_OPTIONS_WITH_OPERANDS: - index += 1 if cluster[position + 1 :] else 2 - break - else: - index += 1 - return None - - -def command_line_runs_inline_source(tokens: list[str]) -> bool: - """True when *tokens* is an interpreter running INLINE SOURCE (``python -c [args]``).""" - return inline_source_flag_index(tokens) is not None - - -# Hermes' own inline bootstraps hand control to a Hermes entry point IN this process, so the argv -# they run with is this process's own identity; every other ``-c`` program keeps its trailing argv -# as data (#107002). Each pattern is one emitted source shape, anchored at both ends so a program -# merely CARRYING a bootstrap command line (the restart watcher's respawn argv) never matches. -_Q = r"""['"]?""" -_MAIN = rf"{_Q}__main__{_Q}" -_RUN_MODULE = rf"runpy\.run_module\(\s*{_Q}(?P[\w.]+){_Q}\s*,\s*run_name\s*=\s*{_MAIN}\s*,\s*alter_sys\s*=\s*True\s*\)" -_BOOTSTRAPS = ( - # hermes_cli._launchers.runtime_command (store launcher, the Windows updater's relaunch) - ("module", re.compile(rf"import os, sys, runpy;.*\b{_RUN_MODULE}", re.S)), - # hermes_cli.venv_sync.relaunch_command: argv is assigned inside the source - ("module", re.compile(rf"import sys, runpy; sys\.path\.insert\(.*\b{_RUN_MODULE}", re.S)), - ("path", re.compile( - rf"import sys, runpy; sys\.path\.insert\(.*\brunpy\.run_path\(\s*{_Q}(?P[^'\"]+?){_Q}\s*,\s*run_name\s*=\s*{_MAIN}\s*\)", - re.S)), - # hermes_cli._launchers._launcher_script (the published POSIX shell / Windows .cmd launcher) - ("entry", re.compile(r"import os, re, sys\s.*\bfrom\s+(?P[\w.]+)\s+import\s+(?P\w+)\b.*\bsys\.exit\(\s*(?P=func)\(\)\s*\)", re.S)), - # hermes_cli._launchers._write_cmd_launcher: the launcher script, base64-encoded - ("base64", re.compile(rf"import base64; exec\(base64\.b64decode\({_Q}(?P[A-Za-z0-9+/=]+){_Q}\)\)")), -) -_ASSIGNED_ARGV = re.compile(r"\bsys\.argv\s*=\s*\[(.*?)\]\s*;") - - -def _bootstrap_entry(source: str, argv: list[str]) -> list[str] | None: - """``[-m, , *argv]`` (or ``[, *argv]``) the inline *source* runs in-process, else None.""" - source = source.strip() - kind, match = next(((k, m) for k, p in _BOOTSTRAPS if (m := p.fullmatch(source))), (None, None)) - if match is None: - return None - target = match["target"] - if kind == "base64": - import base64 - import binascii - try: - return _bootstrap_entry(base64.b64decode(target, validate=True).decode("utf-8"), argv) - except (binascii.Error, UnicodeDecodeError): - return None - if kind == "entry": # the launcher script's own ``--run-module `` switch - return ["-m", argv[1], *argv[2:]] if argv[:1] == ["--run-module"] and len(argv) > 1 else ["-m", target, *argv] - if assigned := _ASSIGNED_ARGV.search(source): - argv = [item.strip().strip("'\"") for item in assigned.group(1).split(",")][1:] - return [target, *argv] if kind == "path" else ["-m", target, *argv] - - -def inline_bootstrap_argv(tokens: list[str]) -> list[str] | None: - """*tokens* as the equivalent ``python -m `` when this interpreter's ``-c`` source - is a Hermes bootstrap running an entry point in-process; None for any other inline source. - - Command lines usually arrive space-joined (``/proc``, psutil, ``ps``), which splits the source - across tokens; the shortest token run that ends in a recognised tail is the source, whatever - joined it, and the tokens after it are the entry point's argv. - """ - index = inline_source_flag_index(tokens) - if index is None: - return None - for end in range(index + 1, len(tokens)): - if tokens[end].rstrip().endswith(")"): - entry = _bootstrap_entry(" ".join(tokens[index + 1 : end + 1]), tokens[end + 1 :]) - if entry is not None: - return [tokens[0], *entry] - return None - - def _gateway_command_subcommand(command: str | None) -> str | None: """Hermes gateway lifecycle subcommand from a command line, or None. No loose substring matches (``"gateway" in cmdline`` also matched ``gateway status`` / ``python -m tui_gateway``): needs a diff --git a/gateway/status_inline_source.py b/gateway/status_inline_source.py new file mode 100644 index 0000000000000..ad069ffb28af5 --- /dev/null +++ b/gateway/status_inline_source.py @@ -0,0 +1,132 @@ +"""Inline-source (``python -c …``) command-line parsing for gateway process identity. + +Everything after ``-c`` is data for the inline program, not the process's own identity -- except +Hermes' own bootstraps, which run an entry point in-process and are mapped back to the equivalent +``python -m `` (see #107002). +""" + +from __future__ import annotations + +import re + + +def inline_source_flag_index(tokens: list[str]) -> int | None: + """Index of the ``-c`` token when *tokens* is an interpreter running INLINE SOURCE, else None. + + Everything after ``-c`` is data the inline program receives, not this process's own identity. + The detached gateway restart watcher (``gateway._spawn_gateway_restart_watcher``) is spawned as + ``python -c -m hermes_cli.main gateway run``: its trailing + argv is the command the watcher will LATER spawn, so every argv matcher used to read it as a + live gateway. See #107002 and the "never infer process identity from argv substrings" rule. + + Only interpreter options may precede ``-c``; the first non-option token ends the option block + (``python -m hermes_cli.main …`` therefore never matches). + + The walk is VALUE-AWARE: ``-X``/``-W``/``-Q`` and ``--check-hash-based-pycs``/``--jit`` take a + SEPARATE operand, so a naive "first non-option token ends the block" walk mistakes that operand + for the end of the block and never reaches the ``-c`` behind it (``python -X utf8 -c …`` + was still read as a live gateway). The operand sets are the canonical ones in + ``hermes_state_holders``, not a second hand-rolled copy. + + *tokens* must be CASE-PRESERVING: the operand-taking ``-Q``/``-W``/``-X`` differ from the + operand-less ``-q``/``-b``, so a lowercased argv would skip the token after a plain ``-q``. + """ + from hermes_state_holders import ( + _PYTHON_LONG_OPTIONS_WITH_OPERANDS, + _PYTHON_SHORT_OPTIONS_WITH_OPERANDS, + ) + + index = 1 + while index < len(tokens): + token = tokens[index] + if token == "--": + return None + if token in _PYTHON_LONG_OPTIONS_WITH_OPERANDS: + index += 2 # the next token is this option's operand, not the end of the option block + continue + if token.startswith("--"): + index += 1 # ``--opt=value`` and operand-less long options + continue + if not token.startswith("-") or token == "-": + return None + # Clustered short options (``-uc``, ``-IsB``). An operand-taking letter consumes the rest of + # the cluster as its attached value, or the following token when the cluster ends there -- + # so ``-Xc`` is ``-X c``, NOT an inline-source ``-c``. + cluster = token[1:] + for position, letter in enumerate(cluster): + if letter == "c": + return index + if letter in _PYTHON_SHORT_OPTIONS_WITH_OPERANDS: + index += 1 if cluster[position + 1 :] else 2 + break + else: + index += 1 + return None + + +def command_line_runs_inline_source(tokens: list[str]) -> bool: + """True when *tokens* is an interpreter running INLINE SOURCE (``python -c [args]``).""" + return inline_source_flag_index(tokens) is not None + + +# Hermes' own inline bootstraps hand control to a Hermes entry point IN this process, so the argv +# they run with is this process's own identity; every other ``-c`` program keeps its trailing argv +# as data (#107002). Each pattern is one emitted source shape, anchored at both ends so a program +# merely CARRYING a bootstrap command line (the restart watcher's respawn argv) never matches. +_Q = r"""['"]?""" +_MAIN = rf"{_Q}__main__{_Q}" +_RUN_MODULE = rf"runpy\.run_module\(\s*{_Q}(?P[\w.]+){_Q}\s*,\s*run_name\s*=\s*{_MAIN}\s*,\s*alter_sys\s*=\s*True\s*\)" +_BOOTSTRAPS = ( + # hermes_cli._launchers.runtime_command (store launcher, the Windows updater's relaunch) + ("module", re.compile(rf"import os, sys, runpy;.*\b{_RUN_MODULE}", re.S)), + # hermes_cli.venv_sync.relaunch_command: argv is assigned inside the source + ("module", re.compile(rf"import sys, runpy; sys\.path\.insert\(.*\b{_RUN_MODULE}", re.S)), + ("path", re.compile( + rf"import sys, runpy; sys\.path\.insert\(.*\brunpy\.run_path\(\s*{_Q}(?P[^'\"]+?){_Q}\s*,\s*run_name\s*=\s*{_MAIN}\s*\)", + re.S)), + # hermes_cli._launchers._launcher_script (the published POSIX shell / Windows .cmd launcher) + ("entry", re.compile(r"import os, re, sys\s.*\bfrom\s+(?P[\w.]+)\s+import\s+(?P\w+)\b.*\bsys\.exit\(\s*(?P=func)\(\)\s*\)", re.S)), + # hermes_cli._launchers._write_cmd_launcher: the launcher script, base64-encoded + ("base64", re.compile(rf"import base64; exec\(base64\.b64decode\({_Q}(?P[A-Za-z0-9+/=]+){_Q}\)\)")), +) +_ASSIGNED_ARGV = re.compile(r"\bsys\.argv\s*=\s*\[(.*?)\]\s*;") + + +def _bootstrap_entry(source: str, argv: list[str]) -> list[str] | None: + """``[-m, , *argv]`` (or ``[, *argv]``) the inline *source* runs in-process, else None.""" + source = source.strip() + kind, match = next(((k, m) for k, p in _BOOTSTRAPS if (m := p.fullmatch(source))), (None, None)) + if match is None: + return None + target = match["target"] + if kind == "base64": + import base64 + import binascii + try: + return _bootstrap_entry(base64.b64decode(target, validate=True).decode("utf-8"), argv) + except (binascii.Error, UnicodeDecodeError): + return None + if kind == "entry": # the launcher script's own ``--run-module `` switch + return ["-m", argv[1], *argv[2:]] if argv[:1] == ["--run-module"] and len(argv) > 1 else ["-m", target, *argv] + if assigned := _ASSIGNED_ARGV.search(source): + argv = [item.strip().strip("'\"") for item in assigned.group(1).split(",")][1:] + return [target, *argv] if kind == "path" else ["-m", target, *argv] + + +def inline_bootstrap_argv(tokens: list[str]) -> list[str] | None: + """*tokens* as the equivalent ``python -m `` when this interpreter's ``-c`` source + is a Hermes bootstrap running an entry point in-process; None for any other inline source. + + Command lines usually arrive space-joined (``/proc``, psutil, ``ps``), which splits the source + across tokens; the shortest token run that ends in a recognised tail is the source, whatever + joined it, and the tokens after it are the entry point's argv. + """ + index = inline_source_flag_index(tokens) + if index is None: + return None + for end in range(index + 1, len(tokens)): + if tokens[end].rstrip().endswith(")"): + entry = _bootstrap_entry(" ".join(tokens[index + 1 : end + 1]), tokens[end + 1 :]) + if entry is not None: + return [tokens[0], *entry] + return None From 4a3e0db73b0994696196ceb150a39bc8369f61c8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:18:51 -0700 Subject: [PATCH 32/63] refactor(update): narrow Windows resume probes' blind excepts Code-health BLE001 in hermes_cli/update_cmd_windows.py: - _service_gateway_ready.under_service: service.pid() raises psutil.Error / OSError and int() ValueError/TypeError; those still mean "owns nothing". - _service_running: psutil missing (ImportError), missing service / SCM access (psutil.Error, OSError), non-Windows psutil without win_service_get (AttributeError) still mean "not running". - _unmapped_ready_filter.matches: Process().cmdline() raises psutil.Error (exited/denied/zombie), OSError, ValueError (invalid pid): still no match. - _resume_paused_set.attempt keeps its catch-all (per-step isolation: a failed step is recorded and re-raised as a summary so it never keeps the other paused runtimes stopped) and is waived on the except line. The narrowed sets are exhaustive for the calls they guard; behaviour unchanged. --- hermes_cli/update_cmd_windows.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 647f6b40f9820..827698ac38b37 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -257,7 +257,7 @@ def _hermes_holder_subcommand(cmdline: str) -> str | None: # ``python -c … -m hermes_cli.main ``: the entry token belongs to the argv the # inline source carries for a LATER spawn, not to this holder (#107002) -- unless the source is a # Hermes bootstrap running the entry point in this process (#124318). - from gateway.status import command_line_runs_inline_source, inline_bootstrap_argv + from gateway.status_inline_source import command_line_runs_inline_source, inline_bootstrap_argv normalized = [t.strip("\"'").replace("\\", "/") for t in tokens] if command_line_runs_inline_source(normalized): tokens = inline_bootstrap_argv(normalized) @@ -1260,7 +1260,7 @@ def _service_gateway_ready(name: str, profile: str | None, timeout_s: float | No def under_service(pids): try: service_pid = int(service.pid() or 0) - except Exception: + except (psutil.Error, OSError, ValueError, TypeError): # SCM/psutil failure or a non-int pid return [] owned = [] for pid in pids: @@ -1276,9 +1276,13 @@ def under_service(pids): def _service_running(name: str) -> bool: + try: + import psutil # noqa: PLC0415 -- the same module _win_service resolves (tests stub it) + except ImportError: + return False try: return _win_service(name)[1].status() == "running" - except Exception: + except (psutil.Error, OSError, AttributeError): # missing service, SCM access, non-Windows psutil return False @@ -1395,7 +1399,7 @@ def matches(pid: int) -> bool: return False try: return list(psutil.Process(pid).cmdline() or [])[1:] == tail - except Exception: + except (psutil.Error, OSError, ValueError): # exited, access denied, zombie, invalid pid return False return lambda pids: [pid for pid in _owned_gateway_pids(pids) if matches(int(pid))] @@ -1502,7 +1506,7 @@ def _resume_paused_set(token: dict) -> None: def attempt(step) -> None: try: step() - except Exception as exc: + except Exception as exc: # health: allow BLE001 -- per-step isolation: any failed step is recorded and re-raised as a summary so it never keeps the other paused runtimes stopped failures.append(str(exc)) attempt(lambda: _resume_windows_services(token)) From 5fb3d944919c42cfc9b1b4cad71ac131e2f09efb Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:18:51 -0700 Subject: [PATCH 33/63] chore(update): waive the pause-record recovery boundaries' catch-alls Code-health BLE001 in hermes_cli/update_pause_record.py. All three are deliberate boundaries, now waived on the except line with the reason: - _venv_is_current: fail closed -- any error reading venv state means "not current", so paused gateways stay stopped. - _resume_fenced: any resume error is reported and the claim handed back in the finally path, never raised into the launch. - recover: start-of-run recovery never bricks a launch. Behaviour unchanged. --- hermes_cli/update_pause_record.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index c5a22b68fb04e..ed0ac38dbbc2d 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -178,7 +178,7 @@ def _venv_is_current(root: Path) -> bool: try: import pm return bool(pm.venv_is_current(project_root=root)) - except Exception: + except Exception: # health: allow BLE001 -- fail closed: any error reading the venv state means "not current", so paused gateways stay stopped return False @@ -675,7 +675,7 @@ def _resume_fenced(claim_path: Path, body: dict) -> None: _resume_windows_gateways_after_update(token) else: token["resume_needed"] = False - except Exception as exc: + except Exception as exc: # health: allow BLE001 -- recovery boundary: any resume error is reported and the claim handed back (finally), never raised into the launch print(f" ⚠ Could not restart every paused gateway: {exc}. Run `hermes update` or " "`hermes gateway start`.", file=sys.stderr) finally: @@ -709,5 +709,5 @@ def recover(argv: list[str] | None = None) -> None: won = claim(src) if won is not None: _resume_claimed(*won) - except Exception as exc: # never brick a launch on recovery + except Exception as exc: # health: allow BLE001 -- never brick a launch on recovery; the record stays for the next one print(f" ⚠ Paused-gateway recovery skipped: {exc}", file=sys.stderr) From e302c46bc29e8b6d8a92dde64a7b9e6ddbfac455 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:42:38 -0700 Subject: [PATCH 34/63] fix(update): keep every pause's tree baseline in the paused-gateway gate merge_into carried an adopted orphan's pre_sha/dirty_at_pause with setdefault, so record_pause's later stamp_tree kept the ORPHAN's baseline and dropped this run's own. When the killed run had moved HEAD X->Y, this run's torn same-HEAD checkout at Y was judged against X: HEAD "moved", so only the venv was checked and the gate passed (probe: composite (True, '') vs own stamp (False, 'half-written')). The token now carries `baselines`: one {pause_id, pre_sha, dirty_at_pause} per pause (stamp_tree is idempotent per pause_id; merge_into, finish_pause and abandon_pause carry the list). The gate fails when ANY baseline taken at the current HEAD sees a tracked change its own pause did not, so the oldest evidence still blocks a torn tree the orphan left and this run's evidence blocks the tree it tore itself. The record format is new on this stack (not on main): no compat read. Review W1. --- hermes_cli/update_pause_record.py | 45 +++++++++++--------- tests/hermes_cli/test_update_pause_record.py | 25 +++++++++++ 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index ed0ac38dbbc2d..cc113894ba084 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -13,8 +13,10 @@ Resuming is gated on a whole tree: no interrupted-pull marker; at the pre-update HEAD, no tracked change beyond the ones present at pause time (git died before moving HEAD); at a moved HEAD, -dependencies current for it (a build step may rewrite tracked files there). Otherwise the record -stays for the next launch, which runs after the interrupted-pull restore and the dependency sync. +dependencies current for it (a build step may rewrite tracked files there). Every pause folded into +a record keeps its own baseline (``baselines``): each one taken at the current HEAD must hold. +Otherwise the record stays for the next launch, which runs after the interrupted-pull restore and +the dependency sync. Custody: every mutation (write, claim, retire, discharge) happens while holding a kernel lock on ``/.hermes-update-paused-gateways.lock`` (flock / msvcrt, released by the kernel when @@ -165,12 +167,13 @@ def tracked_changes(root: Path) -> list[str] | None: def stamp_tree(token: dict, root: Path | None = None) -> dict: - """Record what "the tree before this update" was, for the whole-tree gate.""" + """Record what "the tree before this update" was, for the whole-tree gate: one baseline per + pause (idempotent for this token's ``pause_id``), next to the baselines of the pauses it absorbed.""" root = root or install_root() - if (root / ".git").exists(): - token.setdefault("pre_sha", head_sha(root)) - token.setdefault("dirty_at_pause", tracked_changes(root)) - token.setdefault("pause_id", uuid.uuid4().hex) + pause_id = token.setdefault("pause_id", uuid.uuid4().hex) + baselines = token.setdefault("baselines", []) + if (root / ".git").exists() and not any(b.get("pause_id") == pause_id for b in baselines): + baselines.append({"pause_id": pause_id, "pre_sha": head_sha(root), "dirty_at_pause": tracked_changes(root)}) return token @@ -192,14 +195,18 @@ def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]: head = head_sha(root) if head is None: return False, "the checkout HEAD is unreadable" - if head == token.get("pre_sha"): - # HEAD never moved: a new tracked change is git's half-written checkout. + at_head = [b for b in token.get("baselines") or [] if b.get("pre_sha") == head] + if at_head: + # HEAD never moved for these pauses: a tracked change one of them did not see is git's + # half-written checkout. Each is judged on its own set — a killed run that moved HEAD + # elsewhere vouches for nothing here, and this run's set never certifies an older one's. changes = tracked_changes(root) if changes is None: return False, "git cannot read the checkout state" - unexpected = sorted(set(changes) - set(token.get("dirty_at_pause") or [])) - if unexpected: - return False, f"the checkout has {len(unexpected)} file(s) git left half-written (e.g. {unexpected[0]})" + for baseline in at_head: + unexpected = sorted(set(changes) - set(baseline.get("dirty_at_pause") or [])) + if unexpected: + return False, f"the checkout has {len(unexpected)} file(s) git left half-written (e.g. {unexpected[0]})" return True, "" if not _venv_is_current(root): return False, "dependencies are not current for the updated code yet" @@ -514,7 +521,7 @@ def record_pause(token: dict, adopted: dict | None, claims: list[Path]) -> dict: return token -_CARRIED = ("pause_id", "pre_sha", "dirty_at_pause", "identities", "stop_requested", "stop_sent", "stopper_pid", +_CARRIED = ("pause_id", "baselines", "identities", "stop_requested", "stop_sent", "stopper_pid", "stop_markers", "absorbed") @@ -533,7 +540,7 @@ def abandon_pause(intended: dict, adopted: dict | None) -> None: if adopted is None: discharge(intended) return - carried = {key: intended[key] for key in ("pause_id", "pre_sha", "dirty_at_pause", "absorbed") if key in intended} + carried = {key: intended[key] for key in ("pause_id", "baselines", "absorbed") if key in intended} write({**adopted, **carried}, owner=UNOWNED) @@ -604,11 +611,11 @@ def merge_into(token: dict | None, adopted: dict) -> dict: obligation id (and every id it absorbed) is recorded so a leftover copy is never resumed twice.""" token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []} token["resume_needed"] = True - # The oldest tree evidence wins: stamping the current tree instead would certify whatever a - # killed update left half-written as "dirty before the pause". - for key in ("pre_sha", "dirty_at_pause"): - if key in adopted: - token.setdefault(key, adopted[key]) + # Every pause keeps its own tree evidence: the adopted baseline stops this run's stamp from + # certifying what a killed update left half-written as "dirty before the pause", and this + # run's own baseline still guards the HEAD a killed update moved to. + baselines = token.setdefault("baselines", []) + baselines.extend(b for b in adopted.get("baselines") or [] if b not in baselines) absorbed = token.setdefault("absorbed", []) absorbed.extend(i for i in [adopted.get("pause_id"), *(adopted.get("absorbed") or [])] if i and i not in absorbed) profiles = token.setdefault("profiles", {}) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 4a31f40b38fad..7353f5389efdf 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -134,6 +134,31 @@ def test_an_update_adopting_an_orphan_never_certifies_the_tree_it_left_torn(tmp_ assert not whole and "a.py" in why, "adoption certified the torn tree as the pre-update baseline" +def test_an_adopted_baseline_never_replaces_this_runs_own(tmp_path, monkeypatch): + """Review W1: the killed run moved HEAD X->Y; this run starts at Y and its git dies mid-checkout + without moving HEAD. The orphan's X baseline says nothing about Y — this run's own must still gate.""" + root = tmp_path / "checkout" + root.mkdir() + _git(root, "init", "-q") + (root / "a.py").write_text("v1\n", encoding="utf-8") + _git(root, "add", ".") + _git(root, "commit", "-qm", "X") + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home")) + monkeypatch.setattr(pause_record, "install_root", lambda: root) + pause_record.write(pause_record.stamp_tree({"resume_needed": True, "profiles": {"default": 4242}}), + owner=pause_record.UNOWNED) + (root / "a.py").write_text("v2\n", encoding="utf-8") + _git(root, "commit", "-qam", "Y") # the killed update moved HEAD, synced Y's dependencies, then died + monkeypatch.setattr(pause_record, "_venv_is_current", lambda root: True) + + adopted, claims = pause_record.adopt_orphans() + token = pause_record.record_pause({"resume_needed": True, "profiles": {"beta": 99}}, adopted, claims) + assert pause_record.tree_is_whole(token, root) == (True, ""), "a clean tree at this run's own HEAD must pass" + (root / "a.py").write_text("half-written\n", encoding="utf-8") # this run's git died at HEAD Y + whole, why = pause_record.tree_is_whole(pause_record.read()["token"], root) + assert not whole and "a.py" in why, "this run's own baseline was dropped for the adopted one" + + def _orphan(tmp_path: Path, profiles: dict) -> None: """A record whose owner — a real ``hermes update`` stand-in — was SIGKILLed after writing it.""" owner = _child(""" From 049949e1bfcc164f68a32db8ff795525fce7d6cf Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:43:42 -0700 Subject: [PATCH 35/63] fix(update): narrow the service-readiness parent walk to psutil/OS errors _service_gateway_ready.under_service wrapped psutil.Process(pid).parents() in an unwaived suppress(Exception): any bug in the walk read as "this gateway is not under the service", i.e. a silent not-ready verdict that leaves a resumed service owed forever. Only a vanished gateway or an unreadable process tree (psutil.Error, OSError) means "not the service's"; anything else now reaches the resume boundary, which reports it. Review W2. --- hermes_cli/update_cmd_windows.py | 4 ++- ...t_windows_update_restart_reconciliation.py | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 827698ac38b37..7bf08484b73e9 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -1264,7 +1264,9 @@ def under_service(pids): return [] owned = [] for pid in pids: - with suppress(Exception): + # A gateway that exited or a process tree psutil cannot read is not the service's; any + # other error is a bug the resume boundary reports, not a silent "not under service". + with suppress(psutil.Error, OSError): if service_pid and service_pid in {int(p.pid) for p in psutil.Process(int(pid)).parents()}: owned.append(int(pid)) return owned diff --git a/tests/hermes_cli/test_windows_update_restart_reconciliation.py b/tests/hermes_cli/test_windows_update_restart_reconciliation.py index 6c1a2ac6a1a1f..a7599388c2326 100644 --- a/tests/hermes_cli/test_windows_update_restart_reconciliation.py +++ b/tests/hermes_cli/test_windows_update_restart_reconciliation.py @@ -100,3 +100,37 @@ def test_resume_unregisters_its_own_atexit_fallback_before_running(monkeypatch): assert calls == [update_cmd_windows._resume_windows_gateways_after_update] assert token["resume_needed"] is False + + +def test_service_readiness_filter_skips_vanished_gateways_but_never_swallows_bugs(monkeypatch): + """Review W2: the service-ownership filter reads real process parents; a vanished pid is just + not the service's, but an unexpected error must surface instead of reading as "not ready".""" + import os + import subprocess + import sys + + import psutil + + from hermes_cli import update_cmd_windows + + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], stdin=subprocess.DEVNULL) + gone = subprocess.Popen([sys.executable, "-c", "pass"], stdin=subprocess.DEVNULL) + gone.wait(timeout=30) + service = type("Svc", (), {"pid": staticmethod(lambda: os.getpid())})() # the "service" is this process + monkeypatch.setattr(update_cmd_windows, "_win_service", lambda _name: (psutil, service)) + monkeypatch.setattr(gateway_windows, "_wait_for_gateway_ready", + lambda **kw: kw["pid_filter"]([child.pid, gone.pid])) + try: + assert update_cmd_windows._service_gateway_ready("svc", None, timeout_s=0) == [child.pid] + finally: + child.kill() + child.wait(timeout=30) + + def broken(self): + raise RuntimeError("bug in the parent walk") + + with monkeypatch.context() as m: # scoped: the conftest kill guard walks parents too + m.setattr(psutil.Process, "parents", broken) + m.setattr(gateway_windows, "_wait_for_gateway_ready", lambda **kw: kw["pid_filter"]([os.getpid()])) + with pytest.raises(RuntimeError, match="parent walk"): + update_cmd_windows._service_gateway_ready("svc", None, timeout_s=0) From ced270991e3801b0d3c3fbafb95408d43aa3caf4 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:47:53 -0700 Subject: [PATCH 36/63] fix(update): pause-record temp cleanup, per-thread mutex, corrupt history quarantine Three hygiene gaps in the Windows paused-gateway record: - _atomic_write left `..tmp` beside the record when os.replace was refused (a Windows reader without FILE_SHARE_DELETE) or the write failed: the temp is now unlinked and the error still propagates. - _mutex counted re-entry process-wide, so a second thread of the same process (gateway consumer, watcher) entered while the first held the kernel lock: the depth is now threading.local and each thread's first entry takes its own descriptor + flock/msvcrt lock, so threads exclude each other like processes. - A `.retired` list that reads but does not parse raised RetiredUnknown from adopt_orphans -> every Windows update aborted, forever. With no record or claim of this checkout left the list guards nothing: it is renamed to `.corrupt` (warning on stderr) and the update goes on. While a record/claim exists the list may be the only proof it was completed, so it stays unknown (R9 fail-closed is unchanged); a list that cannot be read at all (sharing violation) is unknown as before. Review W3. --- hermes_cli/update_pause_record.py | 67 +++++++++++++++----- tests/hermes_cli/test_update_pause_record.py | 54 ++++++++++++++++ 2 files changed, 104 insertions(+), 17 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index cc113894ba084..a225adbba335e 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -31,6 +31,8 @@ FILE_SHARE_DELETE) leaves a copy that is redundant by id, never an obligation that executes again. The list is read only under the mutex, and only its absence means "nothing retired": a list that cannot be read means unknown, so nothing is claimed and the list is never rewritten from that read. +A list that reads but does not parse is unknown too while any record or claim exists; with none +left it guards nothing and is set aside as ``.corrupt``. """ from __future__ import annotations @@ -41,6 +43,7 @@ import os import subprocess import sys +import threading import time import uuid from contextlib import contextmanager, suppress @@ -49,7 +52,9 @@ RECORD_STEM = ".hermes-update-paused-gateways" MUTEX_NAME = RECORD_STEM + ".lock" _MUTEX_WAIT_S = 10.0 -_mutex_depth = 0 +# Re-entry depth per THREAD: another thread of this process (a gateway's consumer, a watcher) +# must wait for the kernel lock like any other holder, never ride on this thread's hold. +_mutex_held = threading.local() def install_root() -> Path: @@ -102,14 +107,15 @@ class RetiredUnknown(OSError): @contextmanager def _mutex(wait_s: float = _MUTEX_WAIT_S): - """Exclusive kernel lock on the record directory's sidecar (A7). Re-entrant in-process.""" - global _mutex_depth - if _mutex_depth: - _mutex_depth += 1 + """Exclusive kernel lock on the record directory's sidecar (A7). Re-entrant per thread; each + thread's first entry opens its own descriptor, so threads exclude each other like processes.""" + depth = getattr(_mutex_held, "depth", 0) + if depth: + _mutex_held.depth = depth + 1 try: yield finally: - _mutex_depth -= 1 + _mutex_held.depth -= 1 return from hermes_cli import update_lock path = record_path().with_name(MUTEX_NAME) @@ -119,13 +125,13 @@ def _mutex(wait_s: float = _MUTEX_WAIT_S): deadline = time.monotonic() + wait_s while not update_lock._try_lock(fd): if time.monotonic() > deadline: - raise RecordBusy(f"{path} is held by another process") + raise RecordBusy(f"{path} is held by another process or thread") time.sleep(0.05) - _mutex_depth = 1 + _mutex_held.depth = 1 try: yield finally: - _mutex_depth = 0 + _mutex_held.depth = 0 update_lock._unlock(fd) finally: os.close(fd) @@ -216,11 +222,18 @@ def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]: def _atomic_write(path: Path, body: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") - with open(tmp, "w", encoding="utf-8") as fh: - json.dump(body, fh, indent=1, sort_keys=True) - fh.flush() - os.fsync(fh.fileno()) - os.replace(tmp, path) + try: + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(body, fh, indent=1, sort_keys=True) + fh.flush() + os.fsync(fh.fileno()) + os.replace(tmp, path) + except BaseException: + # A refused replace (a Windows reader without FILE_SHARE_DELETE) or a failed write must + # not leave the half-published temp beside the record; the error itself still propagates. + with suppress(OSError): + tmp.unlink() + raise def read(path: Path | None = None) -> dict | None: @@ -341,16 +354,36 @@ def _retired(path: Path) -> set[str]: claimed again and the list is never rewritten from a read that failed.""" target = _retired_path(path) try: - ids = json.loads(target.read_text(encoding="utf-8-sig")).get("ids") + text = target.read_text(encoding="utf-8-sig", errors="replace") except FileNotFoundError: return set() - except (OSError, ValueError, AttributeError) as exc: + except OSError as exc: # a refused read (sharing violation, AV scanner): unknown raise RetiredUnknown(f"cannot read the retired paused-gateway list {target}: {exc}") from exc + try: + ids = json.loads(text).get("ids") + except (ValueError, AttributeError): # not JSON / not an object: malformed, like a non-list + ids = None if not isinstance(ids, list): - raise RetiredUnknown(f"the retired paused-gateway list {target} is malformed") + return _quarantine_retired(path, target) return {str(i) for i in ids} +def _quarantine_retired(path: Path, target: Path) -> set[str]: + """A retired list that reads but does not parse. While a record or claim of this checkout + exists the list may be the only proof it is complete: unknown, fail closed. With none left it + guards nothing, and keeping it would abort every Windows update: set it aside and go on.""" + if path.exists() or _claims(path): + raise RetiredUnknown(f"the retired paused-gateway list {target} is malformed") + corrupt = target.with_suffix(".corrupt") + try: + os.replace(target, corrupt) + except OSError as exc: + raise RetiredUnknown(f"cannot set aside the malformed retired paused-gateway list {target}: {exc}") from exc + print(f" ⚠ Set aside a malformed paused-gateway history (no paused set refers to it): {corrupt}", + file=sys.stderr) + return set() + + def _retire(path: Path, carriers: list[tuple[Path, dict]]) -> None: """Complete the obligations *carriers* hold (call under the mutex): their ids (and every id they absorbed) go on the durable retired list FIRST; a failed read or write raises and deletes diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 7353f5389efdf..4d2606bcd4167 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -596,3 +596,57 @@ def owed() -> list[str]: retired.chmod(0o644) assert json.loads(retired.read_text(encoding="utf-8-sig"))["ids"] == [done], "the history was rewritten" assert done not in owed() + + + +# --- Review W3: record hygiene -------------------------------------------------------------------- +def test_a_refused_publish_leaves_no_temp_beside_the_record(tmp_path, monkeypatch): + target = tmp_path / "records" / "record.json" + + def refuse(src, dst): + raise PermissionError(13, "Access is denied", str(dst)) # a reader without FILE_SHARE_DELETE + + monkeypatch.setattr(pause_record.os, "replace", refuse) + with pytest.raises(PermissionError): + pause_record._atomic_write(target, {"schema": 1}) + assert sorted(p.name for p in target.parent.iterdir()) == [], "the half-published temp was left behind" + + +def test_the_record_mutex_excludes_other_threads_of_this_process(tmp_path, monkeypatch): + import threading + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + held, release = threading.Event(), threading.Event() + + def holder(): + with pause_record._mutex(): + held.set() + release.wait(30) + + thread = threading.Thread(target=holder) + thread.start() + try: + assert held.wait(30) + with pytest.raises(pause_record.RecordBusy), pause_record._mutex(wait_s=0.3): + pass # rode on the other thread's hold + finally: + release.set() + thread.join(30) + with pause_record._mutex(), pause_record._mutex(): # re-entry on one thread still works + pass + + +def test_a_malformed_retired_list_is_set_aside_only_when_nothing_refers_to_it(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + retired = pause_record._retired_path(pause_record.record_path()) + retired.write_text("{trunc", encoding="utf-8") + assert pause_record.adopt_orphans() == (None, []), "a corrupt history aborted the update" + assert not retired.exists() and retired.with_suffix(".corrupt").read_text(encoding="utf-8-sig") == "{trunc" + + # With a record on disk the list may be the only proof it was completed: still unknown. + pause_record.write({"pause_id": "c" * 32, "resume_needed": True, "profiles": {"default": 4242}}, + owner=pause_record.UNOWNED) + retired.write_text("{trunc", encoding="utf-8") + with pytest.raises(pause_record.RetiredUnknown): + pause_record.orphans() + assert retired.read_text(encoding="utf-8-sig") == "{trunc" From 9d6c1358b76967303d18d24fa974f486a14426c3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 05:59:54 -0700 Subject: [PATCH 37/63] fix(update): the pause record's owner identity uses the reader's creation-time probe identity() took its creation time from process_identity._process_create_time (psutil only), while identity_is_live judges with update_lock.incarnation_live, whose producer update_lock.process_create_time falls back to the stdlib probe. Without psutil every record was written with ct=None, which the reader can only call unprovable (live), so a killed updater's pause was never orphaned on such a host. identity() now calls update_lock.process_create_time, the same producer the marker uses. Review 5411136378 (#132338) fix 2. --- hermes_cli/update_pause_record.py | 14 +++++++------- tests/hermes_cli/test_update_pause_record.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index a225adbba335e..eb9a54516bcb4 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -70,15 +70,15 @@ def record_path() -> Path: return get_default_hermes_root() / f"{RECORD_STEM}.{_install_key()}.json" -def _create_time(pid: int) -> float | None: - from hermes_cli.process_identity import _process_create_time - return _process_create_time(pid) - - def identity(pid: int | None = None) -> dict: - """``{"pid", "ct"}`` with ``ct`` spelled like the update marker's line 3 (``ct:``).""" + """``{"pid", "ct"}`` with ``ct`` spelled like the update marker's line 3 (``ct:``). + + The creation time comes from the probe :func:`identity_is_live` judges with + (``update_lock.process_create_time``: psutil, else stdlib), so a host without psutil still + writes a provable identity instead of ``ct=None``.""" + from hermes_cli import update_lock pid = os.getpid() if pid is None else int(pid) - created = _create_time(pid) + created = update_lock.process_create_time(pid) return {"pid": pid, "ct": None if created is None else f"ct:{created:.3f}"} diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 4d2606bcd4167..fc3333bafe2bd 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -650,3 +650,13 @@ def test_a_malformed_retired_list_is_set_aside_only_when_nothing_refers_to_it(tm with pytest.raises(pause_record.RetiredUnknown): pause_record.orphans() assert retired.read_text(encoding="utf-8-sig") == "{trunc" + + +def test_a_host_without_psutil_records_an_identity_the_liveness_rule_can_prove(monkeypatch): + """Review 5411136378 fix 2: the record's creation time comes from the same probe the reader + judges with, so a psutil-less host never writes ``ct=None`` (unprovable, so never orphaned).""" + from hermes_cli import update_lock + monkeypatch.setitem(sys.modules, "psutil", None) # `import psutil` raises ImportError + ident = pause_record.identity() + assert ident["ct"] is not None, "no creation time without psutil" + assert update_lock.incarnation_live(ident["pid"], ident["ct"]) is True From dafa23fd405b3a259d1918873c4e7fe336552f7f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:01:42 -0700 Subject: [PATCH 38/63] fix(update): the paused-gateway tree gate finds PM's git and runs it in the update's custody update_pause_record._git ran a bare ["git", "-C", root] through subprocess.run. On a Windows install whose only git is the copy install.ps1 staged in PM's store (expose_pm_git puts it on PATH only inside the updater), every gate read died with WinError 2: head_sha returned None and tree_is_whole refused on every launch, so paused gateways stayed stopped. Reads now resolve git like the updater does without installing anything (PATH, then PM's recorded entry, then the lockfile's staged entry) and run through update_custody.run_git, so inside an update they are job-bound on Windows and carry the custody config. The resolver mirrors _early_recovery._git_executable from #132361 minus the marker-recorded path; this lane does not carry that module's version. Review 5411136378 (#132338) fix 3. --- hermes_cli/update_pause_record.py | 35 +++++++++++++++++--- tests/hermes_cli/test_update_pause_record.py | 25 ++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index eb9a54516bcb4..426723cac97b6 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -41,6 +41,7 @@ import hashlib import json import os +import shutil import subprocess import sys import threading @@ -137,12 +138,38 @@ def _mutex(wait_s: float = _MUTEX_WAIT_S): os.close(fd) +def _git_executable() -> str: + """The git ``hermes update`` runs, found without installing anything: PATH's, else the copy + install.ps1 staged in PM's store (``_subprocess_compat.expose_pm_git`` puts it on PATH only + inside the updater). A bare ``git`` on such an install dies with WinError 2, so every gate + read would fail and the paused gateways would never be restarted.""" + found = shutil.which("git") + if found: + return found + with suppress(Exception): # health: allow BLE001 -- no PM, no store, no Windows git package: PATH's answer ("git") stands + import pm + from pm import paths + from pm.lock import Lockfile + + recorded = pm.installed_package("git", allow_outdated=True) + if recorded is not None and recorded.binary is not None and recorded.binary.is_file(): + return str(recorded.binary) + package, target = pm.get_package("git"), pm.current_target() + version = Lockfile(paths.lockfile_path()).version("git") + staged = package.binary(paths.store_root() / package.store_entry(version, target), target) + if staged is not None and staged.is_file(): + return str(staged) + return "git" + + def _git(root: Path, *args: str) -> subprocess.CompletedProcess | None: + """A read-only git query in the updater's custody (``update_custody.run_git``: job-bound on + Windows inside an update, custody config in argv); ``None`` when git cannot run.""" + from hermes_cli.update_custody import run_git try: - return subprocess.run( - ["git", "-C", str(root), *args], capture_output=True, text=True, encoding="utf-8", - errors="replace", stdin=subprocess.DEVNULL, timeout=30, check=False) - except (OSError, subprocess.SubprocessError): + return run_git([_git_executable(), "-C", str(root)], args, capture_output=True, text=True, + encoding="utf-8", errors="replace", stdin=subprocess.DEVNULL, timeout=30, check=False) + except (OSError, ValueError, subprocess.SubprocessError): return None diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index fc3333bafe2bd..92621ec3fcda1 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -660,3 +660,28 @@ def test_a_host_without_psutil_records_an_identity_the_liveness_rule_can_prove(m ident = pause_record.identity() assert ident["ct"] is not None, "no creation time without psutil" assert update_lock.incarnation_live(ident["pid"], ident["ct"]) is True + + +def test_the_tree_gate_finds_pm_git_and_runs_it_in_the_updaters_custody(tmp_path, monkeypatch): + """Review 5411136378 fix 3: on an install whose only git is PM's staged copy (no git on PATH), + a bare ``git`` made ``head_sha`` None, so ``tree_is_whole`` refused forever.""" + import shutil + from types import SimpleNamespace + + import pm + from hermes_cli import update_custody + root = tmp_path / "checkout" + root.mkdir() + _git(root, "init", "-q") + (root / "a.py").write_text("v1\n", encoding="utf-8") + _git(root, "add", ".") + _git(root, "commit", "-qm", "v1") + head, pm_git = _git(root, "rev-parse", "HEAD"), shutil.which("git") + ran, real_run_git = [], update_custody.run_git + monkeypatch.setattr(update_custody, "run_git", lambda cmd, args, **kw: ran.append(cmd[0]) or real_run_git(cmd, args, **kw)) + monkeypatch.setattr(pm, "installed_package", + lambda name, allow_outdated=False: SimpleNamespace(binary=Path(pm_git)) if name == "git" else None) + (tmp_path / "no-git").mkdir() + monkeypatch.setenv("PATH", str(tmp_path / "no-git")) + assert pause_record.head_sha(root) == head + assert ran == [pm_git], "the gate's git bypassed update_custody.run_git" From d4852113275af5e9f89f51d8f736e8532e6d12a8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:04:57 -0700 Subject: [PATCH 39/63] fix(update): a moved HEAD holds paused gateways for dependencies only where a launch syncs them Decision for review 5411136378's "moved HEAD" item. tree_is_whole required pm.venv_is_current at a moved HEAD and counted any exception as "not current". The updater's early exits after the pull (wrong branch, channel sha mismatch) reach the resume before the dependency sync, so they deferred; on a checkout whose launches never sync dependencies (developer checkout, no install stamp, lazy installs disabled, ZIP) or whose probe raises, the deferral never ended and gateways stayed stopped where main restarted them. Rule chosen (gateways never stay stopped after a committed update without an owed follow-up that can succeed): defer only when the install is self-managed (updateMechanism "self", the conditions venv_sync.prepare_launch syncs under), because every later launch syncs dependencies before startup recovery runs, so the owed record resumes on the next launch. Everywhere else, and when currency cannot be established, resume now; the restarted gateway's own launch syncs or prints the remedy. Changed assertions: test_resume_waits_for_a_whole_tree pinned "dependencies" deferral on a stamp-less checkout, which is the stranding this item reports; it now expects a resume. test_an_adopted_baseline_never_replaces_this_runs_own patches the renamed helper (_venv_is_current -> _deps_hold_resume). --- hermes_cli/update_pause_record.py | 25 +++++++++--- tests/hermes_cli/test_update_pause_record.py | 40 +++++++++++++++++--- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 426723cac97b6..51310dda38926 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -13,7 +13,7 @@ Resuming is gated on a whole tree: no interrupted-pull marker; at the pre-update HEAD, no tracked change beyond the ones present at pause time (git died before moving HEAD); at a moved HEAD, -dependencies current for it (a build step may rewrite tracked files there). Every pause folded into +dependencies current for it where a launch syncs them (a build step may rewrite tracked files there). Every pause folded into a record keeps its own baseline (``baselines``): each one taken at the current HEAD must hold. Otherwise the record stays for the next launch, which runs after the interrupted-pull restore and the dependency sync. @@ -210,11 +210,26 @@ def stamp_tree(token: dict, root: Path | None = None) -> dict: return token -def _venv_is_current(root: Path) -> bool: +def _deps_hold_resume(root: Path) -> bool: + """At a moved HEAD: are stale dependencies a reason to keep the paused set stopped? + + Only where waiting converges. A self-managed install (``updateMechanism: self``) syncs its + dependencies at the start of every launch (``venv_sync.prepare_launch``), before startup + recovery runs, so a deferred set comes back on the next launch. Anywhere else nothing a launch + does makes them current (a developer checkout owns its own venv): the set resumes now, as it + did before the pause was durable, and a currency the probe cannot establish never holds it + either — the restarted gateway's own launch syncs or reports the remedy. + """ + if (os.environ.get("HERMES_DISABLE_LAZY_INSTALLS", "").lower() in ("1", "true", "yes") + or not (root / ".git").exists() or not (root / "pyproject.toml").is_file()): + return False # prepare_launch skips the sync: waiting would never end try: + from hermes_cli.steward import read_install_stamp + if read_install_stamp(root).get("updateMechanism") != "self": + return False import pm - return bool(pm.venv_is_current(project_root=root)) - except Exception: # health: allow BLE001 -- fail closed: any error reading the venv state means "not current", so paused gateways stay stopped + return not pm.venv_is_current(project_root=root) + except Exception: # health: allow BLE001 -- unknown currency must not strand the set: resume, the gateway's own launch syncs or names the remedy return False @@ -241,7 +256,7 @@ def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]: if unexpected: return False, f"the checkout has {len(unexpected)} file(s) git left half-written (e.g. {unexpected[0]})" return True, "" - if not _venv_is_current(root): + if _deps_hold_resume(root): return False, "dependencies are not current for the updated code yet" return True, "" diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 92621ec3fcda1..a45388321817c 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -106,12 +106,12 @@ def test_resume_waits_for_a_whole_tree(tmp_path): assert pause_record.tree_is_whole(token, root)[0] is False marker.unlink() - _git(root, "commit", "-qam", "v2") # HEAD moved: only a dependency sync for it makes it whole + _git(root, "commit", "-qam", "v2") # HEAD moved: judged on its dependencies alone (root / "b.py").write_text("rewritten by a build step\n", encoding="utf-8") - whole, why = pause_record.tree_is_whole(token, root) - # A committed update is judged on its dependencies alone; a tracked file the build rewrote - # must not keep the gateways stopped on every later launch. - assert not whole and "dependencies" in why, why + # A committed update: a tracked file the build rewrote must not keep the gateways stopped on + # every later launch, and on a checkout whose launches never sync dependencies (this one has + # no install stamp) waiting for them would never end either (review 5411136378). + assert pause_record.tree_is_whole(token, root) == (True, "") def test_an_update_adopting_an_orphan_never_certifies_the_tree_it_left_torn(tmp_path, monkeypatch): @@ -149,7 +149,7 @@ def test_an_adopted_baseline_never_replaces_this_runs_own(tmp_path, monkeypatch) owner=pause_record.UNOWNED) (root / "a.py").write_text("v2\n", encoding="utf-8") _git(root, "commit", "-qam", "Y") # the killed update moved HEAD, synced Y's dependencies, then died - monkeypatch.setattr(pause_record, "_venv_is_current", lambda root: True) + monkeypatch.setattr(pause_record, "_deps_hold_resume", lambda root: False) adopted, claims = pause_record.adopt_orphans() token = pause_record.record_pause({"resume_needed": True, "profiles": {"beta": 99}}, adopted, claims) @@ -685,3 +685,31 @@ def test_the_tree_gate_finds_pm_git_and_runs_it_in_the_updaters_custody(tmp_path monkeypatch.setenv("PATH", str(tmp_path / "no-git")) assert pause_record.head_sha(root) == head assert ran == [pm_git], "the gate's git bypassed update_custody.run_git" + + +@pytest.mark.parametrize("mechanism, probe, holds", [ + ("self", lambda **kw: False, True), # the next launch syncs first: deferring converges + ("self", lambda **kw: True, False), + ("self", lambda **kw: (_ for _ in ()).throw(ValueError("invalid recorded dependency state")), False), + (None, lambda **kw: False, False), # nothing a launch does makes them current: main resumed +]) +def test_a_moved_head_holds_gateways_for_dependencies_only_where_a_launch_syncs_them( + tmp_path, monkeypatch, mechanism, probe, holds): + """Review 5411136378 decision (moved HEAD): stale dependencies defer the resume only where the + next launch makes them current; an unknown currency never strands a committed update's set.""" + import pm + from hermes_cli import steward + root = tmp_path / "checkout" + root.mkdir() + _git(root, "init", "-q") + (root / "pyproject.toml").write_text("[project]\nname = 'x'\n", encoding="utf-8") + _git(root, "add", ".") + _git(root, "commit", "-qm", "v1") + token = pause_record.stamp_tree({"resume_needed": True}, root) + _git(root, "commit", "-q", "--allow-empty", "-m", "v2") # the update moved HEAD + monkeypatch.delenv("HERMES_DISABLE_LAZY_INSTALLS", raising=False) + monkeypatch.setattr(steward, "read_install_stamp", lambda r: {"updateMechanism": mechanism} if mechanism else {}) + monkeypatch.setattr(pm, "venv_is_current", probe) + whole, why = pause_record.tree_is_whole(token, root) + assert whole is not holds, why + assert not holds or "dependencies" in why From 2799006dfea657116f8c87da749dfa35361b23a6 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:08:26 -0700 Subject: [PATCH 40/63] fix(update): an unmoved HEAD holds paused gateways only for paths the update's move could write Decision for review 5411136378's "same HEAD" item. At the pre-update HEAD the gate counted every tracked change the pause had not seen as git's half-written checkout. A no-op update (already at the target) still runs its dependency and web builds; one that rewrites a tracked file (a lockfile, a generated asset) made the end-of-update resume defer, and every later launch and every later no-op `hermes update` (adoption keeps the old baseline) deferred again, so the gateways stayed stopped until someone cleaned the file. A user editing a tracked file had the same effect. Rule chosen (a committed update's gateways never wait on a follow-up that cannot succeed, and never start on a torn tree): a fast-forward, reset or merge only writes paths that differ between HEAD and its target, and the update fetches that target first. An unseen change now counts only on a path that differs between HEAD and a commit in FETCH_HEAD or refs/remotes. When git cannot say or nothing was fetched, every unseen change still counts (the previous behaviour), so the torn-tree cells keep holding. --- hermes_cli/update_pause_record.py | 55 ++++++++++++++++++-- tests/hermes_cli/test_update_pause_record.py | 30 +++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 51310dda38926..c0799e7b13d32 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -210,6 +210,54 @@ def stamp_tree(token: dict, root: Path | None = None) -> dict: return token +_MAX_MOVE_TARGETS = 16 + + +def _paths_a_move_could_write(root: Path) -> set[str] | None: + """Tracked paths an update's checkout move could have written while HEAD stayed put: the ones + that differ between HEAD and a commit the checkout fetched (``FETCH_HEAD``) or tracks + (``refs/remotes``) — a fast-forward, reset or merge only writes those. ``None`` when git cannot + say or nothing was fetched (the move's target is then unknown).""" + where = _git(root, "rev-parse", "--git-path", "FETCH_HEAD") + refs = _git(root, "for-each-ref", "--format=%(objectname)", "refs/remotes") + if where is None or where.returncode != 0 or refs is None or refs.returncode != 0: + return None + try: + fetched = (root / where.stdout.strip()).read_text(encoding="utf-8-sig", errors="replace").split("\n") + except FileNotFoundError: + fetched = [] + except OSError: + return None + targets = {line.split()[0] for line in fetched if line.split()} | set(refs.stdout.split()) + if not targets or len(targets) > _MAX_MOVE_TARGETS: + return None + paths: set[str] = set() + for target in sorted(targets): + diff = _git(root, "diff", "--name-only", "-z", "--no-renames", "HEAD", target, "--") + if diff is None or diff.returncode != 0: + return None + paths.update(filter(None, diff.stdout.split("\0"))) + return paths + + +def _half_written(root: Path, changes: list[str], at_head: list[dict]) -> list[str]: + """Tracked changes at an unmoved HEAD that git may have left half-written, for the first + baseline that has any. A change the pause did not see counts only on a path the update's move + could write: a build/sync step (or the user) rewriting any other tracked file is not git's + doing, and holding the set for it would keep gateways stopped until someone cleans the file. + When the move's target is unknown every unseen change still counts.""" + writable: set[str] | None = None + probed = False + for baseline in at_head: + unexpected = sorted(set(changes) - set(baseline.get("dirty_at_pause") or [])) + if unexpected and not probed: + writable, probed = _paths_a_move_could_write(root), True + torn = unexpected if writable is None else [path for path in unexpected if path in writable] + if torn: + return torn + return [] + + def _deps_hold_resume(root: Path) -> bool: """At a moved HEAD: are stale dependencies a reason to keep the paused set stopped? @@ -251,10 +299,9 @@ def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]: changes = tracked_changes(root) if changes is None: return False, "git cannot read the checkout state" - for baseline in at_head: - unexpected = sorted(set(changes) - set(baseline.get("dirty_at_pause") or [])) - if unexpected: - return False, f"the checkout has {len(unexpected)} file(s) git left half-written (e.g. {unexpected[0]})" + torn = _half_written(root, changes, at_head) + if torn: + return False, f"the checkout has {len(torn)} file(s) git left half-written (e.g. {torn[0]})" return True, "" if _deps_hold_resume(root): return False, "dependencies are not current for the updated code yet" diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index a45388321817c..b02e8fdb0df26 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -713,3 +713,33 @@ def test_a_moved_head_holds_gateways_for_dependencies_only_where_a_launch_syncs_ whole, why = pause_record.tree_is_whole(token, root) assert whole is not holds, why assert not holds or "dependencies" in why + + +def test_an_unmoved_head_holds_gateways_only_for_paths_the_updates_move_could_write(tmp_path): + """Review 5411136378 decision (same HEAD): a tracked file a build/sync step rewrote is not git's + half-written checkout; holding the set for it kept gateways stopped on every later launch.""" + origin, root = tmp_path / "origin", tmp_path / "checkout" + origin.mkdir() + _git(origin, "init", "-q") + for name in ("a.py", "b.lock"): + (origin / name).write_text("v1\n", encoding="utf-8") + _git(origin, "add", ".") + _git(origin, "commit", "-qm", "v1") + _git(tmp_path, "clone", "-q", str(origin), str(root)) + (origin / "a.py").write_text("v2\n", encoding="utf-8") + _git(origin, "commit", "-qam", "v2") # the update's target changes a.py only + token = pause_record.stamp_tree({"resume_needed": True}, root) + _git(root, "fetch", "-q", "origin") + + (root / "b.lock").write_text("rewritten by the dependency sync\n", encoding="utf-8") + assert pause_record.tree_is_whole(token, root) == (True, ""), "a build rewrite held the set" + (root / "a.py").write_text("v2 half\n", encoding="utf-8") # git wrote a path of the move, died before HEAD + whole, why = pause_record.tree_is_whole(token, root) + assert not whole and "a.py" in why, why + + # A no-op update (HEAD already at the target) that ran its build: nothing git could write. + _git(root, "checkout", "-q", "--", "a.py", "b.lock") + _git(root, "merge", "-q", "--ff-only", "origin/HEAD") + token = pause_record.stamp_tree({"resume_needed": True}, root) + (root / "b.lock").write_text("rewritten by the dependency sync\n", encoding="utf-8") + assert pause_record.tree_is_whole(token, root) == (True, "") From 6b9b532b3395e8f2100936fe9289ccae7a8e66c9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 07:28:01 -0700 Subject: [PATCH 41/63] test: the pause record's git resolver is a reviewed resolution site update_pause_record._git_executable reads the paused gateways' checkout outside the updater, where install.ps1's PM-staged git is not on PATH, so it falls back to that staged git after PATH. Same shape and reason as hermes_cli._early_recovery._git_executable (#132361), which it becomes once both land; the managed-runtime ratchet needs the row until then. --- tests/fixtures/resolution_allowlist.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/fixtures/resolution_allowlist.json b/tests/fixtures/resolution_allowlist.json index 9c16870afeff5..dfdb7c2b45b81 100644 --- a/tests/fixtures/resolution_allowlist.json +++ b/tests/fixtures/resolution_allowlist.json @@ -688,5 +688,11 @@ "path": "tools/file_operations_lint.py", "symbol": "LintMixin._run_managed_node_linter", "kind": "bare_which" + }, + { + "path": "hermes_cli/update_pause_record.py", + "symbol": "_git_executable", + "kind": "bare_which", + "reason": "gate reads of the paused gateways' checkout run outside the updater, where install.ps1's PM-staged git is not on PATH; falls back to that staged git (moves to hermes_cli._early_recovery._git_executable when #132361 lands)" } ] From 6c6c1e28985af2f7d6f902a4f21bcd412649b5ec Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:26:27 -0700 Subject: [PATCH 42/63] fix: an unmapped gateway's restart debt is retired only by its own runtime The pause saved an unmapped Windows gateway (no profile PID file) as pid + argv only, and readiness accepted any owned-fleet gateway with the same argv tail. Two homes can run identical selectorless argv, so home B's gateway (already restored, or service-ready in the same pass) retired home A's debt while A stayed down; the replay also ran A under the updater's HERMES_HOME. Capture the runtime's home (dashboard_procs._hermes_home_for_pid) and birth at pause, replay on that home, and require readiness from a gateway process on the same home born after the one it replaces. Entries from before home capture keep the argv-only match. --- hermes_cli/gateway.py | 10 +-- hermes_cli/gateway_windows.py | 6 +- hermes_cli/update_cmd_windows.py | 35 +++++++-- .../test_update_pause_unmapped_identity.py | 74 +++++++++++++++++++ 4 files changed, 110 insertions(+), 15 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_unmapped_identity.py diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 8bd8a7bff100b..c285babcaa433 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -977,9 +977,9 @@ def _prepare_profile_gateway_update_restart(profile: str, pid: int) -> str | Non return None -def launch_detached_gateway_restart_by_cmdline(old_pid: int, run_argv: list[str]) -> bool: - """Relaunch a gateway with no profile→PID-file mapping by replaying its captured argv after exit.""" - return old_pid > 0 and bool(run_argv) and _spawn_gateway_restart_watcher(old_pid, list(run_argv)) +def launch_detached_gateway_restart_by_cmdline(old_pid: int, run_argv: list[str], home: str | None = None) -> bool: + """Relaunch a gateway with no profile→PID-file mapping by replaying its captured argv (on Windows under ``home``, the one it ran on) after exit.""" + return old_pid > 0 and bool(run_argv) and _spawn_gateway_restart_watcher(old_pid, list(run_argv), home=home) def launch_detached_profile_gateway_restart(profile: str, old_pid: int) -> bool: @@ -1055,7 +1055,7 @@ def _host_gateway_watcher_env() -> dict[str, str]: return env -def _spawn_gateway_restart_watcher(old_pid: int, run_argv: list[str], *, host: bool | None = None) -> bool: +def _spawn_gateway_restart_watcher(old_pid: int, run_argv: list[str], *, host: bool | None = None, home: str | None = None) -> bool: """Spawn the detached watcher that respawns ``run_argv`` once ``old_pid`` exits. Watcher and respawn both need platform-appropriate detach: POSIX setsid; on Windows ``start_new_session`` does NOT detach (the watcher would die with the CLI console), so ``windows_detach_popen_kwargs()`` supplies flags.""" @@ -1074,7 +1074,7 @@ def _spawn_gateway_restart_watcher(old_pid: int, run_argv: list[str], *, host: b if sys.platform == "win32": try: from hermes_cli.gateway_windows import windowless_gateway_restart_spec - run_argv, respawn_cwd, respawn_env_overlay = windowless_gateway_restart_spec(list(run_argv)) + run_argv, respawn_cwd, respawn_env_overlay = windowless_gateway_restart_spec(list(run_argv), home=home) except Exception: # Fall back to the original argv: a visible window beats a failed respawn. respawn_cwd = "" diff --git a/hermes_cli/gateway_windows.py b/hermes_cli/gateway_windows.py index 48219f43e11b8..5ebccca3120f0 100644 --- a/hermes_cli/gateway_windows.py +++ b/hermes_cli/gateway_windows.py @@ -717,7 +717,7 @@ def _build_gateway_argv(home: Path | None = None) -> tuple[list[str], str, dict[ return _gateway_run_argv(python_exe, profile_arg), working_dir, env_overlay -def windowless_gateway_restart_spec(run_argv: list[str]) -> tuple[list[str], str, dict[str, str]]: +def windowless_gateway_restart_spec(run_argv: list[str], home: str | None = None) -> tuple[list[str], str, dict[str, str]]: """(argv, cwd, env overlay) for a hidden-console gateway respawn; arguments after the interpreter are preserved verbatim. Non-Windows or a non-python argv[0] → argv unchanged, empty overlay. @@ -728,6 +728,8 @@ def windowless_gateway_restart_spec(run_argv: list[str]) -> tuple[list[str], str every console-subsystem child allocated a visible conhost). This helper now only normalizes the interpreter via ``_resolve_detached_python`` and supplies the stable cwd + env overlay (HERMES_HOME, VIRTUAL_ENV, PYTHONPATH) so the respawn doesn't depend on the watcher's transient working directory. + ``HERMES_HOME`` is ``home`` when given: a replayed selectorless argv runs on whatever home its + environment names, so the paused runtime's home, not the updater's, must come back. """ if not run_argv or sys.platform != "win32": return run_argv, "", {} @@ -739,7 +741,7 @@ def windowless_gateway_restart_spec(run_argv: list[str]) -> tuple[list[str], str return run_argv, "", {} try: - hermes_home = str(_hermes_home().resolve()) + hermes_home = str(Path(home or _hermes_home()).resolve()) except Exception: hermes_home = "" env_overlay: dict[str, str] = {"PYTHONIOENCODING": "utf-8", "HERMES_GATEWAY_DETACHED": "1", "VIRTUAL_ENV": str(venv_dir)} diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 7bf08484b73e9..fd9c1c53a8bf4 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -948,15 +948,18 @@ def _pause_windows_gateways_for_update() -> dict | None: if not running_pids: return _cold_start_pause_token(adopted, claims) unmapped_pids = [pid for pid in running_pids if pid not in profile_processes and pid not in service_gateway_pids] + from gateway.status import get_process_start_time + from hermes_cli.dashboard_procs import _hermes_home_for_pid + born = {int(pid): get_process_start_time(int(pid)) for pid in running_pids} # the processes discovered # Snapshot unmapped gateways' argv *before* anything stops so resume (or a later launch) can replay it. - # Unmapped = no profile->PID-file mapping (e.g. Scheduled Task ``pythonw.exe -m ...``). + # Unmapped = no profile->PID-file mapping (e.g. Scheduled Task ``pythonw.exe -m ...``). Two homes can + # run identical selectorless argv, so the home and birth identify the runtime the replay must restore. unmapped = [ {"pid": int(pid), "argv": _try_call(lambda p=int(pid): _capture_gateway_argv(p), - "Could not capture argv for unmapped gateway %s: %s", int(pid))} + "Could not capture argv for unmapped gateway %s: %s", int(pid)), + "home": _hermes_home_for_pid(int(pid)), "ct": born[int(pid)]} for pid in unmapped_pids ] - from gateway.status import get_process_start_time - born = {int(pid): get_process_start_time(int(pid)) for pid in running_pids} # the processes discovered intended = { "resume_needed": True, "unmapped_pids": unmapped_pids, "unmapped": unmapped, "profiles": {str(profile_processes[pid].profile): int(pid) for pid in running_pids @@ -1354,7 +1357,9 @@ def _relaunch_paused_gateways(profiles: dict, unmapped: list) -> tuple[dict, lis launched_unmapped = [] for entry in unmapped: argv, old_pid = entry.get("argv"), entry.get("pid") - if argv and old_pid and _try_call(lambda o=old_pid, a=argv: launch_detached_gateway_restart_by_cmdline(int(o), list(a)), + # On the paused runtime's home (None on a pre-home record: the updater's own home, as before). + if argv and old_pid and _try_call(lambda o=old_pid, a=argv, h=entry.get("home"): launch_detached_gateway_restart_by_cmdline( + int(o), list(a), home=h), "Could not restart unmapped Windows gateway (pid %s) after update: %s", old_pid): launched_unmapped.append(entry) return launched, launched_unmapped @@ -1391,16 +1396,30 @@ def _relaunch_verify_timeout_s(profiles: dict, unmapped: list, pid_exists) -> fl def _unmapped_ready_filter(entry: dict, taken: set): - """``pid_filter`` matching the gateway an unmapped entry's argv replay started: same arguments - after the (normalized) interpreter, not the old pid, not one another entry already vouched for.""" + """``pid_filter`` matching the gateway an unmapped entry's replay started: a gateway running the + same arguments after the (normalized) interpreter on the entry's home, born after the process it + replaces, not one another entry already vouched for. Two homes can run identical selectorless argv, + so argv alone would let home B's gateway retire home A's debt (a service gateway on another home + included; on A's home only one gateway can run). An entry recorded before ``home`` was captured + keeps the argv-only match: nothing else identifies it.""" import psutil + from gateway.status import _looks_like_gateway_process, _same_hermes_home, get_process_start_time + from hermes_cli.dashboard_procs import _hermes_home_for_pid tail, old = list(entry.get("argv") or [])[1:], int(entry.get("pid") or 0) + home, old_ct = entry.get("home"), entry.get("ct") + + def same_runtime(pid: int) -> bool: + if not home: + return True + live_home, ct = _hermes_home_for_pid(pid), get_process_start_time(pid) + return live_home is not None and _same_hermes_home(live_home, home) and (old_ct is None or ct is None or ct > old_ct) def matches(pid: int) -> bool: if pid == old or pid in taken: return False try: - return list(psutil.Process(pid).cmdline() or [])[1:] == tail + return (list(psutil.Process(pid).cmdline() or [])[1:] == tail and _looks_like_gateway_process(pid) + and same_runtime(pid)) except (psutil.Error, OSError, ValueError): # exited, access denied, zombie, invalid pid return False return lambda pids: [pid for pid in _owned_gateway_pids(pids) if matches(int(pid))] diff --git a/tests/hermes_cli/test_update_pause_unmapped_identity.py b/tests/hermes_cli/test_update_pause_unmapped_identity.py new file mode 100644 index 0000000000000..43627b97faa84 --- /dev/null +++ b/tests/hermes_cli/test_update_pause_unmapped_identity.py @@ -0,0 +1,74 @@ +"""An unmapped gateway's restart debt is retired only by ITS runtime (#132338 review C). + +The pause records an unmapped gateway (no profile PID file, e.g. a Scheduled Task) by the argv it +replays. Two homes can run identical selectorless argv, their environment picking the home, so the +readiness predicate must also match the home the paused runtime ran on: otherwise home B's gateway, +already back after a partial recovery, retires home A's debt while A stays down. + +Real processes (argv shaped like ``hermes gateway run``, each under its own ``HERMES_HOME``) and the +real predicate; the fleet discovery that hands it candidate PIDs is the only thing bypassed. The +replay restores that home too: the respawn's ``HERMES_HOME`` is the paused runtime's, not the updater's. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time + +import pytest + +from gateway.status import get_process_start_time +from hermes_cli.update_cmd_windows import _unmapped_ready_filter + + +@pytest.fixture +def spawn(tmp_path): + entry = tmp_path / "bin" / "hermes" # a ``hermes`` entry token: the cmdline reads as a gateway + entry.parent.mkdir() + entry.write_text("import time\ntime.sleep(60)\n", encoding="utf-8") + argv = [sys.executable, str(entry), "gateway", "run"] + procs: list[subprocess.Popen] = [] + + def _spawn(home) -> subprocess.Popen: + proc = subprocess.Popen(argv, env={**os.environ, "HERMES_HOME": str(home)}) + procs.append(proc) # Popen returns after exec: the gateway argv and environment are in place + return proc + + yield _spawn, argv + for proc in procs: + proc.kill() + proc.wait(timeout=10) + + +@pytest.mark.platforms("linux") +@pytest.mark.spawns_gateway_lookalike # stub children that sleep; the fixture reaps them +def test_an_equal_argv_gateway_of_another_home_never_retires_the_unmapped_debt(spawn): + spawn, argv = spawn + root = os.environ["HERMES_HOME"] # the sandboxed home this update owns, with two profiles + home_a, home_b = (os.path.join(root, "profiles", name) for name in ("a", "b")) + os.makedirs(home_a) + os.makedirs(home_b) + old = spawn(home_a) + entry = {"pid": old.pid, "argv": argv, "home": home_a, "ct": get_process_start_time(old.pid)} + old.kill() + old.wait(timeout=10) + time.sleep(0.05) # a later clock tick: the replacement is born after the process it replaces + + sibling = spawn(home_b) + assert _unmapped_ready_filter(entry, set())([sibling.pid]) == [], \ + "home B's gateway with A's argv retired A's restart debt" + replacement = spawn(home_a) + assert _unmapped_ready_filter(entry, set())([sibling.pid, replacement.pid]) == [replacement.pid] + + +@pytest.mark.platforms("windows") # the spec overlays the environment only on Windows +def test_an_unmapped_replay_runs_on_the_home_its_runtime_ran_on(tmp_path): + from hermes_cli.gateway_windows import windowless_gateway_restart_spec + + paused_home = tmp_path / "profiles" / "a" + paused_home.mkdir(parents=True) + _argv, _cwd, env = windowless_gateway_restart_spec( + [sys.executable, "-m", "hermes_cli.main", "gateway", "run"], home=str(paused_home)) + assert env["HERMES_HOME"] == str(paused_home.resolve()), "the replay ran on the updater's home" From 4f1f4a55e8a3a7b15e0736b3018ee1d40fb9f9a4 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:29:14 -0700 Subject: [PATCH 43/63] fix: an unreadable saved pause or claim is unknown, never absent read() returned None for ENOENT, a refused read and malformed bytes alike, so the carrier survey dropped an unreadable newer claim (its older alpha+beta copy then executed again, replaying completed alpha) and write() replaced an unreadable orphaned record (losing its restart obligation). Only a missing file is absent now; a refused or malformed one raises RecordUnknown (an OSError), so selection, claiming and the new pause's write fail closed and leave every file for a later, readable retry. --- hermes_cli/update_pause_record.py | 32 +++++++--- tests/hermes_cli/test_update_pause_record.py | 62 ++++++++++++++++++++ 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index c0799e7b13d32..7723837aad9cd 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -106,6 +106,10 @@ class RetiredUnknown(OSError): """The retired-id list exists but cannot be read: which obligations are complete is unknown.""" +class RecordUnknown(OSError): + """A saved pause or claim exists but cannot be read or parsed: what it still owes is unknown.""" + + @contextmanager def _mutex(wait_s: float = _MUTEX_WAIT_S): """Exclusive kernel lock on the record directory's sidecar (A7). Re-entrant per thread; each @@ -326,12 +330,24 @@ def _atomic_write(path: Path, body: dict) -> None: def read(path: Path | None = None) -> dict | None: + """The saved body; ``None`` only when the file is absent. One that exists but cannot be read + (a Windows sharing violation, an AV scanner) or does not parse raises :class:`RecordUnknown`: + it may be the newest copy of an obligation (its partial progress) or the only one, so it never + lets an older copy execute nor a new pause replace it. Callers fail closed and retry later.""" path = path or record_path() try: - body = json.loads(path.read_text(encoding="utf-8-sig")) - except (OSError, ValueError): + text = path.read_text(encoding="utf-8-sig") + except FileNotFoundError: return None - return body if isinstance(body, dict) and isinstance(body.get("token"), dict) else None + except OSError as exc: + raise RecordUnknown(f"cannot read the paused-gateway record {path}: {exc}") from exc + try: + body = json.loads(text) + except ValueError: + body = None + if not isinstance(body, dict) or not isinstance(body.get("token"), dict): + raise RecordUnknown(f"the paused-gateway record {path} is malformed; fix or remove it to resume") + return body UNOWNED = {"pid": 0, "ct": None} @@ -494,7 +510,7 @@ def _holder(src: Path, body: dict) -> dict: def _files(path: Path) -> list[tuple[Path, dict]]: - """This checkout's record and claims, readable ones only.""" + """This checkout's record and claims; one that cannot be read raises :class:`RecordUnknown`.""" found = [] for src in (path, *_claims(path)): body = read(src) @@ -568,12 +584,10 @@ def retire_redundant(path: Path | None = None) -> None: def _prune_retired(path: Path) -> None: - """Forget retired ids no file carries any more — only when every candidate file was readable - (one Windows would not let us read may still carry a retired id).""" + """Forget retired ids no file carries any more (a carrier that cannot be read raises first).""" retired = _retired(path) - candidates = [p for p in (path, *_claims(path)) if p.exists()] - bodies = [read(p) for p in candidates] - if not retired or None in bodies: + bodies = [body for body in map(read, (path, *_claims(path))) if body is not None] + if not retired: return carried = {str(b["token"].get("pause_id")) for b in bodies} keep = retired & carried diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index b02e8fdb0df26..9c762b3cf133e 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -599,6 +599,68 @@ def owed() -> list[str]: +# --- Review A: a carrier that cannot be read is unknown, never absent --------------------------- +def _unreadable(path: Path, how: str) -> None: + if how == "malformed": + path.write_text("{trunc", encoding="utf-8") + else: + path.chmod(0) # stands in for a Windows read refusal (sharing violation, AV scanner) + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root reads a mode-000 file") +@pytest.mark.parametrize("how", ["refused", "malformed"]) +def test_an_unreadable_newer_carrier_never_lets_its_older_copy_execute(tmp_path, monkeypatch, how): + """Partial progress left the newer claim owing only beta beside an older alpha+beta copy of the + same obligation (its unlink was refused). That claim becoming unreadable must not resurrect alpha.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + src = pause_record.record_path() + pause_id = "d" * 32 + pause_record.write({"pause_id": pause_id, "resume_needed": True, "profiles": {"alpha": 11, "beta": 12}}, + owner=pause_record.UNOWNED) + newer = src.with_name(f"{src.name}.999.deadbeef.claim") + pause_record._atomic_write(newer, {"install_root": str(REPO), "claimer": pause_record.UNOWNED, "rev": 1, + "token": {"pause_id": pause_id, "resume_needed": True, "profiles": {"beta": 12}}}) + + def owed() -> list[dict]: + try: + return [body["token"]["profiles"] for _src, body in pause_record.orphans()] + except OSError: # unknown: recovery reports it and claims nothing + return [] + + assert owed() == [{"beta": 12}], "premise: the furthest-progressed copy executes" + _unreadable(newer, how) + try: + assert {"alpha": 11, "beta": 12} not in owed(), "completed alpha would be restarted again" + assert pause_record.claim(src) is None, "the older copy was claimed past an unreadable newer one" + finally: + newer.chmod(0o644) + assert src.exists() and newer.exists(), "a carrier was deleted while its state was unknown" + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root reads a mode-000 file") +@pytest.mark.parametrize("how", ["refused", "malformed"]) +def test_an_unreadable_orphan_is_never_overwritten_by_a_new_pause(tmp_path, monkeypatch, how): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + src = pause_record.record_path() + pause_record.write({"pause_id": "e" * 32, "resume_needed": True, "profiles": {"alpha": 11}}, owner=pause_record.UNOWNED) + saved = src.read_bytes() + _unreadable(src, how) + try: + with pytest.raises(OSError): + pause_record.write({"pause_id": "f" * 32, "resume_needed": True, "profiles": {"beta": 12}}, + owner=pause_record.identity()) + finally: + src.chmod(0o644) + if how == "refused": + assert src.read_bytes() == saved, "alpha's saved restart obligation was replaced" + else: + assert src.read_text(encoding="utf-8") == "{trunc", "an unknown record was replaced" + src.write_bytes(saved) # readable control: the new pause folds the orphan in + token = {"pause_id": "f" * 32, "resume_needed": True, "profiles": {"beta": 12}} + pause_record.write(token, owner=pause_record.identity()) + assert pause_record.read()["token"]["profiles"] == {"alpha": 11, "beta": 12} + + # --- Review W3: record hygiene -------------------------------------------------------------------- def test_a_refused_publish_leaves_no_temp_beside_the_record(tmp_path, monkeypatch): target = tmp_path / "records" / "record.json" From 076ebbb7f137a6e7a22eb08f5b389be2d05534c9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:34:55 -0700 Subject: [PATCH 44/63] fix: an accepted planned stop stays owed for the whole drain When both the producer's mark_stop_sent and the consumer's checkpoint into the pause record were refused (busy mutex, Windows replace refusal), the consumer still accepted the stop and left only the 60-second request marker. An updater that died meanwhile left recovery to drop the still-draining gateway as never asked once that TTL passed, losing its restart. The consumer's fallback now writes a receipt beside the marker (the gateway's own home, not the record) naming the accepting incarnation; recovery treats it as stop evidence without a TTL. Unconsumed requests still expire, and a user's stop (another stopper) is still no evidence. --- gateway/status.py | 8 ++--- hermes_cli/update_pause_record.py | 31 ++++++++++++++--- tests/hermes_cli/test_update_pause_record.py | 35 ++++++++++++++++++++ 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/gateway/status.py b/gateway/status.py index 3fe98e8b662f4..ed4e6ab505b8b 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -1970,11 +1970,11 @@ def consume_planned_stop_marker_for_self() -> bool: ) except OSError as exc: # The pause bookkeeping (a busy mutex, a checkpoint that cannot be written) never decides - # whether this stop was planned: classify without consuming, so the request stays on - # disk as the evidence recovery reads in place of the checkpoint (it expires by TTL). + # whether this stop was planned: classify without consuming, and leave a receipt beside the + # request so recovery keeps this drain owed after the request's TTL (mark_stop_accepted). logger.warning("Planned-stop checkpoint skipped (%s); the stop request stays on disk", exc) - return _consume_pid_marker_for_self( - _get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S, keep=True) + return _consume_pid_marker_for_self(_get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S, + keep=True, on_consume=update_pause_record.mark_stop_accepted) def planned_stop_marker_targets_self() -> bool: diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 7723837aad9cd..aede61c822da2 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -413,6 +413,20 @@ def mark_stop_consumed(path: Path, marker: dict) -> None: _atomic_write(src, body) # Failure leaves the request intact for recovery. +def _accepted_path(marker_path: Path) -> Path: + return marker_path.with_name(marker_path.name + ".accepted") + + +def mark_stop_accepted(path: Path, marker: dict) -> None: + """The consumer's receipt when it cannot checkpoint into the record (busy mutex, refused + replace): this incarnation accepted the request and drains until it exits, however long past + the request's TTL. Written beside the marker, in the gateway's own home, never into the record.""" + try: + _atomic_write(_accepted_path(path), dict(marker)) + except OSError as exc: + print(f" ⚠ Could not record the accepted stop request {path}: {exc}", file=sys.stderr) + + def discharge(token: dict, path: Path | None = None) -> None: """Remove the record only while it still names this pause (decided under the mutex).""" path = path or record_path() @@ -694,16 +708,25 @@ def _without(token: dict, pids: set[str]) -> dict: def _request_on_disk(token: dict, pid: str) -> bool: """The planned-stop marker this update's stopper wrote for *pid* is still on disk (the gateway's watcher has not consumed it yet): the request was issued even if the updater died before - :func:`mark_stop_sent`. A marker naming another stopper (a user's ``hermes gateway stop``) is not.""" + :func:`mark_stop_sent` — or the gateway's receipt of accepting it (:func:`mark_stop_accepted`). + A marker naming another stopper (a user's ``hermes gateway stop``) is not.""" path = (token.get("stop_markers") or {}).get(str(pid)) if not path: return False - from gateway.status import _marker_is_stale, _PLANNED_STOP_MARKER_TTL_S, get_process_start_time + from gateway.status import _PLANNED_STOP_MARKER_TTL_S + # An unconsumed request expires; the receipt of an accepted one is evidence for the whole drain + # (only the accepting incarnation is ever judged: a later one is not this entry's live process). + return (_names_request(token, pid, Path(path), _PLANNED_STOP_MARKER_TTL_S) + or _names_request(token, pid, _accepted_path(Path(path)), None)) + + +def _names_request(token: dict, pid: str, path: Path, ttl_s: int | None) -> bool: + from gateway.status import _marker_is_stale, get_process_start_time try: - marker = json.loads(Path(path).read_text(encoding="utf-8-sig")) + marker = json.loads(path.read_text(encoding="utf-8-sig")) if (int(marker["target_pid"]) != int(pid) or int(marker["stopper_pid"]) != int(token["stopper_pid"]) - or _marker_is_stale(marker.get("written_at") or "", _PLANNED_STOP_MARKER_TTL_S)): + or (ttl_s is not None and _marker_is_stale(marker.get("written_at") or "", ttl_s))): return False expected, actual = marker.get("target_start_time"), get_process_start_time(int(pid)) # Match the consumer's optional birth fingerprint, including unavailable clocks. diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 9c762b3cf133e..ed0c499a0de29 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -661,6 +661,41 @@ def test_an_unreadable_orphan_is_never_overwritten_by_a_new_pause(tmp_path, monk assert pause_record.read()["token"]["profiles"] == {"alpha": 11, "beta": 12} +# --- Review B: an accepted stop stays owed for the whole drain ----------------------------------- +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root writes a read-only directory") +@pytest.mark.parametrize("consumed", [True, False]) +def test_an_accepted_stop_the_record_could_not_checkpoint_outlives_the_request_ttl(tmp_path, monkeypatch, consumed): + """Producer and consumer checkpoints both refused (a read-only record directory stands in for + a Windows replace refusal), the updater gone, the gateway (this process) still draining past the + request's TTL: recovery keeps it owed. A request nobody consumed still expires (control).""" + from datetime import datetime, timedelta, timezone + + from gateway import status + root, home = tmp_path / "root", tmp_path / "root" / "profiles" / "p" + home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(home)) + pid = os.getpid() + marker = status._get_planned_stop_marker_path() + token = pause_record.record_pause({"resume_needed": True, "profiles": {"p": pid}, + "identities": {str(pid): pause_record.identity(pid)["ct"]}}, None, []) + pause_record.mark_stop_requested(token, [pid], markers={pid: marker}) + assert status.write_planned_stop_marker(pid) + root.chmod(0o555) + try: + pause_record.mark_stop_sent(token, pid) # refused: best effort + if consumed: + assert status.consume_planned_stop_marker_for_self() is True + finally: + root.chmod(0o755) + saved = pause_record.read()["token"] + assert saved["stop_sent"] == [], "premise: no checkpoint landed" + body = json.loads(marker.read_text(encoding="utf-8")) + body["written_at"] = (datetime.now(timezone.utc) - timedelta(seconds=120)).isoformat() # past the TTL + marker.write_text(json.dumps(body), encoding="utf-8") + owed = pause_record.drop_never_stopped(dict(saved))["profiles"] + assert owed == ({"p": pid} if consumed else {}), "a gateway draining an accepted stop lost its restart debt" + + # --- Review W3: record hygiene -------------------------------------------------------------------- def test_a_refused_publish_leaves_no_temp_beside_the_record(tmp_path, monkeypatch): target = tmp_path / "records" / "record.json" From 5deafd8fad790e1a7d7c0614af0dfc60bc4a288c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:36:56 -0700 Subject: [PATCH 45/63] fix: same-HEAD gateway admission is judged on the move the update recorded The unmoved-HEAD tree gate decided which paths the failed move could have written from the CURRENT FETCH_HEAD and remote refs, so a later fetch whose target no longer touched a half-written file turned the gate from hold to resume on unchanged partial bytes. And a path dirty at pause was exempt by name alone, so an autostashed edit let different half-written bytes through. _pull_updates now records the resolved move target on the pause's own baseline before git writes (mark_move; a failure aborts the move), and the gate diffs HEAD against that recorded target. The pause also stores each dirty path's digest, so only the same bytes count as "seen at pause". A pause with no recorded move, or a record from before these fields, keeps the fetched-refs and pathname behaviour. --- hermes_cli/update_cmd.py | 3 + hermes_cli/update_pause_record.py | 73 +++++++++++++++----- tests/hermes_cli/test_update_pause_record.py | 53 ++++++++++++++ 3 files changed, 111 insertions(+), 18 deletions(-) diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index ffffa799d1a0d..8c8c68ebe0685 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -29,6 +29,7 @@ from hermes_cli import update_receipt as _completion_receipt, update_cmd_config as _completion_config from hermes_cli._old_updater import stop_for_relaunch from hermes_cli._early_recovery import git_operation_in_progress, interrupted_pull_marker +from hermes_cli import update_pause_record as _pause_record from hermes_cli import update_cmd_check as _check # Re-exports: every split-module name stays reachable (and monkeypatchable) as update_cmd.. @@ -985,6 +986,8 @@ def _pull_updates( encoding="utf-8") try: try: + # The paused gateways' tree gate must know this move's target before git writes a file. + _pause_record.mark_move(_windows_gateway_resume, target_sha) # merge --ff-only the already-fetched ref instead of `git pull`, which would do a # SECOND network fetch; identical in effect given the fresh tracking ref. if merge_ref != f"origin/{branch}": diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index aede61c822da2..6d9deb6758467 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -210,18 +210,51 @@ def stamp_tree(token: dict, root: Path | None = None) -> dict: pause_id = token.setdefault("pause_id", uuid.uuid4().hex) baselines = token.setdefault("baselines", []) if (root / ".git").exists() and not any(b.get("pause_id") == pause_id for b in baselines): - baselines.append({"pause_id": pause_id, "pre_sha": head_sha(root), "dirty_at_pause": tracked_changes(root)}) + dirty = tracked_changes(root) + # The bytes, not just the names: an autostashed edit and git's half-written bytes share a path. + baselines.append({"pause_id": pause_id, "pre_sha": head_sha(root), "dirty_at_pause": dirty, + "dirty_digests": {path: _digest(root / path) for path in dirty or []}}) return token +def _digest(path: Path) -> str | None: + try: + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: # deleted (or unreadable) at pause: it must read the same way to count as unchanged + return None + + +def mark_move(token: dict | None, target: str) -> None: + """Persist, BEFORE git writes the tree, the commit this run's checkout move goes to, on this + pause's own baseline: an unmoved HEAD is later judged against what THAT move could write, never + against refs a later fetch replaced. Raises when it cannot be recorded: the move must not run.""" + if not token or not token.get("pause_id") or not target: + return + for baseline in token.get("baselines") or []: + if baseline.get("pause_id") == token["pause_id"]: + baseline["move_targets"] = sorted({*baseline.get("move_targets", []), target}) + write({**token, "resume_needed": True}) + + _MAX_MOVE_TARGETS = 16 +def _paths_between(root: Path, targets) -> set[str] | None: + """Tracked paths that differ between HEAD and any of *targets* — all a fast-forward, reset or + merge toward them can write; ``None`` when git cannot say.""" + paths: set[str] = set() + for target in sorted(targets): + diff = _git(root, "diff", "--name-only", "-z", "--no-renames", "HEAD", target, "--") + if diff is None or diff.returncode != 0: + return None + paths.update(filter(None, diff.stdout.split("\0"))) + return paths + + def _paths_a_move_could_write(root: Path) -> set[str] | None: - """Tracked paths an update's checkout move could have written while HEAD stayed put: the ones - that differ between HEAD and a commit the checkout fetched (``FETCH_HEAD``) or tracks - (``refs/remotes``) — a fast-forward, reset or merge only writes those. ``None`` when git cannot - say or nothing was fetched (the move's target is then unknown).""" + """For a pause with no recorded move (:func:`mark_move`: none attempted, or a record from before + it was kept): the paths between HEAD and a commit the checkout fetched (``FETCH_HEAD``) or tracks + (``refs/remotes``). ``None`` when git cannot say or nothing was fetched (target unknown).""" where = _git(root, "rev-parse", "--git-path", "FETCH_HEAD") refs = _git(root, "for-each-ref", "--format=%(objectname)", "refs/remotes") if where is None or where.returncode != 0 or refs is None or refs.returncode != 0: @@ -235,13 +268,16 @@ def _paths_a_move_could_write(root: Path) -> set[str] | None: targets = {line.split()[0] for line in fetched if line.split()} | set(refs.stdout.split()) if not targets or len(targets) > _MAX_MOVE_TARGETS: return None - paths: set[str] = set() - for target in sorted(targets): - diff = _git(root, "diff", "--name-only", "-z", "--no-renames", "HEAD", target, "--") - if diff is None or diff.returncode != 0: - return None - paths.update(filter(None, diff.stdout.split("\0"))) - return paths + return _paths_between(root, targets) + + +def _seen_at_pause(root: Path, baseline: dict, path: str) -> bool: + """*path* is as the pause saw it: dirty then, with the same bytes now (a baseline from before + digests were kept has only the name).""" + if path not in (baseline.get("dirty_at_pause") or []): + return False + digests = baseline.get("dirty_digests") + return digests is None or digests.get(path) == _digest(root / path) def _half_written(root: Path, changes: list[str], at_head: list[dict]) -> list[str]: @@ -249,13 +285,14 @@ def _half_written(root: Path, changes: list[str], at_head: list[dict]) -> list[s baseline that has any. A change the pause did not see counts only on a path the update's move could write: a build/sync step (or the user) rewriting any other tracked file is not git's doing, and holding the set for it would keep gateways stopped until someone cleans the file. - When the move's target is unknown every unseen change still counts.""" - writable: set[str] | None = None - probed = False + The move is the one recorded before it ran, so a later fetch never narrows it; with none + recorded the fetched refs stand in, and when even those are unknown every unseen change counts.""" for baseline in at_head: - unexpected = sorted(set(changes) - set(baseline.get("dirty_at_pause") or [])) - if unexpected and not probed: - writable, probed = _paths_a_move_could_write(root), True + unexpected = sorted(path for path in changes if not _seen_at_pause(root, baseline, path)) + if not unexpected: + continue + targets = baseline.get("move_targets") + writable = _paths_between(root, targets) if targets else _paths_a_move_could_write(root) torn = unexpected if writable is None else [path for path in unexpected if path in writable] if torn: return torn diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index ed0c499a0de29..9d9795a49f8f1 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -840,3 +840,56 @@ def test_an_unmoved_head_holds_gateways_only_for_paths_the_updates_move_could_wr token = pause_record.stamp_tree({"resume_needed": True}, root) (root / "b.lock").write_text("rewritten by the dependency sync\n", encoding="utf-8") assert pause_record.tree_is_whole(token, root) == (True, "") + + +def _cloned_checkout(tmp_path: Path) -> tuple[Path, Path]: + origin, root = tmp_path / "origin", tmp_path / "checkout" + origin.mkdir() + _git(origin, "init", "-q") + for name in ("a.py", "b.py"): + (origin / name).write_text("v1\n", encoding="utf-8") + _git(origin, "add", ".") + _git(origin, "commit", "-qm", "X") + _git(tmp_path, "clone", "-q", str(origin), str(root)) + (origin / "a.py").write_text("v2\n", encoding="utf-8") + _git(origin, "commit", "-qam", "B") # the update's target changes a.py + return origin, root + + +def test_a_later_fetch_never_certifies_bytes_an_earlier_recorded_move_left(tmp_path, monkeypatch): + """Review D: the move's target goes on the record before git writes; a later fetch whose refs + no longer touch a.py must not turn the gate from false to true on the same partial bytes.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home")) + origin, root = _cloned_checkout(tmp_path) + token = pause_record.stamp_tree({"resume_needed": True, "profiles": {"default": 4242}}, root) + _git(root, "fetch", "-q", "origin") + pause_record.mark_move(token, _git(root, "rev-parse", "origin/HEAD")) + (root / "a.py").write_text("v2 half\n", encoding="utf-8") # git wrote a.py, died before HEAD + assert not pause_record.tree_is_whole(token, root)[0], "premise: the torn path holds the set" + + (origin / "a.py").write_text("v1\n", encoding="utf-8") + (origin / "b.py").write_text("v2\n", encoding="utf-8") + _git(origin, "commit", "-qam", "C") # a.py back to X's bytes: no fetched ref touches it now + _git(root, "fetch", "-q", "origin") + saved = pause_record.read()["token"] + for judged in (token, saved): + whole, why = pause_record.tree_is_whole(judged, root) + assert not whole and "a.py" in why, "a later fetch certified the earlier move's partial bytes" + + +def test_an_autostashed_edit_vouches_only_for_its_own_bytes(tmp_path, monkeypatch): + """Review D / F3: a.py was dirty at pause, the update stashed it and its move wrote a.py partly. + The pathname alone no longer admits the different bytes; the restored edit itself still passes.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home")) + _origin, root = _cloned_checkout(tmp_path) + (root / "a.py").write_text("user edit\n", encoding="utf-8") + token = pause_record.stamp_tree({"resume_needed": True, "profiles": {"default": 4242}}, root) + _git(root, "stash", "-q") + _git(root, "fetch", "-q", "origin") + pause_record.mark_move(token, _git(root, "rev-parse", "origin/HEAD")) + (root / "a.py").write_text("v2 half\n", encoding="utf-8") + whole, why = pause_record.tree_is_whole(token, root) + assert not whole and "a.py" in why, "the dirty pathname admitted git's half-written bytes" + _git(root, "checkout", "-q", "--", "a.py") + _git(root, "stash", "pop", "-q") # the user's own edit back, byte for byte + assert pause_record.tree_is_whole(token, root) == (True, "") From 277b8d0733c550461d343c55889aec9734386bf6 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:29:32 -0700 Subject: [PATCH 46/63] fix: unreadable or unwritable accepted-stop evidence keeps a draining gateway owed The pause record's second evidence reader (_names_request) turned a refused or malformed planned-stop marker / accepted receipt into False, i.e. "this update never asked it to stop". Past the request's TTL recovery then dropped the live, still-draining gateway (drop_never_stopped) and durably retired its debt. Now only an absent file or a valid one naming something else is "not asked"; refused/malformed is unknown: the entry stays owed (held as draining while its process lives), and unknown is never resolved into stop_sent, so the next pass judges it again. The consumer's receipt (mark_stop_accepted) swallowed its own write failure, so an accepted stop could leave no durable trace at all. When the home refuses the new receipt file it now stamps the kept request in place ("accepted", opened r+, never created); a torn stamp reads malformed, hence unknown, hence owed. An expired request nobody accepted still expires. --- hermes_cli/update_pause_record.py | 65 +++++++---- .../test_update_pause_stop_evidence.py | 103 ++++++++++++++++++ 2 files changed, 148 insertions(+), 20 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_stop_evidence.py diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 6d9deb6758467..e7fa2e1db4484 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -457,11 +457,24 @@ def _accepted_path(marker_path: Path) -> Path: def mark_stop_accepted(path: Path, marker: dict) -> None: """The consumer's receipt when it cannot checkpoint into the record (busy mutex, refused replace): this incarnation accepted the request and drains until it exits, however long past - the request's TTL. Written beside the marker, in the gateway's own home, never into the record.""" + the request's TTL. Written beside the marker, in the gateway's own home, never into the record. + + A home that refuses the new file (an ACL, a read-only directory) still lets the kept request be + rewritten in place: stamped ``accepted`` it is the receipt. Torn mid-write it reads malformed, + which recovery holds as unknown, never as "not asked". The consumer accepts the stop either way.""" try: _atomic_write(_accepted_path(path), dict(marker)) + return + except OSError as exc: + refused = exc + try: + with open(path, "r+", encoding="utf-8") as fh: # r+: never creates a request that was not there + json.dump({**marker, "accepted": True}, fh) + fh.truncate() + fh.flush() + os.fsync(fh.fileno()) except OSError as exc: - print(f" ⚠ Could not record the accepted stop request {path}: {exc}", file=sys.stderr) + print(f" ⚠ Could not record the accepted stop request {path}: {refused}; {exc}", file=sys.stderr) def discharge(token: dict, path: Path | None = None) -> None: @@ -742,40 +755,50 @@ def _without(token: dict, pids: set[str]) -> dict: return token -def _request_on_disk(token: dict, pid: str) -> bool: +def _request_on_disk(token: dict, pid: str) -> bool | None: """The planned-stop marker this update's stopper wrote for *pid* is still on disk (the gateway's watcher has not consumed it yet): the request was issued even if the updater died before :func:`mark_stop_sent` — or the gateway's receipt of accepting it (:func:`mark_stop_accepted`). - A marker naming another stopper (a user's ``hermes gateway stop``) is not.""" + A marker naming another stopper (a user's ``hermes gateway stop``) is not. ``None``: one of them + exists but cannot be read or parsed — the stop may have been accepted, so it is unknown.""" path = (token.get("stop_markers") or {}).get(str(pid)) if not path: return False from gateway.status import _PLANNED_STOP_MARKER_TTL_S # An unconsumed request expires; the receipt of an accepted one is evidence for the whole drain # (only the accepting incarnation is ever judged: a later one is not this entry's live process). - return (_names_request(token, pid, Path(path), _PLANNED_STOP_MARKER_TTL_S) - or _names_request(token, pid, _accepted_path(Path(path)), None)) + verdicts = (_names_request(token, pid, Path(path), _PLANNED_STOP_MARKER_TTL_S), + _names_request(token, pid, _accepted_path(Path(path)), None)) + return True if True in verdicts else None if None in verdicts else False -def _names_request(token: dict, pid: str, path: Path, ttl_s: int | None) -> bool: +def _names_request(token: dict, pid: str, path: Path, ttl_s: int | None) -> bool | None: + """Does *path* hold this stopper's request for *pid*'s incarnation? Only an absent file or a + valid one naming something else is ``False``; refused (a sharing violation, an AV scanner) or + malformed is ``None``, unknown.""" from gateway.status import _marker_is_stale, get_process_start_time try: marker = json.loads(path.read_text(encoding="utf-8-sig")) - if (int(marker["target_pid"]) != int(pid) - or int(marker["stopper_pid"]) != int(token["stopper_pid"]) - or (ttl_s is not None and _marker_is_stale(marker.get("written_at") or "", ttl_s))): - return False - expected, actual = marker.get("target_start_time"), get_process_start_time(int(pid)) - # Match the consumer's optional birth fingerprint, including unavailable clocks. - return None in (expected, actual) or expected == actual + target, stopper = int(marker["target_pid"]), int(marker["stopper_pid"]) + except FileNotFoundError: + return False except (OSError, ValueError, TypeError, KeyError): + return None + if (target != int(pid) or stopper != int(token.get("stopper_pid") or 0) + or (ttl_s is not None and not marker.get("accepted") + and _marker_is_stale(marker.get("written_at") or "", ttl_s))): return False + expected, actual = marker.get("target_start_time"), get_process_start_time(int(pid)) + # Match the consumer's optional birth fingerprint, including unavailable clocks. + return None in (expected, actual) or expected == actual -def _asked(token: dict) -> set[str]: - """Pids this update actually asked to stop: recorded as sent, or whose request is still on disk.""" +def _asked(token: dict) -> tuple[set[str], set[str]]: + """``(asked, unknown)``: pids this update actually asked to stop (recorded as sent, or whose + request is still on disk), and pids whose request evidence exists but cannot be read.""" sent = {str(p) for p in token.get("stop_sent") or []} - return sent | {str(p) for p in token.get("stop_requested") or [] if str(p) not in sent and _request_on_disk(token, str(p))} + verdicts = {str(p): _request_on_disk(token, str(p)) for p in token.get("stop_requested") or [] if str(p) not in sent} + return sent | {p for p, v in verdicts.items() if v}, {p for p, v in verdicts.items() if v is None} def drop_never_stopped(token: dict) -> dict: @@ -784,12 +807,14 @@ def drop_never_stopped(token: dict) -> dict: serving, and a later exit — a user's ``hermes gateway stop`` included — is not this update's to undo. One it asked may be draining: it keeps its restart debt (:func:`split_draining`). The evidence is resolved here, once, into ``stop_sent`` (a merge carries it under another stopper). - No stop record at all (a set from before stop tracking): nothing can be told apart.""" + Evidence that cannot be read keeps the entry too (it may be draining) but resolves nothing: it is + judged again next time. No stop record at all (a set from before stop tracking): nothing can be + told apart.""" if token.get("stop_requested") is None: return token - asked = _asked(token) + asked, unknown = _asked(token) token["stop_sent"] = sorted(asked) - return _without(token, _live_pids(token) - asked) + return _without(token, _live_pids(token) - asked - unknown) def split_draining(token: dict) -> dict: diff --git a/tests/hermes_cli/test_update_pause_stop_evidence.py b/tests/hermes_cli/test_update_pause_stop_evidence.py new file mode 100644 index 0000000000000..d28497c53c8a9 --- /dev/null +++ b/tests/hermes_cli/test_update_pause_stop_evidence.py @@ -0,0 +1,103 @@ +"""Accepted-stop evidence the pause record cannot read or could not write: the debt stays owed. + +The gateway is this test process (a live incarnation that accepted the stop and is draining); the +updater is gone (the record is unowned); every file is real and every refusal is a real one (mode +000, a read-only directory), standing in for a Windows sharing violation or ACL refusal. +""" + +from __future__ import annotations + +import json +import os +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from hermes_cli import update_pause_record as pause_record + +pytestmark = [ + pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes stand in for Windows refusals"), + pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores file modes"), +] + + +def _accepted_then_orphaned(tmp_path: Path, monkeypatch, *, home_writable: bool) -> tuple[Path, str]: + """Record a pause of this process, request its stop, let it accept the request while the record + refuses the consumer's checkpoint (read-only record directory), then age the request past its + TTL and orphan the record. Returns ``(marker, pause_id)``.""" + from gateway import status + root, home = tmp_path / "root", tmp_path / "root" / "profiles" / "p" + home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(home)) + pid = os.getpid() + marker = status._get_planned_stop_marker_path() + token = pause_record.record_pause({"resume_needed": True, "profiles": {"p": pid}, + "identities": {str(pid): pause_record.identity(pid)["ct"]}}, None, []) + pause_record.mark_stop_requested(token, [pid], markers={pid: marker}) + assert status.write_planned_stop_marker(pid) + root.chmod(0o555) + if not home_writable: + home.chmod(0o555) + try: + assert status.consume_planned_stop_marker_for_self() is True, "premise: the stop was accepted" + finally: + home.chmod(0o755) + root.chmod(0o755) + saved = pause_record.read()["token"] + assert saved["stop_sent"] == [], "premise: no checkpoint landed in the record" + body = json.loads(marker.read_text(encoding="utf-8")) + body["written_at"] = (datetime.now(timezone.utc) - timedelta(seconds=120)).isoformat() # past the TTL + marker.write_text(json.dumps(body), encoding="utf-8") + pause_record.write(saved, owner=pause_record.UNOWNED) # the updater died + return marker, saved["pause_id"] + + +def _owed_after_recovery(pause_id: str) -> dict | None: + """Run a launch's recovery; the profiles still owed by the obligation, ``None`` once retired.""" + pause_record.recover(["status"]) + path = pause_record.record_path() + retired = path.with_suffix(".retired") + ids = json.loads(retired.read_text(encoding="utf-8"))["ids"] if retired.exists() else [] + carriers = [pause_record.read(src) for src in (path, *pause_record._claims(path)) if src.exists()] + owed = [body["token"]["profiles"] for body in carriers if body["token"]["pause_id"] == pause_id] + assert bool(owed) != (pause_id in ids), "an obligation is either owed or retired, never both or neither" + return owed[0] if owed else None + + +@pytest.mark.parametrize("receipt", ["refused", "malformed", "missing"]) +def test_an_unreadable_accepted_stop_receipt_keeps_the_draining_gateway_owed(tmp_path, monkeypatch, receipt): + """R1: a receipt that exists but cannot be read or parsed is unknown evidence: the live draining + gateway stays owed (never restarted over, never retired). A receipt that is genuinely absent + beside an expired, unconsumed request is the control: that stop was never accepted.""" + marker, pause_id = _accepted_then_orphaned(tmp_path, monkeypatch, home_writable=True) + accepted = pause_record._accepted_path(marker) + assert accepted.exists(), "premise: the consumer left its receipt" + if receipt == "refused": + accepted.chmod(0) + elif receipt == "malformed": + accepted.write_text("{trunc", encoding="utf-8") + else: + accepted.unlink() + try: + owed = _owed_after_recovery(pause_id) + finally: + if accepted.exists(): + accepted.chmod(0o644) + if receipt == "missing": + assert owed is None, "a request nobody accepted must still expire" + return + assert owed == {"p": os.getpid()}, f"a {receipt} receipt retired a draining gateway's restart debt" + # Unknown is judged again, never resolved into "sent": once readable it is a receipt like any other. + assert str(os.getpid()) not in pause_record.read(pause_record._claims(pause_record.record_path())[0])["token"]["stop_sent"] + assert _owed_after_recovery(pause_id) == {"p": os.getpid()} + + +def test_an_accepted_stop_whose_receipt_cannot_be_written_stays_owed(tmp_path, monkeypatch): + """Thread 6: the gateway home refuses new files too, so neither the record checkpoint nor the + receipt beside the request can be created; the consumer still accepts the stop. The kept + request itself must carry the acceptance, so the debt survives the request's TTL.""" + marker, pause_id = _accepted_then_orphaned(tmp_path, monkeypatch, home_writable=False) + assert not pause_record._accepted_path(marker).exists(), "premise: the receipt could not be created" + assert _owed_after_recovery(pause_id) == {"p": os.getpid()}, "an accepted stop lost its restart debt" From 0b1e0c59df592b6922cb633dd6e877013aa70e19 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:33:38 -0700 Subject: [PATCH 47/63] fix: a recovering launch pays an unready gateway relaunch once per backoff window update_pause_record.recover() runs before every command but `update` and `gateway run` (chat, doctor, config, the serve/dashboard processes Desktop spawns). Its Windows resume waits out the relaunch readiness budget for each profile/unmapped gateway; one that never became ready was handed back unowned "so the next launch retries at once", so every following command stalled ~30 s again (measured: 31 s on each of three consecutive launches). Profile and unmapped entries now get the stored backoff SCM services already have (service_retry): an attempted entry the resume left owed gets relaunch_retry {attempts, next_at} (60 s doubling, capped at 1 h); a launch inside the window holds it owed beside the draining entries without relaunching it. The debt itself is never dropped, a tree-gate deferral does not count as an attempt, and `hermes update` (merge_into does not carry the backoff) still retries at once. --- hermes_cli/update_pause_record.py | 51 ++++++++++++++++-- .../test_update_pause_recover_backoff.py | 52 +++++++++++++++++++ 2 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_recover_backoff.py diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index e7fa2e1db4484..160dec5f1d526 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -895,6 +895,8 @@ def _resume_fenced(claim_path: Path, body: dict) -> None: body = read(claim_path) or body token = drop_never_stopped(dict(body["token"])) draining = split_draining(token) + _hold_backing_off(token, draining) + attempted = _relaunch_keys(token) token.update(resume_needed=True, recovery=True) try: if _has_work(token): @@ -907,13 +909,56 @@ def _resume_fenced(claim_path: Path, body: dict) -> None: print(f" ⚠ Could not restart every paused gateway: {exc}. Run `hermes update` or " "`hermes gateway start`.", file=sys.stderr) finally: + _back_off_unready(token, attempted, draining) _hand_back(claim_path, body, token, draining) +# A recovering launch runs before every command (chat, doctor, a Desktop-spawned serve) and waits +# for each relaunched gateway to become ready. One that does not is re-launched only after this +# backoff (``relaunch_retry``), so an interactive command pays that wait at most once per window; +# the debt itself stays owed, and ``hermes update`` (which drops the backoff) retries at once. +_RELAUNCH_RETRY_BASE_S = 60.0 +_RELAUNCH_RETRY_MAX_S = 3600.0 + + +def _relaunch_keys(token: dict) -> set[str]: + return ({f"profile:{name}" for name in token.get("profiles") or {}} + | {f"unmapped:{u.get('pid')}" for u in token.get("unmapped") or [] if u.get("argv")}) + + +def _hold_backing_off(token: dict, held: dict) -> None: + """Move the profile/unmapped entries still inside their relaunch backoff from *token* into *held* + (beside the draining ones): owed, not relaunched by this launch.""" + now = time.time() + waiting = {key for key, state in (token.get("relaunch_retry") or {}).items() if now < float(state.get("next_at") or 0)} + held["profiles"].update({n: p for n, p in token["profiles"].items() if f"profile:{n}" in waiting}) + held["unmapped"].extend(u for u in token["unmapped"] if u.get("argv") and f"unmapped:{u.get('pid')}" in waiting) + token["profiles"] = {n: p for n, p in token["profiles"].items() if f"profile:{n}" not in waiting} + token["unmapped"] = [u for u in token["unmapped"] if u not in held["unmapped"]] + + +def _back_off_unready(token: dict, attempted: set[str], held: dict) -> None: + """Start or extend the backoff of each attempted entry the resume left owed (not ready, not + launched); forget it for one that came back. A tree-gate deferral attempted nothing.""" + retry = dict(token.get("relaunch_retry") or {}) + if not token.get("resume_deferred"): + unready = _relaunch_keys(token) + for key in attempted: + if key not in unready: + retry.pop(key, None) + continue + attempts = int((retry.get(key) or {}).get("attempts") or 0) + 1 + retry[key] = {"attempts": attempts, "next_at": time.time() + min( + _RELAUNCH_RETRY_BASE_S * 2 ** min(attempts - 1, 6), _RELAUNCH_RETRY_MAX_S)} + owed = _relaunch_keys(token) | _relaunch_keys(held) + token["relaunch_retry"] = {key: state for key, state in retry.items() if key in owed} + + def _hand_back(claim_path: Path, body: dict, token: dict, draining: dict) -> None: - """Retire the claim when nothing is owed; else hand it back unowned so the next launch retries - at once — this launch may live for hours (a chat). A draining process stays owed until it has - exited and been restarted. A failed rewrite leaves our claimer line, dead once we exit.""" + """Retire the claim when nothing is owed; else hand it back unowned for the next launch — this + launch may live for hours (a chat) — which retries at once, except a relaunch that did not + become ready (its backoff, :func:`_back_off_unready`). A draining process stays owed until it + has exited and been restarted. A failed rewrite leaves our claimer line, dead once we exit.""" token["profiles"] = {**(token.get("profiles") or {}), **draining["profiles"]} token["unmapped"] = [*(token.get("unmapped") or []), *draining["unmapped"]] owed = bool(token.get("resume_needed") or token.get("resume_deferred") or draining["profiles"] or draining["unmapped"]) diff --git a/tests/hermes_cli/test_update_pause_recover_backoff.py b/tests/hermes_cli/test_update_pause_recover_backoff.py new file mode 100644 index 0000000000000..a48da682b8d33 --- /dev/null +++ b/tests/hermes_cli/test_update_pause_recover_backoff.py @@ -0,0 +1,52 @@ +"""A launch's paused-gateway recovery never makes every command wait on the same unready relaunch. + +``recover()`` runs before every command (chat, doctor, a Desktop-spawned serve). The Windows resume +it calls waits out a readiness budget per relaunched gateway; that resume cannot run off Windows, so +a stand-in reports what it was handed and fails the way an unready relaunch does. The record, the +claim, the hand-back and the backoff are the real ones, on real files. +""" + +from __future__ import annotations + +import json +import sys + +import pytest + +from hermes_cli import update_pause_record as pause_record + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="the stand-in replaces the native resume") + + +def _claims() -> list: + return pause_record._claims(pause_record.record_path()) + + +def test_an_unready_relaunch_stalls_one_launch_per_backoff_window_and_stays_owed(tmp_path, monkeypatch): + import hermes_cli.update_cmd_windows as w + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + pause_record.write(pause_record.stamp_tree({"resume_needed": True, "profiles": {"alpha": 4194000}}), + owner=pause_record.UNOWNED) # a killed update's set; pid 4194000 is long gone + handed = [] + + def resume(token): + handed.append(sorted(token.get("profiles") or {})) + print("relaunched; waiting for readiness") + raise RuntimeError("Windows gateway relaunch after update was not verified alive: alpha") + + monkeypatch.setattr(w, "_resume_windows_gateways_after_update", resume) + pause_record.recover(["status"]) + pause_record.recover(["chat"]) + assert handed == [["alpha"]], "a launch inside the backoff waited on the unready relaunch again" + [claim] = _claims() + token = pause_record.read(claim)["token"] + assert token["profiles"] == {"alpha": 4194000}, "the unready gateway's restart debt was dropped" + assert token["relaunch_retry"]["profile:alpha"]["attempts"] == 1 + + body = json.loads(claim.read_text(encoding="utf-8")) + body["token"]["relaunch_retry"]["profile:alpha"]["next_at"] = 0 # the window elapsed + claim.write_text(json.dumps(body), encoding="utf-8") + pause_record.recover(["status"]) + assert handed == [["alpha"], ["alpha"]], "the debt was not retried once its backoff elapsed" + [claim] = _claims() + assert pause_record.read(claim)["token"]["relaunch_retry"]["profile:alpha"]["attempts"] == 2 From 6712e26b82ac76775894fac66853551b76a9979d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:34:42 -0700 Subject: [PATCH 48/63] fix: startup recovery keeps the Windows resume's progress lines off the command's stdout recover() runs before unrelated commands (anything printing --json, the serve/dashboard processes Desktop reads over a pipe). Its own notices already go to stderr, but the Windows resume helpers it calls print with bare print(), which landed in that command's stdout. The resume now runs with sys.stdout pointed at stderr. The record tests' resume stand-ins report on the real stdout (sys.__stdout__) accordingly; no assertion changed. --- hermes_cli/update_pause_record.py | 6 ++++-- tests/hermes_cli/test_update_pause_record.py | 9 +++++---- tests/hermes_cli/test_update_pause_recover_backoff.py | 5 ++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 160dec5f1d526..ba57e1eed985a 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -47,7 +47,7 @@ import threading import time import uuid -from contextlib import contextmanager, suppress +from contextlib import contextmanager, redirect_stdout, suppress from pathlib import Path RECORD_STEM = ".hermes-update-paused-gateways" @@ -902,7 +902,9 @@ def _resume_fenced(claim_path: Path, body: dict) -> None: if _has_work(token): print("→ Restarting gateway(s) paused by an interrupted `hermes update`...", file=sys.stderr) from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update - _resume_windows_gateways_after_update(token) + # Whatever command this launch runs owns stdout (``--json`` output, a Desktop pipe). + with redirect_stdout(sys.stderr): + _resume_windows_gateways_after_update(token) else: token["resume_needed"] = False except Exception as exc: # health: allow BLE001 -- recovery boundary: any resume error is reported and the claim handed back (finally), never raised into the launch diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 9d9795a49f8f1..638d8348ac5df 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -173,13 +173,14 @@ def _orphan(tmp_path: Path, profiles: dict) -> None: owner.wait(timeout=10) -# The Windows resume cannot run here: the stand-in for it prints the set it was handed and -# (mode "hang") parks like a resume waiting on relaunch verification. +# The Windows resume cannot run here: the stand-in for it prints the set it was handed (on the real +# stdout: recovery points sys.stdout at stderr) and (mode "hang") parks like a resume waiting on +# relaunch verification. _RECOVER = """ import sys, time import hermes_cli.update_cmd_windows as w def resume(token): - print("resume", sorted(token.get("profiles") or {}), flush=True) + print("resume", sorted(token.get("profiles") or {}), file=sys.__stdout__, flush=True) # recovery sends stdout to stderr if sys.argv[1] == "hang": time.sleep(120) token["resume_needed"] = False @@ -319,7 +320,7 @@ def trace(frame, event, arg): # traces only fn's own frames: the rest runs at f _RESUMES = """ import hermes_cli.update_cmd_windows as w def resume(token): - print("resume", sorted(token.get("profiles") or {}), flush=True) + print("resume", sorted(token.get("profiles") or {}), file=sys.__stdout__, flush=True) # recovery sends stdout to stderr token["resume_needed"] = False w._resume_windows_gateways_after_update = resume r.recover(["status"]) diff --git a/tests/hermes_cli/test_update_pause_recover_backoff.py b/tests/hermes_cli/test_update_pause_recover_backoff.py index a48da682b8d33..3ef66c8812ff5 100644 --- a/tests/hermes_cli/test_update_pause_recover_backoff.py +++ b/tests/hermes_cli/test_update_pause_recover_backoff.py @@ -22,7 +22,7 @@ def _claims() -> list: return pause_record._claims(pause_record.record_path()) -def test_an_unready_relaunch_stalls_one_launch_per_backoff_window_and_stays_owed(tmp_path, monkeypatch): +def test_an_unready_relaunch_stalls_one_launch_per_backoff_window_and_stays_owed(tmp_path, monkeypatch, capsys): import hermes_cli.update_cmd_windows as w monkeypatch.setenv("HERMES_HOME", str(tmp_path)) pause_record.write(pause_record.stamp_tree({"resume_needed": True, "profiles": {"alpha": 4194000}}), @@ -42,6 +42,9 @@ def resume(token): token = pause_record.read(claim)["token"] assert token["profiles"] == {"alpha": 4194000}, "the unready gateway's restart debt was dropped" assert token["relaunch_retry"]["profile:alpha"]["attempts"] == 1 + out = capsys.readouterr() + assert "waiting for readiness" not in out.out and "waiting for readiness" in out.err, \ + "recovery wrote into the stdout of an unrelated command" body = json.loads(claim.read_text(encoding="utf-8")) body["token"]["relaunch_retry"]["profile:alpha"]["next_at"] = 0 # the window elapsed From 6189cdec28cea64beacb23f4d7555e44ded270ee Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:36:20 -0700 Subject: [PATCH 49/63] fix: drop update_pause_record.orphaned_record, which only tests called Its callers (the record test helper and the hostile-host pause cell) now read orphans() directly, the same survey recovery and adoption use. --- hermes_cli/update_pause_record.py | 5 ----- tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py | 2 +- tests/hermes_cli/test_update_pause_record.py | 6 +++--- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index ba57e1eed985a..57690ed1974f9 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -629,11 +629,6 @@ def orphans(path: Path | None = None) -> list[tuple[Path, dict]]: return found -def orphaned_record(path: Path | None = None) -> dict | None: - found = orphans(path) - return found[0][1] if found else None - - def retire_redundant(path: Path | None = None) -> None: """Delete copies of obligations another file already carries (crash leftovers of a transfer).""" path = path or record_path() diff --git a/tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py b/tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py index a3e461cf6772f..d35b607c19a8a 100644 --- a/tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py +++ b/tests/e2e/core/upgrade/hosts/test_hostile_pause_record.py @@ -44,7 +44,7 @@ from hermes_cli.update_lock import UpdateLock lock = UpdateLock(install_root=sys.argv[1]) adopted_claim = lock.acquire() and not lock.acquired -saw_orphan = r.orphaned_record() is not None +saw_orphan = bool(r.orphans()) r.write(r.stamp_tree({"resume_needed": True, "profiles": {"mine": 1}}), owner=r.identity()) print(json.dumps({"adopted_claim": adopted_claim, "saw_orphan": saw_orphan, "recorded": sorted(r.read()["token"]["profiles"])})) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 638d8348ac5df..27b934072a21d 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -46,12 +46,12 @@ def _child(code: str, *argv: str, env: dict) -> subprocess.Popen: def _orphaned_profiles(home: Path) -> dict | None: - """``orphaned_record()`` read by a fresh process (the liveness probe reads the checkout's git dir).""" + """The first orphan's profiles, read by a fresh process (the liveness probe reads the checkout's git dir).""" probe = _child(""" import json from hermes_cli import update_pause_record as r - body = r.orphaned_record() - print(json.dumps(None if body is None else body["token"]["profiles"])) + found = r.orphans() + print(json.dumps(found[0][1]["token"]["profiles"] if found else None)) """, env={"HERMES_HOME": str(home)}) out, _ = probe.communicate(timeout=60) return json.loads(out.strip().splitlines()[-1]) From b8ff7aaf7c7d282622d933407e5d7326d7987fe2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:36:20 -0700 Subject: [PATCH 50/63] fix: the psutil-less identity test judges with one creation-time probe update_lock caches this process's creation time (_OWN_CT). Warmed by an earlier test through psutil, it differed from the stdlib probe identity() uses with psutil hidden by that probe's resolution (macOS `ps -o lstart`: 1 s), so the test failed in a full-file run and passed alone. It now starts from an empty cache, so both sides use the same probe. --- tests/hermes_cli/test_update_pause_record.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py index 27b934072a21d..86d78a6846f09 100644 --- a/tests/hermes_cli/test_update_pause_record.py +++ b/tests/hermes_cli/test_update_pause_record.py @@ -755,6 +755,9 @@ def test_a_host_without_psutil_records_an_identity_the_liveness_rule_can_prove(m judges with, so a psutil-less host never writes ``ct=None`` (unprovable, so never orphaned).""" from hermes_cli import update_lock monkeypatch.setitem(sys.modules, "psutil", None) # `import psutil` raises ImportError + # Our own creation time is cached per process; one an earlier test read with psutil differs + # from the stdlib probe by its resolution (macOS `ps -o lstart`: 1 s). Judge with one probe. + monkeypatch.setattr(update_lock, "_OWN_CT", {}) ident = pause_record.identity() assert ident["ct"] is not None, "no creation time without psutil" assert update_lock.incarnation_live(ident["pid"], ident["ct"]) is True From 78533d30bae558463da451b16d015546ea5221bd Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:21:16 -0700 Subject: [PATCH 51/63] fix: a paused unmapped gateway keeps the home discovery proved Discovery owns an unmapped gateway whose environment is unreadable when the exact-birth spawn ledger names its home, but the pause snapshot re-read only the environment and wrote `home: null`. The readiness predicate read that as a pre-home record and let another home's gateway with the same selectorless argv retire the debt. The snapshot, discovery and readiness now share one resolver (update_fleet_scope.gateway_pid_home: environment, else ledger witness). A home that no longer proves refuses the pause before any stop, and only an entry with no `home` key (an older updater's token) keeps argv-only matching; a recorded empty home is never matched. --- hermes_cli/update_cmd_windows.py | 26 +++-- hermes_cli/update_fleet_scope.py | 15 ++- .../test_update_pause_unmapped_home_proof.py | 101 ++++++++++++++++++ 3 files changed, 130 insertions(+), 12 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_unmapped_home_proof.py diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index fd9c1c53a8bf4..b9daec5f2560f 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -949,15 +949,22 @@ def _pause_windows_gateways_for_update() -> dict | None: return _cold_start_pause_token(adopted, claims) unmapped_pids = [pid for pid in running_pids if pid not in profile_processes and pid not in service_gateway_pids] from gateway.status import get_process_start_time - from hermes_cli.dashboard_procs import _hermes_home_for_pid + from hermes_cli.update_fleet_scope import gateway_pid_home born = {int(pid): get_process_start_time(int(pid)) for pid in running_pids} # the processes discovered # Snapshot unmapped gateways' argv *before* anything stops so resume (or a later launch) can replay it. # Unmapped = no profile->PID-file mapping (e.g. Scheduled Task ``pythonw.exe -m ...``). Two homes can - # run identical selectorless argv, so the home and birth identify the runtime the replay must restore. + # run identical selectorless argv, so the home and birth identify the runtime the replay must restore: + # read with discovery's own resolver (environment, else the exact-birth ledger witness), and a home + # that no longer proves refuses the pause before any stop. ``home: null`` would read as a pre-home + # record, whose argv-only match lets another home's gateway retire this debt. + homes = {int(pid): gateway_pid_home(int(pid)) for pid in unmapped_pids} + if None in homes.values(): + raise RuntimeError("Could not prove the Hermes home of unmapped gateway PID(s) " + + ", ".join(str(p) for p, h in homes.items() if h is None) + "; nothing was stopped") unmapped = [ {"pid": int(pid), "argv": _try_call(lambda p=int(pid): _capture_gateway_argv(p), "Could not capture argv for unmapped gateway %s: %s", int(pid)), - "home": _hermes_home_for_pid(int(pid)), "ct": born[int(pid)]} + "home": homes[int(pid)], "ct": born[int(pid)]} for pid in unmapped_pids ] intended = { @@ -1400,18 +1407,21 @@ def _unmapped_ready_filter(entry: dict, taken: set): same arguments after the (normalized) interpreter on the entry's home, born after the process it replaces, not one another entry already vouched for. Two homes can run identical selectorless argv, so argv alone would let home B's gateway retire home A's debt (a service gateway on another home - included; on A's home only one gateway can run). An entry recorded before ``home`` was captured - keeps the argv-only match: nothing else identifies it.""" + included; on A's home only one gateway can run). Only an entry recorded before ``home`` was + captured (no key: a pre-durable-pause updater's token) keeps the argv-only match; a recorded + ``home`` that is empty proves no runtime and is never matched.""" import psutil from gateway.status import _looks_like_gateway_process, _same_hermes_home, get_process_start_time - from hermes_cli.dashboard_procs import _hermes_home_for_pid + from hermes_cli.update_fleet_scope import gateway_pid_home tail, old = list(entry.get("argv") or [])[1:], int(entry.get("pid") or 0) home, old_ct = entry.get("home"), entry.get("ct") def same_runtime(pid: int) -> bool: - if not home: + if "home" not in entry: return True - live_home, ct = _hermes_home_for_pid(pid), get_process_start_time(pid) + if not home: + return False + live_home, ct = gateway_pid_home(pid), get_process_start_time(pid) return live_home is not None and _same_hermes_home(live_home, home) and (old_ct is None or ct is None or ct > old_ct) def matches(pid: int) -> bool: diff --git a/hermes_cli/update_fleet_scope.py b/hermes_cli/update_fleet_scope.py index 54360c7d39a03..10c5ad12608e1 100644 --- a/hermes_cli/update_fleet_scope.py +++ b/hermes_cli/update_fleet_scope.py @@ -65,14 +65,21 @@ def _ledger_gateway_home(pid: int) -> str | None: return None -def gateway_pid_in_update_scope(pid: int, scope: set[Path] | None = None) -> bool | None: - """Does gateway *pid* run on a home this update owns? ``None`` when its home cannot be read.""" +def gateway_pid_home(pid: int) -> str | None: + """The home gateway *pid* runs on: its live environment, else the exact-birth ledger witness; + ``None`` when neither proves one. Discovery, the Windows pause record and its readiness check + share it, so the home discovery proved is the home the restart debt keeps.""" from hermes_cli.dashboard_procs import _hermes_home_for_pid try: home = _hermes_home_for_pid(pid) - except Exception: + except Exception: # health: allow BLE001 -- tri-state resolver: any environment-read failure is "unknown" and the ledger witness decides home = None - home = home or _ledger_gateway_home(pid) + return home or _ledger_gateway_home(pid) + + +def gateway_pid_in_update_scope(pid: int, scope: set[Path] | None = None) -> bool | None: + """Does gateway *pid* run on a home this update owns? ``None`` when its home cannot be read.""" + home = gateway_pid_home(pid) if home is None: return None return home_in_update_scope(home, scope) diff --git a/tests/hermes_cli/test_update_pause_unmapped_home_proof.py b/tests/hermes_cli/test_update_pause_unmapped_home_proof.py new file mode 100644 index 0000000000000..735049bcc5ab2 --- /dev/null +++ b/tests/hermes_cli/test_update_pause_unmapped_home_proof.py @@ -0,0 +1,101 @@ +"""A paused unmapped gateway keeps the home discovery proved for it (#132338 R2 / review thread 7). + +Discovery owns a gateway whose environment is unreadable (elevated, another account) when this +install's spawn ledger holds an exact-birth witness of its home. The pause record must keep THAT +home: written as ``home: null`` it read as a pre-home record, whose argv-only readiness match let +another home's gateway with the same selectorless argv retire the paused runtime's debt. + +Real processes, each under its own ``HERMES_HOME``, a real ledger entry, the real pause producer and +durable record, and the real readiness predicate. Seams: the OS denying A's environment +(``_pid_environ``), the Windows-only gateway enumeration and the stop itself (never signalled here). +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time + +import psutil +import pytest + +from gateway.status import get_process_start_time +from hermes_cli import dashboard_procs, gateway, process_identity, update_cmd_windows +from hermes_cli import update_pause_record as pause_record +from hermes_cli.update_cmd_windows import _pause_windows_gateways_for_update, _unmapped_ready_filter + + +@pytest.fixture +def homes_and_spawn(tmp_path): + entry = tmp_path / "bin" / "hermes" # a ``hermes`` entry token: the cmdline reads as a gateway + entry.parent.mkdir() + entry.write_text("import time\ntime.sleep(60)\n", encoding="utf-8") + argv = [sys.executable, str(entry), "gateway", "run"] + root = os.environ["HERMES_HOME"] # the sandboxed home this update owns, with two profiles + home_a, home_b = (os.path.join(root, "profiles", name) for name in ("a", "b")) + os.makedirs(home_a) + os.makedirs(home_b) + procs: list[subprocess.Popen] = [] + + def spawn(home) -> subprocess.Popen: + proc = subprocess.Popen(argv, env={**os.environ, "HERMES_HOME": str(home)}) + procs.append(proc) + return proc + + yield home_a, home_b, spawn + for proc in procs: + proc.kill() + proc.wait(timeout=10) + + +@pytest.mark.platforms("linux") +@pytest.mark.spawns_gateway_lookalike # stub children that sleep; the fixture reaps them +def test_a_ledger_proven_unmapped_gateway_is_recorded_with_its_home(homes_and_spawn, monkeypatch): + home_a, home_b, spawn = homes_and_spawn + a = spawn(home_a) + time.sleep(0.05) + ct = process_identity._process_create_time(a.pid) + assert process_identity._append_entry(process_identity.LedgerEntry( + a.pid, ct, "gateway", process_identity.install_id(), None, None, time.time(), "", hermes_home=home_a)) + real_environ = dashboard_procs._pid_environ + monkeypatch.setattr(dashboard_procs, "_pid_environ", lambda pid: None if pid == a.pid else real_environ(pid)) + from hermes_cli import main + monkeypatch.setattr(main, "_is_windows", lambda: True) + monkeypatch.setattr(gateway, "find_gateway_pids", lambda all_profiles=False: [a.pid]) + monkeypatch.setattr(gateway, "find_profile_gateway_processes", lambda strict=False: []) + monkeypatch.setattr(gateway, "find_windows_gateway_services", lambda profile_processes=(): []) + stopped = [] + monkeypatch.setattr(update_cmd_windows, "_stop_windows_gateways", lambda *a, **k: stopped.append(a[0]) or {}) + monkeypatch.setattr(update_cmd_windows, "_record_attested_cold_start_profiles", lambda *a: None) + + _pause_windows_gateways_for_update() + assert stopped == [[a.pid]] + entry = pause_record.read(pause_record.record_path())["token"]["unmapped"][0] + assert entry["home"] == home_a, "the pause dropped the home discovery proved" + + a.kill() + a.wait(timeout=10) + time.sleep(0.05) # a later clock tick: a replacement is born after the process it replaces + sibling = spawn(home_b) + assert _unmapped_ready_filter(entry, set())([sibling.pid]) == [], \ + "home B's gateway with A's argv retired A's restart debt" + replacement = spawn(home_a) + assert _unmapped_ready_filter(entry, set())([sibling.pid, replacement.pid]) == [replacement.pid] + + +@pytest.mark.platforms("linux") +@pytest.mark.spawns_gateway_lookalike +def test_a_recorded_but_empty_home_is_never_matched_only_a_pre_home_entry_is(homes_and_spawn): + home_a, home_b, spawn = homes_and_spawn + old = spawn(home_a) + argv = list(psutil.Process(old.pid).cmdline()) + born = get_process_start_time(old.pid) + old.kill() + old.wait(timeout=10) + time.sleep(0.05) + sibling = spawn(home_b) + assert _unmapped_ready_filter({"pid": old.pid, "argv": argv, "home": None, "ct": born}, set())([sibling.pid]) == [], \ + "a newly incomplete record took the legacy argv-only match" + # A token from an updater that predates the home field has nothing else to match on. + assert _unmapped_ready_filter({"pid": old.pid, "argv": argv}, set())([sibling.pid]) == [sibling.pid] From f93e83aff75c6ee62509cbc3ce6fda328965f42d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:22:41 -0700 Subject: [PATCH 52/63] fix: a failed service pause keeps its record on typed rollback failures _pause_windows_gateways_for_update decided whether to abandon the durable pause record by searching the error text for "rollback failures"; rewording that message would delete the record while gateways are still stopped. The service pause now raises ServicePauseFailed carrying rollback_failures, and the record is abandoned only when that list is empty. Any other exception keeps the record. --- hermes_cli/update_cmd_windows.py | 16 ++++-- .../test_update_pause_service_rollback.py | 51 +++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_service_rollback.py diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index b9daec5f2560f..5f7388b11a4ce 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -811,6 +811,16 @@ def _windows_cold_start_plan() -> dict | None: return None +class ServicePauseFailed(RuntimeError): + """A gateway service could not be stopped; ``rollback_failures`` lists what the in-line + rollback could not restart. Empty = everything stopped was restarted, so the pause is void; + otherwise its record must stay, since gateways are still down.""" + + def __init__(self, detail: str, rollback_failures: list[str]): + super().__init__(detail) + self.rollback_failures = rollback_failures + + def _pause_windows_gateway_services(service_gateways, token: dict, profiles: dict, unmapped: list) -> dict: """Stop each SCM gateway service, recording them on *token*; roll everything back on failure. @@ -848,7 +858,7 @@ def _pause_windows_gateway_services(service_gateways, token: dict, profiles: dic detail = f"Could not stop Windows gateway service {current_service_name or 'unknown'}: {exc}" if rollback_failures: detail += "; rollback failures: " + "; ".join(rollback_failures) - raise RuntimeError(detail) from exc + raise ServicePauseFailed(detail, rollback_failures) from exc def _owned_gateway_pids(pids, *, keep=(), quiet: bool = True) -> list[int]: @@ -993,8 +1003,8 @@ def _pause_windows_gateways_for_update() -> dict | None: _record_attested_cold_start_profiles(token, running_profiles) try: token = _pause_windows_gateway_services(service_gateways, token, profiles, unmapped) - except RuntimeError as exc: - if "rollback failures" not in str(exc): # everything stopped was restarted in-line + except ServicePauseFailed as exc: + if not exc.rollback_failures: # everything stopped was restarted in-line pause_record.abandon_pause(intended, adopted) raise return pause_record.finish_pause(token, intended, adopted) diff --git a/tests/hermes_cli/test_update_pause_service_rollback.py b/tests/hermes_cli/test_update_pause_service_rollback.py new file mode 100644 index 0000000000000..b3b89908e56eb --- /dev/null +++ b/tests/hermes_cli/test_update_pause_service_rollback.py @@ -0,0 +1,51 @@ +"""A failed service pause keeps its record exactly when its rollback left gateways down. + +``_pause_windows_gateways_for_update`` decides whether to abandon the durable pause record from the +service-pause failure. That decision is read from a typed ``rollback_failures`` list, never from the +error message's wording: a reworded message must not delete the record while gateways stay stopped. + +Real pause producer and durable record; seams are the Windows-only discovery, the SCM stop/restore +calls and the ordinary-gateway stop (nothing is signalled). +""" + +from __future__ import annotations + +import os +from types import SimpleNamespace + +import pytest + +from hermes_cli import gateway, update_cmd, update_cmd_windows +from hermes_cli import update_pause_record as pause_record +from hermes_cli.update_cmd_windows import ServicePauseFailed, _pause_windows_gateways_for_update + + +@pytest.mark.parametrize("restore_fails", [False, True]) +def test_the_pause_record_survives_exactly_when_the_service_rollback_failed(monkeypatch, restore_fails): + from hermes_cli import main + pid = os.getpid() # a live identity for the service's gateway; never signalled + service = SimpleNamespace(name="hermes-gw", profile="default", gateway_pid=pid, gateway_create_time=1.0, + service_pid=pid, service_create_time=1.0, descendant_identities=()) + monkeypatch.setattr(main, "_is_windows", lambda: True) + monkeypatch.setattr(gateway, "find_gateway_pids", lambda all_profiles=False: []) + monkeypatch.setattr(gateway, "find_profile_gateway_processes", lambda strict=False: []) + monkeypatch.setattr(gateway, "find_windows_gateway_services", lambda profile_processes=(): [service]) + monkeypatch.setattr(update_cmd_windows, "_stop_windows_gateways", lambda *a, **k: {}) + monkeypatch.setattr(update_cmd_windows, "_record_attested_cold_start_profiles", lambda *a: None) + + def stop(*_a, **_k): + raise OSError("access denied") + + def restore(_name): + if restore_fails: + raise OSError("service will not start") + + monkeypatch.setattr(update_cmd, "_stop_windows_gateway_service", stop) + monkeypatch.setattr(update_cmd, "_restore_windows_gateway_service", restore) + + with pytest.raises(ServicePauseFailed) as failed: + _pause_windows_gateways_for_update() + assert bool(failed.value.rollback_failures) is restore_fails + saved = pause_record.read(pause_record.record_path()) + assert (saved is not None) is restore_fails, \ + "a rolled-back pause kept its record" if saved else "the record of still-stopped gateways was deleted" From bf5575feb1b80779d451a87d5c28e3ae9e4471f3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:23:35 -0700 Subject: [PATCH 53/63] fix: startup gateway recovery prints to stderr, never the command's stdout update_pause_record.recover() resumes paused gateways ahead of every hermes command and prints its own notices to stderr, but the Windows resume path it calls printed deferral/restart notices to stdout, corrupting the output of unrelated commands (--json ones included). A recovering resume now routes its stdout to stderr. --- hermes_cli/update_cmd_windows.py | 10 +++++++- .../test_update_pause_recovery_stdout.py | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/hermes_cli/test_update_pause_recovery_stdout.py diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 5f7388b11a4ce..3a74f8873a915 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -5,7 +5,7 @@ """ import logging -from contextlib import contextmanager, suppress +from contextlib import contextmanager, nullcontext, redirect_stdout, suppress import os import re import shlex @@ -1516,6 +1516,14 @@ def _resume_windows_gateways_after_update(token: dict | None) -> None: if not _m()._is_windows(): token["resume_needed"] = False return + # Startup recovery runs ahead of an unrelated command whose stdout may be parsed (``--json``): + # its notices go to stderr, as ``recover()``'s own do. + with redirect_stdout(sys.stderr) if token.get("recovery") else nullcontext(): + _resume_unless_torn(token) + + +def _resume_unless_torn(token: dict) -> None: + """Resume the paused set, or leave it owed when the checkout is not whole.""" from hermes_cli import update_pause_record as pause_record if token.get("pause_id"): whole, why = pause_record.tree_is_whole(token) diff --git a/tests/hermes_cli/test_update_pause_recovery_stdout.py b/tests/hermes_cli/test_update_pause_recovery_stdout.py new file mode 100644 index 0000000000000..740792ee1824a --- /dev/null +++ b/tests/hermes_cli/test_update_pause_recovery_stdout.py @@ -0,0 +1,25 @@ +"""Startup recovery never writes to the stdout of the command it runs ahead of. + +``update_pause_record.recover()`` resumes gateways an interrupted update paused before ANY ``hermes`` +command (``--json`` ones included) and prints its own notices to stderr; the Windows resume path it +calls must do the same. Real resume entry point; seams are the Windows dispatch and the tree gate's +verdict (a torn checkout), so the deferral notice is what gets printed. +""" + +from __future__ import annotations + +from hermes_cli import update_pause_record as pause_record +from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update + + +def test_a_recovering_resume_prints_its_notices_to_stderr(monkeypatch, capsys): + from hermes_cli import main + monkeypatch.setattr(main, "_is_windows", lambda: True) + monkeypatch.setattr(pause_record, "tree_is_whole", lambda token: (False, "the checkout is mid-pull")) + token = {"resume_needed": True, "pause_id": "p", "recovery": True, "profiles": {"default": 4242}} + + _resume_windows_gateways_after_update(token) + out, err = capsys.readouterr() + assert out == "", "recovery wrote to the unrelated command's stdout" + assert "left stopped: the checkout is mid-pull" in err + assert token["resume_deferred"] == "the checkout is mid-pull" From 6afe92c9216392868995fb108cc0cf6400ce3254 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:29:40 -0700 Subject: [PATCH 54/63] fix: post-update relaunch verification polls every target in one loop Retiring each paused runtime on its own readiness made the wait serial: every profile and unmapped entry ran its own 2 s confirmation window (plus a 6 s per-target floor past the deadline), so 3 profiles took 6 s and 5 took 10 s where the fleet-wide wait had taken one window. All targets now share one poll: one fleet process scan per tick, each target credited only on its own hits for a whole confirmation window, and a PID one target vouched for is never credited to another. Measured with readiness stubbed to succeed: N=1/3/5 went from 2.0/6.0/10.0 s to 2.0 s flat (N=10 also 2.0 s). The reconciliation tests' liveness stub moves to the probe the verifier now polls (_live_gateway_pids); their assertions are unchanged. --- hermes_cli/update_cmd_windows.py | 79 +++++++++++++----- .../test_windows_relaunch_ready_poll.py | 83 +++++++++++++++++++ ...t_windows_update_restart_reconciliation.py | 3 + 3 files changed, 143 insertions(+), 22 deletions(-) create mode 100644 tests/hermes_cli/test_windows_relaunch_ready_poll.py diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 3a74f8873a915..2fa35f07444ba 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -1383,7 +1383,6 @@ def _relaunch_paused_gateways(profiles: dict, unmapped: list) -> tuple[dict, lis _RELAUNCH_VERIFY_TIMEOUT_S = 30.0 -_PER_TARGET_FLOOR_S = 6.0 def _pending_relaunch_pids(profiles: dict, unmapped: list, pid_exists) -> list[int]: @@ -1445,6 +1444,53 @@ def matches(pid: int) -> bool: return lambda pids: [pid for pid in _owned_gateway_pids(pids) if matches(int(pid))] +_READY_INTERVAL_S = 0.4 # gateway_windows._wait_for_gateway_ready's poll interval and +_READY_CONFIRM_S = 2.0 # confirmation window: one hit proves a spawn, not a survived startup + + +def _poll_until_ready(targets: list, deadline: float, taken: set) -> dict: + """``{key: pids}`` for each ``(key, probe, claim_all)`` target whose gateway stayed visible for + a whole confirmation window. All targets share one loop, so N targets cost one window, not N: + one fleet process scan per tick (``probe(scan)`` calls ``scan()`` only if it needs the fleet), + and each target is credited only on its own hits. *taken* holds, while a target is probed, the + PIDs earlier targets vouched for (all of a profile's, the first of an unmapped entry's) so one + gateway never retires two debts. A hit provisional at the deadline still finishes its window; + a probe that raises leaves its target unready.""" + from hermes_cli.gateway import find_gateway_pids + ready: dict = {} + since: dict = {} + failed: set = set() + while True: + now = _time.monotonic() + pending = [t for t in targets if t[0] not in ready and t[0] not in failed and (now < deadline or t[0] in since)] + if not pending: + return ready + fleet: list = [] + + def scan() -> list: + if not fleet: + fleet.append(list(find_gateway_pids(all_profiles=True))) + return fleet[0] + taken.clear() + taken.update(pid for pids in ready.values() for pid in pids) + for key, probe, claim_all in pending: + try: + pids = list(probe(scan)) + except Exception as exc: # health: allow BLE001 -- one target's probe failure must not stall the others + logger.warning("Could not verify Windows gateway %s after update: %s", key, exc) + failed.add(key) + continue + if not pids: + since.pop(key, None) + continue + vouched = pids if claim_all else pids[:1] + taken.update(vouched) + if now - since.setdefault(key, now) >= _READY_CONFIRM_S: + ready[key] = vouched + if len(ready) + len(failed) < len(targets): + _time.sleep(_READY_INTERVAL_S) + + def _verify_relaunched_gateways_alive(token: dict, profiles: dict, unmapped: list) -> None: """Retire each relaunched runtime on ITS OWN readiness: a profile on a stable gateway in its profile home, an unmapped entry on a stable gateway running its argv. A watcher only proves a @@ -1457,27 +1503,16 @@ def _verify_relaunched_gateways_alive(token: dict, profiles: dict, unmapped: lis from hermes_cli import gateway_windows from hermes_cli.profiles import get_profile_dir deadline = _time.monotonic() + _relaunch_verify_timeout_s(profiles, unmapped, _pid_exists) - - def budget() -> float: - return max(deadline - _time.monotonic(), _PER_TARGET_FLOOR_S) - - ready_pids: list[int] = [] - ready_profiles = [] - for name, old_pid in sorted(profiles.items()): - pids = _try_call(lambda n=name, o=old_pid: gateway_windows._wait_for_gateway_ready( - timeout_s=budget(), home=Path(get_profile_dir(n)), pid_filter=lambda ps, o=o: [p for p in ps if int(p) != int(o)]), - "Could not verify Windows gateway profile %s after update: %s", name, default=[]) - if pids: - ready_profiles.append(name) - ready_pids.extend(pids) - ready_unmapped = [] - for entry in unmapped: - pids = _try_call(lambda e=entry: gateway_windows._wait_for_gateway_ready( - timeout_s=budget(), all_profiles=True, pid_filter=_unmapped_ready_filter(e, set(ready_pids))), - "Could not verify unmapped Windows gateway (pid %s) after update: %s", entry.get("pid"), default=[]) - if pids: - ready_unmapped.append(entry) - ready_pids.extend(pids[:1]) + taken: set = set() + # Profiles first: a gateway on a profile's home is that profile's, never an unmapped entry's. + targets = [(("profile", name), lambda _scan, h=Path(get_profile_dir(name)), o=int(old_pid): gateway_windows._live_gateway_pids( + home=h, pid_filter=lambda ps: [p for p in ps if int(p) != o]), True) for name, old_pid in sorted(profiles.items())] + targets += [(("unmapped", i), lambda scan, f=_unmapped_ready_filter(entry, taken): f(scan()), False) + for i, entry in enumerate(unmapped)] + ready = _poll_until_ready(targets, deadline, taken) + ready_profiles = [name for name in sorted(profiles) if ("profile", name) in ready] + ready_unmapped = [entry for i, entry in enumerate(unmapped) if ("unmapped", i) in ready] + ready_pids = [pid for pids in ready.values() for pid in pids] # Fleet reconciliation (#91277) cross-checks every planned runtime against relaunched_profiles: # only a verified one is listed, so an unready one still surfaces as unaccounted. token["relaunched_profiles"] = ready_profiles diff --git a/tests/hermes_cli/test_windows_relaunch_ready_poll.py b/tests/hermes_cli/test_windows_relaunch_ready_poll.py new file mode 100644 index 0000000000000..9bba794414c6b --- /dev/null +++ b/tests/hermes_cli/test_windows_relaunch_ready_poll.py @@ -0,0 +1,83 @@ +"""Post-update relaunch verification polls every paused target at once (#132338 review, kshitij). + +Each relaunched profile / unmapped gateway is retired only on its own stable readiness, and before +this every target ran its own confirmation window in turn: N targets cost N windows (5 profiles, +10 s) where the fleet-wide wait had cost one. One loop keeps the per-target crediting. + +Real ``_verify_relaunched_gateways_alive`` and its poll; the seam is the PID-file read that says +which gateway each profile home runs (``get_running_pid``), so no process is spawned or signalled. +""" + +from __future__ import annotations + +import time + +import pytest + +from gateway import status +from hermes_cli import gateway_windows, update_cmd_windows +from hermes_cli.update_cmd_windows import _READY_CONFIRM_S, _verify_relaunched_gateways_alive + + +@pytest.fixture +def running(monkeypatch): + """``{profile: callable -> pid or None}``: what each profile home's PID file names right now.""" + table: dict = {} + monkeypatch.setattr(status, "get_running_pid", + lambda path, cleanup_stale=False: (table.get(path.parent.name) or (lambda: None))()) + monkeypatch.setattr(status, "_pid_exists", lambda pid: False) # every old gateway is gone + monkeypatch.setattr(gateway_windows, "_write_start_attestation", lambda *a, **k: None) + return table + + +def test_many_ready_targets_cost_one_confirmation_window(running): + profiles = {f"p{i}": 40000 + i for i in range(6)} + for i, name in enumerate(profiles): + running[name] = lambda pid=50000 + i: pid + token = {"profiles": dict(profiles), "unmapped": []} + started = time.monotonic() + _verify_relaunched_gateways_alive(token, profiles, []) + elapsed = time.monotonic() - started + assert token["relaunched_profiles"] == sorted(profiles) and token["profiles"] == {} + assert elapsed < 2 * _READY_CONFIRM_S, f"6 ready targets took {elapsed:.1f}s: verified one after another" + + +def test_each_target_is_credited_only_on_its_own_stable_gateway(running, monkeypatch): + monkeypatch.setattr(update_cmd_windows, "_RELAUNCH_VERIFY_TIMEOUT_S", 3.0) + born = time.monotonic() + running["up"] = lambda: 50001 + running["old"] = lambda: 40002 # only the process the relaunch replaces + running["flaps"] = lambda: 50003 if time.monotonic() - born < 1.0 else None # died during confirmation + profiles = {"up": 40001, "old": 40002, "flaps": 40003, "never": 40004} + token = {"profiles": dict(profiles), "unmapped": []} + with pytest.raises(RuntimeError, match="not verified alive: flaps, never, old"): + _verify_relaunched_gateways_alive(token, profiles, []) + assert token["relaunched_profiles"] == ["up"] + assert sorted(token["profiles"]) == ["flaps", "never", "old"] + + +@pytest.mark.platforms("linux") +@pytest.mark.spawns_gateway_lookalike # a stub child that sleeps; reaped below +def test_one_gateway_never_retires_two_unmapped_debts(running, monkeypatch, tmp_path): + import os + import subprocess + import sys + + from hermes_cli import gateway + monkeypatch.setattr(update_cmd_windows, "_RELAUNCH_VERIFY_TIMEOUT_S", 3.0) + entry = tmp_path / "bin" / "hermes" + entry.parent.mkdir() + entry.write_text("import time\ntime.sleep(60)\n", encoding="utf-8") + argv = [sys.executable, str(entry), "gateway", "run"] + # Pre-home entries (an older updater's token): argv is all that identifies them. + debts = [{"pid": 40011, "argv": argv}, {"pid": 40012, "argv": argv}] + child = subprocess.Popen(argv, env=dict(os.environ)) + try: + monkeypatch.setattr(gateway, "find_gateway_pids", lambda all_profiles=False: [child.pid]) + token = {"profiles": {}, "unmapped": list(debts)} + with pytest.raises(RuntimeError, match="pid 40012"): + _verify_relaunched_gateways_alive(token, {}, debts) + assert token["unmapped"] == [debts[1]], "one relaunched gateway retired both debts" + finally: + child.kill() + child.wait(timeout=10) diff --git a/tests/hermes_cli/test_windows_update_restart_reconciliation.py b/tests/hermes_cli/test_windows_update_restart_reconciliation.py index a7599388c2326..f65cab97d6530 100644 --- a/tests/hermes_cli/test_windows_update_restart_reconciliation.py +++ b/tests/hermes_cli/test_windows_update_restart_reconciliation.py @@ -37,6 +37,9 @@ def _stub_post_relaunch_liveness(monkeypatch): monkeypatch.setattr( gateway_windows, "_wait_for_gateway_ready", lambda **_kw: [4242] ) + # The relaunch verifier polls every target in one loop through the same probe. + monkeypatch.setattr(gateway_windows, "_live_gateway_pids", lambda *_a, **_kw: [4242]) + monkeypatch.setattr("hermes_cli.update_cmd_windows._READY_CONFIRM_S", 0.0) monkeypatch.setattr( gateway_windows, "_write_start_attestation", lambda *_a, **_kw: None ) From 0eed8f29f1d3e39ab51acf3a2bf161931cb4d6c3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:40:37 -0700 Subject: [PATCH 55/63] fix: every checkout-writing update step binds the paused gateways' tree gate first Only the principal pull recorded its target (mark_move) before git wrote the tree. The first branch switch in _prepare_checkout_for_update (and its `checkout -B` fallback), a fork's early and late upstream sync, and the post-pull syntax rollback wrote the tree unrecorded: a torn switch or sync at an unmoved HEAD was judged against whatever refs a later fetch left, and a failed second move (upstream sync after a successful pull) had no baseline at its starting HEAD at all, so the gate admitted the torn tree. All of them now run inside _checkout_move(token, *targets). Before git runs it persists the resolved target(s): on the pause's own baseline when the step starts from the pause HEAD (it knows the edits already there), otherwise on a new baseline taken at the step's starting HEAD with its dirty digests. A record that cannot be written raises, so the step does not run. Once git returns, a per-step baseline is retired if HEAD moved or the tree is exactly as the step found it, so a clean refusal never holds the set. The upstream sync takes the context as `checkout_move`; the pull's move is extracted to _move_checkout_to so _pull_updates does not nest deeper. --- hermes_cli/update_cmd.py | 156 +++++++++++++----- hermes_cli/update_cmd_git.py | 11 +- .../test_update_pause_checkout_moves.py | 145 ++++++++++++++++ 3 files changed, 265 insertions(+), 47 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_checkout_moves.py diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 8c8c68ebe0685..2b04dfec4b6b5 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -6,7 +6,7 @@ """ import logging -from contextlib import suppress +from contextlib import contextmanager, nullcontext, suppress import os import shlex import shutil # noqa: F401 (tests patch update_cmd.shutil.*; split modules resolve it here) @@ -902,7 +902,7 @@ def _reconcile_diverged_checkout(git_cmd, branch: str, pre_pull_sha, *, target_r sys.exit(1) -def _rollback_if_pulled_syntax_error(git_cmd, pre_pull_sha, *, rollback_branch=None) -> None: +def _rollback_if_pulled_syntax_error(git_cmd, pre_pull_sha, *, rollback_branch=None, _windows_gateway_resume=None) -> None: """Post-pull syntax guard: roll back to *pre_pull_sha* and ``sys.exit(1)`` when a critical file no longer compiles (a bad admin-merge past CI must not brick the CLI).""" syntax_ok, failing_path, syntax_error = _validate_critical_files_syntax(_m().PROJECT_ROOT) @@ -926,13 +926,14 @@ def _rollback_if_pulled_syntax_error(git_cmd, pre_pull_sha, *, rollback_branch=N else: rollback_args = ["reset", "--hard", pre_pull_sha] print(f"→ Rolling back to {rollback_branch if parked else pre_pull_sha[:10]}...") - rollback_result = _git_run(git_cmd, rollback_args) - if rollback_result.returncode != 0 and parked: - # The parked branch can be unavailable (e.g. checked out in another worktree): restore - # its commit detached so the install still boots the code it ran before. - print(f" ✗ Could not check out {rollback_branch}; restoring its commit detached.") - rollback_args = ["checkout", "--detach", pre_pull_sha] + with _checkout_move(_windows_gateway_resume, pre_pull_sha): rollback_result = _git_run(git_cmd, rollback_args) + if rollback_result.returncode != 0 and parked: + # The parked branch can be unavailable (e.g. checked out in another worktree): restore + # its commit detached so the install still boots the code it ran before. + print(f" ✗ Could not check out {rollback_branch}; restoring its commit detached.") + rollback_args = ["checkout", "--detach", pre_pull_sha] + rollback_result = _git_run(git_cmd, rollback_args) if rollback_result.returncode == 0: print(" ✓ Rollback complete — your install is unchanged.") print(" Try ``hermes update`` again later once a fix lands.") @@ -960,6 +961,95 @@ def _update_movement_baseline(git_cmd, pre_pull_sha, pre_sync_sha, rollback_bran return pre_sync_sha or pre_pull_sha +def _resolved_commit(git_cmd, ref: str) -> str: + """The commit *ref* names now, or ``""`` when it names none.""" + return (_git_run(git_cmd, ["rev-parse", "-q", "--verify", f"{ref}^{{commit}}"]).stdout or "").strip() + + +@contextmanager +def _checkout_move(token, *targets): + """Run one checkout-writing git step (switch, merge, reset, upstream sync, rollback) with the + paused gateways' tree gate bound to it BEFORE git writes a file: the commit(s) it can move to, + on a baseline taken at the HEAD it starts from. Without that, a step git leaves half-written at + an unmoved HEAD is judged against whatever refs a later fetch left, or against no baseline at + all once an earlier step moved HEAD. A failed record write raises: the step must not run. + + The first step from the pause's own HEAD extends the pause's baseline (it knows the edits that + were already there). A later step gets a baseline of its own, retired once git returns with + HEAD moved or the tree exactly as the step found it, so a clean refusal never holds the set.""" + targets = sorted({t for t in targets if t}) + pause_id = (token or {}).get("pause_id") + if not pause_id or not targets: + yield + return + root = _pause_record.install_root() + baselines = token.setdefault("baselines", []) + head = _pause_record.head_sha(root) + if any(b.get("pause_id") == pause_id and b.get("pre_sha") == head for b in baselines): + for target in targets: + _pause_record.mark_move(token, target) + yield + return + # stamp_tree on a view sharing *token*'s list: the same HEAD + dirty-digest baseline, keyed per move. + move_id = f"{pause_id}@{head}" + _pause_record.stamp_tree({"pause_id": move_id, "baselines": baselines}, root) + baseline = next((b for b in baselines if b.get("pause_id") == move_id), None) + if baseline is None: # not a git checkout: no tree gate to bind + yield + return + baseline["move_targets"] = sorted({*baseline.get("move_targets", []), *targets}) + found = {key: baseline.get(key) for key in ("pre_sha", "dirty_at_pause", "dirty_digests")} + _pause_record.write({**token, "resume_needed": True}) + try: + yield + except Exception: + _settle_checkout_move(token, move_id, found, root) + raise + _settle_checkout_move(token, move_id, found, root) + + +def _settle_checkout_move(token: dict, move_id: str, found: dict, root) -> None: + """Retire a returned step's baseline when git left nothing it could have torn: HEAD moved on, + or HEAD, dirty paths and their bytes are exactly as the step found them.""" + now = _pause_record.stamp_tree({"pause_id": move_id, "baselines": []}, root)["baselines"][0] + if now["pre_sha"] != found["pre_sha"] or all(now.get(key) == found[key] for key in found): + token["baselines"] = [b for b in token["baselines"] if b.get("pause_id") != move_id] + _pause_record.write({**token, "resume_needed": True}) + + +def _move_checkout_to(git_cmd, branch, merge_ref, pre_pull_sha) -> None: + """The principal pull's one checkout move onto *merge_ref*; exits when it cannot be made.""" + # merge --ff-only the already-fetched ref instead of `git pull`, which would do a + # SECOND network fetch; identical in effect given the fresh tracking ref. + if merge_ref != f"origin/{branch}": + # Keep detached local commits reachable, too. Named branches are + # untouched by checkout --detach; an autostash protects dirty files. + _park_detached_head(git_cmd, _m().PROJECT_ROOT, branch) + _git_run(git_cmd, ["checkout", "--detach", merge_ref], check=True) + else: + merge_result = _git_run(git_cmd, ["merge", "--ff-only", merge_ref]) + if merge_result.returncode != 0: + ancestry = _git_run( + git_cmd, ["merge-base", "--is-ancestor", "HEAD", merge_ref]) + if ancestry.returncode == 1: + _reconcile_diverged_checkout( + git_cmd, branch, pre_pull_sha, target_ref=merge_ref) + else: + print("✗ Fast-forward failed; refusing to reset because history divergence was not proven.") + detail = (merge_result.stderr or merge_result.stdout or "").strip() + if detail: + print(f" {detail}") + if ancestry.returncode == 0: + print(f" HEAD is still an ancestor of {merge_ref}.") + else: + print(f" Could not verify whether HEAD is an ancestor of {merge_ref}.") + ancestry_detail = (ancestry.stderr or ancestry.stdout or "").strip() + if ancestry_detail: + print(f" {ancestry_detail}") + print(" Resolve the Git error and re-run `hermes update`; no reset was attempted.") + sys.exit(1) + + def _pull_updates( git_cmd, branch, auto_stash_ref, *, prompt_for_restore, gw_input_fn, discard_local_changes, keep_stash, target_ref=None, pre_sync_sha=None, rollback_branch=None, sync_upstream=False, assume_yes=False, @@ -987,36 +1077,8 @@ def _pull_updates( try: try: # The paused gateways' tree gate must know this move's target before git writes a file. - _pause_record.mark_move(_windows_gateway_resume, target_sha) - # merge --ff-only the already-fetched ref instead of `git pull`, which would do a - # SECOND network fetch; identical in effect given the fresh tracking ref. - if merge_ref != f"origin/{branch}": - # Keep detached local commits reachable, too. Named branches are - # untouched by checkout --detach; an autostash protects dirty files. - _park_detached_head(git_cmd, _m().PROJECT_ROOT, branch) - _git_run(git_cmd, ["checkout", "--detach", merge_ref], check=True) - else: - merge_result = _git_run(git_cmd, ["merge", "--ff-only", merge_ref]) - if merge_result.returncode != 0: - ancestry = _git_run( - git_cmd, ["merge-base", "--is-ancestor", "HEAD", merge_ref]) - if ancestry.returncode == 1: - _reconcile_diverged_checkout( - git_cmd, branch, pre_pull_sha, target_ref=merge_ref) - else: - print("✗ Fast-forward failed; refusing to reset because history divergence was not proven.") - detail = (merge_result.stderr or merge_result.stdout or "").strip() - if detail: - print(f" {detail}") - if ancestry.returncode == 0: - print(f" HEAD is still an ancestor of {merge_ref}.") - else: - print(f" Could not verify whether HEAD is an ancestor of {merge_ref}.") - ancestry_detail = (ancestry.stderr or ancestry.stdout or "").strip() - if ancestry_detail: - print(f" {ancestry_detail}") - print(" Resolve the Git error and re-run `hermes update`; no reset was attempted.") - sys.exit(1) + with _checkout_move(_windows_gateway_resume, target_sha): + _move_checkout_to(git_cmd, branch, merge_ref, pre_pull_sha) except KeyboardInterrupt: raise # Ctrl-C reached git too (same process group): the tree may be torn, keep the marker except BaseException: @@ -1030,13 +1092,15 @@ def _pull_updates( git_cmd, branch, movement_baseline, in_place_update=in_place_update, _windows_gateway_resume=_windows_gateway_resume) _m()._sync_with_upstream_if_needed( - git_cmd, _m().PROJECT_ROOT, assume_yes=assume_yes, input_fn=gw_input_fn) + git_cmd, _m().PROJECT_ROOT, assume_yes=assume_yes, input_fn=gw_input_fn, + checkout_move=lambda target: _checkout_move(_windows_gateway_resume, target)) # Refuse an unexpected branch before syntax rollback can reset its ref. _verify_head_after_pull( git_cmd, branch, movement_baseline, in_place_update=in_place_update, _windows_gateway_resume=_windows_gateway_resume) _rollback_if_pulled_syntax_error( - git_cmd, pre_sync_sha or pre_pull_sha, rollback_branch=rollback_branch) + git_cmd, pre_sync_sha or pre_pull_sha, rollback_branch=rollback_branch, + _windows_gateway_resume=_windows_gateway_resume) update_succeeded = True finally: if auto_stash_ref is not None: @@ -1149,9 +1213,12 @@ def _prepare_checkout_for_update( # Keep the running code AND its branch identity for syntax rollback. moved_from_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) rollback_branch = current_branch - track_result = _git_run(git_cmd, ["checkout", branch]) - if track_result.returncode != 0: - track_result = _git_run(git_cmd, ["checkout", "-B", branch, f"origin/{branch}"]) + # The switch (or its -B fallback) is this update's first tree write: bind both commits first. + with _checkout_move(_windows_gateway_resume, *(_resolved_commit(git_cmd, ref) for ref in ( + f"refs/heads/{branch}", f"origin/{branch}"))): + track_result = _git_run(git_cmd, ["checkout", branch]) + if track_result.returncode != 0: + track_result = _git_run(git_cmd, ["checkout", "-B", branch, f"origin/{branch}"]) if track_result.returncode != 0: # Restore the stash before bailing so the user isn't stranded. if auto_stash_ref is not None: @@ -1211,7 +1278,8 @@ def _prepare_checkout_for_update( if commit_count == 0 and is_fork and branch == "main" and not release_tag: pre_sync_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) upstream_checked = _m()._sync_with_upstream_if_needed( - git_cmd, _m().PROJECT_ROOT, assume_yes=assume_yes, input_fn=gw_input_fn) + git_cmd, _m().PROJECT_ROOT, assume_yes=assume_yes, input_fn=gw_input_fn, + checkout_move=lambda target: _checkout_move(_windows_gateway_resume, target)) post_sync_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) if pre_sync_sha and post_sync_sha and pre_sync_sha != post_sync_sha: synced_count = _count_commits_between( diff --git a/hermes_cli/update_cmd_git.py b/hermes_cli/update_cmd_git.py index dddc07304c6e9..8b5994a77cb88 100644 --- a/hermes_cli/update_cmd_git.py +++ b/hermes_cli/update_cmd_git.py @@ -6,7 +6,7 @@ """ import logging -from contextlib import suppress +from contextlib import nullcontext, suppress import subprocess import sys from datetime import datetime, timedelta, timezone @@ -363,13 +363,17 @@ def _offer_upstream_remote(git_cmd: list[str], cwd: Path, *, assume_yes: bool, i return True -def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path, *, assume_yes: bool = False, input_fn=None) -> bool: +def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path, *, assume_yes: bool = False, input_fn=None, + checkout_move=None) -> bool: """Offer to add ``upstream``, compare origin/main vs upstream/main, ff-pull when strictly behind, then push origin. Returns True only when origin/main was actually verified against upstream/main; False when the check never happened, so the caller never reports "up to date" on an origin-only compare. Fetches only upstream/main: a bare fetch drags in thousands of auto-generated branches. + ``checkout_move(target)`` is the caller's context for the one tree write (the merge): the + paused gateways' tree gate is bound to *target* before git writes a file. + See #97052. """ from hermes_cli.update_cmd import _count_commits_between, _has_upstream_remote, _no_prompt_git_kwargs, _should_skip_upstream_prompt @@ -408,7 +412,8 @@ def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path, *, assume_yes: # The fetch above already brought upstream/main: a local fast-forward (no network, so no # credential helper is started under the checkout lock fd a mutator inherits) to the # very commit counted above. - run_git(git_cmd, ["merge", "--ff-only", upstream], cwd=cwd, check=True, **_no_prompt_git_kwargs()) + with (checkout_move or (lambda _target: nullcontext()))(upstream): + run_git(git_cmd, ["merge", "--ff-only", upstream], cwd=cwd, check=True, **_no_prompt_git_kwargs()) except subprocess.CalledProcessError: print(" ✗ Failed to pull from upstream. You may need to resolve conflicts manually.") return False diff --git a/tests/hermes_cli/test_update_pause_checkout_moves.py b/tests/hermes_cli/test_update_pause_checkout_moves.py new file mode 100644 index 0000000000000..f172eb2d78f0c --- /dev/null +++ b/tests/hermes_cli/test_update_pause_checkout_moves.py @@ -0,0 +1,145 @@ +"""Every checkout-writing step binds the paused gateways' tree gate first (#132338 R3, thread 5). + +The Windows pause records what "the tree before this update" was, and the gate that lets paused +gateways start holds them while git may have left a half-written tree at an unmoved HEAD. Only the +principal pull recorded its target, so a failure in an EARLIER step (the switch onto an existing +local target branch, a fork's early upstream sync) was judged against whatever refs a later fetch +left, and a failure in a LATER step (the upstream sync after a successful pull) had no baseline at +its starting HEAD at all: both torn trees were admitted. + +Real git (local repositories and a bare "upstream"), real update steps, real durable pause record +and gate. git's mid-move failure is real too: a read-only directory, as an unprivileged user. +""" + +from __future__ import annotations + +import os +import subprocess + +import pytest + +from hermes_cli import update_cmd +from hermes_cli import update_pause_record as pause_record + +pytestmark = [ + pytest.mark.platforms("posix"), + pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores read-only dirs"), +] + + +def git(root, *args) -> str: + return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True).stdout.strip() + + +def commit(root, message, **files) -> str: + for name, text in files.items(): + path = root / name.replace("__", "/") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + git(root, "add", ".") + git(root, "commit", "-qm", message) + return git(root, "rev-parse", "HEAD") + + +@pytest.fixture +def repo(tmp_path, monkeypatch): + root = tmp_path / "checkout" + root.mkdir() + git(root, "init", "-q", "-b", "main") + git(root, "config", "user.name", "Fixture") + git(root, "config", "user.email", "fixture@example.com") + monkeypatch.setattr(update_cmd._m(), "PROJECT_ROOT", root) + monkeypatch.setattr(pause_record, "install_root", lambda: root) + locked = root / "locked" + yield root + if locked.exists(): + locked.chmod(0o755) + + +def pause(root) -> dict: + token = pause_record.stamp_tree({"resume_needed": True, "profiles": {"default": 999999}}, root) + pause_record.write(token) + return token + + +def gate(root) -> bool: + """The verdict recovery reaches: the durable record, as the next launch reads it.""" + return pause_record.tree_is_whole(pause_record.read(pause_record.record_path())["token"], root)[0] + + +def bare_upstream(root, tmp_path, tip) -> None: + remote = tmp_path / "upstream.git" + subprocess.run(["git", "clone", "-q", "--bare", str(root), str(remote)], check=True) + git(remote, "update-ref", "refs/heads/main", tip) + git(root, "remote", "add", "upstream", str(remote)) + + +def test_a_torn_first_branch_switch_stays_held_after_a_ref_only_fetch(repo): + a = commit(repo, "A", **{"a.py": "A=1\n"}) + git(repo, "checkout", "-q", "-b", "feature") # parked on a merged branch: the update switches to main + git(repo, "checkout", "-q", "main") + b = commit(repo, "B", **{"a.py": "A=2\n"}) + git(repo, "update-ref", "refs/remotes/origin/main", b) + git(repo, "checkout", "-q", "feature") + git(repo, "branch", "-q", "-D", "main") # no local main: the switch falls back to `checkout -B` + token = pause(repo) + # `checkout -B` writes the tree, then creates the branch ref: an unwritable ref directory stops + # it between the two, as a kill would (HEAD unmoved, the target's bytes already in the tree). + (repo / ".git" / "refs" / "heads").chmod(0o555) + try: + with pytest.raises(SystemExit): + update_cmd._prepare_checkout_for_update( + ["git"], "main", "feature", is_fork=False, assume_yes=True, gateway_mode=False, + gw_input_fn=None, switch_branch=False, _windows_gateway_resume=token) + finally: + (repo / ".git" / "refs" / "heads").chmod(0o755) + assert git(repo, "rev-parse", "HEAD") == a and (repo / "a.py").read_text() == "A=2\n", "fixture: not torn" + c = git(repo, "commit-tree", "-p", a, "-m", "C", git(repo, "rev-parse", f"{a}^{{tree}}")) + git(repo, "update-ref", "refs/remotes/origin/main", c) # a later fetch: refs no longer reach a.py + assert gate(repo) is False, "a torn switch was admitted once a fetch moved the refs" + + +def test_a_torn_early_upstream_sync_stays_held_after_a_ref_only_fetch(repo, tmp_path): + a = commit(repo, "A", **{"a.py": "A=1\n", "locked__z.py": "Z=1\n"}) + b = commit(repo, "B", **{"a.py": "A=2\n", "locked__z.py": "Z=2\n"}) + bare_upstream(repo, tmp_path, b) + git(repo, "reset", "-q", "--hard", a) + git(repo, "update-ref", "refs/remotes/origin/main", a) # the fork matches origin, trails upstream + token = pause(repo) + (repo / "locked").chmod(0o555) + update_cmd._prepare_checkout_for_update( + ["git"], "main", "main", is_fork=True, assume_yes=True, gateway_mode=False, + gw_input_fn=None, switch_branch=False, _windows_gateway_resume=token) + assert git(repo, "rev-parse", "HEAD") == a and (repo / "a.py").read_text() == "A=2\n", "fixture: not torn" + git(tmp_path / "upstream.git", "update-ref", "refs/heads/main", a) # upstream moves on (here: back) + git(repo, "fetch", "-q", "upstream", "+refs/heads/main:refs/remotes/upstream/main") # a later real fetch + assert gate(repo) is False, "a torn upstream sync was admitted once a fetch moved the refs" + + +@pytest.mark.parametrize("second_move", ["torn", "refused", "complete"]) +def test_a_second_move_is_gated_from_the_head_the_first_one_reached(repo, tmp_path, second_move): + commit(repo, "A", **{"a.py": "A=1\n", "locked__z.py": "Z=1\n", "notes.txt": "mine\n"}) + o = commit(repo, "O", **{"origin.py": "O=1\n"}) + # Refused: upstream also touches the user's file, which the user edits again after the update. + edits = {"notes.txt": "upstream\n"} if second_move == "refused" else {} + u = commit(repo, "U", **{"a.py": "A=2\n", "locked__z.py": "Z=2\n", "new.py": "N=1\n", **edits}) + bare_upstream(repo, tmp_path, u) + git(repo, "reset", "-q", "--hard", "HEAD~2") + git(repo, "update-ref", "refs/remotes/origin/main", o) + (repo / "notes.txt").write_text("my edit\n", encoding="utf-8") # dirty before the update: unchanged bytes pass + token = pause(repo) + if second_move == "torn": + (repo / "locked").chmod(0o555) + elif second_move == "refused": + (repo / "new.py").write_text("in the way\n", encoding="utf-8") # git refuses before writing a file + update_cmd._pull_updates( + ["git"], "main", None, prompt_for_restore=False, gw_input_fn=None, discard_local_changes=False, + keep_stash=False, sync_upstream=True, assume_yes=True, _windows_gateway_resume=token) + head = git(repo, "rev-parse", "HEAD") + if second_move == "torn": + assert head == o and (repo / "a.py").read_text() == "A=2\n", "fixture: not torn" + assert gate(repo) is False, "the failed second move's torn tree was admitted at the first move's HEAD" + else: + assert head == (o if second_move == "refused" else u) + (repo / "notes.txt").write_text("my later edit\n", encoding="utf-8") # life goes on after the update + assert gate(repo) is True, "a move that left the tree whole held the paused set" From 2e69a62ea23b91a11dff81c5700cbba4c6fc0496 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:08:31 -0700 Subject: [PATCH 56/63] fix: the paused gateways' tree gate judges a moved HEAD against the commit it reached `git checkout ` can exit 0 and move HEAD while leaving a file unwritten ("unable to unlink old ..."): HEAD and the index name the new commit, the file keeps the old bytes. `tree_is_whole` only judged baselines at an unmoved HEAD, so that torn tree was admitted and the paused gateways started on it. `_checkout_move` now judges every move the moment git returns: the paths between the commit it started from and the one HEAD reached that are changed and were not already there byte for byte (`update_pause_record.torn_by_move`) are recorded on the baseline as `landed[]`. The gate holds while any of those paths is still changed. Judging at return time, not at resume, keeps a later dependency sync or stash restore that rewrites an upstream-changed file (package-lock.json) from holding the set. A recorded move the updater died before judging is judged at resume time against what the pause saw. A step's own baseline is retired only when its move landed whole (it was retired on any HEAD move before). The moved-HEAD gate reuses tracked_changes and the dirty digests the record already carries. --- hermes_cli/update_cmd.py | 85 ++++++++------- hermes_cli/update_pause_record.py | 69 +++++++++--- .../test_update_pause_moved_head_gate.py | 100 ++++++++++++++++++ 3 files changed, 199 insertions(+), 55 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_moved_head_gate.py diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 2b04dfec4b6b5..81e559b796a13 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -967,54 +967,59 @@ def _resolved_commit(git_cmd, ref: str) -> str: @contextmanager -def _checkout_move(token, *targets): - """Run one checkout-writing git step (switch, merge, reset, upstream sync, rollback) with the - paused gateways' tree gate bound to it BEFORE git writes a file: the commit(s) it can move to, - on a baseline taken at the HEAD it starts from. Without that, a step git leaves half-written at - an unmoved HEAD is judged against whatever refs a later fetch left, or against no baseline at - all once an earlier step moved HEAD. A failed record write raises: the step must not run. +def _checkout_move(token, *targets, paths=()): + """Run one checkout-writing git step (switch, merge, reset, upstream sync, rollback, stash) with + the paused gateways' tree gate bound to it BEFORE git writes a file: the commit(s) it can move + to and the tracked *paths* it rewrites in place (stash push/apply, ``reset --hard``, + ``checkout -- ``), on a baseline taken at the HEAD it starts from. Without that, a step + git leaves half-written at an unmoved HEAD is judged against whatever refs a later fetch left, + or against no baseline at all once an earlier step moved HEAD. A failed record write raises: + the step must not run. The first step from the pause's own HEAD extends the pause's baseline (it knows the edits that - were already there). A later step gets a baseline of its own, retired once git returns with - HEAD moved or the tree exactly as the step found it, so a clean refusal never holds the set.""" + were already there). A later step gets a baseline of its own, retired once git returns with the + tree whole: HEAD moved and every file the move wrote holds the new commit, the tree exactly as + the step found it, or the step reported ``whole`` on the dict this yields (a stash restore that + applied cleanly changes the tree by design). A move is judged the moment git returns (see + ``_settle_checkout_move``).""" + step: dict = {} targets = sorted({t for t in targets if t}) pause_id = (token or {}).get("pause_id") - if not pause_id or not targets: - yield + root = _pause_record.install_root() if pause_id and (targets or paths) else None + found = _pause_record.tree_state(root) if root else None + if found is None: # nothing paused, nothing bound, or not a git checkout: no tree gate to bind + yield step return - root = _pause_record.install_root() baselines = token.setdefault("baselines", []) - head = _pause_record.head_sha(root) - if any(b.get("pause_id") == pause_id and b.get("pre_sha") == head for b in baselines): - for target in targets: - _pause_record.mark_move(token, target) - yield - return - # stamp_tree on a view sharing *token*'s list: the same HEAD + dirty-digest baseline, keyed per move. - move_id = f"{pause_id}@{head}" - _pause_record.stamp_tree({"pause_id": move_id, "baselines": baselines}, root) - baseline = next((b for b in baselines if b.get("pause_id") == move_id), None) - if baseline is None: # not a git checkout: no tree gate to bind - yield - return - baseline["move_targets"] = sorted({*baseline.get("move_targets", []), *targets}) - found = {key: baseline.get(key) for key in ("pre_sha", "dirty_at_pause", "dirty_digests")} + move_id = f"{pause_id}@{found['pre_sha']}" + baseline = next((b for b in baselines if b.get("pre_sha") == found["pre_sha"] + and b.get("pause_id") in (pause_id, move_id)), None) + if baseline is None: + baseline = {"pause_id": move_id, **found} + baselines.append(baseline) + for key, bound in (("move_targets", targets), ("move_paths", paths)): + if bound: + baseline[key] = sorted({*baseline.get(key, []), *bound}) _pause_record.write({**token, "resume_needed": True}) try: - yield - except Exception: - _settle_checkout_move(token, move_id, found, root) - raise - _settle_checkout_move(token, move_id, found, root) - - -def _settle_checkout_move(token: dict, move_id: str, found: dict, root) -> None: - """Retire a returned step's baseline when git left nothing it could have torn: HEAD moved on, - or HEAD, dirty paths and their bytes are exactly as the step found them.""" - now = _pause_record.stamp_tree({"pause_id": move_id, "baselines": []}, root)["baselines"][0] - if now["pre_sha"] != found["pre_sha"] or all(now.get(key) == found[key] for key in found): - token["baselines"] = [b for b in token["baselines"] if b.get("pause_id") != move_id] - _pause_record.write({**token, "resume_needed": True}) + yield step + finally: + _settle_checkout_move(token, baseline, found, root, step) + + +def _settle_checkout_move(token: dict, baseline: dict, found: dict, root, step: dict) -> None: + """Record what the returned step left. A moved HEAD gets its verdict now: git can move HEAD + past a file it failed to write, and only now are the move's paths untouched by later steps + (dependency syncs, stash restores). A step's own baseline is retired when the tree is whole.""" + now = _pause_record.tree_state(root) + moved = now["pre_sha"] != found["pre_sha"] + torn = _pause_record.torn_by_move(root, found, now["dirty_at_pause"]) if moved else None + if moved and torn is not None: # unknown stays unjudged: the gate judges it at resume time + baseline.setdefault("landed", {})[now["pre_sha"]] = torn + whole = torn == [] if moved else step.get("whole") or all(now[key] == found[key] for key in found) + if baseline["pause_id"] != token["pause_id"] and whole: + token["baselines"] = [b for b in token["baselines"] if b is not baseline] + _pause_record.write({**token, "resume_needed": True}) def _move_checkout_to(git_cmd, branch, merge_ref, pre_pull_sha) -> None: diff --git a/hermes_cli/update_pause_record.py b/hermes_cli/update_pause_record.py index 57690ed1974f9..3fb2c4d43d483 100644 --- a/hermes_cli/update_pause_record.py +++ b/hermes_cli/update_pause_record.py @@ -209,14 +209,21 @@ def stamp_tree(token: dict, root: Path | None = None) -> dict: root = root or install_root() pause_id = token.setdefault("pause_id", uuid.uuid4().hex) baselines = token.setdefault("baselines", []) - if (root / ".git").exists() and not any(b.get("pause_id") == pause_id for b in baselines): - dirty = tracked_changes(root) - # The bytes, not just the names: an autostashed edit and git's half-written bytes share a path. - baselines.append({"pause_id": pause_id, "pre_sha": head_sha(root), "dirty_at_pause": dirty, - "dirty_digests": {path: _digest(root / path) for path in dirty or []}}) + if not any(b.get("pause_id") == pause_id for b in baselines) and (state := tree_state(root)): + baselines.append({"pause_id": pause_id, **state}) return token +def tree_state(root: Path) -> dict | None: + """HEAD, the tracked changes and their bytes now; ``None`` when *root* is not a git checkout.""" + if not (root / ".git").exists(): + return None + dirty = tracked_changes(root) + # The bytes, not just the names: an autostashed edit and git's half-written bytes share a path. + return {"pre_sha": head_sha(root), "dirty_at_pause": dirty, + "dirty_digests": {path: _digest(root / path) for path in dirty or []}} + + def _digest(path: Path) -> str | None: try: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -291,9 +298,36 @@ def _half_written(root: Path, changes: list[str], at_head: list[dict]) -> list[s unexpected = sorted(path for path in changes if not _seen_at_pause(root, baseline, path)) if not unexpected: continue - targets = baseline.get("move_targets") - writable = _paths_between(root, targets) if targets else _paths_a_move_could_write(root) - torn = unexpected if writable is None else [path for path in unexpected if path in writable] + # move_paths: files a step rewrote toward this HEAD (stash push/apply, reset, checkout --). + targets, paths = baseline.get("move_targets"), baseline.get("move_paths") + writable = _paths_between(root, targets) if targets else set() if paths else _paths_a_move_could_write(root) + torn = unexpected if writable is None else [path for path in unexpected if path in {*writable, *(paths or ())}] + if torn: + return torn + return [] + + +def torn_by_move(root: Path, found: dict, changes: list[str] | None) -> list[str] | None: + """Paths a move from *found*'s HEAD to the current one wrote that do not hold the current + commit's bytes: tracked changes on them that were not already there, byte for byte, when the + move started. ``None`` when git cannot say.""" + writable = _paths_between(root, [found["pre_sha"]]) if found.get("pre_sha") else None + if writable is None or changes is None: + return None + return sorted(path for path in changes if path in writable and not _seen_at_pause(root, found, path)) + + +def _landed_torn(root: Path, head: str, changes: list[str], landed: list[dict]) -> list[str]: + """Tracked changes at a HEAD a recorded move reached that are not that commit's bytes. git can + move HEAD past a file it failed to write (``checkout`` exits 0 after "unable to unlink old"), + so a moved HEAD vouches for nothing by itself. The verdict the updater took the moment git + returned stands while those paths stay changed: a later dependency sync or stash restore + rewriting a file is not the move's doing. A move the updater died before judging is judged now.""" + for baseline in landed: + verdict = (baseline.get("landed") or {}).get(head) + if verdict is None: + verdict = torn_by_move(root, baseline, changes) + torn = list(changes) if verdict is None else [path for path in verdict if path in changes] if torn: return torn return [] @@ -332,18 +366,23 @@ def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]: head = head_sha(root) if head is None: return False, "the checkout HEAD is unreadable" - at_head = [b for b in token.get("baselines") or [] if b.get("pre_sha") == head] - if at_head: - # HEAD never moved for these pauses: a tracked change one of them did not see is git's - # half-written checkout. Each is judged on its own set — a killed run that moved HEAD - # elsewhere vouches for nothing here, and this run's set never certifies an older one's. + baselines = token.get("baselines") or [] + at_head = [b for b in baselines if b.get("pre_sha") == head] + # HEAD reached by a move these pauses recorded: judged against that commit's tree. + landed = [b for b in baselines if b.get("pre_sha") != head + and (head in (b.get("landed") or {}) or head in (b.get("move_targets") or []))] + if at_head or landed: + # HEAD never moved for the at_head pauses: a tracked change one of them did not see is + # git's half-written checkout. Each is judged on its own set — a killed run that moved + # HEAD elsewhere vouches for nothing here, and this run's set never certifies an older one's. changes = tracked_changes(root) if changes is None: return False, "git cannot read the checkout state" - torn = _half_written(root, changes, at_head) + torn = _half_written(root, changes, at_head) or _landed_torn(root, head, changes, landed) if torn: return False, f"the checkout has {len(torn)} file(s) git left half-written (e.g. {torn[0]})" - return True, "" + if at_head: + return True, "" if _deps_hold_resume(root): return False, "dependencies are not current for the updated code yet" return True, "" diff --git a/tests/hermes_cli/test_update_pause_moved_head_gate.py b/tests/hermes_cli/test_update_pause_moved_head_gate.py new file mode 100644 index 0000000000000..f7660beec20dc --- /dev/null +++ b/tests/hermes_cli/test_update_pause_moved_head_gate.py @@ -0,0 +1,100 @@ +"""The paused gateways' tree gate judges a MOVED HEAD too (#132338 R3 follow-up). + +``git checkout `` can exit 0 and move HEAD while leaving a file unwritten +("error: unable to unlink old ..."): the index and HEAD name the new commit, the file keeps the old +bytes. The gate used to judge only an unmoved HEAD, so that torn tree was admitted. Now the commit a +recorded move reached names the expected tree: its paths are judged the moment git returns (before +a dependency sync or stash restore rewrites one), or at resume time when the updater died first. + +Real git, real update step, real durable record and gate; git's failure is a read-only directory. +""" + +from __future__ import annotations + +import os +import subprocess + +import pytest + +from hermes_cli import update_cmd +from hermes_cli import update_pause_record as pause_record + +pytestmark = [ + pytest.mark.platforms("posix"), + pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores read-only dirs"), +] + + +def git(root, *args) -> str: + return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True).stdout.strip() + + +def commit(root, message, **files) -> str: + for name, text in files.items(): + path = root / name.replace("__", "/") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + git(root, "add", ".") + git(root, "commit", "-qm", message) + return git(root, "rev-parse", "HEAD") + + +@pytest.fixture +def parked(tmp_path, monkeypatch): + """Parked on a merged ``feature`` at A; local and origin ``main`` at B, which rewrites locked/z.py.""" + root = tmp_path / "checkout" + root.mkdir() + git(root, "init", "-q", "-b", "main") + git(root, "config", "user.name", "Fixture") + git(root, "config", "user.email", "fixture@example.com") + monkeypatch.setattr(update_cmd._m(), "PROJECT_ROOT", root) + monkeypatch.setattr(pause_record, "install_root", lambda: root) + a = commit(root, "A", **{"a.py": "A=1\n", "locked__z.py": "Z=1\n", "package-lock.json": "{}\n"}) + git(root, "branch", "-q", "feature") + b = commit(root, "B", **{"a.py": "A=2\n", "locked__z.py": "Z=2\n", "package-lock.json": "{\"v\": 2}\n"}) + git(root, "update-ref", "refs/remotes/origin/main", b) + git(root, "checkout", "-q", "feature") + yield root, a, b + (root / "locked").chmod(0o755) + + +def pause(root) -> dict: + token = pause_record.stamp_tree({"resume_needed": True, "profiles": {"default": 999999}}, root) + pause_record.write(token) + return token + + +def gate(root) -> bool: + """The verdict recovery reaches: the durable record, as the next launch reads it.""" + return pause_record.tree_is_whole(pause_record.read(pause_record.record_path())["token"], root)[0] + + +@pytest.mark.parametrize("unlink_fails", [True, False]) +def test_a_switch_that_moved_head_past_an_unwritten_file_stays_held(parked, unlink_fails): + root, _a, b = parked + token = pause(root) + if unlink_fails: + (root / "locked").chmod(0o555) + update_cmd._prepare_checkout_for_update( + ["git"], "main", "feature", is_fork=False, assume_yes=True, gateway_mode=False, + gw_input_fn=None, switch_branch=False, _windows_gateway_resume=token) + assert git(root, "rev-parse", "HEAD") == b, "fixture: the switch did not move HEAD" + assert (root / "a.py").read_text() == "A=2\n" + # Later, npm rewrites a lockfile the move also changed: not the move's doing. + (root / "package-lock.json").write_text("{\"v\": 2, \"churn\": 1}\n", encoding="utf-8") + if unlink_fails: + assert (root / "locked" / "z.py").read_text() == "Z=1\n", "fixture: git wrote the file" + assert gate(root) is False, "a moved HEAD whose tree is not its commit's was admitted" + (root / "locked").chmod(0o755) + git(root, "checkout", "--", "locked/z.py") # the user repairs it: the set may start + assert gate(root) is True, "a switch that left the tree whole held the paused set" + + +def test_a_move_the_updater_died_before_judging_is_judged_at_resume(parked): + root, _a, b = parked + token = pause(root) + pause_record.mark_move(token, b) # recorded before git ran; the updater dies after git returns + (root / "locked").chmod(0o555) + git(root, "checkout", "-q", "main") # exits 0 with "unable to unlink old 'locked/z.py'" + assert git(root, "rev-parse", "HEAD") == b and (root / "locked" / "z.py").read_text() == "Z=1\n" + assert gate(root) is False, "an unjudged move to a recorded target was admitted torn" From fc5465e3a849021dff71f320fbcbb3bad2441efd Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:15:34 -0700 Subject: [PATCH 57/63] fix: the stash and churn steps bind the paused gateways' tree gate before writing files Round 6 bound the branch switch, the upstream syncs, the pull and the rollback. Other steps that run while the Windows pause holds the gateways still rewrote tracked files unbound: - `git stash push` and its cleanup `reset --hard`; - `git stash apply` and the reset after a conflict; - the rejected-restore `reset --hard` + `clean`; - the lockfile-churn `checkout -- ` and the line-ending `checkout`. If one of them was interrupted mid-write, the gate judged the torn file only against whatever refs a fetch had left. A restore after the pull had no baseline at the moved HEAD at all. Either way the paused set was admitted. Each step now runs inside `_checkout_move`. `paths=` binds the files it rewrites in place, and `revert=True` binds every tracked change it finds, because it puts them back to HEAD. A restore binds its stash commit. The gate counts those paths as writable. A restore that applied its tracked changes in full reports `whole`, so the tree it changed by design retires its baseline instead of holding the set. Removed the "already up to date" switch-back `checkout `. It only ran when no checkout had left current_branch: in-place and release updates never switch, and a parked switch is excluded. It was a no-op that nothing gated. Helpers take a `checkout_move` binder (`_moves_for(token)`); test doubles of `_prepare_git_command` accept its new keyword. --- hermes_cli/update_cmd.py | 52 ++++--- hermes_cli/update_cmd_git.py | 19 ++- hermes_cli/update_cmd_stash.py | 70 +++++++--- tests/gateway/test_update_streaming.py | 2 +- .../test_source_channel_integration.py | 6 +- .../test_source_release_channels.py | 2 +- .../test_update_pause_step_interruptions.py | 131 ++++++++++++++++++ .../test_update_venv_holder_retirement.py | 2 +- .../hermes_cli/test_update_zip_completion.py | 6 +- 9 files changed, 235 insertions(+), 55 deletions(-) create mode 100644 tests/hermes_cli/test_update_pause_step_interruptions.py diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 81e559b796a13..b0366857461a2 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -967,11 +967,12 @@ def _resolved_commit(git_cmd, ref: str) -> str: @contextmanager -def _checkout_move(token, *targets, paths=()): +def _checkout_move(token, *targets, paths=(), revert=False): """Run one checkout-writing git step (switch, merge, reset, upstream sync, rollback, stash) with the paused gateways' tree gate bound to it BEFORE git writes a file: the commit(s) it can move to and the tracked *paths* it rewrites in place (stash push/apply, ``reset --hard``, - ``checkout -- ``), on a baseline taken at the HEAD it starts from. Without that, a step + ``checkout -- ``; *revert*: every tracked change it finds, put back to HEAD), on a + baseline taken at the HEAD it starts from. Without that, a step git leaves half-written at an unmoved HEAD is judged against whatever refs a later fetch left, or against no baseline at all once an earlier step moved HEAD. A failed record write raises: the step must not run. @@ -985,11 +986,12 @@ def _checkout_move(token, *targets, paths=()): step: dict = {} targets = sorted({t for t in targets if t}) pause_id = (token or {}).get("pause_id") - root = _pause_record.install_root() if pause_id and (targets or paths) else None + root = _pause_record.install_root() if pause_id and (targets or paths or revert) else None found = _pause_record.tree_state(root) if root else None if found is None: # nothing paused, nothing bound, or not a git checkout: no tree gate to bind yield step return + paths = {*paths, *(found["dirty_at_pause"] or [] if revert else ())} baselines = token.setdefault("baselines", []) move_id = f"{pause_id}@{found['pre_sha']}" baseline = next((b for b in baselines if b.get("pre_sha") == found["pre_sha"] @@ -1007,6 +1009,11 @@ def _checkout_move(token, *targets, paths=()): _settle_checkout_move(token, baseline, found, root, step) +def _moves_for(token): + """``_checkout_move`` bound to *token*: the ``checkout_move`` the git and stash helpers take.""" + return lambda *targets, **bound: _checkout_move(token, *targets, **bound) + + def _settle_checkout_move(token: dict, baseline: dict, found: dict, root, step: dict) -> None: """Record what the returned step left. A moved HEAD gets its verdict now: git can move HEAD past a file it failed to write, and only now are the move's paths untouched by later steps @@ -1098,7 +1105,7 @@ def _pull_updates( _windows_gateway_resume=_windows_gateway_resume) _m()._sync_with_upstream_if_needed( git_cmd, _m().PROJECT_ROOT, assume_yes=assume_yes, input_fn=gw_input_fn, - checkout_move=lambda target: _checkout_move(_windows_gateway_resume, target)) + checkout_move=_moves_for(_windows_gateway_resume)) # Refuse an unexpected branch before syntax rollback can reset its ref. _verify_head_after_pull( git_cmd, branch, movement_baseline, in_place_update=in_place_update, @@ -1123,7 +1130,7 @@ def _pull_updates( else: _m()._restore_stashed_changes( git_cmd, _m().PROJECT_ROOT, auto_stash_ref, prompt_user=prompt_for_restore, - input_fn=gw_input_fn) + input_fn=gw_input_fn, checkout_move=_moves_for(_windows_gateway_resume)) return movement_baseline @@ -1210,7 +1217,8 @@ def _prepare_checkout_for_update( print(f" ⚠ Currently on detached HEAD — switching to {branch} for update...") # Before the stash: its refs/stash would contain HEAD until it is dropped. _park_detached_head(git_cmd, _m().PROJECT_ROOT, branch) - auto_stash_ref = _m()._stash_local_changes_if_needed(git_cmd, _m().PROJECT_ROOT) + auto_stash_ref = _m()._stash_local_changes_if_needed( + git_cmd, _m().PROJECT_ROOT, checkout_move=_moves_for(_windows_gateway_resume)) moved_from_sha = None rollback_branch = None if not release_tag and not in_place_update and current_branch != branch: @@ -1228,7 +1236,8 @@ def _prepare_checkout_for_update( # Restore the stash before bailing so the user isn't stranded. if auto_stash_ref is not None: _m()._restore_stashed_changes( - git_cmd, _m().PROJECT_ROOT, auto_stash_ref, prompt_user=False, input_fn=gw_input_fn) + git_cmd, _m().PROJECT_ROOT, auto_stash_ref, prompt_user=False, input_fn=gw_input_fn, + checkout_move=_moves_for(_windows_gateway_resume)) print(f"✗ Branch '{branch}' does not exist locally or on origin.") if track_result.stderr.strip(): print(f" {track_result.stderr.strip().splitlines()[0]}") @@ -1284,7 +1293,7 @@ def _prepare_checkout_for_update( pre_sync_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) upstream_checked = _m()._sync_with_upstream_if_needed( git_cmd, _m().PROJECT_ROOT, assume_yes=assume_yes, input_fn=gw_input_fn, - checkout_move=lambda target: _checkout_move(_windows_gateway_resume, target)) + checkout_move=_moves_for(_windows_gateway_resume)) post_sync_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) if pre_sync_sha and post_sync_sha and pre_sync_sha != post_sync_sha: synced_count = _count_commits_between( @@ -1389,9 +1398,10 @@ def _record_update_initiator() -> None: _completion_receipt.record_fact("initiator", "desktop") -def _prepare_git_command() -> tuple[bool, list, bool]: +def _prepare_git_command(*, checkout_move=None) -> tuple[bool, list, bool]: """Return ``(use_zip_update, git_cmd, is_fork)``; ``sys.exit(1)`` when not a git repo - on a non-Windows host (Windows falls back to ZIP: broken git file I/O, AV, NTFS filters).""" + on a non-Windows host (Windows falls back to ZIP: broken git file I/O, AV, NTFS filters). + *checkout_move* binds the churn cleanups' file writes to the paused gateways' tree gate.""" git_dir = _m().PROJECT_ROOT / ".git" use_zip_update = not git_dir.exists() if use_zip_update and sys.platform != "win32": @@ -1416,8 +1426,8 @@ def _prepare_git_command() -> tuple[bool, list, bool]: # Before stash/branch logic: npm rewrites package-lock.json non-deterministically and # line-ending churn is machine-made dirt; both would otherwise force an autostash every update. - _discard_lockfile_churn(git_cmd, _m().PROJECT_ROOT) - _normalize_managed_eol(git_cmd, _m().PROJECT_ROOT) + _discard_lockfile_churn(git_cmd, _m().PROJECT_ROOT, checkout_move=checkout_move) + _normalize_managed_eol(git_cmd, _m().PROJECT_ROOT, checkout_move=checkout_move) origin_url = _m()._get_origin_url(git_cmd, _m().PROJECT_ROOT) is_fork = _is_fork(origin_url) @@ -1520,15 +1530,17 @@ def _finalize_receipt(status: str, debug_message: str) -> None: def _finish_already_up_to_date( - git_cmd, branch: str, current_branch: str, _plan, *, gw_input_fn, completion_request: dict) -> None: - """"Already up to date" path: restore stash/branch, repair the checkout, catch up the fleet. + git_cmd, branch: str, current_branch: str, _plan, *, gw_input_fn, completion_request: dict, + _windows_gateway_resume=None) -> None: + """"Already up to date" path: restore stash, repair the checkout, catch up the fleet. ``sys.exit(1)`` when the repair is incomplete (after gateway exit code + partial receipt).""" - # Restore stash and switch back if we moved. EXCEPTION: a parked branch verified clean + - # fully merged stays on the target — re-parking on the stale branch recreates the incident. + # A parked branch the update switched off stays on the target: re-parking on the stale branch + # recreates the incident. No other checkout left current_branch (in-place and release updates + # never switch), so there is nothing to switch back to. if _plan.auto_stash_ref is not None: _m()._restore_stashed_changes( git_cmd, _m().PROJECT_ROOT, _plan.auto_stash_ref, prompt_user=_plan.prompt_for_restore, - input_fn=gw_input_fn) + input_fn=gw_input_fn, checkout_move=_moves_for(_windows_gateway_resume)) if _plan.parked_branch_switched: if _plan.switch_block_reason.startswith("unmerged:"): _count = _plan.switch_block_reason.split(":", 1)[1] @@ -1537,8 +1549,6 @@ def _finish_already_up_to_date( f"{_count} unmerged commit(s) kept on '{current_branch}'.") else: print(f" ✓ Checkout was parked on '{current_branch}' (fully merged) — switched back to {branch}.") - elif current_branch not in {branch, "HEAD"}: - _git_run(git_cmd, ["checkout", current_branch]) if completion_request is not None: # Same code, same host obligation: an SHA-less arm would REPLACE the standing record @@ -1612,7 +1622,7 @@ def _cmd_update_impl(args, gateway_mode: bool): or _m()._desktop_dist_exists(desktop_dir) or bool(_m()._installed_desktop_apps())) - use_zip_update, git_cmd, is_fork = _prepare_git_command() + use_zip_update, git_cmd, is_fork = _prepare_git_command(checkout_move=_moves_for(_windows_gateway_resume)) completion_request = _source_completion_request( opts, _pre_update_plan, pre_update_snapshot_id, _windows_gateway_resume, @@ -1731,7 +1741,7 @@ def _cmd_update_impl(args, gateway_mode: bool): if commit_count == 0: _finish_already_up_to_date( git_cmd, branch, current_branch, _plan, gw_input_fn=gw_input_fn, - completion_request=completion_request) + completion_request=completion_request, _windows_gateway_resume=_windows_gateway_resume) return if release_sha: diff --git a/hermes_cli/update_cmd_git.py b/hermes_cli/update_cmd_git.py index 8b5994a77cb88..be48874309ad1 100644 --- a/hermes_cli/update_cmd_git.py +++ b/hermes_cli/update_cmd_git.py @@ -412,7 +412,7 @@ def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path, *, assume_yes: # The fetch above already brought upstream/main: a local fast-forward (no network, so no # credential helper is started under the checkout lock fd a mutator inherits) to the # very commit counted above. - with (checkout_move or (lambda _target: nullcontext()))(upstream): + with (checkout_move or _no_move)(upstream): run_git(git_cmd, ["merge", "--ff-only", upstream], cwd=cwd, check=True, **_no_prompt_git_kwargs()) except subprocess.CalledProcessError: print(" ✗ Failed to pull from upstream. You may need to resolve conflicts manually.") @@ -569,7 +569,12 @@ def _npm_lockfile_owners(repo_root: Path) -> set[Path]: return owners -def _discard_lockfile_churn(git_cmd, repo_root): +def _no_move(*_targets, **_bound): + """The ``checkout_move`` of a run with no paused gateways: the step runs unbound.""" + return nullcontext({}) + + +def _discard_lockfile_churn(git_cmd, repo_root, *, checkout_move=None): """Restore ``package-lock.json`` files npm rewrote non-deterministically, so the update sees a clean tree instead of autostashing every run. A lockfile is kept when a manifest it records is dirty: for the root lock that is the root or ANY workspace ``package.json`` (reverting it under a dirty ``apps/desktop`` @@ -595,11 +600,12 @@ def _discard_lockfile_churn(git_cmd, repo_root): dirty.append(path) if not dirty: return - _git_run(git_cmd, ["checkout", "--", *dirty], repo_root) + with (checkout_move or _no_move)(paths=dirty): + _git_run(git_cmd, ["checkout", "--", *dirty], repo_root) print(f"→ Discarded npm lockfile churn ({len(dirty)} file(s))") -def _normalize_managed_eol(git_cmd, repo_root): +def _normalize_managed_eol(git_cmd, repo_root, *, checkout_move=None): """Take a managed checkout off ``core.autocrlf=true`` without leaving it dirty. Git for Windows sets ``autocrlf=true`` system-wide, turning LF files CRLF and breaking ``git checkout`` @@ -644,8 +650,9 @@ def _eol_only(): return if eol_only: # Pathspec via stdin: thousands of paths exceed the Windows argv limit. - _probe_run("checkout", "--pathspec-from-file=-", "--pathspec-file-nul", "--", - input="\0".join(sorted(eol_only)), check=False) + with (checkout_move or _no_move)(paths=sorted(eol_only)): + _probe_run("checkout", "--pathspec-from-file=-", "--pathspec-file-nul", "--", + input="\0".join(sorted(eol_only)), check=False) if _eol_only(): # still dirty: pinning would only surface churn we failed to clear return print(f"→ Normalized line-ending churn ({len(eol_only)} file(s))") diff --git a/hermes_cli/update_cmd_stash.py b/hermes_cli/update_cmd_stash.py index b2164955edc00..591a32de5250e 100644 --- a/hermes_cli/update_cmd_stash.py +++ b/hermes_cli/update_cmd_stash.py @@ -83,9 +83,11 @@ def _print_first_line(text: str) -> None: print(f" {text.strip().splitlines()[0]}") -def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[str]: +def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path, *, checkout_move=None) -> Optional[str]: + """Stash local changes before the update moves the checkout; the stash commit, or None when clean. + ``checkout_move`` binds the push (and its cleanup reset) to the paused gateways' tree gate.""" global _pending_autostash - from hermes_cli.update_cmd_git import _git_run + from hermes_cli.update_cmd_git import _git_run, _no_move _pending_autostash = None status = _git_run(git_cmd, ["status", "--porcelain", "-z"], cwd, check=True) if not status.stdout.strip(): @@ -110,6 +112,17 @@ def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[st stash_name = datetime.now(timezone.utc).strftime(f"{_AUTOSTASH_NAME_PREFIX}%Y%m%d-%H%M%S") print("→ Local changes detected — stashing before update...") prev_stash = _git_run(git_cmd, ["rev-parse", "--verify", "refs/stash"], cwd).stdout.strip() + with (checkout_move or _no_move)(revert=True): # push and reset put every tracked change back to HEAD + stash_ref = _push_stash(git_cmd, cwd, stash_name, prev_stash) + _pending_autostash = ( + stash_ref, len([entry for entry in status.stdout.split("\0") if entry])) + return stash_ref + + +def _push_stash(git_cmd: list[str], cwd: Path, stash_name: str, prev_stash: str) -> str: + """``git stash push``; the stash commit. Raises when nothing was saved; resets tracked leftovers + when a push saved everything but could not clean up.""" + from hermes_cli.update_cmd_git import _git_run push = _git_run(git_cmd, ["stash", "push", "--include-untracked", "-m", stash_name], cwd) _print_nonempty(push.stdout) stash_probe = _git_run(git_cmd, ["rev-parse", "--verify", "refs/stash"], cwd) @@ -132,8 +145,6 @@ def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[st # A partially-failed push also skips cleanup of TRACKED modifications; they'd break the following # pull. Safe to reset: all is in the stash. _reset_hard(git_cmd, cwd) - _pending_autostash = ( - stash_ref, len([entry for entry in status.stdout.split("\0") if entry])) return stash_ref @@ -324,10 +335,11 @@ def _restored_python_paths(git_cmd: list[str], cwd: Path) -> tuple[str, ...] | N def _reject_unsafe_stash_restore( git_cmd: list[str], cwd: Path, stash_ref: str, preexisting_untracked: set[str], failing_target: str, - detail: str | None, + detail: str | None, *, checkout_move=None, ) -> None: """Restore the clean updated tree, preserve the stash, and abort the update.""" from hermes_cli.update_cmd import _git_untracked_paths + from hermes_cli.update_cmd_git import _no_move print() print("✗ Restored local changes made the Hermes agent unexecutable.") print(f" Health check failed: {failing_target}") @@ -340,8 +352,9 @@ def _ok(result) -> bool: current_untracked = _git_untracked_paths(git_cmd, cwd) restored_untracked = current_untracked - preexisting_untracked if current_untracked is not None else set() - reset = _git_quiet(git_cmd, ["reset", "--hard", "HEAD"], cwd) - clean = _git_quiet(git_cmd, ["clean", "-fd", "--", *sorted(restored_untracked)], cwd) if restored_untracked else None + with (checkout_move or _no_move)(revert=True): + reset = _git_quiet(git_cmd, ["reset", "--hard", "HEAD"], cwd) + clean = _git_quiet(git_cmd, ["clean", "-fd", "--", *sorted(restored_untracked)], cwd) if restored_untracked else None cleanup_ok = current_untracked is not None and _ok(reset) and (not restored_untracked or _ok(clean)) if cleanup_ok: cleanup_ok = _ok(_git_quiet(git_cmd, ["diff", "--quiet", "HEAD", "--"], cwd)) @@ -382,17 +395,21 @@ def _confirm_restore(stash_ref: str, input_fn) -> bool: return False -def _apply_stash(git_cmd: list[str], cwd: Path, stash_ref: str) -> list[str] | None: +def _apply_stash(git_cmd: list[str], cwd: Path, stash_ref: str, checkout_move=None) -> list[str] | None: """``git stash apply``; the untracked paths the update replaced (empty on a full restore), or - None (tree reset, stash kept) on conflicts or any failure other than the already-exists class.""" - from hermes_cli.update_cmd_git import _git_run + None (tree reset, stash kept) on conflicts or any failure other than the already-exists class. + ``checkout_move`` binds the apply (and a conflict's reset) to the paused gateways' tree gate.""" + from hermes_cli.update_cmd_git import _no_move print("→ Restoring local changes...") - restore = _git_run(git_cmd, ["stash", "apply", stash_ref], cwd) - unmerged = _git_run(git_cmd, ["diff", "--name-only", "--diff-filter=U"], cwd) # conflicts can exist even on rc 0 - conflicted_files = unmerged.stdout.strip() - if restore.returncode == 0 and not conflicted_files: + with (checkout_move or _no_move)(stash_ref) as step: + restore, conflicted_files = _apply_stash_step(git_cmd, cwd, stash_ref) + # Tracked changes applied in full (the already-exists class refuses only untracked files): + # the tree this changed is the restore's result, not a tear. + step["whole"] = not conflicted_files and ( + restore.returncode == 0 or _stash_apply_failed_only_on_existing_untracked(restore.stderr)) + if step["whole"] and restore.returncode == 0: return [] - if not conflicted_files and _stash_apply_failed_only_on_existing_untracked(restore.stderr): + if step["whole"]: # Tracked changes applied; git refused to overwrite untracked files that already exist. That is # harmless only when the occupant IS the stashed copy (undeletable at stash time, #70127). When # the update added its own file at that path, the stash is the only copy of the user's file. @@ -409,13 +426,26 @@ def _apply_stash(git_cmd: list[str], cwd: Path, stash_ref: str) -> list[str] | N print(f" • {f}") print("\nYour stashed changes are preserved — nothing is lost.") print(f" Stash ref: {stash_ref}") - _reset_hard(git_cmd, cwd) # conflict markers make hermes unrunnable; changes stay in the stash print("Working tree reset to clean state.") print(f"Restore your changes later with: git stash apply {stash_ref}") _record_stash_disposition("parked", stash_ref, "restore hit conflicts") return None # code update succeeded; cmd_update continues (deps, skills, gateway) +def _apply_stash_step(git_cmd: list[str], cwd: Path, stash_ref: str): + """``git stash apply`` and its conflicted paths. Any failure but the already-exists class is + reset to the clean tree here, inside the bound step: conflict markers make hermes unrunnable, + and the changes stay in the stash.""" + from hermes_cli.update_cmd_git import _git_run + restore = _git_run(git_cmd, ["stash", "apply", stash_ref], cwd) + unmerged = _git_run(git_cmd, ["diff", "--name-only", "--diff-filter=U"], cwd) # conflicts can exist even on rc 0 + conflicted_files = unmerged.stdout.strip() + if conflicted_files or (restore.returncode != 0 + and not _stash_apply_failed_only_on_existing_untracked(restore.stderr)): + _reset_hard(git_cmd, cwd) + return restore, conflicted_files + + def _drop_restored_stash(git_cmd: list[str], cwd: Path, stash_ref: str) -> None: from hermes_cli.update_cmd_git import _git_run stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) @@ -434,7 +464,8 @@ def _drop_restored_stash(git_cmd: list[str], cwd: Path, stash_ref: str) -> None: def _restore_stashed_changes( - git_cmd: list[str], cwd: Path, stash_ref: str, prompt_user: bool = False, input_fn=None, + git_cmd: list[str], cwd: Path, stash_ref: str, prompt_user: bool = False, input_fn=None, *, + checkout_move=None, ) -> bool: from hermes_cli.update_cmd import _critical_module_import_failures, _git_untracked_paths, _restored_python_paths, _validate_python_files_syntax if prompt_user and not _confirm_restore(stash_ref, input_fn): @@ -447,12 +478,13 @@ def _restore_stashed_changes( _record_stash_disposition("parked", stash_ref, "untracked baseline unknown") return False clean_import_failures = _critical_module_import_failures(cwd, report_runtime_errors=True) - replaced = _apply_stash(git_cmd, cwd, stash_ref) + replaced = _apply_stash(git_cmd, cwd, stash_ref, checkout_move) if replaced is None: return False # disposition already recorded inside _apply_stash def reject(failing_target: str, detail) -> None: - _reject_unsafe_stash_restore(git_cmd, cwd, stash_ref, preexisting_untracked, failing_target, detail) + _reject_unsafe_stash_restore( + git_cmd, cwd, stash_ref, preexisting_untracked, failing_target, detail, checkout_move=checkout_move) restored_python = _restored_python_paths(git_cmd, cwd) if restored_python is None: diff --git a/tests/gateway/test_update_streaming.py b/tests/gateway/test_update_streaming.py index 69b76b12b709f..f0193e5db6e8b 100644 --- a/tests/gateway/test_update_streaming.py +++ b/tests/gateway/test_update_streaming.py @@ -367,7 +367,7 @@ def git(*args): monkeypatch.setattr(main, "_run_pre_update_backup", lambda args: None) monkeypatch.setattr(main, "_pause_windows_gateways_for_update", lambda: None) monkeypatch.setattr("hermes_cli.update_inventory.collect_runtime_inventory", lambda: None) - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (False, ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (False, ["git"], False)) monkeypatch.setattr(update_cmd, "run_completion", lambda request: {"exit_code": 0, "receipt": None}) gateway_prompt = MagicMock(return_value="n") monkeypatch.setattr(update_cmd, "_gateway_prompt", gateway_prompt) diff --git a/tests/hermes_cli/test_source_channel_integration.py b/tests/hermes_cli/test_source_channel_integration.py index 10d8704e1df84..adbde5a722e33 100644 --- a/tests/hermes_cli/test_source_channel_integration.py +++ b/tests/hermes_cli/test_source_channel_integration.py @@ -52,7 +52,7 @@ def source(tmp_path, monkeypatch): monkeypatch.setattr(update_cmd, "_begin_update_receipt_and_plan", lambda *_: None) monkeypatch.setattr(main, "_run_pre_update_backup", lambda *_: None) monkeypatch.setattr(main, "_pause_windows_gateways_for_update", lambda: None) - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (False, ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (False, ["git"], False)) return SimpleNamespace(home=home, origin=origin, root=checkout, commits=commits, parser=parser) @@ -350,7 +350,7 @@ def test_retirement_refuses_to_downgrade_newer_source( git(source.root.parent, "clone", "--depth=1", source.origin.as_uri(), str(source.root)) if transport == "zip": (source.root / "pyproject.toml").write_text('[project]\nversion = "1.2.2"\n') - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (True, ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (True, ["git"], False)) original = deepcopy(saved(source)) completed = [] monkeypatch.setattr(update_cmd, "_complete_source_update", lambda request: completed.append(request)) @@ -380,7 +380,7 @@ def download(url, filename): urls.append(url) shutil.copyfile(archive, filename) monkeypatch.setattr(urllib.request, "urlretrieve", download) - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (True, ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (True, ["git"], False)) completed = [] monkeypatch.setattr(update_cmd, "_complete_source_update", lambda req: completed.append(deepcopy(req))) if dirty: diff --git a/tests/hermes_cli/test_source_release_channels.py b/tests/hermes_cli/test_source_release_channels.py index 543df7b53d811..caa6c2342d7f7 100644 --- a/tests/hermes_cli/test_source_release_channels.py +++ b/tests/hermes_cli/test_source_release_channels.py @@ -146,7 +146,7 @@ def test_source_check_and_apply_land_on_selected_release(releases, monkeypatch, monkeypatch.setattr(update_cmd, "_begin_update_receipt_and_plan", lambda *_: None) monkeypatch.setattr(main, "_run_pre_update_backup", lambda *_: None) monkeypatch.setattr(main, "_pause_windows_gateways_for_update", lambda: None) - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (False, ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (False, ["git"], False)) applied = [] monkeypatch.setattr(update_cmd, "_complete_source_update", lambda request: applied.append(request)) args = SimpleNamespace(branch=None, channel=None, force_venv=True) diff --git a/tests/hermes_cli/test_update_pause_step_interruptions.py b/tests/hermes_cli/test_update_pause_step_interruptions.py new file mode 100644 index 0000000000000..815862bbdf262 --- /dev/null +++ b/tests/hermes_cli/test_update_pause_step_interruptions.py @@ -0,0 +1,131 @@ +"""The stash and churn steps bind the paused gateways' tree gate too (#132338 R3 follow-up). + +``git stash push`` / ``stash apply`` and the lockfile-churn ``checkout -- `` rewrite tracked +files while the Windows pause holds the gateways. Each now records, before git runs, the HEAD it +starts from, the tree's dirty bytes and what it may write; a step interrupted mid-write keeps the +set held instead of being judged against unrelated refs (or, at a moved HEAD, against nothing). + +Real git, real update steps, real durable record and gate. The interruption is real too: a git +wrapper copies the durable record as git starts (what was on disk before the write), runs real git, +leaves one file half-written and sends the updater SIGINT, as a terminal's Ctrl-C reaches both. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess + +import pytest + +from hermes_cli import update_cmd, update_cmd_git, update_cmd_stash +from hermes_cli import update_pause_record as pause_record + +pytestmark = pytest.mark.platforms("posix") + + +def git(root, *args) -> str: + return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True).stdout.strip() + + +def commit(root, message, **files) -> str: + for name, text in files.items(): + (root / name).write_text(text, encoding="utf-8") + git(root, "add", ".") + git(root, "commit", "-qm", message) + return git(root, "rev-parse", "HEAD") + + +@pytest.fixture +def repo(tmp_path, monkeypatch): + root = tmp_path / "checkout" + root.mkdir() + git(root, "init", "-q", "-b", "main") + git(root, "config", "user.name", "Fixture") + git(root, "config", "user.email", "fixture@example.com") + monkeypatch.setattr(update_cmd._m(), "PROJECT_ROOT", root) + monkeypatch.setattr(pause_record, "install_root", lambda: root) + a = commit(root, "A", **{"a.py": "A=1\n", "mine.py": "M=1\n", "package-lock.json": "{}\n"}) + t = commit(root, "T", **{"a.py": "A=2\n"}) + git(root, "reset", "-q", "--hard", a) + # A fetched ref that never touches the files these steps write: what the gate fell back to. + git(root, "update-ref", "refs/remotes/origin/main", t) + return root, a, t + + +def interrupting_git(tmp_path, match: str, tear) -> list[str]: + """A git that, on the *match* subcommand, snapshots the durable record, runs real git, leaves + *tear* half-written and interrupts the updater.""" + wrapper = tmp_path / "git-interrupted" + wrapper.write_text( + "#!/bin/sh\n" + f'case " $* " in *" {match} "*) cp "{pause_record.record_path()}" "{tmp_path / "at-start.json"}"\n' + f' "{shutil.which("git")}" "$@"\n' + f" printf 'M=' > \"{tear}\"; kill -INT $PPID; sleep 5; exit 130;;\n" + f'esac\nexec "{shutil.which("git")}" "$@"\n', + encoding="utf-8") + wrapper.chmod(0o755) + return [str(wrapper)] + + +def pause(root) -> dict: + token = pause_record.stamp_tree({"resume_needed": True, "profiles": {"default": 999999}}, root) + pause_record.write(token) + return token + + +def gate(root) -> bool: + return pause_record.tree_is_whole(pause_record.read(pause_record.record_path())["token"], root)[0] + + +def at_start(tmp_path, head) -> dict: + """The baseline at *head* in the record as it was on disk the moment git started.""" + token = pause_record.read(tmp_path / "at-start.json")["token"] + return next(b for b in token["baselines"] if b["pre_sha"] == head) + + +def test_an_interrupted_stash_push_stays_held(repo, tmp_path): + root, a, _t = repo + (root / "mine.py").write_text("M=2 # my edit\n", encoding="utf-8") + token = pause(root) + with pytest.raises(KeyboardInterrupt): + update_cmd_stash._stash_local_changes_if_needed( + interrupting_git(tmp_path, "stash push", root / "mine.py"), root, + checkout_move=update_cmd._moves_for(token)) + started = at_start(tmp_path, a) + assert "mine.py" in started["move_paths"] and started["dirty_at_pause"] == ["mine.py"] + assert gate(root) is False, "a stash push interrupted mid-write was admitted" + + +def test_an_interrupted_lockfile_churn_cleanup_stays_held(repo, tmp_path): + root, a, _t = repo + (root / "package-lock.json").write_text('{"npm": "rewrote me"}\n', encoding="utf-8") + token = pause(root) + with pytest.raises(KeyboardInterrupt): + update_cmd_git._discard_lockfile_churn( + interrupting_git(tmp_path, "checkout --", root / "package-lock.json"), root, + checkout_move=update_cmd._moves_for(token)) + assert at_start(tmp_path, a)["move_paths"] == ["package-lock.json"] + assert gate(root) is False, "a lockfile cleanup interrupted mid-write was admitted" + + +@pytest.mark.parametrize("interrupted", [True, False]) +def test_a_stash_restore_at_the_moved_head_is_gated_from_that_head(repo, tmp_path, interrupted): + root, a, t = repo + (root / "mine.py").write_text("M=2 # my edit\n", encoding="utf-8") + token = pause(root) + moves = update_cmd._moves_for(token) + stash = update_cmd_stash._stash_local_changes_if_needed(["git"], root, checkout_move=moves) + with update_cmd._checkout_move(token, t): + git(root, "merge", "-q", "--ff-only", t) # the pull lands whole + git_cmd = interrupting_git(tmp_path, "stash apply", root / "mine.py") if interrupted else ["git"] + if interrupted: + with pytest.raises(KeyboardInterrupt): + update_cmd_stash._restore_stashed_changes(git_cmd, root, stash, checkout_move=moves) + started = at_start(tmp_path, t) + assert started["move_targets"] == [stash] and started["dirty_at_pause"] == [] + assert gate(root) is False, "a stash restore interrupted mid-write at the moved HEAD was admitted" + else: + assert update_cmd_stash._restore_stashed_changes(git_cmd, root, stash, checkout_move=moves) + assert (root / "mine.py").read_text() == "M=2 # my edit\n" + assert gate(root) is True, "a stash restore that applied cleanly held the paused set" diff --git a/tests/hermes_cli/test_update_venv_holder_retirement.py b/tests/hermes_cli/test_update_venv_holder_retirement.py index 431fc1461c10a..6cc84e480e073 100644 --- a/tests/hermes_cli/test_update_venv_holder_retirement.py +++ b/tests/hermes_cli/test_update_venv_holder_retirement.py @@ -100,7 +100,7 @@ class ReachedCheckout(BaseException): monkeypatch.setattr(main, "_is_windows", forbidden) monkeypatch.setattr(os, "kill", forbidden) - def prepare_checkout(): + def prepare_checkout(**_): reached.append("checkout") raise ReachedCheckout diff --git a/tests/hermes_cli/test_update_zip_completion.py b/tests/hermes_cli/test_update_zip_completion.py index 514179e00add9..0d9e699a032a6 100644 --- a/tests/hermes_cli/test_update_zip_completion.py +++ b/tests/hermes_cli/test_update_zip_completion.py @@ -123,7 +123,7 @@ def test_zip_command_migrates_profiles_recovers_snapshot_and_verifies_fleet( zip_update, monkeypatch, route, gateway_mode, ): state = zip_update - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (route == "direct", ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (route == "direct", ["git"], False)) monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) def fail_fetch(*args, **kwargs): @@ -193,7 +193,7 @@ def test_zip_failure_recovers_pause_without_completion_mutations(zip_update, mon old_project = (state.root / "pyproject.toml").read_bytes() original_tree = {p.relative_to(state.root): p.read_bytes() for p in state.root.rglob("*") if p.is_file()} - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (route == "direct", ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (route == "direct", ["git"], False)) monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) monkeypatch.setattr(update_cmd, "_should_zip_fallback_on_update_error", lambda exc: True) @@ -423,7 +423,7 @@ def test_installed_app_without_a_checkout_build_is_still_rebuilt(zip_update, mon built = [] monkeypatch.setattr("hermes_cli.source_build.build_update_products", lambda selected, *, desktop: built.append(desktop)) - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (True, ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (True, ["git"], False)) monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), False) From 387789d7d491c55a331f415023ffce9cbde3bf72 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:05:28 -0700 Subject: [PATCH 58/63] fix: a second stop signal never deletes a stamped accepted stop request When the gateway's home refuses a new .accepted receipt, mark_stop_accepted stamps the kept request itself with accepted=true; that stamp is then the only durable trace that this draining incarnation accepted the update's stop. _read_live_pid_marker unlinked every expired marker, so a second signal past the request TTL (another Ctrl+C / SIGTERM, which calls the consumer again) or a planned-stop watcher probe in the same home deleted it, and recovery then retired the restart debt of a gateway that was still draining. An expired stamped request still matches nobody (no PID-reuse grief), but the TTL cleanup no longer unlinks it. --- gateway/status.py | 8 +- .../test_planned_stop_accepted_debt.py | 82 +++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 tests/gateway/test_planned_stop_accepted_debt.py diff --git a/gateway/status.py b/gateway/status.py index ed4e6ab505b8b..d0ec62ab887f4 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -1673,13 +1673,17 @@ def _marker_is_stale(written_at: str, ttl_s: int) -> bool: def _read_live_pid_marker(path: Path, ttl_s: int) -> Optional[tuple[dict[str, Any], int, Any]]: """``(record, target_pid, target_start_time)`` for a usable marker, else None. Malformed/expired - markers can never match anyone, so they are unlinked here (must not wedge a new instance).""" + markers can never match anyone, so they are unlinked here (must not wedge a new instance) -- + except a request stamped ``accepted`` (``update_pause_record.mark_stop_accepted``): expired, it + matches nobody, but it can be the only trace that a still-draining gateway accepted an update's + stop, so it stays until that pause is settled.""" record = _read_json_file(path) if not record: return None target_pid = _pid_from_record(record, "target_pid") if target_pid is None or _marker_is_stale(record.get("written_at") or "", ttl_s): - _unlink_quietly(path) + if record.get("accepted") is not True: + _unlink_quietly(path) return None return record, target_pid, record.get("target_start_time") diff --git a/tests/gateway/test_planned_stop_accepted_debt.py b/tests/gateway/test_planned_stop_accepted_debt.py new file mode 100644 index 0000000000000..7315ab811f038 --- /dev/null +++ b/tests/gateway/test_planned_stop_accepted_debt.py @@ -0,0 +1,82 @@ +"""The gateway's planned-stop consumer and the accepted-stop evidence a Windows update pause relies on. + +The gateway is this test process; every file is real and every refusal a real one (a read-only +directory standing in for a Windows ACL or sharing refusal). Recovery is the real +``update_pause_record.recover``. +""" + +from __future__ import annotations + +import json +import os +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from gateway import status +from hermes_cli import update_pause_record as pause_record + +pytestmark = [ + pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes stand in for Windows refusals"), + pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores file modes"), +] + + +def _home(tmp_path: Path, monkeypatch) -> tuple[Path, Path]: + root, home = tmp_path / "root", tmp_path / "root" / "profiles" / "p" + home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(home)) + return root, home + + +def _accepted_unrecorded(tmp_path: Path, monkeypatch, *, home_writable: bool) -> tuple[Path, dict]: + """A paused gateway (this process) accepts its update's stop request while the record refuses the + checkpoint; the request then ages past its TTL and the updater dies. The acceptance survives only + beside the request: the ``.accepted`` receipt, or (home refusing new files) the stamped request.""" + root, home = _home(tmp_path, monkeypatch) + pid = os.getpid() + marker = status._get_planned_stop_marker_path() + token = pause_record.record_pause({"resume_needed": True, "profiles": {"p": pid}, + "identities": {str(pid): pause_record.identity(pid)["ct"]}}, None, []) + pause_record.mark_stop_requested(token, [pid], markers={pid: marker}) + assert status.write_planned_stop_marker(pid) + root.chmod(0o555) + if not home_writable: + home.chmod(0o555) + try: + assert status.consume_planned_stop_marker_for_self() is True, "premise: the stop was accepted" + finally: + home.chmod(0o755) + root.chmod(0o755) + body = json.loads(marker.read_text(encoding="utf-8")) + assert body.get("accepted") is (None if home_writable else True), "premise: receipt vs stamped request" + body["written_at"] = (datetime.now(timezone.utc) - timedelta(seconds=120)).isoformat() + marker.write_text(json.dumps(body), encoding="utf-8") + saved = pause_record.read()["token"] + assert saved["stop_sent"] == [], "premise: no checkpoint landed in the record" + pause_record.write(saved, owner=pause_record.UNOWNED) + return marker, saved + + +def _owed(pause_id: str) -> dict | None: + pause_record.recover(["status"]) + path = pause_record.record_path() + bodies = [pause_record.read(src) for src in (path, *pause_record._claims(path)) if src.exists()] + owed = [b["token"]["profiles"] for b in bodies if b["token"]["pause_id"] == pause_id] + return owed[0] if owed else None + + +@pytest.mark.parametrize("second_look", ["second_signal", "watcher_probe"]) +def test_a_stamped_accepted_request_outlives_its_ttl_while_the_pause_is_owed(tmp_path, monkeypatch, second_look): + """A second shutdown signal (another Ctrl+C) or a watcher probe after the request's TTL must not + delete the stamped request: it is the only durable trace that this draining gateway accepted the + stop, and recovery would otherwise retire its restart debt while it still drains.""" + marker, saved = _accepted_unrecorded(tmp_path, monkeypatch, home_writable=False) + if second_look == "second_signal": + assert status.consume_planned_stop_marker_for_self() is False, "an expired request matches nobody" + else: + assert status.planned_stop_marker_targets_self() is False + assert marker.exists(), f"the {second_look} deleted the stamped accepted request" + assert _owed(saved["pause_id"]) == {"p": os.getpid()}, "an accepted stop lost its restart debt" From f6c7199567e95c26516e55bbf672266bedbdd297 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:09:16 -0700 Subject: [PATCH 59/63] fix: the planned-stop consumer takes the pause lock only for a pause on disk and drops settled receipts consume_planned_stop_marker_for_self took .hermes-update-paused-gateways.lock on every platform for every stop (up to 2 s when busy), creating the lock file in the root home, and on a busy lock fell back to keeping the request and writing a .accepted receipt that nothing ever removed. The lock only protects the checkpoint into a Windows update pause record; with no record or claim of this checkout on disk (every non-Windows host, or a pause recovery/the updater already retired) there is nothing to checkpoint. The consumer now holds the lock only when a pause is on disk. The checkpoint hook re-checks after the request was read, so a pause that appeared after the first look (an update writes its record before its request) is still checkpointed under the lock. With no pause on disk, the consumer removes the .accepted receipt and an expired stamped request an earlier drain left: their debt is settled. While any pause is on disk both stay (never earlier). test_a_busy_pause_mutex_never_turns_a_planned_stop_into_an_unplanned_one now puts a pause record on disk first: without one, keeping the request on a busy lock was the leftover this fixes; with one, its assertions (planned, request retained, no block) are unchanged. --- gateway/status.py | 44 +++++++++++++---- .../test_planned_stop_accepted_debt.py | 47 +++++++++++++++++++ .../test_update_pause_boundaries.py | 1 + 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/gateway/status.py b/gateway/status.py index d0ec62ab887f4..3275cf5cfab66 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -1682,7 +1682,7 @@ def _read_live_pid_marker(path: Path, ttl_s: int) -> Optional[tuple[dict[str, An return None target_pid = _pid_from_record(record, "target_pid") if target_pid is None or _marker_is_stale(record.get("written_at") or "", ttl_s): - if record.get("accepted") is not True: + if record.get("accepted") is not True or not _update_pause_on_disk(): _unlink_quietly(path) return None return record, target_pid, record.get("target_start_time") @@ -1961,17 +1961,45 @@ def write_planned_stop_marker(target_pid: int) -> bool: _PLANNED_STOP_MUTEX_WAIT_S = 2.0 +def _update_pause_on_disk() -> bool: + """A Windows update pause of this checkout is on disk (its record or a recovery's claim), or the + record directory cannot be listed. Only then does a consume need the pause lock and a checkpoint, + and only then can accepted-stop evidence beside a request still be owed: with none, every pause + was settled by recovery or the updater (or never made: every non-Windows host).""" + from hermes_cli import update_pause_record + record = update_pause_record.record_path() + try: + with os.scandir(record.parent) as entries: + return any(e.name == record.name or (e.name.startswith(record.name + ".") and e.name.endswith(".claim")) + for e in entries) + except FileNotFoundError: + return False + except OSError: + return True + + +def _checkpoint_planned_stop(path: Path, record: dict[str, Any]) -> None: + """``on_consume``, decided after the request was read: an update writes its record before its + request, so a pause that appeared since the consumer first looked is checkpointed too.""" + from hermes_cli import update_pause_record + if _update_pause_on_disk(): + with update_pause_record._mutex(_PLANNED_STOP_MUTEX_WAIT_S): + update_pause_record.mark_stop_consumed(path, record) + + def consume_planned_stop_marker_for_self() -> bool: """Return True when the current process is being intentionally stopped.""" from hermes_cli import update_pause_record - # Recovery must see either the request or its checkpoint, never the gap - # between validating/consuming the marker and scheduling asynchronous stop. + path = _get_planned_stop_marker_path() try: - with update_pause_record._mutex(_PLANNED_STOP_MUTEX_WAIT_S): - return _consume_pid_marker_for_self( - _get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S, - on_consume=update_pause_record.mark_stop_consumed, - ) + paused = _update_pause_on_disk() + if not paused: # an earlier drain's receipt: its debt is settled + _unlink_quietly(update_pause_record._accepted_path(path)) + # With a pause on disk, recovery must see either the request or its checkpoint, never the + # gap between validating/consuming the marker and scheduling asynchronous stop. + with update_pause_record._mutex(_PLANNED_STOP_MUTEX_WAIT_S) if paused else contextlib.nullcontext(): + return _consume_pid_marker_for_self(path, ttl_s=_PLANNED_STOP_MARKER_TTL_S, + on_consume=_checkpoint_planned_stop) except OSError as exc: # The pause bookkeeping (a busy mutex, a checkpoint that cannot be written) never decides # whether this stop was planned: classify without consuming, and leave a receipt beside the diff --git a/tests/gateway/test_planned_stop_accepted_debt.py b/tests/gateway/test_planned_stop_accepted_debt.py index 7315ab811f038..aa7672ab5977a 100644 --- a/tests/gateway/test_planned_stop_accepted_debt.py +++ b/tests/gateway/test_planned_stop_accepted_debt.py @@ -10,6 +10,8 @@ import json import os import sys +import threading +import time from datetime import datetime, timedelta, timezone from pathlib import Path @@ -80,3 +82,48 @@ def test_a_stamped_accepted_request_outlives_its_ttl_while_the_pause_is_owed(tmp assert status.planned_stop_marker_targets_self() is False assert marker.exists(), f"the {second_look} deleted the stamped accepted request" assert _owed(saved["pause_id"]) == {"p": os.getpid()}, "an accepted stop lost its restart debt" + + +@pytest.mark.parametrize("home_writable", [True, False], ids=["receipt", "stamped_request"]) +def test_accepted_stop_evidence_is_removed_once_its_pause_is_settled_and_never_before( + tmp_path, monkeypatch, home_writable): + """``.accepted`` receipts and stamped requests are cleaned up by the next consume once no pause of + this checkout is on disk (recovery or the updater retired it), and kept while one still is.""" + marker, saved = _accepted_unrecorded(tmp_path, monkeypatch, home_writable=home_writable) + evidence = pause_record._accepted_path(marker) if home_writable else marker + status.consume_planned_stop_marker_for_self() + assert evidence.exists(), "accepted-stop evidence removed while its pause is still owed" + pause_record.discharge(saved) + assert not pause_record.record_path().exists(), "premise: the pause is settled" + status.consume_planned_stop_marker_for_self() + assert not pause_record._accepted_path(marker).exists(), "a settled receipt was left behind" + assert not marker.exists(), "a settled stamped request was left behind" + + +def test_with_no_pause_on_disk_the_consumer_never_waits_on_the_pause_lock(tmp_path, monkeypatch): + """No pause record (every non-Windows host, or a settled pause): a plain ``hermes gateway stop`` is + consumed without the pause lock, so another holder costs it no wait and leaves no request or + ``.accepted`` receipt behind.""" + _home(tmp_path, monkeypatch) + held, release = threading.Event(), threading.Event() + + def holder(): + with pause_record._mutex(): + held.set() + release.wait(10) + + thread = threading.Thread(target=holder) + thread.start() + try: + assert held.wait(5), "premise: another thread holds the pause lock" + marker = status._get_planned_stop_marker_path() + assert status.write_planned_stop_marker(os.getpid()) + started = time.monotonic() + assert status.consume_planned_stop_marker_for_self() is True + elapsed = time.monotonic() - started + finally: + release.set() + thread.join() + assert elapsed < 1.0, f"the consumer waited {elapsed:.2f}s on a pause lock no pause needed" + assert not marker.exists(), "the consumed request was left on disk" + assert not pause_record._accepted_path(marker).exists(), "an .accepted receipt was written with no pause" diff --git a/tests/hermes_cli/test_update_pause_boundaries.py b/tests/hermes_cli/test_update_pause_boundaries.py index 0da9e794c36f5..9dae296ca045c 100644 --- a/tests/hermes_cli/test_update_pause_boundaries.py +++ b/tests/hermes_cli/test_update_pause_boundaries.py @@ -169,6 +169,7 @@ def test_failed_consumer_checkpoint_keeps_request_and_user_stop_is_not_update_de def test_a_busy_pause_mutex_never_turns_a_planned_stop_into_an_unplanned_one(cell): base, _, launch = cell holder = launch(''' + r.write({"pause_id": "busy", "resume_needed": True}) # a pause on disk: the consume needs the mutex with r._mutex(): Path(sys.argv[2]).touch() while not Path(sys.argv[3]).exists(): time.sleep(.05) From 9b18b42d6c670ce4949900d60368c15381f980f8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:21:51 -0700 Subject: [PATCH 60/63] test: the cold-start lifecycle fixture answers the probe the relaunch verifier polls The post-update relaunch verifier now polls every target in one loop through `gateway_windows._live_gateway_pids` (d243ecf107f); these Windows-only tests still stubbed only `_wait_for_gateway_ready`, so the relaunched `beta` was never seen and resume raised "relaunch ... was not verified alive: beta". The same probe also answers "is this profile already running?" before a cold start, so the stub follows the lifecycle: a home answers (with a fresh PID, through the verifier's own PID filter) only once its profile was relaunched via `launch_detached_profile_gateway_restart`. A profile that has not started yet still reads as down and is cold-started, and one profile never vouches for another. Assertions are unchanged. --- ...st_windows_gateway_cold_start_desktop_lifecycle.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/hermes_cli/test_windows_gateway_cold_start_desktop_lifecycle.py b/tests/hermes_cli/test_windows_gateway_cold_start_desktop_lifecycle.py index 7d30978192c90..ff20add1a13c3 100644 --- a/tests/hermes_cli/test_windows_gateway_cold_start_desktop_lifecycle.py +++ b/tests/hermes_cli/test_windows_gateway_cold_start_desktop_lifecycle.py @@ -112,9 +112,18 @@ def _running_beta_pause_fixture(monkeypatch, tmp_path): monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda name: homes[name]) # Resume side. monkeypatch.setattr(cli_main, "_refresh_windows_gateway_launchers", lambda: None) - monkeypatch.setattr(hermes_gateway, "launch_detached_profile_gateway_restart", lambda p, o: True) + relaunched: set = set() + monkeypatch.setattr(hermes_gateway, "launch_detached_profile_gateway_restart", + lambda p, o: relaunched.add(p) or True) ready_probes: list = [] monkeypatch.setattr(gateway_windows, "_wait_for_gateway_ready", lambda *a, **k: ready_probes.append(k) or [4242]) + # The relaunch verifier polls this probe: a home answers only once its profile was relaunched + # (a fresh PID), so a not-yet-started profile still reads as down and is cold-started. + by_home = {str(h): n for n, h in homes.items()} + monkeypatch.setattr(gateway_windows, "_live_gateway_pids", + lambda home=None, pid_filter=None, **_k: (pid_filter or list)( + [888] if by_home.get(str(home)) in relaunched else [])) + monkeypatch.setattr(update_cmd_windows, "_READY_CONFIRM_S", 0.0) homes["_ready_probes"] = ready_probes return homes From 8495d7d90300a7cfebcadc08edfecf0287fedbe2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 6 Oct 2026 04:48:00 -0700 Subject: [PATCH 61/63] test(update): the post-swap ZIP tests' git-command fakes accept checkout_move #132386 added three ZIP tests that replace update_cmd._prepare_git_command with a no-argument lambda. This branch calls it as _prepare_git_command(checkout_move=...), so after the rebase onto main each fake died with "unexpected keyword argument 'checkout_move'". They now take **_ like every other fake of that function in the tree; the assertions are unchanged. --- tests/hermes_cli/test_update_zip_completion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/hermes_cli/test_update_zip_completion.py b/tests/hermes_cli/test_update_zip_completion.py index 0d9e699a032a6..f9ecb7d98e343 100644 --- a/tests/hermes_cli/test_update_zip_completion.py +++ b/tests/hermes_cli/test_update_zip_completion.py @@ -253,7 +253,7 @@ def test_zip_build_failure_after_swap_is_a_followup(zip_update, monkeypatch, rou import pm state = zip_update - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (route == "direct", ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (route == "direct", ["git"], False)) monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) monkeypatch.setattr(update_cmd, "_should_zip_fallback_on_update_error", lambda exc: True) @@ -285,7 +285,7 @@ def test_zip_dependency_sync_failure_after_swap_is_a_followup(zip_update, monkey from hermes_cli.venv_sync import completion_pending_path state = zip_update - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (route == "direct", ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (route == "direct", ["git"], False)) monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) monkeypatch.setattr(update_cmd, "_should_zip_fallback_on_update_error", lambda exc: True) @@ -324,7 +324,7 @@ def test_ctrl_c_after_the_swap_reports_the_new_code_not_a_failure(zip_update, mo # The tree is new, so the run is ``interrupted`` (never "failed" / "still on the previous # version") and the message says what is owed; a non-zero exit for an interrupt is fine. state = zip_update - monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (True, ["git"], False)) + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda **_: (True, ["git"], False)) monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) def interrupted(request): From c8264772db0a87b7b3fbc4576716d9bb80b6472d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 6 Oct 2026 05:02:30 -0700 Subject: [PATCH 62/63] refactor(update): the startup half of a previous update runs from main_install_repair On main, #132365 moved container-exec routing out of main.py, which lowered main.py's code-health line cap to its new size (3643). This branch's paused-gateway recovery added four lines to main() and pushed the file to 3647, so the code-health check refused it. The owed-fleet-restart warning and the paused-gateway recovery move together into main_install_repair._recover_update_debts_on_startup, next to the Windows quarantine sweep that already lives there; main() calls it at the same point, so startup order and behaviour are unchanged. Both steps stay late-imported, so tests patching update_cmd_fleet._warn_pending_fleet_restart_on_startup or update_pause_record.recover still intercept them. The fleet warning's silent `except Exception: pass` now logs at debug with the traceback, which is what the health check asks of a catch-all boundary in new code; a failure there still never blocks the launch. --- hermes_cli/main.py | 16 ++-------------- hermes_cli/main_install_repair.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index dcb4ee27f8b6f..57d85dcce2e9c 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -863,7 +863,7 @@ def _apply_profile_override() -> None: _windows_shim_in_process_chain, _write_web_ui_build_stamp, ) -from hermes_cli.main_install_repair import _cleanup_quarantined_exes +from hermes_cli.main_install_repair import _cleanup_quarantined_exes, _recover_update_debts_on_startup from hermes_cli.main_install_repair import ( # frozen updater surface: update_cmd*.py resolve these via _m() _UPDATE_REEXEC_ENV, _clear_lazy_refresh_incomplete_marker, @@ -3553,19 +3553,7 @@ def main(): # process resolves fresh source against old bytecode. Never raises. _sweep_stale_bytecode_if_checkout_changed() - # Dependency recovery already ran before imports. Report any fleet restart - # still owed by a previous update without restarting services here. - if "update" not in sys.argv[1:]: - try: - from hermes_cli.update_cmd_fleet import _warn_pending_fleet_restart_on_startup - - _warn_pending_fleet_restart_on_startup() - except Exception: - pass - from hermes_cli.update_pause_record import recover as _recover_paused_gateways - # Unconditional: recover() itself skips the PARSED `update`/`gateway run` commands; a raw argv - # value equal to "update" (`--resume update`) is no reason to strand paused gateways. - _recover_paused_gateways() # a killed `hermes update` left gateways paused (Windows) + _recover_update_debts_on_startup() # owed fleet restarts, gateways a killed update paused if _first_positional_argv() != "update": from hermes_cli.boot_bootstrap import maybe_run_boot_bootstrap diff --git a/hermes_cli/main_install_repair.py b/hermes_cli/main_install_repair.py index 676c1607572f7..1bc842588011f 100644 --- a/hermes_cli/main_install_repair.py +++ b/hermes_cli/main_install_repair.py @@ -186,6 +186,23 @@ def _quarantine_stamp_ms(stale: Path) -> int | None: return None +def _recover_update_debts_on_startup() -> None: + """The startup half of a previous update, run on every launch before dispatch.""" + # Dependency recovery already ran before imports. Report any fleet restart + # still owed by a previous update without restarting services here. + if "update" not in sys.argv[1:]: + try: + from hermes_cli.update_cmd_fleet import _warn_pending_fleet_restart_on_startup + + _warn_pending_fleet_restart_on_startup() + except Exception: # a startup warning never blocks the launch it warns about + logger.debug("pending fleet restart check failed", exc_info=True) + from hermes_cli.update_pause_record import recover as _recover_paused_gateways + # Unconditional: recover() itself skips the PARSED `update`/`gateway run` commands; a raw argv + # value equal to "update" (`--resume update`) is no reason to strand paused gateways. + _recover_paused_gateways() # a killed `hermes update` left gateways paused (Windows) + + def _cleanup_quarantined_exes(scripts_dir: Path | None = None) -> None: """Sweep — and where necessary RESCUE — ``hermes.exe.old.*`` from updates. From 6bb7daa909e7b811196de3805a3cc7a70f4edb15 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 6 Oct 2026 05:12:37 -0700 Subject: [PATCH 63/63] test(update): the post-commit ZIP resume test's git-command fake accepts checkout_move The same #132386 interaction as the ZIP completion tests: this test (from main) replaces update_cmd._prepare_git_command with a no-argument lambda inside its child process, and this branch calls it with checkout_move=..., so the child died with TypeError and the test read it as a failed update. The fake takes **_ like every other fake of it. --- tests/hermes_cli/test_update_post_commit_safety.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/hermes_cli/test_update_post_commit_safety.py b/tests/hermes_cli/test_update_post_commit_safety.py index 7b7c4d92193f0..bf5e466f5d3e6 100644 --- a/tests/hermes_cli/test_update_post_commit_safety.py +++ b/tests/hermes_cli/test_update_post_commit_safety.py @@ -306,7 +306,7 @@ def scm(resume_token): if resume_token and resume_token.get("resume_needed"): raise RuntimeError("Could not restart Windows gateway service(s): HermesGatewayProbe") hermes_main._resume_windows_gateways_after_update = scm - update_cmd._prepare_git_command = lambda: (True, None, False) + update_cmd._prepare_git_command = lambda **_: (True, None, False) def via_zip(args, *, had_desktop_app_before_update, target_sha=None, completion_request=None, **kwargs): completion_request["expected_sha"] = target_sha completion_request["apply_mode"] = "zip"