From 3d320133318c3c377bbf76752f7128eb93e5dfc9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 11:31:33 -0700 Subject: [PATCH 01/62] fix(update): after the commit point nothing fails hermes update (C3) --- hermes_cli/main_dashboard.py | 9 +- hermes_cli/source_build.py | 142 +++-- hermes_cli/source_completion.py | 51 +- hermes_cli/update_cmd.py | 15 +- hermes_cli/update_cmd_fleet.py | 105 +++- hermes_cli/update_cmd_maint.py | 56 +- hermes_cli/update_cmd_stale_survivors.py | 9 +- hermes_cli/update_completion.py | 83 +-- hermes_cli/update_inventory.py | 2 +- hermes_cli/update_receipt.py | 160 +++++- .../e2e/core/upgrade/git/test_hostile_post.py | 533 ++++++++++++++++++ .../test_fleet_matrix_self_restart_pending.py | 9 +- .../hermes_cli/test_manual_serve_deferral.py | 12 +- tests/hermes_cli/test_source_build.py | 19 +- .../test_source_completion_stamp.py | 22 +- .../test_update_completion_process.py | 33 +- ..._update_external_profile_reconciliation.py | 16 +- .../test_update_fleet_check_fail_closed.py | 43 +- .../test_update_fleet_completion.py | 18 +- .../test_update_fleet_restart_pending.py | 19 +- .../test_update_hangup_protection.py | 4 +- .../test_update_post_commit_followups.py | 178 ++++++ .../test_update_stale_gateway_yield.py | 7 +- .../hermes_cli/test_update_zip_completion.py | 59 +- .../source-update-completion.md | 31 +- website/docs/getting-started/updating.md | 4 +- website/docs/reference/cli-commands.md | 4 +- 27 files changed, 1407 insertions(+), 236 deletions(-) create mode 100644 tests/e2e/core/upgrade/git/test_hostile_post.py create mode 100644 tests/hermes_cli/test_update_post_commit_followups.py diff --git a/hermes_cli/main_dashboard.py b/hermes_cli/main_dashboard.py index 5a5ca614f417..0b884efd9059 100644 --- a/hermes_cli/main_dashboard.py +++ b/hermes_cli/main_dashboard.py @@ -515,10 +515,11 @@ def _install_hangup_protection(gateway_mode: bool = False): # Any failure here is non-fatal; we just skip the wrap. try: - # Late-bound import so tests can monkeypatch - # hermes_cli.config.get_hermes_home to simulate setup failure. - from hermes_cli.config import get_hermes_home as _get_hermes_home - logs_dir = _get_hermes_home() / "logs" + # Late-bound import so tests can monkeypatch it to simulate setup failure. The ROOT + # home, never a sticky profile's: the update mutates the shared checkout, and the + # Desktop and the hand-off scripts read /logs/update.log. + from hermes_constants import get_default_hermes_root as _get_root_home + logs_dir = _get_root_home() / "logs" logs_dir.mkdir(parents=True, exist_ok=True) log_file = open(logs_dir / "update.log", "a", buffering=1, encoding="utf-8") diff --git a/hermes_cli/source_build.py b/hermes_cli/source_build.py index 2559379ab2e9..f9208d71bd83 100644 --- a/hermes_cli/source_build.py +++ b/hermes_cli/source_build.py @@ -99,66 +99,70 @@ def source_frontends(project_root: Path) -> tuple[str, ...]: return tuple(name for name in ("ui-tui", "web") if (project_root / name / "package.json").is_file()) +class ProductBuildError(RuntimeError): + """One or more products failed; every independent product was still attempted.""" + + def __init__(self, failures: list[tuple[str, BaseException]]): + self.failures = failures + super().__init__("; ".join(f"{name}: {_failure_text(exc)}" for name, exc in failures)) + + +def _failure_text(exc: BaseException) -> str: + if isinstance(exc, subprocess.CalledProcessError): + return f"{' '.join(map(str, exc.cmd)) if isinstance(exc.cmd, (list, tuple)) else exc.cmd} exited {exc.returncode}" + return f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__ + + def build_update_products(project_root: Path, *, desktop: bool) -> None: - """Prepare the selected union once; a failed product aborts the update.""" + """Prepare the selected union once, attempting every independent product. + + A failed product never skips the next one (a broken web build must not leave the TUI or the + desktop app stale); the failures are raised together at the end as ``ProductBuildError``. + """ # Both current updates and historical takeover reach this in a fresh target # interpreter, never in the updater's pre-sync import graph. from hermes_cli.main_install_repair import _install_configured_features_missing_deps from hermes_cli.update_stage import publish_stage - _install_configured_features_missing_deps(project_root) + failures: list[tuple[str, BaseException]] = [] + + def attempt(name: str, step) -> bool: + try: + step() + return True + except Exception as exc: # noqa: BLE001 — collected and raised after the independent steps + print(f" ⚠ {name} failed: {_failure_text(exc)}") + failures.append((name, exc)) + return False + + attempt("feature dependencies", lambda: _install_configured_features_missing_deps(project_root)) frontends = source_frontends(project_root) - if not frontends: - return - env = source_build_env(explicit=True) - workspaces = frontends + (("apps/desktop",) if desktop else ()) - publish_stage("Updating Node dependencies") - prepare_source_dependencies(project_root, workspaces, env=env, explicit=True) - # An update that changed no TUI/web input reuses the receipted output, as the - # launch path already does; recompiling it produces the same bytes. Desktop - # additionally needs the packaged app to name HEAD (its baked stamp carries the - # commit), so it is reused only when HEAD did not move: "Already up to date", - # a retried tail, a takeover re-entry. - if "ui-tui" in frontends: - if source_product_current(project_root, "tui", project_root / "ui-tui/dist"): - print(" ✓ TUI is up to date") - else: - publish_stage("Building the TUI") - build_source_tui(project_root, env=env) - if "web" in frontends: - if source_product_current(project_root, "web", project_root / "hermes_cli/web_dist"): - print(" ✓ Web UI is up to date") - else: - publish_stage("Building the web UI") - build_source_web(project_root, env=env) - if desktop: - from hermes_cli.main_desktop import ( - _packaged_desktop_current_for_head, _refresh_installed_desktop_apps, build_prepared_desktop) - - desktop_dir = project_root / "apps/desktop" - if _packaged_desktop_current_for_head(desktop_dir, project_root): - print(" ✓ Desktop app is up to date") - else: - publish_stage("Building the desktop app") - # The desktop build mutates checkout-scoped node_modules and - # apps/desktop/release; serialize it against a concurrent manual - # `hermes desktop` (#93940). The update path waits rather than exits: - # the in-flight build it queues behind produces the same fresh tree - # this update needs. - from hermes_cli.desktop_build_lock import DesktopBuildLock - - build_lock = DesktopBuildLock(project_root) - build_lock.acquire(wait=True) - try: - build_prepared_desktop( - desktop_dir, source_mode=False, - npm=shutil.which("npm", path=env["PATH"]), env=env, icons=project_root, - ) - finally: - build_lock.release() - # A current release/ can still sit beside a stale installed copy (an earlier - # update rebuilt but never installed); healing must not wait for the next build. - _refresh_installed_desktop_apps(desktop_dir) + if frontends: + env = source_build_env(explicit=True) + workspaces = frontends + (("apps/desktop",) if desktop else ()) + publish_stage("Updating Node dependencies") + # Every product compiles from these node_modules: without them there is nothing to build. + if attempt("Node dependencies", lambda: prepare_source_dependencies( + project_root, workspaces, env=env, explicit=True)): + # An update that changed no TUI/web input reuses the receipted output, as the + # launch path already does; recompiling it produces the same bytes. Desktop + # additionally needs the packaged app to name HEAD (its baked stamp carries the + # commit), so it is reused only when HEAD did not move: "Already up to date", + # a retried tail, a takeover re-entry. + if "ui-tui" in frontends: + if source_product_current(project_root, "tui", project_root / "ui-tui/dist"): + print(" ✓ TUI is up to date") + else: + publish_stage("Building the TUI") + attempt("TUI build", lambda: build_source_tui(project_root, env=env)) + if "web" in frontends: + if source_product_current(project_root, "web", project_root / "hermes_cli/web_dist"): + print(" ✓ Web UI is up to date") + else: + publish_stage("Building the web UI") + attempt("web UI build", lambda: build_source_web(project_root, env=env)) + if desktop: + attempt("desktop app build", lambda: _build_desktop_product(project_root, env, publish_stage)) # A configured memory provider that no longer ships in core is installed from the # catalog for every profile home sharing this venv (config, data and tool names # unchanged). The update must finish even if the migration blows up. @@ -176,6 +180,38 @@ def build_update_products(project_root: Path, *, desktop: bool) -> None: migrate_left_core() except Exception as exc: print(f" ⚠ Plugin migration skipped: {exc}") + if failures: + raise ProductBuildError(failures) + + +def _build_desktop_product(project_root: Path, env: dict, publish_stage) -> None: + from hermes_cli.main_desktop import ( + _packaged_desktop_current_for_head, _refresh_installed_desktop_apps, build_prepared_desktop) + + desktop_dir = project_root / "apps/desktop" + if _packaged_desktop_current_for_head(desktop_dir, project_root): + print(" ✓ Desktop app is up to date") + else: + publish_stage("Building the desktop app") + # The desktop build mutates checkout-scoped node_modules and + # apps/desktop/release; serialize it against a concurrent manual + # `hermes desktop` (#93940). The update path waits rather than exits: + # the in-flight build it queues behind produces the same fresh tree + # this update needs. + from hermes_cli.desktop_build_lock import DesktopBuildLock + + build_lock = DesktopBuildLock(project_root) + build_lock.acquire(wait=True) + try: + build_prepared_desktop( + desktop_dir, source_mode=False, + npm=shutil.which("npm", path=env["PATH"]), env=env, icons=project_root, + ) + finally: + build_lock.release() + # A current release/ can still sit beside a stale installed copy (an earlier + # update rebuilt but never installed); healing must not wait for the next build. + _refresh_installed_desktop_apps(desktop_dir) if __name__ == "__main__": diff --git a/hermes_cli/source_completion.py b/hermes_cli/source_completion.py index f3df7d9cf479..603690ea25d1 100644 --- a/hermes_cli/source_completion.py +++ b/hermes_cli/source_completion.py @@ -36,11 +36,13 @@ def complete_source_checkout( pre_update_version: str | None = None, completion_message: str | None = None, announce: str | None = None, + followups: list[tuple[str, str]] | None = None, ) -> bool: """Publish commands, build the products, then run post-build maintenance. - Returns the SQLite runtime verdict: a positive unsafe-runtime probe withholds - success here exactly as it does at the end of an update. + Every step runs even when an earlier one failed; each failure is printed as ``⚠``, + recorded on the open update receipt and appended to ``followups`` as ``(step, reason)``. + Returns True only when every step finished and the SQLite runtime is not unsafe. """ from hermes_cli.update_lock import UpdateLock, describe_holder @@ -65,7 +67,7 @@ def complete_source_checkout( root, desktop=desktop, assume_yes=assume_yes, gateway_mode=gateway_mode, pre_update_snapshot_id=pre_update_snapshot_id, pre_update_version=pre_update_version, - completion_message=completion_message, announce=announce, + completion_message=completion_message, announce=announce, followups=followups, ) finally: lock.release() @@ -81,6 +83,7 @@ def _complete_locked( pre_update_version: str | None, completion_message: str | None, announce: str | None, + followups: list[tuple[str, str]] | None = None, ) -> bool: """The completion body; callers hold the update lock already.""" from hermes_cli.source_build import build_update_products @@ -95,19 +98,36 @@ def _complete_locked( expose_pm_git(root) except Exception as exc: # noqa: BLE001 — git-less steps below still complete print(f"⚠ Could not provide git for the source completion: {exc}", file=sys.stderr) - publish_launchers(root) - build_update_products(root, desktop=desktop) + owed: list[tuple[str, str]] = [] + + def step(name: str, run) -> None: + # Each step is independent: a failed launcher publish or web build must not skip the + # config migration, the maintenance, or (in an update) the gateway restart after it. + try: + run() + except (Exception, SystemExit) as exc: # noqa: BLE001 — reported as an owed follow-up + from hermes_cli.update_receipt import record_followup + + reason = str(exc) or type(exc).__name__ + record_followup(name, reason) + owed.append((name, reason)) + + step("launchers", lambda: publish_launchers(root)) + step("build", lambda: build_update_products(root, desktop=desktop)) if announce: print(announce) - complete = _run_post_update_maintenance( + verdict: list[bool] = [] + step("maintenance", lambda: verdict.append(_run_post_update_maintenance( assume_yes=assume_yes, gateway_mode=gateway_mode, pre_update_snapshot_id=pre_update_snapshot_id, had_desktop_app_before_update=desktop, pre_update_version=pre_update_version, completion_message=completion_message, - ) - if complete: + followups=owed, + ))) + tail_done = not owed + if tail_done: from hermes_cli.source_stamp import write_source_stamp try: @@ -115,7 +135,11 @@ def _complete_locked( except (OSError, ValueError) as exc: print(f"⚠ Source update completed, but the install stamp could not be written: {exc}", file=sys.stderr) - return complete + if followups is not None: + followups.extend(owed) + # The SQLite verdict is not tail work (re-running the tail cannot fix the interpreter); it + # is reported by the maintenance step itself and only withholds an install's success. + return tail_done and all(verdict) def _bootstrap_command(root: Path, argv: list[str]) -> list[str]: @@ -155,11 +179,14 @@ def main(argv: list[str] | None = None) -> int: # pre-handoff release cannot flip during `hermes update`, so its # update ends with the tree at HEAD and nothing built -- lands here # on the next ordinary startup. Same tail as an install, so the two - # states cannot drift apart. - ok = complete_source_checkout( + # states cannot drift apart. Only owed TAIL work keeps it pending: an + # unsafe SQLite runtime is reported, but re-running the tail cannot fix it. + owed: list[tuple[str, str]] = [] + complete_source_checkout( root, desktop=args.desktop, assume_yes=True, - completion_message=None, announce="\n✓ Code updated!", + completion_message=None, announce="\n✓ Code updated!", followups=owed, ) + ok = not owed else: ok = complete_source_checkout( root, desktop=args.desktop, assume_yes=not args.interactive, diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 13509f0674d9..2d82730d8676 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -544,7 +544,9 @@ def _log_only_write(text: str) -> None: log_file = getattr(stream, "_log", None) with suppress(Exception): if log_file is None: - log_path = get_hermes_home() / "logs" / "update.log" + from hermes_constants import get_default_hermes_root + + log_path = get_default_hermes_root() / "logs" / "update.log" # the root home's tee log_path.parent.mkdir(parents=True, exist_ok=True) with log_path.open("a", encoding="utf-8") as fallback: fallback.write(text) @@ -749,8 +751,15 @@ def _complete_source_update(request: dict | None) -> None: _completion_receipt._current.reset(current.current_token) if result["exit_code"]: raise SystemExit(result["exit_code"]) - if adopt_retired_channel(request): - print(f"→ Source subscription moved to {request['channel_retirement']['destination']}") + try: + if adopt_retired_channel(request): + print(f"→ Source subscription moved to {request['channel_retirement']['destination']}") + except Exception as exc: # noqa: BLE001 — the code is committed (C3); the next update re-adopts + from hermes_cli.update_receipt import amend_terminal_followup, record_followup + + reason = str(exc) or type(exc).__name__ + record_followup("channel_adoption", reason, retry="the next `hermes update` adopts it again") + amend_terminal_followup(request["receipt"]["update_id"], "channel_adoption", reason) def _reconcile_diverged_checkout(git_cmd, branch: str, pre_pull_sha, *, target_ref=None) -> None: diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index aec6e660123e..1ebb1f3089be 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -401,7 +401,7 @@ def _marker_only_restart_obsolete() -> bool: undischargeable on every host that runs a dashboard. A manual-serve row still needs its durable handoff (``defer_manual_serve``), and an unclassified backend stays fail-closed. Discharging here strands nobody: the same row is still accounted at update time by - ``update_inventory.report_unaccounted_runtimes``, which prints it and exits 1 when the restart + ``update_inventory.report_unaccounted_runtimes``, which prints it and records an owed restart when the restart phase never touched it — this marker only stops re-warning about it on every later startup. """ from hermes_cli.update_cmd_fleet_checkout import checkout_contains @@ -1839,7 +1839,7 @@ def _collect_fleet_snapshot(restart, rows_expected: bool) -> list: Gateways need time to rewrite gateway_state.json; Windows resumes DETACHED (~10s boot), so a single 2s sleep reported "no rows" on healthy resumes. A "down" row may be a detached replacement still booting: poll until none remain or the deadline passes. - Pre-restart PIDs make a gateway stopped WITHOUT verified replacement a DOWN row (exit 1) + Pre-restart PIDs make a gateway stopped WITHOUT verified replacement a DOWN row (owed restart) instead of no row at all. An ``unknown`` row whose pid is NOT a pre-restart pid is a successor that has not published its code identity yet (a relaunched gateway can sit ~10s between process start and its first runtime-status write, #112634) — keep polling; at the deadline it is flagged @@ -1917,13 +1917,52 @@ def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict: return live +def _record_owed_gateway_inventory(plan) -> None: + """Name the pre-update gateways on an inventory-less obligation once their restart is owed. + + The pull arms the obligation before the restart phase knows what it owes. An inventory-less + record on a host whose gateway then died at boot is settled by the gateway-less discharge + (nothing live, nothing named), which would silence the owed-restart warning that replaces a + failed update under contract C3. A named inventory keeps it until those gateways run HEAD. + """ + if plan is None: + return + armed = _fleet_restart_obligation_armed() + fields = _obligation_fields() if armed else {} + if fields is None or fields.get("inventory") not in (None, "", "null"): + return # unreadable terms stay fail-closed; a recorded inventory is never rewritten + from dataclasses import asdict + rows = [ + asdict(runtime) for runtime in getattr(plan, "runtimes", ()) or () + if getattr(runtime, "kind", None) == "gateway" + and isinstance(getattr(runtime, "profile", None), str) + and runtime.profile.strip() and runtime.profile != "unknown" + ] + if rows: + # Not armed any more (a pre-restart probe settled an inventory-less record): re-arm, the + # restart is owed. The SHA is the code the fleet must be proven current on. + sha = fields.get("expected_sha", "") if armed else (_current_checkout_sha() or "") + _write_fleet_restart_pending_marker(expected_sha=sha, runtimes=rows) + + +def _named_gateways_still_owed() -> bool: + """True when the armed obligation NAMES gateways (recorded by an earlier owed restart) and the + live fleet still does not prove them current. An inventory-less record keeps its old rules.""" + if not _fleet_restart_obligation_armed(): + return False + fields = _obligation_fields() + if fields is None or fields.get("inventory") in (None, "", "null"): + return False + return not _marker_only_restart_obsolete() + + def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, update_complete): """Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize. - Exits 1 (leaving ``fleet_restart_pending`` for the next catch-up) when any gateway - may still be stale; otherwise clears the marker. A failed SQLite verdict also - exits 1, without retaining a fulfilled fleet-restart obligation. + Never fails the committed update (contract C3): when any gateway may still be stale it + records a ``gateway_restart`` follow-up and leaves ``fleet_restart_pending`` armed for the + next catch-up and the startup warning; otherwise clears the marker. """ from hermes_cli.update_cmd import ( _m, _surviving_pre_update_serve_runtimes, _warn_stale_serve_runtimes, @@ -1943,13 +1982,13 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re # Success-path twin of the abort-recovery probe: the restart phase only touches # units, so a unit-less `hermes serve` keeps stale sys.modules. Runs AFTER # dashboard cleanup so a respawned manual dashboard isn't a survivor. Rows feed - # reconciliation (survivor → exit 1); ``None`` = probe failed, stays fail-closed. + # reconciliation (survivor → owed restart); ``None`` = probe failed, stays fail-closed. # Check if any pre-update serve/dashboard runtimes survived on pre-update code generations (#100479). # This is the SUCCESS-path twin of the abort-recovery probe above: the restart phase only restarts # units, so an sshd-spawned `serve --isolated` or a manual `hermes serve` (no unit) is left running its # pre-update sys.modules graph — and its cron ticker keeps firing agent jobs that ImportError on every # symbol added in the pulled range. The rows also feed the plan-vs-execution reconciliation below, so a - # survivor is escalated (exit 1) instead of merely printed. + # survivor is escalated (an owed gateway_restart follow-up) instead of merely printed. _stale_serve_rows: "list | None" = None with _best_effort('Failed to check for surviving serve runtimes: %s'): _stale_serve_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan) @@ -1972,7 +2011,7 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re # See #93406. # A gateway stopped WITHOUT a successor ("Restart manually") publishes no row by design, # so it must not count as an expected one — otherwise an update whose only live gateways - # were unmapped exits 1 with "no rows" after correctly stopping them. + # were unmapped reports "no rows" after correctly stopping them. _pre_restart, _killed = restart.fleet_probe_signals() _fleet_rows_expected = _m()._fleet_probe_expected_runtimes( _pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed, @@ -1986,14 +2025,14 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re signal_stale_fleet_survivors(_fleet_snapshot, restart, _gateway_drain_budget()) elif not _fleet_snapshot and _fleet_rows_expected: # collect_fleet_versions() swallows every failure, so zero rows with - # expected runtimes is indistinguishable from health — fail (partial, exit 1). + # expected runtimes is indistinguishable from health — keep the restart owed. print( # Fleet probe returned zero rows even though at least one gateway runtime was (or may have # been) live pre-update — POSIX restart bookkeeping, the pre-restart PID snapshot, the # pre-update plan inventory, or the Windows pause/resume token all count as that signal. # Every failure path inside collect_fleet_versions() is swallowed via logger.debug(), so an # empty list is indistinguishable from a healthy fleet in the current output. Treat it as - # verification failure so the receipt records "partial" and the exit code is 1 (#93406). + # verification failure so the receipt carries a gateway_restart follow-up (#93406). "\n⚠ Fleet version check returned no rows even though" " gateway runtimes were expected — verification incomplete." ) @@ -2047,28 +2086,54 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re if _active is not None: _active.data["runtime_outcomes"] = _runtime_outcomes + if not restart.incomplete: + with _best_effort('Owed-gateway check failed: %s'): + if _named_gateways_still_owed(): + # A gateway a previous run could not restart is still not serving HEAD (nothing + # live to restart this time): keep owing it rather than clearing the obligation. + print(" ⚠ A gateway owed a restart by a previous update is still not serving this checkout.") + restart.incomplete = True + if getattr(restart, "stopped_unmapped_pids", None): + # A gateway stopped with no successor is owed a restart, not "accounted for": keep the + # fleet obligation armed so the startup warning names it until someone restarts it. + restart.incomplete = True + if restart.incomplete: + # Code is committed (contract C3): a gateway that may still run stale modules is a + # follow-up, never a failed update. The fleet obligation stays armed, every CLI start + # warns about it, and the next `hermes update` retries the restart. + from hermes_cli.update_receipt import record_followup + stopped = sorted(getattr(restart, "stopped_unmapped_pids", None) or ()) + record_followup( + "gateway_restart", + "gateways may still run pre-update code or were stopped without a successor" + + (f" (stopped PIDs {', '.join(map(str, stopped))})" if stopped else "") + + "; recover with `hermes gateway restart`", + ) + with _best_effort('Fleet restart inventory not recorded: %s'): + _record_owed_gateway_inventory(_pre_update_plan) with _best_effort('Update receipt finalize failed: %s'): from hermes_cli.update_receipt import finalize_update_receipt - _receipt_path = finalize_update_receipt( - "partial" if restart.incomplete or not update_complete else "success", - fleet=_fleet_snapshot, - ) + # ``update_complete`` is kept for the historical takeover caller; a False verdict (the + # SQLite runtime) is reported as a follow-up by the maintenance step, not a failed run. + _receipt_path = finalize_update_receipt("success", fleet=_fleet_snapshot) if _receipt_path is not None: logger.info("Update receipt written: %s", _receipt_path) if restart.incomplete: - # Code updated but a gateway may still run stale modules: fail so automation - # doesn't treat the fleet as healthy; leave the pending marker for catch-up. - sys.exit(1) + return _clear_fleet_restart_pending_marker() if not update_complete: - # Fleet caught up, but the independently checked SQLite runtime is unsafe. - sys.exit(1) + # Fleet caught up, but the selected runtime's SQLite is unsafe (reported by maintenance): + # leave the topology alone until the runtime is repaired. + return # Fleet is healthy on the new code: fold per-profile gateways into one multiplexer when nothing # blocks it (deterministic; never prompts), else print the blockers and the one-liner to run later. - with _best_effort('Multiplex auto-migration after update failed: %s'): + try: from hermes_cli.gateway_migrate import maybe_auto_migrate_after_update maybe_auto_migrate_after_update() + except (Exception, SystemExit) as exc: # noqa: BLE001 — a SystemExit here must not fail a committed update + logger.warning('Multiplex auto-migration after update failed: %s', exc) + print(f" ⚠ Gateway multiplex migration did not finish: {exc} (run `hermes gateway migrate` later)") def _restart_phase_failure_is_incomplete(surviving, pre_restart_pids) -> bool: diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py index 193ac25e27af..3cb353f7b5a0 100644 --- a/hermes_cli/update_cmd_maint.py +++ b/hermes_cli/update_cmd_maint.py @@ -289,7 +289,7 @@ def _refresh_dashboard_after_update(*, already_restarted_units: set[str] | None # Isolated like every sibling post-update step: a failure here (#112604) used to abort # the fleet matrix, reconciliation and the inner receipt finalize that follow it. A # dashboard/serve left on pre-update code is still caught by the survivor probe → - # reconciliation (exit 1). + # reconciliation (an owed gateway_restart follow-up). logger.warning("Post-update dashboard cleanup failed: %s", exc) _record_update_step("dashboard_cleanup", False, f"{type(exc).__name__}: {exc}") print() @@ -809,14 +809,26 @@ def _run_pre_update_backup(args) -> Optional[str]: def _sweep_bytecode_after_update(branch: str) -> None: """Clear stale ``__pycache__`` (else gateway restart ImportErrors on names absent from old - bytecode), re-stamp the fingerprint, refresh the bootstrap cache scripts.""" + bytecode), re-stamp the fingerprint, refresh the bootstrap cache scripts. + + Never fails the committed update: a failed sweep is a ``bytecode_sweep`` follow-up and the + fingerprint stays un-stamped, so the launch-time sweep (fingerprint mismatch) retries it. + """ from hermes_cli.update_cmd import _m + from hermes_cli.update_receipt import record_followup # Timestamp-based .pyc validation can accept old bytecode after the source swap. - removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) - if removed: - print(f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}") - _m()._record_bytecode_fingerprint() - _m()._refresh_bootstrap_cache_scripts(branch) + try: + removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) + if removed: + print(f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}") + _m()._record_bytecode_fingerprint() + except Exception as exc: # noqa: BLE001 — owed to the next launch's fingerprint sweep + record_followup("bytecode_sweep", str(exc) or type(exc).__name__) + try: + _m()._refresh_bootstrap_cache_scripts(branch) + except Exception as exc: # noqa: BLE001 — refreshed again by the next update + record_followup("bootstrap_scripts", str(exc) or type(exc).__name__, + retry="the next `hermes update` refreshes them") def _profile_skill_sync_status(r) -> str: @@ -967,12 +979,26 @@ def _migrate_relay_exporter_env() -> None: def _run_post_update_maintenance( *, assume_yes, gateway_mode, pre_update_snapshot_id, had_desktop_app_before_update, - pre_update_version, completion_message=None, + pre_update_version, completion_message=None, followups=None, ) -> bool: """Post-build housekeeping and completion, returning the SQLite runtime verdict. - Ancillary repairs and notices are best-effort; an unsafe runtime withholds success. + Ancillary repairs and notices are best-effort. The profile sync and the config migration run + after a failed build too and never fail the committed update: a failure is printed as ``⚠``, + recorded as a receipt follow-up and appended to ``followups`` so the tail stays owed. An + unsafe runtime withholds success (and is reported) but is not tail work. """ + from hermes_cli.update_receipt import record_followup + + def owed_step(name, run): + try: + run() + except (Exception, SystemExit) as exc: # noqa: BLE001 — the code is committed; retry later + reason = str(exc) or type(exc).__name__ + record_followup(name, reason) + if followups is not None: + followups.append((name, reason)) + from hermes_cli.update_cmd import _check_and_apply_config_migration, _m # macOS TCC: Desktop bundles are re-signed each update, so old grants can go stale # (toggle ON, yet macOS re-prompts with no Allow button). Tell users how to re-grant. @@ -1030,11 +1056,11 @@ def _run_post_update_maintenance( print("→ Syncing bundled skills...") _print_bundled_skills_sync_report() - _sync_profiles_after_update() + owed_step("profile_sync", _sync_profiles_after_update) - _check_and_apply_config_migration( + owed_step("config_migration", lambda: _check_and_apply_config_migration( assume_yes=assume_yes, gateway_mode=gateway_mode, pre_update_snapshot_id=pre_update_snapshot_id, - ) + )) print() update_complete = _print_verified_update_completion(completion_message or _update_complete_message(pre_update_version)) @@ -1045,5 +1071,9 @@ def _run_post_update_maintenance( for line in [*consume_rewritten_notice(), *recorded_standalone_warning_lines()]: print(line) - _print_post_update_notices_and_self_heals() + with _best_effort('Post-update notices failed: %s'): + _print_post_update_notices_and_self_heals() + if not update_complete: + record_followup("sqlite_runtime", "the selected Python links an unsafe SQLite runtime", + retry="run the installer again") return update_complete diff --git a/hermes_cli/update_cmd_stale_survivors.py b/hermes_cli/update_cmd_stale_survivors.py index e1716ff81438..d70c408117de 100644 --- a/hermes_cli/update_cmd_stale_survivors.py +++ b/hermes_cli/update_cmd_stale_survivors.py @@ -1,7 +1,7 @@ """Post-update escalation for gateways the fleet matrix proved stale. ``_verify_fleet_after_update`` compares every live gateway's stamped ``code_sha`` against the -fresh checkout. Until now a ``stale`` row only failed the update (exit 1) and left the process +fresh checkout. A ``stale`` row once only failed the update (exit 1) and left the process running pre-update modules: its cron ticker then yields every tick to the "fresh gateway" it assumes exists, and nothing ever restarts it (#117275). A proven-stale survivor is now handed to the same drain-first ``request_restart`` path (SIGUSR1) the restart phase uses — a supervised @@ -29,7 +29,7 @@ def signal_stale_fleet_survivors(fleet: list, restart, drain_budget: float) -> l """Drain-first restart every proven-stale gateway; returns the PIDs signalled. Bookkeeping lands in ``restart.killed_pids`` so the receipt and the survivor sweep see them. - Never raises: verification must still finalize the receipt and exit 1. + Never raises: verification must still finalize the receipt (with an owed restart). """ from hermes_cli.update_cmd_fleet import _drain_or_signal_gateway_for_update @@ -58,6 +58,11 @@ def signal_stale_fleet_survivors(fleet: list, restart, drain_budget: float) -> l logger.warning("Could not signal stale gateway PID %s: %s", pid, exc) print(f" ⚠ {label}: could not be signalled ({exc}) — restart it by hand") if manual: + # No successor: the receipt's gateway_restart follow-up names them and the fleet + # obligation stays armed, so the stop is never silent (codemap §6 V7). + stopped = getattr(restart, "stopped_unmapped_pids", None) + if isinstance(stopped, set): + stopped.update(manual) print(f" → Stopped {len(manual)} manual gateway process(es) that had no supervisor to respawn them") print(" Restart manually: hermes gateway run") if len(manual) > 1: diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index 011e57a89339..d3732b724408 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -132,8 +132,11 @@ def _resume_receipt(data: dict) -> None: def _read_terminal_receipt(request: dict) -> dict | None: - directory = Path(request["home"]) / "logs/update_receipts" - # Never latest.json: another profile/context may have finalized more recently. + from hermes_cli.update_receipt import _receipt_dir + + # The ROOT home's receipts (the run's own file), never latest.json: another + # profile/context may have finalized more recently. + directory = _receipt_dir() for path in directory.glob(f"update_*_{request['receipt']['update_id']}.json"): data = json.loads(path.read_text(encoding="utf-8-sig")) if data.get("update_id") == request["receipt"]["update_id"] and data.get("finished_at"): @@ -181,9 +184,16 @@ def _prepare(request: dict, request_path: Path, result_path: Path) -> int: def _complete_selected(request: dict) -> None: + """Everything after the commit point. Nothing here fails the update (contract C3). + + Each step is independent; a failed one prints ``⚠``, lands on the receipt as a follow-up + and keeps its own obligation armed (``source-completion-pending`` for the tail, the fleet + restart obligation for gateways), so the next launch or ``hermes update`` retries it. + """ from hermes_cli import main, update_cmd, update_cmd_config from hermes_cli.source_completion import complete_source_checkout from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan + from hermes_cli.update_receipt import TAIL_FOLLOWUPS, record_followup, record_skip, record_stage root = Path(request["source"]) main.PROJECT_ROOT = root @@ -194,50 +204,49 @@ def _complete_selected(request: dict) -> None: update_cmd._sweep_bytecode_after_update(request["branch"]) # Launchers, products and post-build maintenance live in one place so an # install and an update cannot end in different states. - complete = complete_source_checkout( - root, desktop=request["desktop"], assume_yes=request["assume_yes"], - gateway_mode=request["gateway_mode"], pre_update_snapshot_id=request["snapshot_id"], - pre_update_version=request["pre_update_version"], - completion_message=request.get("completion_message"), - announce=None if request.get("completion_message") else "\n✓ Code updated!") - from hermes_cli.update_receipt import record_stage - record_stage("build", "success" if complete else "failed") - if complete: + followups: list[tuple[str, str]] = [] + try: + complete_source_checkout( + root, desktop=request["desktop"], assume_yes=request["assume_yes"], + gateway_mode=request["gateway_mode"], pre_update_snapshot_id=request["snapshot_id"], + pre_update_version=request["pre_update_version"], + completion_message=request.get("completion_message"), + announce=None if request.get("completion_message") else "\n✓ Code updated!", + followups=followups) + except (Exception, SystemExit) as exc: # noqa: BLE001 — e.g. the shared update lock refused the tail + reason = str(exc) or type(exc).__name__ + record_followup("completion", reason) + followups.append(("completion", reason)) + tail_owed = any(step in TAIL_FOLLOWUPS for step, _ in followups) + record_stage("build", "failed" if tail_owed else "success") + if not tail_owed: from hermes_cli.venv_sync import clear_completion clear_completion(root) # systemctl's KillMode=mixed fallback can kill this whole cgroup. Publish the - # gateway watcher's status BEFORE that operation, and demote on later failure. + # gateway watcher's status BEFORE that operation: the code is committed, so it is 0. if request["gateway_mode"]: - update_cmd._write_gateway_update_exit_code(complete) + update_cmd._write_gateway_update_exit_code(True) if request.get("no_gateway_restart", False): - from hermes_cli.update_receipt import record_skip - record_skip("gateway_restart", "--no-gateway-restart: deferred, marker kept") record_stage("restart", "skipped") print("→ Gateway restart deferred (--no-gateway-restart); restart gateways separately.") - if not complete: - raise SystemExit(1) return skip = update_cmd._fleet_restart_skip_reason(plan) + if skip and update_cmd._pending_fleet_restart_needed(): + # A host already stamped "restarted" for this SHA whose fleet is still off the checkout + # (a stale sibling, a failed resume) gets the restart again instead of a dead end. + print(f" → Gateway restart not skipped ({skip}): gateways are still off the checkout code.") + skip = None if skip: - from hermes_cli.update_receipt import record_skip - record_skip("gateway_restart", skip) record_stage("restart", "skipped") print(f" ✓ Gateway restart skipped: {skip}.") - # Discharges the obligation this run armed when the live fleet vouches for it; a - # fleet still owing the restart fails closed exactly like a stale matrix would. - if update_cmd._pending_fleet_restart_needed(): - print(" ⚠ Gateways are still off the checkout code. Recover with: hermes gateway restart") - raise SystemExit(1) - if not complete: - raise SystemExit(1) return restart = update_cmd._restart_gateway_fleet_after_update(plan, request["gateway_mode"]) record_stage("restart", "failed" if getattr(restart, "incomplete", False) else "success") update_cmd._resume_windows_gateways_and_merge_outcome(restart, request["windows_resume"], request["gateway_mode"]) update_cmd._verify_fleet_after_update( - restart, _pre_update_plan=plan, _windows_gateway_resume=request["windows_resume"], update_complete=complete) + restart, _pre_update_plan=plan, _windows_gateway_resume=request["windows_resume"], update_complete=True) class _ForwardedOutput: @@ -263,13 +272,15 @@ def _finish(request: dict, result_path: Path) -> int: code, reason = 0, "source update completion" try: _complete_selected(request) + except KeyboardInterrupt: + # An operator interrupt is not a step failure; the armed obligations finish the tail. + code, reason = 1, "KeyboardInterrupt: completion interrupted" + print("✗ Source update completion interrupted; the next launch or `hermes update` finishes it") except SystemExit as exc: - code = _exit_status(exc.code) if isinstance(exc.code, int) else 1 - reason = f"completion exited {code}" - except BaseException as exc: - code = _exit_status(exc.returncode) if isinstance(exc, subprocess.CalledProcessError) else 1 - reason = f"{type(exc).__name__}: {exc}" - print(f"✗ Source update completion failed: {reason}") + if exc.code not in (0, None): + update_receipt.record_followup("completion", f"completion exited {exc.code}") + except BaseException as exc: # noqa: BLE001 — after the commit point nothing fails the update + update_receipt.record_followup("completion", f"{type(exc).__name__}: {exc}") finally: if code and request["gateway_mode"]: from hermes_cli.update_cmd import _write_gateway_update_exit_code @@ -280,8 +291,10 @@ def _finish(request: dict, result_path: Path) -> int: from hermes_cli.update_cmd import _resume_windows_gateways_after_update _resume_windows_gateways_after_update(request["windows_resume"]) except Exception as exc: - code, reason = 1, f"Windows gateway recovery failed: {exc}" - print(f"✗ {reason}") + step_reason = f"Windows gateway recovery failed: {exc}" + update_receipt.record_followup("windows_resume", step_reason) + if update_receipt._current.get() is None: # verification already finalized this run + update_receipt.amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", step_reason) update_receipt.finalize_pending_update_receipt(code, reason) terminal_receipt = _read_terminal_receipt(request) if not terminal_receipt: diff --git a/hermes_cli/update_inventory.py b/hermes_cli/update_inventory.py index a5762a519ddd..fac8b12a26ca 100644 --- a/hermes_cli/update_inventory.py +++ b/hermes_cli/update_inventory.py @@ -542,7 +542,7 @@ def report_unaccounted_runtimes(outcomes: list[dict[str, Any]]) -> bool: """Print a loud warning for runtimes the restart phase never touched. Returns True when at least one planned runtime is unaccounted; the caller escalates like a - STALE/DOWN fleet row (exit 1) — a promised restart silently missed is the class this phase + STALE/DOWN fleet row (an owed ``gateway_restart`` follow-up) — a promised restart silently missed is the class this phase exists to kill. """ manual = [o for o in outcomes if o.get("outcome") == "deferred" and o.get("mechanism") == "respawn-argv"] diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index b1a6c72c7dc1..1f479bc52806 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -33,6 +33,7 @@ import json import logging import os +import re import sys import time import uuid @@ -142,6 +143,9 @@ def stage(self, name: str, outcome: str, **facts: str) -> None: def fact(self, key: str, value: Any) -> None: self.data[key] = value + def followup(self, step: str, reason: str) -> None: + self.data.setdefault("followups", []).append({"step": step, "reason": reason, "at": _utc_now_iso()}) + def gateway_restart_result( self, *, restarted_services: list | None = None, relaunched_profiles: list | None = None, externally_supervised_profiles: list | None = None, killed_pids: list | None = None, @@ -192,14 +196,78 @@ def finalize(self, outcome: str) -> None: def _receipt_dir() -> Path: # ``hermes_constants`` (stdlib-only), never ``hermes_cli.config``: the receipt must be # writable from the refused/failed paths where config loading itself may be what broke - # (#112465, #112558). - from hermes_constants import get_hermes_home + # (#112465, #112558). The ROOT home, never a sticky profile's: an update mutates the + # checkout every profile shares, and the Desktop and the hand-off scripts read the root. + from hermes_constants import get_default_hermes_root + + return get_default_hermes_root() / "logs" / "update_receipts" + + +def _run_file(directory: Path, data: dict[str, Any]) -> Path: + """One archive file per run, named at begin so the running and terminal records coincide.""" + stamp = re.sub(r"[^0-9]", "", str(data.get("started_at") or ""))[:14] or time.strftime("%Y%m%d%H%M%S") + return directory / f"update_{stamp[:8]}_{stamp[8:]}_{data.get('pid') or os.getpid()}_{data.get('update_id')}.json" + + +def _persist_running(data: dict[str, Any]) -> None: + """Write the open run to disk: a killed update still leaves its own record. Never raises.""" + with suppress(Exception): + from hermes_cli.runtime_state import _atomic_bytes + + directory = _receipt_dir() + directory.mkdir(parents=True, exist_ok=True) + payload = (json.dumps({**data, "writer_pid": os.getpid()}, indent=2, default=str) + "\n").encode("utf-8") + _atomic_bytes(_run_file(directory, data), payload) + _atomic_bytes(directory / "latest.json", payload) - return get_hermes_home() / "logs" / "update_receipts" + +def _owner_alive(record: dict[str, Any]) -> bool: + from hermes_cli.process_identity import _pid_alive_matches + + for key in ("pid", "writer_pid"): + pid = record.get(key) + if isinstance(pid, int) and pid > 0 and pid != os.getpid(): + create_time = record.get("pid_create_time") if key == "pid" else None + if _pid_alive_matches(pid, create_time) is not False: + return True + return False + + +def reconcile_interrupted_runs() -> list[dict[str, Any]]: + """Mark ``running`` records whose processes are gone as ``interrupted``; returns them. + + The start-of-run reclaimer for the durable running receipt: the next update says plainly + that the previous one was killed and at which stage, instead of reporting its outcome as + whatever ran before it. Never raises. + """ + interrupted: list[dict[str, Any]] = [] + with suppress(Exception): + from hermes_cli.runtime_state import _atomic_bytes + + directory = _receipt_dir() + latest = read_latest_receipt() or {} + for path in sorted(directory.glob("update_*.json"), key=lambda p: p.stat().st_mtime)[-_RECEIPT_KEEP:]: + with suppress(Exception): + record = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(record, dict) or record.get("outcome") != "running" or _owner_alive(record): + continue + stages = record.get("stages") or [] + last = stages[-1].get("name") if stages and isinstance(stages[-1], dict) else None + record.update(outcome="interrupted", interrupted_detected_at=_utc_now_iso(), + stop_reason=f"process exited during the update (last stage: {last or 'start'})") + payload = (json.dumps(record, indent=2, default=str) + "\n").encode("utf-8") + _atomic_bytes(path, payload) + if latest.get("update_id") == record.get("update_id"): + _atomic_bytes(directory / "latest.json", payload) + interrupted.append(record) + for record in interrupted: + print(f"⚠ The previous update ({record.get('started_at')}) was interrupted before it finished " + f"({record.get('stop_reason')}); this update completes the work it still owed.") + return interrupted def begin_update_receipt(*, previous: dict | None = None, correlation_id: str | None = None) -> None: - """Start recording a new update receipt. + """Start recording a new update receipt, durably ``running`` until it finalizes. Nested updates are safe: the previous receipt (if any) is preserved behind the ContextVar token and comes back when this one finalizes — @@ -213,10 +281,71 @@ def begin_update_receipt(*, previous: dict | None = None, correlation_id: str | receipt.data.update(update_id=receipt.correlation_id, outcome="running", finished_at=None) # A handoff receipt from an older interpreter may predate the field. receipt.data["correlation_id"] = receipt.data.get("correlation_id") or _launcher_correlation_id() + with suppress(Exception): + from hermes_cli.process_identity import _process_create_time + + receipt.data.setdefault("pid_create_time", _process_create_time(receipt.data["pid"])) + if not previous: + reconcile_interrupted_runs() + # The running record is about to replace latest.json: snapshot the previous record's + # manual serve rows (raw, no side effects) for finalize's carry-forward. + with suppress(Exception): + prior = read_latest_receipt() or {} + rows = list((prior.get("plan") or {}).get("runtimes") or []) + rows += list(prior.get("pending_manual_serves") or []) + receipt.data["carried_manual_serves"] = [ + row for row in rows if isinstance(row, dict) and row.get("supervisor") == "manual-serve"] except Exception as exc: # pragma: no cover - defensive logger.debug("Could not start update receipt: %s", exc) return receipt.current_token = _current.set(receipt) + _persist_running(receipt.data) + + +def persist_running_receipt() -> None: + """Re-assert the open run on disk (the completion child calls this after resuming it).""" + current = _current.get() + if current is not None and current.data.get("outcome") == "running": + _persist_running(current.data) + + +#: Follow-up steps whose failure leaves the source-update tail owed (``source-completion-pending``). +TAIL_FOLLOWUPS = frozenset({"launchers", "build", "maintenance", "config_migration", "completion"}) + + +def record_followup(step: str, reason: str, *, retry: str = "the next launch or `hermes update` retries it") -> None: + """A post-commit step failed: print ⚠, keep the run a success, and say what is still owed. + + The code already committed, so the update does not fail; the step's own obligation (the + source tail, the fleet restart, the launch-time bytecode sweep) stays armed and the next + launch or ``hermes update`` retries it. Never raises. + """ + reason = " ".join(str(reason).split())[:500] or "failed" + print(f" ⚠ Update follow-up '{step}' did not finish: {reason} ({retry})", flush=True) + current = _current.get() + if current is None: + return + _record("followup", f"update followup {step}", step, reason) + current = _current.get() + if current is not None: + _persist_running(current.data) + + +def amend_terminal_followup(update_id: str, step: str, reason: str) -> None: + """A follow-up that failed after the run's receipt was finalized lands on that receipt. Never raises.""" + with suppress(Exception): + from hermes_cli.runtime_state import _atomic_bytes + + directory = _receipt_dir() + for path in directory.glob(f"update_*_{update_id}.json"): + record = json.loads(path.read_text(encoding="utf-8-sig")) + if record.get("update_id") != update_id: + continue + record.setdefault("followups", []).append({"step": step, "reason": reason, "at": _utc_now_iso()}) + payload = (json.dumps(record, indent=2, default=str) + "\n").encode("utf-8") + _atomic_bytes(path, payload) + if (read_latest_receipt() or {}).get("update_id") == update_id: + _atomic_bytes(directory / "latest.json", payload) def _record(method: str, what: str, *args: Any, **kwargs: Any) -> None: @@ -254,6 +383,8 @@ def record_skip(name: str, reason: str) -> None: def record_stage(name: str, outcome: str, **facts: str) -> None: """Mark the END of a pipeline stage (``success``/``failed``/``skipped``) with a timestamp.""" _record("stage", f"update stage {name}", name, outcome, **facts) + # Each stage boundary refreshes the durable running record (a kill names its last stage). + persist_running_receipt() def record_fact(key: str, value: Any) -> None: @@ -299,7 +430,11 @@ def finalize_update_receipt(outcome: str, fleet: list | None = None, stop_reason # receipt's still-pending rows forward so the startup warning survives (see # update_serve_obligations). from hermes_cli.update_serve_obligations import retain_receipt_manual_serves - pending = retain_receipt_manual_serves(read_latest_receipt() or {}) + if "carried_manual_serves" in receipt.data: + prior = {"pending_manual_serves": receipt.data.pop("carried_manual_serves") or []} + else: # a receipt begun before the running record existed + prior = read_latest_receipt() or {} + pending = retain_receipt_manual_serves(prior) if pending: receipt.data["pending_manual_serves"] = pending # EMBED the pm sync sections (the settled receipts contract): the @@ -335,10 +470,8 @@ def finalize_update_receipt(outcome: str, fleet: list | None = None, stop_reason # latest.json pointer (no torn readers). from hermes_cli.runtime_state import _atomic_bytes - path = directory / ( - f"update_{time.strftime('%Y%m%d_%H%M%S')}_{os.getpid()}_" - f"{receipt.correlation_id}.json" - ) + # The run's own file (named at begin): the durable running record becomes terminal. + path = _run_file(directory, receipt.data) payload = (json.dumps(receipt.data, indent=2, default=str) + "\n").encode("utf-8") _atomic_bytes(path, payload) with suppress(Exception): # stable pointer for the dashboard/desktop @@ -459,7 +592,7 @@ def settle_latest_receipt_fleet(fleet: list[dict[str, Any]], *, discharges) -> b matrix as the receipt's post-restart ``fleet`` (and un-flagging ``gateway_restart``) is what lets the stale-runtime readers see the recovery. ``discharges(settled_receipt)`` decides on the in-memory copy; ``latest.json`` is rewritten only when it answers True, so a catch-up - that still exits 1 leaves the receipt byte-identical. Only the ``latest.json`` pointer is + that still owes the restart leaves the receipt byte-identical. Only the ``latest.json`` pointer is rewritten; the archived per-run file keeps the original outcome. Never raises. """ try: @@ -784,12 +917,11 @@ def print_fleet_version_matrix(fleet: list[dict[str, Any]]) -> bool: print(" ⚠ Stale gateways keep serving pre-update code until restarted.") if "down" in states: print(" ⚠ Down gateways stopped serving messaging entirely.") - # ``✓ Update complete!`` was already printed before the restart phase (the exit code - # must land before a systemd restart can kill this process), so this verdict has to - # supersede it explicitly — otherwise the output says success while the exit code is 1. + # The code is committed (the update exits 0), but the fleet is not on it yet: say what + # is still owed after ``✓ Update complete!``; every CLI start repeats it until restarted. print() print( - f"✗ Update not complete: {stale_or_down} gateway(s) still running the old code (or stopped).") + f"⚠ Gateway restart still owed: {stale_or_down} gateway(s) still running the old code (or stopped).") print(" Run `hermes gateway restart` (or `hermes -p gateway restart` for a named") print(" profile), then `hermes gateway status` to confirm.") return stale_or_down > 0 diff --git a/tests/e2e/core/upgrade/git/test_hostile_post.py b/tests/e2e/core/upgrade/git/test_hostile_post.py new file mode 100644 index 000000000000..db598f6ef5eb --- /dev/null +++ b/tests/e2e/core/upgrade/git/test_hostile_post.py @@ -0,0 +1,533 @@ +"""Once ``hermes update`` has committed the new code, the update never fails (contract C3). + +One real HEAD install (``_git_world``: the real ``scripts/install.sh`` over real smart HTTP, every +command in a ``bwrap`` sandbox) is shared by the cells. Each cell publishes a hostile upstream +release and breaks something that runs AFTER the checkout moved, for real: + +(a) the web UI build fails: the release's own ``scripts/build/web.mjs`` exits 1 while + ``$HOME/.e2e-break-web-build`` exists, and the release changes a web build input (``web.mjs`` + itself lives under ``scripts/build``, plus a new file under ``web/``) so ``freshness.mjs`` reports + the web UI stale and the updater must build it. A config migration is owed too + (``_config_version`` lowered by one): one failed step must not skip later independent ones. +(b) a gateway that refuses to restart: a credential-free, platform-less (cron-only) ``hermes + gateway run`` lives in the same PID namespace as the updater (``handoff._nshost``). The release + makes ``hermes_cli.gateway.run_gateway`` exit 1 before booting while + ``$HOME/.e2e-break-gateway-boot`` exists. The updater drains the OLD gateway (pre-update code, + no fault) and its relaunch boots the NEW code, which refuses to start: the restart fails for real, + no gateway serves the new commit. Removing the toggle and updating again must retry the owed + restart. +(c) SIGKILL during the post-commit tail: the release's ``web.mjs`` writes + ``$HOME/.e2e-web-build-started`` and sleeps 120 s while ``$HOME/.e2e-slow-web-build`` exists. The + update runs in the shared namespace; once the slow build started, every process born after the + update was spawned is SIGSTOPped then SIGKILLed (that is the whole update tree, including a + completion child in its own session or reparented to the namespace init). +(d) a sticky profile (``hermes profile use e2epost``) must not move update.log / receipts off the + root hermes home. + +What is asserted is what the user sees: the exit code, ``⚠`` lines, the receipt at +``/logs/update_receipts/latest.json`` (``outcome``, ``followups``), the +``source-completion-pending`` obligation in the install-state dir, the config version, the +owed-restart warning a later command prints, and which gateway (by its control-socket ``identify``) +serves the checkout. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import re +import shutil +import signal +import tempfile +import time +from pathlib import Path +from typing import Iterator + +import pytest + +import hermes_yaml as yaml +from tests.e2e.core.upgrade import _helpers as H +from tests.e2e.core.upgrade import _install_helpers as I +from tests.e2e.core.upgrade.git import _git_world as G +from tests.e2e.core.upgrade.handoff._nshost import NamespaceHost + +pytestmark = G.PYTESTMARK + +WARN = "⚠" +BREAK_WEB = ".e2e-break-web-build" +SLOW_WEB = ".e2e-slow-web-build" +WEB_STARTED = ".e2e-web-build-started" +BREAK_GATEWAY = ".e2e-break-gateway-boot" +TOGGLES = (BREAK_WEB, SLOW_WEB, WEB_STARTED, BREAK_GATEWAY) +WEB_FAULT_TEXT = "e2e: the web build fails on purpose" +GATEWAY_FAULT_TEXT = "e2e: gateway boot refused on purpose" +OWED_RESTART = "did not restart running gateways" # update_cmd_fleet._warn_pending_fleet_restart +PROFILE = "e2epost" +UPDATE_TIMEOUT = 1500 +_WEB_MARK = "// e2e (test_hostile_post): hostile build toggles" +_GATEWAY_MARK = "# e2e (test_hostile_post): hostile gateway boot toggle" + +_HOSTILE_WEB = f"""{_WEB_MARK}, keyed on files in $HOME. +import {{ existsSync as e2eExists, writeFileSync as e2eWrite }} from 'node:fs' +import {{ homedir as e2eHomedir }} from 'node:os' +const e2eHomes = [...new Set([process.env.HOME, e2eHomedir()].filter(Boolean))] +const e2eToggle = name => e2eHomes.some(home => e2eExists(`${{home}}/${{name}}`)) +if (e2eToggle('{BREAK_WEB}')) {{ + console.error('{WEB_FAULT_TEXT} ({BREAK_WEB})') + process.exit(1) +}} +if (e2eToggle('{SLOW_WEB}')) {{ + for (const home of e2eHomes) {{ try {{ e2eWrite(`${{home}}/{WEB_STARTED}`, String(process.pid)) }} catch {{}} }} + console.log('e2e: slow web build started ({SLOW_WEB})') + await new Promise(resolve => setTimeout(resolve, 120000)) +}} +""" + +_HOSTILE_GATEWAY = f""" {_GATEWAY_MARK} + if __import__("os").path.exists(__import__("os").path.join(__import__("os").path.expanduser("~"), "{BREAK_GATEWAY}")): + print("{GATEWAY_FAULT_TEXT} ({BREAK_GATEWAY})", file=__import__("sys").stderr, flush=True) + raise SystemExit(1) +""" + + +# -- the shared install -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def w() -> Iterator[G.World]: + """A SHORT root (the gateway's AF_UNIX control socket lives under ``$HERMES_HOME``; pytest's deep + ``tmp_path`` overflows ``sun_path``). Set ``HERMES_E2E_KEEP_ROOT=1`` to keep it for a post-mortem.""" + base = os.environ.get("TMPDIR") or tempfile.gettempdir() + root = Path(tempfile.mkdtemp(prefix="hp", dir=base)) + try: + with G.world(root, base=I.head_sha()) as world: + _edit_config(world, lambda cfg: cfg.update(updates={**(cfg.get("updates") or {}), "check": False})) + yield world + finally: + if not os.environ.get("HERMES_E2E_KEEP_ROOT"): + shutil.rmtree(root, ignore_errors=True) + + +def _config_path(w: G.World) -> Path: + return w.sb.hermes_home / "config.yaml" + + +def _load_config(w: G.World) -> dict: + p = _config_path(w) + data = yaml.safe_load(p.read_text(encoding="utf-8-sig")) if p.is_file() else None + return data if isinstance(data, dict) else {} + + +def _edit_config(w: G.World, edit) -> None: + cfg = _load_config(w) + edit(cfg) + _config_path(w).write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8") + + +def _config_version(w: G.World) -> int | None: + v = _load_config(w).get("_config_version") + return int(v) if isinstance(v, int) or (isinstance(v, str) and v.isdigit()) else None + + +def _toggle(w: G.World, name: str, on: bool) -> None: + p = w.sb.home / name + if on: + p.write_text("on\n", encoding="utf-8") + else: + p.unlink(missing_ok=True) + + +def _clear_toggles(w: G.World) -> None: + for name in TOGGLES: + _toggle(w, name, False) + + +def _pending_markers(w: G.World) -> list[str]: + """The ``source-completion-pending`` obligation(s) in the install-state dir(s).""" + installs = w.sb.hermes_home / "installs" + return sorted(str(p) for p in installs.glob("**/source-completion-pending")) if installs.is_dir() else [] + + +def _followups(rec: dict) -> list[dict]: + items = rec.get("followups") or [] + return [f for f in items if isinstance(f, dict)] + + +def _followup_steps(rec: dict) -> set[str]: + return {str(f.get("step")) for f in _followups(rec)} + + +def _update_log(w: G.World) -> Path: + return w.sb.hermes_home / "logs" / "update.log" + + +def _read(p: Path) -> str: + return p.read_text(encoding="utf-8-sig", errors="replace") if p.is_file() else "" + + +def _read_json(path: Path) -> dict: + try: + data = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def _warn_lines(text: str) -> list[str]: + return [line.strip() for line in text.splitlines() if WARN in line] + + +def _obligation(w: G.World) -> str: + """The host fleet-restart obligation record(s) anywhere in the sandbox (diagnostics only).""" + found = [] + for base, dirs, files in os.walk(w.sb.root): + dirs[:] = [d for d in dirs if d not in ("node_modules", ".git", "venv", ".venv", "hermes-agent")] + if "host-update-restart.json" in files: + path = Path(base) / "host-update-restart.json" + found.append(f"{path}: {_read(path)[:600]}") + return "; ".join(found) or "none" + + +def _facts(w: G.World, cp=None, **extra) -> str: + """A one-glance summary heading every failure message.""" + rec = w.receipt() + parts = [f"rc={cp.returncode}" if cp is not None else "rc=-", + f"head={w.head()[:12]}", + f"receipt.outcome={rec.get('outcome')!r} update_id={str(rec.get('update_id'))[:12]} " + f"pid={rec.get('pid')} followups={_followups(rec)}", + f"pending={_pending_markers(w)}", + f"config_version={_config_version(w)}", + f"obligation={_obligation(w)}"] + if cp is not None: + parts.append(f"⚠ lines={_warn_lines(G.output(cp))[:8]}") + parts += [f"{k}={v}" for k, v in extra.items()] + return "\n".join(parts) + "\n" + + +# -- hostile releases ---------------------------------------------------------------------------- + + +def _head_file(w: G.World, rel: str) -> str: + return w.git("show", f"HEAD:{rel}") + "\n" + + +def _hostile_web_mjs(w: G.World) -> str: + """The checkout's ``scripts/build/web.mjs`` with the toggle block after its imports.""" + text = _head_file(w, "scripts/build/web.mjs") + if _WEB_MARK in text: + return text + lines = text.splitlines(keepends=True) + last_import = max(i for i, line in enumerate(lines) if line.startswith("import ")) + while " from " not in lines[last_import] and last_import + 1 < len(lines): # a multi-line import + last_import += 1 + out = "".join(lines[:last_import + 1]) + _HOSTILE_WEB + "".join(lines[last_import + 1:]) + assert out.count(_WEB_MARK) == 1 + return out + + +def _publish_web_release(w: G.World, tag: str) -> str: + """A release whose web build obeys the toggles and whose web inputs changed (web UI stale).""" + return w.publish(f"release: e2e hostile post web {tag}", { + "scripts/build/web.mjs": _hostile_web_mjs(w), + f"web/e2e-hostile-post-{tag}.txt": f"web input changed by release {tag}\n", + }) + + +def _publish_gateway_fault_release(w: G.World, tag: str) -> str: + """A release whose ``hermes gateway run`` refuses to boot while the toggle exists.""" + text = _head_file(w, "hermes_cli/gateway.py") + if _GATEWAY_MARK not in text: + m = re.search(r"^def run_gateway\(.*?\):\n \"\"\".*?\"\"\"\n", text, re.S | re.M) + assert m, "premise: hermes_cli/gateway.py has no run_gateway() with a docstring to inject after" + text = text[:m.end()] + _HOSTILE_GATEWAY + text[m.end():] + return w.publish(f"release: e2e hostile post gateway {tag}", { + "hermes_cli/gateway.py": text, + f"docs/e2e-hostile-post-{tag}.txt": f"release {tag}\n", + }) + + +# -- one shared PID namespace (gateway / killable update) ---------------------------------------- + + +@contextlib.contextmanager +def _namespace(w: G.World) -> Iterator[NamespaceHost]: + """Every process of the cell in ONE sandbox, as on a user's machine: the updater's process scans + see the gateway it restarts. Closing kills the namespace init, which takes everything with it.""" + host = NamespaceHost(w.sb.root, w.sb.env) + try: + yield host + finally: + leaked = host.close() + assert not leaked, f"processes outlived the sandbox: {leaked}" + + +def _ns_cli(host: NamespaceHost, w: G.World, *args: str, timeout: float = 600): + cp = host.run([w.sb.hermes, *args], timeout=timeout) + w.transcripts.append(f"$ hermes {' '.join(args)} -> rc={cp.returncode}\n{(cp.stdout or '')[-6000:]}\n" + f"{(cp.stderr or '')[-3000:]}") + return cp + + +def _ns_update(host: NamespaceHost, w: G.World, *extra: str): + return _ns_cli(host, w, "update", "--yes", "--branch", "main", *extra, timeout=UPDATE_TIMEOUT) + + +def _identify(w: G.World) -> dict | None: + from gateway.control_socket import identify_gateway + + with contextlib.suppress(Exception): + return identify_gateway(w.sb.hermes_home, timeout=5.0) + return None + + +def _serving(w: G.World, sha: str) -> dict | None: + ident = _identify(w) + return ident if ident and ident.get("code_sha") == sha else None + + +def _kill_born_after(host: NamespaceHost, before: set[int]) -> list[int]: + """SIGSTOP then SIGKILL every process born after ``before`` was taken (by PID, namespace view): + the update, its re-exec'd interpreter, build children and any detached completion child (own + session, or reparented to the namespace init). Repeats until none is left.""" + killed: list[int] = [] + for _ in range(10): + live = [p["pid"] for p in host.procs() if p["pid"] not in before and p["state"] not in ("Z", "X")] + if not live: + break + for pid in live: + host.kill(pid, signal.SIGSTOP) + for pid in live: + host.kill(pid, signal.SIGKILL) # windows-footgun: ok - linux-only (bwrap) module + if pid not in killed: + killed.append(pid) + time.sleep(0.5) + return killed + + +def _gateway_logs(w: G.World) -> str: + logs = w.sb.hermes_home / "logs" + chunks = [_read(p)[-4000:] for p in sorted(logs.glob("*.log"))] if logs.is_dir() else [] + chunks += [_read(p)[-4000:] for p in sorted(w.sb.root.glob("gateway-*.log"))] + return "\n".join(chunks) + + +# -- cells --------------------------------------------------------------------------------------- + + +def test_failed_web_build_after_commit_is_a_followup_not_a_failed_update(w): + w.reset_clean() + _clear_toggles(w) + latest = _config_version(w) + if latest is None: + from hermes_cli.config_defaults import DEFAULT_CONFIG + + latest = int(DEFAULT_CONFIG["_config_version"]) + owed = latest - 1 + _edit_config(w, lambda cfg: cfg.update(_config_version=owed)) + target = _publish_web_release(w, "a") + log0 = len(_read(_update_log(w))) + + _toggle(w, BREAK_WEB, True) + try: + cp = w.update("--no-gateway-restart") + finally: + _toggle(w, BREAK_WEB, False) + out = G.output(cp) + log = _read(_update_log(w))[log0:] + rec = w.receipt() + facts = _facts(w, cp, config_version_lowered_to=owed, latest_config_version=latest) + diag = facts + w.diag(cp) + + assert w.head() == target, f"premise: the release never committed, so this is no post-commit cell:\n{diag}" + assert WEB_FAULT_TEXT in out + log, f"premise: the hostile web build never ran (web UI not seen stale?):\n{diag}" + assert cp.returncode == 0, f"the code committed, yet a failed web build failed `hermes update`:\n{diag}" + assert _warn_lines(out), f"the failed web build was not surfaced with a ⚠ line:\n{diag}" + assert rec.get("outcome") == "success", f"receipt outcome is not success after the commit:\n{diag}" + assert "build" in _followup_steps(rec), f"receipt followups do not name the failed `build` step:\n{diag}" + assert _pending_markers(w), f"the failed build disarmed the source-completion obligation:\n{diag}" + assert (_config_version(w) or 0) >= latest, ( + f"the failed web build skipped the independent config migration (_config_version still " + f"{_config_version(w)}, latest {latest}):\n{diag}") + + # The fault clears; the next plain update (no new release) pays the owed tail. + cp = w.update() + rec = w.receipt() + diag = _facts(w, cp) + w.diag(cp) + assert cp.returncode == 0 and w.head() == target, f"the healing update failed:\n{diag}" + assert not _followups(rec), f"the healing update still reports followups:\n{diag}" + assert not _pending_markers(w), f"the healing update left the completion obligation armed:\n{diag}" + assert (w.checkout / "hermes_cli" / "web_dist" / "index.html").is_file(), \ + f"the healing update did not build the web UI:\n{diag}" + + +def test_gateway_that_cannot_reboot_is_an_owed_restart_not_a_failed_update(w): + """Fault: the RELEASE's gateway cannot boot (``run_gateway`` exits 1 while the toggle exists). The + old gateway drains/stops normally; the relaunch runs the new code and dies at boot, so no gateway + serves the new commit. A SIGSTOPped old gateway would not do: the survivor sweep SIGKILLs it and + the relaunch then succeeds.""" + w.reset_clean() + _clear_toggles(w) + with _namespace(w) as host: + try: + gw = host.spawn([w.sb.hermes, "gateway", "run"], log=w.sb.root / "gateway-b.log") # sandbox-writable + before_sha = w.head() + try: + ident = H.wait_for(lambda: _serving(w, before_sha), timeout=240, interval=1.0, + what="the cron-only gateway's control-socket identify") + except AssertionError as exc: + raise AssertionError(f"premise: the gateway never came up: {exc}\n{_gateway_logs(w)[-4000:]}\n" + f"{host.ps_text()}") from None + target = _publish_gateway_fault_release(w, "b") + _toggle(w, BREAK_GATEWAY, True) + cp = _ns_update(host, w) + out = G.output(cp) + rec = w.receipt() + obligation_after_update = _obligation(w) + status = _ns_cli(host, w, "status", timeout=300) + facts = _facts(w, cp, gateway_before=f"pid {ident.get('pid')} sha {before_sha[:12]} (spawned {gw})", + identify_after=_identify(w), obligation_after_update=obligation_after_update) + diag = (facts + w.diag(cp) + "\n--- gateway logs ---\n" + _gateway_logs(w)[-6000:] + + "\n--- sandbox ---\n" + host.ps_text() + "\n--- hermes status ---\n" + G.output(status)[-3000:]) + + assert w.head() == target, f"premise: the release never committed:\n{diag}" + assert not _serving(w, target), f"premise: a gateway serves the new commit; the restart fault never fired:\n{diag}" + assert cp.returncode == 0, f"the code committed, yet a failed gateway restart failed `hermes update`:\n{diag}" + assert _warn_lines(out), f"the failed gateway restart was not surfaced with a ⚠ line:\n{diag}" + assert rec.get("outcome") == "success", f"receipt outcome is not success after the commit:\n{diag}" + assert "gateway_restart" in _followup_steps(rec), \ + f"receipt followups do not name the failed `gateway_restart` step:\n{diag}" + assert OWED_RESTART in G.output(status), \ + f"a later command does not warn that a gateway restart is still owed:\n{diag}" + + # The fault clears; the next plain update retries the owed restart. This gateway is a + # manual `gateway run` that died at boot: no supervisor and no live process, so the + # retry has nothing it can relaunch. It must still exit 0 and must NOT silently discharge + # the obligation (no gateway serves HEAD yet). + _toggle(w, BREAK_GATEWAY, False) + cp = _ns_update(host, w) + status = _ns_cli(host, w, "status", timeout=300) + diag = (_facts(w, cp, identify_after=_identify(w)) + w.diag(cp) + "\n--- gateway logs ---\n" + + _gateway_logs(w)[-6000:] + "\n--- sandbox ---\n" + host.ps_text() + + "\n--- hermes status ---\n" + G.output(status)[-3000:]) + assert cp.returncode == 0, f"the retrying update failed:\n{diag}" + assert OWED_RESTART in G.output(status), \ + f"the retry discharged the owed restart while no gateway serves HEAD:\n{diag}" + + # The operator restarts it (the remedy the warning names); once a gateway serves HEAD the + # obligation discharges and the warning goes away. + host.spawn([w.sb.hermes, "gateway", "run"], log=w.sb.root / "gateway-b2.log") + served = None + with contextlib.suppress(AssertionError): + served = H.wait_for(lambda: _serving(w, target), timeout=240, interval=1.0, what="a gateway on HEAD") + status = _ns_cli(host, w, "status", timeout=300) + diag = (_facts(w, cp, identify_after=_identify(w)) + "\n--- gateway logs ---\n" + + _gateway_logs(w)[-6000:] + "\n--- sandbox ---\n" + host.ps_text() + + "\n--- hermes status ---\n" + G.output(status)[-3000:]) + assert served, f"premise: the restarted gateway never served {target[:12]}:\n{diag}" + assert OWED_RESTART not in G.output(status), \ + f"the owed-restart warning survived a gateway serving HEAD:\n{diag}" + finally: + _toggle(w, BREAK_GATEWAY, False) + for p in host.procs(): + if p["pid"] > 2 and "gateway" in p["cmdline"]: + host.kill(p["pid"], signal.SIGKILL) # windows-footgun: ok - linux-only (bwrap) module + # The namespace (and its gateways) is gone; its PID file names a PID of a dead namespace. + (w.sb.hermes_home / "gateway.pid").unlink(missing_ok=True) + + +def test_sigkill_during_post_commit_tail_leaves_a_running_receipt(w): + w.reset_clean() + _clear_toggles(w) + prev = w.receipt() + target = _publish_web_release(w, "c") + log0 = len(_read(_update_log(w))) + started = w.sb.home / WEB_STARTED + _toggle(w, SLOW_WEB, True) + killed: list[int] = [] + try: + with _namespace(w) as host: + before = {p["pid"] for p in host.procs()} + spawn_log = w.sb.root / "update-c.log" # sandbox-writable + upd = host.spawn([w.sb.hermes, "update", "--yes", "--branch", "main", "--no-gateway-restart"], + log=spawn_log) + + def _slow_build_running(): + if started.is_file(): + return True + if not host.alive(upd): + raise AssertionError(f"premise: the update exited before the slow web build started:\n" + f"{_read(spawn_log)[-6000:]}\n{w.diag()}") + return False + + H.wait_for(_slow_build_running, timeout=1200, interval=1.0, what="the slow web build to start") + with contextlib.suppress(AssertionError): + H.wait_for(lambda: re.search(r"web UI|slow web build", _read(_update_log(w))[log0:]), + timeout=30, interval=0.5, what="update.log to show the web build") + log_shows_build = bool(re.search(r"web UI|slow web build", _read(_update_log(w))[log0:])) + killed = _kill_born_after(host, before) + rec = w.receipt() + facts = _facts(w, killed=killed, update_spawn_pid=upd, log_shows_web_build=log_shows_build, + previous_update_id=prev.get("update_id")) + diag = facts + w.diag() + "\n--- spawned update output ---\n" + _read(spawn_log)[-6000:] + finally: + _toggle(w, SLOW_WEB, False) + started.unlink(missing_ok=True) + + assert killed, f"premise: nothing was killed:\n{diag}" + assert w.head() == target, f"premise: the kill landed before the release committed:\n{diag}" + assert rec.get("outcome") == "running" and rec.get("update_id") and \ + rec.get("update_id") != prev.get("update_id"), \ + f"latest.json is not a `running` record of the killed update:\n{json.dumps(rec, default=str)[:3000]}\n{diag}" + assert rec.get("pid") in killed, f"the running receipt names pid {rec.get('pid')}, not a killed process:\n{diag}" + killed_id = rec["update_id"] + + cp = w.update("--no-gateway-restart") + rec = w.receipt() + diag = _facts(w, cp, killed_update_id=killed_id) + w.diag(cp) + assert "interrupted" in G.output(cp), f"the next update did not report the interrupted run:\n{diag}" + assert cp.returncode == 0, f"the update after the interrupted one failed:\n{diag}" + assert rec.get("outcome") == "success" and rec.get("update_id") != killed_id, \ + f"latest.json is not this run's terminal success:\n{diag}" + + +def test_sticky_profile_update_logs_and_receipts_land_in_the_root_home(w): + w.reset_clean() + _clear_toggles(w) + root = w.sb.hermes_home + prof_receipts = root / "profiles" / PROFILE / "logs" / "update_receipts" + if not (root / "profiles" / PROFILE).is_dir(): + cp = w.sb.cli("profile", "create", PROFILE, "--no-alias", timeout=300) + assert cp.returncode == 0, f"premise: profile create failed:\n{H.describe(cp)}" + cp = w.sb.cli("profile", "use", PROFILE, timeout=300) + assert cp.returncode == 0, f"premise: profile use failed:\n{H.describe(cp)}" + try: + assert (root / "active_profile").read_text(encoding="utf-8-sig").strip() == PROFILE, \ + "premise: the sticky profile was not recorded" + prof_before = {p.name for p in prof_receipts.iterdir()} if prof_receipts.is_dir() else set() + log0 = _update_log(w).stat().st_size if _update_log(w).is_file() else 0 + prev = w.receipt() + target = w.publish("release: e2e hostile post profile d", {"docs/e2e-hostile-post-d.txt": "d\n"}) + + cp = w.update("--no-gateway-restart") + + prof_after = {p.name for p in prof_receipts.iterdir()} if prof_receipts.is_dir() else set() + log1 = _update_log(w).stat().st_size if _update_log(w).is_file() else 0 + rec = w.receipt() + diag = _facts(w, cp, profile_receipts_new=sorted(prof_after - prof_before), root_update_log_bytes=f"{log0}->{log1}", + previous_update_id=prev.get("update_id")) + w.diag(cp) + assert w.head() == target, f"premise: the release never committed:\n{diag}" + assert cp.returncode == 0, f"the sticky-profile update failed:\n{diag}" + # Only the UPDATE receipt is this lane's: pm's own sync receipts (pm_*sync*.json, its + # latest.json with kind "sync") still follow the process home — pm/ is out of scope here. + update_files = sorted(n for n in prof_after - prof_before if n.startswith("update_")) + prof_latest = _read_json(prof_receipts / "latest.json") + assert not update_files, f"the update wrote update receipts into the profile home {prof_receipts}:\n{diag}" + # pm's sync receipt carries the update's correlation id, so "kind" tells them apart. + assert prof_latest.get("kind", "sync") == "sync", \ + f"the profile home's latest.json is this update's receipt:\n{diag}" + assert log1 > log0, f"the root update.log did not grow:\n{diag}" + assert rec.get("update_id") != prev.get("update_id") and rec.get("outcome") == "success", \ + f"root latest.json is not this run's success:\n{diag}" + finally: + w.sb.cli("profile", "use", "default", timeout=300) diff --git a/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py b/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py index 1a0929113024..a8b9f92800d8 100644 --- a/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py +++ b/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py @@ -96,9 +96,12 @@ def test_restart_phase_records_accepted_self_restart_and_verify_exits_clean(monk assert "restart pending" in out.getvalue() assert "Update not complete" not in out.getvalue() assert cleared == [True] - with contextlib.redirect_stdout(io.StringIO()), pytest.raises(SystemExit) as exc: - restart.self_restart_pending_pids = set() # same fleet, identity not threaded → STALE, exit 1 + with contextlib.redirect_stdout(io.StringIO()) as out: + restart.self_restart_pending_pids = set() # same fleet, identity not threaded → STALE fleet_mod._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=True, ) - assert exc.value.code == 1 + # Contract C3: STALE is an owed restart (was SystemExit(1)): flagged, follow-up, marker kept. + assert restart.incomplete + assert "follow-up 'gateway_restart'" in out.getvalue() + assert cleared == [True] diff --git a/tests/hermes_cli/test_manual_serve_deferral.py b/tests/hermes_cli/test_manual_serve_deferral.py index 05c14404544f..5d01d360d728 100644 --- a/tests/hermes_cli/test_manual_serve_deferral.py +++ b/tests/hermes_cli/test_manual_serve_deferral.py @@ -37,11 +37,15 @@ def test_manual_deferral_survives_receipt_rotation(monkeypatch, capsys, kind, co restart.failed_or_stale_units.append("hermes-serve-work.service") restart.incomplete = True if condition != "alive": - with pytest.raises(SystemExit) as exc: - fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) - assert exc.value.code == 1 + # Contract C3: a restart that is still owed no longer fails the committed update (was + # SystemExit(1) + "partial") ... + fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) + # ... the obligation stays armed (unchanged) ... assert fleet._fleet_restart_obligation_armed() - assert update_receipt.read_latest_receipt()["outcome"] == "partial" + receipt = update_receipt.read_latest_receipt() + # ... and the receipt is a success naming the owed restart. + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] return fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) receipt = update_receipt.read_latest_receipt() diff --git a/tests/hermes_cli/test_source_build.py b/tests/hermes_cli/test_source_build.py index 9c3c9e85d9c6..dbdea3e14772 100644 --- a/tests/hermes_cli/test_source_build.py +++ b/tests/hermes_cli/test_source_build.py @@ -299,21 +299,26 @@ def products(): @pytest.mark.platforms("linux") @pytest.mark.parametrize("step", ["tui", "web", "desktop"]) def test_update_failure_raises_without_retries_or_replacing_live_app(source_products, step): - from hermes_cli.source_build import build_update_products + from hermes_cli.source_build import ProductBuildError, build_update_products root, acquired = source_products app = root / "apps/desktop/release/linux-unpacked/hermes" app.parent.mkdir(parents=True) app.write_text("previous app") (root / f"fail-{step}").touch() - with pytest.raises(subprocess.CalledProcessError): + with pytest.raises(ProductBuildError) as failure: build_update_products(root, desktop=True) - assert app.read_text() == "previous app" - assert not list((root / "apps/desktop").glob(".staging-*")) - order = ["deps", "tui", "web", "desktop"] - assert [event["step"] for event in _events(root)] == order[:order.index(step) + 1] + # The failure is still raised, naming the one product that failed (no retries) ... + assert len(failure.value.failures) == 1 + assert isinstance(failure.value.failures[0][1], subprocess.CalledProcessError) + # ... but it no longer skips the independent products after it (was order[:index + 1]). + assert [event["step"] for event in _events(root)] == ["deps", "tui", "web", "desktop"] assert acquired == ["npm"] - assert not (Path(os.environ["HERMES_HOME"]) / "desktop-build-stamp.json").exists() + assert not list((root / "apps/desktop").glob(".staging-*")) + if step == "desktop": + # A failed desktop build never replaces the live app nor stamps it. + assert app.read_text() == "previous app" + assert not (Path(os.environ["HERMES_HOME"]) / "desktop-build-stamp.json").exists() @pytest.mark.platforms("linux") diff --git a/tests/hermes_cli/test_source_completion_stamp.py b/tests/hermes_cli/test_source_completion_stamp.py index c445aafcd374..6ca887dae08f 100644 --- a/tests/hermes_cli/test_source_completion_stamp.py +++ b/tests/hermes_cli/test_source_completion_stamp.py @@ -52,12 +52,30 @@ def maintenance(**_kwargs): def test_failed_source_completion_does_not_publish_identity(tmp_path, monkeypatch): root = _repo(tmp_path) - _completion_dependencies(monkeypatch, lambda **_kwargs: False) + _completion_dependencies(monkeypatch, lambda **_kwargs: True) - assert not complete_source_checkout(root, desktop=False, assume_yes=True) + def broken_build(root, *, desktop): + raise RuntimeError("web UI build: npm exited 1") + + monkeypatch.setattr("hermes_cli.source_build.build_update_products", broken_build) + owed: list = [] + assert not complete_source_checkout(root, desktop=False, assume_yes=True, followups=owed) + # The tail is owed, so the checkout identity is not published. + assert [step for step, _ in owed] == ["build"] assert not (root / "install-stamp.json").exists() +def test_unsafe_sqlite_withholds_success_but_not_the_finished_tail(tmp_path, monkeypatch): + # Was: an unsafe SQLite verdict also withheld the stamp, so the tail stayed owed forever + # (re-running it cannot fix the interpreter, codemap §8 V1). The install still reports failure. + root = _repo(tmp_path) + _completion_dependencies(monkeypatch, lambda **_kwargs: False) + owed: list = [] + assert not complete_source_checkout(root, desktop=False, assume_yes=True, followups=owed) + assert owed == [] + assert (root / "install-stamp.json").is_file() + + def _sandboxed_marker(tmp_path, monkeypatch): """Point every update-lock reader at a marker inside this test's sandbox.""" marker = tmp_path / ".hermes-update-in-progress" diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index 91158c51c39b..8fef6c7e6009 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -155,6 +155,15 @@ def git(*args): "_current = contextvars.ContextVar('receipt', default=None)\n" "class UpdateReceipt: pass\n" "def record_stage(*args, **kwargs): pass\n" + "def record_skip(*args, **kwargs): pass\n" + "TAIL_FOLLOWUPS = frozenset({'launchers', 'build', 'maintenance', 'config_migration', 'completion'})\n" + "def record_followup(step, reason, **kwargs):\n" + " print(f' ⚠ Update follow-up {step!r} did not finish: {reason}')\n" + " r = _current.get()\n" + " if r is not None: r.data.setdefault('followups', []).append({'step': step, 'reason': reason})\n" + "def amend_terminal_followup(*args): pass\n" + "def _receipt_dir():\n" + " return pathlib.Path(os.environ['HERMES_HOME']) / 'logs/update_receipts'\n" "def finalize_pending_update_receipt(code, reason):\n" " r = _current.get()\n" " if r is None: return\n" @@ -370,7 +379,8 @@ def test_killed_selected_python_returns_signal_exit_status(transition): assert result["pm_receipt"]["update_id"] == request["receipt"]["update_id"] -def test_failed_build_preserves_exit_status_without_maintenance(transition): +def test_failed_build_after_commit_is_a_followup_and_later_steps_still_run(transition): + """Contract C3: the code is committed, so a failed product build cannot fail the update.""" from hermes_cli import update_completion root, git, old, new, request = transition @@ -380,13 +390,24 @@ def test_failed_build_preserves_exit_status_without_maintenance(transition): "def build_update_products(*a, **kw): raise subprocess.CalledProcessError(23, ['builder'])\n" ) result = update_completion.run_completion(request) - assert result["exit_code"] == 23 - assert result["receipt"]["outcome"] == "failed" + # Was 23 / "failed": a post-commit build failure exits 0 with a success receipt. + assert result["exit_code"] == 0 + assert result["receipt"]["outcome"] == "success" + # The failure is not silent: it is a named receipt follow-up. + assert [f["step"] for f in result["receipt"]["followups"]] == ["build"] + # The tail obligation stays armed so the next launch / `hermes update` rebuilds. assert (Path(request["home"]) / "completion-pending").read_text() == "owed" events = [json.loads(line)["name"] for line in (root / "events.jsonl").read_text().splitlines()] - assert "maintenance" not in events - assert "restart" not in events - assert "emergency_resume" in events + # Was "not in": the failed build no longer skips maintenance (config migration) ... + assert "maintenance" in events + # ... nor the gateway restart and its verification. + assert "restart" in events and "verify" in events + # The tail was not complete, so the install stamp is not written. + assert "stamp" not in events + # The gateway watcher hears success: the code it runs is the committed code. + assert {"name": "exit_marker", "ok": True}.items() <= next( + json.loads(line) for line in (root / "events.jsonl").read_text().splitlines() + if json.loads(line)["name"] == "exit_marker").items() def test_prepare_failure_preserves_correlated_pm_receipt(transition, monkeypatch): diff --git a/tests/hermes_cli/test_update_external_profile_reconciliation.py b/tests/hermes_cli/test_update_external_profile_reconciliation.py index 4ce7666af1e6..2c79bc921bde 100644 --- a/tests/hermes_cli/test_update_external_profile_reconciliation.py +++ b/tests/hermes_cli/test_update_external_profile_reconciliation.py @@ -94,20 +94,22 @@ def test_symlinked_external_profile_gateway_does_not_fail_update(external_profil def test_external_profile_does_not_mask_a_missed_own_gateway(external_profile_host): - """The root's own gateway missed by the restart phase still fails the update.""" + """The root's own gateway missed by the restart phase is still an owed restart.""" plan = UpdatePlan(runtimes=[_gateway("default", 1111), _gateway("work", 4242)]) restart = _restart_outcome() restart.restarted_services.clear() update_receipt.begin_update_receipt() - with pytest.raises(SystemExit) as exc: - fleet._verify_fleet_after_update( - restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True, - ) + # Contract C3: no longer SystemExit(1); the miss is flagged and owed instead. + fleet._verify_fleet_after_update( + restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True, + ) - assert exc.value.code == 1 - outcomes = {o["profile"]: o["outcome"] for o in update_receipt.read_latest_receipt()["runtime_outcomes"]} + assert restart.incomplete + receipt = update_receipt.read_latest_receipt() + outcomes = {o["profile"]: o["outcome"] for o in receipt["runtime_outcomes"]} assert outcomes == {"default": "unaccounted", "work": "external"} + assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] def test_external_requires_verified_pid_evidence(): diff --git a/tests/hermes_cli/test_update_fleet_check_fail_closed.py b/tests/hermes_cli/test_update_fleet_check_fail_closed.py index 42dd9aae9659..f78a39ebf6e0 100644 --- a/tests/hermes_cli/test_update_fleet_check_fail_closed.py +++ b/tests/hermes_cli/test_update_fleet_check_fail_closed.py @@ -72,19 +72,23 @@ def verify(): fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) + cleared = [] + monkeypatch.setattr(fleet, "_clear_fleet_restart_pending_marker", lambda: cleared.append(True)) + # Contract C3: the code is committed, so verification never fails the update (was SystemExit(1)). + verify() if had_gateway: - with pytest.raises(SystemExit) as failure: - verify() - assert failure.value.code == 1 assert events[0] == "settle" assert events.count("probe") > 1 assert "returned no rows" in capsys.readouterr().out else: - verify() assert events == ["probe"] + # Fail-closed survives as an OWED restart: the fleet obligation is kept armed ... assert restart.incomplete is had_gateway + assert cleared == ([] if had_gateway else [True]) receipt = json.loads((update_cmd.get_hermes_home() / "logs/update_receipts/latest.json").read_text()) - assert receipt["outcome"] == ("partial" if had_gateway else "success") + # ... and the receipt is a success that names the owed step (was outcome "partial"). + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt.get("followups", [])] == (["gateway_restart"] if had_gateway else []) @@ -105,3 +109,32 @@ def test_unmapped_stops_are_not_expected_rows(): out.stopped_unmapped_pids.discard(102) pre, killed = out.fleet_probe_signals() assert _fleet_probe_expected_runtimes(_plan([]), pre, None, out.restarted_services, killed) + + +def test_unmapped_stop_keeps_the_restart_owed(monkeypatch, tmp_path): + # Codemap §6 V7/V12: an unmapped gateway stopped with no successor used to be "accounted for" + # (update exit 0, obligation cleared) and stayed down silently. It still predicts no row (the + # test above), but the restart is now OWED: a receipt follow-up and an armed fleet obligation. + import json + from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_receipt + + monkeypatch.setattr(main, "PROJECT_ROOT", tmp_path) + monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) + monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) + monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: []) + cleared = [] + monkeypatch.setattr(fleet, "_clear_fleet_restart_pending_marker", lambda: cleared.append(True)) + out = fleet._GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[101], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], + killed_pids={101}, stopped_unmapped_pids={101}, + ) + update_receipt.begin_update_receipt() + fleet._verify_fleet_after_update(out, _pre_update_plan=_plan([]), _windows_gateway_resume=None, + update_complete=True) + receipt = json.loads((update_cmd.get_hermes_home() / "logs/update_receipts/latest.json").read_text()) + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] + assert "101" in receipt["followups"][0]["reason"] + assert cleared == [] diff --git a/tests/hermes_cli/test_update_fleet_completion.py b/tests/hermes_cli/test_update_fleet_completion.py index 984bfb0b1ffb..2a458ff58de6 100644 --- a/tests/hermes_cli/test_update_fleet_completion.py +++ b/tests/hermes_cli/test_update_fleet_completion.py @@ -1,6 +1,5 @@ """SQLite completion and fleet verification remain independent update outcomes.""" -from contextlib import nullcontext import json import pytest @@ -45,17 +44,18 @@ def collect(outcome, rows_expected): healthy = update_complete and state == "current" with update_receipt.update_receipt_scope(): update_receipt.begin_update_receipt() - with nullcontext() if healthy else pytest.raises(SystemExit) as exc: - update_cmd_fleet._verify_fleet_after_update( - restart, _pre_update_plan=None, _windows_gateway_resume=None, - update_complete=update_complete, - ) - if not healthy: - assert exc.value.code == 1 + # Contract C3: neither verdict fails the committed update any more (was SystemExit(1)). + update_cmd_fleet._verify_fleet_after_update( + restart, _pre_update_plan=None, _windows_gateway_resume=None, + update_complete=update_complete, + ) receipt = json.loads((get_hermes_home() / "logs/update_receipts/latest.json").read_text()) - assert receipt["outcome"] == ("success" if healthy else "partial") + # Was "partial" when not healthy: the run is a success; an owed fleet restart is a follow-up. + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt.get("followups", [])] == ([] if state == "current" else ["gateway_restart"]) assert receipt["fleet"] == snapshot + # The two verdicts stay independent: only the fleet keeps the restart obligation armed. assert restart.incomplete is (state != "current") assert update_cmd_fleet._fleet_restart_obligation_armed() is (state != "current") assert migrated == ([True] if healthy else []) diff --git a/tests/hermes_cli/test_update_fleet_restart_pending.py b/tests/hermes_cli/test_update_fleet_restart_pending.py index 357dbd8c79ab..d284761ecbd2 100644 --- a/tests/hermes_cli/test_update_fleet_restart_pending.py +++ b/tests/hermes_cli/test_update_fleet_restart_pending.py @@ -412,8 +412,9 @@ def test_clean_update_escalates_surviving_serve_as_unaccounted( ``hermes-gateway.service``) and an unmanaged ``serve`` on the same default profile. The serve survives the update as the SAME process, so the update must (1) warn, (2) reconcile it as ``unaccounted`` instead of - borrowing the gateway's restart, and (3) exit 1 with a ``partial`` - receipt — not print a clean success.""" + borrowing the gateway's restart, and (3) under contract C3 (the code is + committed) end ``success`` with an owed ``gateway_restart`` follow-up and + the restart obligation still armed — never a silent clean success.""" from hermes_cli.update_inventory import ( RuntimeRecord, UpdatePlan, _restart_mechanism, ) @@ -460,18 +461,24 @@ def _match(p, **kw): lambda **_k: [{"pid": 5555, "purpose": "serve", "create_time": 1000.0}], ) - with pytest.raises(SystemExit) as excinfo: - hermes_main.cmd_update(args) - assert excinfo.value.code == 1 + # Was SystemExit(1): the committed update no longer fails (contract C3). + hermes_main.cmd_update(args) out = capsys.readouterr().out + # (1) still warned, unchanged. assert "pid 5555" in out and "pre-update code" in out assert "Planned runtimes the restart phase never touched" in out assert "serve [default] pid 5555" in out + assert "follow-up 'gateway_restart'" in out latest = get_hermes_home() / "logs" / "update_receipts" / "latest.json" receipt = json.loads(latest.read_text(encoding="utf-8")) - assert receipt["outcome"] == "partial" + # (3) was "partial": a success that names the owed restart. + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] + from hermes_cli import update_cmd_fleet + assert update_cmd_fleet._fleet_restart_obligation_armed() + # (2) the reconciliation verdict is unchanged. by_pid = {o["pid"]: o["outcome"] for o in receipt["runtime_outcomes"]} assert by_pid == {4444: "restarted", 5555: "unaccounted"} diff --git a/tests/hermes_cli/test_update_hangup_protection.py b/tests/hermes_cli/test_update_hangup_protection.py index a4b6c5a54133..b924353f09cd 100644 --- a/tests/hermes_cli/test_update_hangup_protection.py +++ b/tests/hermes_cli/test_update_hangup_protection.py @@ -118,7 +118,7 @@ def test_wraps_stdout_and_stderr_with_mirror(self, tmp_path, monkeypatch): def test_non_fatal_if_log_setup_fails(self, monkeypatch): - """If get_hermes_home() raises, stdio must be left untouched but SIGHUP still handled.""" + """If the root-home lookup raises, stdio must be left untouched but SIGHUP still handled.""" prev_out, prev_err = sys.stdout, sys.stderr def _boom(): @@ -126,7 +126,7 @@ def _boom(): # Patch the import inside _install_hangup_protection. monkeypatch.setattr( - "hermes_cli.config.get_hermes_home", _boom, raising=True + "hermes_constants.get_default_hermes_root", _boom, raising=True ) original_handler = ( diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py new file mode 100644 index 000000000000..f7fe1f61561b --- /dev/null +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -0,0 +1,178 @@ +"""Contract C3: after the commit point nothing fails `hermes update`. + +Post-commit step failures are ⚠ + a receipt follow-up + an armed obligation; the receipt +is durable while the run is open; receipts resolve to the root home. +""" + +from __future__ import annotations + +import json +import os +import subprocess + +import pytest + +from hermes_cli import update_receipt + + +def _latest(home): + return json.loads((home / "logs/update_receipts/latest.json").read_text()) + + +def test_open_receipt_is_durably_running_and_finalizes_in_place(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.begin_update_receipt() + running = _latest(tmp_path) + assert running["outcome"] == "running" and running["pid"] == os.getpid() + run_files = list((tmp_path / "logs/update_receipts").glob(f"update_*_{running['update_id']}.json")) + assert len(run_files) == 1 + update_receipt.record_stage("apply", "success") + assert [s["name"] for s in _latest(tmp_path)["stages"]] == ["apply"] + update_receipt.record_followup("build", "web UI build: npm exited 1") + update_receipt.finalize_update_receipt("success") + final = _latest(tmp_path) + assert final["outcome"] == "success" + assert [(f["step"], f["reason"]) for f in final["followups"]] == [("build", "web UI build: npm exited 1")] + # The terminal record replaced the running one in the SAME archive file (no duplicate per run). + assert list((tmp_path / "logs/update_receipts").glob(f"update_*_{final['update_id']}.json")) == run_files + + +def test_dead_running_record_is_reported_interrupted_by_the_next_run(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + dead = subprocess.Popen(["true"]) + dead.wait() + update_receipt.begin_update_receipt() + update_receipt.record_stage("deps", "success") + killed = update_receipt._current.get().data + killed_id = killed["update_id"] + # Simulate the kill: the record on disk names a process that is gone. + for path in (tmp_path / "logs/update_receipts").glob("*.json"): + data = json.loads(path.read_text()) + data.update(pid=dead.pid, writer_pid=dead.pid, pid_create_time=None) + path.write_text(json.dumps(data)) + update_receipt._current.set(None) + + update_receipt.begin_update_receipt() + out = capsys.readouterr().out + assert "interrupted" in out and "last stage: deps" in out + (archived,) = (tmp_path / "logs/update_receipts").glob(f"update_*_{killed_id}.json") + assert json.loads(archived.read_text())["outcome"] == "interrupted" + assert _latest(tmp_path)["update_id"] != killed_id + update_receipt.finalize_update_receipt("success") + + +def test_live_running_record_is_not_reclaimed(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.begin_update_receipt() + own_id = update_receipt._current.get().data["update_id"] + for path in (tmp_path / "logs/update_receipts").glob("*.json"): + data = json.loads(path.read_text()) + data.update(pid=os.getppid(), writer_pid=os.getppid(), pid_create_time=None) # alive + path.write_text(json.dumps(data)) + assert update_receipt.reconcile_interrupted_runs() == [] + (archived,) = (tmp_path / "logs/update_receipts").glob(f"update_*_{own_id}.json") + assert json.loads(archived.read_text())["outcome"] == "running" + update_receipt.finalize_update_receipt("success") + + +def test_profile_process_writes_receipts_to_the_root_home(tmp_path, monkeypatch): + root = tmp_path / ".hermes" + profile = root / "profiles/work" + profile.mkdir(parents=True) + monkeypatch.setattr("hermes_constants._get_platform_default_hermes_home", lambda: root) + monkeypatch.setattr("hermes_constants._default_hermes_root_memo", None) + monkeypatch.setenv("HERMES_HOME", str(profile)) + assert update_receipt._receipt_dir() == root / "logs/update_receipts" + + +def test_failed_web_build_does_not_skip_the_tui(tmp_path, monkeypatch, capsys): + from hermes_cli import source_build + + built = [] + monkeypatch.setattr("hermes_cli.main_install_repair._install_configured_features_missing_deps", lambda root: None) + monkeypatch.setattr("hermes_cli.update_stage.publish_stage", lambda text: None) + monkeypatch.setattr("hermes_cli.memory_provider_migration.migrate_all_homes", lambda: None) + monkeypatch.setattr(source_build, "source_frontends", lambda root: ("web", "ui-tui")) + monkeypatch.setattr(source_build, "source_build_env", lambda **kw: {"PATH": ""}) + monkeypatch.setattr(source_build, "prepare_source_dependencies", lambda *a, **kw: None) + monkeypatch.setattr(source_build, "source_product_current", lambda *a: False) + monkeypatch.setattr(source_build, "build_source_tui", lambda root, env: built.append("tui")) + + def broken_web(root, env): + raise subprocess.CalledProcessError(1, ["npm", "run", "build"]) + + monkeypatch.setattr(source_build, "build_source_web", broken_web) + with pytest.raises(source_build.ProductBuildError) as failure: + source_build.build_update_products(tmp_path, desktop=False) + assert built == ["tui"] + assert [name for name, _ in failure.value.failures] == ["web UI build"] + assert "npm run build exited 1" in str(failure.value) + + +def test_failed_config_migration_is_owed_and_later_maintenance_runs(tmp_path, monkeypatch, capsys): + from hermes_cli import update_cmd, update_cmd_maint as maint + + calls = [] + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + for name in ("_verify_and_restore_state_dbs_post_update", "_invalidate_live_plugin_catalog_caches", + "_print_bundled_skills_sync_report", "_sync_profiles_after_update"): + monkeypatch.setattr(maint, name, lambda name=name: calls.append(name)) + monkeypatch.setattr("hermes_cli.gitlock.fetch_full_commit_graph", lambda *a, **kw: False) + monkeypatch.setattr("hermes_cli.model_catalog.seed_cache_from_checkout", lambda root: False) + + def broken_migration(**kwargs): + raise SystemExit(1) # a migration helper that exits must not fail the committed update + + monkeypatch.setattr(update_cmd, "_check_and_apply_config_migration", broken_migration) + monkeypatch.setattr(maint, "_print_verified_update_completion", lambda message: calls.append("verdict") or True) + monkeypatch.setattr(maint, "_print_post_update_notices_and_self_heals", lambda: calls.append("notices")) + update_receipt.begin_update_receipt() + owed: list = [] + assert maint._run_post_update_maintenance( + assume_yes=True, gateway_mode=False, pre_update_snapshot_id=None, + had_desktop_app_before_update=False, pre_update_version=None, followups=owed) is True + assert [step for step, _ in owed] == ["config_migration"] + assert calls[-2:] == ["verdict", "notices"] + assert "⚠ Update follow-up 'config_migration'" in capsys.readouterr().out + update_receipt.finalize_update_receipt("success") + assert [f["step"] for f in _latest(tmp_path)["followups"]] == ["config_migration"] + + +def test_owed_restart_names_its_gateways_so_a_dead_fleet_cannot_discharge_it(tmp_path, monkeypatch): + # A gateway that died at boot leaves no live row. An inventory-less obligation would be settled + # by the gateway-less discharge, silencing the warning that replaces exit 1 under C3. + from hermes_cli import update_cmd_fleet as fleet + from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("HERMES_GATEWAY_LOCK_DIR", str(tmp_path / "locks")) + fleet._write_fleet_restart_pending_marker(expected_sha="a" * 40) + plan = UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default", pid=4242), + RuntimeRecord(kind="dashboard", profile="default", pid=4343)]) + fleet._record_owed_gateway_inventory(plan) + + inventory = json.loads(fleet._obligation_fields()["inventory"]) + assert [(r["kind"], r["profile"]) for r in inventory["runtimes"]] == [("gateway", "default")] + # HEAD still holds the pulled code, so only the fleet evidence can settle it. + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "a" * 40) + monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda: []) + assert fleet._marker_only_restart_obsolete() is False + assert fleet._fleet_restart_obligation_armed() + + +def test_owed_restart_rearms_a_settled_obligation_and_a_later_run_keeps_owing_it(tmp_path, monkeypatch): + # A pre-restart probe can settle an inventory-less record; the owed restart must re-arm it, and + # a later verify that finds nothing to restart must keep owing the named gateway. + from hermes_cli import update_cmd_fleet as fleet + from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("HERMES_GATEWAY_LOCK_DIR", str(tmp_path / "locks")) + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "b" * 40) + monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda: []) + assert not fleet._fleet_restart_obligation_armed() + fleet._record_owed_gateway_inventory(UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default")])) + assert fleet._obligation_fields()["expected_sha"] == "b" * 40 + # A later verify with nothing live to restart still owes the named gateway (and keeps it armed). + assert fleet._named_gateways_still_owed() + assert fleet._fleet_restart_obligation_armed() diff --git a/tests/hermes_cli/test_update_stale_gateway_yield.py b/tests/hermes_cli/test_update_stale_gateway_yield.py index 9a2fcba478c6..11b9937fffe9 100644 --- a/tests/hermes_cli/test_update_stale_gateway_yield.py +++ b/tests/hermes_cli/test_update_stale_gateway_yield.py @@ -103,7 +103,7 @@ def test_update_signals_proven_stale_gateway_survivor(cron_home, tmp_path): def test_verify_fleet_hands_stale_rows_to_survivor_signalling(monkeypatch): - """The wiring: a stale fleet matrix reaches signal_stale_fleet_survivors before exit 1.""" + """The wiring: a stale fleet matrix reaches signal_stale_fleet_survivors; the restart stays owed.""" import hermes_cli.update_cmd_fleet as fleet_mod import hermes_cli.update_cmd_stale_survivors as surv import hermes_cli.update_cmd as update_cmd @@ -121,9 +121,10 @@ def test_verify_fleet_hands_stale_rows_to_survivor_signalling(monkeypatch): incomplete=False, phase_errors=[], pre_restart_gateway_pids=[4242], restarted_services=[], failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set(), ) - with contextlib.redirect_stdout(io.StringIO()), pytest.raises(SystemExit) as exc: + with contextlib.redirect_stdout(io.StringIO()): fleet_mod._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=True, ) - assert exc.value.code == 1 + # Contract C3: was SystemExit(1); now flagged incomplete (owed restart, marker kept). + assert restart.incomplete assert seen["fleet"] == stale_fleet diff --git a/tests/hermes_cli/test_update_zip_completion.py b/tests/hermes_cli/test_update_zip_completion.py index a60d76b59603..2eda523fa2e9 100644 --- a/tests/hermes_cli/test_update_zip_completion.py +++ b/tests/hermes_cli/test_update_zip_completion.py @@ -168,26 +168,24 @@ def test_zip_helper_propagates_completion_status_after_real_verification(zip_upd {"pid": plan.runtimes[0].pid, "profile": "default"}]) request = update_cmd._source_completion_request( update_cmd._resolve_update_options(args, True), plan, snapshot, state.token, False, True) - if verdict == "healthy": - assert update_cmd_zip._update_via_zip(args, completion_request=request) is True - else: - with pytest.raises(SystemExit) as error: - update_cmd_zip._update_via_zip(args, completion_request=request) - assert error.value.code == 1 - assert state.events == ["prepare", ("marker", verdict != "unsafe-sqlite"), - "restart", "resume", "finalize"] + # Contract C3: the swapped code is committed, so neither verdict fails the update (was SystemExit(1)). + assert update_cmd_zip._update_via_zip(args, completion_request=request) is True + # The gateway watcher hears success for every verdict (was False for unsafe SQLite). + assert state.events == ["prepare", ("marker", True), "restart", "resume", "finalize"] receipt = json.loads((state.active / "logs/update_receipts/latest.json").read_text()) - assert receipt["outcome"] == ("success" if verdict == "healthy" else "partial") + # Was "partial": a success that names what is still owed. + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt.get("followups", [])] == { + "healthy": [], "unsafe-sqlite": ["sqlite_runtime"], "stale-fleet": ["gateway_restart"]}[verdict] assert receipt["runtime_outcomes"][0]["outcome"] == ( "unaccounted" if verdict == "stale-fleet" else "restarted") assert json.loads(state.jobs.read_text()) == state.original_jobs @pytest.mark.parametrize("route", ["direct", "git-failure"]) -@pytest.mark.parametrize("failure", ["swap", "late-swap", "stage", "preparation"]) +@pytest.mark.parametrize("failure", ["swap", "late-swap", "stage"]) def test_zip_failure_recovers_pause_without_completion_mutations(zip_update, monkeypatch, route, failure): import os - import pm state = zip_update before = {profile: (profile / "config.yaml").read_bytes() @@ -225,12 +223,6 @@ def fail_copy(src, dst, *args, **kwargs): return copytree(src, dst, *args, **kwargs) monkeypatch.setattr(shutil, "copytree", fail_copy) expected = SystemExit - else: - def fail_preparation(*args, **kwargs): - assert (state.root / "payload.txt").read_text() == "new" - raise pm.InstallError("venv", "preparation stopped") - monkeypatch.setattr("hermes_cli.source_build.build_update_products", fail_preparation) - expected = pm.InstallError with pytest.raises(expected) as raised: update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), gateway_mode=True) if failure in {"swap", "late-swap"}: @@ -246,10 +238,41 @@ def fail_preparation(*args, **kwargs): assert {profile: (profile / "config.yaml").read_bytes() for profile in before} == before assert not (state.sibling / ".env").exists() assert json.loads(state.jobs.read_text()) == state.original_jobs - assert not (state.active / "logs/update_receipts/latest.json").exists() + # The receipt is durable from the start (was: no latest.json at all). This direct + # ``_cmd_update_impl`` call has no command-boundary finalizer, so it is still the open + # record of THIS run, never a success. + latest = state.active / "logs/update_receipts/latest.json" + assert not latest.exists() or json.loads(latest.read_text())["outcome"] != "success" assert not list(state.root.glob("*.hermes-update-*")) +@pytest.mark.parametrize("route", ["direct", "git-failure"]) +def test_zip_build_failure_after_swap_is_a_followup(zip_update, monkeypatch, route): + # Was the "preparation" case above (pm.InstallError propagated): the build runs after the + # swap committed the new tree, so under contract C3 it is an owed follow-up, not a failure. + import pm + + state = zip_update + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (route == "direct", ["git"], False)) + monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) + monkeypatch.setattr(update_cmd, "_should_zip_fallback_on_update_error", lambda exc: True) + + def fail_fetch(*args, **kwargs): + raise subprocess.CalledProcessError(1, ["git", "fetch"]) + monkeypatch.setattr(update_cmd, "_git_run", fail_fetch) + + def fail_build(*args, **kwargs): + assert (state.root / "payload.txt").read_text() == "new" + raise pm.InstallError("venv", "preparation stopped") + monkeypatch.setattr("hermes_cli.source_build.build_update_products", fail_build) + update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), gateway_mode=True) + assert (state.root / "payload.txt").read_text() == "new" + receipt = json.loads((state.active / "logs/update_receipts/latest.json").read_text()) + assert receipt["outcome"] == "success" + assert "build" in [f["step"] for f in receipt["followups"]] + assert state.token["resume_needed"] is False + + @pytest.mark.parametrize("entry", ["payload.txt", "tools"]) def test_zip_recovers_crashed_backup_before_failed_copy_and_retry(zip_update, monkeypatch, entry): import shutil diff --git a/website/docs/developer-guide/source-update-completion.md b/website/docs/developer-guide/source-update-completion.md index 87221130b791..8f43012b1399 100644 --- a/website/docs/developer-guide/source-update-completion.md +++ b/website/docs/developer-guide/source-update-completion.md @@ -35,13 +35,38 @@ fleet restart, Windows resume, dashboard deduplication and verification. The existing per-kind restart and abort-recovery algorithms remain; transient supervisor/process failures are real even without mixed-generation imports. Only the purge/reload workaround and independent retry/ZIP tail compositions disappear. -Gateway exit status is written before a restart can terminate the updater's cgroup, -and is demoted on later failure. Verification publishes the final receipt. +Gateway exit status is written before a restart can terminate the updater's cgroup. +Verification publishes the final receipt. + +## After the commit point nothing fails the update + +Once the tree has moved, `hermes update` exits 0 unless the code itself was rolled back. +Every completion step is independent: a failed launcher publish, product build (each of +TUI/web/desktop is attempted even when another failed), profile sync, config migration, +bytecode sweep, gateway restart/verification, Windows resume or retired-channel adoption +prints a `⚠` line and is appended to the receipt's `followups` as `{step, reason}` while the +receipt's `outcome` stays `"success"`. The step's own obligation stays armed: +`source-completion-pending` for the tail (launchers, build, maintenance, config migration), +the host fleet-restart obligation for gateways (the CLI startup warning keeps naming it; an +owed restart records the pre-update gateways on the obligation, so a gateway that died at boot +stays owed until it serves the checkout instead of being settled by the gateway-less discharge), +the unstamped bytecode fingerprint for the sweep. The next launch or `hermes update` — +including the "Already up to date" path, which runs the same completion — retries it. A host +stamped "restarted" for this commit whose fleet is still off the checkout restarts again +instead of dead-ending. Exit 2 (refused / concurrent) and exit 1 (nothing committed, or +rolled back) keep their meaning. + +The receipt is durable while the run is open: `begin_update_receipt` writes it as +`outcome: "running"` to the run's own archive file and `latest.json`, and each stage +boundary refreshes it, so a killed update leaves its own record. The next update marks a +`running` record whose processes are gone `interrupted` (naming its last stage) and reports +it. Receipts and the `update.log` tee resolve to the root home +(`hermes_constants.get_default_hermes_root()`), never a sticky profile's. ## Parent lifecycle and failures The parent waits and propagates the child's exact nonzero result (a signal is -mapped to shell-style 128+signal). A child cannot succeed by merely exiting zero: +mapped to shell-style 128+signal); post-commit step failures never produce one. A child cannot succeed by merely exiting zero: a terminal response with the matching receipt identity is required. The response returns the mutated Windows token so the parent's registered emergency resume does not repeat completed work. Normal parent completion performs no maintenance. diff --git a/website/docs/getting-started/updating.md b/website/docs/getting-started/updating.md index e7b9c2962e0f..8923f884e932 100644 --- a/website/docs/getting-started/updating.md +++ b/website/docs/getting-started/updating.md @@ -97,7 +97,7 @@ For an admitted source checkout, `hermes update` runs these phases: 3. **Post-pull syntax validation + auto-rollback** — after the pull, Hermes compiles the nine critical files every `hermes` invocation imports at startup. If any fails to parse (e.g. an orphan merge-conflict marker, an accidentally truncated file), Hermes rolls the install back so your shell stays bootable: it resets the checkout to the commit it ran before the pull (`git reset --hard `), or, when the update switched away from a detached checkout or a parked feature branch, returns there with `git checkout --detach ` / `git checkout ` (a branch it cannot check out again is restored detached at the same commit). Re-run `hermes update` once the upstream fix lands. 4. **Dependency preparation** — PM provisions required tools and prepares a complete Python environment from the new lock, existing extras, and enabled plugin requirements. It validates that environment before publishing its selection. A plugin never fails the update. A plugin that no longer fits the new core is added to `plugins.disabled` in every profile that enables it (a memory provider has `memory.provider` cleared). That covers a `requires-python` that excludes Hermes's Python, a `manifest_version` newer than this Hermes supports, and dependencies that the resolver proves can't resolve alongside core and earlier plugins in config order, or that fail their own build. A download or network failure gets one retry and is disabled if it fails again. The update prints `⚠ Disabled plugin '' in : `, records it in the receipt's warnings, and continues. Re-enable it with `hermes plugins enable ` once the plugin ships a compatible release, or once the network is back. A `requires_hermes` range the running version misses never disables a plugin, because a source checkout without release tags can read as an older release. That plugin sits out instead (`⚠ Left plugin '' … out of this update`), stays enabled, and rejoins once Hermes reports a version it accepts. A secondary profile whose config cannot be read sits out the same way until the config is fixed. Only a core that cannot build on its own fails this step. 5. **Config migration** — detects new config options added since your version and prompts you to set them -6. **Desktop rebuild (stage-and-swap)** — if the Hermes Desktop app was built from this checkout, it is rebuilt so the GUI matches the new code. The rebuild packs into a temporary staging directory next to `apps/desktop/release/`, verifies the staged app, and only then renames it over the previous build (on Windows a real-time scanner briefly holding `release/win-unpacked` is ridden out with a few short retries). A rebuild that fails at any point — corrupt Electron download, missing dependency, disk full — leaves the previous app untouched and launchable; the update fails at that step, and `hermes desktop --build-only --force-build` or the next `hermes update` retries the rebuild. On macOS the rebuilt bundle is then copied (with `ditto`, signature intact) over a stale `/Applications/Hermes.app` or `~/Applications/Hermes.app`, so the copy Finder and the Dock launch matches the backend; an installed copy that is currently running is left alone and the update tells you to quit it and run `hermes update` again. If a copy an earlier update installed has since been removed (including dragged to the Trash), the update rebuilds it if needed, puts it back and says so, so Finder, the Dock and Spotlight find Hermes again; `hermes uninstall` removes the app for good. On Windows, when the update finishes from inside the Desktop app it would rebuild (the app completing an interrupted update at launch), the rebuild is skipped with a notice instead: Windows locks a running app's files, and stopping the app would end the update with it. The rest of the update completes; quit Hermes Desktop and run `hermes desktop` from a terminal, or use **Update now** in **Settings → About**, to rebuild and reopen it. +6. **Desktop rebuild (stage-and-swap)** — if the Hermes Desktop app was built from this checkout, it is rebuilt so the GUI matches the new code. The rebuild packs into a temporary staging directory next to `apps/desktop/release/`, verifies the staged app, and only then renames it over the previous build (on Windows a real-time scanner briefly holding `release/win-unpacked` is ridden out with a few short retries). A rebuild that fails at any point — corrupt Electron download, missing dependency, disk full — leaves the previous app untouched and launchable; the update prints a `⚠` line, still finishes the remaining steps (config migration, gateway restart) and exits 0 with a `build` follow-up in its receipt, and the next launch, `hermes update` or `hermes desktop --build-only --force-build` retries the rebuild. On macOS the rebuilt bundle is then copied (with `ditto`, signature intact) over a stale `/Applications/Hermes.app` or `~/Applications/Hermes.app`, so the copy Finder and the Dock launch matches the backend; an installed copy that is currently running is left alone and the update tells you to quit it and run `hermes update` again. If a copy an earlier update installed has since been removed (including dragged to the Trash), the update rebuilds it if needed, puts it back and says so, so Finder, the Dock and Spotlight find Hermes again; `hermes uninstall` removes the app for good. On Windows, when the update finishes from inside the Desktop app it would rebuild (the app completing an interrupted update at launch), the rebuild is skipped with a notice instead: Windows locks a running app's files, and stopping the app would end the update with it. The rest of the update completes; quit Hermes Desktop and run `hermes desktop` from a terminal, or use **Update now** in **Settings → About**, to rebuild and reopen it. 7. **Gateway auto-restart**: running gateways are refreshed after the update completes. Service-managed gateways (systemd on Linux, launchd on macOS) restart through the service manager. Manual gateways are relaunched when Hermes can map their PID to a profile. Manually launched `hermes serve` / `hermes dashboard` backends are different: the updater leaves them running and asks their owner to restart them. See [Manual backend restart reminders](#manual-backend-restart-reminders). Backends owned by a running Desktop app remain the app's responsibility. 8. **Multiplex migration (multi-profile installs)** — once the fleet is verified on the new code, an install with two or more profiles that still run **one gateway per profile** is folded into a single multiplexed default gateway when nothing blocks it (same as `hermes gateway migrate --multiplex --yes`); if a blocker exists (a bot token shared by two profiles, a secondary profile binding a port with no `/p//` ingress) the update prints the blockers with their fixes and changes nothing. Single-profile installs are never touched. See [Migrating from per-profile gateways](../user-guide/multi-profile-gateways.md#migrating-from-per-profile-gateways). @@ -314,7 +314,7 @@ fleet caused only by the deferral does not make the update `partial`. ### Interrupted gateway restarts -If an earlier update pulled code but did not finish restarting the fleet, the next `hermes update` retries even when the checkout is already current. An empty process scan does not prove recovery: failed systemd units and installed launchd jobs may have no live PID. If discovery or restart fails, or a requested service cannot be verified active, the marker stays pending and the update exits nonzero. Recover the affected services with the printed commands and retry. +If an earlier update pulled code but did not finish restarting the fleet, the next `hermes update` retries even when the checkout is already current. An empty process scan does not prove recovery: failed systemd units and installed launchd jobs may have no live PID. If discovery or restart fails, or a requested service cannot be verified active, the marker stays pending: the update still exits 0 (the new code is in place), prints a `⚠` line and a `gateway_restart` follow-up in its receipt, every CLI start warns about the owed restart, and the next `hermes update` retries it. Recover the affected services with the printed commands, or rerun `hermes update`. A failed historical receipt alone does not prove that gateways are still stale. Startup and gateway-status warnings, as well as update catch-up, check for a live successor on the current checkout for every gateway profile recorded in that receipt. A manual gateway restart can settle receipt-only advice without rewriting the historical outcome. Manual backend obligations must first transfer to their separate reminders. diff --git a/website/docs/reference/cli-commands.md b/website/docs/reference/cli-commands.md index e9c28573fbc9..4cb19a237409 100644 --- a/website/docs/reference/cli-commands.md +++ b/website/docs/reference/cli-commands.md @@ -2017,8 +2017,8 @@ external update owner. See [Updating & Uninstalling](../getting-started/updating Additional behavior: - **Gateway restart.** After a successful update, Hermes attempts to restart all running gateway profiles of the home being updated (its root and every `profiles/` under it) automatically so they pick up the new code. Gateways and `hermes-gateway*` services that belong to a different `HERMES_HOME` on the same machine — another install, or a scratch home running `hermes update` — are named in the output and left alone. Use `hermes gateway restart` when you want to restart a gateway without applying an update. -- **Restart-phase recovery.** If the in-process restart phase aborts while importing the freshly pulled tree, supervised gateway profiles are retried through a clean Python process. Only restarts independently confirmed by systemd (`systemctl --user is-active`) are reported as verified; a relaunch that merely exited 0 is recorded as `relaunch_attempted` and still fails the update conservatively. Manual gateways and serve/dashboard runtimes are never killed without a relaunch authority; they are recorded as skipped with a reason and remain in the incomplete-update report with the exact restart command. -- **Update receipts + fleet version check.** Every run writes a machine-readable receipt to `~/.hermes/logs/update_receipts/` (pre-update fleet plan, steps, skips with reasons, restart outcome; `latest.json` points at the newest). After the restart phase the updater verifies each live gateway's running code against the updated checkout and prints a per-profile version matrix; a gateway still on pre-update code fails the update (exit 1) with the exact restart command. +- **Restart-phase recovery.** If the in-process restart phase aborts while importing the freshly pulled tree, supervised gateway profiles are retried through a clean Python process. Only restarts independently confirmed by systemd (`systemctl --user is-active`) are reported as verified; a relaunch that merely exited 0 is recorded as `relaunch_attempted` and the restart stays owed (a `gateway_restart` follow-up; the update itself still exits 0 because the code is in place). Manual gateways and serve/dashboard runtimes are never killed without a relaunch authority; they are recorded as skipped with a reason and remain in the incomplete-update report with the exact restart command. +- **Update receipts + fleet version check.** Every run writes a machine-readable receipt to `~/.hermes/logs/update_receipts/` in the root Hermes home, even from a sticky profile (pre-update fleet plan, steps, skips with reasons, restart outcome, follow-ups; `latest.json` points at the newest and reads `running` while an update is in progress). After the restart phase the updater verifies each live gateway's running code against the updated checkout and prints a per-profile version matrix. A gateway still on pre-update code does not fail the update once the new code is in place: the update prints a `⚠` line with the exact restart command, exits 0, records a `gateway_restart` follow-up, and keeps the restart owed — every CLI start warns about it and the next `hermes update` retries it. - **Local source changes.** For git installs, dirty tracked files and untracked files are auto-stashed before branch checkout or pull (`git stash push --include-untracked`). Interactive terminal updates ask before restoring the stash. Non-interactive updates restore it by default; set `updates.non_interactive_local_changes: discard` only on managed installs where local source edits should be thrown away after a successful pull. If stash restore conflicts or the pull fails, the stash is left in place for manual recovery. - **npm lockfile churn.** Before stashing or switching branches, Hermes makes a best-effort cleanup of tracked `package-lock.json` diffs produced by npm install/build steps. Commit or manually stash intentional lockfile edits before running `hermes update`. - **Pairing data snapshot.** Even when `--backup` is off, `hermes update` takes a lightweight snapshot of `~/.hermes/pairing/` and the Feishu comment rules before `git pull`. You can roll it back with `hermes backup restore --state pre-update` if a pull rewrites a file you were editing. From fa5dbd96641d5c48c16ac3d58dfa30d2e646f464 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:39:40 -0700 Subject: [PATCH 02/62] refactor(update): move post-restart fleet verification into update_cmd_fleet_verify update_cmd_fleet.py had grown past the 2,000-line cap (2,230). The verify phase (_verify_fleet_after_update, the settle-polled fleet snapshot, owed-inventory recording, legacy-unit warning) is its own topic: it runs after the restart phase and decides follow-up vs clear. It now lives in a sibling; restart-phase helpers it uses are read through update_cmd_fleet so their test patches still apply. update_cmd re-imports _verify_fleet_after_update from the new module. Tests retarget their patches to where production reads. No behaviour change. --- hermes_cli/AGENTS.md | 2 +- hermes_cli/update_cmd.py | 3 +- hermes_cli/update_cmd_fleet.py | 329 ---------------- hermes_cli/update_cmd_fleet_verify.py | 350 ++++++++++++++++++ .../test_fleet_matrix_self_restart_pending.py | 7 +- .../hermes_cli/test_manual_serve_deferral.py | 7 +- ..._update_external_profile_reconciliation.py | 9 +- tests/hermes_cli/test_update_finish.py | 5 +- .../test_update_fleet_check_fail_closed.py | 14 +- .../test_update_fleet_completion.py | 8 +- .../test_update_fleet_snapshot_settle.py | 25 +- .../test_update_post_commit_followups.py | 8 +- .../test_update_stale_gateway_yield.py | 7 +- .../hermes_cli/test_update_zip_completion.py | 6 +- 14 files changed, 405 insertions(+), 375 deletions(-) create mode 100644 hermes_cli/update_cmd_fleet_verify.py diff --git a/hermes_cli/AGENTS.md b/hermes_cli/AGENTS.md index 4fea926e603c..23bc568dd049 100644 --- a/hermes_cli/AGENTS.md +++ b/hermes_cli/AGENTS.md @@ -184,7 +184,7 @@ target; table-driven `_PREFLIGHT_CHECKS`; manifest `/gateway_migration. a re-run resumes from it; a named profile's `gateway install|start|run` refuse without `--force` via `gateway.py::_named_profile_refused_under_multiplexer`, dashboard twin `web_server_gateway.py::multiplexed_profile_refusal`); -`update_cmd_fleet._verify_fleet_after_update` calls `maybe_auto_migrate_after_update` on the success +`update_cmd_fleet_verify._verify_fleet_after_update` calls `maybe_auto_migrate_after_update` on the success path only; `gateway_migrate_guards.py` holds the auto-path-only refusals (table `_AUTO_MIGRATION_GUARDS`: other service domain / UNIX user / HERMES_HOME outside `profiles/` — notices for the explicit command, blockers for the hook) and the `gateway.auto_multiplex_migration` opt-out (#109954). Blockers reuse `GatewayRunner._adapter_credential_fingerprint` and `platform_binds_port`; diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 2d82730d8676..38eef9c30929 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -61,11 +61,12 @@ _restart_systemd_gateway_units, _run_pending_fleet_restart, _service_restart_sec, _service_unit_supports_graceful_sigusr1_restart, _surviving_gateway_pids_after_failed_restart, - _systemctl, _systemctl_reset_and_restart, _verify_fleet_after_update, + _systemctl, _systemctl_reset_and_restart, _wait_for_service_active, _warn_gateway_restart_phase_aborted, _warn_incomplete_gateway_fleet_restart, _warn_pending_fleet_restart, _warn_pending_fleet_restart_on_startup, _write_fleet_restart_pending_marker, _write_gateway_update_exit_code) +from hermes_cli.update_cmd_fleet_verify import _verify_fleet_after_update # noqa: F401 from hermes_cli.update_cmd_zip import ( # noqa: F401 _ZIP_PRESERVED_TOP_LEVEL, _ZIP_STAGING_ARTIFACT_SUFFIXES, _abort_zip_update_if_dirty_tree, _atomic_replace_dir, _commit_staged_replacements, _discard_staged, diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index 1ebb1f3089be..aca620b1b524 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -30,11 +30,6 @@ _FRESH_RESTART_SUPERVISORS = frozenset({"systemd", "launchd", "service", "s6"}) -# A supervisor can report a restarted unit active before the gateway finishes its -# bootstrap and publishes ``gateway_state.json``. Keep the readiness poll bounded, -# but allow the default systemd startup budget plus status-publication slack. -_FLEET_PROBE_SETTLE_TIMEOUT_SECONDS = 120.0 - _SYSTEMD_SCOPES = (("user", ["systemctl", "--user"]), ("system", ["systemctl"])) _LIST_GATEWAY_UNITS = [ "list-units", "hermes-gateway*", "hermes-serve*", "hermes-dashboard*", @@ -1812,330 +1807,6 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): return out -def _print_legacy_units_warning() -> None: - """Legacy hermes.service fights hermes-gateway.service over the bot token; warn on - every update until migrated.""" - from hermes_cli.gateway import (has_legacy_hermes_units, _find_legacy_hermes_units, supports_systemd_services) - if not (supports_systemd_services() and has_legacy_hermes_units()): - return - print() - print("⚠ Legacy Hermes gateway unit(s) detected:") - for name, path, is_sys in _find_legacy_hermes_units(): - scope = "system" if is_sys else "user" - print(f" {path} ({scope} scope)") - print() - print(" These pre-rename units (hermes.service) fight the current") - print(" hermes-gateway.service for the bot token and cause SIGTERM") - print(" flap loops. Remove them with:") - print() - print(" hermes gateway migrate-legacy") - print() - print(" (add `sudo` if any are in system scope)") - - -def _collect_fleet_snapshot(restart, rows_expected: bool) -> list: - """Fleet version rows, polled over a bounded settle window when runtimes are expected. - - Gateways need time to rewrite gateway_state.json; Windows resumes DETACHED (~10s boot), - so a single 2s sleep reported "no rows" on healthy resumes. A "down" row may be a - detached replacement still booting: poll until none remain or the deadline passes. - Pre-restart PIDs make a gateway stopped WITHOUT verified replacement a DOWN row (owed restart) - instead of no row at all. An ``unknown`` row whose pid is NOT a pre-restart pid is a successor - that has not published its code identity yet (a relaunched gateway can sit ~10s between process - start and its first runtime-status write, #112634) — keep polling; at the deadline it is flagged - ``identity_pending`` so the matrix does not call it a pre-stamping gateway. - """ - from hermes_cli.update_receipt import collect_fleet_versions - pending = getattr(restart, "self_restart_pending_pids", None) or None - if not rows_expected: - return collect_fleet_versions( - pre_restart_pids=restart.pre_restart_gateway_pids, self_restart_pending=pending) - pre_pids = restart.pre_restart_gateway_pids - _fleet_deadline = _time.monotonic() + _FLEET_PROBE_SETTLE_TIMEOUT_SECONDS - while True: - _time.sleep(2.0) - snapshot = collect_fleet_versions(pre_restart_pids=pre_pids, self_restart_pending=pending) - unstamped = [row for row in snapshot if _fleet_row_identity_pending(row, pre_pids)] - if snapshot and not unstamped and not any(row.get("state") == "down" for row in snapshot): - return snapshot - if _time.monotonic() >= _fleet_deadline or _restarted_units_gone( - getattr(restart, "restarted_scoped_units", ())): - for row in unstamped: - row["identity_pending"] = True - return snapshot - - -def _fleet_row_identity_pending(row: dict, pre_restart_pids) -> bool: - """An ``unknown`` row with no sha from a pid that did not exist at update start: a relaunched - gateway still booting, not a gateway that predates version stamping. A surviving pre-restart pid - (or no pid snapshot at all) is settled as-is — waiting cannot change what it publishes.""" - if row.get("state") != "unknown" or row.get("code_sha"): - return False - if pre_restart_pids is None: - return False - return row.get("pid") not in {int(p) for p in pre_restart_pids if isinstance(p, int)} - - -def _restarted_units_gone(scoped_units) -> bool: - """True when every restarted systemd unit is LOADED in its scope and neither active nor - activating: the successor died, nothing will publish a state stamp, so the settle poll should fail - closed now instead of at the deadline. Anything inconclusive keeps waiting: no units, systemctl - missing/slow, or ``LoadState=not-found`` — a unit name asked in a scope that does not own it - answers ``inactive`` exactly like a dead unit (#112466), so only a loaded unit can prove death.""" - if not scoped_units: - return False - scope_cmds = dict(_SYSTEMD_SCOPES) - for scoped in scoped_units: - scope, _, name = scoped.partition("/") - try: - stdout = _systemctl(scope_cmds[scope] + ["show", "-p", "LoadState,ActiveState", name], timeout=5).stdout - except (KeyError, FileNotFoundError, subprocess.TimeoutExpired): - return False - props = dict(line.split("=", 1) for line in stdout.splitlines() if "=" in line) - if props.get("LoadState") != "loaded": - return False - if props.get("ActiveState") in ("active", "activating", "reloading"): - return False - return True - - -def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict: - """Profile -> gateway PIDs alive after the restart phase, from the fleet snapshot. - - Incarnation evidence for gateway reconciliation (``match_runtime_outcomes``'s - ``live_gateway_pids``): the plan identifies a gateway by the profile it SERVES while the restart - bookkeeping names the SERVICE, and a service can serve a profile its name does not encode — the - profile-scoped name matcher then never credits the planned runtime. A ``down`` row reports the - PRE-restart PID (nothing replaced it), so it is never a successor; rows without a usable - profile/PID are skipped. - """ - live: dict = {} - for row in fleet_rows: - pid, profile = row.get("pid"), row.get("profile") - if profile and isinstance(pid, int) and row.get("state") != "down": - live.setdefault(str(profile), set()).add(pid) - return live - - -def _record_owed_gateway_inventory(plan) -> None: - """Name the pre-update gateways on an inventory-less obligation once their restart is owed. - - The pull arms the obligation before the restart phase knows what it owes. An inventory-less - record on a host whose gateway then died at boot is settled by the gateway-less discharge - (nothing live, nothing named), which would silence the owed-restart warning that replaces a - failed update under contract C3. A named inventory keeps it until those gateways run HEAD. - """ - if plan is None: - return - armed = _fleet_restart_obligation_armed() - fields = _obligation_fields() if armed else {} - if fields is None or fields.get("inventory") not in (None, "", "null"): - return # unreadable terms stay fail-closed; a recorded inventory is never rewritten - from dataclasses import asdict - rows = [ - asdict(runtime) for runtime in getattr(plan, "runtimes", ()) or () - if getattr(runtime, "kind", None) == "gateway" - and isinstance(getattr(runtime, "profile", None), str) - and runtime.profile.strip() and runtime.profile != "unknown" - ] - if rows: - # Not armed any more (a pre-restart probe settled an inventory-less record): re-arm, the - # restart is owed. The SHA is the code the fleet must be proven current on. - sha = fields.get("expected_sha", "") if armed else (_current_checkout_sha() or "") - _write_fleet_restart_pending_marker(expected_sha=sha, runtimes=rows) - - -def _named_gateways_still_owed() -> bool: - """True when the armed obligation NAMES gateways (recorded by an earlier owed restart) and the - live fleet still does not prove them current. An inventory-less record keeps its old rules.""" - if not _fleet_restart_obligation_armed(): - return False - fields = _obligation_fields() - if fields is None or fields.get("inventory") in (None, "", "null"): - return False - return not _marker_only_restart_obsolete() - - -def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, update_complete): - """Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve - probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize. - - Never fails the committed update (contract C3): when any gateway may still be stale it - records a ``gateway_restart`` follow-up and leaves ``fleet_restart_pending`` armed for the - next catch-up and the startup warning; otherwise clears the marker. - """ - from hermes_cli.update_cmd import ( - _m, _surviving_pre_update_serve_runtimes, _warn_stale_serve_runtimes, - ) - from hermes_cli.update_cmd_maint import _refresh_dashboard_after_update - with _best_effort('Legacy unit check during update failed: %s'): - _print_legacy_units_warning() - - # Restart a managed dashboard via systemd or stop stale manual ones (raw-killing - # a systemd-owned PID reads as clean stop and leaves the Cloudflare origin dead). - # Already-restarted units aren't redone. - # A dashboard it stopped and could not bring back is a promised restart that did not happen. - _dashboards_down = _refresh_dashboard_after_update(already_restarted_units=set(restart.restarted_services)) - if _dashboards_down: - restart.incomplete = True - - # Success-path twin of the abort-recovery probe: the restart phase only touches - # units, so a unit-less `hermes serve` keeps stale sys.modules. Runs AFTER - # dashboard cleanup so a respawned manual dashboard isn't a survivor. Rows feed - # reconciliation (survivor → owed restart); ``None`` = probe failed, stays fail-closed. - # Check if any pre-update serve/dashboard runtimes survived on pre-update code generations (#100479). - # This is the SUCCESS-path twin of the abort-recovery probe above: the restart phase only restarts - # units, so an sshd-spawned `serve --isolated` or a manual `hermes serve` (no unit) is left running its - # pre-update sys.modules graph — and its cron ticker keeps firing agent jobs that ImportError on every - # symbol added in the pulled range. The rows also feed the plan-vs-execution reconciliation below, so a - # survivor is escalated (an owed gateway_restart follow-up) instead of merely printed. - _stale_serve_rows: "list | None" = None - with _best_effort('Failed to check for surviving serve runtimes: %s'): - _stale_serve_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan) - if _stale_serve_rows: - _warn_stale_serve_runtimes(_stale_serve_rows) - - print() - print("Tip: You can now select a provider and model:") - print(" hermes model # Select provider and model") - - # Compare every live gateway's stamped code_sha against the fresh checkout - # instead of assuming the restart phase worked. - # Phase 1 (#91277): post-update fleet version verification. - _fleet_snapshot: list = [] - with _best_effort('Fleet version verification failed: %s'): - from hermes_cli.update_receipt import print_fleet_version_matrix - # Cross-platform "rows expected" signal: (restarted_services or killed_pids) - # never fires on Windows (pause/resume populates neither), so a healthy - # resumed gateway yielded zero rows and exit 0. - # See #93406. - # A gateway stopped WITHOUT a successor ("Restart manually") publishes no row by design, - # so it must not count as an expected one — otherwise an update whose only live gateways - # were unmapped reports "no rows" after correctly stopping them. - _pre_restart, _killed = restart.fleet_probe_signals() - _fleet_rows_expected = _m()._fleet_probe_expected_runtimes( - _pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed, - ) - _fleet_snapshot = _collect_fleet_snapshot(restart, _fleet_rows_expected) - if print_fleet_version_matrix(_fleet_snapshot): - restart.incomplete = True - # A proven-stale survivor must not keep running (its ticker yields every tick and - # nothing else restarts it, #117275): hand it to the drain-first restart path. - from hermes_cli.update_cmd_stale_survivors import signal_stale_fleet_survivors - signal_stale_fleet_survivors(_fleet_snapshot, restart, _gateway_drain_budget()) - elif not _fleet_snapshot and _fleet_rows_expected: - # collect_fleet_versions() swallows every failure, so zero rows with - # expected runtimes is indistinguishable from health — keep the restart owed. - print( - # Fleet probe returned zero rows even though at least one gateway runtime was (or may have - # been) live pre-update — POSIX restart bookkeeping, the pre-restart PID snapshot, the - # pre-update plan inventory, or the Windows pause/resume token all count as that signal. - # Every failure path inside collect_fleet_versions() is swallowed via logger.debug(), so an - # empty list is indistinguishable from a healthy fleet in the current output. Treat it as - # verification failure so the receipt carries a gateway_restart follow-up (#93406). - "\n⚠ Fleet version check returned no rows even though" - " gateway runtimes were expected — verification incomplete." - ) - restart.incomplete = True - - # Every runtime the PLAN saw must appear in restart bookkeeping; an - # unaccounted one is a silent miss and escalates like a STALE/DOWN row. - with _best_effort('Runtime-outcome reconciliation failed: %s'): - # An unaccounted runtime is the silent-miss class (a platform branch re-discovered its own targets - # and skipped one the inventory knew about) — escalate it exactly like a STALE/DOWN fleet row. See - # #91277. - if _pre_update_plan is not None and _pre_update_plan.runtimes: - from hermes_cli.update_inventory import (match_runtime_outcomes, report_unaccounted_runtimes) - from hermes_cli.update_receipt import row_is_external - # Gateway incarnation evidence, from the post-restart fleet snapshot collected above: a - # service can serve a profile its own name does not encode (root-home launchd label + - # sticky active profile, hashed custom HERMES_HOME), so the bookkeeping's service names - # alone cannot credit the planned runtime. A profile the probe produced no row for simply - # has no evidence — that runtime stays on the name-matching path (and logs it). - _live_gateway_pids = _live_gateway_pids_from_fleet(_fleet_snapshot) - _runtime_outcomes = match_runtime_outcomes( - _pre_update_plan, - restarted_services=restart.restarted_services, - relaunched_profiles=restart.relaunched_profiles, - externally_supervised_profiles=restart.externally_supervised_profiles, - killed_pids=restart.killed_pids, - failed_units=restart.failed_or_stale_units, - # Serve/dashboard reconcile by incarnation liveness, not unit names. - # See #100479. - stale_serve_pids=( - {row.get("pid") for row in _stale_serve_rows} - if _stale_serve_rows is not None - else None - ), - failed_respawn_pids=_dashboards_down, - # A symlinked profile served by another install's checkout (#120240). - external_gateway_pids={row.get("pid") for row in _fleet_snapshot if row_is_external(row)}, - live_gateway_pids=_live_gateway_pids, - ) - from dataclasses import asdict - from hermes_cli.update_serve_obligations import defer_manual_serve - - for runtime, outcome in zip(_pre_update_plan.runtimes, _runtime_outcomes): - if outcome["outcome"] == "unaccounted" and defer_manual_serve(asdict(runtime), require_alive=True): - outcome["outcome"] = "deferred" - if report_unaccounted_runtimes(_runtime_outcomes): - restart.incomplete = True - with suppress(Exception): - import hermes_cli.update_receipt as _ur - _active = _ur._current.get() - if _active is not None: - _active.data["runtime_outcomes"] = _runtime_outcomes - - if not restart.incomplete: - with _best_effort('Owed-gateway check failed: %s'): - if _named_gateways_still_owed(): - # A gateway a previous run could not restart is still not serving HEAD (nothing - # live to restart this time): keep owing it rather than clearing the obligation. - print(" ⚠ A gateway owed a restart by a previous update is still not serving this checkout.") - restart.incomplete = True - if getattr(restart, "stopped_unmapped_pids", None): - # A gateway stopped with no successor is owed a restart, not "accounted for": keep the - # fleet obligation armed so the startup warning names it until someone restarts it. - restart.incomplete = True - if restart.incomplete: - # Code is committed (contract C3): a gateway that may still run stale modules is a - # follow-up, never a failed update. The fleet obligation stays armed, every CLI start - # warns about it, and the next `hermes update` retries the restart. - from hermes_cli.update_receipt import record_followup - stopped = sorted(getattr(restart, "stopped_unmapped_pids", None) or ()) - record_followup( - "gateway_restart", - "gateways may still run pre-update code or were stopped without a successor" - + (f" (stopped PIDs {', '.join(map(str, stopped))})" if stopped else "") - + "; recover with `hermes gateway restart`", - ) - with _best_effort('Fleet restart inventory not recorded: %s'): - _record_owed_gateway_inventory(_pre_update_plan) - with _best_effort('Update receipt finalize failed: %s'): - from hermes_cli.update_receipt import finalize_update_receipt - # ``update_complete`` is kept for the historical takeover caller; a False verdict (the - # SQLite runtime) is reported as a follow-up by the maintenance step, not a failed run. - _receipt_path = finalize_update_receipt("success", fleet=_fleet_snapshot) - if _receipt_path is not None: - logger.info("Update receipt written: %s", _receipt_path) - - if restart.incomplete: - return - _clear_fleet_restart_pending_marker() - if not update_complete: - # Fleet caught up, but the selected runtime's SQLite is unsafe (reported by maintenance): - # leave the topology alone until the runtime is repaired. - return - # Fleet is healthy on the new code: fold per-profile gateways into one multiplexer when nothing - # blocks it (deterministic; never prompts), else print the blockers and the one-liner to run later. - try: - from hermes_cli.gateway_migrate import maybe_auto_migrate_after_update - maybe_auto_migrate_after_update() - except (Exception, SystemExit) as exc: # noqa: BLE001 — a SystemExit here must not fail a committed update - logger.warning('Multiplex auto-migration after update failed: %s', exc) - print(f" ⚠ Gateway multiplex migration did not finish: {exc} (run `hermes gateway migrate` later)") - - def _restart_phase_failure_is_incomplete(surviving, pre_restart_pids) -> bool: """Whether an escaped restart-phase exception must fail the update. diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py new file mode 100644 index 000000000000..c7be2947416a --- /dev/null +++ b/hermes_cli/update_cmd_fleet_verify.py @@ -0,0 +1,350 @@ +"""Post-restart fleet verification for ``hermes update`` (``update_cmd_fleet`` sibling). + +After the restart phase: compare every live gateway's stamped code against the checkout, reconcile +the pre-update plan against what the restart did, keep the restart owed (a ``gateway_restart`` +follow-up, never a failed update — contract C3) or clear the obligation, and finalize the receipt. +Restart-phase helpers stay in ``update_cmd_fleet`` and are read through it so their patches apply. +""" + +import logging +import subprocess +import time as _time +from contextlib import suppress + +from hermes_cli.update_cmd_common import _best_effort + +# Log-record parity with the origin module. +logger = logging.getLogger("hermes_cli.update_cmd") + +# A supervisor can report a restarted unit active before the gateway finishes its +# bootstrap and publishes ``gateway_state.json``. Keep the readiness poll bounded, +# but allow the default systemd startup budget plus status-publication slack. +_FLEET_PROBE_SETTLE_TIMEOUT_SECONDS = 120.0 + + +def _print_legacy_units_warning() -> None: + """Legacy hermes.service fights hermes-gateway.service over the bot token; warn on + every update until migrated.""" + from hermes_cli.gateway import (has_legacy_hermes_units, _find_legacy_hermes_units, supports_systemd_services) + if not (supports_systemd_services() and has_legacy_hermes_units()): + return + print() + print("⚠ Legacy Hermes gateway unit(s) detected:") + for name, path, is_sys in _find_legacy_hermes_units(): + scope = "system" if is_sys else "user" + print(f" {path} ({scope} scope)") + print() + print(" These pre-rename units (hermes.service) fight the current") + print(" hermes-gateway.service for the bot token and cause SIGTERM") + print(" flap loops. Remove them with:") + print() + print(" hermes gateway migrate-legacy") + print() + print(" (add `sudo` if any are in system scope)") + + +def _collect_fleet_snapshot(restart, rows_expected: bool) -> list: + """Fleet version rows, polled over a bounded settle window when runtimes are expected. + + Gateways need time to rewrite gateway_state.json; Windows resumes DETACHED (~10s boot), + so a single 2s sleep reported "no rows" on healthy resumes. A "down" row may be a + detached replacement still booting: poll until none remain or the deadline passes. + Pre-restart PIDs make a gateway stopped WITHOUT verified replacement a DOWN row (owed restart) + instead of no row at all. An ``unknown`` row whose pid is NOT a pre-restart pid is a successor + that has not published its code identity yet (a relaunched gateway can sit ~10s between process + start and its first runtime-status write, #112634) — keep polling; at the deadline it is flagged + ``identity_pending`` so the matrix does not call it a pre-stamping gateway. + """ + from hermes_cli.update_receipt import collect_fleet_versions + pending = getattr(restart, "self_restart_pending_pids", None) or None + if not rows_expected: + return collect_fleet_versions( + pre_restart_pids=restart.pre_restart_gateway_pids, self_restart_pending=pending) + pre_pids = restart.pre_restart_gateway_pids + _fleet_deadline = _time.monotonic() + _FLEET_PROBE_SETTLE_TIMEOUT_SECONDS + while True: + _time.sleep(2.0) + snapshot = collect_fleet_versions(pre_restart_pids=pre_pids, self_restart_pending=pending) + unstamped = [row for row in snapshot if _fleet_row_identity_pending(row, pre_pids)] + if snapshot and not unstamped and not any(row.get("state") == "down" for row in snapshot): + return snapshot + if _time.monotonic() >= _fleet_deadline or _restarted_units_gone( + getattr(restart, "restarted_scoped_units", ())): + for row in unstamped: + row["identity_pending"] = True + return snapshot + + +def _fleet_row_identity_pending(row: dict, pre_restart_pids) -> bool: + """An ``unknown`` row with no sha from a pid that did not exist at update start: a relaunched + gateway still booting, not a gateway that predates version stamping. A surviving pre-restart pid + (or no pid snapshot at all) is settled as-is — waiting cannot change what it publishes.""" + if row.get("state") != "unknown" or row.get("code_sha"): + return False + if pre_restart_pids is None: + return False + return row.get("pid") not in {int(p) for p in pre_restart_pids if isinstance(p, int)} + + +def _restarted_units_gone(scoped_units) -> bool: + """True when every restarted systemd unit is LOADED in its scope and neither active nor + activating: the successor died, nothing will publish a state stamp, so the settle poll should fail + closed now instead of at the deadline. Anything inconclusive keeps waiting: no units, systemctl + missing/slow, or ``LoadState=not-found`` — a unit name asked in a scope that does not own it + answers ``inactive`` exactly like a dead unit (#112466), so only a loaded unit can prove death.""" + from hermes_cli import update_cmd_fleet as fleet + if not scoped_units: + return False + scope_cmds = dict(fleet._SYSTEMD_SCOPES) + for scoped in scoped_units: + scope, _, name = scoped.partition("/") + try: + stdout = fleet._systemctl(scope_cmds[scope] + ["show", "-p", "LoadState,ActiveState", name], timeout=5).stdout + except (KeyError, FileNotFoundError, subprocess.TimeoutExpired): + return False + props = dict(line.split("=", 1) for line in stdout.splitlines() if "=" in line) + if props.get("LoadState") != "loaded": + return False + if props.get("ActiveState") in ("active", "activating", "reloading"): + return False + return True + + +def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict: + """Profile -> gateway PIDs alive after the restart phase, from the fleet snapshot. + + Incarnation evidence for gateway reconciliation (``match_runtime_outcomes``'s + ``live_gateway_pids``): the plan identifies a gateway by the profile it SERVES while the restart + bookkeeping names the SERVICE, and a service can serve a profile its name does not encode — the + profile-scoped name matcher then never credits the planned runtime. A ``down`` row reports the + PRE-restart PID (nothing replaced it), so it is never a successor; rows without a usable + profile/PID are skipped. + """ + live: dict = {} + for row in fleet_rows: + pid, profile = row.get("pid"), row.get("profile") + if profile and isinstance(pid, int) and row.get("state") != "down": + live.setdefault(str(profile), set()).add(pid) + return live + + +def _record_owed_gateway_inventory(plan) -> None: + """Name the pre-update gateways on an inventory-less obligation once their restart is owed. + + The pull arms the obligation before the restart phase knows what it owes. An inventory-less + record on a host whose gateway then died at boot is settled by the gateway-less discharge + (nothing live, nothing named), which would silence the owed-restart warning that replaces a + failed update under contract C3. A named inventory keeps it until those gateways run HEAD. + """ + from hermes_cli import update_cmd_fleet as fleet + if plan is None: + return + armed = fleet._fleet_restart_obligation_armed() + fields = fleet._obligation_fields() if armed else {} + if fields is None or fields.get("inventory") not in (None, "", "null"): + return # unreadable terms stay fail-closed; a recorded inventory is never rewritten + from dataclasses import asdict + rows = [ + asdict(runtime) for runtime in getattr(plan, "runtimes", ()) or () + if getattr(runtime, "kind", None) == "gateway" + and isinstance(getattr(runtime, "profile", None), str) + and runtime.profile.strip() and runtime.profile != "unknown" + ] + if rows: + # Not armed any more (a pre-restart probe settled an inventory-less record): re-arm, the + # restart is owed. The SHA is the code the fleet must be proven current on. + sha = fields.get("expected_sha", "") if armed else (fleet._current_checkout_sha() or "") + fleet._write_fleet_restart_pending_marker(expected_sha=sha, runtimes=rows) + + +def _named_gateways_still_owed() -> bool: + """True when the armed obligation NAMES gateways (recorded by an earlier owed restart) and the + live fleet still does not prove them current. An inventory-less record keeps its old rules.""" + from hermes_cli import update_cmd_fleet as fleet + if not fleet._fleet_restart_obligation_armed(): + return False + fields = fleet._obligation_fields() + if fields is None or fields.get("inventory") in (None, "", "null"): + return False + return not fleet._marker_only_restart_obsolete() + + +def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, update_complete): + """Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve + probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize. + + Never fails the committed update (contract C3): when any gateway may still be stale it + records a ``gateway_restart`` follow-up and leaves ``fleet_restart_pending`` armed for the + next catch-up and the startup warning; otherwise clears the marker. + """ + from hermes_cli import update_cmd_fleet as fleet + from hermes_cli.update_cmd import ( + _m, _surviving_pre_update_serve_runtimes, _warn_stale_serve_runtimes, + ) + from hermes_cli.update_cmd_maint import _refresh_dashboard_after_update + with _best_effort('Legacy unit check during update failed: %s'): + _print_legacy_units_warning() + + # Restart a managed dashboard via systemd or stop stale manual ones (raw-killing + # a systemd-owned PID reads as clean stop and leaves the Cloudflare origin dead). + # Already-restarted units aren't redone. + # A dashboard it stopped and could not bring back is a promised restart that did not happen. + _dashboards_down = _refresh_dashboard_after_update(already_restarted_units=set(restart.restarted_services)) + if _dashboards_down: + restart.incomplete = True + + # Success-path twin of the abort-recovery probe: the restart phase only touches + # units, so a unit-less `hermes serve` keeps stale sys.modules. Runs AFTER + # dashboard cleanup so a respawned manual dashboard isn't a survivor. Rows feed + # reconciliation (survivor → owed restart); ``None`` = probe failed, stays fail-closed. + # Check if any pre-update serve/dashboard runtimes survived on pre-update code generations (#100479). + # This is the SUCCESS-path twin of the abort-recovery probe above: the restart phase only restarts + # units, so an sshd-spawned `serve --isolated` or a manual `hermes serve` (no unit) is left running its + # pre-update sys.modules graph — and its cron ticker keeps firing agent jobs that ImportError on every + # symbol added in the pulled range. The rows also feed the plan-vs-execution reconciliation below, so a + # survivor is escalated (an owed gateway_restart follow-up) instead of merely printed. + _stale_serve_rows: "list | None" = None + with _best_effort('Failed to check for surviving serve runtimes: %s'): + _stale_serve_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan) + if _stale_serve_rows: + _warn_stale_serve_runtimes(_stale_serve_rows) + + print() + print("Tip: You can now select a provider and model:") + print(" hermes model # Select provider and model") + + # Compare every live gateway's stamped code_sha against the fresh checkout + # instead of assuming the restart phase worked. + # Phase 1 (#91277): post-update fleet version verification. + _fleet_snapshot: list = [] + with _best_effort('Fleet version verification failed: %s'): + from hermes_cli.update_receipt import print_fleet_version_matrix + # Cross-platform "rows expected" signal: (restarted_services or killed_pids) + # never fires on Windows (pause/resume populates neither), so a healthy + # resumed gateway yielded zero rows and exit 0. + # See #93406. + # A gateway stopped WITHOUT a successor ("Restart manually") publishes no row by design, + # so it must not count as an expected one — otherwise an update whose only live gateways + # were unmapped reports "no rows" after correctly stopping them. + _pre_restart, _killed = restart.fleet_probe_signals() + _fleet_rows_expected = _m()._fleet_probe_expected_runtimes( + _pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed, + ) + _fleet_snapshot = _collect_fleet_snapshot(restart, _fleet_rows_expected) + if print_fleet_version_matrix(_fleet_snapshot): + restart.incomplete = True + # A proven-stale survivor must not keep running (its ticker yields every tick and + # nothing else restarts it, #117275): hand it to the drain-first restart path. + from hermes_cli.update_cmd_stale_survivors import signal_stale_fleet_survivors + signal_stale_fleet_survivors(_fleet_snapshot, restart, fleet._gateway_drain_budget()) + elif not _fleet_snapshot and _fleet_rows_expected: + # collect_fleet_versions() swallows every failure, so zero rows with + # expected runtimes is indistinguishable from health — keep the restart owed. + print( + # Fleet probe returned zero rows even though at least one gateway runtime was (or may have + # been) live pre-update — POSIX restart bookkeeping, the pre-restart PID snapshot, the + # pre-update plan inventory, or the Windows pause/resume token all count as that signal. + # Every failure path inside collect_fleet_versions() is swallowed via logger.debug(), so an + # empty list is indistinguishable from a healthy fleet in the current output. Treat it as + # verification failure so the receipt carries a gateway_restart follow-up (#93406). + "\n⚠ Fleet version check returned no rows even though" + " gateway runtimes were expected — verification incomplete." + ) + restart.incomplete = True + + # Every runtime the PLAN saw must appear in restart bookkeeping; an + # unaccounted one is a silent miss and escalates like a STALE/DOWN row. + with _best_effort('Runtime-outcome reconciliation failed: %s'): + # An unaccounted runtime is the silent-miss class (a platform branch re-discovered its own targets + # and skipped one the inventory knew about) — escalate it exactly like a STALE/DOWN fleet row. See + # #91277. + if _pre_update_plan is not None and _pre_update_plan.runtimes: + from hermes_cli.update_inventory import (match_runtime_outcomes, report_unaccounted_runtimes) + from hermes_cli.update_receipt import row_is_external + # Gateway incarnation evidence, from the post-restart fleet snapshot collected above: a + # service can serve a profile its own name does not encode (root-home launchd label + + # sticky active profile, hashed custom HERMES_HOME), so the bookkeeping's service names + # alone cannot credit the planned runtime. A profile the probe produced no row for simply + # has no evidence — that runtime stays on the name-matching path (and logs it). + _live_gateway_pids = _live_gateway_pids_from_fleet(_fleet_snapshot) + _runtime_outcomes = match_runtime_outcomes( + _pre_update_plan, + restarted_services=restart.restarted_services, + relaunched_profiles=restart.relaunched_profiles, + externally_supervised_profiles=restart.externally_supervised_profiles, + killed_pids=restart.killed_pids, + failed_units=restart.failed_or_stale_units, + # Serve/dashboard reconcile by incarnation liveness, not unit names. + # See #100479. + stale_serve_pids=( + {row.get("pid") for row in _stale_serve_rows} + if _stale_serve_rows is not None + else None + ), + failed_respawn_pids=_dashboards_down, + # A symlinked profile served by another install's checkout (#120240). + external_gateway_pids={row.get("pid") for row in _fleet_snapshot if row_is_external(row)}, + live_gateway_pids=_live_gateway_pids, + ) + from dataclasses import asdict + from hermes_cli.update_serve_obligations import defer_manual_serve + + for runtime, outcome in zip(_pre_update_plan.runtimes, _runtime_outcomes): + if outcome["outcome"] == "unaccounted" and defer_manual_serve(asdict(runtime), require_alive=True): + outcome["outcome"] = "deferred" + if report_unaccounted_runtimes(_runtime_outcomes): + restart.incomplete = True + with suppress(Exception): + import hermes_cli.update_receipt as _ur + _active = _ur._current.get() + if _active is not None: + _active.data["runtime_outcomes"] = _runtime_outcomes + + if not restart.incomplete: + with _best_effort('Owed-gateway check failed: %s'): + if _named_gateways_still_owed(): + # A gateway a previous run could not restart is still not serving HEAD (nothing + # live to restart this time): keep owing it rather than clearing the obligation. + print(" ⚠ A gateway owed a restart by a previous update is still not serving this checkout.") + restart.incomplete = True + if getattr(restart, "stopped_unmapped_pids", None): + # A gateway stopped with no successor is owed a restart, not "accounted for": keep the + # fleet obligation armed so the startup warning names it until someone restarts it. + restart.incomplete = True + if restart.incomplete: + # Code is committed (contract C3): a gateway that may still run stale modules is a + # follow-up, never a failed update. The fleet obligation stays armed, every CLI start + # warns about it, and the next `hermes update` retries the restart. + from hermes_cli.update_receipt import record_followup + stopped = sorted(getattr(restart, "stopped_unmapped_pids", None) or ()) + record_followup( + "gateway_restart", + "gateways may still run pre-update code or were stopped without a successor" + + (f" (stopped PIDs {', '.join(map(str, stopped))})" if stopped else "") + + "; recover with `hermes gateway restart`", + ) + with _best_effort('Fleet restart inventory not recorded: %s'): + _record_owed_gateway_inventory(_pre_update_plan) + with _best_effort('Update receipt finalize failed: %s'): + from hermes_cli.update_receipt import finalize_update_receipt + # ``update_complete`` is kept for the historical takeover caller; a False verdict (the + # SQLite runtime) is reported as a follow-up by the maintenance step, not a failed run. + _receipt_path = finalize_update_receipt("success", fleet=_fleet_snapshot) + if _receipt_path is not None: + logger.info("Update receipt written: %s", _receipt_path) + + if restart.incomplete: + return + fleet._clear_fleet_restart_pending_marker() + if not update_complete: + # Fleet caught up, but the selected runtime's SQLite is unsafe (reported by maintenance): + # leave the topology alone until the runtime is repaired. + return + # Fleet is healthy on the new code: fold per-profile gateways into one multiplexer when nothing + # blocks it (deterministic; never prompts), else print the blockers and the one-liner to run later. + try: + from hermes_cli.gateway_migrate import maybe_auto_migrate_after_update + maybe_auto_migrate_after_update() + except (Exception, SystemExit) as exc: # noqa: BLE001 — a SystemExit here must not fail a committed update + logger.warning('Multiplex auto-migration after update failed: %s', exc) + print(f" ⚠ Gateway multiplex migration did not finish: {exc} (run `hermes gateway migrate` later)") diff --git a/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py b/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py index a8b9f92800d8..4664dde8b8d4 100644 --- a/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py +++ b/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py @@ -13,6 +13,7 @@ import pytest import hermes_cli.update_cmd_fleet as fleet_mod +import hermes_cli.update_cmd_fleet_verify as fleet_verify import hermes_cli.update_receipt as ur OLD = "a" * 40 @@ -77,7 +78,7 @@ def test_restart_phase_records_accepted_self_restart_and_verify_exits_clean(monk assert pending == {ancestor} _fleet_homes(monkeypatch, tmp_path, {"default": {"pid": ancestor, "gateway_state": "running", "code_sha": OLD}}) - monkeypatch.setattr(fleet_mod, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda *a, **k: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) monkeypatch.setattr(fleet_mod._time, "sleep", lambda s: None) @@ -90,7 +91,7 @@ def test_restart_phase_records_accepted_self_restart_and_verify_exits_clean(monk self_restart_pending_pids=pending, ) with contextlib.redirect_stdout(io.StringIO()) as out: - fleet_mod._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=True, ) # a SystemExit(1) here is the #119597 symptom assert "restart pending" in out.getvalue() @@ -98,7 +99,7 @@ def test_restart_phase_records_accepted_self_restart_and_verify_exits_clean(monk assert cleared == [True] with contextlib.redirect_stdout(io.StringIO()) as out: restart.self_restart_pending_pids = set() # same fleet, identity not threaded → STALE - fleet_mod._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=True, ) # Contract C3: STALE is an owed restart (was SystemExit(1)): flagged, follow-up, marker kept. diff --git a/tests/hermes_cli/test_manual_serve_deferral.py b/tests/hermes_cli/test_manual_serve_deferral.py index 5d01d360d728..4de557fd33eb 100644 --- a/tests/hermes_cli/test_manual_serve_deferral.py +++ b/tests/hermes_cli/test_manual_serve_deferral.py @@ -7,6 +7,7 @@ from hermes_cli import process_identity from hermes_cli import update_cmd_fleet as fleet +from hermes_cli import update_cmd_fleet_verify as fleet_verify from hermes_cli import update_receipt from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan from hermes_cli.update_serve_obligations import defer_manual_serve, retain_receipt_manual_serves @@ -20,7 +21,7 @@ def test_manual_deferral_survives_receipt_rotation(monkeypatch, capsys, kind, co plan = UpdatePlan(runtimes=[runtime]) monkeypatch.setattr(process_identity, "_pid_alive_matches", lambda *a: True) monkeypatch.setattr(process_identity, "ledger_entries", lambda: [{"pid": 900, "purpose": kind, "create_time": 1000.0}]) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd._finish_dashboard_update_cleanup", lambda *a, **k: None) monkeypatch.setattr("hermes_cli.gateway_migrate.maybe_auto_migrate_after_update", lambda: None) monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **k: []) @@ -39,7 +40,7 @@ def test_manual_deferral_survives_receipt_rotation(monkeypatch, capsys, kind, co if condition != "alive": # Contract C3: a restart that is still owed no longer fails the committed update (was # SystemExit(1) + "partial") ... - fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) + fleet_verify._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) # ... the obligation stays armed (unchanged) ... assert fleet._fleet_restart_obligation_armed() receipt = update_receipt.read_latest_receipt() @@ -47,7 +48,7 @@ def test_manual_deferral_survives_receipt_rotation(monkeypatch, capsys, kind, co assert receipt["outcome"] == "success" assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] return - fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) + fleet_verify._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) receipt = update_receipt.read_latest_receipt() assert receipt["runtime_outcomes"][0]["outcome"] == "deferred" assert not fleet._fleet_restart_obligation_armed() diff --git a/tests/hermes_cli/test_update_external_profile_reconciliation.py b/tests/hermes_cli/test_update_external_profile_reconciliation.py index 2c79bc921bde..32439847d8e0 100644 --- a/tests/hermes_cli/test_update_external_profile_reconciliation.py +++ b/tests/hermes_cli/test_update_external_profile_reconciliation.py @@ -14,6 +14,7 @@ import pytest from hermes_cli import update_cmd_fleet as fleet +from hermes_cli import update_cmd_fleet_verify as fleet_verify from hermes_cli import update_receipt from hermes_cli.update_inventory import ( RuntimeRecord, @@ -61,8 +62,8 @@ def external_profile_host(tmp_path, monkeypatch): monkeypatch.setattr(update_receipt, "_socket_identity", lambda _home: None) live = {root.resolve(): 1111, other_home.resolve(): 4242} monkeypatch.setattr("gateway.status.live_gateway_pid_for_home", lambda home: live.get(Path(home).resolve())) - monkeypatch.setattr(fleet, "_time", SimpleNamespace(sleep=lambda _s: None, monotonic=time.monotonic)) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_time", SimpleNamespace(sleep=lambda _s: None, monotonic=time.monotonic)) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd._finish_dashboard_update_cleanup", lambda *a, **k: None) monkeypatch.setattr("hermes_cli.gateway_migrate.maybe_auto_migrate_after_update", lambda: None) return SimpleNamespace(other_checkout=other_checkout) @@ -79,7 +80,7 @@ def test_symlinked_external_profile_gateway_does_not_fail_update(external_profil plan = UpdatePlan(runtimes=[_gateway("default", 1111), _gateway("work", 4242)]) update_receipt.begin_update_receipt() - fleet._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( _restart_outcome(), _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True, ) @@ -101,7 +102,7 @@ def test_external_profile_does_not_mask_a_missed_own_gateway(external_profile_ho update_receipt.begin_update_receipt() # Contract C3: no longer SystemExit(1); the miss is flagged and owed instead. - fleet._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True, ) diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index be9047b7447d..a5a9535de106 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -154,6 +154,7 @@ def execute(module): from hermes_cli import update_cmd as update from hermes_cli import update_cmd_maint as maint from hermes_cli import update_cmd_fleet as fleet + from hermes_cli import update_cmd_fleet_verify as fleet_verify from hermes_cli import gateway_migrate from hermes_cli import macos_tcc_anchor from hermes_cli import source_build @@ -196,11 +197,11 @@ def maintain(**kwargs): maint._print_post_update_notices_and_self_heals = lambda: None maint._sync_profiles_after_update = lambda: None maint._print_bundled_skills_sync_report = lambda: None - fleet._print_legacy_units_warning = lambda: None + fleet_verify._print_legacy_units_warning = lambda: None maint._refresh_dashboard_after_update = lambda **kwargs: None gateway_migrate.maybe_auto_migrate_after_update = lambda: None update._surviving_pre_update_serve_runtimes = lambda plan: [] - fleet._collect_fleet_snapshot = lambda *args: [] + fleet_verify._collect_fleet_snapshot = lambda *args: [] def restart(plan, gateway_mode): assert isinstance(plan, UpdatePlan) assert isinstance(plan.runtimes[0], RuntimeRecord) diff --git a/tests/hermes_cli/test_update_fleet_check_fail_closed.py b/tests/hermes_cli/test_update_fleet_check_fail_closed.py index f78a39ebf6e0..cfdd26d8b6c9 100644 --- a/tests/hermes_cli/test_update_fleet_check_fail_closed.py +++ b/tests/hermes_cli/test_update_fleet_check_fail_closed.py @@ -39,10 +39,10 @@ class TestCallSiteWiring: @pytest.mark.parametrize("had_gateway", [False, True], ids=["idle", "plan-saw-gateway"]) def test_empty_probe_settles_and_fails_only_when_rows_expected(self, monkeypatch, tmp_path, capsys, had_gateway): import json - from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_receipt + from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as fleet_verify, update_receipt monkeypatch.setattr(main, "PROJECT_ROOT", tmp_path) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) # Reconciliation is a separate guard; it must not supply this test's failure. @@ -59,7 +59,7 @@ def collect(**kwargs): events.append("probe") return [] - monkeypatch.setattr(fleet, "_time", types.SimpleNamespace(monotonic=lambda: now[0], sleep=sleep)) + monkeypatch.setattr(fleet_verify, "_time", types.SimpleNamespace(monotonic=lambda: now[0], sleep=sleep)) monkeypatch.setattr(update_receipt, "collect_fleet_versions", collect) restart = fleet._GatewayRestartOutcome( incomplete=False, phase_errors=[], pre_restart_gateway_pids=[], restarted_services=[], @@ -69,7 +69,7 @@ def collect(**kwargs): update_receipt.begin_update_receipt() def verify(): - fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, + fleet_verify._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) cleared = [] @@ -116,10 +116,10 @@ def test_unmapped_stop_keeps_the_restart_owed(monkeypatch, tmp_path): # (update exit 0, obligation cleared) and stayed down silently. It still predicts no row (the # test above), but the restart is now OWED: a receipt follow-up and an armed fleet obligation. import json - from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_receipt + from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as fleet_verify, update_receipt monkeypatch.setattr(main, "PROJECT_ROOT", tmp_path) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: []) @@ -131,7 +131,7 @@ def test_unmapped_stop_keeps_the_restart_owed(monkeypatch, tmp_path): killed_pids={101}, stopped_unmapped_pids={101}, ) update_receipt.begin_update_receipt() - fleet._verify_fleet_after_update(out, _pre_update_plan=_plan([]), _windows_gateway_resume=None, + fleet_verify._verify_fleet_after_update(out, _pre_update_plan=_plan([]), _windows_gateway_resume=None, update_complete=True) receipt = json.loads((update_cmd.get_hermes_home() / "logs/update_receipts/latest.json").read_text()) assert receipt["outcome"] == "success" diff --git a/tests/hermes_cli/test_update_fleet_completion.py b/tests/hermes_cli/test_update_fleet_completion.py index 2a458ff58de6..b423756dd225 100644 --- a/tests/hermes_cli/test_update_fleet_completion.py +++ b/tests/hermes_cli/test_update_fleet_completion.py @@ -4,7 +4,7 @@ import pytest -from hermes_cli import update_cmd, update_cmd_fleet, update_cmd_maint, update_receipt +from hermes_cli import update_cmd, update_cmd_fleet, update_cmd_fleet_verify as fleet_verify, update_cmd_maint, update_receipt from hermes_constants import get_hermes_home @@ -22,7 +22,7 @@ def test_fleet_completion_preserves_runtime_verdict_and_restart_obligation( restarted_services=["hermes-gateway"], failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set(), ) - monkeypatch.setattr(update_cmd_fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) monkeypatch.setattr( update_cmd_maint, "_refresh_dashboard_after_update", @@ -38,14 +38,14 @@ def collect(outcome, rows_expected): assert rows_expected is True return snapshot - monkeypatch.setattr(update_cmd_fleet, "_collect_fleet_snapshot", collect) + monkeypatch.setattr(fleet_verify, "_collect_fleet_snapshot", collect) update_cmd_fleet._write_fleet_restart_pending_marker() assert update_cmd_fleet._fleet_restart_obligation_armed() healthy = update_complete and state == "current" with update_receipt.update_receipt_scope(): update_receipt.begin_update_receipt() # Contract C3: neither verdict fails the committed update any more (was SystemExit(1)). - update_cmd_fleet._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=update_complete, ) diff --git a/tests/hermes_cli/test_update_fleet_snapshot_settle.py b/tests/hermes_cli/test_update_fleet_snapshot_settle.py index 62b55d3494d3..894b152da53a 100644 --- a/tests/hermes_cli/test_update_fleet_snapshot_settle.py +++ b/tests/hermes_cli/test_update_fleet_snapshot_settle.py @@ -14,6 +14,7 @@ from hermes_cli import update_cmd import hermes_cli.update_cmd_fleet as update_cmd_fleet +import hermes_cli.update_cmd_fleet_verify as fleet_verify from hermes_constants import get_hermes_home @@ -45,11 +46,11 @@ def test_snapshot_waits_for_late_current_gateway_state(monkeypatch) -> None: restart = SimpleNamespace(pre_restart_gateway_pids=[101]) - result = update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) + result = fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) assert result == [expected] assert clock.now > 30.0 - assert clock.now <= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS + assert clock.now <= fleet_verify._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS def test_snapshot_stops_waiting_once_the_restarted_unit_is_dead(monkeypatch) -> None: @@ -64,7 +65,7 @@ def test_snapshot_stops_waiting_once_the_restarted_unit_is_dead(monkeypatch) -> stdout="LoadState=loaded\nActiveState=failed\n", stderr="", returncode=0)) restart = SimpleNamespace(pre_restart_gateway_pids=[101], restarted_scoped_units={"user/hermes-gateway.service"}) - assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == [] + assert fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) == [] assert clock.now < 30.0 @@ -85,11 +86,11 @@ def systemctl(cmd, *, timeout): stderr="", returncode=0) monkeypatch.setattr(update_cmd_fleet, "_systemctl", systemctl) - assert update_cmd_fleet._restarted_units_gone(("system/hermes-gateway.service",)) is False + assert fleet_verify._restarted_units_gone(("system/hermes-gateway.service",)) is False restart = SimpleNamespace(pre_restart_gateway_pids=[101], restarted_scoped_units={"system/hermes-gateway.service"}) - assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == [] - assert clock.now >= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS + assert fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) == [] + assert clock.now >= fleet_verify._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS def test_snapshot_waits_for_the_relaunched_pid_to_publish_its_identity(monkeypatch) -> None: @@ -104,20 +105,20 @@ def test_snapshot_waits_for_the_relaunched_pid_to_publish_its_identity(monkeypat monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda **_kwargs: next(snapshots)) restart = SimpleNamespace(pre_restart_gateway_pids=[32512], restarted_scoped_units=set()) - assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == [current] + assert fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) == [current] assert clock.now == 12.0 # Control: the same unknown row for a pid that was already running pre-update is settled as-is. clock.now = 0.0 monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda **_kwargs: [dict(unknown)]) survivor = SimpleNamespace(pre_restart_gateway_pids=[34516], restarted_scoped_units=set()) - result = update_cmd_fleet._collect_fleet_snapshot(survivor, rows_expected=True) + result = fleet_verify._collect_fleet_snapshot(survivor, rows_expected=True) assert clock.now == 2.0 and "identity_pending" not in result[0] # Still unknown at the deadline: flagged so the matrix prints restart-aware copy, never "predates". clock.now = 0.0 - result = update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) - assert clock.now >= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS + result = fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) + assert clock.now >= fleet_verify._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS assert result[0]["identity_pending"] is True @@ -139,7 +140,7 @@ def test_verifier_clears_marker_after_late_current_gateway_state(monkeypatch) -> "hermes_cli.update_receipt.print_fleet_version_matrix", lambda _fleet: False, ) - monkeypatch.setattr(update_cmd_fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr(update_cmd, "_finish_dashboard_update_cleanup", lambda *a, **k: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda _plan: []) monkeypatch.setattr(update_cmd, "_warn_stale_serve_runtimes", lambda _rows: None) @@ -157,7 +158,7 @@ def test_verifier_clears_marker_after_late_current_gateway_state(monkeypatch) -> fleet_probe_signals=lambda: ([101], set()), ) - update_cmd_fleet._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py index f7fe1f61561b..c91576436c54 100644 --- a/tests/hermes_cli/test_update_post_commit_followups.py +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -142,6 +142,7 @@ def test_owed_restart_names_its_gateways_so_a_dead_fleet_cannot_discharge_it(tmp # A gateway that died at boot leaves no live row. An inventory-less obligation would be settled # by the gateway-less discharge, silencing the warning that replaces exit 1 under C3. from hermes_cli import update_cmd_fleet as fleet + from hermes_cli import update_cmd_fleet_verify as fleet_verify from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan monkeypatch.setenv("HERMES_HOME", str(tmp_path)) @@ -149,7 +150,7 @@ def test_owed_restart_names_its_gateways_so_a_dead_fleet_cannot_discharge_it(tmp fleet._write_fleet_restart_pending_marker(expected_sha="a" * 40) plan = UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default", pid=4242), RuntimeRecord(kind="dashboard", profile="default", pid=4343)]) - fleet._record_owed_gateway_inventory(plan) + fleet_verify._record_owed_gateway_inventory(plan) inventory = json.loads(fleet._obligation_fields()["inventory"]) assert [(r["kind"], r["profile"]) for r in inventory["runtimes"]] == [("gateway", "default")] @@ -164,6 +165,7 @@ def test_owed_restart_rearms_a_settled_obligation_and_a_later_run_keeps_owing_it # A pre-restart probe can settle an inventory-less record; the owed restart must re-arm it, and # a later verify that finds nothing to restart must keep owing the named gateway. from hermes_cli import update_cmd_fleet as fleet + from hermes_cli import update_cmd_fleet_verify as fleet_verify from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan monkeypatch.setenv("HERMES_HOME", str(tmp_path)) @@ -171,8 +173,8 @@ def test_owed_restart_rearms_a_settled_obligation_and_a_later_run_keeps_owing_it monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "b" * 40) monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda: []) assert not fleet._fleet_restart_obligation_armed() - fleet._record_owed_gateway_inventory(UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default")])) + fleet_verify._record_owed_gateway_inventory(UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default")])) assert fleet._obligation_fields()["expected_sha"] == "b" * 40 # A later verify with nothing live to restart still owes the named gateway (and keeps it armed). - assert fleet._named_gateways_still_owed() + assert fleet_verify._named_gateways_still_owed() assert fleet._fleet_restart_obligation_armed() diff --git a/tests/hermes_cli/test_update_stale_gateway_yield.py b/tests/hermes_cli/test_update_stale_gateway_yield.py index 11b9937fffe9..f967d21b1579 100644 --- a/tests/hermes_cli/test_update_stale_gateway_yield.py +++ b/tests/hermes_cli/test_update_stale_gateway_yield.py @@ -105,16 +105,17 @@ def test_update_signals_proven_stale_gateway_survivor(cron_home, tmp_path): def test_verify_fleet_hands_stale_rows_to_survivor_signalling(monkeypatch): """The wiring: a stale fleet matrix reaches signal_stale_fleet_survivors; the restart stays owed.""" import hermes_cli.update_cmd_fleet as fleet_mod + import hermes_cli.update_cmd_fleet_verify as fleet_verify import hermes_cli.update_cmd_stale_survivors as surv import hermes_cli.update_cmd as update_cmd from hermes_cli.update_cmd_fleet import _GatewayRestartOutcome stale_fleet = [{"profile": "default", "pid": 4242, "state": "stale", "code_sha": "a" * 40}] seen = {} - monkeypatch.setattr(fleet_mod, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr(update_cmd, "_finish_dashboard_update_cleanup", lambda *a, **k: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) - monkeypatch.setattr(fleet_mod, "_collect_fleet_snapshot", lambda restart, expected: stale_fleet) + monkeypatch.setattr(fleet_verify, "_collect_fleet_snapshot", lambda restart, expected: stale_fleet) monkeypatch.setattr(surv, "signal_stale_fleet_survivors", lambda fleet, restart, budget: seen.setdefault("fleet", fleet) and []) restart = _GatewayRestartOutcome( @@ -122,7 +123,7 @@ def test_verify_fleet_hands_stale_rows_to_survivor_signalling(monkeypatch): failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set(), ) with contextlib.redirect_stdout(io.StringIO()): - fleet_mod._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=True, ) # Contract C3: was SystemExit(1); now flagged incomplete (owed restart, marker kept). diff --git a/tests/hermes_cli/test_update_zip_completion.py b/tests/hermes_cli/test_update_zip_completion.py index 2eda523fa2e9..6f990d8feefa 100644 --- a/tests/hermes_cli/test_update_zip_completion.py +++ b/tests/hermes_cli/test_update_zip_completion.py @@ -11,7 +11,7 @@ import pytest from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_cmd_maint as maint -from hermes_cli import update_cmd_zip, update_receipt +from hermes_cli import update_cmd_fleet_verify as fleet_verify, update_cmd_zip, update_receipt from hermes_cli.config_defaults import DEFAULT_CONFIG from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan import hermes_yaml @@ -84,10 +84,10 @@ def prepare(selected, *, desktop): monkeypatch.setattr("hermes_cli.profiles.seed_profile_skills", lambda *a, **kw: {}) monkeypatch.setattr(update_cmd, "_reload_config_modules", lambda: None) monkeypatch.setattr(update_cmd, "_post_update_sqlite_runtime_status", lambda: (True, None)) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr(maint, "_refresh_dashboard_after_update", lambda **kwargs: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) - monkeypatch.setattr(fleet, "_collect_fleet_snapshot", lambda *args: []) + monkeypatch.setattr(fleet_verify, "_collect_fleet_snapshot", lambda *args: []) monkeypatch.setattr("hermes_cli.gateway_migrate.maybe_auto_migrate_after_update", lambda: None) def resume(received): From bf4758a07ea3277c63db770aa28519228610c0e3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:52:45 -0700 Subject: [PATCH 03/62] fix(update): a dependency sync that fails after the tree moved is a follow-up (A6); Ctrl-C says the code moved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The completion bootstrap's PM step (ensure_tools_for_sync / pm.sync_venv) runs after the checkout moved. Its failure ended in exit 1 with no receipt, which the Desktop reads as "still on the previous version" while the tree is new. - update_completion._bootstrap: a failure there is now settled as a post-commit follow-up: the tail obligation stays armed, the run's receipt is a success naming a `dependencies` follow-up ("dependencies not installed yet — the next launch retries"), exit 0 (.update_exit_code 0 in gateway mode). A prepared child that dies without a result settles the same way (`completion`). - The parent resumes paused Windows gateways itself when the bootstrap could not (it has no dependencies to do so). - Ctrl-C after the commit point (parent in run_completion, or the child) closes the run as `interrupted` (never `failed`), exit 130, and prints that the new code is in place and its remaining steps are owed. - TAIL_FOLLOWUPS gains `dependencies`. Restores the dropped zip-completion "preparation" case with the new expectation. --- hermes_cli/update_cmd.py | 17 +++- hermes_cli/update_completion.py | 82 ++++++++++++++++--- hermes_cli/update_receipt.py | 32 +++++++- .../test_update_completion_process.py | 60 +++++++------- .../test_update_fleet_restart_pending.py | 4 +- .../hermes_cli/test_update_zip_completion.py | 69 ++++++++++++++++ 6 files changed, 220 insertions(+), 44 deletions(-) diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 38eef9c30929..dd292ba496b8 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -739,13 +739,28 @@ def _complete_source_update(request: dict | None) -> None: # A head capture that came back empty must not arm an SHA-less record: it names no code the # fleet can be proven current on, so the warning could never clear (#125952). _write_fleet_restart_pending_marker(expected_sha=request.get("expected_sha") or _current_checkout_sha() or "") - result = run_completion(request) + try: + result = run_completion(request) + except KeyboardInterrupt as interrupt: + # Ctrl-C after the commit point: the tree is new, so the run is interrupted, never "failed". + _completion_receipt.finalize_interrupted_update_receipt("KeyboardInterrupt: interrupted after the code was updated") + raise SystemExit(130) from interrupt _accept_completion_pm_receipt(result.get("pm_receipt"), request["receipt"]["update_id"]) token = request["windows_resume"] if token is not None and result.get("windows_resume") is not None: resumed = dict(result["windows_resume"]) token.clear() token.update(resumed) + elif token and token.get("resume_needed") and not result["exit_code"]: + # Dependencies owed (A6): the bootstrap child cannot resume paused gateways; this process can. + try: + _m()._resume_windows_gateways_after_update(token) + except Exception as exc: # noqa: BLE001 — the code is committed (C3) + from hermes_cli.update_receipt import amend_terminal_followup, record_followup + + reason = f"Windows gateway recovery failed: {exc}" + record_followup("windows_resume", reason) + amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", reason) if result.get("receipt") is not None: current = _completion_receipt._current.get() if current is not None: diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index d3732b724408..91325ce89755 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -26,11 +26,11 @@ def _exit_status(code: int) -> int: return code if code >= 0 else 128 - code -def _failed_result(request: dict, result_path: Path, code: int) -> int: - code = _exit_status(code) or 1 +def _write_bootstrap_result(request: dict, result_path: Path, code: int, receipt: dict | None) -> int: + """The bootstrap child's answer; windows_resume stays the parent's token (and its atexit net).""" _write_json(result_path, { "schema": 1, "update_id": request["receipt"]["update_id"], "exit_code": code, - "receipt": None, "windows_resume": None, "pm_receipt": request.get("pm_receipt"), + "receipt": receipt, "windows_resume": None, "pm_receipt": request.get("pm_receipt"), }) return code @@ -144,6 +144,51 @@ def _read_terminal_receipt(request: dict) -> dict | None: return None +def _running_record(request: dict) -> dict | None: + from hermes_cli.update_receipt import _receipt_dir + + for path in _receipt_dir().glob(f"update_*_{request['receipt']['update_id']}.json"): + try: + data = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, ValueError): + continue + if data.get("update_id") == request["receipt"]["update_id"] and data.get("outcome") == "running": + data.pop("writer_pid", None) + return data + return None + + +def _settle_after_commit(request: dict, result_path: Path, step: str, reason: str) -> int: + """The tree already moved, so a failure here is owed work, never a failed update (C3, A6). + + Runs in the bootstrap interpreter (stdlib + the new tree) when dependency preparation failed or + the prepared child died without a result. The tail obligation stays armed, so the next launch + syncs the dependencies and finishes the tail; the run's receipt is a success that names the + follow-up, so nothing reports "still on the previous version" while the tree is new. + """ + from hermes_cli import update_receipt + from hermes_cli.venv_sync import arm_completion + + try: + arm_completion(Path(request["source"])) + except OSError as exc: # the next launch still sees stale dependencies and syncs them + print(f" ⚠ Could not record the owed source-update tail: {exc}") + receipt = _read_terminal_receipt(request) # a prepared child that finalized, then died + if receipt is None: + # A prepared child that died mid-tail persisted stages the parent's snapshot lacks. + _resume_receipt(_running_record(request) or request["receipt"]) + update_receipt.record_stage("deps" if step == "dependencies" else "build", "failed") + update_receipt.record_followup(step, reason, retry="dependencies not installed yet — the next launch retries" + if step == "dependencies" else "the next launch or `hermes update` retries it") + update_receipt.finalize_pending_update_receipt(0, f"{step} owed after the code was updated") + receipt = _read_terminal_receipt(request) + code = 0 if receipt is not None and receipt.get("outcome") == "success" else 1 + if code == 0 and request["gateway_mode"]: + # The gateway's /update watcher reads this; the code is committed (same as _complete_selected). + (Path(request["home"]) / ".update_exit_code").write_text("0", encoding="utf-8") + return _write_bootstrap_result(request, result_path, code, receipt) + + def _prepare(request: dict, request_path: Path, result_path: Path) -> int: import pm from pm import receipt @@ -179,7 +224,8 @@ def _prepare(request: dict, request_path: Path, result_path: Path) -> int: # and dependency graph. No application maintenance runs in this bootstrap. code = _exit_status(subprocess.call(command, cwd=root, env=activation_environment(root))) if not result_path.exists(): - return _failed_result(request, result_path, code) + return _settle_after_commit(request, result_path, "completion", + f"the completion process exited {code} without a result") return code @@ -273,9 +319,10 @@ def _finish(request: dict, result_path: Path) -> int: try: _complete_selected(request) except KeyboardInterrupt: - # An operator interrupt is not a step failure; the armed obligations finish the tail. - code, reason = 1, "KeyboardInterrupt: completion interrupted" - print("✗ Source update completion interrupted; the next launch or `hermes update` finishes it") + # An operator interrupt is not a step failure, and the code already moved: the run is + # ``interrupted`` (never "failed"), and the armed obligations finish the tail. + code, reason = 130, "KeyboardInterrupt: interrupted after the code was updated" + update_receipt.finalize_interrupted_update_receipt(reason, exit_code=code) except SystemExit as exc: if exc.code not in (0, None): update_receipt.record_followup("completion", f"completion exited {exc.code}") @@ -323,14 +370,23 @@ def main() -> int: from pm.environments import activate_dependencies activate_dependencies(root) return _finish(request, result_path) + return _bootstrap(request, request_path, result_path) + + +def _bootstrap(request: dict, request_path: Path, result_path: Path) -> int: + """Dependency preparation in the parent's interpreter; its failure is a follow-up (A6).""" try: return _prepare(request, request_path, result_path) - except BaseException as exc: - # PM failed before application dependencies were ready. Leave the parent - # receipt and paused-gateway obligation intact for boundary recovery. - print(f"✗ Source update preparation failed: {exc}") - code = exc.returncode if isinstance(exc, subprocess.CalledProcessError) else 1 - return _failed_result(request, result_path, code) + except KeyboardInterrupt: + _resume_receipt(request["receipt"]) + from hermes_cli import update_receipt + update_receipt.finalize_interrupted_update_receipt( + "KeyboardInterrupt: interrupted while installing dependencies", exit_code=130) + return _write_bootstrap_result(request, result_path, 130, _read_terminal_receipt(request)) + except (Exception, SystemExit) as exc: # noqa: BLE001 — after the commit point nothing fails the update + # The code is committed but its dependencies are not (A6): an owed follow-up, exit 0. + # Paused Windows gateways stay the parent's to resume (it has the dependencies). + return _settle_after_commit(request, result_path, "dependencies", str(exc) or type(exc).__name__) if __name__ == "__main__": diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index 1f479bc52806..d00a0473bea6 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -310,7 +310,7 @@ def persist_running_receipt() -> None: #: Follow-up steps whose failure leaves the source-update tail owed (``source-completion-pending``). -TAIL_FOLLOWUPS = frozenset({"launchers", "build", "maintenance", "config_migration", "completion"}) +TAIL_FOLLOWUPS = frozenset({"dependencies", "launchers", "build", "maintenance", "config_migration", "completion"}) def record_followup(step: str, reason: str, *, retry: str = "the next launch or `hermes update` retries it") -> None: @@ -574,6 +574,36 @@ def finalize_pending_update_receipt(exit_code: Optional[int] = None, stop_reason return finalize_update_receipt(outcome, stop_reason=stop_reason) +def finalize_interrupted_update_receipt(stop_reason: str, *, exit_code: int = 130) -> Optional[Path]: + """Close a run the operator interrupted AFTER the commit point as ``interrupted``. Never raises. + + Not ``failed``: the code already moved, so "still on the previous version" would be false; the + armed obligations finish the rest. The run's own on-disk record is preferred when it is further + along (the completion child persisted stages this process never saw). + """ + print("⚠ Interrupted after the code was updated: the new code is in place; its remaining steps are owed " + "and the next launch or `hermes update` finishes them.", flush=True) + current = _current.get() + if current is None: + return None + with suppress(Exception): + clone = copy.copy(current) + clone.data = copy.deepcopy(current.data) + on_disk: dict = {} + with suppress(Exception): + path = _run_file(_receipt_dir(), current.data) + on_disk = json.loads(path.read_text(encoding="utf-8-sig")) + if on_disk.get("update_id") == current.data.get("update_id"): + if on_disk.get("finished_at"): # the completion child already closed the run + _current.reset(current.current_token) + return path + on_disk.pop("writer_pid", None) + clone.data = on_disk + clone.data["exit_code"] = int(exit_code) + _current.set(clone) + return finalize_update_receipt("interrupted", stop_reason=stop_reason) + + def _prune_old_receipts(directory: Path) -> None: with suppress(Exception): receipts = (p for p in directory.glob("update_*.json") if p.is_file()) diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index 8fef6c7e6009..747d014f41cc 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -156,7 +156,7 @@ def git(*args): "class UpdateReceipt: pass\n" "def record_stage(*args, **kwargs): pass\n" "def record_skip(*args, **kwargs): pass\n" - "TAIL_FOLLOWUPS = frozenset({'launchers', 'build', 'maintenance', 'config_migration', 'completion'})\n" + "TAIL_FOLLOWUPS = frozenset({'dependencies', 'launchers', 'build', 'maintenance', 'config_migration', 'completion'})\n" "def record_followup(step, reason, **kwargs):\n" " print(f' ⚠ Update follow-up {step!r} did not finish: {reason}')\n" " r = _current.get()\n" @@ -343,21 +343,28 @@ def complete(args, gateway_mode): monkeypatch.setattr(subprocess, "Popen", capture_child) update_cmd._complete_source_update(request) - monkeypatch.setattr(sys, "stdout", Interrupt()) + stdout = Interrupt() + monkeypatch.setattr(sys, "stdout", stdout) monkeypatch.setattr(update_cmd, "_cmd_update_impl", complete) - with pytest.raises(KeyboardInterrupt) as error: + # Was KeyboardInterrupt + a ``failed`` receipt: Ctrl-C after the commit point is an interrupt + # (exit 130), and the receipt keeps the truth the child already wrote (C3, item 5). + with pytest.raises(SystemExit) as error: main.cmd_update(SimpleNamespace(gateway=True)) + assert error.value.code == 130 assert interrupted, "child never published success before cancellation" assert children[0].poll() is not None assert children[0].stdout.closed assert waits and all(timeout is not None and timeout > 0 for timeout in waits) + assert isinstance(error.value.__cause__, KeyboardInterrupt) if cleanup_failure: - assert error.value.__cause__ is cleanup_error + assert error.value.__cause__.__cause__ is cleanup_error + assert "Interrupted after the code was updated" in stdout.getvalue() receipt = update_receipt.read_latest_receipt() assert receipt["update_id"] == request["receipt"]["update_id"] - assert receipt["outcome"] == "failed" - assert receipt["exit_code"] == 1 + assert receipt["outcome"] == "interrupted" # the child was stopped before it closed the run + assert receipt["exit_code"] == 130 assert receipt["stop_reason"].startswith("KeyboardInterrupt:") + # The gateway watcher still hears the interrupt (a non-zero exit is fine for Ctrl-C). assert marker.read_text().strip() == "1" lock = update_lock.UpdateLock() assert lock.acquire() @@ -375,7 +382,12 @@ def test_killed_selected_python_returns_signal_exit_status(transition): "def build_update_products(*a, **kw): os.kill(os.getpid(), signal.SIGKILL)\n" ) result = update_completion.run_completion(request) - assert result["exit_code"] == 137 + # Was 137: the tree already moved, so a prepared child that dies without a result is an owed + # ``completion`` follow-up (C3/A6), the tail obligation armed for the next launch. + assert result["exit_code"] == 0 + assert result["receipt"]["outcome"] == "success" + assert [f["step"] for f in result["receipt"]["followups"]] == ["completion"] + assert "exited 137" in result["receipt"]["followups"][0]["reason"] assert result["pm_receipt"]["update_id"] == request["receipt"]["update_id"] @@ -410,9 +422,8 @@ def test_failed_build_after_commit_is_a_followup_and_later_steps_still_run(trans if json.loads(line)["name"] == "exit_marker").items() -def test_prepare_failure_preserves_correlated_pm_receipt(transition, monkeypatch): - from types import SimpleNamespace - from hermes_cli import main, update_cmd, update_completion, update_receipt +def test_prepare_failure_preserves_correlated_pm_receipt(transition): + from hermes_cli import update_completion root, git, old, new, request = transition shutil.copy2(update_completion.__file__, root / "hermes_cli/update_completion.py") @@ -421,24 +432,17 @@ def test_prepare_failure_preserves_correlated_pm_receipt(transition, monkeypatch ) with (root / "pm/receipt.py").open("a") as stream: stream.write("last_for_update = lambda update_id: {'update_id': update_id, 'outcome': 'refused', 'refusal': {'reason': 'dependency refused'}}\n") - monkeypatch.setenv("HERMES_HOME", request["home"]) - monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) - monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) - monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) - - def complete(args, gateway_mode): - update_receipt.begin_update_receipt() - request["receipt"] = update_receipt._current.get().data - update_cmd._complete_source_update(request) - - monkeypatch.setattr(update_cmd, "_cmd_update_impl", complete) - with pytest.raises(SystemExit) as error: - main.cmd_update(SimpleNamespace(gateway=True)) - assert error.value.code == 1 - receipt = update_receipt.read_latest_receipt() - assert receipt["update_id"] == request["receipt"]["update_id"] - assert receipt["pm_sync_outcome"] == "refused" - assert receipt["pm_refusal"] == {"reason": "dependency refused"} + result = update_completion.run_completion(request) + # Was exit 1 / no receipt: the dependency sync runs after the tree moved, so its failure is a + # ``dependencies`` follow-up (A6) with the tail obligation armed — and the correlated pm + # refusal still travels back for the receipt. + assert result["exit_code"] == 0 + assert result["receipt"]["outcome"] == "success" + assert [f["step"] for f in result["receipt"]["followups"]] == ["dependencies"] + assert "dependency refused" in result["receipt"]["followups"][0]["reason"] + assert (Path(request["home"]) / "completion-pending").read_text() == "owed" + assert result["pm_receipt"] == {"update_id": request["receipt"]["update_id"], "outcome": "refused", + "refusal": {"reason": "dependency refused"}} def test_bootstrap_does_not_initialize_old_site_packages(transition, tmp_path, monkeypatch): diff --git a/tests/hermes_cli/test_update_fleet_restart_pending.py b/tests/hermes_cli/test_update_fleet_restart_pending.py index d284761ecbd2..d7417177170a 100644 --- a/tests/hermes_cli/test_update_fleet_restart_pending.py +++ b/tests/hermes_cli/test_update_fleet_restart_pending.py @@ -559,9 +559,11 @@ def _interrupt(*_a, **_k): monkeypatch.setattr(hermes_main, "_clear_bytecode_cache", _interrupt) - with pytest.raises(KeyboardInterrupt): + # Was KeyboardInterrupt: Ctrl-C after the commit point exits 130 as an interrupt, not a failure. + with pytest.raises(SystemExit) as exc: hermes_main.cmd_update(args) + assert exc.value.code == 130 assert update_cmd_fleet._fleet_restart_obligation_armed() record = json.loads(host_obligation.host_obligation_path().read_text(encoding="utf-8")) assert record["expected_sha"] == "def456" diff --git a/tests/hermes_cli/test_update_zip_completion.py b/tests/hermes_cli/test_update_zip_completion.py index 6f990d8feefa..514179e00add 100644 --- a/tests/hermes_cli/test_update_zip_completion.py +++ b/tests/hermes_cli/test_update_zip_completion.py @@ -273,6 +273,75 @@ def fail_build(*args, **kwargs): assert state.token["resume_needed"] is False +@pytest.mark.parametrize("route", ["direct", "git-failure"]) +def test_zip_dependency_sync_failure_after_swap_is_a_followup(zip_update, monkeypatch, tmp_path, route): + # The dropped "preparation" case, restored at the real seam: PM's dependency sync in the + # completion bootstrap fails AFTER the swap. Was pm.InstallError / exit 1 / no receipt, which + # the Desktop reads as "still on the previous version" while the tree is new. Contract C3/A6: + # exit 0, a ``dependencies`` follow-up, the tail obligation armed, nothing of the tail run. + import pm + import pm.client + from hermes_cli import update_completion + from hermes_cli.venv_sync import completion_pending_path + + state = zip_update + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (route == "direct", ["git"], False)) + monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) + monkeypatch.setattr(update_cmd, "_should_zip_fallback_on_update_error", lambda exc: True) + + def fail_fetch(*args, **kwargs): + raise subprocess.CalledProcessError(1, ["git", "fetch"]) + monkeypatch.setattr(update_cmd, "_git_run", fail_fetch) + monkeypatch.setattr(pm.client, "ensure_tools_for_sync", lambda: None) + + def fail_sync(*args, **kwargs): + assert (state.root / "payload.txt").read_text() == "new" + raise pm.InstallError("venv", "dependency sync stopped") + monkeypatch.setattr(pm, "sync_venv", fail_sync) + + def bootstrap(request): # the completion child's first interpreter, in-process + request = {**request, "bytecode_cache": str(tmp_path / "bytecode")} + request_path, result_path = tmp_path / "request.json", tmp_path / "result.json" + update_completion._write_json(request_path, request) + update_completion._bootstrap(request, request_path, result_path) + return json.loads(result_path.read_text()) + monkeypatch.setattr(update_cmd, "run_completion", bootstrap) + + update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), gateway_mode=True) + + assert (state.root / "payload.txt").read_text() == "new" + receipt = json.loads((state.active / "logs/update_receipts/latest.json").read_text()) + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt["followups"]] == ["dependencies"] + assert completion_pending_path(state.root).is_file() # the next launch syncs and finishes the tail + assert "prepare" not in state.events # no build/maintenance ran against stale dependencies + assert (state.active / ".update_exit_code").read_text() == "0" + assert "resume" in state.events and state.token["resume_needed"] is False + + +def test_ctrl_c_after_the_swap_reports_the_new_code_not_a_failure(zip_update, monkeypatch, capsys): + # Ctrl-C while the completion child runs: run_completion kills the child group and re-raises. + # The tree is new, so the run is ``interrupted`` (never "failed" / "still on the previous + # version") and the message says what is owed; a non-zero exit for an interrupt is fine. + state = zip_update + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (True, ["git"], False)) + monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) + + def interrupted(request): + assert (state.root / "payload.txt").read_text() == "new" + raise KeyboardInterrupt + monkeypatch.setattr(update_cmd, "run_completion", interrupted) + + with pytest.raises(SystemExit) as exc: + update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), gateway_mode=False) + + assert exc.value.code == 130 + assert "Interrupted after the code was updated: the new code is in place" in capsys.readouterr().out + receipt = json.loads((state.active / "logs/update_receipts/latest.json").read_text()) + assert receipt["outcome"] == "interrupted" and receipt["exit_code"] == 130 + assert (state.root / "payload.txt").read_text() == "new" + + @pytest.mark.parametrize("entry", ["payload.txt", "tools"]) def test_zip_recovers_crashed_backup_before_failed_copy_and_retry(zip_update, monkeypatch, entry): import shutil From b2134ce2aee09b4b1997a36ae906c81639d20d0c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:52:45 -0700 Subject: [PATCH 04/62] fix(update): keep the unsaved manual-serve restart warning while an update runs The durable running latest.json carries the previous receipt's manual-serve rows as `carried_manual_serves` (no plan yet), and the startup warning read only plan.runtimes + pending_manual_serves: the warning vanished during a run and after a run killed before its plan stage. Read carried rows too, and carry a killed run's rows forward into the next run. --- hermes_cli/update_cmd_fleet.py | 1 + hermes_cli/update_receipt.py | 1 + hermes_cli/update_serve_obligations.py | 8 +++++- .../hermes_cli/test_manual_serve_deferral.py | 28 +++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index aca620b1b524..8c49bf1946d0 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -259,6 +259,7 @@ def _receipt_owed_gateways(receipt: dict, pending_manual: list[dict]) -> set[tup plan = receipt.get("plan") or {} entries: list[tuple[object, str | None]] = [(entry, None) for entry in plan.get("runtimes") or []] entries.extend((entry, None) for entry in receipt.get("pending_manual_serves") or []) + entries.extend((entry, None) for entry in receipt.get("carried_manual_serves") or []) entries.extend((entry, "gateway") for entry in receipt.get("fleet") or []) owed: set[tuple[str, str]] = set() unverified = False diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index d00a0473bea6..1e5e1a3db408 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -293,6 +293,7 @@ def begin_update_receipt(*, previous: dict | None = None, correlation_id: str | prior = read_latest_receipt() or {} rows = list((prior.get("plan") or {}).get("runtimes") or []) rows += list(prior.get("pending_manual_serves") or []) + rows += list(prior.get("carried_manual_serves") or []) # a run killed before finalize receipt.data["carried_manual_serves"] = [ row for row in rows if isinstance(row, dict) and row.get("supervisor") == "manual-serve"] except Exception as exc: # pragma: no cover - defensive diff --git a/hermes_cli/update_serve_obligations.py b/hermes_cli/update_serve_obligations.py index 8d5b71a40e68..e27cb8e9da67 100644 --- a/hermes_cli/update_serve_obligations.py +++ b/hermes_cli/update_serve_obligations.py @@ -60,9 +60,15 @@ def defer_manual_serve(runtime: dict, *, require_alive: bool = False) -> bool: def retain_receipt_manual_serves(receipt: dict) -> list[dict]: - """Return transfers still owed so receipt rotation cannot discard failed writes.""" + """Return transfers still owed so receipt rotation cannot discard failed writes. + + ``carried_manual_serves`` is the durable running record's copy of the previous receipt's rows + (update_receipt.begin): read it too, or the warning vanishes while an update runs and after a + run killed before its ``plan`` stage. + """ plan = receipt.get("plan") or {} rows = list(plan.get("runtimes") or []) + list(receipt.get("pending_manual_serves") or []) + rows += list(receipt.get("carried_manual_serves") or []) pending = [] for row in rows: if not isinstance(row, dict) or row.get("kind") not in ("serve", "dashboard") or row.get("supervisor") != "manual-serve": diff --git a/tests/hermes_cli/test_manual_serve_deferral.py b/tests/hermes_cli/test_manual_serve_deferral.py index 4de557fd33eb..eb8ae394b0ee 100644 --- a/tests/hermes_cli/test_manual_serve_deferral.py +++ b/tests/hermes_cli/test_manual_serve_deferral.py @@ -209,6 +209,34 @@ def replace(src, dst, *args, **kwargs): assert "900" not in capsys.readouterr().err +def test_unsaved_manual_restart_warning_survives_a_running_and_a_killed_update(monkeypatch, capsys): + """The durable running latest.json holds the prior rows as ``carried_manual_serves`` (no plan + yet): the startup warning must read them while an update runs, after one is killed before its + ``plan`` stage, and the next run must carry them forward again.""" + manual = asdict(RuntimeRecord(kind="serve", profile="work", pid=900, supervisor="manual-serve", restart_via="respawn-argv", detail={"create_time": 1000.0})) + root = get_hermes_home() / "logs" / "update_receipts" + root.mkdir(parents=True, exist_ok=True) + (root / "latest.json").write_text(json.dumps({"outcome": "partial", "plan": {"runtimes": [manual]}})) + (get_hermes_home() / "serve_restart_pending").write_text("not a directory") # the reminder can't be filed + monkeypatch.setattr(process_identity, "_pid_alive_matches", lambda *a: True) + + update_receipt.begin_update_receipt() # running record replaces latest.json + running = json.loads((root / "latest.json").read_text()) + assert running["outcome"] == "running" and "plan" not in running + fleet._warn_pending_fleet_restart_on_startup() + assert "serve [work] pid 900" in capsys.readouterr().err + + # Killed before the plan stage: the in-memory receipt is gone, latest.json stays the running record. + update_receipt._current.set(None) + fleet._warn_pending_fleet_restart_on_startup() + assert "serve [work] pid 900" in capsys.readouterr().err + + update_receipt.begin_update_receipt() # the next run reconciles the killed one and carries again + update_receipt.finalize_update_receipt("success", fleet=[]) + fleet._warn_pending_fleet_restart_on_startup() + assert "serve [work] pid 900" in capsys.readouterr().err + + @pytest.mark.parametrize("kind", ["serve", "dashboard"]) @pytest.mark.parametrize("alive", [True, None, False]) def test_unreadable_create_time_discharges_only_a_proven_dead_pid(monkeypatch, kind, alive): From 01766d9102012bf89ae80285b7bcc4eb9f5dafc4 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:52:45 -0700 Subject: [PATCH 05/62] fix(update): read update.log and receipts from the root home The writers moved to the root home; the readers did not. `hermes logs update` / `handoff`, `hermes logs list`, the debug bundle, the dashboard's durable update-completion marker, the post-swap hand-off file and pm's sync receipts (same update_receipts dir) now use get_default_hermes_root(), so a sticky profile no longer hides the update log. --- hermes_cli/debug.py | 5 ++-- hermes_cli/logs.py | 30 +++++++++++++++---- hermes_cli/update_handoff.py | 6 ++-- hermes_cli/web_routers/actions.py | 7 ++++- pm/receipt.py | 7 +++-- .../test_debug_share_update_logs.py | 17 +++++++++++ tests/hermes_cli/test_logs.py | 23 ++++++++++++++ tests/hermes_cli/test_web_server.py | 8 +++-- 8 files changed, 85 insertions(+), 18 deletions(-) diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index fafdf3a2249d..fa6e9a4c68c2 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -219,9 +219,8 @@ class LogSnapshot: def _primary_log_path(log_name: str) -> Optional[Path]: """Where *log_name* would live if present. Doesn't check existence.""" - from hermes_cli.logs import LOG_FILES - filename = LOG_FILES.get(log_name) - return (get_hermes_home() / "logs" / filename) if filename else None + from hermes_cli.logs import log_file_path + return log_file_path(log_name) # update/handoff: the ROOT home's, whatever profile is active # Logs written by a client process, invisible to a remote/docker/SSH backend running `debug diff --git a/hermes_cli/logs.py b/hermes_cli/logs.py index 42ea725f67d6..4b98c5389bfb 100644 --- a/hermes_cli/logs.py +++ b/hermes_cli/logs.py @@ -28,7 +28,7 @@ from pathlib import Path from typing import Optional, Sequence -from hermes_constants import get_hermes_home, display_hermes_home +from hermes_constants import display_hermes_home, get_default_hermes_root, get_hermes_home # Known log files (name → filename) LOG_FILES = { @@ -53,6 +53,20 @@ "mcp": "mcp-stderr.log", } +# Written to the ROOT home whatever profile is active: ``hermes update`` mutates the checkout every +# profile shares, and its mirror (main_dashboard) + the Desktop hand-off scripts write /logs. +ROOT_HOME_LOGS = frozenset({"update", "handoff"}) + + +def log_file_path(log_name: str) -> Optional[Path]: + """Where *log_name* lives (doesn't check existence); None for an unknown name.""" + filename = LOG_FILES.get(log_name) + if filename is None: + return None + home = get_default_hermes_root() if log_name in ROOT_HOME_LOGS else get_hermes_home() + return home / "logs" / filename + + # "2026-04-05 22:35:00[,123]" at the start of a line; update.log / # desktop-update-handoff.log stamp with the shell's ISO-8601 "T" shape # ("2026-09-29T21:36:18+08:00", "=== hermes update started 2026-09-29T21:36:18 ==="). @@ -174,7 +188,7 @@ def tail_log( print(f"Unknown log: {log_name!r}. Available: {', '.join(sorted(LOG_FILES))}") sys.exit(1) - log_path = get_hermes_home() / "logs" / filename + log_path = log_file_path(log_name) if not log_path.exists(): print(f"Log file not found: {log_path}") print("(Logs are created when Hermes runs — try 'hermes chat' first)") @@ -313,19 +327,23 @@ def _age_label(mtime: datetime) -> str: def list_logs() -> None: - """Print available log files with sizes.""" + """Print available log files with sizes (plus the root home's update logs under a profile).""" log_dir = get_hermes_home() / "logs" - if not log_dir.exists(): + root_logs = [path for name in sorted(ROOT_HOME_LOGS) + if (path := log_file_path(name)).parent != log_dir and path.is_file()] + if not log_dir.exists() and not root_logs: print(f"No logs directory at {display_hermes_home()}/logs/") return print(f"Log files in {display_hermes_home()}/logs/:\n") found = False - for entry in sorted(log_dir.iterdir()): + entries = sorted(log_dir.iterdir()) if log_dir.exists() else [] + for entry in [*entries, *root_logs]: if entry.is_file() and entry.suffix == ".log": st = entry.stat() age_str = _age_label(datetime.fromtimestamp(st.st_mtime)) - print(f" {entry.name:<25} {_size_label(st.st_size):>8} {age_str}") + label = entry.name if entry.parent == log_dir else f"{entry.name} (root)" + print(f" {label:<25} {_size_label(st.st_size):>8} {age_str}") found = True if not found: diff --git a/hermes_cli/update_handoff.py b/hermes_cli/update_handoff.py index c62e9e58e46a..4ff89ab049ed 100644 --- a/hermes_cli/update_handoff.py +++ b/hermes_cli/update_handoff.py @@ -30,10 +30,10 @@ def is_post_swap_child() -> bool: def write_handoff(payload: dict[str, Any]) -> Path: - """Persist the post-swap payload under HERMES_HOME; returns its path.""" - from hermes_constants import get_hermes_home + """Persist the post-swap payload beside the receipts (the ROOT home); returns its path.""" + from hermes_constants import get_default_hermes_root - directory = get_hermes_home() / "logs" / "update_receipts" + directory = get_default_hermes_root() / "logs" / "update_receipts" directory.mkdir(parents=True, exist_ok=True) path = directory / f"post_swap_{os.getpid()}.json" path.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") diff --git a/hermes_cli/web_routers/actions.py b/hermes_cli/web_routers/actions.py index 966662c56935..b128705aabaf 100644 --- a/hermes_cli/web_routers/actions.py +++ b/hermes_cli/web_routers/actions.py @@ -357,7 +357,12 @@ async def get_action_status(name: str, lines: int = 200): durable_update_action_id = None update_receipt_summary = None if name == "hermes-update": - durable_update_action_id = _durable_completed_update_action_id(_tail_lines(log_dir / "update.log", 2000)) + # ``hermes update`` mirrors to the ROOT home's update.log (main_dashboard), never this + # dashboard's profile home: read it where it is written. + from hermes_constants import get_default_hermes_root + + durable_update_action_id = _durable_completed_update_action_id( + _tail_lines(get_default_hermes_root() / "logs" / "update.log", 2000)) if durable_update_action_id: marker = f"=== hermes-update completed {durable_update_action_id} ===" if marker not in tail: diff --git a/pm/receipt.py b/pm/receipt.py index 34cca627827b..b20402aa6b9f 100644 --- a/pm/receipt.py +++ b/pm/receipt.py @@ -123,10 +123,11 @@ def _utc_now_iso() -> str: def _receipt_dir() -> Path: """The receipts dir — a pure path computation, NO mkdir side - effect: readers (latest()) must not create state.""" - from hermes_constants import get_hermes_home + effect: readers (latest()) must not create state. The ROOT home, like the + updater's receipts (hermes_cli.update_receipt): one dir whatever profile runs.""" + from hermes_constants import get_default_hermes_root - return get_hermes_home() / "logs" / "update_receipts" + return get_default_hermes_root() / "logs" / "update_receipts" def begin(kind: str) -> contextvars.Token: diff --git a/tests/hermes_cli/test_debug_share_update_logs.py b/tests/hermes_cli/test_debug_share_update_logs.py index 211cfe7ebe3d..449f71698cac 100644 --- a/tests/hermes_cli/test_debug_share_update_logs.py +++ b/tests/hermes_cli/test_debug_share_update_logs.py @@ -67,6 +67,23 @@ def test_snapshots_include_update_and_handoff_keys(self, home_with_update_logs): assert "Desktop GUI build failed" in snaps["update"].tail_text assert "ELIFECYCLE" in snaps["handoff"].tail_text + def test_snapshots_read_the_root_home_update_logs_under_a_profile(self, tmp_path, monkeypatch): + """``hermes update`` and the hand-off scripts write the ROOT home's logs, whatever profile.""" + from hermes_cli.debug import _capture_default_log_snapshots + + root = tmp_path / "root" + profile = root / "profiles" / "coder" + (root / "logs").mkdir(parents=True) + (profile / "logs").mkdir(parents=True) + (root / "logs" / "update.log").write_text(UPDATE_LOG_BODY) + (root / "logs" / "desktop-update-handoff.log").write_text(HANDOFF_LOG_BODY) + monkeypatch.setenv("HERMES_HOME", str(profile)) + + snaps = _capture_default_log_snapshots(100) + + assert "Desktop GUI build failed" in snaps["update"].tail_text + assert "ELIFECYCLE" in snaps["handoff"].tail_text + def test_missing_files_report_absence_without_raising( self, home_without_update_logs ): diff --git a/tests/hermes_cli/test_logs.py b/tests/hermes_cli/test_logs.py index 3ac6b1d65b1f..9643f7217be6 100644 --- a/tests/hermes_cli/test_logs.py +++ b/tests/hermes_cli/test_logs.py @@ -228,3 +228,26 @@ def test_every_log_file_writes_a_stamp_hermes_logs_since_can_read(): # gateway.error.log (launchd stderr, not in LOG_FILES) uses the shared stamper. from hermes_cli.stderr_timestamp import stamp_line assert _parse_line_timestamp(stamp_line("raw gateway stderr")) is not None + + +def test_update_logs_are_read_from_the_root_home_under_a_profile(tmp_path, monkeypatch, capsys): + # ``hermes update`` mirrors to /logs/update.log and the hand-off scripts write the root + # too; `hermes logs update` under a sticky profile read the profile's (absent) copy. + from hermes_cli.logs import list_logs, log_file_path, tail_log + + root = tmp_path / "root" + profile = root / "profiles" / "coder" + (root / "logs").mkdir(parents=True) + (profile / "logs").mkdir(parents=True) + (root / "logs" / "update.log").write_text("=== hermes update started 2026-10-03T10:00:00 ===\n✓ Update complete!\n") + (profile / "logs" / "agent.log").write_text("2026-10-03 10:00:00,000 INFO run_agent: hi\n") + monkeypatch.setenv("HERMES_HOME", str(profile)) + + assert log_file_path("update") == root / "logs" / "update.log" + assert log_file_path("handoff") == root / "logs" / "desktop-update-handoff.log" + assert log_file_path("agent") == profile / "logs" / "agent.log" + tail_log("update", num_lines=5) + assert "✓ Update complete!" in capsys.readouterr().out + list_logs() + listing = capsys.readouterr().out + assert "agent.log" in listing and "update.log (root)" in listing diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 9157381ec55e..46ed79fd11de 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -1455,14 +1455,18 @@ def fail_spawn(*_args, **_kwargs): assert check_data["update_command"] == data["update_command"] def test_update_status_recovers_completed_result_after_dashboard_restart(self, monkeypatch, tmp_path): - + # The dashboard runs under a profile home; ``hermes update`` mirrors to the ROOT home's + # update.log (main_dashboard), so the durable completion marker is read there. + root = tmp_path / "root" + (root / "logs").mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "coder")) action_id = "c" * 32 (tmp_path / "hermes-update.log").write_text( "=== hermes-update started 2026-08-17 11:19:34 ===\n" "pulling updates...\n", encoding="utf-8", ) - (tmp_path / "update.log").write_text( + (root / "logs" / "update.log").write_text( "=== hermes update started 2026-08-17T11:19:35 ===\n" "✓ Update complete!\n" f"=== hermes-update completed {action_id} ===\n", From ecc0d8d5b439ec9ae98a12d3fe7a97259c3b41aa Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:52:45 -0700 Subject: [PATCH 06/62] test(e2e): hostile post cells for a failed dependency sync, Ctrl-C after commit, and profile log readers --- .../e2e/core/upgrade/git/test_hostile_post.py | 149 ++++++++++++++++-- 1 file changed, 139 insertions(+), 10 deletions(-) diff --git a/tests/e2e/core/upgrade/git/test_hostile_post.py b/tests/e2e/core/upgrade/git/test_hostile_post.py index db598f6ef5eb..67b329a072c7 100644 --- a/tests/e2e/core/upgrade/git/test_hostile_post.py +++ b/tests/e2e/core/upgrade/git/test_hostile_post.py @@ -22,7 +22,14 @@ update was spawned is SIGSTOPped then SIGKILLed (that is the whole update tree, including a completion child in its own session or reparented to the namespace init). (d) a sticky profile (``hermes profile use e2epost``) must not move update.log / receipts off the - root hermes home. + root hermes home, and ``hermes logs update`` under that profile reads the root's update.log. +(e) the dependency sync fails after the tree moved (contract amendment A6): the release's + ``pm.client.ensure_tools_for_sync`` raises while ``$HOME/.e2e-break-deps-sync`` exists; it runs in + the completion bootstrap from the NEW tree, after the checkout moved. Exit 0, a ``dependencies`` + follow-up, the tail obligation armed; the next update completes it. +(f) Ctrl-C after the commit point: SIGINT (what a terminal's Ctrl-C delivers to the foreground + ``hermes update``; the completion child runs in its own session) while the slow web build runs. + The user must read that the new code is in place and the rest is owed, never a failure. What is asserted is what the user sees: the exit code, ``⚠`` lines, the receipt at ``/logs/update_receipts/latest.json`` (``outcome``, ``followups``), the @@ -59,7 +66,11 @@ SLOW_WEB = ".e2e-slow-web-build" WEB_STARTED = ".e2e-web-build-started" BREAK_GATEWAY = ".e2e-break-gateway-boot" -TOGGLES = (BREAK_WEB, SLOW_WEB, WEB_STARTED, BREAK_GATEWAY) +BREAK_DEPS = ".e2e-break-deps-sync" +TOGGLES = (BREAK_WEB, SLOW_WEB, WEB_STARTED, BREAK_GATEWAY, BREAK_DEPS) +DEPS_FAULT_TEXT = "e2e: dependency sync fails on purpose" +DEPS_OWED = "dependencies not installed yet" # update_completion._settle_after_commit +INTERRUPTED_AFTER_COMMIT = "Interrupted after the code was updated" # update_receipt WEB_FAULT_TEXT = "e2e: the web build fails on purpose" GATEWAY_FAULT_TEXT = "e2e: gateway boot refused on purpose" OWED_RESTART = "did not restart running gateways" # update_cmd_fleet._warn_pending_fleet_restart @@ -67,6 +78,7 @@ UPDATE_TIMEOUT = 1500 _WEB_MARK = "// e2e (test_hostile_post): hostile build toggles" _GATEWAY_MARK = "# e2e (test_hostile_post): hostile gateway boot toggle" +_DEPS_MARK = "# e2e (test_hostile_post): hostile dependency sync toggle" _HOSTILE_WEB = f"""{_WEB_MARK}, keyed on files in $HOME. import {{ existsSync as e2eExists, writeFileSync as e2eWrite }} from 'node:fs' @@ -90,6 +102,11 @@ raise SystemExit(1) """ +_HOSTILE_DEPS = f""" {_DEPS_MARK} + if __import__("os").path.exists(__import__("os").path.join(__import__("os").path.expanduser("~"), "{BREAK_DEPS}")): + raise RuntimeError("{DEPS_FAULT_TEXT} ({BREAK_DEPS})") +""" + # -- the shared install -------------------------------------------------------------------------- @@ -247,6 +264,19 @@ def _publish_gateway_fault_release(w: G.World, tag: str) -> str: }) +def _publish_deps_fault_release(w: G.World, tag: str) -> str: + """A release whose dependency preparation (``ensure_tools_for_sync``) fails while the toggle exists.""" + text = _head_file(w, "pm/client.py") + if _DEPS_MARK not in text: + m = re.search(r"^def ensure_tools_for_sync\(\) -> None:\n \"\"\".*?\"\"\"\n", text, re.S | re.M) + assert m, "premise: pm/client.py has no ensure_tools_for_sync() with a docstring to inject after" + text = text[:m.end()] + _HOSTILE_DEPS + text[m.end():] + return w.publish(f"release: e2e hostile post deps {tag}", { + "pm/client.py": text, + f"docs/e2e-hostile-post-{tag}.txt": f"release {tag}\n", + }) + + # -- one shared PID namespace (gateway / killable update) ---------------------------------------- @@ -518,16 +548,115 @@ def test_sticky_profile_update_logs_and_receipts_land_in_the_root_home(w): previous_update_id=prev.get("update_id")) + w.diag(cp) assert w.head() == target, f"premise: the release never committed:\n{diag}" assert cp.returncode == 0, f"the sticky-profile update failed:\n{diag}" - # Only the UPDATE receipt is this lane's: pm's own sync receipts (pm_*sync*.json, its - # latest.json with kind "sync") still follow the process home — pm/ is out of scope here. - update_files = sorted(n for n in prof_after - prof_before if n.startswith("update_")) - prof_latest = _read_json(prof_receipts / "latest.json") - assert not update_files, f"the update wrote update receipts into the profile home {prof_receipts}:\n{diag}" - # pm's sync receipt carries the update's correlation id, so "kind" tells them apart. - assert prof_latest.get("kind", "sync") == "sync", \ - f"the profile home's latest.json is this update's receipt:\n{diag}" + # Neither the update's receipts nor pm's sync receipts (same dir, ``kind`` field) follow + # the sticky profile: one root dir is what the Desktop and `hermes pm status` read. + assert not sorted(prof_after - prof_before), \ + f"the update wrote receipts into the profile home {prof_receipts}:\n{diag}" assert log1 > log0, f"the root update.log did not grow:\n{diag}" assert rec.get("update_id") != prev.get("update_id") and rec.get("outcome") == "success", \ f"root latest.json is not this run's success:\n{diag}" + # The readers follow the writers: `hermes logs update` under the profile shows the root log. + logs_cp = w.sb.cli("logs", "update", "-n", "400", timeout=300) + assert logs_cp.returncode == 0 and "hermes update started" in G.output(logs_cp), \ + f"`hermes logs update` under the sticky profile does not show the root update.log:\n{H.describe(logs_cp)}\n{diag}" finally: w.sb.cli("profile", "use", "default", timeout=300) + + +def test_dependency_sync_failure_after_commit_is_a_followup_not_a_failed_update(w): + w.reset_clean() + _clear_toggles(w) + target = _publish_deps_fault_release(w, "e") + _toggle(w, BREAK_DEPS, True) + try: + cp = w.update("--no-gateway-restart") + finally: + _toggle(w, BREAK_DEPS, False) + out = G.output(cp) + rec = w.receipt() + diag = _facts(w, cp) + w.diag(cp) + + assert w.head() == target, f"premise: the release never committed, so this is no post-commit cell:\n{diag}" + assert DEPS_FAULT_TEXT in out, f"premise: the hostile dependency sync never ran:\n{diag}" + assert cp.returncode == 0, f"the code committed, yet a failed dependency sync failed `hermes update`:\n{diag}" + assert any(DEPS_OWED in line for line in _warn_lines(out)), \ + f"no ⚠ line says the dependencies are not installed yet:\n{diag}" + assert rec.get("outcome") == "success", f"receipt outcome is not success after the commit:\n{diag}" + assert "dependencies" in _followup_steps(rec), f"receipt followups do not name `dependencies`:\n{diag}" + assert _pending_markers(w), f"the failed dependency sync disarmed the source-completion obligation:\n{diag}" + + # The fault clears; the next plain update completes the owed dependencies and tail. + cp = w.update("--no-gateway-restart") + rec = w.receipt() + diag = _facts(w, cp) + w.diag(cp) + assert cp.returncode == 0 and w.head() == target, f"the completing update failed:\n{diag}" + assert not _followups(rec), f"the completing update still reports followups:\n{diag}" + assert not _pending_markers(w), f"the completing update left the completion obligation armed:\n{diag}" + + +def _foreground_group(host: NamespaceHost, upd: int) -> list[int]: + """What a terminal's Ctrl-C reaches: the update and its descendants, minus the completion child's + subtree (it runs in its own session, so the terminal's SIGINT never reaches it).""" + procs = {p["pid"]: p for p in host.procs() if p["state"] not in ("Z", "X")} + group, frontier = [], [upd] + while frontier: + pid = frontier.pop() + if pid not in procs or "update_completion.py" in " ".join(procs[pid]["cmdline"]): + continue + group.append(pid) + frontier += [p for p, row in procs.items() if row["ppid"] == pid] + return group + + +def test_ctrl_c_after_commit_reports_the_new_code_not_a_failed_update(w): + w.reset_clean() + _clear_toggles(w) + prev = w.receipt() + target = _publish_web_release(w, "f") + started = w.sb.home / WEB_STARTED + _toggle(w, SLOW_WEB, True) + try: + with _namespace(w) as host: + spawn_log = w.sb.root / "update-f.log" # sandbox-writable + upd = host.spawn([w.sb.hermes, "update", "--yes", "--branch", "main", "--no-gateway-restart"], + log=spawn_log) + + def _slow_build_running(): + if started.is_file(): + return True + if not host.alive(upd): + raise AssertionError(f"premise: the update exited before the slow web build started:\n" + f"{_read(spawn_log)[-6000:]}\n{w.diag()}") + return False + + H.wait_for(_slow_build_running, timeout=1200, interval=1.0, what="the slow web build to start") + interrupted = _foreground_group(host, upd) + for pid in interrupted: + host.kill(pid, signal.SIGINT) + exited = True + try: + H.wait_for(lambda: not host.alive(upd), timeout=120, interval=0.5, what="the interrupted update to exit") + except AssertionError: + exited = False + out = _read(spawn_log) + rec = w.receipt() + diag = (_facts(w, sigint=interrupted, update_exited=exited, previous_update_id=prev.get("update_id")) + + w.diag() + "\n--- sandbox ---\n" + host.ps_text() + + "\n--- interrupted update output ---\n" + out[-6000:]) + finally: + _toggle(w, SLOW_WEB, False) + started.unlink(missing_ok=True) + + assert w.head() == target, f"premise: the SIGINT landed before the release committed:\n{diag}" + assert exited, f"the update did not exit after Ctrl-C:\n{diag}" + assert INTERRUPTED_AFTER_COMMIT in out, \ + f"after Ctrl-C the user is not told the new code is in place and the rest is owed:\n{diag}" + assert rec.get("update_id") != prev.get("update_id") and rec.get("outcome") == "interrupted", \ + f"latest.json is not this run's `interrupted` record (a `failed` one reads as the previous version):\n{diag}" + assert _pending_markers(w), f"the interrupt disarmed the source-completion obligation:\n{diag}" + + cp = w.update("--no-gateway-restart") + rec = w.receipt() + diag = _facts(w, cp) + w.diag(cp) + assert cp.returncode == 0 and rec.get("outcome") == "success", f"the update after Ctrl-C failed:\n{diag}" + assert not _pending_markers(w), f"the update after Ctrl-C left the completion obligation armed:\n{diag}" From adb18b53ddae55bc8bcddd3c2566e5cc3c568a22 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:02:43 -0700 Subject: [PATCH 07/62] docs(update): dependency follow-up, Ctrl-C after commit, root-home readers, fleet exit 0 --- .../docs/developer-guide/source-update-completion.md | 10 +++++++++- website/docs/getting-started/updating.md | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/website/docs/developer-guide/source-update-completion.md b/website/docs/developer-guide/source-update-completion.md index 8f43012b1399..0e0e47aa83a4 100644 --- a/website/docs/developer-guide/source-update-completion.md +++ b/website/docs/developer-guide/source-update-completion.md @@ -61,7 +61,15 @@ The receipt is durable while the run is open: `begin_update_receipt` writes it a boundary refreshes it, so a killed update leaves its own record. The next update marks a `running` record whose processes are gone `interrupted` (naming its last stage) and reports it. Receipts and the `update.log` tee resolve to the root home -(`hermes_constants.get_default_hermes_root()`), never a sticky profile's. +(`hermes_constants.get_default_hermes_root()`), never a sticky profile's; so do their readers +(`hermes logs update`, the debug bundle, the dashboard's update status, pm's sync receipts). + +The completion bootstrap's dependency preparation (`ensure_tools_for_sync`, `pm.sync_venv`) also +runs after the tree moved: its failure is a `dependencies` follow-up ("dependencies not installed +yet — the next launch retries"), exit 0, with the tail obligation armed; a prepared child that +dies without a result is a `completion` follow-up. A Ctrl-C after the commit point closes the run +as `interrupted` (exit 130, never `failed`) and says the new code is in place with its remaining +steps owed. ## Parent lifecycle and failures diff --git a/website/docs/getting-started/updating.md b/website/docs/getting-started/updating.md index 8923f884e932..7fb59be1e7c5 100644 --- a/website/docs/getting-started/updating.md +++ b/website/docs/getting-started/updating.md @@ -286,7 +286,7 @@ The same inventory is embedded in every real update's receipt (`~/.hermes/logs/u ### Update receipts and the fleet version check -Every `hermes update` run writes a machine-readable receipt to `~/.hermes/logs/update_receipts/` (last 20 kept, `latest.json` always points at the most recent): the pre-update fleet plan, each step taken, anything skipped and why, the gateway restart outcome, and the final fleet version matrix. The SQLite runtime repair is one of those steps (`sqlite_runtime_repair`): a failed repair records the actual reason (for example the `uv sync` error) and the SQLite version pair, a deferred or not-applicable repair lands in the skips with its reason. After the restart phase the updater compares each live gateway's running code against the freshly updated checkout and prints a per-profile matrix — a gateway still serving pre-update code is reported loudly with the exact restart command, and the update exits non-zero so automation never treats a mixed-version fleet as healthy. Both `--plan` and the fleet check ask each running gateway directly over its local control socket (`gateway.sock` in the profile's data directory, a named pipe on Windows) when available, so version and supervisor information comes from the gateway itself; gateways from older versions are still discovered through their state files as before. +Every `hermes update` run writes a machine-readable receipt to `~/.hermes/logs/update_receipts/` (last 20 kept, `latest.json` always points at the most recent): the pre-update fleet plan, each step taken, anything skipped and why, the gateway restart outcome, and the final fleet version matrix. The SQLite runtime repair is one of those steps (`sqlite_runtime_repair`): a failed repair records the actual reason (for example the `uv sync` error) and the SQLite version pair, a deferred or not-applicable repair lands in the skips with its reason. After the restart phase the updater compares each live gateway's running code against the freshly updated checkout and prints a per-profile matrix — a gateway still serving pre-update code is reported loudly with the exact restart command and recorded as a `gateway_restart` follow-up in the receipt; the code is already in place, so the update still exits 0, and every later command warns until the fleet serves the new code, so a mixed-version fleet is never silently healthy. Both `--plan` and the fleet check ask each running gateway directly over its local control socket (`gateway.sock` in the profile's data directory, a named pipe on Windows) when available, so version and supervisor information comes from the gateway itself; gateways from older versions are still discovered through their state files as before. A multiplexed default gateway is one process serving several profiles, so it appears once in the matrix and vouches for every profile in its `served_profiles` record. The same coverage clears the "A previous `hermes update` pulled new code but did not restart running gateways" hint: once that gateway (or, after a manual `git pull`, every gateway an update restarted) runs the current code, `hermes gateway restart` is enough — the hint no longer waits for the next `hermes update` to write a fresh receipt. The same is true of the restart obligation left by an update that died before recording which gateways it owed (or by an older updater that never recorded them): once every live gateway runs the current checkout, the obligation is retired and the hint stops. On a host that runs no gateway at all (the Desktop app alone), it is retired once no profile has a gateway that went away without a clean stop and every running backend is restarted by its own supervisor or has its own reminder. That obligation is recorded once per HOST, in the cross-profile rendezvous directory (`$HERMES_GATEWAY_LOCK_DIR`, else `$XDG_STATE_HOME/hermes/gateway-locks`) as `host-update-restart.json`, so every profile's CLI sees the same one: `hermes -p coder update` and `hermes -p writer update` restart the shared multiplexed gateway once between them, not once each. An obligation left behind by an older per-profile updater (`fleet_restart_pending` in one profile's Hermes home) is still read and cleared. An update whose pre-update plan found no gateway at all owes nothing and leaves no breadcrumb. A backend supervised by Desktop, systemd or launchd is restarted by its supervisor and never blocks this settlement; only a manual backend whose reminder could not be saved keeps the obligation open. @@ -392,7 +392,7 @@ If `git status --short` shows unexpected changes after `hermes update`, stop and tail -f ~/.hermes/logs/update.log ``` -- `Ctrl-C` (SIGINT) and system shutdown (SIGTERM) are still honored — those are deliberate cancellations, not accidents. +- `Ctrl-C` (SIGINT) and system shutdown (SIGTERM) are still honored — those are deliberate cancellations, not accidents. A `Ctrl-C` after the new code is in place exits 130 and says so: the remaining steps (dependencies, builds, gateway restart) are owed, and the next launch or `hermes update` finishes them. You no longer need to wrap `hermes update` in `screen` or `tmux` to survive a terminal drop. From d4f8d94d30cd838a67c1c4eb0589e9ab801f0f36 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:25:12 -0700 Subject: [PATCH 08/62] revert(pm): leave pm receipts where pm writes them pm/ is bundled package-manager code outside this lane. pm.receipt.latest() (hermes pm status) reads the same dir pm writes, so nothing is lost: the updater embeds the pm sync section in memory (last_for_update) and its own receipts stay under the root home. --- pm/receipt.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pm/receipt.py b/pm/receipt.py index b20402aa6b9f..34cca627827b 100644 --- a/pm/receipt.py +++ b/pm/receipt.py @@ -123,11 +123,10 @@ def _utc_now_iso() -> str: def _receipt_dir() -> Path: """The receipts dir — a pure path computation, NO mkdir side - effect: readers (latest()) must not create state. The ROOT home, like the - updater's receipts (hermes_cli.update_receipt): one dir whatever profile runs.""" - from hermes_constants import get_default_hermes_root + effect: readers (latest()) must not create state.""" + from hermes_constants import get_hermes_home - return get_default_hermes_root() / "logs" / "update_receipts" + return get_hermes_home() / "logs" / "update_receipts" def begin(kind: str) -> contextvars.Token: From 745ab88a5b383e4c3ebf7206d0df4c18ad6df1f2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:30:17 -0700 Subject: [PATCH 09/62] test: follow the fleet-verify split and the root-home hand-off in pinned tests - _live_gateway_pids_from_fleet moved to update_cmd_fleet_verify (no shim: internal path). - The takeover fixture stubs hermes_constants; write_handoff now resolves the ROOT home, so the stub provides get_default_hermes_root too. - The PATH-scoped npm lookup moved from build_update_products into _build_desktop_product: rename its resolution-allowlist row (same site, same reason). --- tests/fixtures/resolution_allowlist.json | 2 +- tests/hermes_cli/test_old_updater_takeover.py | 6 ++++-- tests/hermes_cli/test_restart_plan_reconciliation.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/resolution_allowlist.json b/tests/fixtures/resolution_allowlist.json index 41f37f2824f7..30a0d3d46163 100644 --- a/tests/fixtures/resolution_allowlist.json +++ b/tests/fixtures/resolution_allowlist.json @@ -356,7 +356,7 @@ }, { "path": "hermes_cli/source_build.py", - "symbol": "build_update_products", + "symbol": "_build_desktop_product", "kind": "bare_which" }, { diff --git a/tests/hermes_cli/test_old_updater_takeover.py b/tests/hermes_cli/test_old_updater_takeover.py index 465e5ec5c148..32870b23557b 100644 --- a/tests/hermes_cli/test_old_updater_takeover.py +++ b/tests/hermes_cli/test_old_updater_takeover.py @@ -54,10 +54,12 @@ def test_historical_payload_maps_to_takeover_request_schema(tmp_path, desktop, r # checkout is the whole tree the child sees (CI has no editable finder for the source). shutil.copy2(source / "hermes_cli/update_handoff.py", package / "update_handoff.py") # write_handoff resolves the home through hermes_constants; the child tree is the whole - # sys.path (CI has no editable finder), so give it the one name the hand-off reads. + # sys.path (CI has no editable finder), so give it the names the hand-off reads (the + # payload lands beside the receipts in the ROOT home, which is HERMES_HOME here). (root / "hermes_constants.py").write_text( "import os\nfrom pathlib import Path\n" - "def get_hermes_home():\n return Path(os.environ['HERMES_HOME'])\n", encoding="utf-8", + "def get_hermes_home():\n return Path(os.environ['HERMES_HOME'])\n" + "get_default_hermes_root = get_hermes_home\n", encoding="utf-8", ) program = root / "historical.py" program.write_text( diff --git a/tests/hermes_cli/test_restart_plan_reconciliation.py b/tests/hermes_cli/test_restart_plan_reconciliation.py index b74d533df871..f69a5eb77070 100644 --- a/tests/hermes_cli/test_restart_plan_reconciliation.py +++ b/tests/hermes_cli/test_restart_plan_reconciliation.py @@ -494,7 +494,7 @@ def test_live_gateway_pids_from_fleet_skips_down_and_unusable_rows(): PRE-restart PID (nothing replaced it) so it must never count as a successor; rows without a usable profile/PID are skipped; a ``stale`` row is a live successor (the fleet matrix escalates staleness on its own).""" - from hermes_cli.update_cmd_fleet import _live_gateway_pids_from_fleet + from hermes_cli.update_cmd_fleet_verify import _live_gateway_pids_from_fleet rows = [ {"profile": "coder", "pid": 76796, "state": "current"}, From ba2fbcc2533e37de442865fe7b0971f5b4b99db2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:46:40 -0700 Subject: [PATCH 10/62] fix(update): a conflicted autostash is owed to the user, never a plain success Contract C3 keeps post-commit step failures at exit 0, but the parked stash is not a step anything retries: the receipt records a user_action, finalizes partial and hermes update exits 1 (#122557). The Desktop handoff reports an owed app rebuild as an action ("could not be rebuilt ... hermes desktop --force-build"), and the pm e2e follows A6 (dependency sync after the tree moved is an owed follow-up). --- apps/desktop/e2e/update/build-fail.spec.ts | 32 +++++++++----- hermes_cli/update_cmd_maint.py | 4 +- hermes_cli/update_completion.py | 42 +++++++++++++++---- hermes_cli/update_receipt.py | 18 +++++++- scripts/desktop-update/posix.sh | 23 +++++++--- .../core/upgrade/pm/test_stamps_and_doctor.py | 31 +++++++++----- .../test_update_completion_process.py | 1 + .../test_update_post_commit_followups.py | 15 +++++++ 8 files changed, 130 insertions(+), 36 deletions(-) diff --git a/apps/desktop/e2e/update/build-fail.spec.ts b/apps/desktop/e2e/update/build-fail.spec.ts index 2db60bdf6291..f7aa698a72e1 100644 --- a/apps/desktop/e2e/update/build-fail.spec.ts +++ b/apps/desktop/e2e/update/build-fail.spec.ts @@ -4,10 +4,15 @@ * #124040: the Desktop build failed inside the update takeover's completion * stage and the failure was swallowed / kept looping instead of reaching the * user. Here upstream main carries a Desktop source file that does not - * compile; the user clicks "Update now". The update must end with the app - * telling the user the update FAILED (the relaunched app's result dialog is - * logged as `[updates] detached update FAILED`), never "finished OK", and the - * updater must not be left running. + * compile; the user clicks "Update now". + * + * The build runs after the code committed, so under contract C3 the update + * succeeds with an owed follow-up: Hermes IS on the new code and only this + * app was not rebuilt. The relaunched (old-build) app must say exactly that + * and what to do — the action-required dialog, logged as `[updates] detached + * update finished with manual action` — never "finished OK" (a silent stale + * app) and never "FAILED / still on the previous version" (false: the code + * moved). The updater must not be left running. */ import * as path from 'node:path' @@ -32,7 +37,7 @@ import { waitForUpdateOffer } from './harness' -test('a Desktop build failure during Update now is reported to the user as a failed update', async () => { +test('a Desktop build failure during Update now tells the user the app was not rebuilt and how to fix it', async () => { test.setTimeout(20 * 60_000) const session = await startInstallSession() const { facts, env } = session @@ -52,7 +57,7 @@ test('a Desktop build failure during Update now is reported to the user as a fai await clickUpdateNowAndExpectHandoff(launched, facts) }) - await test.step('the failed build reaches the user as a failed update', async () => { + await test.step('the failed build reaches the user as an app that still needs rebuilding', async () => { await waitFor( 'the relaunched app to report the update outcome', () => /\[updates\] detached update (finished|FAILED)/.test(desktopLog(facts)), @@ -64,8 +69,15 @@ test('a Desktop build failure during Update now is reported to the user as a fai .filter(line => /\[updates\] detached update/.test(line)) .join('\n') - expect(outcome, `the update is reported as FAILED, not finished\n${explain()}`).toMatch(/detached update FAILED/) + expect(outcome, `the owed rebuild reaches the user as an action, not a silent OK\n${explain()}`).toMatch( + /detached update finished with manual action/ + ) + expect(outcome, 'the user is told the Desktop app could not be rebuilt and how to rebuild it').toMatch( + /Desktop app could not be rebuilt.*hermes desktop --force-build/ + ) expect(outcome).not.toMatch(/detached update finished OK/) + // The code committed: "FAILED / still on the previous version" would be false. + expect(outcome).not.toMatch(/detached update FAILED|previous version/) // The relaunched app reports the result while posix.sh is still inside launch_app's 1.5 s // acceptance window, so the updater gets a bounded moment to exit instead of none. await expect @@ -77,10 +89,8 @@ test('a Desktop build failure during Update now is reported to the user as a fai { timeout: 30_000, message: 'no updater is left running after the failure' } ) .toEqual([]) - // What the user is left on: record it for triage (the checkout moved; the app bundle did not). - test - .info() - .annotations.push({ type: 'checkout-after-failure', description: git(facts.checkout, 'rev-parse', 'HEAD') }) + // The checkout moved (that is why it is not "previous version"); the app bundle did not. + expect(git(facts.checkout, 'rev-parse', 'HEAD'), 'the code update committed').toBe(target) }) } finally { await session.close() diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py index 3cb353f7b5a0..3080b3e81acb 100644 --- a/hermes_cli/update_cmd_maint.py +++ b/hermes_cli/update_cmd_maint.py @@ -1073,7 +1073,9 @@ def owed_step(name, run): with _best_effort('Post-update notices failed: %s'): _print_post_update_notices_and_self_heals() - if not update_complete: + # A non-✓ completion message (parked local changes) withholds success on its own; only a + # ✓ message that still came back False is the SQLite verdict. + if not update_complete and (completion_message or "✓").startswith("✓"): record_followup("sqlite_runtime", "the selected Python links an unsafe SQLite runtime", retry="run the installer again") return update_complete diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index 91325ce89755..d71216038b42 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -158,6 +158,26 @@ def _running_record(request: dict) -> dict | None: return None +def _owed_user_action(request: dict) -> str | None: + """The unsettled-autostash notice (#122557) when this run parked the user's local changes. + + ``_complete_source_update`` hands it over as the completion message, and a completion message + that is not a ``✓`` line is never a success (``_print_verified_update_completion``). + """ + message = request.get("completion_message") or "" + return message if message and not message.startswith("✓") else None + + +def _record_owed_user_action(request: dict) -> bool: + """Land the parked local changes on the receipt; True when nothing is owed to the user.""" + from hermes_cli.update_receipt import record_user_action + + notice = _owed_user_action(request) + if notice: + record_user_action("local_changes", notice) + return notice is None + + def _settle_after_commit(request: dict, result_path: Path, step: str, reason: str) -> int: """The tree already moved, so a failure here is owed work, never a failed update (C3, A6). @@ -180,6 +200,8 @@ def _settle_after_commit(request: dict, result_path: Path, step: str, reason: st update_receipt.record_stage("deps" if step == "dependencies" else "build", "failed") update_receipt.record_followup(step, reason, retry="dependencies not installed yet — the next launch retries" if step == "dependencies" else "the next launch or `hermes update` retries it") + if not _record_owed_user_action(request): + print(_owed_user_action(request)) # the completion child that would print it never ran update_receipt.finalize_pending_update_receipt(0, f"{step} owed after the code was updated") receipt = _read_terminal_receipt(request) code = 0 if receipt is not None and receipt.get("outcome") == "success" else 1 @@ -229,12 +251,14 @@ def _prepare(request: dict, request_path: Path, result_path: Path) -> int: return code -def _complete_selected(request: dict) -> None: - """Everything after the commit point. Nothing here fails the update (contract C3). +def _complete_selected(request: dict) -> bool: + """Everything after the commit point. No step failure fails the update (contract C3). Each step is independent; a failed one prints ``⚠``, lands on the receipt as a follow-up and keeps its own obligation armed (``source-completion-pending`` for the tail, the fleet restart obligation for gateways), so the next launch or ``hermes update`` retries it. + Returns False only when the user's local changes were left parked in the stash: nothing + retries that, so the run is ``partial`` and exits 1 (#122557), never "Update complete". """ from hermes_cli import main, update_cmd, update_cmd_config from hermes_cli.source_completion import complete_source_checkout @@ -243,6 +267,7 @@ def _complete_selected(request: dict) -> None: root = Path(request["source"]) main.PROJECT_ROOT = root + complete = _record_owed_user_action(request) update_cmd_config._LAST_SIBLING_SNAPSHOTS = request["sibling_snapshots"] plan_data = request["plan"] plan = None if plan_data is None else UpdatePlan(**{ @@ -269,14 +294,15 @@ def _complete_selected(request: dict) -> None: from hermes_cli.venv_sync import clear_completion clear_completion(root) # systemctl's KillMode=mixed fallback can kill this whole cgroup. Publish the - # gateway watcher's status BEFORE that operation: the code is committed, so it is 0. + # gateway watcher's status BEFORE that operation: the code is committed, so it is 0 unless + # the user's own changes are still parked. if request["gateway_mode"]: - update_cmd._write_gateway_update_exit_code(True) + update_cmd._write_gateway_update_exit_code(complete) if request.get("no_gateway_restart", False): record_skip("gateway_restart", "--no-gateway-restart: deferred, marker kept") record_stage("restart", "skipped") print("→ Gateway restart deferred (--no-gateway-restart); restart gateways separately.") - return + return complete skip = update_cmd._fleet_restart_skip_reason(plan) if skip and update_cmd._pending_fleet_restart_needed(): # A host already stamped "restarted" for this SHA whose fleet is still off the checkout @@ -287,12 +313,13 @@ def _complete_selected(request: dict) -> None: record_skip("gateway_restart", skip) record_stage("restart", "skipped") print(f" ✓ Gateway restart skipped: {skip}.") - return + return complete restart = update_cmd._restart_gateway_fleet_after_update(plan, request["gateway_mode"]) record_stage("restart", "failed" if getattr(restart, "incomplete", False) else "success") update_cmd._resume_windows_gateways_and_merge_outcome(restart, request["windows_resume"], request["gateway_mode"]) update_cmd._verify_fleet_after_update( restart, _pre_update_plan=plan, _windows_gateway_resume=request["windows_resume"], update_complete=True) + return complete class _ForwardedOutput: @@ -317,7 +344,8 @@ def _finish(request: dict, result_path: Path) -> int: update_receipt.record_stage("deps", "success") # only a completed PM preparation reaches --prepared code, reason = 0, "source update completion" try: - _complete_selected(request) + if not _complete_selected(request): + code, reason = 1, "local changes left in the stash; re-apply them by hand" except KeyboardInterrupt: # An operator interrupt is not a step failure, and the code already moved: the run is # ``interrupted`` (never "failed"), and the armed obligations finish the tail. diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index 1e5e1a3db408..16c11f7c3d37 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -188,6 +188,8 @@ def gateway_restart_result( self.data["gateway_restart"] = result def finalize(self, outcome: str) -> None: + if outcome == "success" and self.data.get("user_action"): + outcome = "partial" # committed, but the user still has to act (record_user_action) self.data["outcome"] = outcome self.data["finished_at"] = _utc_now_iso() self.data["post_update"] = _code_identity(refresh=True) @@ -332,6 +334,16 @@ def record_followup(step: str, reason: str, *, retry: str = "the next launch or _persist_running(current.data) +def record_user_action(step: str, reason: str) -> None: + """The code committed, but something only the user can do is still owed. Never raises. + + Unlike a follow-up nothing retries it (a stash whose restore conflicted stays parked until the + user re-applies it), so the run can never be a plain success: it finalizes ``partial`` and + ``hermes update`` exits 1, as #122557 established for an unrestored autostash. + """ + _record("fact", f"update user action {step}", "user_action", {"step": step, "reason": " ".join(str(reason).split())[:500]}) + + def amend_terminal_followup(update_id: str, step: str, reason: str) -> None: """A follow-up that failed after the run's receipt was finalized lands on that receipt. Never raises.""" with suppress(Exception): @@ -557,7 +569,8 @@ def finalize_pending_update_receipt(exit_code: Optional[int] = None, stop_reason ``hermes update`` has many early ``sys.exit`` paths (preflight refusals, venv-holder refusal, fetch failure) predating the inner finalize calls; finalizing here means refused/failed runs — where a receipt matters most — leave a record. Exit 0/None → ``success``, exit 2 → ``refused`` - (preflight convention), else → ``failed``. + (preflight convention), else → ``failed`` (``partial`` when the run committed and only owes a + user action, see ``record_user_action``). No-op when no receipt is open (the inner paths already finalized — exactly-once via the popped per-context receipt) or when recording was never started. See #91283. @@ -565,7 +578,8 @@ def finalize_pending_update_receipt(exit_code: Optional[int] = None, stop_reason current = _current.get() if current is None: return None - outcome = "success" if exit_code in (0, None) else "refused" if exit_code == 2 else "failed" + outcome = ("success" if exit_code in (0, None) else "refused" if exit_code == 2 + else "partial" if current.data.get("user_action") else "failed") if exit_code is not None: with suppress(Exception): clone = copy.copy(current) diff --git a/scripts/desktop-update/posix.sh b/scripts/desktop-update/posix.sh index b1c86d26ed70..84e15ea6335b 100755 --- a/scripts/desktop-update/posix.sh +++ b/scripts/desktop-update/posix.sh @@ -88,7 +88,8 @@ STARTED_AT="$(date +%s)" # the shim's elapsed clock; see serve-ui.py UI_SERVER_PID="" UI_BROWSER_PID="" UI_PANEL_PID="" UI_PROFILE_DIR="" FINAL_CODE=1 FINAL_MSG="update did not complete" -DONE_NOTE="" # set when the update succeeded but the app will NOT reopen itself +DONE_NOTE="" # set when the update succeeded but the user must act (reopen, reinstall, rebuild) +APP_REBUILD_FAILED=0 # 1 = the code committed but the Desktop app build is an owed follow-up log() { echo "$(date +%Y-%m-%dT%H:%M:%S%z) $1" | tee -a "$LOG" 2>/dev/null; } @@ -400,7 +401,9 @@ mac_swap() { # Transactional swap: stage a full copy, move the old bundle aside, move # the copy in. Every step checked; a failed final move ROLLS BACK so the # user always has a launchable app, and the result file tells the truth. - if [ "$FINAL_CODE" -eq 0 ] && [ -n "$rebuilt" ] && [ -d "$RELAUNCH_TARGET" ] && [ "$rebuilt" != "$RELAUNCH_TARGET" ]; then + # A failed rebuild left release/ holding the OLD build: nothing new to install. + if [ "$FINAL_CODE" -eq 0 ] && [ "$APP_REBUILD_FAILED" -eq 0 ] && [ -n "$rebuilt" ] \ + && [ -d "$RELAUNCH_TARGET" ] && [ "$rebuilt" != "$RELAUNCH_TARGET" ]; then publish_stage "Installing the new app" rm -rf "$RELAUNCH_TARGET.new" "$RELAUNCH_TARGET.old" 2>/dev/null || true if ! /usr/bin/ditto "$rebuilt" "$RELAUNCH_TARGET.new"; then @@ -434,7 +437,7 @@ deliver_outcome() { # the truth-determining half: swap bundles / gate the relaun else linux_gate if [ "$GATE" != "relaunch" ] && [ "$FINAL_CODE" -eq 0 ]; then - DONE_NOTE="$GATE_MSG" + DONE_NOTE="${DONE_NOTE:+$DONE_NOTE }$GATE_MSG" log "no relaunch ($GATE): $GATE_MSG" fi fi @@ -505,9 +508,10 @@ finish() { fi if [ -n "$DONE_NOTE" ]; then - if [ "$(uname)" = "Darwin" ]; then + if [ "$(uname)" = "Darwin" ] || [ "$GATE" = "relaunch" ]; then # mac DONE_NOTE = swap failed but the PREVIOUS bundle was kept/rolled - # back — bring it back up; the note still tells the user to re-run. + # back, or (any OS) the Desktop rebuild is owed — bring the kept app + # back up; the note still tells the user what to do. # A gated linux outcome (skew/manual) skips the launch BY DESIGN. if ! launch_app; then # Even the kept bundle didn't come back: the durable message must @@ -885,6 +889,15 @@ if [ "$LEGACY_INSTALL" -eq 1 ] && [ "$CODE" -eq 0 ] && printf '%s' "$OUT" | grep fi if [ "$CODE" -eq 0 ]; then FINAL_CODE=0 FINAL_MSG="Update complete." + # Contract C3: a Desktop build that fails after the code committed is an owed + # follow-up (hermes update exits 0 and prints it). The user is on the new + # Hermes, but this app was not rebuilt: say so and say how to fix it, never + # "finished OK" and never "still on the previous version". + if printf '%s' "$OUT" | grep -Eq "Update follow-up 'build' did not finish: .*(desktop app build|Node dependencies)"; then + APP_REBUILD_FAILED=1 + DONE_NOTE="Hermes was updated, but the Desktop app could not be rebuilt, so it still runs its old build. Run hermes desktop --force-build in a terminal to rebuild it; the update log has the build error." + log "desktop app build is an owed follow-up of the committed update" + fi else FINAL_CODE="$CODE" FINAL_MSG="Update failed (exit $CODE). Run hermes debug share in a terminal to send a report." # The bricked-venv class is fixable and must not read as a generic exit 1: diff --git a/tests/e2e/core/upgrade/pm/test_stamps_and_doctor.py b/tests/e2e/core/upgrade/pm/test_stamps_and_doctor.py index 0edc3e265537..8dd99f0b8a22 100644 --- a/tests/e2e/core/upgrade/pm/test_stamps_and_doctor.py +++ b/tests/e2e/core/upgrade/pm/test_stamps_and_doctor.py @@ -13,8 +13,10 @@ * real drift is caught and healed: a user uninstalls fastapi (the ``web`` extra the dashboard imports) from the selected environment. ``hermes doctor`` must say so, and ``hermes pm repair`` must bring the dashboard's import back; -* a dependency update that cannot resolve fails loudly, ``hermes pm status`` reports it as failed, - and the previous generation stays selected and working. +* a dependency update that cannot resolve after the tree moved is an owed follow-up (contract C3, + A6): ``hermes update`` exits 0 but says loudly that the dependencies are not installed yet, the + receipt names the ``dependencies`` follow-up, the source-update tail stays armed for the next + launch, and the previous generation stays selected and working. """ from __future__ import annotations @@ -149,20 +151,29 @@ def test_pm_repair_heals_the_drift(drifted): f"`hermes pm repair` exited 0 but the dashboard still cannot import: {imports}\n" + P.diagnostics(sb, rp)) -def test_failed_dependency_update_is_reported_as_failed(updated): - """A release whose uv.lock uv rejects: the update must fail loudly, the receipt must say so, and - the install must keep running on the generation it had.""" +def test_failed_dependency_update_is_an_owed_followup(updated): + """A release whose uv.lock uv rejects, synced after the tree moved: the code is committed, so + the update exits 0 (A6) but must say loudly that the dependencies are not installed yet, the + receipt must name the owed follow-up, the tail must stay armed, and the install must keep + running on the generation it had.""" sb = updated["sb"] before = P.selected_generation(sb) origin, scratch = updated["origin"], updated["root"] lock = I.git("show", "main:uv.lock", cwd=origin) + '\n[[package]]\nname = "e2e-broken"\n' - I.publish_commit(origin, scratch, "release: e2e broken lockfile", {"uv.lock": lock}) + target = I.publish_commit(origin, scratch, "release: e2e broken lockfile", {"uv.lock": lock}) up = P.update(sb) receipt = json.loads(P.ok(sb.cli("pm", "status")).stdout) - assert up.returncode != 0, "`hermes update` exited 0 on a release whose uv.lock uv rejects\n" + P.diagnostics(sb, up) - assert receipt.get("outcome") not in ("ok", "success"), ( - f"`hermes pm status` reports the failed update as a success: {receipt}\n" + P.diagnostics(sb, up)) - assert P.selected_generation(sb) == before, "a failed update switched the selected generation" + assert up.returncode == 0, ( + "`hermes update` failed although the code committed; a dependency sync after the tree moved " + "is an owed follow-up (A6)\n" + P.diagnostics(sb, up)) + assert I.git("rev-parse", "HEAD", cwd=sb.checkout) == target, "the code update did not land\n" + P.diagnostics(sb, up) + assert re.search(r"⚠ Update follow-up 'dependencies' did not finish: .*dependencies not installed yet", + up.stdout + up.stderr), "the owed dependency sync is not reported loudly\n" + P.diagnostics(sb, up) + assert receipt.get("outcome") == "success" and "dependencies" in [ + f.get("step") for f in receipt.get("followups") or []], ( + f"the receipt does not name the owed dependency sync: {receipt}\n" + P.diagnostics(sb, up)) + assert P.pending_marker(sb).exists(), "the owed source-update tail is not armed for the next launch" + assert P.selected_generation(sb) == before, "a failed dependency sync switched the selected generation" imports = P.managed_imports(sb, "pydantic", "openai") assert set(imports.values()) == {"ok"}, f"the install no longer works after a failed update: {imports}" cp = sb.cli("--version") diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index 747d014f41cc..c10009a538fb 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -162,6 +162,7 @@ def git(*args): " r = _current.get()\n" " if r is not None: r.data.setdefault('followups', []).append({'step': step, 'reason': reason})\n" "def amend_terminal_followup(*args): pass\n" + "def record_user_action(*args): pass\n" "def _receipt_dir():\n" " return pathlib.Path(os.environ['HERMES_HOME']) / 'logs/update_receipts'\n" "def finalize_pending_update_receipt(code, reason):\n" diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py index c91576436c54..e928f1b75d16 100644 --- a/tests/hermes_cli/test_update_post_commit_followups.py +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -37,6 +37,21 @@ def test_open_receipt_is_durably_running_and_finalizes_in_place(tmp_path, monkey assert list((tmp_path / "logs/update_receipts").glob(f"update_*_{final['update_id']}.json")) == run_files +@pytest.mark.parametrize("finish", [lambda: update_receipt.finalize_update_receipt("success"), + lambda: update_receipt.finalize_pending_update_receipt(1, "local changes parked")]) +def test_parked_local_changes_never_finalize_as_success(tmp_path, monkeypatch, finish): + # A follow-up is retried by the next launch; a stash whose restore conflicted is not, so the + # committed run must stay partial (#122557) on both the verify path and the boundary net. + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.begin_update_receipt() + update_receipt.record_user_action("local_changes", "⚠ hermes update stashed 1 local modification(s)\n Stash ref: abc") + finish() + final = _latest(tmp_path) + assert final["outcome"] == "partial" + assert final["user_action"] == {"step": "local_changes", + "reason": "⚠ hermes update stashed 1 local modification(s) Stash ref: abc"} + + def test_dead_running_record_is_reported_interrupted_by_the_next_run(tmp_path, monkeypatch, capsys): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) dead = subprocess.Popen(["true"]) From 1e4d5f82b90f04ad1682ca10f467ca47a5842f96 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:12:01 -0700 Subject: [PATCH 11/62] test(e2e): pm sync receipts stay in the active home; only update receipts are root-only The pm/receipt.py hunk is reverted (pm is not the updater's), so a sticky-profile update leaves pm's own sync receipt under the profile, where `hermes pm status` reads it. --- tests/e2e/core/upgrade/git/test_hostile_post.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/e2e/core/upgrade/git/test_hostile_post.py b/tests/e2e/core/upgrade/git/test_hostile_post.py index 67b329a072c7..da20f90536f2 100644 --- a/tests/e2e/core/upgrade/git/test_hostile_post.py +++ b/tests/e2e/core/upgrade/git/test_hostile_post.py @@ -21,8 +21,9 @@ update runs in the shared namespace; once the slow build started, every process born after the update was spawned is SIGSTOPped then SIGKILLed (that is the whole update tree, including a completion child in its own session or reparented to the namespace init). -(d) a sticky profile (``hermes profile use e2epost``) must not move update.log / receipts off the - root hermes home, and ``hermes logs update`` under that profile reads the root's update.log. +(d) a sticky profile (``hermes profile use e2epost``) must not move update.log / update receipts off + the root hermes home, and ``hermes logs update`` under that profile reads the root's update.log; + pm's own sync receipts stay where pm writes them and ``hermes pm status`` there still shows them. (e) the dependency sync fails after the tree moved (contract amendment A6): the release's ``pm.client.ensure_tools_for_sync`` raises while ``$HOME/.e2e-break-deps-sync`` exists; it runs in the completion bootstrap from the NEW tree, after the checkout moved. Exit 0, a ``dependencies`` @@ -548,10 +549,14 @@ def test_sticky_profile_update_logs_and_receipts_land_in_the_root_home(w): previous_update_id=prev.get("update_id")) + w.diag(cp) assert w.head() == target, f"premise: the release never committed:\n{diag}" assert cp.returncode == 0, f"the sticky-profile update failed:\n{diag}" - # Neither the update's receipts nor pm's sync receipts (same dir, ``kind`` field) follow - # the sticky profile: one root dir is what the Desktop and `hermes pm status` read. - assert not sorted(prof_after - prof_before), \ - f"the update wrote receipts into the profile home {prof_receipts}:\n{diag}" + # The update's receipts never follow the sticky profile: the root dir is what the Desktop + # and the hand-off scripts read. pm's own sync receipts stay where pm writes them (the + # active home; pm is not the updater's), and pm's reader under the profile still finds them. + assert not [n for n in prof_after - prof_before if n.startswith("update_")], \ + f"the update wrote update receipts into the profile home {prof_receipts}:\n{diag}" + pm_cp = w.sb.cli("pm", "status", timeout=300) + assert pm_cp.returncode == 0 and json.loads(pm_cp.stdout).get("kind") == "sync", \ + f"`hermes pm status` under the profile does not show pm's own receipt:\n{H.describe(pm_cp)}\n{diag}" assert log1 > log0, f"the root update.log did not grow:\n{diag}" assert rec.get("update_id") != prev.get("update_id") and rec.get("outcome") == "success", \ f"root latest.json is not this run's success:\n{diag}" From 45023322aa3163fea193b0e5e790ac8927ee6e33 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:02:20 -0700 Subject: [PATCH 12/62] fix(update): keep runtime safety, config debt and carried serves separate from exit 0 Round 5 (R8, R14) on the post-commit completion: - update_completion: the completion's runtime-safety verdict reaches fleet verification, so an unsafe SQLite runtime still vetoes gateway multiplex auto-migration; the committed update keeps exit 0 + the sqlite_runtime follow-up. - update_cmd_config: a failed config-format write raises into the owed-step guard: config_migration lands on the receipt and the tail stays pending. - update_receipt: a handed-off receipt from an older interpreter (no carried_manual_serves) snapshots the previous receipt's manual-serve debt before the running record replaces latest.json. - update_finish (historical takeover child): a post-commit build failure is a build follow-up, maintenance and the fleet restart still run, exit 0, tail stays pending until it finishes. --- hermes_cli/update_cmd_config.py | 4 ++ hermes_cli/update_cmd_fleet_verify.py | 8 +-- hermes_cli/update_completion.py | 10 +++- hermes_cli/update_finish.py | 71 ++++++++++++++++++++------- hermes_cli/update_receipt.py | 6 ++- 5 files changed, 74 insertions(+), 25 deletions(-) diff --git a/hermes_cli/update_cmd_config.py b/hermes_cli/update_cmd_config.py index fd99c81a6b19..5e91cc1c7267 100644 --- a/hermes_cli/update_cmd_config.py +++ b/hermes_cli/update_cmd_config.py @@ -205,6 +205,10 @@ def _check_and_apply_config_migration( except Exception as _mig_err: print(f" ⚠️ Config format update failed: {_mig_err}") print(" Run 'hermes config migrate' to retry.") + # Not a silent success: the caller's owed-step guard records ``config_migration`` as + # a receipt follow-up and keeps the source-update tail pending, so it is retried. + raise RuntimeError( + f"config format v{current_ver} → v{latest_ver} was not written: {_mig_err}") from _mig_err elif needs_migration: print() # Show WHAT changed, not just a count, for an informed yes/no. diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py index c7be2947416a..4197ad1b32b5 100644 --- a/hermes_cli/update_cmd_fleet_verify.py +++ b/hermes_cli/update_cmd_fleet_verify.py @@ -327,8 +327,8 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re _record_owed_gateway_inventory(_pre_update_plan) with _best_effort('Update receipt finalize failed: %s'): from hermes_cli.update_receipt import finalize_update_receipt - # ``update_complete`` is kept for the historical takeover caller; a False verdict (the - # SQLite runtime) is reported as a follow-up by the maintenance step, not a failed run. + # A False ``update_complete`` (unsafe SQLite runtime, owed tail work) never fails the + # run: maintenance already reported it as a follow-up. It only vetoes migration below. _receipt_path = finalize_update_receipt("success", fleet=_fleet_snapshot) if _receipt_path is not None: logger.info("Update receipt written: %s", _receipt_path) @@ -337,8 +337,8 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re return fleet._clear_fleet_restart_pending_marker() if not update_complete: - # Fleet caught up, but the selected runtime's SQLite is unsafe (reported by maintenance): - # leave the topology alone until the runtime is repaired. + # Fleet caught up, but the selected runtime's SQLite is unsafe or the tail is still owed + # (both reported by the completion): leave the topology alone until they are repaired. return # Fleet is healthy on the new code: fold per-profile gateways into one multiplexer when nothing # blocks it (deterministic; never prompts), else print the blockers and the one-liner to run later. diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index d71216038b42..e0112f8fcb39 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -276,8 +276,13 @@ def _complete_selected(request: dict) -> bool: # Launchers, products and post-build maintenance live in one place so an # install and an update cannot end in different states. followups: list[tuple[str, str]] = [] + # Exit status and runtime safety are separate facts (R8): an unsafe SQLite runtime keeps the + # committed update at exit 0 (reported as the ``sqlite_runtime`` follow-up), but fleet + # verification still needs the real verdict so it never auto-migrates the gateway topology + # on a runtime that can corrupt sessions. Unknown (the tail raised) is not proven safe. + runtime_safe = False try: - complete_source_checkout( + runtime_safe = complete_source_checkout( root, desktop=request["desktop"], assume_yes=request["assume_yes"], gateway_mode=request["gateway_mode"], pre_update_snapshot_id=request["snapshot_id"], pre_update_version=request["pre_update_version"], @@ -318,7 +323,8 @@ def _complete_selected(request: dict) -> bool: record_stage("restart", "failed" if getattr(restart, "incomplete", False) else "success") update_cmd._resume_windows_gateways_and_merge_outcome(restart, request["windows_resume"], request["gateway_mode"]) update_cmd._verify_fleet_after_update( - restart, _pre_update_plan=plan, _windows_gateway_resume=request["windows_resume"], update_complete=True) + restart, _pre_update_plan=plan, _windows_gateway_resume=request["windows_resume"], + update_complete=bool(runtime_safe) and complete) return complete diff --git a/hermes_cli/update_finish.py b/hermes_cli/update_finish.py index 32b3845dd0e1..ac2801e3c8bc 100644 --- a/hermes_cli/update_finish.py +++ b/hermes_cli/update_finish.py @@ -8,31 +8,55 @@ def finish_update(*, root, assume_yes, gateway_mode, pre_update_snapshot_id, had_desktop_app_before_update, pre_update_version, - plan, windows_resume) -> None: - """Finish the selected checkout; never fetch, switch branches or restore a stash.""" + plan, windows_resume, followups=None) -> None: + """Finish the selected checkout; never fetch, switch branches or restore a stash. + + Same contract as the current completion (C3): the code is committed, so a failed build or + maintenance step is owed work, never a failed update. ``followups`` carries the steps that + already failed (the product build); while any is owed the install stamp is withheld, the + source-update tail stays pending for the next launch, and the gateway topology is left + alone. An unsafe SQLite runtime is reported by maintenance and vetoes migration only. + """ from hermes_cli.update_cmd import ( _run_post_update_maintenance, _restart_gateway_fleet_after_update, _verify_fleet_after_update, _write_gateway_update_exit_code, _resume_windows_gateways_and_merge_outcome, ) + from hermes_cli.update_receipt import TAIL_FOLLOWUPS, record_followup, record_stage - complete = _run_post_update_maintenance( - assume_yes=assume_yes, gateway_mode=gateway_mode, - pre_update_snapshot_id=pre_update_snapshot_id, - had_desktop_app_before_update=had_desktop_app_before_update, - pre_update_version=pre_update_version, - ) - if complete: + owed = followups if followups is not None else [] + runtime_safe = False + try: + runtime_safe = _run_post_update_maintenance( + assume_yes=assume_yes, gateway_mode=gateway_mode, + pre_update_snapshot_id=pre_update_snapshot_id, + had_desktop_app_before_update=had_desktop_app_before_update, + pre_update_version=pre_update_version, followups=owed, + ) + except (Exception, SystemExit) as exc: # noqa: BLE001 — the code is committed; retry later + reason = str(exc) or type(exc).__name__ + record_followup("maintenance", reason) + owed.append(("maintenance", reason)) + tail_owed = any(step in TAIL_FOLLOWUPS for step, _ in owed) + record_stage("build", "failed" if tail_owed else "success") + if not tail_owed: from hermes_cli.source_stamp import write_source_stamp - - write_source_stamp(Path(root)) - # Restart can kill this process's gateway cgroup; record its result first. + from hermes_cli.venv_sync import clear_completion + + try: + write_source_stamp(Path(root)) + except (OSError, ValueError) as exc: + print(f"⚠ Source update completed, but the install stamp could not be written: {exc}", + file=sys.stderr) + clear_completion(Path(root)) + # Restart can kill this process's gateway cgroup; record its result first. The code is + # committed, so the watcher sees success; owed work is on the receipt. if gateway_mode: - _write_gateway_update_exit_code(complete) + _write_gateway_update_exit_code(True) restarted = _restart_gateway_fleet_after_update(plan, gateway_mode) _resume_windows_gateways_and_merge_outcome(restarted, windows_resume, gateway_mode) - _verify_fleet_after_update(restarted, _pre_update_plan=plan, - _windows_gateway_resume=windows_resume, update_complete=complete) + _verify_fleet_after_update(restarted, _pre_update_plan=plan, _windows_gateway_resume=windows_resume, + update_complete=bool(runtime_safe) and not tail_owed) def _restore_plan(data): @@ -80,6 +104,7 @@ def main(context: Path, result: Path) -> int: from hermes_cli.source_build import build_update_products from hermes_cli.update_lock import UpdateLock, describe_holder from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update + from hermes_cli.venv_sync import arm_completion cli.PROJECT_ROOT = root if restarting: @@ -105,13 +130,25 @@ def main(context: Path, result: Path) -> int: desktop_dir = root / "apps" / "desktop" desktop = (_desktop_packaged_executable(desktop_dir) is not None or _desktop_dist_exists(desktop_dir)) - build_update_products(root, desktop=desktop) + # The tail is owed from here until it finishes: a failure or a kill leaves the + # pending marker, so the next launch finishes it (same as the current completion). + try: + arm_completion(root) + except OSError as exc: # stale dependencies still trigger the next launch's sync + print(f" ⚠ Could not record the owed source-update tail: {exc}") + owed: list[tuple[str, str]] = [] + try: + build_update_products(root, desktop=desktop) + except (Exception, SystemExit) as exc: # noqa: BLE001 — committed code: owed, not failed + reason = str(exc) or type(exc).__name__ + update_receipt.record_followup("build", reason) + owed.append(("build", reason)) finish_update( root=root, assume_yes=request["assume_yes"], gateway_mode=request["gateway_mode"], pre_update_snapshot_id=request.get("pre_update_snapshot_id"), had_desktop_app_before_update=desktop, pre_update_version=request.get("pre_update_version"), - plan=plan, windows_resume=token, + plan=plan, windows_resume=token, followups=owed, ) code = 0 except SystemExit as exc: diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index 16c11f7c3d37..d7a423f4ed65 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -289,8 +289,10 @@ def begin_update_receipt(*, previous: dict | None = None, correlation_id: str | receipt.data.setdefault("pid_create_time", _process_create_time(receipt.data["pid"])) if not previous: reconcile_interrupted_runs() - # The running record is about to replace latest.json: snapshot the previous record's - # manual serve rows (raw, no side effects) for finalize's carry-forward. + # The running record is about to replace latest.json: snapshot the previous record's + # manual serve rows (raw, no side effects) for finalize's carry-forward. A handed-off + # receipt from an older interpreter predates the field, so it needs the same snapshot. + if not previous or "carried_manual_serves" not in previous: with suppress(Exception): prior = read_latest_receipt() or {} rows = list((prior.get("plan") or {}).get("runtimes") or []) From 3dc67d8025dfca198cde1faf52fde55cb76ff1e8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:07:25 -0700 Subject: [PATCH 13/62] test(update): real-process invariants for runtime safety, config debt, carried serves, historical C3 - unsafe SQLite (3.46.1 interpreter) keeps exit 0 + sqlite_runtime follow-up but never enters multiplex auto-migration; healthy 3.53.1 control does. - a late PermissionError on the profile config write is a config_migration follow-up, keeps the tail pending and withholds the install stamp. - an old-schema handoff receipt keeps the previous run's manual-serve debt (1 -> 1). - historical takeover: a real TypeScript build failure is a build follow-up, exit 0, config migration + fleet restart still run (rewritten from the old exit-1 pin). --- tests/hermes_cli/test_update_finish.py | 30 ++- .../test_update_post_commit_safety.py | 204 ++++++++++++++++++ 2 files changed, 224 insertions(+), 10 deletions(-) create mode 100644 tests/hermes_cli/test_update_post_commit_safety.py diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index a5a9535de106..536fe7742845 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -400,30 +400,40 @@ def test_selected_child_builds_and_finalizes_under_parent_lock(completion): @pytest.mark.platforms("posix") -def test_real_compiler_failure_retains_receipt_and_skips_completion(completion): +def test_real_compiler_failure_is_owed_build_and_completion_continues(completion): + """Contract C3 for the historical takeover child: the code is committed, so a failed product + build is an owed ``build`` follow-up, never a failed update. Maintenance (config migration) + and the fleet restart still run, the exit is 0, and the tail stays pending until it builds.""" + from hermes_cli.config_defaults import DEFAULT_CONFIG + from hermes_cli.venv_sync import completion_pending_path + import yaml + source, home, request, context, result, run = completion _npm_graph(source) - # A real TypeScript error: dependency installation and TUI succeed, but - # neither the web product nor config/fleet completion may claim success. + # A real TypeScript error: dependency installation and TUI succeed; the web product fails. _put(source, "web/src/main.ts", "const broken: string = 42; document.body.textContent = broken;\n") - config_before = (home / "config.yaml").read_bytes() context_before = context.read_bytes() child = run() - assert child.returncode == 1, child.stdout + child.stderr - # Contained output reports a failure through the step's tail on stdout. + assert child.returncode == 0, child.stdout + child.stderr assert "TypeScript build failed" in child.stdout + child.stderr assert (source / "npm-python.json").is_file() assert (source / "ui-tui/dist/entry.js").is_file() assert not (source / "hermes_cli/web_dist/index.html").exists() - assert not (source / "restarted-plan.json").exists() - assert (home / "config.yaml").read_bytes() == config_before + # Maintenance and the fleet restart were not skipped by the build failure. + config = yaml.safe_load((home / "config.yaml").read_text()) + assert config["_config_version"] == DEFAULT_CONFIG["_config_version"] + assert config["model"]["default"] == "retained-model" + assert (source / "restarted-plan.json").is_file() assert context.read_bytes() == context_before receipt = json.loads((home / "logs/update_receipts/latest.json").read_text()) - assert receipt["outcome"] == "failed" - assert receipt["exit_code"] == child.returncode + assert receipt["outcome"] == "success" + assert [row["step"] for row in receipt["followups"]] == ["build"] assert receipt["update_id"] == request["update_id"] assert receipt["steps"][0] == request["receipt"]["steps"][0] assert receipt["pm_venv_rebuild"] == request["pm_receipt"]["venv_rebuild"] + # The owed tail is retried by the next launch; nothing claims the tree was built. + assert completion_pending_path(source).is_file() + assert not (source / "install-stamp.json").exists() assert json.loads(result.read_text()) == {"resume_handled": True, "receipt_handled": True} diff --git a/tests/hermes_cli/test_update_post_commit_safety.py b/tests/hermes_cli/test_update_post_commit_safety.py new file mode 100644 index 000000000000..218342a53f73 --- /dev/null +++ b/tests/hermes_cli/test_update_post_commit_safety.py @@ -0,0 +1,204 @@ +"""Post-commit completion keeps exit status, runtime safety and owed debt as separate facts. + +Each test runs the real completion (or receipt rotation) in a separate process against a +disposable home and a real git checkout. The only seams are machine boundaries: the selected +interpreter's linked SQLite (a shim interpreter that answers the SQLite probe and execs the real +Python for everything else), and the gateway fleet restart / dashboard / multiplex migration +service calls, which are replaced by recorders so the test can never touch a host's services. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import textwrap + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + +_CHILD = textwrap.dedent(''' + import copy, json, os, subprocess, sys + from pathlib import Path + root_checkout, cell, mode = sys.argv[1:4] + sys.path.insert(0, root_checkout) + cell = Path(cell) + source = cell / "source" + source.mkdir() + (source / "pyproject.toml").write_text('[project]\\nname="completion-probe"\\nversion="0.0.0"\\n') + git = ["git", "-c", "user.email=probe@example.invalid", "-c", "user.name=Probe", "-c", "commit.gpgsign=false"] + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run([*git, "-C", str(source), "commit", "-qm", "fixture"], check=True) + if mode.startswith("sqlite"): + # The selected interpreter is the shim: its SQLite probe is the runtime-safety fact. + sys.executable = os.environ["PROBE_SHIM"] + import yaml + from hermes_cli.config import DEFAULT_CONFIG + home = Path(os.environ["HERMES_HOME"]) + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text(yaml.safe_dump({ + "_config_version": DEFAULT_CONFIG["_config_version"] - 1, + "telemetry": {"shared_metrics": {"enabled": False}}, + "updates": {"refresh_cua_driver": False}})) + # Service boundaries: record instead of touching the host's gateways/dashboards. + from hermes_cli import gateway_migrate, update_cmd, update_cmd_fleet as fleet + from hermes_cli import update_cmd_fleet_verify as verify, update_cmd_maint as maint + entered = cell / "migration-entered" + gateway_migrate.maybe_auto_migrate_after_update = lambda: entered.write_text("entered") + update_cmd._restart_gateway_fleet_after_update = lambda plan, gateway_mode: fleet._GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set()) + update_cmd._fleet_restart_skip_reason = lambda plan: None + update_cmd._surviving_pre_update_serve_runtimes = lambda plan: [] + maint._refresh_dashboard_after_update = lambda **kwargs: None + verify._print_legacy_units_warning = lambda: None + verify._collect_fleet_snapshot = lambda *args: [] + from hermes_cli import update_receipt + from hermes_cli.venv_sync import arm_completion, completion_pending_path + arm_completion(source) + update_receipt.begin_update_receipt() + receipt = copy.deepcopy(update_receipt._current.get().data) + # The completion process never inherits the parent's open receipt context. + update_receipt._current.reset(update_receipt._current.get().current_token) + request = {"schema": 1, "source": str(source), "home": str(home), "branch": "main", "desktop": False, + "assume_yes": True, "gateway_mode": False, "no_gateway_restart": mode == "config-ro", + "pre_update_version": None, "snapshot_id": None, "sibling_snapshots": {}, "plan": None, + "receipt": receipt, "windows_resume": None} + if mode == "config-ro": + home.chmod(0o555) # a late filesystem failure: the profile home refuses the config write + from hermes_cli.update_completion import _finish + try: + code = _finish(request, cell / "result.json") + finally: + home.chmod(0o755) + latest = update_receipt.read_latest_receipt() or {} + (cell / "summary.json").write_text(json.dumps({ + "code": code, "outcome": latest.get("outcome"), + "followups": [row["step"] for row in latest.get("followups", [])], + "config_version": yaml.safe_load((home / "config.yaml").read_text())["_config_version"], + "latest_config_version": DEFAULT_CONFIG["_config_version"], + "pending": completion_pending_path(source).is_file(), + "stamp": (source / "install-stamp.json").is_file(), + "migration_entered": entered.is_file()})) +''') + + +def _run_completion(tmp_path: Path, mode: str, *, sqlite: str | None = None) -> dict: + cell = tmp_path / "cell" + root_home = cell / "home" / ".hermes" + # A named profile: receipts stay in the (writable) root home while the profile home refuses. + profile = root_home / "profiles" / "probe" + for name in ("logs/update_receipts", "skills", "cache", "profiles/probe"): + (root_home / name).mkdir(parents=True, exist_ok=True) + env = {key: value for key, value in os.environ.items() if not key.startswith(("HERMES_", "PYTEST_"))} + env.update(HOME=str(cell / "home"), HERMES_HOME=str(profile), HERMES_DISABLE_LAZY_INSTALLS="1", + HERMES_RUNTIME_DIR=str(tmp_path / "store"), PYTHONDONTWRITEBYTECODE="1", + XDG_CONFIG_HOME=str(cell / "home/.config"), XDG_CACHE_HOME=str(cell / "home/.cache")) + if sqlite is not None: + payload = json.dumps({"base_prefix": sys.base_prefix, "executable": sys.executable, + "python_version": list(sys.version_info[:3]), + "sqlite_version": [int(p) for p in sqlite.split(".")], + "sqlite_version_string": sqlite, "sqlite_source_id": "fixture"}) + shim = tmp_path / "python-shim" + script = (f"#!/bin/sh\ncase \"$3\" in *sqlite_source_id*) printf '%s\\n' '{payload}'; exit 0;; esac\n" + f"exec '{sys.executable}' \"$@\"\n") + shim.write_text(script, encoding="utf-8") + shim.chmod(0o755) + env["PROBE_SHIM"] = str(shim) + child = tmp_path / "child.py" + child.write_text(_CHILD, encoding="utf-8") + done = subprocess.run([sys.executable, "-B", str(child), str(ROOT), str(cell), mode], + env=env, cwd=tmp_path, capture_output=True, text=True, timeout=240) + assert done.returncode == 0, done.stdout + done.stderr + summary = json.loads((cell / "summary.json").read_text(encoding="utf-8-sig")) + summary["output"] = done.stdout + done.stderr + return summary + + +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("sqlite, safe", [("3.46.1", False), ("3.53.1", True)]) +def test_unsafe_sqlite_runtime_keeps_exit_zero_but_vetoes_topology_migration(tmp_path, sqlite, safe): + summary = _run_completion(tmp_path, f"sqlite-{sqlite}", sqlite=sqlite) + # Committed code: success and exit 0 either way. + assert summary["code"] == 0, summary["output"] + assert summary["outcome"] == "success" + if safe: + assert summary["followups"] == [] + assert summary["migration_entered"] is True + else: + # The unsafe runtime is a visible, owed step — and it must still veto the migration. + assert f"SQLite ({sqlite})" in summary["output"] + assert summary["followups"] == ["sqlite_runtime"] + assert summary["migration_entered"] is False + + +@pytest.mark.platforms("posix") +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores directory modes") +def test_config_format_write_failure_is_owed_not_reported_complete(tmp_path): + summary = _run_completion(tmp_path, "config-ro") + assert summary["code"] == 0, summary["output"] + assert summary["config_version"] == summary["latest_config_version"] - 1 + # The receipt names the debt, the tail stays pending and nothing stamps the tree as done. + assert summary["followups"] == ["config_migration"] + assert summary["pending"] is True + assert summary["stamp"] is False + + +_RECEIPT_CHILD = textwrap.dedent(''' + import json, sys + root_checkout, action = sys.argv[1:3] + sys.path.insert(0, root_checkout) + from hermes_cli import update_receipt as ur + if action == "seed": + ur.begin_update_receipt() + ur.record_fact("plan", {"runtimes": [json.loads(sys.argv[3])]}) + else: + # An older interpreter's frozen handoff payload: no ``carried_manual_serves`` field. + old = ur.UpdateReceipt().data + old.pop("carried_manual_serves", None) + ur.begin_update_receipt(previous=old, correlation_id=old["update_id"]) + ur.finalize_pending_update_receipt(0) +''') + + +@pytest.mark.platforms("posix") +def test_old_schema_handoff_receipt_keeps_the_previous_runs_manual_serve_debt(tmp_path): + from hermes_cli.process_identity import _process_create_time + + home = tmp_path / "home" / ".hermes" + home.mkdir(parents=True) + # The durable reminder store is obstructed, so the receipt is the only record of the debt. + (home / "serve_restart_pending").write_text("not a directory\n", encoding="utf-8") + env = {key: value for key, value in os.environ.items() if not key.startswith(("HERMES_", "PYTEST_"))} + env.update(HOME=str(tmp_path / "home"), HERMES_HOME=str(home), PYTHONDONTWRITEBYTECODE="1") + child = tmp_path / "receipt_child.py" + child.write_text(_RECEIPT_CHILD, encoding="utf-8") + serve = subprocess.Popen([sys.executable, "-I", "-c", "import time; print('ready', flush=True); time.sleep(120)"], + stdout=subprocess.PIPE, text=True) + assert serve.stdout is not None + try: + assert serve.stdout.readline().strip() == "ready" + row = {"kind": "serve", "profile": "fixture", "supervisor": "manual-serve", "restart_via": "respawn-argv", + "pid": serve.pid, "detail": {"create_time": _process_create_time(serve.pid)}} + + def run(*args): + done = subprocess.run([sys.executable, "-B", str(child), str(ROOT), *args], + env=env, capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stdout + done.stderr + + def owed(): + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text(encoding="utf-8-sig")) + rows = (receipt.get("plan") or {}).get("runtimes", []) + receipt.get("pending_manual_serves", []) + return [r["pid"] for r in rows if r.get("supervisor") == "manual-serve"] + + run("seed", json.dumps(row)) + assert owed() == [serve.pid] + run("handoff") + assert owed() == [serve.pid] + finally: + serve.terminate() + serve.wait(timeout=10) + serve.stdout.close() From 09b06e760fed3b45e8d5c5b770a35a48d76e9a28 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:15:50 -0700 Subject: [PATCH 14/62] test(update): a failed config-format write is owed, not swallowed; children inherit the runner's dependency path --- ...test_update_config_migration_on_current.py | 7 ++++++- .../test_update_post_commit_safety.py | 19 +++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/hermes_cli/test_update_config_migration_on_current.py b/tests/hermes_cli/test_update_config_migration_on_current.py index e511d9920ce6..700f9f5bb400 100644 --- a/tests/hermes_cli/test_update_config_migration_on_current.py +++ b/tests/hermes_cli/test_update_config_migration_on_current.py @@ -49,7 +49,12 @@ def prompt(text): monkeypatch.setattr('builtins.input', prompt) gateway_prompts = [] monkeypatch.setattr(update_cmd, '_gateway_prompt', lambda text, default: gateway_prompts.append((text, default)) or 'y') - update_cmd._check_and_apply_config_migration(assume_yes=case == 'yes', gateway_mode=case == 'gateway') + if case == 'migration-error': + # A failed write is owed, not silently completed: the owed-step guard needs the raise. + with pytest.raises(RuntimeError, match=r'v2 → v3 was not written: cannot write config'): + update_cmd._check_and_apply_config_migration(assume_yes=False, gateway_mode=False) + else: + update_cmd._check_and_apply_config_migration(assume_yes=case == 'yes', gateway_mode=case == 'gateway') assert calls == expected assert bool(prompts) is (case in {'tty-yes', 'tty-decline', 'eof', 'unicode'}) assert bool(gateway_prompts) is (case == 'gateway') diff --git a/tests/hermes_cli/test_update_post_commit_safety.py b/tests/hermes_cli/test_update_post_commit_safety.py index 218342a53f73..1729116989fa 100644 --- a/tests/hermes_cli/test_update_post_commit_safety.py +++ b/tests/hermes_cli/test_update_post_commit_safety.py @@ -19,6 +19,15 @@ ROOT = Path(__file__).resolve().parents[2] + +def _child_env(**values: str) -> dict[str, str]: + """A fresh env for a child of THIS interpreter: same dependency path, no hermes state.""" + env = {key: value for key, value in os.environ.items() if not key.startswith(("HERMES_", "PYTEST_"))} + # The test runner may provide dependencies through sys.path rather than site-packages. + env["PYTHONPATH"] = os.pathsep.join(path for path in sys.path if path) + env.update(values) + return env + _CHILD = textwrap.dedent(''' import copy, json, os, subprocess, sys from pathlib import Path @@ -93,10 +102,9 @@ def _run_completion(tmp_path: Path, mode: str, *, sqlite: str | None = None) -> profile = root_home / "profiles" / "probe" for name in ("logs/update_receipts", "skills", "cache", "profiles/probe"): (root_home / name).mkdir(parents=True, exist_ok=True) - env = {key: value for key, value in os.environ.items() if not key.startswith(("HERMES_", "PYTEST_"))} - env.update(HOME=str(cell / "home"), HERMES_HOME=str(profile), HERMES_DISABLE_LAZY_INSTALLS="1", - HERMES_RUNTIME_DIR=str(tmp_path / "store"), PYTHONDONTWRITEBYTECODE="1", - XDG_CONFIG_HOME=str(cell / "home/.config"), XDG_CACHE_HOME=str(cell / "home/.cache")) + env = _child_env(HOME=str(cell / "home"), HERMES_HOME=str(profile), HERMES_DISABLE_LAZY_INSTALLS="1", + HERMES_RUNTIME_DIR=str(tmp_path / "store"), PYTHONDONTWRITEBYTECODE="1", + XDG_CONFIG_HOME=str(cell / "home/.config"), XDG_CACHE_HOME=str(cell / "home/.cache")) if sqlite is not None: payload = json.dumps({"base_prefix": sys.base_prefix, "executable": sys.executable, "python_version": list(sys.version_info[:3]), @@ -172,8 +180,7 @@ def test_old_schema_handoff_receipt_keeps_the_previous_runs_manual_serve_debt(tm home.mkdir(parents=True) # The durable reminder store is obstructed, so the receipt is the only record of the debt. (home / "serve_restart_pending").write_text("not a directory\n", encoding="utf-8") - env = {key: value for key, value in os.environ.items() if not key.startswith(("HERMES_", "PYTEST_"))} - env.update(HOME=str(tmp_path / "home"), HERMES_HOME=str(home), PYTHONDONTWRITEBYTECODE="1") + env = _child_env(HOME=str(tmp_path / "home"), HERMES_HOME=str(home), PYTHONDONTWRITEBYTECODE="1") child = tmp_path / "receipt_child.py" child.write_text(_RECEIPT_CHILD, encoding="utf-8") serve = subprocess.Popen([sys.executable, "-I", "-c", "import time; print('ready', flush=True); time.sleep(120)"], From 3a94ebd31a010c9d9c50910b326e210301f7311c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:26:01 -0700 Subject: [PATCH 15/62] test(update): round-5 invariants read config through hermes_yaml (PyYAML is not a dependency) --- tests/hermes_cli/test_update_finish.py | 4 ++-- tests/hermes_cli/test_update_post_commit_safety.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index 536fe7742845..2a8b401a524e 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -406,7 +406,7 @@ def test_real_compiler_failure_is_owed_build_and_completion_continues(completion and the fleet restart still run, the exit is 0, and the tail stays pending until it builds.""" from hermes_cli.config_defaults import DEFAULT_CONFIG from hermes_cli.venv_sync import completion_pending_path - import yaml + import hermes_yaml source, home, request, context, result, run = completion _npm_graph(source) @@ -420,7 +420,7 @@ def test_real_compiler_failure_is_owed_build_and_completion_continues(completion assert (source / "ui-tui/dist/entry.js").is_file() assert not (source / "hermes_cli/web_dist/index.html").exists() # Maintenance and the fleet restart were not skipped by the build failure. - config = yaml.safe_load((home / "config.yaml").read_text()) + config = hermes_yaml.safe_load((home / "config.yaml").read_text()) assert config["_config_version"] == DEFAULT_CONFIG["_config_version"] assert config["model"]["default"] == "retained-model" assert (source / "restarted-plan.json").is_file() diff --git a/tests/hermes_cli/test_update_post_commit_safety.py b/tests/hermes_cli/test_update_post_commit_safety.py index 1729116989fa..a52a64b64b77 100644 --- a/tests/hermes_cli/test_update_post_commit_safety.py +++ b/tests/hermes_cli/test_update_post_commit_safety.py @@ -44,7 +44,7 @@ def _child_env(**values: str) -> dict[str, str]: if mode.startswith("sqlite"): # The selected interpreter is the shim: its SQLite probe is the runtime-safety fact. sys.executable = os.environ["PROBE_SHIM"] - import yaml + import hermes_yaml as yaml from hermes_cli.config import DEFAULT_CONFIG home = Path(os.environ["HERMES_HOME"]) home.mkdir(parents=True, exist_ok=True) From 4d66135a1def1d940b939684521c539afbe353ef Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:00:09 -0700 Subject: [PATCH 16/62] fix(update): the receipt's build stage fails only for a build follow-up (C3) Both completion paths marked the build stage failed whenever ANY tail follow-up was owed, so a run whose only debt was config_migration (or maintenance / dependencies) reported a failed build. record_build_stage marks it failed only for launchers/build (what the build stage publishes) or completion (the tail raised before proving them); other owed steps stay their own follow-ups. --- hermes_cli/update_completion.py | 5 +++-- hermes_cli/update_finish.py | 4 ++-- hermes_cli/update_receipt.py | 11 +++++++++++ tests/hermes_cli/test_update_completion_process.py | 1 + tests/hermes_cli/test_update_finish.py | 2 ++ tests/hermes_cli/test_update_post_commit_safety.py | 3 +++ 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index e0112f8fcb39..d609adf5ab93 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -263,7 +263,8 @@ def _complete_selected(request: dict) -> bool: from hermes_cli import main, update_cmd, update_cmd_config from hermes_cli.source_completion import complete_source_checkout from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan - from hermes_cli.update_receipt import TAIL_FOLLOWUPS, record_followup, record_skip, record_stage + from hermes_cli.update_receipt import ( + TAIL_FOLLOWUPS, record_build_stage, record_followup, record_skip, record_stage) root = Path(request["source"]) main.PROJECT_ROOT = root @@ -294,7 +295,7 @@ def _complete_selected(request: dict) -> bool: record_followup("completion", reason) followups.append(("completion", reason)) tail_owed = any(step in TAIL_FOLLOWUPS for step, _ in followups) - record_stage("build", "failed" if tail_owed else "success") + record_build_stage(followups) if not tail_owed: from hermes_cli.venv_sync import clear_completion clear_completion(root) diff --git a/hermes_cli/update_finish.py b/hermes_cli/update_finish.py index ac2801e3c8bc..96527643b768 100644 --- a/hermes_cli/update_finish.py +++ b/hermes_cli/update_finish.py @@ -22,7 +22,7 @@ def finish_update(*, root, assume_yes, gateway_mode, pre_update_snapshot_id, _restart_gateway_fleet_after_update, _verify_fleet_after_update, _write_gateway_update_exit_code, _resume_windows_gateways_and_merge_outcome, ) - from hermes_cli.update_receipt import TAIL_FOLLOWUPS, record_followup, record_stage + from hermes_cli.update_receipt import TAIL_FOLLOWUPS, record_build_stage, record_followup owed = followups if followups is not None else [] runtime_safe = False @@ -38,7 +38,7 @@ def finish_update(*, root, assume_yes, gateway_mode, pre_update_snapshot_id, record_followup("maintenance", reason) owed.append(("maintenance", reason)) tail_owed = any(step in TAIL_FOLLOWUPS for step, _ in owed) - record_stage("build", "failed" if tail_owed else "success") + record_build_stage(owed) if not tail_owed: from hermes_cli.source_stamp import write_source_stamp from hermes_cli.venv_sync import clear_completion diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index d7a423f4ed65..9668a2f2aaa6 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -317,6 +317,17 @@ def persist_running_receipt() -> None: #: Follow-up steps whose failure leaves the source-update tail owed (``source-completion-pending``). TAIL_FOLLOWUPS = frozenset({"dependencies", "launchers", "build", "maintenance", "config_migration", "completion"}) +#: Follow-ups that mean the BUILD stage did not succeed (C3: the receipt names what actually failed): +#: the products or the launchers the build publishes failed, or the tail raised before proving them. +#: Owed dependencies, config migration or maintenance keep the tail armed but say nothing about the +#: build; they are reported as their own follow-ups. +BUILD_FOLLOWUPS = frozenset({"launchers", "build", "completion"}) + + +def record_build_stage(followups) -> None: + """Mark the build stage from the owed ``(step, reason)`` follow-ups: failed only for a build one.""" + record_stage("build", "failed" if any(step in BUILD_FOLLOWUPS for step, _ in followups) else "success") + def record_followup(step: str, reason: str, *, retry: str = "the next launch or `hermes update` retries it") -> None: """A post-commit step failed: print ⚠, keep the run a success, and say what is still owed. diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index c10009a538fb..013f13e9611e 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -155,6 +155,7 @@ def git(*args): "_current = contextvars.ContextVar('receipt', default=None)\n" "class UpdateReceipt: pass\n" "def record_stage(*args, **kwargs): pass\n" + "def record_build_stage(*args, **kwargs): pass\n" "def record_skip(*args, **kwargs): pass\n" "TAIL_FOLLOWUPS = frozenset({'dependencies', 'launchers', 'build', 'maintenance', 'config_migration', 'completion'})\n" "def record_followup(step, reason, **kwargs):\n" diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index 2a8b401a524e..781a71139c8c 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -428,6 +428,8 @@ def test_real_compiler_failure_is_owed_build_and_completion_continues(completion receipt = json.loads((home / "logs/update_receipts/latest.json").read_text()) assert receipt["outcome"] == "success" assert [row["step"] for row in receipt["followups"]] == ["build"] + # Only a build follow-up marks the build stage failed (an owed config/dependency step does not). + assert {mark["name"]: mark["outcome"] for mark in receipt["stages"]}.get("build") == "failed" assert receipt["update_id"] == request["update_id"] assert receipt["steps"][0] == request["receipt"]["steps"][0] assert receipt["pm_venv_rebuild"] == request["pm_receipt"]["venv_rebuild"] diff --git a/tests/hermes_cli/test_update_post_commit_safety.py b/tests/hermes_cli/test_update_post_commit_safety.py index a52a64b64b77..555f535eac47 100644 --- a/tests/hermes_cli/test_update_post_commit_safety.py +++ b/tests/hermes_cli/test_update_post_commit_safety.py @@ -87,6 +87,7 @@ def _child_env(**values: str) -> dict[str, str]: (cell / "summary.json").write_text(json.dumps({ "code": code, "outcome": latest.get("outcome"), "followups": [row["step"] for row in latest.get("followups", [])], + "stages": {mark["name"]: mark["outcome"] for mark in latest.get("stages", [])}, "config_version": yaml.safe_load((home / "config.yaml").read_text())["_config_version"], "latest_config_version": DEFAULT_CONFIG["_config_version"], "pending": completion_pending_path(source).is_file(), @@ -153,6 +154,8 @@ def test_config_format_write_failure_is_owed_not_reported_complete(tmp_path): assert summary["followups"] == ["config_migration"] assert summary["pending"] is True assert summary["stamp"] is False + # C3: the receipt names what actually failed. The build succeeded; only the config is owed. + assert summary["stages"].get("build") == "success", summary["stages"] _RECEIPT_CHILD = textwrap.dedent(''' From ef2c5382c11891da18331492350c3f25c51b1664 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:20:00 -0700 Subject: [PATCH 17/62] fix(update): preserve receipt identity and durable terminal facts --- hermes_cli/update_receipt.py | 32 +++-- .../test_update_post_commit_followups.py | 6 +- .../test_update_receipt_durability.py | 112 ++++++++++++++++++ 3 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 tests/hermes_cli/test_update_receipt_durability.py diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index 9668a2f2aaa6..c8aec94bc120 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -218,21 +218,36 @@ def _persist_running(data: dict[str, Any]) -> None: directory = _receipt_dir() directory.mkdir(parents=True, exist_ok=True) - payload = (json.dumps({**data, "writer_pid": os.getpid()}, indent=2, default=str) + "\n").encode("utf-8") - _atomic_bytes(_run_file(directory, data), payload) + path = _run_file(directory, data) + # A completion child can finalize while its parent still holds the pre-child + # context. That stale running snapshot must never undo the terminal receipt. + with suppress(OSError, ValueError): + stored = json.loads(path.read_text(encoding="utf-8-sig")) + if stored.get("update_id") == data.get("update_id") and stored.get("finished_at"): + return + from hermes_cli.process_identity import _process_create_time + + payload = (json.dumps({**data, "writer_pid": os.getpid(), + "writer_create_time": _process_create_time(os.getpid())}, + indent=2, default=str) + "\n").encode("utf-8") + _atomic_bytes(path, payload) _atomic_bytes(directory / "latest.json", payload) def _owner_alive(record: dict[str, Any]) -> bool: from hermes_cli.process_identity import _pid_alive_matches - for key in ("pid", "writer_pid"): + identities = {} + for key, time_key in (("pid", "pid_create_time"), ("writer_pid", "writer_create_time")): pid = record.get(key) - if isinstance(pid, int) and pid > 0 and pid != os.getpid(): - create_time = record.get("pid_create_time") if key == "pid" else None - if _pid_alive_matches(pid, create_time) is not False: - return True - return False + if isinstance(pid, int) and pid > 0: + # Older receipts recorded no writer incarnation. Do not let a duplicate + # bare PID override the original owner's proven creation-time mismatch. + create_time = record.get(time_key) + if pid not in identities or identities[pid] is None: + identities[pid] = create_time + return any(_pid_alive_matches(pid, create_time) is not False + for pid, create_time in identities.items()) def reconcile_interrupted_runs() -> list[dict[str, Any]]: @@ -355,6 +370,7 @@ def record_user_action(step: str, reason: str) -> None: ``hermes update`` exits 1, as #122557 established for an unrestored autostash. """ _record("fact", f"update user action {step}", "user_action", {"step": step, "reason": " ".join(str(reason).split())[:500]}) + persist_running_receipt() def amend_terminal_followup(update_id: str, step: str, reason: str) -> None: diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py index e928f1b75d16..f07ed21e03b0 100644 --- a/tests/hermes_cli/test_update_post_commit_followups.py +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -82,7 +82,11 @@ def test_live_running_record_is_not_reclaimed(tmp_path, monkeypatch): own_id = update_receipt._current.get().data["update_id"] for path in (tmp_path / "logs/update_receipts").glob("*.json"): data = json.loads(path.read_text()) - data.update(pid=os.getppid(), writer_pid=os.getppid(), pid_create_time=None) # alive + from hermes_cli.process_identity import _process_create_time + + parent_time = _process_create_time(os.getppid()) + data.update(pid=os.getppid(), writer_pid=os.getppid(), + pid_create_time=parent_time, writer_create_time=parent_time) path.write_text(json.dumps(data)) assert update_receipt.reconcile_interrupted_runs() == [] (archived,) = (tmp_path / "logs/update_receipts").glob(f"update_*_{own_id}.json") diff --git a/tests/hermes_cli/test_update_receipt_durability.py b/tests/hermes_cli/test_update_receipt_durability.py new file mode 100644 index 000000000000..11a8ead1b559 --- /dev/null +++ b/tests/hermes_cli/test_update_receipt_durability.py @@ -0,0 +1,112 @@ +"""Receipt recovery must preserve terminal and user-action facts across processes.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import textwrap + +ROOT = Path(__file__).resolve().parents[2] + + +def _run(tmp_path: Path, body: str) -> dict: + home = tmp_path / "account" + state = home / ".hermes" + state.mkdir(parents=True) + script = tmp_path / "receipt_probe.py" + script.write_text(textwrap.dedent(""" + import json, os, subprocess, sys + from pathlib import Path + sys.path.insert(0, sys.argv[1]) + from hermes_cli import update_receipt as receipts + from hermes_cli.process_identity import _process_create_time + """) + textwrap.dedent(body), encoding="utf-8") + env = {key: value for key, value in os.environ.items() + if not key.startswith(("HERMES_", "PYTEST_"))} + env.update(HOME=str(home), USERPROFILE=str(home), HERMES_HOME=str(state), + HERMES_RUNTIME_DIR=str(tmp_path / "tools"), + XDG_CONFIG_HOME=str(home / ".config"), XDG_CACHE_HOME=str(home / ".cache"), + PYTHONPATH=os.pathsep.join([str(ROOT), *filter(None, sys.path)]), + PYTHONDONTWRITEBYTECODE="1") + done = subprocess.run([sys.executable, str(script), str(ROOT)], env=env, cwd=tmp_path, + text=True, capture_output=True, timeout=60) + assert done.returncode == 0, done.stdout + done.stderr + return json.loads((tmp_path / "result.json").read_text(encoding="utf-8")) + + +def test_nested_begin_does_not_reconcile_live_outer_receipt(tmp_path): + result = _run(tmp_path, """ + receipts.begin_update_receipt() + outer = dict(receipts._current.get().data) + receipts.begin_update_receipt() + archive = receipts._run_file(receipts._receipt_dir(), outer) + before = json.loads(archive.read_text()) + receipts.finalize_pending_update_receipt(0) + restored = receipts.current_correlation_id() + receipts.finalize_pending_update_receipt(0) + Path('result.json').write_text(json.dumps({ + 'outcome': before['outcome'], 'restored': restored, 'outer': outer['update_id']})) + """) + assert result["outcome"] == "running" + assert result["restored"] == result["outer"] + + +def test_writer_identity_cannot_resurrect_a_mismatched_owner(tmp_path): + result = _run(tmp_path, """ + child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)']) + try: + ct = _process_create_time(child.pid) + assert ct is not None + same = {'pid': child.pid, 'writer_pid': child.pid, 'pid_create_time': ct - 60} + writer = {'pid': 0, 'writer_pid': child.pid, 'writer_create_time': ct - 60} + control = {'pid': child.pid, 'writer_pid': child.pid, 'pid_create_time': ct} + Path('result.json').write_text(json.dumps({ + 'same': receipts._owner_alive(same), 'writer': receipts._owner_alive(writer), + 'control': receipts._owner_alive(control)})) + finally: + child.terminate() + child.wait(timeout=10) + """) + assert result == {"same": False, "writer": False, "control": True} + + +def test_user_action_survives_interrupt_before_next_stage(tmp_path): + result = _run(tmp_path, """ + receipts.begin_update_receipt() + receipts.record_user_action('local_changes', 'Reapply stash@{0} by hand') + running = receipts.read_latest_receipt() + receipts.finalize_interrupted_update_receipt('operator interrupted') + terminal = receipts.read_latest_receipt() + Path('result.json').write_text(json.dumps({'running': running, 'terminal': terminal})) + """) + expected = {"step": "local_changes", "reason": "Reapply stash@{0} by hand"} + assert result["running"].get("user_action") == expected + assert result["terminal"].get("user_action") == expected + assert result["terminal"]["outcome"] == "interrupted" + + +def test_parent_followup_does_not_replace_child_terminal_with_running(tmp_path): + result = _run(tmp_path, """ + receipts.begin_update_receipt() + frozen = dict(receipts._current.get().data) + Path('request.json').write_text(json.dumps(frozen)) + child = subprocess.run([sys.executable, '-c', + 'import json,sys; from pathlib import Path; ' + 'from hermes_cli.update_completion import _resume_receipt; ' + 'from hermes_cli.update_receipt import finalize_pending_update_receipt; ' + '_resume_receipt(json.loads(Path("request.json").read_text())); ' + 'finalize_pending_update_receipt(0)'], check=True) + before = receipts.read_latest_receipt() + receipts.record_followup('windows_resume', 'recovery refused') + receipts.amend_terminal_followup(frozen['update_id'], 'windows_resume', 'recovery refused') + after = receipts.read_latest_receipt() + archive = json.loads(receipts._run_file(receipts._receipt_dir(), frozen).read_text()) + Path('result.json').write_text(json.dumps({'before': before, 'after': after, 'archive': archive})) + """) + assert result["before"]["outcome"] == "success" + for terminal in (result["after"], result["archive"]): + assert terminal["outcome"] == "success" + assert terminal["finished_at"] == result["before"]["finished_at"] + assert [row["step"] for row in terminal["followups"]] == ["windows_resume"] From dbf710dcf0f88d54c36589d8638ae5442be4fdf1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:26:16 -0700 Subject: [PATCH 18/62] fix(update): no post-commit receipt-store, restart-IPC or profile-sync path fails the update (D5) - A receipt store that refuses the terminal write after the commit point no longer selects exit 1: update_receipt keeps the correlated terminal record this process finalized (finalized_receipt) and the completion child answers its parent from it; run_completion's correlation gate is unchanged. - Restart failures (failed systemd unit, abort recovery, Windows service resume) no longer write 1 to the gateway /update marker; the debt stays the gateway_restart follow-up, the host obligation and the warning lines. - TAIL_FOLLOWUPS includes profile_sync, so an owed profile sync keeps source-completion-pending armed and withholds the install stamp. --- hermes_cli/update_cmd_fleet.py | 10 +- hermes_cli/update_cmd_windows.py | 6 +- hermes_cli/update_completion.py | 19 +-- hermes_cli/update_receipt.py | 19 ++- .../test_update_completion_process.py | 4 +- .../test_update_post_commit_safety.py | 110 +++++++++++++++++- 6 files changed, 144 insertions(+), 24 deletions(-) diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index 8c49bf1946d0..3e297e48c632 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -1644,7 +1644,7 @@ def _recover_after_restart_phase_abort( from hermes_cli.update_abort_recovery import _owed_stale_serve_rows from hermes_cli.update_cmd import ( _abort_recovery_is_complete, _recover_gateway_restart_after_abort, _surviving_pre_update_serve_runtimes, - _warn_stale_serve_runtimes, _write_gateway_update_exit_code, + _warn_stale_serve_runtimes, ) logger.debug("Gateway restart during update failed: %s", e) out.phase_errors.append(str(e)) @@ -1704,11 +1704,10 @@ def _recover_after_restart_phase_abort( or _owed_stale_serve_rows(_stale_runtime_rows) or _serve_units_failed ): + # Owed restart (C3): verification records the follow-up; the /update marker stays committed. out.incomplete = True _warn_gateway_restart_phase_aborted(e, _surviving) _warn_stale_serve_runtimes(_stale_runtime_rows) - if gateway_mode: - _write_gateway_update_exit_code(False) out.record_receipt(phase_error=str(e), fresh_recovery=_recovery_result) @@ -1727,7 +1726,7 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): Never raises: a phase abort runs fresh-child recovery and fails closed unless every planned gateway is verifiably covered. """ - from hermes_cli.update_cmd import _m, _write_gateway_update_exit_code + from hermes_cli.update_cmd import _m # All bookkeeping is declared before the try so abort recovery and fleet reconciliation # can read it even if the phase raises early. ``pre_restart_gateway_pids`` stays empty # until we are about to stop/drain, so an early exception has nothing to fail closed on, @@ -1787,9 +1786,8 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): _restart_manual_gateways(out, _drain_budget) if out.failed_or_stale_units: + # Owed restart (C3): verification records the follow-up; the /update marker stays committed. out.incomplete = True - if gateway_mode: - _write_gateway_update_exit_code(False) _warn_incomplete_gateway_fleet_restart(out.failed_or_stale_units) out.record_receipt() _force_kill_stuck_gateways(out.killed_pids) diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 2dd7c35d03ea..21b490525a1d 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -1331,15 +1331,15 @@ def _resume_windows_gateways_after_update(token: dict | None) -> None: def _resume_windows_gateways_and_merge_outcome(outcome, _windows_gateway_resume, gateway_mode: bool): """Resume gateways paused for a Windows update and fold the token into ``outcome``'s systemd/launchd-style bookkeeping so reconciliation never reports a healthy gateway as unaccounted. Must never abort the update.""" - from hermes_cli.update_cmd import _m, _write_gateway_update_exit_code + from hermes_cli.update_cmd import _m try: _m()._resume_windows_gateways_after_update(_windows_gateway_resume) except Exception as _windows_resume_exc: + # Owed restart, not a failed update (C3): verification records the ``gateway_restart`` + # follow-up; the gateway's /update marker keeps the committed result. outcome.incomplete = True outcome.phase_errors.append(str(_windows_resume_exc)) print(f" ⚠ Windows gateway service restart incomplete: {_windows_resume_exc}") - if gateway_mode: - _write_gateway_update_exit_code(False) if not isinstance(_windows_gateway_resume, dict): return diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index d609adf5ab93..d6ba20863e5e 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -132,16 +132,21 @@ def _resume_receipt(data: dict) -> None: def _read_terminal_receipt(request: dict) -> dict | None: - from hermes_cli.update_receipt import _receipt_dir + from hermes_cli import update_receipt # The ROOT home's receipts (the run's own file), never latest.json: another # profile/context may have finalized more recently. - directory = _receipt_dir() - for path in directory.glob(f"update_*_{request['receipt']['update_id']}.json"): - data = json.loads(path.read_text(encoding="utf-8-sig")) - if data.get("update_id") == request["receipt"]["update_id"] and data.get("finished_at"): - return data - return None + update_id = request["receipt"]["update_id"] + try: + for path in update_receipt._receipt_dir().glob(f"update_*_{update_id}.json"): + data = json.loads(path.read_text(encoding="utf-8-sig")) + if data.get("update_id") == update_id and data.get("finished_at"): + return data + except (OSError, ValueError): + pass + # The store refused the terminal write after the commit point (loudly): this process still + # holds the correlated terminal record it finalized, so the exit status never turns into 1. + return update_receipt.finalized_receipt(update_id) def _running_record(request: dict) -> dict | None: diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index c8aec94bc120..cb764ceeb6e7 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -45,6 +45,8 @@ logger = logging.getLogger(__name__) _RECEIPT_KEEP = 20 # keep the last N receipts per profile home +#: Terminal records finalized by THIS process, by update id (see ``finalized_receipt``). +_FINALIZED: dict[str, dict[str, Any]] = {} COMMAND_BOUNDARY_STOP_REASON = "completed at command boundary" # Receipt state is per-CONTEXT, not a module global: a nested @@ -330,7 +332,8 @@ def persist_running_receipt() -> None: #: Follow-up steps whose failure leaves the source-update tail owed (``source-completion-pending``). -TAIL_FOLLOWUPS = frozenset({"dependencies", "launchers", "build", "maintenance", "config_migration", "completion"}) +TAIL_FOLLOWUPS = frozenset({"dependencies", "launchers", "build", "maintenance", "profile_sync", + "config_migration", "completion"}) #: Follow-ups that mean the BUILD stage did not succeed (C3: the receipt names what actually failed): #: the products or the launchers the build publishes failed, or the tail raised before proving them. @@ -466,6 +469,9 @@ def finalize_update_receipt(outcome: str, fleet: list | None = None, stop_reason receipt.finalize(outcome) if stop_reason: receipt.data["stop_reason"] = stop_reason + # The terminal facts exist before the store is touched; a store that refuses the write + # (after the commit point) must not erase them for this process's own caller. + _FINALIZED[str(receipt.data.get("update_id"))] = receipt.data if fleet is not None: receipt.data["fleet"] = fleet # Manual serve restart obligations outlive one receipt rotation: carry the previous @@ -592,6 +598,17 @@ def _publish_shared_metrics(data: dict[str, Any]) -> None: record_update_receipt(data) +def finalized_receipt(update_id: str) -> Optional[dict[str, Any]]: + """The terminal record this process finalized for ``update_id``, published or not. + + The completion child answers its parent from this when the receipt store refused the + terminal write: the run is still correlated and terminal, only its archive is missing + (already reported by ``⚠ Update receipt not written``). + """ + data = _FINALIZED.get(str(update_id)) + return copy.deepcopy(data) if data is not None else None + + def finalize_pending_update_receipt(exit_code: Optional[int] = None, stop_reason: str = "") -> Optional[Path]: """Command-boundary safety net: persist a still-open receipt, if any. Never raises. diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index 013f13e9611e..a5f7610288a5 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -157,7 +157,9 @@ def git(*args): "def record_stage(*args, **kwargs): pass\n" "def record_build_stage(*args, **kwargs): pass\n" "def record_skip(*args, **kwargs): pass\n" - "TAIL_FOLLOWUPS = frozenset({'dependencies', 'launchers', 'build', 'maintenance', 'config_migration', 'completion'})\n" + "TAIL_FOLLOWUPS = frozenset({'dependencies', 'launchers', 'build', 'maintenance', 'profile_sync',\n" + " 'config_migration', 'completion'})\n" + "finalized_receipt = lambda update_id: None\n" "def record_followup(step, reason, **kwargs):\n" " print(f' ⚠ Update follow-up {step!r} did not finish: {reason}')\n" " r = _current.get()\n" diff --git a/tests/hermes_cli/test_update_post_commit_safety.py b/tests/hermes_cli/test_update_post_commit_safety.py index 555f535eac47..66cd90d0ea5c 100644 --- a/tests/hermes_cli/test_update_post_commit_safety.py +++ b/tests/hermes_cli/test_update_post_commit_safety.py @@ -57,6 +57,7 @@ def _child_env(**values: str) -> dict[str, str]: from hermes_cli import update_cmd_fleet_verify as verify, update_cmd_maint as maint entered = cell / "migration-entered" gateway_migrate.maybe_auto_migrate_after_update = lambda: entered.write_text("entered") + real_restart = update_cmd._restart_gateway_fleet_after_update update_cmd._restart_gateway_fleet_after_update = lambda plan, gateway_mode: fleet._GatewayRestartOutcome( incomplete=False, phase_errors=[], pre_restart_gateway_pids=[], restarted_services=[], failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set()) @@ -65,6 +66,43 @@ def _child_env(**values: str) -> dict[str, str]: maint._refresh_dashboard_after_update = lambda **kwargs: None verify._print_legacy_units_warning = lambda: None verify._collect_fleet_snapshot = lambda *args: [] + windows_resume = None + marker = home / ".update_exit_code" + if mode.startswith("ipc-"): + # Barrier: verification starts after the restart phase returned. A gateway /update reader + # polling the marker can consume it from here on, before the run ever finalizes. + verify._print_legacy_units_warning = lambda: (cell / "marker-at-verify").write_text( + marker.read_text() if marker.exists() else "absent") + if mode in ("ipc-failed-unit", "ipc-abort"): + # The real restart phase; only the host's service manager and process table are replaced. + from hermes_cli import gateway as gateway_module + update_cmd._restart_gateway_fleet_after_update = real_restart + fleet._restart_manual_gateways = lambda out, budget: None + fleet._force_kill_stuck_gateways = lambda pids: None + if mode == "ipc-failed-unit": + gateway_module.find_gateway_pids = lambda **kwargs: [] + fleet._restart_systemd_gateway_units = lambda restarted, failed, *rest: failed.append( + "hermes-gateway-probe.service") + else: + def unavailable(*args, **kwargs): + raise RuntimeError("systemctl unavailable") + gateway_module.find_gateway_pids = unavailable + fleet._restart_systemd_gateway_units = unavailable + update_cmd._recover_gateway_restart_after_abort = lambda plan, **kwargs: {} + if mode == "ipc-windows": + # The Windows service manager refuses the paused gateway's restart. + from hermes_cli import main as hermes_main + def scm(token): + if token and token.get("resume_needed"): + raise RuntimeError("Could not restart Windows gateway service(s): HermesGatewayProbe") + hermes_main._resume_windows_gateways_after_update = scm + update_cmd._resume_windows_gateways_after_update = scm + windows_resume = {"resume_needed": True, "services": ["HermesGatewayProbe"], + "service_profiles": {"HermesGatewayProbe": "probe"}} + if mode == "profile-sync": + def profile_sync(): + raise SystemExit("profile skill sync exited 3") + maint._sync_profiles_after_update = profile_sync from hermes_cli import update_receipt from hermes_cli.venv_sync import arm_completion, completion_pending_path arm_completion(source) @@ -73,25 +111,37 @@ def _child_env(**values: str) -> dict[str, str]: # The completion process never inherits the parent's open receipt context. update_receipt._current.reset(update_receipt._current.get().current_token) request = {"schema": 1, "source": str(source), "home": str(home), "branch": "main", "desktop": False, - "assume_yes": True, "gateway_mode": False, "no_gateway_restart": mode == "config-ro", + "assume_yes": True, "gateway_mode": mode.startswith("ipc-"), + "no_gateway_restart": mode == "config-ro", "pre_update_version": None, "snapshot_id": None, "sibling_snapshots": {}, "plan": None, - "receipt": receipt, "windows_resume": None} + "receipt": receipt, "windows_resume": windows_resume} if mode == "config-ro": home.chmod(0o555) # a late filesystem failure: the profile home refuses the config write - from hermes_cli.update_completion import _finish + if mode.startswith("receipts-blocked"): + # The receipt store fails after the commit point: its directory is now an ordinary file. + import shutil + shutil.rmtree(update_receipt._receipt_dir()) + update_receipt._receipt_dir().write_text("not a directory") + from hermes_cli.update_completion import _finish, _settle_after_commit try: - code = _finish(request, cell / "result.json") + if mode == "receipts-blocked-settle": + code = _settle_after_commit(request, cell / "result.json", "dependencies", "dependency sync refused") + else: + code = _finish(request, cell / "result.json") finally: home.chmod(0o755) latest = update_receipt.read_latest_receipt() or {} + result = json.loads((cell / "result.json").read_text()) if (cell / "result.json").exists() else None (cell / "summary.json").write_text(json.dumps({ - "code": code, "outcome": latest.get("outcome"), + "code": code, "outcome": latest.get("outcome"), "update_id": receipt["update_id"], "result": result, "followups": [row["step"] for row in latest.get("followups", [])], "stages": {mark["name"]: mark["outcome"] for mark in latest.get("stages", [])}, "config_version": yaml.safe_load((home / "config.yaml").read_text())["_config_version"], "latest_config_version": DEFAULT_CONFIG["_config_version"], "pending": completion_pending_path(source).is_file(), "stamp": (source / "install-stamp.json").is_file(), + "marker_at_verify": (cell / "marker-at-verify").read_text() if (cell / "marker-at-verify").exists() else None, + "marker_final": marker.read_text() if marker.exists() else None, "migration_entered": entered.is_file()})) ''') @@ -105,7 +155,9 @@ def _run_completion(tmp_path: Path, mode: str, *, sqlite: str | None = None) -> (root_home / name).mkdir(parents=True, exist_ok=True) env = _child_env(HOME=str(cell / "home"), HERMES_HOME=str(profile), HERMES_DISABLE_LAZY_INSTALLS="1", HERMES_RUNTIME_DIR=str(tmp_path / "store"), PYTHONDONTWRITEBYTECODE="1", - XDG_CONFIG_HOME=str(cell / "home/.config"), XDG_CACHE_HOME=str(cell / "home/.cache")) + XDG_CONFIG_HOME=str(cell / "home/.config"), XDG_CACHE_HOME=str(cell / "home/.cache"), + XDG_STATE_HOME=str(cell / "home/.local/state"), + HERMES_GATEWAY_LOCK_DIR=str(cell / "home/.local/state/hermes/gateway-locks")) if sqlite is not None: payload = json.dumps({"base_prefix": sys.base_prefix, "executable": sys.executable, "python_version": list(sys.version_info[:3]), @@ -158,6 +210,52 @@ def test_config_format_write_failure_is_owed_not_reported_complete(tmp_path): assert summary["stages"].get("build") == "success", summary["stages"] +@pytest.mark.platforms("posix") +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores directory modes") +def test_profile_sync_followup_keeps_the_tail_owed(tmp_path): + summary = _run_completion(tmp_path, "profile-sync") + assert summary["code"] == 0, summary["output"] + assert summary["outcome"] == "success" + assert summary["followups"] == ["profile_sync"] + # A profile sync that did not finish is tail work: retried by the next launch, never stamped done. + assert summary["pending"] is True + assert summary["stamp"] is False + + +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("path", ["finish", "settle"]) +def test_receipt_store_failure_after_commit_keeps_exit_zero_and_a_correlated_receipt(tmp_path, path): + """The receipt store is written after the commit point: losing it is loud, never exit 1.""" + summary = _run_completion(tmp_path, f"receipts-blocked-{path}") + assert "Update receipt not written" in summary["output"] + assert summary["code"] == 0, summary["output"] + result = summary["result"] + assert result["exit_code"] == 0 + # run_completion's correlation gate: this run's id, terminal, and its outcome agrees with exit 0. + receipt = result["receipt"] + assert receipt is not None, summary["output"] + assert receipt["update_id"] == summary["update_id"] + assert receipt["finished_at"] + assert receipt["outcome"] == "success" + # Retry obligations are unchanged: owed dependencies stay armed, a finished tail is cleared. + if path == "settle": + assert [row["step"] for row in receipt["followups"]] == ["dependencies"] + assert summary["pending"] is (path == "settle") + + +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("branch", ["failed-unit", "abort", "windows"]) +def test_restart_failure_is_owed_and_never_reported_as_a_failed_update(tmp_path, branch): + """The gateway /update reader hears the committed success; the restart is a follow-up.""" + summary = _run_completion(tmp_path, f"ipc-{branch}") + assert summary["code"] == 0, summary["output"] + assert summary["outcome"] == "success" + assert "gateway_restart" in summary["followups"], summary["output"] + # What a reader sees between the failed restart and the run's finalize, and at the end. + assert summary["marker_at_verify"] == "0", summary["output"] + assert summary["marker_final"] == "0" + + _RECEIPT_CHILD = textwrap.dedent(''' import json, sys root_checkout, action = sys.argv[1:3] From 4e2a0e728ab37ef7a2595de4423da8d8f4961673 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:26:16 -0700 Subject: [PATCH 19/62] docs(update): receipt-store and gateway-marker rules after the commit point --- .../docs/developer-guide/source-update-completion.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/website/docs/developer-guide/source-update-completion.md b/website/docs/developer-guide/source-update-completion.md index 0e0e47aa83a4..6c860b1c5cb1 100644 --- a/website/docs/developer-guide/source-update-completion.md +++ b/website/docs/developer-guide/source-update-completion.md @@ -46,7 +46,7 @@ TUI/web/desktop is attempted even when another failed), profile sync, config mig bytecode sweep, gateway restart/verification, Windows resume or retired-channel adoption prints a `⚠` line and is appended to the receipt's `followups` as `{step, reason}` while the receipt's `outcome` stays `"success"`. The step's own obligation stays armed: -`source-completion-pending` for the tail (launchers, build, maintenance, config migration), +`source-completion-pending` for the tail (launchers, build, maintenance, profile sync, config migration), the host fleet-restart obligation for gateways (the CLI startup warning keeps naming it; an owed restart records the pre-update gateways on the obligation, so a gateway that died at boot stays owed until it serves the checkout instead of being settled by the gateway-less discharge), @@ -71,6 +71,14 @@ dies without a result is a `completion` follow-up. A Ctrl-C after the commit poi as `interrupted` (exit 130, never `failed`) and says the new code is in place with its remaining steps owed. +Two more post-commit channels follow the same rule. The gateway `/update` marker +(`.update_exit_code`) keeps the committed result when a gateway restart fails (systemd unit, +abort recovery or Windows service resume): the restart debt is the `gateway_restart` follow-up, +the host obligation and the `⚠` lines in the forwarded output, never a "failed" notice. A receipt +store that refuses the terminal write prints `⚠ Update receipt not written` and the completion +child answers its parent with the correlated terminal record it finalized in memory, so the exit +status stays 0; only a user action (local changes left in the stash) still exits 1. + ## Parent lifecycle and failures The parent waits and propagates the child's exact nonzero result (a signal is From 6ce051eaea49763e9c2f5c21b9529845bc40f731 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:39:11 -0700 Subject: [PATCH 20/62] fix(update): a refused Windows resume after the commit point never fails the update (M4) - After the completion child answers (or the A6 branch resumes), the post-commit resume attempt is settled: its failure is already the windows_resume follow-up, so the parent's atexit net is unregistered and the command's finally does not replay it. A replay waited again and, on the ZIP route, its RuntimeError reached main.py's BaseException handler: exit 1 and "/update failed". - The ZIP and CalledProcessError finally blocks resume only what is still owed and report a failure as the windows_resume follow-up instead of raising. - update_finish (historical takeover) records a refused resume as a follow-up and keeps its exit status (was code = 1), and reports the attempt as handled so the historical parent does not replay it. Tests (real processes, refusing SCM resume is the only service seam): test_zip_update_never_replays_a_refused_resume_after_the_commit_point, test_refused_gateway_resume_after_commit_is_a_followup_not_exit_1. --- hermes_cli/update_cmd.py | 43 ++++++++- hermes_cli/update_finish.py | 13 ++- tests/hermes_cli/test_update_finish.py | 32 +++++++ .../test_update_post_commit_safety.py | 91 +++++++++++++++++++ 4 files changed, 171 insertions(+), 8 deletions(-) diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index dd292ba496b8..52c03ec25fe9 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -721,6 +721,41 @@ def _source_completion_request(opts, plan, snapshot_id, windows_resume, desktop, } +def _settle_windows_resume(request: dict) -> None: + """This run's post-commit resume attempt is over (the completion child's, or the A6 branch's). + + A failure is already the ``windows_resume`` follow-up, so neither the command's ``finally`` + nor the atexit net may replay it: a replay waits again and its error would fail a committed + update (C3). + """ + import atexit + + from hermes_cli import update_cmd_windows + + atexit.unregister(_m()._resume_windows_gateways_after_update) + atexit.unregister(update_cmd_windows._resume_windows_gateways_after_update) + request["windows_resume_settled"] = True + + +def _resume_paused_gateways_at_exit(token: dict | None, request: dict | None) -> None: + """The command's last resume of gateways it paused; a failure is reported, never raised. + + Raising here would replace the run's own exit (a committed update's 0, or the original + failure) with the restart's error. Skipped when the post-commit attempt already ran. + """ + if not token or not token.get("resume_needed") or (request or {}).get("windows_resume_settled"): + return + try: + _m()._resume_windows_gateways_after_update(token) + except Exception as exc: # noqa: BLE001 — a restart failure is owed, not the update's status + from hermes_cli.update_receipt import _current, amend_terminal_followup, record_followup + + reason = f"Windows gateway recovery failed: {exc}" + record_followup("windows_resume", reason) + if _current.get() is None and request: # the run already finalized its receipt + amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", reason) + + def _complete_source_update(request: dict | None) -> None: # Never "Update complete!" while this run's local patches sit unrestored in the stash (#122557). unrestored = _unrestored_autostash_notice() @@ -751,6 +786,7 @@ def _complete_source_update(request: dict | None) -> None: resumed = dict(result["windows_resume"]) token.clear() token.update(resumed) + _settle_windows_resume(request) elif token and token.get("resume_needed") and not result["exit_code"]: # Dependencies owed (A6): the bootstrap child cannot resume paused gateways; this process can. try: @@ -761,6 +797,7 @@ def _complete_source_update(request: dict | None) -> None: reason = f"Windows gateway recovery failed: {exc}" record_followup("windows_resume", reason) amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", reason) + _settle_windows_resume(request) if result.get("receipt") is not None: current = _completion_receipt._current.get() if current is not None: @@ -1535,8 +1572,7 @@ def _cmd_update_impl(args, gateway_mode: bool): target_sha=release_sha, completion_request=completion_request, **({"target_repository": target_repository} if target_repository else {})) finally: - if _windows_gateway_resume and _windows_gateway_resume.get("resume_needed"): - _m()._resume_windows_gateways_after_update(_windows_gateway_resume) + _resume_paused_gateways_at_exit(_windows_gateway_resume, completion_request) return @@ -1630,5 +1666,4 @@ def _cmd_update_impl(args, gateway_mode: bool): e, args, gateway_mode, had_desktop_app_before_update, target_sha=release_sha, target_repository=target_repository, completion_request=completion_request) finally: - if _windows_gateway_resume and _windows_gateway_resume.get("resume_needed"): - _m()._resume_windows_gateways_after_update(_windows_gateway_resume) + _resume_paused_gateways_at_exit(_windows_gateway_resume, completion_request) diff --git a/hermes_cli/update_finish.py b/hermes_cli/update_finish.py index 96527643b768..6ccc5df1bb6b 100644 --- a/hermes_cli/update_finish.py +++ b/hermes_cli/update_finish.py @@ -168,16 +168,21 @@ def main(context: Path, result: Path) -> int: # receipt. Preserve the original handoff, just like preparation. update_receipt.begin_update_receipt(previous=request.get("receipt"), correlation_id=request["update_id"]) begun = update_receipt._current.get() is not None + attempted = False if resume is not None: try: resume(token) - except Exception as exc: - code = 1 - print(f"Gateway recovery failed: {exc}", file=sys.stderr, flush=True) + except Exception as exc: # noqa: BLE001 — a restart failure is owed, never the exit status (C3) + attempted = True + reason = f"Windows gateway recovery failed: {exc}" + update_receipt.record_followup("windows_resume", reason) + if update_receipt._current.get() is None: # the run already finalized its receipt + update_receipt.amend_terminal_followup(request["update_id"], "windows_resume", reason) handled = cli_started or (begun and update_receipt._current.get() is None) written = update_receipt.finalize_pending_update_receipt(code, "historical takeover completion") result.write_text(json.dumps({ - "resume_handled": not token or not token.get("resume_needed"), + # An attempt that failed is reported here: the historical parent must not replay it. + "resume_handled": attempted or not token or not token.get("resume_needed"), "receipt_handled": handled or written is not None, }), encoding="utf-8") return code diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index 781a71139c8c..c244330e63e5 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -219,6 +219,14 @@ def resume(outcome, token, gateway_mode): (root / 'resumed-token.json').write_text(json.dumps(token)) return merge(outcome, token, gateway_mode) update._resume_windows_gateways_and_merge_outcome = resume + if fault == 'scm': + # The Windows service manager refuses the paused gateway's restart. + from hermes_cli import update_cmd_windows + def scm(token): + if token and token.get('resume_needed'): + raise RuntimeError('Could not restart Windows gateway service(s): HermesGatewayProbe') + module._resume_windows_gateways_after_update = scm + update_cmd_windows._resume_windows_gateways_after_update = scm spec.loader.exec_module = execute return spec sys.meta_path.insert(0, CompletionImports()) @@ -270,6 +278,30 @@ def test_failure_preserves_original_receipt_before_build(completion, fault): assert not (source / "build-environment.json").exists() +@pytest.mark.platforms("posix") +def test_refused_gateway_resume_after_commit_is_a_followup_not_exit_1(completion): + """C3 for the historical takeover child: the code is committed, so a Windows gateway resume + that the service manager refuses is an owed ``windows_resume`` follow-up, never exit 1.""" + source, home, request, context, result, run = completion + request["gateway_mode"] = True + # A real start time names the run's archive file, so the running and terminal records share one + # file (the fixture's placeholder falls back to the clock and can split them across a second). + request["receipt"]["started_at"] = "2026-10-04T12:00:00+00:00" + request["windows_resume"] = {"resume_needed": True, "profiles": {}, "unmapped": [], + "services": ["HermesGatewayProbe"], + "service_profiles": {"HermesGatewayProbe": "default"}} + context.write_text(json.dumps(request), encoding="utf-8") + child = run("scm") + assert child.returncode == 0, child.stdout + child.stderr + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text()) + assert receipt["update_id"] == request["update_id"] + assert receipt["outcome"] == "success" + assert "windows_resume" in [row["step"] for row in receipt["followups"]], child.stdout + child.stderr + assert (home / ".update_exit_code").read_text().strip() == "0" + # The attempt is reported: the historical parent must not replay it at its own exit. + assert json.loads(result.read_text())["resume_handled"] is True + + @pytest.mark.platforms("posix") @pytest.mark.parametrize("captured, installed, expected", [ (None, None, False), diff --git a/tests/hermes_cli/test_update_post_commit_safety.py b/tests/hermes_cli/test_update_post_commit_safety.py index 66cd90d0ea5c..7b7c4d92193f 100644 --- a/tests/hermes_cli/test_update_post_commit_safety.py +++ b/tests/hermes_cli/test_update_post_commit_safety.py @@ -256,6 +256,97 @@ def test_restart_failure_is_owed_and_never_reported_as_a_failed_update(tmp_path, assert summary["marker_final"] == "0" +# The NEW tree's completion child after a committed update whose Windows resume the service +# manager refused (what _finish answers): the run is a success with a windows_resume follow-up, +# and the token it hands back still owes the restart. +_REFUSED_RESUME_COMPLETION = textwrap.dedent(''' + import json, os, sys + from pathlib import Path + request = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8-sig")) + receipts = Path(os.environ["PROBE_RECEIPTS"]) + receipt = dict(request["receipt"], outcome="success", exit_code=0, finished_at="now", followups=[ + {"step": "windows_resume", "reason": "Windows gateway recovery failed: refused"}]) + payload = json.dumps(receipt) + (receipts / ("update_probe_" + receipt["update_id"] + ".json")).write_text(payload) + (receipts / "latest.json").write_text(payload) + (Path(request["home"]) / ".update_exit_code").write_text("0") + Path(sys.argv[2]).write_text(json.dumps({ + "schema": 1, "update_id": receipt["update_id"], "exit_code": 0, "receipt": receipt, + "windows_resume": dict(request["windows_resume"], resume_needed=True)})) +''') + +_ZIP_PARENT = textwrap.dedent(''' + import json, os, sys + from pathlib import Path + from types import SimpleNamespace + root_checkout, cell = sys.argv[1:3] + sys.path.insert(0, root_checkout) + cell = Path(cell) + calls = cell / "parent-resume-calls.jsonl" + from hermes_cli import main as hermes_main, update_cmd, update_inventory, update_receipt + hermes_main.PROJECT_ROOT = cell / "source" + receipts = update_receipt._receipt_dir() + receipts.mkdir(parents=True, exist_ok=True) + os.environ["PROBE_RECEIPTS"] = str(receipts) + # Machine boundaries only: the host's process inventory, backups, terminal/hangup handling, + # the Windows service manager, and the ZIP download + swap (a no-.git install). + def no_inventory(): + raise RuntimeError("the host's runtime inventory is not part of this probe") + update_inventory.collect_runtime_inventory = no_inventory + hermes_main._update_preflight_handled = lambda args: False + hermes_main._install_hangup_protection = lambda **kwargs: None + hermes_main._finalize_update_output = lambda state: None + hermes_main._run_pre_update_backup = lambda args: None + token = {"resume_needed": True, "profiles": {}, "unmapped": [], "services": ["HermesGatewayProbe"], + "service_profiles": {"HermesGatewayProbe": "probe"}} + hermes_main._pause_windows_gateways_for_update = lambda: token + def scm(resume_token): + with calls.open("a") as handle: + handle.write(json.dumps(bool(resume_token and resume_token.get("resume_needed"))) + "\\n") + if resume_token and resume_token.get("resume_needed"): + raise RuntimeError("Could not restart Windows gateway service(s): HermesGatewayProbe") + hermes_main._resume_windows_gateways_after_update = scm + update_cmd._prepare_git_command = lambda: (True, None, False) + def via_zip(args, *, had_desktop_app_before_update, target_sha=None, completion_request=None, **kwargs): + completion_request["expected_sha"] = target_sha + completion_request["apply_mode"] = "zip" + update_cmd._complete_source_update(completion_request) + update_cmd._update_via_zip = via_zip + class Args(SimpleNamespace): + def __getattr__(self, name): + return None + hermes_main.cmd_update(Args(gateway=True, branch="main", yes=True)) +''') + + +@pytest.mark.platforms("posix") +def test_zip_update_never_replays_a_refused_resume_after_the_commit_point(tmp_path): + """The child already tried the resume and recorded the follow-up: the parent's finally and its + atexit net must not run it again, and a restart error must not become exit 1 / "/update failed".""" + cell = tmp_path / "cell" + home = cell / "home" / ".hermes" + (home / "logs/update_receipts").mkdir(parents=True) + completion = cell / "source" / "hermes_cli" / "update_completion.py" + completion.parent.mkdir(parents=True) + completion.write_text(_REFUSED_RESUME_COMPLETION, encoding="utf-8") + env = _child_env(HOME=str(cell / "home"), HERMES_HOME=str(home), HERMES_DISABLE_LAZY_INSTALLS="1", + HERMES_RUNTIME_DIR=str(tmp_path / "store"), PYTHONDONTWRITEBYTECODE="1", + XDG_CONFIG_HOME=str(cell / "home/.config"), XDG_CACHE_HOME=str(cell / "home/.cache"), + XDG_STATE_HOME=str(cell / "home/.local/state")) + driver = tmp_path / "zip_parent.py" + driver.write_text(_ZIP_PARENT, encoding="utf-8") + done = subprocess.run([sys.executable, "-B", str(driver), str(ROOT), str(cell)], + env=env, cwd=tmp_path, capture_output=True, text=True, timeout=240) + output = done.stdout + done.stderr + assert done.returncode == 0, output + assert (home / ".update_exit_code").read_text().strip() == "0", output + calls = cell / "parent-resume-calls.jsonl" + assert not calls.exists(), calls.read_text() + output + latest = json.loads((home / "logs/update_receipts/latest.json").read_text(encoding="utf-8-sig")) + assert latest["outcome"] == "success" + assert [row["step"] for row in latest["followups"]] == ["windows_resume"] + + _RECEIPT_CHILD = textwrap.dedent(''' import json, sys root_checkout, action = sys.argv[1:3] From 1e08ef1fe676f76e8d155b21a87014f3d0772def Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:39:11 -0700 Subject: [PATCH 21/62] docs(update): profile sync is owed only when the step escapes; one post-commit Windows resume (m6, M4) --- .../developer-guide/source-update-completion.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/website/docs/developer-guide/source-update-completion.md b/website/docs/developer-guide/source-update-completion.md index 6c860b1c5cb1..fd0a75b22ce7 100644 --- a/website/docs/developer-guide/source-update-completion.md +++ b/website/docs/developer-guide/source-update-completion.md @@ -42,11 +42,11 @@ Verification publishes the final receipt. Once the tree has moved, `hermes update` exits 0 unless the code itself was rolled back. Every completion step is independent: a failed launcher publish, product build (each of -TUI/web/desktop is attempted even when another failed), profile sync, config migration, +TUI/web/desktop is attempted even when another failed), config migration, bytecode sweep, gateway restart/verification, Windows resume or retired-channel adoption prints a `⚠` line and is appended to the receipt's `followups` as `{step, reason}` while the receipt's `outcome` stays `"success"`. The step's own obligation stays armed: -`source-completion-pending` for the tail (launchers, build, maintenance, profile sync, config migration), +`source-completion-pending` for the tail (launchers, build, maintenance, config migration), the host fleet-restart obligation for gateways (the CLI startup warning keeps naming it; an owed restart records the pre-update gateways on the obligation, so a gateway that died at boot stays owed until it serves the checkout instead of being settled by the gateway-less discharge), @@ -56,6 +56,16 @@ stamped "restarted" for this commit whose fleet is still off the checkout restar instead of dead-ending. Exit 2 (refused / concurrent) and exit 1 (nothing committed, or rolled back) keep their meaning. +Profile sync is best-effort: `_sync_profiles_after_update` prints a per-profile error and +carries on, so that error is not owed. Only a sync that escapes the step (for example with +`SystemExit`) becomes a `profile_sync` follow-up and keeps `source-completion-pending` armed. + +A Windows gateway resume is attempted once after the commit point: by the completion child, or +by the parent when dependencies are owed. Its failure is the `windows_resume` follow-up. The +command's own exit path and its atexit net do not run it again, and a resume they still owe +(the child never answered) is reported the same way instead of raising. The historical takeover +completion (`update_finish`) also records the failure as a follow-up and keeps its exit status. + The receipt is durable while the run is open: `begin_update_receipt` writes it as `outcome: "running"` to the run's own archive file and `latest.json`, and each stage boundary refreshes it, so a killed update leaves its own record. The next update marks a From 3d4e3bfdb070a6f5324c3964516708713543d297 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:04:26 -0700 Subject: [PATCH 22/62] fix(update): a stopped unmapped gateway keeps its restart owed across startups (review F7-unmapped-stop-debt-lost) What: when the restart phase stops a gateway it cannot map to a profile, the owed obligation now records it as an unmapped inventory row; _marker_owed_gateways turns that row into a sentinel that only a live, current gateway of this checkout settles. Why: the row-less obligation looked gateway-less at the next startup (an unmapped process leaves no gateway_state.json), so the gateway-less discharge erased the debt while the gateway stayed down. Red: pre-fix fleet/fleet_verify -> test_unmapped_stop_debt_survives_startup_until_a_current_gateway_runs fails (_update_owes_fleet_restart False, marker cleared). Green: same test passes; reviewer probe unmapped_debt2 (with _pytest_owns_live_checkout stub) exits 0, inventory carries the unmapped row. --- hermes_cli/update_cmd_fleet.py | 12 ++++++- hermes_cli/update_cmd_fleet_verify.py | 11 +++--- .../test_update_fleet_check_fail_closed.py | 35 +++++++++++++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index 3e297e48c632..3d5cd4d1dc84 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -325,6 +325,11 @@ def _live_fleet_covers_receipt(expected_sha: str | None, receipt: dict, owed: se return False +# A gateway the restart phase stopped without knowing its profile: no live row can be matched to it, +# so only a live, current gateway of this checkout settles it, never an empty host (""=no profile). +_UNMAPPED_GATEWAY = ("gateway", "") + + def _marker_owed_gateways(inventory: object) -> set[tuple[str, str]] | None: """The ``("gateway", profile)`` set a marker's inventory owes; None when it recorded none. @@ -345,6 +350,9 @@ def _marker_owed_gateways(inventory: object) -> set[tuple[str, str]] | None: raise ValueError("fleet-restart inventory row is not an object") if runtime_outside_gateway_evidence(runtime): continue + if runtime.get("kind") == "gateway" and runtime.get("stopped_unmapped"): + owed.add(_UNMAPPED_GATEWAY) + continue profile = runtime.get("profile") if runtime.get("kind") != "gateway" or not isinstance(profile, str) or not profile.strip() or profile == "unknown": raise ValueError("fleet-restart inventory row is not an identified gateway") @@ -447,8 +455,10 @@ def _marker_only_restart_obsolete() -> bool: continue if row.get("state") != "current" or str(row.get("code_sha")) != target_sha: return False # stale / down / unknown-identity row still owes the restart - if owed is not None and not owed <= covered: + if owed is not None and not owed - {_UNMAPPED_GATEWAY} <= covered: return False # A gateway this marker owns is absent (down) or unidentifiable. + if owed and _UNMAPPED_GATEWAY in owed and all(row_is_external(row) for row in fleet): + return False # another checkout's gateway is no successor for the one the update stopped _clear_fleet_restart_pending_marker() logger.debug( "Fleet-restart-pending marker discharged: %d gateway(s) already serve %s", diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py index 4197ad1b32b5..9275e0f06605 100644 --- a/hermes_cli/update_cmd_fleet_verify.py +++ b/hermes_cli/update_cmd_fleet_verify.py @@ -128,17 +128,17 @@ def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict: return live -def _record_owed_gateway_inventory(plan) -> None: +def _record_owed_gateway_inventory(plan, stopped_unmapped_pids=()) -> None: """Name the pre-update gateways on an inventory-less obligation once their restart is owed. The pull arms the obligation before the restart phase knows what it owes. An inventory-less record on a host whose gateway then died at boot is settled by the gateway-less discharge (nothing live, nothing named), which would silence the owed-restart warning that replaces a failed update under contract C3. A named inventory keeps it until those gateways run HEAD. + A gateway stopped with no known profile is recorded as an unmapped row for the same reason: it + has no ``gateway_state.json`` the gateway-less probe could see, so absence would settle it. """ from hermes_cli import update_cmd_fleet as fleet - if plan is None: - return armed = fleet._fleet_restart_obligation_armed() fields = fleet._obligation_fields() if armed else {} if fields is None or fields.get("inventory") not in (None, "", "null"): @@ -149,7 +149,8 @@ def _record_owed_gateway_inventory(plan) -> None: if getattr(runtime, "kind", None) == "gateway" and isinstance(getattr(runtime, "profile", None), str) and runtime.profile.strip() and runtime.profile != "unknown" - ] + ] + [{"kind": "gateway", "profile": None, "pid": pid, "stopped_unmapped": True} + for pid in sorted(stopped_unmapped_pids)] if rows: # Not armed any more (a pre-restart probe settled an inventory-less record): re-arm, the # restart is owed. The SHA is the code the fleet must be proven current on. @@ -324,7 +325,7 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re + "; recover with `hermes gateway restart`", ) with _best_effort('Fleet restart inventory not recorded: %s'): - _record_owed_gateway_inventory(_pre_update_plan) + _record_owed_gateway_inventory(_pre_update_plan, stopped) with _best_effort('Update receipt finalize failed: %s'): from hermes_cli.update_receipt import finalize_update_receipt # A False ``update_complete`` (unsafe SQLite runtime, owed tail work) never fails the diff --git a/tests/hermes_cli/test_update_fleet_check_fail_closed.py b/tests/hermes_cli/test_update_fleet_check_fail_closed.py index cfdd26d8b6c9..8b10342837fd 100644 --- a/tests/hermes_cli/test_update_fleet_check_fail_closed.py +++ b/tests/hermes_cli/test_update_fleet_check_fail_closed.py @@ -138,3 +138,38 @@ def test_unmapped_stop_keeps_the_restart_owed(monkeypatch, tmp_path): assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] assert "101" in receipt["followups"][0]["reason"] assert cleared == [] + + +def test_unmapped_stop_debt_survives_startup_until_a_current_gateway_runs(monkeypatch, tmp_path): + # An unmapped gateway leaves no gateway_state.json, so on the next start the empty host looked + # gateway-less and the inventory-less obligation was discharged: the stopped gateway's restart + # debt vanished. Only a live gateway serving the checkout may settle it. + from hermes_cli import update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as fleet_verify + from hermes_cli import update_cmd_fleet_gatewayless as gatewayless, update_host_obligation as host, update_receipt + + monkeypatch.setattr(host, "host_obligation_path", lambda: tmp_path / "host-update-restart.json") + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "head") + monkeypatch.setattr(update_cmd, "_current_checkout_sha", lambda: "head") + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) + monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) + monkeypatch.setattr(gatewayless, "host_owes_no_gateway_restart", lambda: True) + fleet_rows: list = [] + monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: list(fleet_rows)) + fleet._write_fleet_restart_pending_marker(expected_sha="head") # armed by the pull, inventory-less + out = fleet._GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[101], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], + killed_pids={101}, stopped_unmapped_pids={101}, + ) + update_receipt.begin_update_receipt() + fleet_verify._verify_fleet_after_update(out, _pre_update_plan=_plan([]), _windows_gateway_resume=None, + update_complete=True) + + assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is True + assert fleet._pending_fleet_restart_needed(receipt={}, pending_manual=[]) is True # already-current update + assert fleet._fleet_restart_obligation_armed() + + fleet_rows.append({"profile": "default", "state": "current", "code_sha": "head"}) # `hermes gateway run` + assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is False + assert not fleet._fleet_restart_obligation_armed() From 967add639d43f0f9a8d1d91287437f043cc050ec Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:05:39 -0700 Subject: [PATCH 23/62] fix(update): managed SSH observer reads a profile update's receipt from the root home (review F2-ssh-receipt-root) What: OBSERVATION_SCRIPT's receipt() scans /logs/update_receipts (install_root was already derived for the marker); status/ready/intent stay on the launch home. Why: the CLI writes receipts under get_default_hermes_root(), so a named-profile managed SSH update never found its correlated receipt and the Desktop reported a committed update as failed. Red: pre-fix managed-ssh-update.ts -> new vitest case fails (reads the stale profile-path receipt). Green: managed-ssh-update.test.ts 36/36 passed. --- .../electron/managed-ssh-update.test.ts | 47 +++++++++++++++++++ apps/desktop/electron/managed-ssh-update.ts | 9 ++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/apps/desktop/electron/managed-ssh-update.test.ts b/apps/desktop/electron/managed-ssh-update.test.ts index adef9261f098..30d8a89c650e 100644 --- a/apps/desktop/electron/managed-ssh-update.test.ts +++ b/apps/desktop/electron/managed-ssh-update.test.ts @@ -538,6 +538,53 @@ test.runIf(process.platform !== 'win32')( } ) +test.runIf(process.platform !== 'win32')( + 'managed observer reads a named profile update receipt from the root home the CLI writes', + async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-profile-receipt-')) + const profileHome = path.join(root, 'profiles', 'research') + + const receipt = (outcome: string) => + JSON.stringify({ + correlation_id: CORRELATION, + outcome, + started_at: '2026-08-23T00:00:00Z', + finished_at: '2026-08-23T00:01:00Z' + }) + + try { + // The CLI's receipt store is install-wide; a stale copy under the profile is not its source. + await mkdir(path.join(root, 'logs', 'update_receipts'), { recursive: true }) + await mkdir(path.join(profileHome, 'logs', 'update_receipts'), { recursive: true }) + await writeFile(path.join(profileHome, `.update_exit_code.${CORRELATION}`), '0') + await writeFile(path.join(root, 'logs', 'update_receipts', `update_${CORRELATION}.json`), receipt('success')) + await writeFile( + path.join(profileHome, 'logs', 'update_receipts', `update_${CORRELATION}.json`), + receipt('failed') + ) + + const command = buildRemoteUpdateObservationCommand( + { + ssh: { exec: async () => '' }, + platform: 'Linux', + hermesPath: '/opt/hermes/hermes', + hermesHome: profileHome + }, + CORRELATION + ) + + const { stdout } = await exec(command, { shell: 'sh' }) + const parsed = parseRemoteUpdateObservation(stdout, CORRELATION) + + assert.equal(parsed.exitCode, 0) + assert.equal(parsed.receipt?.correlationId, CORRELATION) + assert.equal(parsed.receipt?.outcome, 'success') + } finally { + await rm(root, { force: true, recursive: true }) + } + } +) + test('Windows coordinator handoff is pending until its marker clears and correlated receipt is durable', async () => { const replies = [ observation({ marker: 'live', markerPid: 44 }), diff --git a/apps/desktop/electron/managed-ssh-update.ts b/apps/desktop/electron/managed-ssh-update.ts index c87796d11d09..c9712f5165e8 100644 --- a/apps/desktop/electron/managed-ssh-update.ts +++ b/apps/desktop/electron/managed-ssh-update.ts @@ -324,9 +324,10 @@ from pathlib import Path home=Path(os.path.expanduser(sys.argv[1])) correlation=sys.argv[2] -# The update marker is install-wide even when the launcher was invoked with a -# named profile home (/profiles/). Correlated status/intent/receipt -# remain under the launch home, matching the CLI writer. +# The update marker and the receipt store are install-wide even when the launcher +# was invoked with a named profile home (/profiles/): the CLI writes +# receipts under the root home. Correlated status/ready/intent stay under the +# launch home, matching their writers. profile_parent=home.parent.name is_profile_home=(profile_parent.lower()=='profiles') if os.name=='nt' else (profile_parent=='profiles') install_root=home.parent.parent if is_profile_home else home @@ -416,7 +417,7 @@ def terminal_code(): except ValueError:return 'malformed' def receipt(): - directory=home/'logs'/'update_receipts' + directory=install_root/'logs'/'update_receipts' try:paths=sorted(directory.glob('update_*.json'),key=lambda p:p.stat().st_mtime_ns,reverse=True) except OSError:return None for path in paths: From 5f4ae82c74171bc4f1f14d645c2c3ac3a7e9a3a0 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:14:16 -0700 Subject: [PATCH 24/62] fix(update): a dashboard accepts only its own action's completion from the shared root update.log (review actions-root-marker-uncorrelated) What: _spawn_hermes_action writes the action id into the action log's start header; get_action_status accepts a root `hermes-update completed ` marker only when is that header's id, else falls back to the receipt / null. Why: the root update.log is shared by every profile, so profile A's success certified profile B's failed (or never-started) action as exit 0 after B's dashboard lost its in-memory result. Fixture: test_update_status_recovers_completed_result_after_dashboard_restart's action-log header now carries the action id it recovers (its old id-less header is exactly the uncorrelated case). Red: pre-fix actions.py/web_server_gateway.py -> new test fails (foreign completion -> exit_code 0). Green: new test + recovery test pass; matching id still yields exit 0 and action_id. --- hermes_cli/web_routers/actions.py | 14 +++++++++++++ hermes_cli/web_server_gateway.py | 7 +++++-- tests/hermes_cli/test_web_server.py | 31 ++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/hermes_cli/web_routers/actions.py b/hermes_cli/web_routers/actions.py index b128705aabaf..30dea8f558b3 100644 --- a/hermes_cli/web_routers/actions.py +++ b/hermes_cli/web_routers/actions.py @@ -53,6 +53,7 @@ def _server_path(name: str) -> Path: _ACTION_LOG_TAIL_MAX_CHUNK_BYTES = 64 * 1024 _UPDATE_ACTION_COMPLETED_RE = re.compile(r"^=== hermes-update completed ([0-9a-f]{32}) ===$") +_UPDATE_ACTION_STARTED_RE = re.compile(r"^=== hermes-update started .* ([0-9a-f]{32}) ===$") _MANAGED_EXTERNALLY_MESSAGE = "Hermes updates are managed outside this dashboard in containerized environments." @@ -137,6 +138,15 @@ def _durable_completed_update_action_id(lines: List[str]) -> Optional[str]: return completed_action_id if completed_action_id and last_completed > last_start else None +def _latest_spawned_update_action_id(lines: List[str]) -> Optional[str]: + """Action id named by the latest ``hermes-update started`` header of THIS dashboard's log.""" + for line in reversed(lines): + if line.startswith("=== hermes-update started "): + match = _UPDATE_ACTION_STARTED_RE.fullmatch(line.strip()) + return match.group(1) if match else None + return None + + @router.post("/api/gateway/restart") async def restart_gateway(profile: Optional[str] = None): """Kick off a ``hermes gateway restart`` in the background.""" @@ -363,6 +373,10 @@ async def get_action_status(name: str, lines: int = 200): durable_update_action_id = _durable_completed_update_action_id( _tail_lines(get_default_hermes_root() / "logs" / "update.log", 2000)) + if durable_update_action_id != _latest_spawned_update_action_id(_tail_lines(log_dir / log_file_name, 2000)): + # The root log is shared by every profile: another profile's (or an older) run's + # completion never certifies the action this dashboard started. + durable_update_action_id = None if durable_update_action_id: marker = f"=== hermes-update completed {durable_update_action_id} ===" if marker not in tail: diff --git a/hermes_cli/web_server_gateway.py b/hermes_cli/web_server_gateway.py index 232d04524647..9f4b5bb0c0ad 100644 --- a/hermes_cli/web_server_gateway.py +++ b/hermes_cli/web_server_gateway.py @@ -457,7 +457,11 @@ def _spawn_hermes_action( from hermes_cli.web_server import PROJECT_ROOT _ACTION_LOG_DIR.mkdir(parents=True, exist_ok=True) log_file = open(_ACTION_LOG_DIR / _ACTION_LOG_FILES[name], "ab", buffering=0) - log_file.write(f"\n=== {name} started {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n".encode()) + # The header carries the action id durably: the update's completion marker lands in the ROOT + # update.log every profile shares, and only this id ties one to the action THIS log started. + action_id = (env_overrides or {}).get("HERMES_ACTION_ID") + stamp = time.strftime('%Y-%m-%d %H:%M:%S') + (f" {action_id}" if action_id else "") + log_file.write(f"\n=== {name} started {stamp} ===\n".encode()) from hermes_cli._launchers import runtime_command cmd = runtime_command(PROJECT_ROOT, subcommand) @@ -494,7 +498,6 @@ def _spawn_hermes_action( _ACTION_RESULTS.pop(name, None) _ACTION_COMMANDS[name] = tuple(subcommand) _ACTION_PROCS[name] = proc - action_id = (env_overrides or {}).get("HERMES_ACTION_ID") if action_id: _ACTION_IDS[name] = action_id else: diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 46ed79fd11de..171df54c6d7e 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -8,6 +8,7 @@ import sys import threading import time +import types from pathlib import Path from unittest.mock import patch, MagicMock @@ -1462,7 +1463,7 @@ def test_update_status_recovers_completed_result_after_dashboard_restart(self, m monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "coder")) action_id = "c" * 32 (tmp_path / "hermes-update.log").write_text( - "=== hermes-update started 2026-08-17 11:19:34 ===\n" + f"=== hermes-update started 2026-08-17 11:19:34 {action_id} ===\n" "pulling updates...\n", encoding="utf-8", ) @@ -1487,6 +1488,34 @@ def test_update_status_recovers_completed_result_after_dashboard_restart(self, m assert data["action_id"] == action_id assert f"=== hermes-update completed {action_id} ===" in data["lines"] + def test_update_status_ignores_another_profiles_completion_in_the_shared_root_log(self, monkeypatch, tmp_path): + # Every profile's ``hermes update`` mirrors into the ROOT update.log; profile A's success + # there must not certify the action profile B's dashboard started (and lost on restart). + root = tmp_path / "root" + (root / "logs").mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + b_logs = tmp_path / "b-logs" + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", b_logs) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + monkeypatch.setattr(_web_server_gateway.subprocess, "Popen", lambda *a, **kw: types.SimpleNamespace(pid=7)) + b_id = "b" * 32 + _web_server_gateway._spawn_hermes_action(["update"], "hermes-update", env_overrides={"HERMES_ACTION_ID": b_id}) + _web_server_gateway._ACTION_PROCS.clear() # B's dashboard restarted: in-memory result lost + + def status_after_root_completion(action_id): + (root / "logs" / "update.log").write_text( + "=== hermes update started 2026-08-17T11:19:35 ===\n" + f"=== hermes-update completed {action_id} ===\n", encoding="utf-8") + return self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + foreign = status_after_root_completion("a" * 32) + assert foreign["exit_code"] is None + assert "action_id" not in foreign + own = status_after_root_completion(b_id) + assert own["exit_code"] == 0 + assert own["action_id"] == b_id + def test_update_hermes_spawns_with_action_id(self, monkeypatch): import hermes_cli.web_server as web_server From cc009e9cd36e75f6f886275c1a2018b80b7dc598 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:15:28 -0700 Subject: [PATCH 25/62] docs(update): maintenance docstring says profile sync is best-effort per profile (review F1-profile-sync, declined) The behaviour stays as designed (source-update-completion.md): a per-profile sync error is printed and not owed; only a sync that escapes the step becomes the profile_sync follow-up. The docstring claimed every profile-sync failure was owed, contradicting the code and the developer doc. --- hermes_cli/update_cmd_maint.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py index 3080b3e81acb..2c35430dc796 100644 --- a/hermes_cli/update_cmd_maint.py +++ b/hermes_cli/update_cmd_maint.py @@ -984,9 +984,11 @@ def _run_post_update_maintenance( """Post-build housekeeping and completion, returning the SQLite runtime verdict. Ancillary repairs and notices are best-effort. The profile sync and the config migration run - after a failed build too and never fail the committed update: a failure is printed as ``⚠``, - recorded as a receipt follow-up and appended to ``followups`` so the tail stays owed. An - unsafe runtime withholds success (and is reported) but is not tail work. + after a failed build too and never fail the committed update. A config-migration failure is + printed as ``⚠``, recorded as a receipt follow-up and appended to ``followups`` so the tail + stays owed. Profile sync is best-effort per profile (``_sync_profiles_after_update`` prints + that profile's error and carries on); only a sync that escapes the step is owed. An unsafe + runtime withholds success (and is reported) but is not tail work. """ from hermes_cli.update_receipt import record_followup From cb620d83001de84d50c75ef8b8a3f786eacdc4de Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:53:33 -0700 Subject: [PATCH 26/62] fix(update): an unbuilt Desktop app is reported by one whole line the hand-offs match (review F6-desktop-build-debt-grep) What: build_update_products acquires npm inside the "Node dependencies" attempt and, whenever the desktop product was requested but not built, prints `Desktop app build owed: failed`. posix.sh matches that line instead of the truncated follow-up text; windows.ps1 now projects it as a manual outcome (it had no projection of the current follow-up at all). Why: the 500-char follow-up leads with whichever product failed first, and an npm acquisition failure escaped un-named, so a committed update whose Desktop was not rebuilt said "Update complete." Red: pre-fix source_build.py -> test_unbuilt_desktop_is_named_on_one_whole_line[both] fail; pre-fix posix.sh -> truncated-followup hand-off test reports manual=false. Green: test_source_build.py 15 passed; test_desktop_update_gateway_flag.py 3 passed. windows.ps1 needs a native run. --- hermes_cli/source_build.py | 18 +++++++++--- scripts/desktop-update/posix.sh | 9 +++--- scripts/desktop-update/windows.ps1 | 9 ++++++ tests/hermes_cli/test_source_build.py | 29 +++++++++++++++++++ .../test_desktop_update_gateway_flag.py | 23 +++++++++++++-- 5 files changed, 78 insertions(+), 10 deletions(-) diff --git a/hermes_cli/source_build.py b/hermes_cli/source_build.py index f9208d71bd83..086a1e9c12d6 100644 --- a/hermes_cli/source_build.py +++ b/hermes_cli/source_build.py @@ -138,12 +138,18 @@ def attempt(name: str, step) -> bool: attempt("feature dependencies", lambda: _install_configured_features_missing_deps(project_root)) frontends = source_frontends(project_root) if frontends: - env = source_build_env(explicit=True) + desktop_built = False + env: dict = {} workspaces = frontends + (("apps/desktop",) if desktop else ()) publish_stage("Updating Node dependencies") + + def node_dependencies() -> None: + # Acquiring npm is part of this step: its failure must be reported like the install's. + env.update(source_build_env(explicit=True)) + prepare_source_dependencies(project_root, workspaces, env=env, explicit=True) + # Every product compiles from these node_modules: without them there is nothing to build. - if attempt("Node dependencies", lambda: prepare_source_dependencies( - project_root, workspaces, env=env, explicit=True)): + if attempt("Node dependencies", node_dependencies): # An update that changed no TUI/web input reuses the receipted output, as the # launch path already does; recompiling it produces the same bytes. Desktop # additionally needs the packaged app to name HEAD (its baked stamp carries the @@ -162,7 +168,11 @@ def attempt(name: str, step) -> bool: publish_stage("Building the web UI") attempt("web UI build", lambda: build_source_web(project_root, env=env)) if desktop: - attempt("desktop app build", lambda: _build_desktop_product(project_root, env, publish_stage)) + desktop_built = attempt("desktop app build", lambda: _build_desktop_product(project_root, env, publish_stage)) + if desktop and not desktop_built: + # One whole line the Desktop hand-off scripts match: the follow-up text is truncated + # and names whichever products failed first, so it cannot say if THIS app was rebuilt. + print(f" Desktop app build owed: {failures[-1][0]} failed") # A configured memory provider that no longer ships in core is installed from the # catalog for every profile home sharing this venv (config, data and tool names # unchanged). The update must finish even if the migration blows up. diff --git a/scripts/desktop-update/posix.sh b/scripts/desktop-update/posix.sh index 84e15ea6335b..aad25662d0a7 100755 --- a/scripts/desktop-update/posix.sh +++ b/scripts/desktop-update/posix.sh @@ -890,10 +890,11 @@ fi if [ "$CODE" -eq 0 ]; then FINAL_CODE=0 FINAL_MSG="Update complete." # Contract C3: a Desktop build that fails after the code committed is an owed - # follow-up (hermes update exits 0 and prints it). The user is on the new - # Hermes, but this app was not rebuilt: say so and say how to fix it, never - # "finished OK" and never "still on the previous version". - if printf '%s' "$OUT" | grep -Eq "Update follow-up 'build' did not finish: .*(desktop app build|Node dependencies)"; then + # follow-up (hermes update exits 0 and prints one whole "Desktop app build + # owed:" line for it; the follow-up text itself is truncated). The user is on + # the new Hermes, but this app was not rebuilt: say so and say how to fix it, + # never "finished OK" and never "still on the previous version". + if printf '%s\n' "$OUT" | grep -Eq '^[[:space:]]*Desktop app build owed: '; then APP_REBUILD_FAILED=1 DONE_NOTE="Hermes was updated, but the Desktop app could not be rebuilt, so it still runs its old build. Run hermes desktop --force-build in a terminal to rebuild it; the update log has the build error." log "desktop app build is an owed follow-up of the committed update" diff --git a/scripts/desktop-update/windows.ps1 b/scripts/desktop-update/windows.ps1 index 2bda29767dcd..6b97693f5d59 100644 --- a/scripts/desktop-update/windows.ps1 +++ b/scripts/desktop-update/windows.ps1 @@ -1731,6 +1731,15 @@ try { } } + # Contract C3: a Desktop build that failed after the code committed is an owed follow-up + # (exit 0); the CLI prints one whole "Desktop app build owed:" line for it. The user is on + # the new Hermes but this app was not rebuilt: a manual outcome, never plain success. + if ($res.Code -eq 0 -and -not $desktopBuildFailed -and $res.Output -match '(?m)^\s*Desktop app build owed: ') { + $manualAction = $true + $manualMsg = ("Hermes was updated, but the Desktop app could not be rebuilt, so it still runs its old build. Run ``hermes desktop --force-build`` in a terminal to rebuild it; the update log has the build error. " + $manualMsg).Trim() + Write-HandoffLog $manualMsg + } + if ($res.Code -eq 0 -and -not $desktopBuildFailed) { $finalCode = 0 $finalMsg = "Update complete." diff --git a/tests/hermes_cli/test_source_build.py b/tests/hermes_cli/test_source_build.py index dbdea3e14772..afe7de03449f 100644 --- a/tests/hermes_cli/test_source_build.py +++ b/tests/hermes_cli/test_source_build.py @@ -321,6 +321,35 @@ def test_update_failure_raises_without_retries_or_replacing_live_app(source_prod assert not (Path(os.environ["HERMES_HOME"]) / "desktop-build-stamp.json").exists() +@pytest.mark.platforms("linux") +@pytest.mark.parametrize("failure", ["npm-unavailable", "desktop-after-long-feature-failure"]) +def test_unbuilt_desktop_is_named_on_one_whole_line(source_products, monkeypatch, capsys, failure): + # The Desktop hand-off keys on this line: the receipt follow-up is truncated and leads with + # whichever product failed first, and an npm acquisition failure used to escape un-named. + import hermes_cli.main_install_repair as install_repair + from hermes_cli.source_build import ProductBuildError, build_update_products + + root, _ = source_products + + def fail(error): + def raiser(*_args, **_kwargs): + raise error + return raiser + + if failure == "npm-unavailable": + monkeypatch.setattr(pm, "ensure", fail(pm.InstallError("npm", "unavailable"))) + expected = "Desktop app build owed: Node dependencies failed" + else: + monkeypatch.setattr(install_repair, "_install_configured_features_missing_deps", + fail(RuntimeError("pip install failed: " + "x" * 600))) + (root / "fail-desktop").touch() + expected = "Desktop app build owed: desktop app build failed" + with pytest.raises(ProductBuildError): + build_update_products(root, desktop=True) + lines = [line.strip() for line in capsys.readouterr().out.splitlines()] + assert [line for line in lines if line.startswith("Desktop app build owed:")] == [expected] + + @pytest.mark.platforms("linux") @pytest.mark.parametrize("desktop", [False, True]) def test_module_cli_builds_the_requested_products(source_products, desktop, monkeypatch): diff --git a/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py b/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py index 36133d6ce78a..8f3d05b9185e 100644 --- a/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py +++ b/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py @@ -13,6 +13,7 @@ from __future__ import annotations +import json import os import subprocess import time @@ -34,10 +35,11 @@ FAKE_HERMES = """#!/usr/bin/env bash case "$*" in *--help*) echo "--keep-stash"; exit 0 ;; esac printf '%s\\n' "$*" >> "$HERMES_TEST_ARGV" +case " $* " in *" update "*) printf '%s\\n' "${HERMES_TEST_UPDATE_OUTPUT:-}" ;; esac exit 0 """ -def _run_handoff(tmp_path: Path, extra_args: list[str]) -> list[str]: +def _run_handoff(tmp_path: Path, extra_args: list[str], update_output: str = "") -> list[str]: """Run the real hand-off end to end; return the argv of each hermes call.""" install_root = tmp_path / "hermes-agent" (install_root / "venv" / "bin").mkdir(parents=True) @@ -47,7 +49,8 @@ def _run_handoff(tmp_path: Path, extra_args: list[str]) -> list[str]: argv_log = tmp_path / "argv.jsonl" # posix.sh honours an ambient HERMES_HOME; the result file must land where this test looks. - env = {**os.environ, "TMPDIR": str(tmp_path), "HERMES_TEST_ARGV": str(argv_log), "HERMES_HOME": str(tmp_path)} + env = {**os.environ, "TMPDIR": str(tmp_path), "HERMES_TEST_ARGV": str(argv_log), "HERMES_HOME": str(tmp_path), + "HERMES_TEST_UPDATE_OUTPUT": update_output} subprocess.run( ["/bin/bash", str(SHIM_DIR / "posix.sh"), "--install-root", str(install_root), "--no-ui", *extra_args], env=env, @@ -89,3 +92,19 @@ def test_no_gateway_flag_omits_gateway_from_update(tmp_path): assert "--gateway" not in argv, f"--gateway reappeared in update argv: {call}" assert "--keep-stash" in argv, "--no-gateway must not disturb --keep-stash" assert "--yes" in argv + + +@requires_posix_handoff +def test_owed_desktop_build_is_a_manual_outcome_even_when_the_followup_is_truncated(tmp_path): + """Contract C3: the update committed (exit 0) but this app was not rebuilt. + + The receipt follow-up is capped at 500 chars and leads with whichever product failed first, + so it can lose the desktop part; the CLI's own whole-line marker is what the hand-off reads. + """ + followup = " ⚠ Update follow-up 'build' did not finish: feature dependencies: " + "x" * 440 + _run_handoff(tmp_path, [], f"{followup}\n Desktop app build owed: desktop app build failed\n✓ Update complete!") + + result = json.loads((tmp_path / ".hermes-update-result.json").read_text(encoding="utf-8-sig")) + assert result["exit_code"] == 0 + assert result["manual"] is True + assert "could not be rebuilt" in result["message"] From 7e2f44041c3b7735d8e8aa2377b9f0b1d58bbcaa Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:53:33 -0700 Subject: [PATCH 27/62] fix(update): run the isolated completion child in UTF-8 mode (hardening, win-utf8 review) The parent already decodes the child's piped output as UTF-8; -X utf8 makes the child encode it that way on any console code page. The reviewed failure was invalid on re-verification; this is the one-token hardening the triage allowed. test_update_completion_process.py failure set unchanged. --- hermes_cli/update_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index d6ba20863e5e..5767e2d8823c 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -56,7 +56,7 @@ def run_completion(request: dict) -> dict: request = {**request, "stdout_isatty": sys.stdout.isatty()} request["bytecode_cache"] = str(Path(directory) / "bytecode") _write_json(request_path, request) - command = [sys.executable, "-I", "-S", "-u", "-X", f"pycache_prefix={request['bytecode_cache']}", + command = [sys.executable, "-I", "-S", "-u", "-X", "utf8", "-X", f"pycache_prefix={request['bytecode_cache']}", str(root / "hermes_cli/update_completion.py"), str(request_path), str(result_path)] proc = subprocess.Popen( From 15eb3240dea21f6ac0c7bae16297ed090f4f0341 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:02:16 -0700 Subject: [PATCH 28/62] docs(update): post-commit contract in the compacted update-pipeline docs main moved the update-pipeline long form from hermes_cli/AGENTS.md into developer-guide/cli-internals.md (f3af9366945). The C3 contract (nothing fails after the commit point; root-home receipts that are on disk from the start) now lives there, and the AGENTS.md rule bullet states it in the 30k chain budget. --- hermes_cli/AGENTS.md | 3 ++- website/docs/developer-guide/cli-internals.md | 21 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/hermes_cli/AGENTS.md b/hermes_cli/AGENTS.md index 23bc568dd049..825fc63365aa 100644 --- a/hermes_cli/AGENTS.md +++ b/hermes_cli/AGENTS.md @@ -136,7 +136,8 @@ The stage-by-stage contract: `website/docs/developer-guide/cli-internals.md` § `_ZIP_PRESERVED_NESTED` build outputs into the swap. - Restarts are fleet-wide and drain-first, bounded by the home each unit runs on (`update_fleet_scope.py`); another install's runtime is never restarted. -- A provably-stale gateway fails the update (exit 1); every run writes a receipt under +- After the commit point nothing fails `hermes update`: a stale gateway or failed post-commit + step is a receipt `followup` with its obligation kept armed. Receipts live under the ROOT home's `logs/update_receipts/`. - Nothing runs pulled code in the pre-pull interpreter. `update_handoff.py` and `update_serve_obligations.py` are a FROZEN compat surface (`tests/compat/old_updater_surface.json`): diff --git a/website/docs/developer-guide/cli-internals.md b/website/docs/developer-guide/cli-internals.md index d3610e70e917..a4c848b0d615 100644 --- a/website/docs/developer-guide/cli-internals.md +++ b/website/docs/developer-guide/cli-internals.md @@ -51,10 +51,23 @@ it guards. `plan → snapshot → apply → restart-per-kind → verify → repo left alone, never restarted (#93349). - **Verify**: gateways stamp `code_sha`/`code_version` into `gateway_state.json` on every runtime-status write (`gateway/status.py`); the updater compares each live gateway against the - fresh checkout and prints a fleet version matrix. A provably-stale gateway fails the update - (exit 1) — automation must never treat a mixed-version fleet as healthy. -- **Report**: every run writes a machine-readable receipt to `~/.hermes/logs/update_receipts/` - (`latest.json` pointer; steps, skips WITH reasons, restart outcome, plan, fleet snapshot). + fresh checkout and prints a fleet version matrix. After the commit point (the tree moved) nothing + fails `hermes update` (exit 0 unless the code was rolled back): a provably-stale or stopped + gateway, a failed build/launcher/config migration/bytecode sweep/channel adoption/Windows resume + each print `⚠`, land in the receipt as `outcome: "success"` with `followups: [{step, reason}]`, + and keep their own obligation armed (`source-completion-pending` for the tail, the fleet-restart + obligation for gateways) so the next launch or `hermes update` — including "Already up to date" — + retries them. The owed-fleet-restart CLI startup warning is what keeps a mixed-version fleet from + ever looking healthy. Exit 2 (refused/concurrent) and exit 1 (nothing committed / rolled back) + keep their meaning. +- **Report**: every run writes a machine-readable receipt to the ROOT home's + `logs/update_receipts/` (never a sticky profile's; `update.log` likewise) — `latest.json` pointer; + steps, skips WITH reasons, restart outcome, plan, fleet snapshot, `followups`. The receipt is on + disk as `outcome: "running"` from the start and refreshed at each stage boundary; the next + update marks a `running` record whose processes are gone `interrupted` and says so. Readers + (`hermes logs update`, debug bundle, dashboard status, pm receipts) use the root home too. + A dependency-sync failure in the completion bootstrap is a `dependencies` follow-up (A6), and + Ctrl-C after the commit point finalizes `interrupted` (exit 130), never `failed`. Before a source swap, the parent captures plan/snapshots/receipt and its Windows pause token. `update_completion.py` runs new-code PM preparation with site initialization disabled, then selected-Python builds, maintenance, scans/restarts and verification. Git/current/ZIP share From bc4f5cc7c69b79a17e10ede683ddeddab18ef7e3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:05:18 -0700 Subject: [PATCH 29/62] chore(update): post-commit catch-all boundaries carry the code-health waiver main's code-health ratchet (41567159605) ignores `# noqa: BLE001`, and ruff does not select BLE here, so those markers waived nothing. Each catch-all is the C3 boundary: after the commit point the failure is a receipt follow-up, never the update's status. The stated reason moves into `# health: allow BLE001 -- ` unchanged. --- hermes_cli/source_build.py | 6 +- hermes_cli/source_completion.py | 2 +- hermes_cli/update_cmd.py | 6 +- hermes_cli/update_cmd_fleet_verify.py | 2 +- hermes_cli/update_cmd_maint.py | 6 +- hermes_cli/update_completion.py | 4 +- hermes_cli/update_finish.py | 4 +- .../test_web_server_update_status.py | 88 +++++++++++++++++++ 8 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 tests/hermes_cli/test_web_server_update_status.py diff --git a/hermes_cli/source_build.py b/hermes_cli/source_build.py index 086a1e9c12d6..37e94aab5b4d 100644 --- a/hermes_cli/source_build.py +++ b/hermes_cli/source_build.py @@ -130,7 +130,7 @@ def attempt(name: str, step) -> bool: try: step() return True - except Exception as exc: # noqa: BLE001 — collected and raised after the independent steps + except Exception as exc: # health: allow BLE001 -- collected and raised after the independent steps print(f" ⚠ {name} failed: {_failure_text(exc)}") failures.append((name, exc)) return False @@ -180,7 +180,7 @@ def node_dependencies() -> None: from hermes_cli.memory_provider_migration import migrate_all_homes migrate_all_homes() - except Exception as exc: + except Exception as exc: # health: allow BLE001 -- post-commit boundary: the failure is reported as a follow-up, never fails the committed update print(f" ⚠ Memory provider migration skipped: {exc}") # Same for a gateway platform / toolset that left core (Home Assistant): every home that used # it gets its catalog plugin (hermes_cli/left_core_migration.py). @@ -188,7 +188,7 @@ def node_dependencies() -> None: from hermes_cli.left_core_migration import migrate_all_homes as migrate_left_core migrate_left_core() - except Exception as exc: + except Exception as exc: # health: allow BLE001 -- post-commit boundary: the failure is reported as a follow-up, never fails the committed update print(f" ⚠ Plugin migration skipped: {exc}") if failures: raise ProductBuildError(failures) diff --git a/hermes_cli/source_completion.py b/hermes_cli/source_completion.py index 603690ea25d1..40cdccfdd317 100644 --- a/hermes_cli/source_completion.py +++ b/hermes_cli/source_completion.py @@ -105,7 +105,7 @@ def step(name: str, run) -> None: # config migration, the maintenance, or (in an update) the gateway restart after it. try: run() - except (Exception, SystemExit) as exc: # noqa: BLE001 — reported as an owed follow-up + except (Exception, SystemExit) as exc: # health: allow BLE001 -- reported as an owed follow-up from hermes_cli.update_receipt import record_followup reason = str(exc) or type(exc).__name__ diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 52c03ec25fe9..965dec28612d 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -747,7 +747,7 @@ def _resume_paused_gateways_at_exit(token: dict | None, request: dict | None) -> return try: _m()._resume_windows_gateways_after_update(token) - except Exception as exc: # noqa: BLE001 — a restart failure is owed, not the update's status + except Exception as exc: # health: allow BLE001 -- a restart failure is owed, not the update's status from hermes_cli.update_receipt import _current, amend_terminal_followup, record_followup reason = f"Windows gateway recovery failed: {exc}" @@ -791,7 +791,7 @@ def _complete_source_update(request: dict | None) -> None: # Dependencies owed (A6): the bootstrap child cannot resume paused gateways; this process can. try: _m()._resume_windows_gateways_after_update(token) - except Exception as exc: # noqa: BLE001 — the code is committed (C3) + except Exception as exc: # health: allow BLE001 -- the code is committed (C3) from hermes_cli.update_receipt import amend_terminal_followup, record_followup reason = f"Windows gateway recovery failed: {exc}" @@ -807,7 +807,7 @@ def _complete_source_update(request: dict | None) -> None: try: if adopt_retired_channel(request): print(f"→ Source subscription moved to {request['channel_retirement']['destination']}") - except Exception as exc: # noqa: BLE001 — the code is committed (C3); the next update re-adopts + except Exception as exc: # health: allow BLE001 -- the code is committed (C3); the next update re-adopts from hermes_cli.update_receipt import amend_terminal_followup, record_followup reason = str(exc) or type(exc).__name__ diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py index 9275e0f06605..59f092d95fee 100644 --- a/hermes_cli/update_cmd_fleet_verify.py +++ b/hermes_cli/update_cmd_fleet_verify.py @@ -346,6 +346,6 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re try: from hermes_cli.gateway_migrate import maybe_auto_migrate_after_update maybe_auto_migrate_after_update() - except (Exception, SystemExit) as exc: # noqa: BLE001 — a SystemExit here must not fail a committed update + except (Exception, SystemExit) as exc: # health: allow BLE001 -- a SystemExit here must not fail a committed update logger.warning('Multiplex auto-migration after update failed: %s', exc) print(f" ⚠ Gateway multiplex migration did not finish: {exc} (run `hermes gateway migrate` later)") diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py index 2c35430dc796..5504968d507c 100644 --- a/hermes_cli/update_cmd_maint.py +++ b/hermes_cli/update_cmd_maint.py @@ -822,11 +822,11 @@ def _sweep_bytecode_after_update(branch: str) -> None: if removed: print(f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}") _m()._record_bytecode_fingerprint() - except Exception as exc: # noqa: BLE001 — owed to the next launch's fingerprint sweep + except Exception as exc: # health: allow BLE001 -- owed to the next launch's fingerprint sweep record_followup("bytecode_sweep", str(exc) or type(exc).__name__) try: _m()._refresh_bootstrap_cache_scripts(branch) - except Exception as exc: # noqa: BLE001 — refreshed again by the next update + except Exception as exc: # health: allow BLE001 -- refreshed again by the next update record_followup("bootstrap_scripts", str(exc) or type(exc).__name__, retry="the next `hermes update` refreshes them") @@ -995,7 +995,7 @@ def _run_post_update_maintenance( def owed_step(name, run): try: run() - except (Exception, SystemExit) as exc: # noqa: BLE001 — the code is committed; retry later + except (Exception, SystemExit) as exc: # health: allow BLE001 -- the code is committed; retry later reason = str(exc) or type(exc).__name__ record_followup(name, reason) if followups is not None: diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index 5767e2d8823c..26d4f567a731 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -295,7 +295,7 @@ def _complete_selected(request: dict) -> bool: completion_message=request.get("completion_message"), announce=None if request.get("completion_message") else "\n✓ Code updated!", followups=followups) - except (Exception, SystemExit) as exc: # noqa: BLE001 — e.g. the shared update lock refused the tail + except (Exception, SystemExit) as exc: # health: allow BLE001 -- e.g. the shared update lock refused the tail reason = str(exc) or type(exc).__name__ record_followup("completion", reason) followups.append(("completion", reason)) @@ -423,7 +423,7 @@ def _bootstrap(request: dict, request_path: Path, result_path: Path) -> int: update_receipt.finalize_interrupted_update_receipt( "KeyboardInterrupt: interrupted while installing dependencies", exit_code=130) return _write_bootstrap_result(request, result_path, 130, _read_terminal_receipt(request)) - except (Exception, SystemExit) as exc: # noqa: BLE001 — after the commit point nothing fails the update + except (Exception, SystemExit) as exc: # health: allow BLE001 -- after the commit point nothing fails the update # The code is committed but its dependencies are not (A6): an owed follow-up, exit 0. # Paused Windows gateways stay the parent's to resume (it has the dependencies). return _settle_after_commit(request, result_path, "dependencies", str(exc) or type(exc).__name__) diff --git a/hermes_cli/update_finish.py b/hermes_cli/update_finish.py index 6ccc5df1bb6b..acb56066c935 100644 --- a/hermes_cli/update_finish.py +++ b/hermes_cli/update_finish.py @@ -33,7 +33,7 @@ def finish_update(*, root, assume_yes, gateway_mode, pre_update_snapshot_id, had_desktop_app_before_update=had_desktop_app_before_update, pre_update_version=pre_update_version, followups=owed, ) - except (Exception, SystemExit) as exc: # noqa: BLE001 — the code is committed; retry later + except (Exception, SystemExit) as exc: # health: allow BLE001 -- the code is committed; retry later reason = str(exc) or type(exc).__name__ record_followup("maintenance", reason) owed.append(("maintenance", reason)) @@ -139,7 +139,7 @@ def main(context: Path, result: Path) -> int: owed: list[tuple[str, str]] = [] try: build_update_products(root, desktop=desktop) - except (Exception, SystemExit) as exc: # noqa: BLE001 — committed code: owed, not failed + except (Exception, SystemExit) as exc: # health: allow BLE001 -- committed code: owed, not failed reason = str(exc) or type(exc).__name__ update_receipt.record_followup("build", reason) owed.append(("build", reason)) diff --git a/tests/hermes_cli/test_web_server_update_status.py b/tests/hermes_cli/test_web_server_update_status.py new file mode 100644 index 000000000000..7656849f5cb1 --- /dev/null +++ b/tests/hermes_cli/test_web_server_update_status.py @@ -0,0 +1,88 @@ +"""Dashboard update status reads the ROOT home's shared update.log, correlated by action id.""" + +import types + +import pytest + +import hermes_cli.web_server_gateway as _web_server_gateway + + +class TestUpdateStatusRootLog: + @pytest.fixture(autouse=True) + def _setup_test_client(self, monkeypatch, _isolate_hermes_home): + """Create a TestClient and isolate the state DB under the test HERMES_HOME.""" + try: + from starlette.testclient import TestClient + except ImportError: + pytest.skip("fastapi/starlette not installed") + + import hermes_state + from hermes_constants import get_hermes_home + from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN + + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + + self.client = TestClient(app) + self.client.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN + + def test_update_status_recovers_completed_result_after_dashboard_restart(self, monkeypatch, tmp_path): + # The dashboard runs under a profile home; ``hermes update`` mirrors to the ROOT home's + # update.log (main_dashboard), so the durable completion marker is read there. + root = tmp_path / "root" + (root / "logs").mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "coder")) + action_id = "c" * 32 + (tmp_path / "hermes-update.log").write_text( + f"=== hermes-update started 2026-08-17 11:19:34 {action_id} ===\n" + "pulling updates...\n", + encoding="utf-8", + ) + (root / "logs" / "update.log").write_text( + "=== hermes update started 2026-08-17T11:19:35 ===\n" + "✓ Update complete!\n" + f"=== hermes-update completed {action_id} ===\n", + encoding="utf-8", + ) + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path) + monkeypatch.setattr(_web_server_gateway, "_ACTION_PROCS", {}) + monkeypatch.setattr(_web_server_gateway, "_ACTION_RESULTS", {}) + monkeypatch.setattr(_web_server_gateway, "_ACTION_COMMANDS", {}) + monkeypatch.setattr(_web_server_gateway, "_ACTION_IDS", {}) + + status = self.client.get("/api/actions/hermes-update/status?lines=2000") + + assert status.status_code == 200 + data = status.json() + assert data["running"] is False + assert data["exit_code"] == 0 + assert data["action_id"] == action_id + assert f"=== hermes-update completed {action_id} ===" in data["lines"] + + + def test_update_status_ignores_another_profiles_completion_in_the_shared_root_log(self, monkeypatch, tmp_path): + # Every profile's ``hermes update`` mirrors into the ROOT update.log; profile A's success + # there must not certify the action profile B's dashboard started (and lost on restart). + root = tmp_path / "root" + (root / "logs").mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + b_logs = tmp_path / "b-logs" + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", b_logs) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + monkeypatch.setattr(_web_server_gateway.subprocess, "Popen", lambda *a, **kw: types.SimpleNamespace(pid=7)) + b_id = "b" * 32 + _web_server_gateway._spawn_hermes_action(["update"], "hermes-update", env_overrides={"HERMES_ACTION_ID": b_id}) + _web_server_gateway._ACTION_PROCS.clear() # B's dashboard restarted: in-memory result lost + + def status_after_root_completion(action_id): + (root / "logs" / "update.log").write_text( + "=== hermes update started 2026-08-17T11:19:35 ===\n" + f"=== hermes-update completed {action_id} ===\n", encoding="utf-8") + return self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + foreign = status_after_root_completion("a" * 32) + assert foreign["exit_code"] is None + assert "action_id" not in foreign + own = status_after_root_completion(b_id) + assert own["exit_code"] == 0 + assert own["action_id"] == b_id From 8cf0455297336280caac78a3dd0b25bba5d9e51d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:05:18 -0700 Subject: [PATCH 30/62] test(dashboard): update-status root-log cells move to their own file test_web_server.py is over the 2,000-line target and may only shrink; the two root-home update.log status cells (one changed, one added by this PR) move unchanged into test_web_server_update_status.py. --- tests/hermes_cli/test_web_server.py | 62 ----------------------------- 1 file changed, 62 deletions(-) diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 171df54c6d7e..ade89ea0ea9a 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -8,7 +8,6 @@ import sys import threading import time -import types from pathlib import Path from unittest.mock import patch, MagicMock @@ -1455,67 +1454,6 @@ def fail_spawn(*_args, **_kwargs): assert check_data["can_apply"] is False assert check_data["update_command"] == data["update_command"] - def test_update_status_recovers_completed_result_after_dashboard_restart(self, monkeypatch, tmp_path): - # The dashboard runs under a profile home; ``hermes update`` mirrors to the ROOT home's - # update.log (main_dashboard), so the durable completion marker is read there. - root = tmp_path / "root" - (root / "logs").mkdir(parents=True) - monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "coder")) - action_id = "c" * 32 - (tmp_path / "hermes-update.log").write_text( - f"=== hermes-update started 2026-08-17 11:19:34 {action_id} ===\n" - "pulling updates...\n", - encoding="utf-8", - ) - (root / "logs" / "update.log").write_text( - "=== hermes update started 2026-08-17T11:19:35 ===\n" - "✓ Update complete!\n" - f"=== hermes-update completed {action_id} ===\n", - encoding="utf-8", - ) - monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path) - monkeypatch.setattr(_web_server_gateway, "_ACTION_PROCS", {}) - monkeypatch.setattr(_web_server_gateway, "_ACTION_RESULTS", {}) - monkeypatch.setattr(_web_server_gateway, "_ACTION_COMMANDS", {}) - monkeypatch.setattr(_web_server_gateway, "_ACTION_IDS", {}) - - status = self.client.get("/api/actions/hermes-update/status?lines=2000") - - assert status.status_code == 200 - data = status.json() - assert data["running"] is False - assert data["exit_code"] == 0 - assert data["action_id"] == action_id - assert f"=== hermes-update completed {action_id} ===" in data["lines"] - - def test_update_status_ignores_another_profiles_completion_in_the_shared_root_log(self, monkeypatch, tmp_path): - # Every profile's ``hermes update`` mirrors into the ROOT update.log; profile A's success - # there must not certify the action profile B's dashboard started (and lost on restart). - root = tmp_path / "root" - (root / "logs").mkdir(parents=True) - monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) - b_logs = tmp_path / "b-logs" - monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", b_logs) - for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): - monkeypatch.setattr(_web_server_gateway, registry, {}) - monkeypatch.setattr(_web_server_gateway.subprocess, "Popen", lambda *a, **kw: types.SimpleNamespace(pid=7)) - b_id = "b" * 32 - _web_server_gateway._spawn_hermes_action(["update"], "hermes-update", env_overrides={"HERMES_ACTION_ID": b_id}) - _web_server_gateway._ACTION_PROCS.clear() # B's dashboard restarted: in-memory result lost - - def status_after_root_completion(action_id): - (root / "logs" / "update.log").write_text( - "=== hermes update started 2026-08-17T11:19:35 ===\n" - f"=== hermes-update completed {action_id} ===\n", encoding="utf-8") - return self.client.get("/api/actions/hermes-update/status?lines=2000").json() - - foreign = status_after_root_completion("a" * 32) - assert foreign["exit_code"] is None - assert "action_id" not in foreign - own = status_after_root_completion(b_id) - assert own["exit_code"] == 0 - assert own["action_id"] == b_id - def test_update_hermes_spawns_with_action_id(self, monkeypatch): import hermes_cli.web_server as web_server From 281c68a8f2a019420587db30ef38855f22077031 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:49:41 -0700 Subject: [PATCH 31/62] fix(update): an unwritable gateway status never fails a settled commit _settle_after_commit wrote /.update_exit_code unguarded. An OSError there (a home that is a file, a read-only mount, ENOSPC) escaped before _write_bootstrap_result, so the parent saw no result and reported "Source update completion did not finish" + exit 1 for a run whose code was committed and whose receipt already said success. The write now goes through _publish_gateway_success, guarded exactly like update_cmd_fleet._write_gateway_update_exit_code (which this stdlib-only bootstrap cannot import): it warns and the result is still written, so the run answers 0. The gateway watcher falls back to its own timeout. Review P1 (invariant 3). --- hermes_cli/update_completion.py | 20 ++++++++++++++++--- .../test_update_completion_process.py | 18 +++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index 26d4f567a731..e64b8b6bd1c2 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -183,6 +183,21 @@ def _record_owed_user_action(request: dict) -> bool: return notice is None +def _publish_gateway_success(request: dict) -> None: + """The gateway /update watcher's status: the code is committed (same as _complete_selected). + + Guarded like ``update_cmd_fleet._write_gateway_update_exit_code`` (which this stdlib-only + process cannot import): an unwritable home must not turn the committed run into an error + before its result is written (review P1). The watcher then falls back to its own timeout. + """ + if not request.get("gateway_mode"): + return + try: + (Path(request["home"]) / ".update_exit_code").write_text("0", encoding="utf-8") + except OSError as exc: + print(f" ⚠ Could not publish the gateway's update status: {exc}") + + def _settle_after_commit(request: dict, result_path: Path, step: str, reason: str) -> int: """The tree already moved, so a failure here is owed work, never a failed update (C3, A6). @@ -210,9 +225,8 @@ def _settle_after_commit(request: dict, result_path: Path, step: str, reason: st update_receipt.finalize_pending_update_receipt(0, f"{step} owed after the code was updated") receipt = _read_terminal_receipt(request) code = 0 if receipt is not None and receipt.get("outcome") == "success" else 1 - if code == 0 and request["gateway_mode"]: - # The gateway's /update watcher reads this; the code is committed (same as _complete_selected). - (Path(request["home"]) / ".update_exit_code").write_text("0", encoding="utf-8") + if code == 0: + _publish_gateway_success(request) return _write_bootstrap_result(request, result_path, code, receipt) diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index a5f7610288a5..8f8ed40c1769 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -280,6 +280,24 @@ def complete(args, gateway_mode): lock.release() +def test_unwritable_gateway_status_never_fails_a_settled_commit(tmp_path, monkeypatch): + """Review P1: the bootstrap's gateway status write is best effort, like + ``_write_gateway_update_exit_code``; the committed run still answers its result with exit 0.""" + from hermes_cli import venv_sync + + monkeypatch.setattr(venv_sync, "arm_completion", lambda root: None) + receipt = {"update_id": "u1", "outcome": "success", "finished_at": "now"} + monkeypatch.setattr(update_completion, "_read_terminal_receipt", lambda request: receipt) + home = tmp_path / "home-is-a-file" + home.write_text("x", encoding="utf-8") + result = tmp_path / "result.json" + request = {"source": str(tmp_path), "receipt": {"update_id": "u1"}, "gateway_mode": True, + "home": str(home), "pm_receipt": None} + assert update_completion._settle_after_commit(request, result, "dependencies", "pip failed") == 0 + answer = json.loads(result.read_text(encoding="utf-8")) + assert answer["exit_code"] == 0 and answer["receipt"] == receipt + + @pytest.mark.parametrize("cleanup_failure", [None, "kill", "wait"]) def test_interrupt_after_child_success_demotes_gateway_marker_at_boundary(transition, monkeypatch, cleanup_failure): import io From 2b594187c908957842367f2e80121f9ac1e33f48 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:50:40 -0700 Subject: [PATCH 32/62] fix(update): a Desktop build that never ran is named for the hand-off The only "Desktop app build owed:" line came from source_build when the Desktop build itself failed. A committed run whose build never ran -- the dependency sync failed (A6), the source-update tail was refused by the update lock, the prepared completion child died -- printed no such line, so posix.sh / windows.ps1 reported a plain "Update complete." while the app was stale (F6 residual). update_completion now owns the prefix (DESKTOP_BUILD_OWED, same text the scripts match) and _report_desktop_build_owed prints the whole line whenever the run requested the Desktop build and the run's record has no `build` stage (the build never reached its own verdict, where the builder prints the line itself): in _settle_after_commit (deps owed / prepared child died), in _complete_selected when the tail raised, and in _finish when the tail raised before the build stage. Review P4. --- hermes_cli/update_completion.py | 37 ++++++++++++++++++- .../test_update_completion_process.py | 31 ++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index e64b8b6bd1c2..135caec8827b 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -198,6 +198,26 @@ def _publish_gateway_success(request: dict) -> None: print(f" ⚠ Could not publish the gateway's update status: {exc}") +#: The one whole line the Desktop hand-off scripts match (scripts/desktop-update/posix.sh and +#: windows.ps1); ``source_build.build_update_products`` prints the same prefix when the build fails. +DESKTOP_BUILD_OWED = "Desktop app build owed:" + + +def _report_desktop_build_owed(request: dict, record: dict | None, reason: str) -> None: + """Name the owed Desktop build when this run never got the build to its own verdict (F6, review P4). + + The builder prints the line only when the build itself fails; a run that never reached it + (dependencies owed, the tail refused by the lock, a completion process lost) would otherwise end + as a plain "Update complete." in the hand-off. A ``build`` stage on the run's record means the + build ran and printed this line itself when the Desktop app failed. + """ + if not request.get("desktop"): + return + if any(isinstance(mark, dict) and mark.get("name") == "build" for mark in (record or {}).get("stages") or ()): + return + print(f" {DESKTOP_BUILD_OWED} {reason}", flush=True) + + def _settle_after_commit(request: dict, result_path: Path, step: str, reason: str) -> int: """The tree already moved, so a failure here is owed work, never a failed update (C3, A6). @@ -216,7 +236,9 @@ def _settle_after_commit(request: dict, result_path: Path, step: str, reason: st receipt = _read_terminal_receipt(request) # a prepared child that finalized, then died if receipt is None: # A prepared child that died mid-tail persisted stages the parent's snapshot lacks. - _resume_receipt(_running_record(request) or request["receipt"]) + record = _running_record(request) or request["receipt"] + _report_desktop_build_owed(request, record, f"{step} did not finish") + _resume_receipt(record) update_receipt.record_stage("deps" if step == "dependencies" else "build", "failed") update_receipt.record_followup(step, reason, retry="dependencies not installed yet — the next launch retries" if step == "dependencies" else "the next launch or `hermes update` retries it") @@ -311,6 +333,8 @@ def _complete_selected(request: dict) -> bool: followups=followups) except (Exception, SystemExit) as exc: # health: allow BLE001 -- e.g. the shared update lock refused the tail reason = str(exc) or type(exc).__name__ + # The tail's steps catch their own failures, so a raise here means the build never ran. + _report_desktop_build_owed(request, None, "the source-update tail did not run") record_followup("completion", reason) followups.append(("completion", reason)) tail_owed = any(step in TAIL_FOLLOWUPS for step, _ in followups) @@ -361,6 +385,15 @@ def __getattr__(self, name): return getattr(self.stream, name) +def _report_unbuilt_desktop(request: dict) -> None: + """The tail raised out of ``_complete_selected``: before the build stage, the build never ran.""" + from hermes_cli import update_receipt + + current = update_receipt._current.get() + if current is not None: # a finalized run got past the build (verification finalizes it) + _report_desktop_build_owed(request, current.data, "the source-update tail did not finish") + + def _finish(request: dict, result_path: Path) -> int: from hermes_cli import update_receipt from pm.receipt import accept_worker_receipt @@ -379,8 +412,10 @@ def _finish(request: dict, result_path: Path) -> int: update_receipt.finalize_interrupted_update_receipt(reason, exit_code=code) except SystemExit as exc: if exc.code not in (0, None): + _report_unbuilt_desktop(request) update_receipt.record_followup("completion", f"completion exited {exc.code}") except BaseException as exc: # noqa: BLE001 — after the commit point nothing fails the update + _report_unbuilt_desktop(request) update_receipt.record_followup("completion", f"{type(exc).__name__}: {exc}") finally: if code and request["gateway_mode"]: diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index 8f8ed40c1769..2dc5d811e29e 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -280,6 +280,37 @@ def complete(args, gateway_mode): lock.release() +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("never_built", ["dependencies", "refused", "died"]) +@pytest.mark.parametrize("desktop", [True, False]) +def test_desktop_build_that_never_ran_is_named_for_the_handoff(transition, capsys, never_built, desktop): + """Review P4 (F6): the hand-off scripts read ``Desktop app build owed:`` to tell the user the app is + stale. The builder prints it only when the build fails; a run whose build never ran (dependency + sync failed, the tail refused by the update lock, the prepared child died) must print it too.""" + scripts = Path(update_completion.__file__).parents[1] / "scripts/desktop-update" + for script in ("posix.sh", "windows.ps1"): # the exact prefix the hand-off scripts match + assert "Desktop app build owed: " in (scripts / script).read_text(encoding="utf-8") + assert update_completion.DESKTOP_BUILD_OWED == "Desktop app build owed:" + root, git, old, new, request = transition + request["desktop"] = desktop + shutil.copy2(update_completion.__file__, root / "hermes_cli/update_completion.py") + if never_built == "dependencies": + (root / "pm/__init__.py").write_text("def sync_venv(**kw): raise RuntimeError('dependency refused')\n") + elif never_built == "refused": + (root / "hermes_cli/source_completion.py").write_text( + "def complete_source_checkout(*args, **kwargs):\n" + " raise RuntimeError('an update is still running (pid 1); wait for it to exit')\n") + else: + (root / "hermes_cli/source_completion.py").write_text( + "import os, signal\n" + "def complete_source_checkout(*args, **kwargs): os.kill(os.getpid(), signal.SIGKILL)\n") + result = update_completion.run_completion(request) + out = capsys.readouterr().out + assert result["exit_code"] == 0, out + owed = [line.strip() for line in out.splitlines() if line.strip().startswith("Desktop app build owed: ")] + assert len(owed) == (1 if desktop else 0), out + + def test_unwritable_gateway_status_never_fails_a_settled_commit(tmp_path, monkeypatch): """Review P1: the bootstrap's gateway status write is best effort, like ``_write_gateway_update_exit_code``; the committed run still answers its result with exit 0.""" From e5c8e5993413a8f3e6d4c3d0c88322494e6796e0 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:52:00 -0700 Subject: [PATCH 33/62] fix(update): a lost completion after the commit point is owed, not failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the tree moved, an OSError from run_completion's TemporaryDirectory, _write_json or Popen (or anything the spawn path raises) reached main.cmd_update as "Update failed": receipt failed, .update_exit_code 1, exit 1. A completion child killed by OOM/SIGKILL, or one that exited without a correlated terminal result, returned `code or 1` and exited non-zero the same way. Both contradict invariant 3: the code is committed, only the tail is owed. _complete_source_update now settles both through update_completion.settle_lost_completion (pre-swap module, everything it uses is already imported in the parent): it arms the source-update tail for the next launch (the bootstrap child may never have run to arm it), names the owed Desktop build (P4), records a `completion` follow-up and finalizes the run success-with-follow-ups, publishes the gateway watcher's 0, and returns exit 0 -- so the parent also resumes paused Windows gateways the lost child never resumed. Meanings kept: Ctrl-C stays 130 (a 130 without a result finalizes `interrupted`, not `failed`), parked local changes stay `partial` / exit 1 (#122557), and a child that finalized then died keeps its own terminal record. run_completion no longer prints "✗ ... did not finish"; it hands the reason to the caller. Test change: test_missing_child_result_fails_boundary_receipt_and_releases_lock pinned exit `code or 1`, receipt `failed`, .update_exit_code 1 and resume_needed still True -- exactly the outcome invariant 3 forbids. It is now test_missing_child_result_is_an_owed_completion_and_releases_lock (adds a SIGKILL case): exit 0, receipt success with a `completion` follow-up, watcher 0, gateways resumed, lock released. New: spawn OSError is owed; a lost completion with parked changes stays partial/1. Review P2. --- hermes_cli/update_cmd.py | 11 ++- hermes_cli/update_completion.py | 40 +++++++++- .../test_update_completion_process.py | 80 ++++++++++++++++--- 3 files changed, 116 insertions(+), 15 deletions(-) diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 965dec28612d..d9d4697f17ee 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -774,12 +774,21 @@ def _complete_source_update(request: dict | None) -> None: # A head capture that came back empty must not arm an SHA-less record: it names no code the # fleet can be proven current on, so the warning could never clear (#125952). _write_fleet_restart_pending_marker(expected_sha=request.get("expected_sha") or _current_checkout_sha() or "") + # Pre-swap module (imported with run_completion above): never the replacement tree's. + from hermes_cli.update_completion import settle_lost_completion try: result = run_completion(request) except KeyboardInterrupt as interrupt: # Ctrl-C after the commit point: the tree is new, so the run is interrupted, never "failed". _completion_receipt.finalize_interrupted_update_receipt("KeyboardInterrupt: interrupted after the code was updated") raise SystemExit(130) from interrupt + except Exception as exc: # health: allow BLE001 -- after the commit point a lost completion is owed, never a failed update (C3) + result = settle_lost_completion(request, f"{type(exc).__name__}: {exc}") + if result.get("receipt") is None and result["exit_code"] == 130: + _completion_receipt.finalize_interrupted_update_receipt("the completion was interrupted after the code was updated") + elif result.get("receipt") is None: + # The child crashed (OOM, SIGKILL) or never answered: the same owed tail (review P2). + result = settle_lost_completion(request, result.get("error") or f"the completion process exited {result['exit_code']}") _accept_completion_pm_receipt(result.get("pm_receipt"), request["receipt"]["update_id"]) token = request["windows_resume"] if token is not None and result.get("windows_resume") is not None: @@ -788,7 +797,7 @@ def _complete_source_update(request: dict | None) -> None: token.update(resumed) _settle_windows_resume(request) elif token and token.get("resume_needed") and not result["exit_code"]: - # Dependencies owed (A6): the bootstrap child cannot resume paused gateways; this process can. + # Dependencies owed (A6) or a lost completion: no child resumed paused gateways; this process can. try: _m()._resume_windows_gateways_after_update(token) except Exception as exc: # health: allow BLE001 -- the code is committed (C3) diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index 135caec8827b..bcd6b3cd620c 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -116,8 +116,9 @@ def run_completion(request: dict) -> dict: if code == 0 and receipt is None: raise ValueError("completion did not publish a terminal receipt") except (OSError, ValueError, KeyError, TypeError) as exc: - print(f"✗ Source update completion did not finish: {exc}") - return {"exit_code": code or 1, "receipt": None, "windows_resume": None} + # No correlated terminal result: the caller settles it (settle_lost_completion). + return {"exit_code": code or 1, "receipt": None, "windows_resume": None, + "error": f"{exc} (the completion process exited {code})"} return result @@ -218,6 +219,41 @@ def _report_desktop_build_owed(request: dict, record: dict | None, reason: str) print(f" {DESKTOP_BUILD_OWED} {reason}", flush=True) +def settle_lost_completion(request: dict, reason: str) -> dict: + """The parent's side of C3: the code is committed, but the completion never answered (review P2). + + Reached when spawning or reading the completion failed (an OSError from the temporary + directory, the request file or Popen) or the child died without a correlated terminal result + (OOM, SIGKILL). The tail is owed, never a failed update: it is armed for the next launch, the + run's receipt is a success that names the ``completion`` follow-up, and the exit is 0 -- unless + the user's local changes are still parked (#122557: ``partial``, exit 1). Runs in the parent + (pre-swap) interpreter: everything it uses is already imported there. + """ + from hermes_cli import update_receipt + + print(f" ⚠ Source update completion did not finish: {reason}", flush=True) + receipt = _read_terminal_receipt(request) # the child finalized the run, then died + if receipt is None: + try: + from hermes_cli.venv_sync import arm_completion + + arm_completion(Path(request["source"])) + except Exception as exc: # health: allow BLE001 -- post-commit boundary: stale dependencies still make the next launch sync + print(f" ⚠ Could not record the owed source-update tail: {exc}") + _report_desktop_build_owed(request, _running_record(request), "the update completion did not finish") + update_receipt.record_followup("completion", reason, retry="the next launch or `hermes update` finishes it") + parked = not _record_owed_user_action(request) + if parked: + print(_owed_user_action(request)) # the completion child that prints it may never have run + update_receipt.finalize_pending_update_receipt(1 if parked else 0, "completion owed after the code was updated") + receipt = _read_terminal_receipt(request) + outcome = (receipt or {}).get("outcome") + code = 130 if outcome == "interrupted" else 1 if outcome == "partial" else 0 + if code == 0: + _publish_gateway_success(request) + return {"exit_code": code, "receipt": receipt, "windows_resume": None, "pm_receipt": None} + + def _settle_after_commit(request: dict, result_path: Path, step: str, reason: str) -> int: """The tree already moved, so a failure here is owed work, never a failed update (C3, A6). diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index 2dc5d811e29e..032092faadca 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -248,13 +248,11 @@ def test_old_process_new_git_tree_completes_in_fresh_python(transition, tmp_path assert [e["name"] for e in events].index("exit_marker") < [e["name"] for e in events].index("restart") -@pytest.mark.parametrize("code", [0, 23]) -def test_missing_child_result_fails_boundary_receipt_and_releases_lock(transition, monkeypatch, code): +def _run_cmd_update(monkeypatch, request, capsys): + """``hermes update --gateway`` whose impl hands this request to the real post-commit completion.""" from types import SimpleNamespace - from hermes_cli import main, update_cmd, update_receipt, update_lock + from hermes_cli import main, update_cmd, update_receipt - root, git, old, new, request = transition - (root / "hermes_cli/update_completion.py").write_text(f"import os\nos._exit({code})\n") monkeypatch.setenv("HERMES_HOME", request["home"]) monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) @@ -266,20 +264,78 @@ def complete(args, gateway_mode): update_cmd._complete_source_update(request) monkeypatch.setattr(update_cmd, "_cmd_update_impl", complete) - with pytest.raises(SystemExit) as error: + try: main.cmd_update(SimpleNamespace(gateway=True)) - assert error.value.code == (code or 1) - receipt = update_receipt.read_latest_receipt() - assert receipt["outcome"] == "failed" - assert receipt["exit_code"] == (code or 1) + code = 0 + except SystemExit as exc: + code = exc.code + return code, update_receipt.read_latest_receipt(), capsys.readouterr().out + + +@pytest.mark.parametrize("code", [0, 23, -9]) +def test_missing_child_result_is_an_owed_completion_and_releases_lock(transition, monkeypatch, capsys, code): + """Review P2 (invariant 3): the tree already moved, so a completion that died without a result + (exit 0/23 with no answer, SIGKILL/OOM) is an owed ``completion`` follow-up, never "Update failed". + + Was exit ``code or 1``, receipt ``failed``, ``.update_exit_code`` 1 and the paused gateways left + paused: that pinned the failure invariant 3 forbids. + """ + from hermes_cli import update_lock + + root, git, old, new, request = transition + die = f"os._exit({code})" if code >= 0 else "os.kill(os.getpid(), 9)" + (root / "hermes_cli/update_completion.py").write_text(f"import os\n{die}\n") + exit_code, receipt, out = _run_cmd_update(monkeypatch, request, capsys) + assert exit_code == 0, out assert receipt["update_id"] == request["receipt"]["update_id"] - assert request["windows_resume"]["resume_needed"] is True - assert (Path(request["home"]) / ".update_exit_code").read_text().strip() == "1" + assert receipt["outcome"] == "success" + assert receipt["exit_code"] == 0 + assert [row["step"] for row in receipt["followups"]] == ["completion"] + assert "Update failed" not in out + # The Desktop hand-off scripts must not read this as a plain "Update complete." (review P4). + assert "Desktop app build owed: the update completion did not finish" in out + # This process resumes what the lost child never did, and the gateway watcher hears success. + assert request["windows_resume"]["resume_needed"] is False + assert (Path(request["home"]) / ".update_exit_code").read_text().strip() == "0" lock = update_lock.UpdateLock() assert lock.acquire() lock.release() +def test_completion_spawn_failure_after_commit_is_owed(transition, monkeypatch, capsys): + """Review P2: an OSError while starting the completion (temporary directory, request file, + Popen) after the commit point is the same owed ``completion`` follow-up and exit 0.""" + import tempfile + + root, git, old, new, request = transition + + def no_temporary_directory(*args, **kwargs): + raise FileNotFoundError(2, "No usable temporary directory found") + + monkeypatch.setattr(tempfile, "TemporaryDirectory", no_temporary_directory) + exit_code, receipt, out = _run_cmd_update(monkeypatch, request, capsys) + assert exit_code == 0, out + assert receipt["outcome"] == "success" + assert [row["step"] for row in receipt["followups"]] == ["completion"] + assert "No usable temporary directory" in receipt["followups"][0]["reason"] + assert (Path(request["home"]) / ".update_exit_code").read_text().strip() == "0" + + +def test_lost_completion_with_parked_local_changes_stays_partial(transition, monkeypatch, capsys): + """The one documented exception survives the owed path: parked user changes still exit 1.""" + from hermes_cli import update_cmd + + root, git, old, new, request = transition + (root / "hermes_cli/update_completion.py").write_text("import os\nos._exit(0)\n") + notice = "⚠ Your local changes are parked in stash@{0}; re-apply them by hand." + monkeypatch.setattr(update_cmd, "_unrestored_autostash_notice", lambda: notice) + exit_code, receipt, out = _run_cmd_update(monkeypatch, request, capsys) + assert exit_code == 1, out + assert receipt["outcome"] == "partial" + assert [row["step"] for row in receipt["followups"]] == ["completion"] + assert notice in out + + @pytest.mark.platforms("posix") @pytest.mark.parametrize("never_built", ["dependencies", "refused", "died"]) @pytest.mark.parametrize("desktop", [True, False]) From 35042af076657d38cdeac2e709de7dca225a2775 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:52:51 -0700 Subject: [PATCH 34/62] fix(update): a raising fleet restart in the takeover child is owed, not exit 1 update_finish.finish_update (the historical takeover's completion) ran _restart_gateway_fleet_after_update, the Windows resume merge and _verify_fleet_after_update unguarded. They run after the commit point, yet a raise (a systemctl timeout, a service-manager error) reached update_finish.main's catch-all: record_step failed, receipt `failed`, .update_exit_code rewritten to 1, exit 1. The three calls now sit behind the same post-commit boundary as the current completion: the error is a `gateway_restart` follow-up (amended onto the terminal receipt when verification already finalized it), the fleet restart obligation stays armed for the next launch / `hermes update`, and the run exits 0 with the watcher's 0 intact. Review P3 (invariant 3). --- hermes_cli/update_finish.py | 28 +++++++++++++++++--------- tests/hermes_cli/test_update_finish.py | 21 +++++++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/hermes_cli/update_finish.py b/hermes_cli/update_finish.py index acb56066c935..e1aae542c899 100644 --- a/hermes_cli/update_finish.py +++ b/hermes_cli/update_finish.py @@ -11,17 +11,19 @@ def finish_update(*, root, assume_yes, gateway_mode, pre_update_snapshot_id, plan, windows_resume, followups=None) -> None: """Finish the selected checkout; never fetch, switch branches or restore a stash. - Same contract as the current completion (C3): the code is committed, so a failed build or - maintenance step is owed work, never a failed update. ``followups`` carries the steps that - already failed (the product build); while any is owed the install stamp is withheld, the - source-update tail stays pending for the next launch, and the gateway topology is left - alone. An unsafe SQLite runtime is reported by maintenance and vetoes migration only. + Same contract as the current completion (C3): the code is committed, so a failed build, + maintenance step or fleet restart/verification is owed work, never a failed update. + ``followups`` carries the steps that already failed (the product build); while any is owed + the install stamp is withheld, the source-update tail stays pending for the next launch, and + the gateway topology is left alone. An unsafe SQLite runtime is reported by maintenance and + vetoes migration only. """ from hermes_cli.update_cmd import ( _run_post_update_maintenance, _restart_gateway_fleet_after_update, _verify_fleet_after_update, _write_gateway_update_exit_code, _resume_windows_gateways_and_merge_outcome, ) + from hermes_cli import update_receipt from hermes_cli.update_receipt import TAIL_FOLLOWUPS, record_build_stage, record_followup owed = followups if followups is not None else [] @@ -53,10 +55,18 @@ def finish_update(*, root, assume_yes, gateway_mode, pre_update_snapshot_id, # committed, so the watcher sees success; owed work is on the receipt. if gateway_mode: _write_gateway_update_exit_code(True) - restarted = _restart_gateway_fleet_after_update(plan, gateway_mode) - _resume_windows_gateways_and_merge_outcome(restarted, windows_resume, gateway_mode) - _verify_fleet_after_update(restarted, _pre_update_plan=plan, _windows_gateway_resume=windows_resume, - update_complete=bool(runtime_safe) and not tail_owed) + run = update_receipt._current.get() + update_id = run.data.get("update_id") if run is not None else None + try: + restarted = _restart_gateway_fleet_after_update(plan, gateway_mode) + _resume_windows_gateways_and_merge_outcome(restarted, windows_resume, gateway_mode) + _verify_fleet_after_update(restarted, _pre_update_plan=plan, _windows_gateway_resume=windows_resume, + update_complete=bool(runtime_safe) and not tail_owed) + except (Exception, SystemExit) as exc: # health: allow BLE001 -- the code is committed (C3): the fleet obligation stays armed + reason = str(exc) or type(exc).__name__ + record_followup("gateway_restart", reason) + if update_receipt._current.get() is None and update_id: # verification already finalized the run + update_receipt.amend_terminal_followup(update_id, "gateway_restart", reason) def _restore_plan(data): diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index c244330e63e5..23fe4eceb5c9 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -203,6 +203,8 @@ def maintain(**kwargs): update._surviving_pre_update_serve_runtimes = lambda plan: [] fleet_verify._collect_fleet_snapshot = lambda *args: [] def restart(plan, gateway_mode): + if fault == 'restart': + raise RuntimeError('systemctl restart timed out') assert isinstance(plan, UpdatePlan) assert isinstance(plan.runtimes[0], RuntimeRecord) assert plan.to_dict() == request['plan'] | { @@ -302,6 +304,25 @@ def test_refused_gateway_resume_after_commit_is_a_followup_not_exit_1(completion assert json.loads(result.read_text())["resume_handled"] is True +@pytest.mark.platforms("posix") +def test_raising_fleet_restart_after_commit_is_a_followup_not_exit_1(completion): + """Review P3 (invariant 3): the takeover child's restart/verify run after the commit point, so + a raising fleet restart is an owed ``gateway_restart`` follow-up and exit 0, never exit 1 with a + ``failed`` receipt and a gateway watcher told 1.""" + source, home, request, context, result, run = completion + request["gateway_mode"] = True + request["receipt"]["started_at"] = "2026-10-04T12:00:00+00:00" + context.write_text(json.dumps(request), encoding="utf-8") + child = run("restart") + assert child.returncode == 0, child.stdout + child.stderr + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text()) + assert receipt["update_id"] == request["update_id"] + assert receipt["outcome"] == "success" + owed = {row["step"]: row["reason"] for row in receipt["followups"]} + assert "systemctl restart timed out" in owed.get("gateway_restart", ""), child.stdout + child.stderr + assert (home / ".update_exit_code").read_text().strip() == "0" + + @pytest.mark.platforms("posix") @pytest.mark.parametrize("captured, installed, expected", [ (None, None, False), From 964e3dc8fe522eb6f64948ff43937c6442e8a2bd Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:53:03 -0700 Subject: [PATCH 35/62] fix(update): source_build migration waivers say what actually happens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two BLE001 waivers around the memory-provider and left-core plugin migrations claimed the failure "is reported as a follow-up", but neither calls record_followup: the failure is printed as ⚠ only. Recording a follow-up here would be wrong for the install path that shares this builder (it would print "Update follow-up ..." during an install), and the migrations are re-run by every later update/tail anyway. Fix the wording to name the real behaviour instead. Review P5. --- hermes_cli/source_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermes_cli/source_build.py b/hermes_cli/source_build.py index 37e94aab5b4d..240170e7d6b6 100644 --- a/hermes_cli/source_build.py +++ b/hermes_cli/source_build.py @@ -180,7 +180,7 @@ def node_dependencies() -> None: from hermes_cli.memory_provider_migration import migrate_all_homes migrate_all_homes() - except Exception as exc: # health: allow BLE001 -- post-commit boundary: the failure is reported as a follow-up, never fails the committed update + except Exception as exc: # health: allow BLE001 -- post-commit boundary: printed as ⚠, never fails the committed update; every later update/tail re-runs it print(f" ⚠ Memory provider migration skipped: {exc}") # Same for a gateway platform / toolset that left core (Home Assistant): every home that used # it gets its catalog plugin (hermes_cli/left_core_migration.py). @@ -188,7 +188,7 @@ def node_dependencies() -> None: from hermes_cli.left_core_migration import migrate_all_homes as migrate_left_core migrate_left_core() - except Exception as exc: # health: allow BLE001 -- post-commit boundary: the failure is reported as a follow-up, never fails the committed update + except Exception as exc: # health: allow BLE001 -- post-commit boundary: printed as ⚠, never fails the committed update; every later update/tail re-runs it print(f" ⚠ Plugin migration skipped: {exc}") if failures: raise ProductBuildError(failures) From 506ea4b9c3356e93e69623150bbf3ba7e10d828a Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:53:21 -0700 Subject: [PATCH 36/62] fix(update): docs name the parked-stash exit 1 and the owed completion hermes_cli/AGENTS.md and cli-internals.md stated "after the commit point nothing fails" without the one documented exception (an autostash whose restore conflicted stays parked: `partial`, exit 1, #122557) and without the lost-completion outcome P2 introduces (a completion that could not start or died without a result is an owed `completion` follow-up, tail re-armed, exit 0). update_receipt's module docstring still said receipts live under ; they live under the ROOT home (get_default_hermes_root()), never a sticky profile's. Review P6. --- hermes_cli/AGENTS.md | 5 ++++- hermes_cli/update_receipt.py | 5 +++-- website/docs/developer-guide/cli-internals.md | 9 +++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/hermes_cli/AGENTS.md b/hermes_cli/AGENTS.md index 825fc63365aa..996c89418e47 100644 --- a/hermes_cli/AGENTS.md +++ b/hermes_cli/AGENTS.md @@ -137,7 +137,10 @@ The stage-by-stage contract: `website/docs/developer-guide/cli-internals.md` § - Restarts are fleet-wide and drain-first, bounded by the home each unit runs on (`update_fleet_scope.py`); another install's runtime is never restarted. - After the commit point nothing fails `hermes update`: a stale gateway or failed post-commit - step is a receipt `followup` with its obligation kept armed. Receipts live under the ROOT home's + step is a receipt `followup` with its obligation kept armed, and a completion process that + could not start or died without a result is an owed `completion` follow-up (tail re-armed), + exit 0. One exception: the user's autostash left parked is `partial`, exit 1 (#122557); + Ctrl-C after the commit point is `interrupted`, exit 130. Receipts live under the ROOT home's `logs/update_receipts/`. - Nothing runs pulled code in the pre-pull interpreter. `update_handoff.py` and `update_serve_obligations.py` are a FROZEN compat surface (`tests/compat/old_updater_surface.json`): diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index cb764ceeb6e7..e4b595f8fd75 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -8,7 +8,8 @@ 1. **Update receipt** — a machine-readable JSON record of what one ``hermes update`` run discovered, did, skipped (and why), written to - ``/logs/update_receipts/``. Silent-failure classes this + the ROOT home's ``logs/update_receipts/`` (``get_default_hermes_root()``, + never a sticky profile's ``HERMES_HOME``). Silent-failure classes this makes visible: #88848 (helper died after "success" printed), #74973 (restart silently skipped), #85753 (restart phase never ran), #81193 (desktop shows failure for a successful update). @@ -44,7 +45,7 @@ logger = logging.getLogger(__name__) -_RECEIPT_KEEP = 20 # keep the last N receipts per profile home +_RECEIPT_KEEP = 20 # keep the last N receipts in the root home's receipt directory #: Terminal records finalized by THIS process, by update id (see ``finalized_receipt``). _FINALIZED: dict[str, dict[str, Any]] = {} COMMAND_BOUNDARY_STOP_REASON = "completed at command boundary" diff --git a/website/docs/developer-guide/cli-internals.md b/website/docs/developer-guide/cli-internals.md index a4c848b0d615..5728006c804c 100644 --- a/website/docs/developer-guide/cli-internals.md +++ b/website/docs/developer-guide/cli-internals.md @@ -58,8 +58,13 @@ it guards. `plan → snapshot → apply → restart-per-kind → verify → repo and keep their own obligation armed (`source-completion-pending` for the tail, the fleet-restart obligation for gateways) so the next launch or `hermes update` — including "Already up to date" — retries them. The owed-fleet-restart CLI startup warning is what keeps a mixed-version fleet from - ever looking healthy. Exit 2 (refused/concurrent) and exit 1 (nothing committed / rolled back) - keep their meaning. + ever looking healthy. A completion process that could not be started (temporary directory, + request file, spawn) or died without a correlated result (OOM, SIGKILL) is the same kind of + debt: an owed `completion` follow-up with the source-update tail re-armed, exit 0. Exit 2 + (refused/concurrent) and exit 1 (nothing committed / rolled back) keep their meaning; the one + committed run that exits 1 is an autostash whose restore conflicted and stays parked: the + receipt is `partial` with a `user_action`, since nothing retries what only the user can re-apply + (#122557). - **Report**: every run writes a machine-readable receipt to the ROOT home's `logs/update_receipts/` (never a sticky profile's; `update.log` likewise) — `latest.json` pointer; steps, skips WITH reasons, restart outcome, plan, fleet snapshot, `followups`. The receipt is on From 87fbf03a0aaa8373494882b31603fad9d4a2dec9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 02:04:26 -0700 Subject: [PATCH 37/62] fix(update): a lost completion is owed without adopting a retired channel Follow-up to the lost-completion settle (review P2): - A retired channel is adopted only after a VERIFIED completion: a lost one exits 0 with the owed `completion` follow-up but leaves the config on the retired channel; the next `hermes update` resolves the retirement again and adopts it (the same retry the adoption-failure follow-up already names). - The settle triggers on a non-zero answer without a receipt (what run_completion returns for every answer that is not a correlated terminal receipt) and runs at most once per run (a spawn failure that could not finalize does not settle twice). Test changes (test_source_channel_integration.py), each pinning the failure invariant 3 forbids: - test_retirement_adopts_destination_only_after_success[failed]: the completion answered exit 1 without a receipt and the test expected SystemExit; it is now an owed completion (exit 0). The channel assertion is unchanged: the destination is still not adopted. - test_retirement_waits_for_correlated_completion_process[uncorrelated| missing]: same, an uncorrelated/missing answer was SystemExit and is now owed; the channel still stays on the retired one. [failure] (the child answered a correlated failed receipt) still raises. Review P2. --- hermes_cli/update_cmd.py | 15 ++++++++++----- .../test_source_channel_integration.py | 16 ++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index d9d4697f17ee..cb817be5f385 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -776,6 +776,7 @@ def _complete_source_update(request: dict | None) -> None: _write_fleet_restart_pending_marker(expected_sha=request.get("expected_sha") or _current_checkout_sha() or "") # Pre-swap module (imported with run_completion above): never the replacement tree's. from hermes_cli.update_completion import settle_lost_completion + lost = False try: result = run_completion(request) except KeyboardInterrupt as interrupt: @@ -783,12 +784,14 @@ def _complete_source_update(request: dict | None) -> None: _completion_receipt.finalize_interrupted_update_receipt("KeyboardInterrupt: interrupted after the code was updated") raise SystemExit(130) from interrupt except Exception as exc: # health: allow BLE001 -- after the commit point a lost completion is owed, never a failed update (C3) - result = settle_lost_completion(request, f"{type(exc).__name__}: {exc}") - if result.get("receipt") is None and result["exit_code"] == 130: + result, lost = settle_lost_completion(request, f"{type(exc).__name__}: {exc}"), True + if not lost and result.get("receipt") is None and result["exit_code"] == 130: _completion_receipt.finalize_interrupted_update_receipt("the completion was interrupted after the code was updated") - elif result.get("receipt") is None: - # The child crashed (OOM, SIGKILL) or never answered: the same owed tail (review P2). - result = settle_lost_completion(request, result.get("error") or f"the completion process exited {result['exit_code']}") + elif not lost and result.get("receipt") is None and result["exit_code"]: + # The child crashed (OOM, SIGKILL) or never answered (run_completion maps every answer + # without a correlated terminal receipt to non-zero): the same owed tail (review P2). + result, lost = settle_lost_completion( + request, result.get("error") or f"the completion process exited {result['exit_code']}"), True _accept_completion_pm_receipt(result.get("pm_receipt"), request["receipt"]["update_id"]) token = request["windows_resume"] if token is not None and result.get("windows_resume") is not None: @@ -813,6 +816,8 @@ def _complete_source_update(request: dict | None) -> None: _completion_receipt._current.reset(current.current_token) if result["exit_code"]: raise SystemExit(result["exit_code"]) + if lost: + return # a retired channel is adopted after a verified completion only; the next update adopts it try: if adopt_retired_channel(request): print(f"→ Source subscription moved to {request['channel_retirement']['destination']}") diff --git a/tests/hermes_cli/test_source_channel_integration.py b/tests/hermes_cli/test_source_channel_integration.py index 5695dc06c058..10d8704e1df8 100644 --- a/tests/hermes_cli/test_source_channel_integration.py +++ b/tests/hermes_cli/test_source_channel_integration.py @@ -125,11 +125,9 @@ def completion(request): monkeypatch.setattr(update_cmd, "_write_fleet_restart_pending_marker", lambda **kw: None) flags = ["--channel", name] if outcome == "transient" else [] args = source.parser.parse_args(["update", "--yes", *flags]) - if outcome == "failed": - with pytest.raises(SystemExit): - update_cmd._cmd_update_impl(args, False) - else: - update_cmd._cmd_update_impl(args, False) + # A completion that answered no terminal receipt ("failed") was SystemExit: after the commit + # point it is an owed completion, exit 0 (review P2) -- and still never adopts the destination. + update_cmd._cmd_update_impl(args, False) expected = {"success": "stable", "already-current": "stable", "failed": name, "transient": name, "concurrent": "my-new-choice"}[outcome] assert saved(source)["channel"] == expected @@ -430,11 +428,13 @@ def test_retirement_waits_for_correlated_completion_process(source, monkeypatch, "receipt": deepcopy(update_receipt._current.get().data), "channel_retirement": {"original": original, "destination": "stable"}} monkeypatch.setattr(update_cmd, "_write_fleet_restart_pending_marker", lambda **kw: None) - if outcome == "success": - update_cmd._complete_source_update(request) - else: + if outcome == "failure": # the child itself answered a correlated failure with pytest.raises(SystemExit): update_cmd._complete_source_update(request) + else: + # An uncorrelated or missing answer was SystemExit: after the commit point it is an owed + # completion, exit 0 (review P2). Adoption still waits for a verified completion. + update_cmd._complete_source_update(request) assert saved(source)["channel"] == ("stable" if outcome == "success" else name) child_request = json.loads((source.home / "child-request.json").read_text()) assert child_request["channel_retirement"]["original"] == original From f179b2f7165805faa9f8959f3388114044a54c46 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 02:04:26 -0700 Subject: [PATCH 38/62] fix(update): new post-commit tests use explicit encodings The tests added for review P1-P4 wrote and read text without an encoding (or read with BOM-intolerant utf-8) and used a bare signal.SIGKILL in a child script, which check-windows-footguns flags (invariant 5). Writes now pass encoding="utf-8", reads "utf-8-sig", and the posix-only child kills itself with signal 9 by number. Review P1-P4 (invariant 5 hygiene for the new tests). --- .../test_update_completion_process.py | 22 ++++++++++--------- tests/hermes_cli/test_update_finish.py | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index 032092faadca..9b3fb3dd0f1f 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -284,7 +284,7 @@ def test_missing_child_result_is_an_owed_completion_and_releases_lock(transition root, git, old, new, request = transition die = f"os._exit({code})" if code >= 0 else "os.kill(os.getpid(), 9)" - (root / "hermes_cli/update_completion.py").write_text(f"import os\n{die}\n") + (root / "hermes_cli/update_completion.py").write_text(f"import os\n{die}\n", encoding="utf-8") exit_code, receipt, out = _run_cmd_update(monkeypatch, request, capsys) assert exit_code == 0, out assert receipt["update_id"] == request["receipt"]["update_id"] @@ -296,7 +296,7 @@ def test_missing_child_result_is_an_owed_completion_and_releases_lock(transition assert "Desktop app build owed: the update completion did not finish" in out # This process resumes what the lost child never did, and the gateway watcher hears success. assert request["windows_resume"]["resume_needed"] is False - assert (Path(request["home"]) / ".update_exit_code").read_text().strip() == "0" + assert (Path(request["home"]) / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0" lock = update_lock.UpdateLock() assert lock.acquire() lock.release() @@ -318,7 +318,7 @@ def no_temporary_directory(*args, **kwargs): assert receipt["outcome"] == "success" assert [row["step"] for row in receipt["followups"]] == ["completion"] assert "No usable temporary directory" in receipt["followups"][0]["reason"] - assert (Path(request["home"]) / ".update_exit_code").read_text().strip() == "0" + assert (Path(request["home"]) / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0" def test_lost_completion_with_parked_local_changes_stays_partial(transition, monkeypatch, capsys): @@ -326,7 +326,7 @@ def test_lost_completion_with_parked_local_changes_stays_partial(transition, mon from hermes_cli import update_cmd root, git, old, new, request = transition - (root / "hermes_cli/update_completion.py").write_text("import os\nos._exit(0)\n") + (root / "hermes_cli/update_completion.py").write_text("import os\nos._exit(0)\n", encoding="utf-8") notice = "⚠ Your local changes are parked in stash@{0}; re-apply them by hand." monkeypatch.setattr(update_cmd, "_unrestored_autostash_notice", lambda: notice) exit_code, receipt, out = _run_cmd_update(monkeypatch, request, capsys) @@ -345,21 +345,23 @@ def test_desktop_build_that_never_ran_is_named_for_the_handoff(transition, capsy sync failed, the tail refused by the update lock, the prepared child died) must print it too.""" scripts = Path(update_completion.__file__).parents[1] / "scripts/desktop-update" for script in ("posix.sh", "windows.ps1"): # the exact prefix the hand-off scripts match - assert "Desktop app build owed: " in (scripts / script).read_text(encoding="utf-8") + assert "Desktop app build owed: " in (scripts / script).read_text(encoding="utf-8-sig") assert update_completion.DESKTOP_BUILD_OWED == "Desktop app build owed:" root, git, old, new, request = transition request["desktop"] = desktop shutil.copy2(update_completion.__file__, root / "hermes_cli/update_completion.py") if never_built == "dependencies": - (root / "pm/__init__.py").write_text("def sync_venv(**kw): raise RuntimeError('dependency refused')\n") + (root / "pm/__init__.py").write_text("def sync_venv(**kw): raise RuntimeError('dependency refused')\n", + encoding="utf-8") elif never_built == "refused": (root / "hermes_cli/source_completion.py").write_text( "def complete_source_checkout(*args, **kwargs):\n" - " raise RuntimeError('an update is still running (pid 1); wait for it to exit')\n") + " raise RuntimeError('an update is still running (pid 1); wait for it to exit')\n", encoding="utf-8") else: (root / "hermes_cli/source_completion.py").write_text( - "import os, signal\n" - "def complete_source_checkout(*args, **kwargs): os.kill(os.getpid(), signal.SIGKILL)\n") + "import os\n" + "def complete_source_checkout(*args, **kwargs): os.kill(os.getpid(), 9) # posix-only test\n", + encoding="utf-8") result = update_completion.run_completion(request) out = capsys.readouterr().out assert result["exit_code"] == 0, out @@ -381,7 +383,7 @@ def test_unwritable_gateway_status_never_fails_a_settled_commit(tmp_path, monkey request = {"source": str(tmp_path), "receipt": {"update_id": "u1"}, "gateway_mode": True, "home": str(home), "pm_receipt": None} assert update_completion._settle_after_commit(request, result, "dependencies", "pip failed") == 0 - answer = json.loads(result.read_text(encoding="utf-8")) + answer = json.loads(result.read_text(encoding="utf-8-sig")) assert answer["exit_code"] == 0 and answer["receipt"] == receipt diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index 23fe4eceb5c9..3f2f6b41eb6b 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -315,12 +315,12 @@ def test_raising_fleet_restart_after_commit_is_a_followup_not_exit_1(completion) context.write_text(json.dumps(request), encoding="utf-8") child = run("restart") assert child.returncode == 0, child.stdout + child.stderr - receipt = json.loads((home / "logs/update_receipts/latest.json").read_text()) + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text(encoding="utf-8-sig")) assert receipt["update_id"] == request["update_id"] assert receipt["outcome"] == "success" owed = {row["step"]: row["reason"] for row in receipt["followups"]} assert "systemctl restart timed out" in owed.get("gateway_restart", ""), child.stdout + child.stderr - assert (home / ".update_exit_code").read_text().strip() == "0" + assert (home / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0" @pytest.mark.platforms("posix") From eb0d19f3fe4f751c26944044908fe6e679d32431 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:15:17 -0700 Subject: [PATCH 39/62] fix(update): an unmapped stopped gateway's restart debt needs its own successor (C13) What: an inventoried marker that owes an unmapped gateway (one the update stopped without a profile it could name) now clears only when a local gateway no mapped owed identity explains is running the checkout. The old guard only refused an all-external fleet. Why: with `default` and an unmapped gateway both owed, `default` coming back current cleared the whole marker, so the unmapped gateway stayed down with no warning and no owed restart. The new rule subsumes the all-external refusal (no local row can be the successor). Red: test_unmapped_stop_debt_is_not_settled_by_the_mapped_gateways_restart -> owes False with only `default` current (review probe f7_mixed: marker cleared, pid 101 still down). Green: owed while only `default` runs; settled once a further local gateway runs the checkout; 14 fleet/obligation files, 124 passed. --- hermes_cli/update_cmd_fleet.py | 7 ++++-- .../test_update_fleet_check_fail_closed.py | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index 3d5cd4d1dc84..5ca7fd6768dc 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -457,8 +457,11 @@ def _marker_only_restart_obsolete() -> bool: return False # stale / down / unknown-identity row still owes the restart if owed is not None and not owed - {_UNMAPPED_GATEWAY} <= covered: return False # A gateway this marker owns is absent (down) or unidentifiable. - if owed and _UNMAPPED_GATEWAY in owed and all(row_is_external(row) for row in fleet): - return False # another checkout's gateway is no successor for the one the update stopped + if owed and _UNMAPPED_GATEWAY in owed and not (_fleet_covered_gateways( + [row for row in fleet if not row_is_external(row)]) or set()) - owed: + # An unmapped stopped gateway has no identity to match, so only a local gateway that no + # mapped owed one explains can be its successor; another checkout's gateway never is. + return False _clear_fleet_restart_pending_marker() logger.debug( "Fleet-restart-pending marker discharged: %d gateway(s) already serve %s", diff --git a/tests/hermes_cli/test_update_fleet_check_fail_closed.py b/tests/hermes_cli/test_update_fleet_check_fail_closed.py index 8b10342837fd..c6de5605ef16 100644 --- a/tests/hermes_cli/test_update_fleet_check_fail_closed.py +++ b/tests/hermes_cli/test_update_fleet_check_fail_closed.py @@ -173,3 +173,26 @@ def test_unmapped_stop_debt_survives_startup_until_a_current_gateway_runs(monkey fleet_rows.append({"profile": "default", "state": "current", "code_sha": "head"}) # `hermes gateway run` assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is False assert not fleet._fleet_restart_obligation_armed() + + +def test_unmapped_stop_debt_is_not_settled_by_the_mapped_gateways_restart(monkeypatch, tmp_path): + # The inventory owes `default` AND an unmapped gateway: `default` coming back current is the + # successor of `default`, not of the unmapped one, so the debt stays until a further local + # gateway (the unmapped one's successor, whatever profile it names) runs the checkout. + from hermes_cli import update_cmd_fleet as fleet, update_host_obligation as host, update_receipt + + monkeypatch.setattr(host, "host_obligation_path", lambda: tmp_path / "host-update-restart.json") + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "head") + fleet._write_fleet_restart_pending_marker(expected_sha="head", runtimes=[ + {"kind": "gateway", "profile": "default", "pid": 7}, + {"kind": "gateway", "profile": None, "pid": 101, "stopped_unmapped": True}, + ]) + fleet_rows = [{"profile": "default", "state": "current", "code_sha": "head", "pid": 8}] + monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: list(fleet_rows)) + + assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is True + assert fleet._fleet_restart_obligation_armed() + + fleet_rows.append({"profile": "work", "state": "current", "code_sha": "head", "pid": 9}) + assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is False + assert not fleet._fleet_restart_obligation_armed() From 05355f76565e340b6229768a4566497bfbc53a2f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:59:58 -0700 Subject: [PATCH 40/62] docs(update): hermes_cli/AGENTS.md states the post-commit exit rule and points at its table The full outcome list (stale gateway, owed completion, re-armed tail) lives in developer-guide/source-update-completion.md; the AGENTS.md chain stays under its 30,000-char cap. --- hermes_cli/AGENTS.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/hermes_cli/AGENTS.md b/hermes_cli/AGENTS.md index 996c89418e47..9b7ce0f94da3 100644 --- a/hermes_cli/AGENTS.md +++ b/hermes_cli/AGENTS.md @@ -136,11 +136,9 @@ The stage-by-stage contract: `website/docs/developer-guide/cli-internals.md` § `_ZIP_PRESERVED_NESTED` build outputs into the swap. - Restarts are fleet-wide and drain-first, bounded by the home each unit runs on (`update_fleet_scope.py`); another install's runtime is never restarted. -- After the commit point nothing fails `hermes update`: a stale gateway or failed post-commit - step is a receipt `followup` with its obligation kept armed, and a completion process that - could not start or died without a result is an owed `completion` follow-up (tail re-armed), - exit 0. One exception: the user's autostash left parked is `partial`, exit 1 (#122557); - Ctrl-C after the commit point is `interrupted`, exit 130. Receipts live under the ROOT home's +- After the commit point nothing fails `hermes update`: failures are receipt follow-ups with + their obligations kept armed, exit 0, except a parked autostash (`partial`, exit 1, #122557) and + Ctrl-C (`interrupted`, 130); table in `developer-guide/source-update-completion.md`. Receipts live under the ROOT home's `logs/update_receipts/`. - Nothing runs pulled code in the pre-pull interpreter. `update_handoff.py` and `update_serve_obligations.py` are a FROZEN compat surface (`tests/compat/old_updater_surface.json`): From 1b51277533d94a42a3a670bbc6b33a6b68c6c314 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:10:54 -0700 Subject: [PATCH 41/62] fix(update): every owed follow-up reaches the Desktop hand-off result A committed update exits 0 and prints one whole "Update follow-up '' did not finish:" line per owed step (update_receipt.record_followup). The hand-offs only reacted to the Desktop build line, so a gateway proven to run old code (gateway_restart), a refused Windows resume, a lost completion or a failed channel adoption ended as a plain "Update complete.". posix.sh collects the distinct owed steps from that line (owed_followup_steps) and turns them into the manual result message (DONE_NOTE, the only channel the main-based script has); a gateway_restart adds the `hermes gateway restart` hint. windows.ps1 does the same through its manual outcome. With #132354's add_warning protocol the same owed_followup_steps output feeds one add_warning per step. record_followup's docstring names the line as a protocol; a contract test runs the real posix.sh parser and the windows.ps1 regex against it. Review: kshitijk4poor 5411222454 regression 1 / open thread F6. --- hermes_cli/update_receipt.py | 4 ++- scripts/desktop-update/posix.sh | 17 +++++++++++++ scripts/desktop-update/windows.ps1 | 12 +++++++++ .../test_update_post_commit_followups.py | 21 ++++++++++++++++ .../test_desktop_update_gateway_flag.py | 25 +++++++++++++++++++ 5 files changed, 78 insertions(+), 1 deletion(-) diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index e4b595f8fd75..6f6c10078e06 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -353,7 +353,9 @@ def record_followup(step: str, reason: str, *, retry: str = "the next launch or The code already committed, so the update does not fail; the step's own obligation (the source tail, the fleet restart, the launch-time bytecode sweep) stays armed and the next - launch or ``hermes update`` retries it. Never raises. + launch or ``hermes update`` retries it. Never raises. The printed line is a protocol: the + Desktop hand-offs (scripts/desktop-update/posix.sh, windows.ps1) parse ``Update follow-up + '' did not finish:`` to report every owed step, so keep it one whole line. """ reason = " ".join(str(reason).split())[:500] or "failed" print(f" ⚠ Update follow-up '{step}' did not finish: {reason} ({retry})", flush=True) diff --git a/scripts/desktop-update/posix.sh b/scripts/desktop-update/posix.sh index aad25662d0a7..76e7cc8f311e 100755 --- a/scripts/desktop-update/posix.sh +++ b/scripts/desktop-update/posix.sh @@ -93,6 +93,12 @@ APP_REBUILD_FAILED=0 # 1 = the code committed but the Desktop app build is an o log() { echo "$(date +%Y-%m-%dT%H:%M:%S%z) $1" | tee -a "$LOG" 2>/dev/null; } +owed_followup_steps() { # $OUT -> the distinct owed steps, space-separated, in print order + # hermes_cli/update_receipt.record_followup prints each as one whole line. + printf '%s\n' "$OUT" | sed -n "s/^.*Update follow-up '\([A-Za-z0-9_]*\)' did not finish: .*$/\1/p" \ + | awk '!seen[$0]++' | tr '\n' ' ' | sed 's/ *$//' +} + # Keep a durable signal breadcrumb. A detached hand-off used to leave only the # generic FINAL_MSG when it was terminated while the updater child was running, # which erased the one fact needed to diagnose the failure. @@ -899,6 +905,17 @@ if [ "$CODE" -eq 0 ]; then FINAL_CODE=0 FINAL_MSG="Update complete." DONE_NOTE="Hermes was updated, but the Desktop app could not be rebuilt, so it still runs its old build. Run hermes desktop --force-build in a terminal to rebuild it; the update log has the build error." log "desktop app build is an owed follow-up of the committed update" fi + # Every other owed follow-up (a gateway still on the old code, a Windows + # resume, a lost completion, channel adoption, maintenance...) prints one + # whole "Update follow-up '' did not finish:" line. None of them may + # end as a plain "Update complete." (review regression 1). + OWED_STEPS="$(owed_followup_steps)" + if [ -n "$OWED_STEPS" ]; then + case " $OWED_STEPS " in *" gateway_restart "*) OWED_HINT=" Run hermes gateway restart to move the messaging gateway onto the new code now." ;; *) OWED_HINT="" ;; esac + if [ -n "$DONE_NOTE" ]; then DONE_NOTE="$DONE_NOTE Follow-up steps still owed: $OWED_STEPS.$OWED_HINT" + else DONE_NOTE="Hermes was updated, but some follow-up steps did not finish ($OWED_STEPS). The next launch or hermes update retries them; the update log has the details.$OWED_HINT"; fi + log "owed follow-ups of the committed update: $OWED_STEPS" + fi else FINAL_CODE="$CODE" FINAL_MSG="Update failed (exit $CODE). Run hermes debug share in a terminal to send a report." # The bricked-venv class is fixable and must not read as a generic exit 1: diff --git a/scripts/desktop-update/windows.ps1 b/scripts/desktop-update/windows.ps1 index 6b97693f5d59..e50f058f1d07 100644 --- a/scripts/desktop-update/windows.ps1 +++ b/scripts/desktop-update/windows.ps1 @@ -1740,6 +1740,18 @@ try { Write-HandoffLog $manualMsg } + # Every other owed follow-up of the committed update (a gateway still on the old code, a + # Windows resume, a lost completion...) prints one whole "Update follow-up '' did not + # finish:" line (hermes_cli/update_receipt.record_followup): never a plain success either. + $owedSteps = @([regex]::Matches(($res.Output -join "`n"), "Update follow-up '([A-Za-z0-9_]+)' did not finish: ") | + ForEach-Object { $_.Groups[1].Value } | Select-Object -Unique) + if ($res.Code -eq 0 -and -not $desktopBuildFailed -and $owedSteps.Count -gt 0) { + $manualAction = $true + $owedHint = if ($owedSteps -contains 'gateway_restart') { ' Run `hermes gateway restart` to move the messaging gateway onto the new code now.' } else { '' } + $manualMsg = ($manualMsg + " Hermes was updated, but some follow-up steps did not finish (" + ($owedSteps -join ', ') + "). The next launch or ``hermes update`` retries them; the update log has the details." + $owedHint).Trim() + Write-HandoffLog $manualMsg + } + if ($res.Code -eq 0 -and -not $desktopBuildFailed) { $finalCode = 0 $finalMsg = "Update complete." diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py index f07ed21e03b0..daee3ea287fa 100644 --- a/tests/hermes_cli/test_update_post_commit_followups.py +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -197,3 +197,24 @@ def test_owed_restart_rearms_a_settled_obligation_and_a_later_run_keeps_owing_it # A later verify with nothing live to restart still owes the named gateway (and keeps it armed). assert fleet_verify._named_gateways_still_owed() assert fleet._fleet_restart_obligation_armed() + + +def test_followup_line_is_the_one_both_desktop_handoffs_parse(tmp_path, monkeypatch, capsys): + """The hand-offs read owed follow-ups from this printed line (review regression 1): pin the + producer against the exact parsers in posix.sh (sed) and windows.ps1 (.NET regex).""" + import re + from pathlib import Path + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.record_followup("gateway_restart", "x" * 900) + update_receipt.record_followup("windows_resume", "Windows gateway recovery failed: 'quoted' reason") + out = capsys.readouterr().out + scripts = Path(__file__).resolve().parents[2] / "scripts/desktop-update" + posix = (scripts / "posix.sh").read_text(encoding="utf-8") + function = re.search(r"^owed_followup_steps\(\) \{.*?^\}", posix, re.S | re.M).group(0) + steps = subprocess.run(["bash", "-c", f"{function}\nowed_followup_steps"], env={**os.environ, "OUT": out}, + capture_output=True, text=True, encoding="utf-8", check=True).stdout.strip() + assert steps == "gateway_restart windows_resume" + pattern = re.search(r"\[regex\]::Matches\(\(\$res\.Output -join \"`n\"\), \"([^\"]+)\"\)", + (scripts / "windows.ps1").read_text(encoding="utf-8")).group(1) + assert [m.group(1) for m in re.finditer(pattern, out)] == ["gateway_restart", "windows_resume"] diff --git a/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py b/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py index 8f3d05b9185e..3905d819b6bc 100644 --- a/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py +++ b/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py @@ -108,3 +108,28 @@ def test_owed_desktop_build_is_a_manual_outcome_even_when_the_followup_is_trunca assert result["exit_code"] == 0 assert result["manual"] is True assert "could not be rebuilt" in result["message"] + + +@requires_posix_handoff +@pytest.mark.parametrize("step", ["gateway_restart", "windows_resume", "channel_adoption", "completion"]) +def test_every_owed_followup_reaches_the_desktop_result(tmp_path, step): + """Contract C3 (review regression 1): a committed update exits 0, but a follow-up it still owes + (a gateway proven to run old code, a refused resume, a lost completion) must never end as a + plain "Update complete." -- the result names the owed step.""" + line = f" ⚠ Update follow-up '{step}' did not finish: old code still serving (the next launch retries it)" + _run_handoff(tmp_path, [], f"{line}\n✓ Update complete!") + + result = json.loads((tmp_path / ".hermes-update-result.json").read_text(encoding="utf-8-sig")) + assert result["exit_code"] == 0 and result["ok"] is True + assert result["manual"] is True + assert step in result["message"] + assert ("hermes gateway restart" in result["message"]) is (step == "gateway_restart") + + +@requires_posix_handoff +def test_a_clean_update_stays_plain_success(tmp_path): + _run_handoff(tmp_path, [], "✓ Update complete!") + + result = json.loads((tmp_path / ".hermes-update-result.json").read_text(encoding="utf-8-sig")) + assert result["exit_code"] == 0 and result["manual"] is False + assert result["message"] == "Update complete." From 9586d9934c83ec91a7af598db9694062ee742d42 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:11:12 -0700 Subject: [PATCH 42/62] fix(update): the gateway /update status follows a committed run's receipt After the run's receipt closed as `success`, three boundaries could still write 1 to .update_exit_code (and the completion child could still answer 130), so the gateway watcher reported a failed update the receipt calls a success: - cmd_update's catch-alls (main.py, unchanged) call _write_gateway_update_exit_code(False) for any escaping exception. The writer now keeps 0 when this command's run is closed as a success (update_receipt.committed_success: the receipt scope records the terminal outcome it finalized, or adopted from the completion child). - The parent adopts the child's terminal receipt and drops its stale pre-child context as soon as the result is accepted, before the parent- side Windows resume, so nothing after it can write that context. - update_completion._finish: an interrupt that lands after verification finalized the run answers 0 (the receipt's verdict); the 1 for a parked stash (partial) or an interrupt before the run closed is unchanged, and the status is written once the code is settled. - update_finish (historical takeover): the same rule for its finally. An interrupted run (receipt `interrupted`, exit 130) still writes 1, as test_interrupt_after_child_success_demotes_gateway_marker_at_boundary pins. Review: kshitijk4poor 5411222454 regression 3 / open thread F4. --- hermes_cli/update_cmd.py | 12 ++-- hermes_cli/update_cmd_fleet.py | 5 ++ hermes_cli/update_completion.py | 10 +-- hermes_cli/update_finish.py | 7 +- hermes_cli/update_receipt.py | 25 +++++++ tests/hermes_cli/test_update_finish.py | 21 ++++++ .../test_update_post_commit_followups.py | 66 +++++++++++++++++++ 7 files changed, 136 insertions(+), 10 deletions(-) diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index cb817be5f385..6d0929508622 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -793,6 +793,14 @@ def _complete_source_update(request: dict | None) -> None: result, lost = settle_lost_completion( request, result.get("error") or f"the completion process exited {result['exit_code']}"), True _accept_completion_pm_receipt(result.get("pm_receipt"), request["receipt"]["update_id"]) + if result.get("receipt") is not None: + # The child closed the run: drop the stale pre-child context before anything below can + # write it (a later follow-up amends the terminal archive), and let the command boundary + # answer the gateway status from the receipt (C3, review regression 3). + _completion_receipt.adopt_terminal_receipt(result["receipt"]) + current = _completion_receipt._current.get() + if current is not None: + _completion_receipt._current.reset(current.current_token) token = request["windows_resume"] if token is not None and result.get("windows_resume") is not None: resumed = dict(result["windows_resume"]) @@ -810,10 +818,6 @@ def _complete_source_update(request: dict | None) -> None: record_followup("windows_resume", reason) amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", reason) _settle_windows_resume(request) - if result.get("receipt") is not None: - current = _completion_receipt._current.get() - if current is not None: - _completion_receipt._current.reset(current.current_token) if result["exit_code"]: raise SystemExit(result["exit_code"]) if lost: diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index 5ca7fd6768dc..f37e91eb67a1 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -39,6 +39,11 @@ def _write_gateway_update_exit_code(ok: bool) -> None: from hermes_cli.update_cmd import get_hermes_home + from hermes_cli.update_receipt import committed_success + + # A run whose receipt already closed as a success is never reported failed (C3, review + # regression 3): the command boundary's catch-alls call this with False. + ok = ok or committed_success() path = get_hermes_home() / ".update_exit_code" with suppress(OSError): path.write_text("0" if ok else "1", encoding="utf-8") diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index bcd6b3cd620c..6ab8b6d1e6d4 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -454,9 +454,6 @@ def _finish(request: dict, result_path: Path) -> int: _report_unbuilt_desktop(request) update_receipt.record_followup("completion", f"{type(exc).__name__}: {exc}") finally: - if code and request["gateway_mode"]: - from hermes_cli.update_cmd import _write_gateway_update_exit_code - _write_gateway_update_exit_code(False) # The new interpreter owns recovery too. The original parent's atexit # token is updated from the response; it acts only if this process dies. try: @@ -469,8 +466,13 @@ def _finish(request: dict, result_path: Path) -> int: update_receipt.amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", step_reason) update_receipt.finalize_pending_update_receipt(code, reason) terminal_receipt = _read_terminal_receipt(request) + if terminal_receipt and terminal_receipt.get("outcome") == "success": + code = 0 # an interrupt that landed after verification closed the run (review regression 3) + if code and request["gateway_mode"]: + from hermes_cli.update_cmd import _write_gateway_update_exit_code + _write_gateway_update_exit_code(False) if not terminal_receipt: - code = code or 1 + code = code or 1 # the parent settles a lost result (settle_lost_completion) _write_json(result_path, { "schema": 1, "update_id": request["receipt"]["update_id"], "exit_code": code, "receipt": terminal_receipt, "windows_resume": request["windows_resume"], diff --git a/hermes_cli/update_finish.py b/hermes_cli/update_finish.py index e1aae542c899..beb386eb45d5 100644 --- a/hermes_cli/update_finish.py +++ b/hermes_cli/update_finish.py @@ -167,12 +167,15 @@ def main(context: Path, result: Path) -> int: update_receipt.record_step("historical_completion", False, str(exc)) print(f"Update completion failed: {exc}", file=sys.stderr, flush=True) finally: + terminal = update_receipt.finalized_receipt(request["update_id"]) or {} if code and request.get("gateway_mode"): - # Even an application import failure must wake the gateway watcher. + # Even an application import failure must wake the gateway watcher. A run that already + # closed as a success (an interrupt after verification) stays 0 (review regression 3). from hermes_constants import get_hermes_home from hermes_cli.runtime_state import _atomic_bytes - _atomic_bytes(get_hermes_home() / ".update_exit_code", b"1") + _atomic_bytes(get_hermes_home() / ".update_exit_code", + b"0" if terminal.get("outcome") == "success" else b"1") if restarting and not cli_started: # Startup failed before the replacement command could own a # receipt. Preserve the original handoff, just like preparation. diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index 6f6c10078e06..9aadd1117e21 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -58,18 +58,42 @@ _current: contextvars.ContextVar[Optional["UpdateReceipt"]] = contextvars.ContextVar( "update_receipt_current", default=None ) +#: The terminal outcome of the run the enclosing command scope finalized or adopted (from a +#: completion child): ``committed_success`` answers the command boundary's gateway status from it. +_scope_terminal: contextvars.ContextVar[Optional[dict[str, Any]]] = contextvars.ContextVar( + "update_receipt_scope_terminal", default=None +) @contextmanager def update_receipt_scope(): """Keep the command's finalization guard away from an enclosing update.""" token = _current.set(None) + terminal = _scope_terminal.set({}) try: yield finally: + _scope_terminal.reset(terminal) _current.reset(token) +def adopt_terminal_receipt(data: Any) -> None: + """A completion child finalized this command's run: its terminal outcome is this scope's too.""" + cell = _scope_terminal.get() + if cell is not None and isinstance(data, dict) and data.get("finished_at"): + cell["outcome"] = data.get("outcome") + + +def committed_success() -> bool: + """True when this command's run is closed and its receipt says ``success`` (C3). + + The gateway ``/update`` status written at the command boundary follows the receipt: an + interrupt or error that escapes after the run closed as a success never reports it failed. + """ + cell = _scope_terminal.get() + return _current.get() is None and bool(cell) and cell.get("outcome") == "success" + + def current_correlation_id() -> Optional[str]: """The update correlation id in force in this context, or None. @@ -475,6 +499,7 @@ def finalize_update_receipt(outcome: str, fleet: list | None = None, stop_reason # The terminal facts exist before the store is touched; a store that refuses the write # (after the commit point) must not erase them for this process's own caller. _FINALIZED[str(receipt.data.get("update_id"))] = receipt.data + adopt_terminal_receipt(receipt.data) if fleet is not None: receipt.data["fleet"] = fleet # Manual serve restart obligations outlive one receipt rotation: carry the previous diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index 3f2f6b41eb6b..fe2874c2e048 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -215,6 +215,12 @@ def restart(plan, gateway_mode): restarted_services=[], failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set()) update._restart_gateway_fleet_after_update = restart + if fault == 'interrupt': + verify = update._verify_fleet_after_update + def verify_then_interrupt(*args, **kwargs): + verify(*args, **kwargs) # closes the run as a success + raise KeyboardInterrupt() + update._verify_fleet_after_update = verify_then_interrupt merge = update._resume_windows_gateways_and_merge_outcome def resume(outcome, token, gateway_mode): assert token == request['windows_resume'] @@ -323,6 +329,21 @@ def test_raising_fleet_restart_after_commit_is_a_followup_not_exit_1(completion) assert (home / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0" +@pytest.mark.platforms("posix") +def test_interrupt_after_verification_closed_the_run_keeps_gateway_status_0(completion): + """Review regression 3: verification already finalized the takeover's run as ``success``; an + interrupt landing after it must not rewrite the gateway /update status to 1.""" + source, home, request, context, result, run = completion + request["gateway_mode"] = True + request["receipt"]["started_at"] = "2026-10-04T12:00:00+00:00" + context.write_text(json.dumps(request), encoding="utf-8") + child = run("interrupt") + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text(encoding="utf-8-sig")) + assert receipt["update_id"] == request["update_id"] + assert receipt["outcome"] == "success", child.stdout + child.stderr + assert (home / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0", child.stdout + child.stderr + + @pytest.mark.platforms("posix") @pytest.mark.parametrize("captured, installed, expected", [ (None, None, False), diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py index daee3ea287fa..522bfedde00d 100644 --- a/tests/hermes_cli/test_update_post_commit_followups.py +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -218,3 +218,69 @@ def test_followup_line_is_the_one_both_desktop_handoffs_parse(tmp_path, monkeypa pattern = re.search(r"\[regex\]::Matches\(\(\$res\.Output -join \"`n\"\), \"([^\"]+)\"\)", (scripts / "windows.ps1").read_text(encoding="utf-8")).group(1) assert [m.group(1) for m in re.finditer(pattern, out)] == ["gateway_restart", "windows_resume"] + + +def test_interrupt_after_the_run_closed_as_success_never_tells_the_gateway_1(tmp_path, monkeypatch): + """Review regression 3, the ``cmd_update`` boundary: once this run's receipt says ``success``, + an interrupt escaping afterwards must not write 1 to the gateway /update status.""" + from types import SimpleNamespace + from hermes_cli import main, update_cmd + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) + monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) + monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) + + def committed_then_interrupted(args, gateway_mode): + update_receipt.begin_update_receipt() + update_receipt.finalize_update_receipt("success") + raise KeyboardInterrupt() + + monkeypatch.setattr(update_cmd, "_cmd_update_impl", committed_then_interrupted) + with pytest.raises(KeyboardInterrupt): + main.cmd_update(SimpleNamespace(gateway=True)) + assert _latest(tmp_path)["outcome"] == "success" + assert (tmp_path / ".update_exit_code").read_text(encoding="utf-8").strip() == "0" + + +def test_a_failure_before_the_run_closed_still_tells_the_gateway_1(tmp_path, monkeypatch): + from types import SimpleNamespace + from hermes_cli import main, update_cmd + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) + monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) + monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) + + def interrupted_while_open(args, gateway_mode): + update_receipt.begin_update_receipt() + raise KeyboardInterrupt() + + monkeypatch.setattr(update_cmd, "_cmd_update_impl", interrupted_while_open) + with pytest.raises(KeyboardInterrupt): + main.cmd_update(SimpleNamespace(gateway=True)) + assert (tmp_path / ".update_exit_code").read_text(encoding="utf-8").strip() == "1" + + +def test_completion_child_interrupted_after_verification_answers_its_success(tmp_path, monkeypatch): + """Review regression 3, the selected completion child: verification finalized ``success`` and + then an interrupt landed. Exit, result and gateway status follow the receipt (0), not 130/1.""" + from hermes_cli import update_completion + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.begin_update_receipt() + data = dict(update_receipt._current.get().data) + update_receipt._current.set(None) + + def verified_then_interrupted(request): + update_receipt.finalize_update_receipt("success") + raise KeyboardInterrupt() + + monkeypatch.setattr(update_completion, "_complete_selected", verified_then_interrupted) + request = {"receipt": data, "pm_receipt": None, "gateway_mode": True, "windows_resume": None} + result_path = tmp_path / "result.json" + assert update_completion._finish(request, result_path) == 0 + answer = json.loads(result_path.read_text(encoding="utf-8")) + assert answer["exit_code"] == 0 and answer["receipt"]["outcome"] == "success" + assert not (tmp_path / ".update_exit_code").exists() or \ + (tmp_path / ".update_exit_code").read_text(encoding="utf-8").strip() == "0" From be62eeb9824dae3df6af74bd85c2dc156a1d8ceb Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:11:12 -0700 Subject: [PATCH 43/62] fix(update): the prepared completion child runs in UTF-8 mode too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 89e06e62dbe gave the bootstrap completion child -X utf8; the prepared child (_prepare -> --prepared, which runs _complete_selected and prints the ✓/⚠ follow-up protocol into the same pipe) still started with -I and no UTF-8 mode, so it relied on hermes_cli's import-time stdio repair alone. Both children now carry -X utf8; a test captures both argvs. Review: kshitijk4poor 5411222454 open thread "UTF-8 under -I". --- hermes_cli/update_completion.py | 4 ++- .../test_update_post_commit_followups.py | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index 6ab8b6d1e6d4..c209e851a8e4 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -315,8 +315,10 @@ def _prepare(request: dict, request_path: Path, result_path: Path) -> int: finally: request["pm_receipt"] = receipt.last_for_update(update_id) _write_json(request_path, request) + # -X utf8 like the bootstrap child: -I ignores PYTHONIOENCODING, and this child prints ✓/⚠ + # (the follow-up protocol) into the parent's pipe, whatever the console code page. command = [str(project_python(root)), - "-I", "-S", "-u", "-X", f"pycache_prefix={request['bytecode_cache']}", + "-I", "-S", "-u", "-X", "utf8", "-X", f"pycache_prefix={request['bytecode_cache']}", str(root / "hermes_cli/update_completion.py"), str(request_path), str(result_path), "--prepared"] # A second interpreter is mandatory: PM may have selected a different Python diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py index 522bfedde00d..4156f06274c0 100644 --- a/tests/hermes_cli/test_update_post_commit_followups.py +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -284,3 +284,36 @@ def verified_then_interrupted(request): assert answer["exit_code"] == 0 and answer["receipt"]["outcome"] == "success" assert not (tmp_path / ".update_exit_code").exists() or \ (tmp_path / ".update_exit_code").read_text(encoding="utf-8").strip() == "0" + + +def test_both_completion_children_run_in_utf8_mode(tmp_path, monkeypatch): + """-I drops PYTHONIOENCODING, and both completion children print the ✓/⚠ follow-up protocol + into a pipe: each argv carries -X utf8 (win-utf8 review; the bootstrap child got it first).""" + import pm + from pm import client, environments, receipt + from hermes_cli import update_completion, venv_sync + + calls = [] + + def popen(command, **kwargs): + calls.append(command) + raise OSError("stop after capturing the argv") + + monkeypatch.setattr(subprocess, "call", lambda command, **kw: calls.append(command) or 0) + monkeypatch.setattr(subprocess, "Popen", popen) + for module, name in ((venv_sync, "refuse_foreign_owned_venv"), (venv_sync, "arm_completion"), + (venv_sync, "collect_superseded_generations"), (client, "ensure_tools_for_sync"), + (pm, "sync_venv")): + monkeypatch.setattr(module, name, lambda *a, **k: None) + monkeypatch.setattr(receipt, "last_for_update", lambda *a, **k: None) + monkeypatch.setattr(environments, "project_python", lambda root: "python") + monkeypatch.setattr(environments, "activation_environment", lambda root: {}) + monkeypatch.setattr(update_completion, "_settle_after_commit", lambda *a: 0) + request = {"source": str(tmp_path), "receipt": {"update_id": "u1"}, "bytecode_cache": str(tmp_path / "bc")} + update_completion._prepare(request, tmp_path / "request.json", tmp_path / "result.json") + monkeypatch.setattr(update_completion.sys, "platform", "win32") # skip the Linux libatomic pre-install + with pytest.raises(OSError): + update_completion.run_completion({**request, "home": str(tmp_path)}) + assert len(calls) == 2 + for command in calls: + assert command[command.index("-I"):].count("utf8") == 1 and "-X" in command, command From 69f6f5c1efed47ced5dee12759e4949d47f4faf6 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:13:57 -0700 Subject: [PATCH 44/62] test(update): the gateway-status boundary tests stub the owning-install retarget cmd_update first retargets to the owning install, which probes the PM test environment under the test HOME and trips the home I/O guard in a mixed file run. The retarget is not what these tests (review regression 3) pin. --- tests/hermes_cli/test_update_post_commit_followups.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py index 4156f06274c0..b985bc0c3dc2 100644 --- a/tests/hermes_cli/test_update_post_commit_followups.py +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -224,12 +224,13 @@ def test_interrupt_after_the_run_closed_as_success_never_tells_the_gateway_1(tmp """Review regression 3, the ``cmd_update`` boundary: once this run's receipt says ``success``, an interrupt escaping afterwards must not write 1 to the gateway /update status.""" from types import SimpleNamespace - from hermes_cli import main, update_cmd + from hermes_cli import main, update_cmd, update_owning_install monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) + monkeypatch.setattr(update_owning_install, "retarget_to_owning_install", lambda root: None) # not the target def committed_then_interrupted(args, gateway_mode): update_receipt.begin_update_receipt() @@ -245,12 +246,13 @@ def committed_then_interrupted(args, gateway_mode): def test_a_failure_before_the_run_closed_still_tells_the_gateway_1(tmp_path, monkeypatch): from types import SimpleNamespace - from hermes_cli import main, update_cmd + from hermes_cli import main, update_cmd, update_owning_install monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) + monkeypatch.setattr(update_owning_install, "retarget_to_owning_install", lambda root: None) # not the target def interrupted_while_open(args, gateway_mode): update_receipt.begin_update_receipt() From b2be7496b253b874d5ad4bf924526d31c0297479 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:16:21 -0700 Subject: [PATCH 45/62] docs(update): hermes_cli/AGENTS.md keeps the post-commit rule within the size cap The C3 rewrite of the stale-gateway line (review 5411222454 'Rebase against #132361': the AGENTS.md line) pushed AGENTS.md + hermes_cli/AGENTS.md to 30217 chars, past the 30000 cap. The rule stays (owed follow-ups, exit 0; parked stash 1, Ctrl-C 130; root receipts); the table lives in developer-guide/source-update-completion.md. --- hermes_cli/AGENTS.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hermes_cli/AGENTS.md b/hermes_cli/AGENTS.md index 9b7ce0f94da3..b7aa264b163c 100644 --- a/hermes_cli/AGENTS.md +++ b/hermes_cli/AGENTS.md @@ -136,10 +136,8 @@ The stage-by-stage contract: `website/docs/developer-guide/cli-internals.md` § `_ZIP_PRESERVED_NESTED` build outputs into the swap. - Restarts are fleet-wide and drain-first, bounded by the home each unit runs on (`update_fleet_scope.py`); another install's runtime is never restarted. -- After the commit point nothing fails `hermes update`: failures are receipt follow-ups with - their obligations kept armed, exit 0, except a parked autostash (`partial`, exit 1, #122557) and - Ctrl-C (`interrupted`, 130); table in `developer-guide/source-update-completion.md`. Receipts live under the ROOT home's - `logs/update_receipts/`. +- Post-commit failures are owed follow-ups, exit 0 (stash left 1, Ctrl-C 130); + receipts in ROOT `logs/update_receipts/`. - Nothing runs pulled code in the pre-pull interpreter. `update_handoff.py` and `update_serve_obligations.py` are a FROZEN compat surface (`tests/compat/old_updater_surface.json`): removing a name bricks releases mid-update. From 3b2d4ee2977d8c2303ff1f60de5224614fdf8ebb Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:18:34 -0700 Subject: [PATCH 46/62] fix: an active config write failure still migrates siblings and restores snapshots The owed-migration raise escaped before the sibling config migration and the pre-update snapshot safety nets, so a failed active write left sibling configs on the old version and skipped the cron/model-settings restore that only this call can do (the snapshot ids are not carried to the next-launch retry). Hold the error, run the independent work, then raise so config_migration stays owed. --- hermes_cli/update_cmd_config.py | 12 ++++-- ...test_update_config_migration_on_current.py | 40 +++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/hermes_cli/update_cmd_config.py b/hermes_cli/update_cmd_config.py index 5e91cc1c7267..554165d9f1f4 100644 --- a/hermes_cli/update_cmd_config.py +++ b/hermes_cli/update_cmd_config.py @@ -182,6 +182,7 @@ def _check_and_apply_config_migration( print(" Run 'hermes config migrate' to check manually.") return + owed_error = None has_new_options = bool(missing_env or missing_config) version_bump_only = not has_new_options and current_ver < latest_ver needs_migration = has_new_options or current_ver < latest_ver @@ -205,10 +206,11 @@ def _check_and_apply_config_migration( except Exception as _mig_err: print(f" ⚠️ Config format update failed: {_mig_err}") print(" Run 'hermes config migrate' to retry.") - # Not a silent success: the caller's owed-step guard records ``config_migration`` as - # a receipt follow-up and keeps the source-update tail pending, so it is retried. - raise RuntimeError( - f"config format v{current_ver} → v{latest_ver} was not written: {_mig_err}") from _mig_err + # Not a silent success: raised after the sibling migration and snapshot safety nets + # below (independent of the active write, and the snapshot id is only in hand now), so + # the caller's owed-step guard records ``config_migration`` and keeps the tail pending. + owed_error = RuntimeError(f"config format v{current_ver} → v{latest_ver} was not written: {_mig_err}") + owed_error.__cause__ = _mig_err elif needs_migration: print() # Show WHAT changed, not just a count, for an informed yes/no. @@ -245,6 +247,8 @@ def _check_and_apply_config_migration( print(f" ✓ Profile '{_name}': config format updated (v{_from_ver} → v{_to_ver})") _restore_snapshot_safety_nets(pre_update_snapshot_id) + if owed_error is not None: + raise owed_error # {profile: snapshot_id} from this run's pre-update backup, consumed by the per-profile diff --git a/tests/hermes_cli/test_update_config_migration_on_current.py b/tests/hermes_cli/test_update_config_migration_on_current.py index 700f9f5bb400..d7bf507967ca 100644 --- a/tests/hermes_cli/test_update_config_migration_on_current.py +++ b/tests/hermes_cli/test_update_config_migration_on_current.py @@ -95,3 +95,43 @@ def test_update_copies_bundled_skill_bytes_to_default_active_and_sibling(tmp_pat witness = next(bundled.rglob('SKILL.md')) for profile in homes: assert (profile / 'skills' / witness.relative_to(bundled)).read_bytes() == witness.read_bytes() + + +def test_active_config_write_failure_still_migrates_siblings_and_restores_snapshots(tmp_path, monkeypatch): + """A failed ACTIVE config write is owed debt, but the independent sibling migration and the + pre-update snapshot safety nets (the only holders of this run's snapshot ids) still run.""" + import json + from pathlib import Path + + from hermes_cli import backup, update_cmd_config + + root = tmp_path / '.hermes' + sibling = root / 'profiles' / 'sibling' + jobs = {'jobs': [{'id': 'nightly', 'name': 'nightly', 'schedule': '0 3 * * *', 'prompt': 'report'}]} + for home in (root, sibling): + (home / 'cron').mkdir(parents=True) + (home / 'config.yaml').write_text('_config_version: 48\n', encoding='utf-8') + (home / 'cron' / 'jobs.json').write_text(json.dumps(jobs), encoding='utf-8') + monkeypatch.setattr(Path, 'home', lambda: tmp_path) + monkeypatch.setenv('HERMES_HOME', str(root)) + active_snapshot = backup.create_quick_snapshot(label='pre-update', hermes_home=root) + monkeypatch.setattr(update_cmd_config, '_LAST_SIBLING_SNAPSHOTS', + backup.create_pre_update_snapshots_all_profiles(root)) + for home in (root, sibling): # the documented mutation-window loss the safety nets repair + (home / 'cron' / 'jobs.json').write_text('{"jobs": []}', encoding='utf-8') + real_replace = config.atomic_config_replace + + def replace(path, *args, **kwargs): + if Path(path) == root / 'config.yaml': + raise OSError('disk full') + return real_replace(path, *args, **kwargs) + + monkeypatch.setattr(config, 'atomic_config_replace', replace) + with pytest.raises(RuntimeError, match='was not written: disk full'): + update_cmd_config._check_and_apply_config_migration(pre_update_snapshot_id=active_snapshot) + for home in (root, sibling): + assert json.loads((home / 'cron' / 'jobs.json').read_text(encoding='utf-8')) == jobs + assert config.check_config_version()[0] == 48 # the active write stays owed + monkeypatch.setenv('HERMES_HOME', str(sibling)) + sibling_version, latest = config.check_config_version() + assert sibling_version == latest > 48 From 78cd7d34ce5784e2859def93af11483c0d7a91a1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:36:27 -0700 Subject: [PATCH 47/62] fix: a same-SHA catch-up adds newly stopped unmapped gateways to the owed inventory _record_owed_gateway_inventory returned for any existing inventory, so a same-SHA retry that stopped an unmapped gateway while the mapped one stayed down recorded a gateway_restart follow-up but never added the stopped PID to the durable owed set; once the mapped gateway recovered, startup settled the obligation with no successor for the unmapped one. A readable inventory now only grows (earlier rows and record provenance kept); a malformed or unsupported one is still left untouched. --- hermes_cli/update_cmd_fleet_verify.py | 20 +++++++-- .../test_update_fleet_check_fail_closed.py | 42 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py index 59f092d95fee..6097bce476dd 100644 --- a/hermes_cli/update_cmd_fleet_verify.py +++ b/hermes_cli/update_cmd_fleet_verify.py @@ -6,6 +6,7 @@ Restart-phase helpers stay in ``update_cmd_fleet`` and are read through it so their patches apply. """ +import json import logging import subprocess import time as _time @@ -129,7 +130,7 @@ def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict: def _record_owed_gateway_inventory(plan, stopped_unmapped_pids=()) -> None: - """Name the pre-update gateways on an inventory-less obligation once their restart is owed. + """Name the pre-update gateways on the obligation once their restart is owed. The pull arms the obligation before the restart phase knows what it owes. An inventory-less record on a host whose gateway then died at boot is settled by the gateway-less discharge @@ -137,12 +138,22 @@ def _record_owed_gateway_inventory(plan, stopped_unmapped_pids=()) -> None: failed update under contract C3. A named inventory keeps it until those gateways run HEAD. A gateway stopped with no known profile is recorded as an unmapped row for the same reason: it has no ``gateway_state.json`` the gateway-less probe could see, so absence would settle it. + A readable inventory (a same-SHA catch-up keeps the standing record) only GROWS: rows this run + newly owes are appended, so a mapped successor cannot discharge a newly stopped unmapped one. """ from hermes_cli import update_cmd_fleet as fleet armed = fleet._fleet_restart_obligation_armed() fields = fleet._obligation_fields() if armed else {} - if fields is None or fields.get("inventory") not in (None, "", "null"): - return # unreadable terms stay fail-closed; a recorded inventory is never rewritten + if fields is None: + return # unreadable terms stay fail-closed + recorded: list = [] + if fields.get("inventory") not in (None, "", "null"): + try: + inventory = json.loads(fields["inventory"]) + fleet._marker_owed_gateways(inventory) # validates; malformed/unsupported stays untouched + except ValueError: + return + recorded = inventory["runtimes"] from dataclasses import asdict rows = [ asdict(runtime) for runtime in getattr(plan, "runtimes", ()) or () @@ -151,11 +162,12 @@ def _record_owed_gateway_inventory(plan, stopped_unmapped_pids=()) -> None: and runtime.profile.strip() and runtime.profile != "unknown" ] + [{"kind": "gateway", "profile": None, "pid": pid, "stopped_unmapped": True} for pid in sorted(stopped_unmapped_pids)] + rows = [row for row in rows if row not in recorded] if rows: # Not armed any more (a pre-restart probe settled an inventory-less record): re-arm, the # restart is owed. The SHA is the code the fleet must be proven current on. sha = fields.get("expected_sha", "") if armed else (fleet._current_checkout_sha() or "") - fleet._write_fleet_restart_pending_marker(expected_sha=sha, runtimes=rows) + fleet._write_fleet_restart_pending_marker(expected_sha=sha, runtimes=recorded + rows) def _named_gateways_still_owed() -> bool: diff --git a/tests/hermes_cli/test_update_fleet_check_fail_closed.py b/tests/hermes_cli/test_update_fleet_check_fail_closed.py index c6de5605ef16..179c46f2d5a7 100644 --- a/tests/hermes_cli/test_update_fleet_check_fail_closed.py +++ b/tests/hermes_cli/test_update_fleet_check_fail_closed.py @@ -23,6 +23,7 @@ from __future__ import annotations +import json import types import pytest @@ -196,3 +197,44 @@ def test_unmapped_stop_debt_is_not_settled_by_the_mapped_gateways_restart(monkey fleet_rows.append({"profile": "work", "state": "current", "code_sha": "head", "pid": 9}) assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is False assert not fleet._fleet_restart_obligation_armed() + + +def test_same_sha_retry_adds_newly_stopped_unmapped_debt_to_an_existing_inventory(monkeypatch, tmp_path): + # A same-SHA catch-up keeps the standing inventory (gateway `default`, still down). If that retry + # also stops an unmapped gateway, its debt must join the inventory: otherwise `default` coming + # back settles the obligation while the unmapped gateway still has no successor. + from hermes_cli import update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as fleet_verify + from hermes_cli import update_host_obligation as host, update_receipt + from hermes_cli.update_inventory import UpdatePlan + + monkeypatch.setattr(host, "host_obligation_path", lambda: tmp_path / "host-update-restart.json") + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "head") + monkeypatch.setattr(update_cmd, "_current_checkout_sha", lambda: "head") + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) + monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) + monkeypatch.setattr(fleet_verify, "_FLEET_PROBE_SETTLE_TIMEOUT_SECONDS", 0) # A stays down: one poll + # Signal delivery is the external boundary: the down row has no process to drain. + monkeypatch.setattr("hermes_cli.update_cmd_stale_survivors.signal_stale_fleet_survivors", lambda *a: None) + fleet_rows = [{"profile": "default", "state": "down", "code_sha": "old", "pid": 7}] + monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: list(fleet_rows)) + fleet._write_fleet_restart_pending_marker(expected_sha="head", runtimes=[ + {"kind": "gateway", "profile": "default", "pid": 7}]) + out = fleet._GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[7999], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], + killed_pids={7999}, stopped_unmapped_pids={7999}, + ) + update_receipt.begin_update_receipt() + fleet_verify._verify_fleet_after_update( + out, _pre_update_plan=UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default", pid=7)]), + _windows_gateway_resume=None, update_complete=True) + + rows = json.loads(fleet._obligation_fields()["inventory"])["runtimes"] + assert {"kind": "gateway", "profile": "default", "pid": 7} in rows # earlier debt kept + assert {"kind": "gateway", "profile": None, "pid": 7999, "stopped_unmapped": True} in rows + fleet_rows[:] = [{"profile": "default", "state": "current", "code_sha": "head", "pid": 8}] + assert fleet._marker_only_restart_obsolete() is False # `default`'s successor is not 7999's + assert fleet._fleet_restart_obligation_armed() + fleet_rows.append({"profile": "work", "state": "current", "code_sha": "head", "pid": 9}) + assert fleet._marker_only_restart_obsolete() is True From c9f47c0d9048e61a59a9190d5736a4709fb9f2a9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:47:47 -0700 Subject: [PATCH 48/62] fix: the dashboard update status certifies an outcome only from its own action's receipt The receipt store moved to the root home, so profile A's later success in latest.json became profile B's exit 0 once B's dashboard lost its in-memory result: the receipt fallback had no identity to compare. Each receipt now records the dashboard action (HERMES_ACTION_ID) that spawned the run, and the status route resolves the action this dashboard started (its log header, else the in-memory id), reads that action's receipt from latest.json or the retained archive, and maps an outcome to an exit code only on an exact match. The attached summary carries action_id so clients can tell whose run it is; the update_id/PM correlation identity is unchanged. --- hermes_cli/update_receipt.py | 18 ++++++++++ hermes_cli/web_routers/actions.py | 35 ++++++++++++------- .../test_update_receipt_endpoint.py | 7 ++++ .../test_web_server_update_status.py | 33 +++++++++++++++++ 4 files changed, 81 insertions(+), 12 deletions(-) diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index 9aadd1117e21..8e7596500a66 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -156,6 +156,9 @@ def __init__(self) -> None: self.correlation_id = uuid.uuid4().hex self.data["update_id"] = self.correlation_id self.data["correlation_id"] = _launcher_correlation_id() + # The dashboard action that spawned this run: the receipt store is root-wide, so the + # action-status route certifies an outcome only from a receipt naming ITS action. + self.data["action_id"] = os.environ.get("HERMES_ACTION_ID", "").strip() or None def step(self, name: str, ok: bool, detail: str = "") -> None: self.data["steps"].append({"name": name, "ok": bool(ok), "detail": detail, "at": _utc_now_iso()}) @@ -746,6 +749,21 @@ def read_latest_receipt() -> Optional[dict[str, Any]]: return None +def read_receipt_for_action(action_id: str) -> Optional[dict[str, Any]]: + """Newest receipt written by the dashboard action ``action_id`` (``latest.json`` first, then + the retained archive), or None. Never raises.""" + latest = read_latest_receipt() + if latest and latest.get("action_id") == action_id: + return latest + with suppress(Exception): + for path in sorted(_receipt_dir().glob("update_*.json"), key=lambda p: p.stat().st_mtime, reverse=True): + with suppress(Exception): + payload = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(payload, dict) and payload.get("action_id") == action_id: + return payload + return None + + def _profile_homes() -> list[tuple[str, Path]]: """``(profile, home)`` for the default home plus every valid named profile dir, sorted.""" from hermes_cli.profiles import _get_default_hermes_home, _get_profiles_root, _PROFILE_ID_RE diff --git a/hermes_cli/web_routers/actions.py b/hermes_cli/web_routers/actions.py index 30dea8f558b3..f6c0845f2ea1 100644 --- a/hermes_cli/web_routers/actions.py +++ b/hermes_cli/web_routers/actions.py @@ -346,9 +346,9 @@ def _completed_exit_code( if durable_action_id: return 0 if receipt is not None and receipt.get("outcome") in ("success", "partial"): - # No in-memory result and no log marker (e.g. log rotated), but the - # receipt proves a completed run: report its outcome rather than a - # null clients time out on. ``partial`` maps to exit 1 like the CLI. + # No in-memory result and no log marker (e.g. log rotated), but THIS + # action's receipt proves a completed run: report its outcome rather than + # a null clients time out on. ``partial`` maps to exit 1 like the CLI. return 0 if receipt["outcome"] == "success" else 1 return None @@ -365,7 +365,7 @@ async def get_action_status(name: str, lines: int = 200): tail = _tail_lines(log_dir / log_file_name, requested_lines) durable_update_action_id = None - update_receipt_summary = None + update_receipt_summary = action_receipt_summary = None if name == "hermes-update": # ``hermes update`` mirrors to the ROOT home's update.log (main_dashboard), never this # dashboard's profile home: read it where it is written. @@ -373,7 +373,9 @@ async def get_action_status(name: str, lines: int = 200): durable_update_action_id = _durable_completed_update_action_id( _tail_lines(get_default_hermes_root() / "logs" / "update.log", 2000)) - if durable_update_action_id != _latest_spawned_update_action_id(_tail_lines(log_dir / log_file_name, 2000)): + spawned_action_id = (_latest_spawned_update_action_id(_tail_lines(log_dir / log_file_name, 2000)) + or _ACTION_IDS.get(name)) + if durable_update_action_id != spawned_action_id: # The root log is shared by every profile: another profile's (or an older) run's # completion never certifies the action this dashboard started. durable_update_action_id = None @@ -386,14 +388,17 @@ async def get_action_status(name: str, lines: int = 200): # dashboard restarting itself mid-action). Surface it so clients READ # the outcome instead of inferring it from liveness probes. # See #81193, #87359, #91277. - update_receipt_summary = _latest_update_receipt_summary() + # The store is root-wide (every profile's runs): show THIS action's receipt when one exists. + update_receipt_summary = _latest_update_receipt_summary(spawned_action_id) + if spawned_action_id and (update_receipt_summary or {}).get("action_id") == spawned_action_id: + action_receipt_summary = update_receipt_summary proc = _ACTION_PROCS.get(name) if proc is None: result = _ACTION_RESULTS.get(name) running = False pid = result.get("pid") if result else None - exit_code = _completed_exit_code(result, durable_update_action_id, update_receipt_summary) + exit_code = _completed_exit_code(result, durable_update_action_id, action_receipt_summary) else: exit_code = proc.poll() running = exit_code is None @@ -420,22 +425,28 @@ def _read_latest_receipt() -> Optional[Dict[str, Any]]: return None -def _latest_update_receipt_summary() -> Optional[Dict[str, Any]]: - """Compact summary of the latest receipt (written by EVERY ``hermes update`` run, - incl. refused/failed), or None; never raises. Steps/skips stay in the full endpoint. +def _latest_update_receipt_summary(action_id: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Compact summary of dashboard action ``action_id``'s receipt when one exists, else of the + latest receipt (written by EVERY ``hermes update`` run, incl. refused/failed), or None; never + raises. Steps/skips stay in the full endpoint. ``action_id`` names the writer so a client + never credits another action's run. Phase-1 bullet 3 (#91277): the receipt (written by EVERY ``hermes update`` run since #91283, including refused and failed ones, with a ``latest.json`` pointer) is the durable success signal the Desktop and dashboard should read instead of inferring outcomes from liveness probes across the update's stop/start gap (#81193, #87359). """ - receipt = _read_latest_receipt() + receipt = None + if action_id: + from hermes_cli.update_receipt import read_receipt_for_action + receipt = read_receipt_for_action(action_id) + receipt = receipt or _read_latest_receipt() if not receipt: return None try: post = receipt.get("post_update") or {} return { - **{k: receipt.get(k) for k in ("outcome", "started_at", "finished_at")}, + **{k: receipt.get(k) for k in ("outcome", "started_at", "finished_at", "action_id")}, "pre_sha": (receipt.get("pre_update") or {}).get("sha"), "post_sha": post.get("sha"), "post_version": post.get("version"), "fleet_states": sorted({str(e.get("state")) for e in receipt.get("fleet") or [] if isinstance(e, dict)}), diff --git a/tests/hermes_cli/test_update_receipt_endpoint.py b/tests/hermes_cli/test_update_receipt_endpoint.py index bcdc34fd1449..e9d4d5daed4e 100644 --- a/tests/hermes_cli/test_update_receipt_endpoint.py +++ b/tests/hermes_cli/test_update_receipt_endpoint.py @@ -32,6 +32,9 @@ def client(): return c +_ACTION_ID = "d" * 32 + + def _write_receipt(tmp_path: Path, monkeypatch, *, outcome="success") -> dict: receipt = { "schema": 1, @@ -40,6 +43,7 @@ def _write_receipt(tmp_path: Path, monkeypatch, *, outcome="success") -> dict: "argv": ["hermes", "update"], "pid": 12345, "outcome": outcome, + "action_id": _ACTION_ID, "pre_update": {"sha": "a" * 40, "version": "0.20.4"}, "post_update": {"sha": "b" * 40, "version": "0.20.5"}, "steps": [{"name": "pre_update_backup", "ok": True, "detail": "", "at": ""}], @@ -91,6 +95,9 @@ class TestUpdateStatusReadsReceipt: def _clear_registries(self, monkeypatch, tmp_path): monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path / "actions") (tmp_path / "actions").mkdir(exist_ok=True) + # The dashboard's own log names the action it spawned; the receipt must name the same one. + (tmp_path / "actions" / "hermes-update.log").write_text( + f"=== hermes-update started 2026-08-23 07:00:00 {_ACTION_ID} ===\n", encoding="utf-8") monkeypatch.setattr(_web_server_gateway, "_ACTION_PROCS", {}) monkeypatch.setattr(_web_server_gateway, "_ACTION_RESULTS", {}) monkeypatch.setattr(_web_server_gateway, "_ACTION_COMMANDS", {}) diff --git a/tests/hermes_cli/test_web_server_update_status.py b/tests/hermes_cli/test_web_server_update_status.py index 7656849f5cb1..132e486291b7 100644 --- a/tests/hermes_cli/test_web_server_update_status.py +++ b/tests/hermes_cli/test_web_server_update_status.py @@ -86,3 +86,36 @@ def status_after_root_completion(action_id): own = status_after_root_completion(b_id) assert own["exit_code"] == 0 assert own["action_id"] == b_id + + @pytest.mark.parametrize("b_outcome", [None, "failed", "success"]) + def test_root_latest_receipt_of_another_profile_never_certifies_this_action(self, monkeypatch, tmp_path, b_outcome): + # The receipt store is root-wide too: A's later success in latest.json must not become B's + # exit 0. Only a receipt naming B's action (found in the archive when A is latest) counts. + from hermes_cli import update_receipt + + root = tmp_path / "root" + (root / "logs").mkdir(parents=True) + b_id, a_id = "b" * 32, "a" * 32 + for profile, action_id, outcome in (("b", b_id, b_outcome), ("a", a_id, "success")): + if outcome: + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / profile)) + monkeypatch.setenv("HERMES_ACTION_ID", action_id) + update_receipt.begin_update_receipt() + update_receipt.finalize_update_receipt(outcome) + monkeypatch.delenv("HERMES_ACTION_ID") + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + assert update_receipt.read_latest_receipt()["action_id"] == a_id + (tmp_path / "hermes-update.log").write_text( + f"=== hermes-update started 2026-08-17 11:19:34 {b_id} ===\n", encoding="utf-8") + (root / "logs" / "update.log").write_text(f"=== hermes-update completed {a_id} ===\n", encoding="utf-8") + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + + data = self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + assert data["exit_code"] == (0 if b_outcome == "success" else None) + assert "action_id" not in data # A's root completion marker is not B's + # The attached receipt names its writer: B's own when it exists, else A's (never B's). + assert data["receipt"]["action_id"] == (b_id if b_outcome else a_id) + assert data["receipt"]["outcome"] == (b_outcome or "success") From 8c4a3d118f3f99d64f178f1435277f8c8a12c55e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:51:15 -0700 Subject: [PATCH 49/62] fix: dashboard, Desktop and managed SSH clients report a committed update's owed steps Under C3 a run whose code committed is a success (exit 0) even while a post-commit step is owed, but the dashboard status summary and the managed SSH observer dropped the receipt's followups/user_action, so every client outside the local shell hand-off reported plain success. Both summary protocols now carry followups and user_action; the Desktop backend apply and managed SSH result keep their C3 success/restore status but name each owed step and its remedy, and the web dashboard toast does the same. The Desktop receipt fallback also requires the receipt's action_id to match the started action (when the backend reports one), so another action's run cannot certify it. --- .../electron/managed-ssh-update.test.ts | 63 +++++++++++++++++ apps/desktop/electron/managed-ssh-update.ts | 67 +++++++++++++++++-- apps/desktop/src/global.d.ts | 4 ++ apps/desktop/src/i18n/en.ts | 6 +- apps/desktop/src/i18n/types.ts | 4 +- apps/desktop/src/store/updates.test.ts | 52 ++++++++++++++ apps/desktop/src/store/updates.ts | 38 +++++++++-- apps/desktop/src/types/hermes.ts | 5 ++ hermes_cli/web_routers/actions.py | 5 ++ locales/_keys.desktop.json | 1 + .../test_web_server_update_status.py | 32 +++++++++ web/src/contexts/SystemActions.tsx | 21 +++++- web/src/i18n/en.ts | 1 + web/src/i18n/types.ts | 1 + 14 files changed, 279 insertions(+), 21 deletions(-) diff --git a/apps/desktop/electron/managed-ssh-update.test.ts b/apps/desktop/electron/managed-ssh-update.test.ts index 30d8a89c650e..8196caf91fd4 100644 --- a/apps/desktop/electron/managed-ssh-update.test.ts +++ b/apps/desktop/electron/managed-ssh-update.test.ts @@ -507,6 +507,69 @@ test.runIf(process.platform !== 'win32')( } ) +test.runIf(process.platform !== 'win32')( + 'a committed update that still owes a step stays a success but names the owed step', + async () => { + const home = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-owed-')) + + const run = async (followups: unknown[]) => { + const receipts = path.join(home, 'logs', 'update_receipts') + await mkdir(receipts, { recursive: true }) + await writeFile(path.join(home, `.update_exit_code.${CORRELATION}`), '0') + await writeFile( + path.join(receipts, 'update_x.json'), + JSON.stringify({ + correlation_id: CORRELATION, + outcome: 'success', + started_at: '2026-08-23T00:00:00Z', + finished_at: '2026-08-23T00:01:00Z', + followups + }) + ) + + const command = buildRemoteUpdateObservationCommand( + { ssh: { exec: async () => '' }, platform: 'Linux', hermesPath: '/opt/hermes/hermes', hermesHome: home }, + CORRELATION + ) + + const observed = parseRemoteUpdateObservation((await exec(command, { shell: 'sh' })).stdout, CORRELATION) + + return runManagedSshUpdate({ + connectionId: 'home', + correlationId: CORRELATION, + scopes: [{ key: 'conn:home::default', profile: 'default' }], + preflightRemote: async () => {}, + drainScope: async () => {}, + updateRemote: async () => ({ exitCode: observed.exitCode!, receipt: observed.receipt! }), + awaitRestoreClearance: async () => {}, + closeTransports: async () => {}, + restoreScope: async () => {}, + releaseGate: () => {} + }) + } + + try { + const owed = await run([{ step: 'dependencies', reason: 'sync failed', at: '2026-08-23T00:01:00Z' }]) + + assert.equal(owed.ok, true) + assert.equal(owed.updateOk, true) + assert.equal(owed.restoreOk, true) + assert.deepEqual(owed.owed, [{ step: 'dependencies', reason: 'sync failed' }]) + assert.match(owed.message || '', /still owed: dependencies \(sync failed\)/) + assert.match(owed.message || '', /hermes update/) + assert.match(managedSshUpdateAllRow({ id: 'home' }, owed).detail || '', /dependencies/) + + const clean = await run([]) + + assert.equal(clean.ok, true) + assert.equal(clean.owed, undefined) + assert.equal(clean.message, 'Remote Hermes updated and every managed SSH profile is ready.') + } finally { + await rm(home, { force: true, recursive: true }) + } + } +) + test.runIf(process.platform !== 'win32')( 'managed observer unwraps a named profile home for the install-wide marker', async () => { diff --git a/apps/desktop/electron/managed-ssh-update.ts b/apps/desktop/electron/managed-ssh-update.ts index c9712f5165e8..3e9b1bd07aaa 100644 --- a/apps/desktop/electron/managed-ssh-update.ts +++ b/apps/desktop/electron/managed-ssh-update.ts @@ -29,6 +29,11 @@ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0- type ManagedUpdateOutcome = 'updated' | 'update-failed' | 'restore-failed' | 'update-and-restore-failed' | 'refused' +interface ManagedUpdateOwedStep { + step: string + reason: string +} + interface ManagedUpdateReceiptSummary { correlationId: string outcome: string @@ -39,6 +44,9 @@ interface ManagedUpdateReceiptSummary { preVersion?: string postVersion?: string stopReason?: string + // Post-commit steps a successful update still owes (contract C3). + followups?: ManagedUpdateOwedStep[] + userAction?: ManagedUpdateOwedStep | null } interface ManagedUpdateScopeResult { @@ -57,6 +65,8 @@ interface ManagedConnectionUpdateResult { exitCode: null | number receipt: ManagedUpdateReceiptSummary | null scopes: ManagedUpdateScopeResult[] + // Every step a committed update still owes; set only when non-empty. + owed?: ManagedUpdateOwedStep[] error?: string message?: string // Set when the connection was deliberately not attempted (a known safety @@ -427,12 +437,15 @@ def receipt(): if payload.get('outcome')=='running' or not payload.get('finished_at'):continue pre=payload.get('pre_update') if isinstance(payload.get('pre_update'),dict) else {} post=payload.get('post_update') if isinstance(payload.get('post_update'),dict) else {} + action=payload.get('user_action') return { 'correlationId':correlation,'outcome':str(payload.get('outcome') or 'unknown'), 'startedAt':payload.get('started_at'),'finishedAt':payload.get('finished_at'), 'preSha':pre.get('sha'),'postSha':post.get('sha'), 'preVersion':pre.get('version'),'postVersion':post.get('version'), 'stopReason':payload.get('stop_reason'), + 'followups':[{'step':str(f.get('step')),'reason':str(f.get('reason') or '')} for f in payload.get('followups') or [] if isinstance(f,dict)], + 'userAction':{'step':str(action.get('step')),'reason':str(action.get('reason') or '')} if isinstance(action,dict) else None, } return None @@ -481,6 +494,50 @@ function buildRemoteUpdateObservationCommand(target: RemoteUpdateTarget, correla return `python3 -c ${shq(OBSERVATION_SCRIPT)} ${shq(home)} ${shq(correlation)}` } +function owedStep(value: any): ManagedUpdateOwedStep | null { + return value && typeof value === 'object' ? { step: String(value.step), reason: String(value.reason ?? '') } : null +} + +// An older remote observer omits the debt fields: read them as none. +function withReceiptDebt(receipt: any): ManagedUpdateReceiptSummary { + const followups = Array.isArray(receipt.followups) ? receipt.followups.map(owedStep).filter(Boolean) : [] + + return { ...receipt, followups, userAction: owedStep(receipt.userAction) } +} + +// C3: a committed update is a success even when post-commit steps are owed; +// name each one and its remedy instead of claiming everything is ready. +function managedUpdateMessage( + outcome: ManagedUpdateOutcome, + restoreOk: boolean, + receipt: ManagedUpdateReceiptSummary | undefined +): { owed?: ManagedUpdateOwedStep[]; message: string } { + if (outcome !== 'updated') { + return { + message: restoreOk + ? 'The remote update failed, but every managed SSH profile was restored.' + : 'The remote update transaction could not restore every managed SSH profile.' + } + } + + const followups = receipt?.followups ?? [] + const userAction = receipt?.userAction + const owed = [...followups, ...(userAction ? [userAction] : [])] + + if (!owed.length) { + return { message: 'Remote Hermes updated and every managed SSH profile is ready.' } + } + + const message = + 'Remote Hermes updated, but these steps are still owed: ' + + owed.map(step => (step.reason ? `${step.step} (${step.reason})` : step.step)).join('; ') + + '.' + + (followups.length ? ' Re-run `hermes update` on the remote to finish them.' : '') + + (userAction?.reason ? ` ${userAction.reason}` : '') + + return { owed, message } +} + function parseRemoteUpdateObservation(raw: string, correlationId: string): RemoteUpdateObservation { const correlation = validateCorrelationId(correlationId) let parsed: any @@ -526,7 +583,7 @@ function parseRemoteUpdateObservation(raw: string, correlationId: string): Remot throw new Error('Remote update receipt did not match this transaction.') } - receipt = parsed.receipt as ManagedUpdateReceiptSummary + receipt = withReceiptDebt(parsed.receipt) } let coordinatorReady: RemoteUpdateObservation['coordinatorReady'] = null @@ -938,12 +995,7 @@ async function runManagedSshUpdate( receipt: proof?.receipt ?? null, scopes: restoreResults, ...(error ? { error } : {}), - message: - outcome === 'updated' - ? 'Remote Hermes updated and every managed SSH profile is ready.' - : restoreOk - ? 'The remote update failed, but every managed SSH profile was restored.' - : 'The remote update transaction could not restore every managed SSH profile.' + ...managedUpdateMessage(outcome, restoreOk, proof?.receipt) } } @@ -1180,6 +1232,7 @@ export { managedSshUpdateAllRow, type ManagedUpdateDeps, type ManagedUpdateOutcome, + type ManagedUpdateOwedStep, type ManagedUpdateReceiptSummary, type ManagedUpdateScopeResult, markerIsClear, diff --git a/apps/desktop/src/global.d.ts b/apps/desktop/src/global.d.ts index e0b63fe03465..0159edf0902d 100644 --- a/apps/desktop/src/global.d.ts +++ b/apps/desktop/src/global.d.ts @@ -1232,6 +1232,8 @@ export interface DesktopManagedUpdateReceipt { preVersion?: string postVersion?: string stopReason?: string + followups?: Array<{ step: string; reason: string }> + userAction?: { step: string; reason: string } | null } export interface DesktopManagedConnectionUpdateResult { @@ -1244,6 +1246,8 @@ export interface DesktopManagedConnectionUpdateResult { exitCode: number | null receipt: DesktopManagedUpdateReceipt | null scopes: Array<{ profile: string; restored: boolean; error?: string }> + /** Post-commit steps a successful update still owes (named in `message`). */ + owed?: Array<{ step: string; reason: string }> error?: string message?: string } diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index cec308d4b0c9..f1d6cfead876 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -4232,10 +4232,10 @@ export const en: Translations = { restarting: 'Backend restarting to load the update…', notAvailable: 'Update not available for this backend.', failed: 'Backend update failed.', - noReturn: 'Backend didn’t come back online. The update may not have completed — check the backend host.' + noReturn: 'Backend didn’t come back online. The update may not have completed — check the backend host.', + owed: steps => `Backend updated, but still owed: ${steps}. Re-run \`hermes update\` to finish them.` }, - // Update-status overlay + version-details (mechanism-aware update UI). - appName: 'Hermes', + appName: 'Hermes', // Update-status overlay + version-details (mechanism-aware update UI). version: (value: string) => `Version ${value}`, versionUnavailable: 'Version unavailable', checkNow: 'Check now', diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index 7645d14cd8ff..9a88a48f8d2a 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -3521,9 +3521,9 @@ export interface Translations { notAvailable: string failed: string noReturn: string + owed: (steps: string) => string } - /** Update-status overlay + version-details (mechanism-aware update UI): - * the overlay reads these off t.updates directly. */ + /** Update-status overlay + version-details (mechanism-aware update UI), read off t.updates directly. */ appName: string version: (value: string) => string versionUnavailable: string diff --git a/apps/desktop/src/store/updates.test.ts b/apps/desktop/src/store/updates.test.ts index 173ff4c36a80..a0cb66cb6375 100644 --- a/apps/desktop/src/store/updates.test.ts +++ b/apps/desktop/src/store/updates.test.ts @@ -1487,6 +1487,58 @@ describe('applyBackendUpdate recovery', () => { expect(result.ok).toBe(false) expect($backendUpdateApply.get().stage).toBe('error') }, 10000) + + it('reports a committed update with owed follow-ups as success plus an owed-step warning', async () => { + updateHermesSpy.mockResolvedValue({ action_id: 'e'.repeat(32), ok: true, name: 'hermes-update', pid: 1 }) + getActionStatusSpy.mockResolvedValue({ + exit_code: 0, + lines: ['Update complete!'], + name: 'hermes-update', + pid: null, + running: false, + receipt: { outcome: 'success', followups: [{ step: 'dependencies', reason: 'sync failed' }], user_action: null } + }) + notifySpy.mockClear() + + const promise = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(1500) + const result = await promise + + expect(result).toMatchObject({ ok: true }) + expect(result.message).toContain('dependencies (sync failed)') + expect(notifySpy).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'warning', message: expect.stringContaining('dependencies (sync failed)') }) + ) + }) + + it("never lets another action's receipt certify this update", async () => { + const receipt = (actionId: string) => ({ + exit_code: null, + lines: [], + name: 'hermes-update', + pid: null, + running: false, + receipt: { + action_id: actionId, + outcome: 'success', + started_at: new Date().toISOString(), + finished_at: new Date().toISOString(), + followups: [], + user_action: null + } + }) + + updateHermesSpy.mockResolvedValue({ action_id: 'b'.repeat(32), ok: true, name: 'hermes-update', pid: 1 }) + getActionStatusSpy.mockResolvedValue(receipt('a'.repeat(32))) + const foreign = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(6 * 60 * 1000 + 1500) + await expect(foreign).resolves.toMatchObject({ ok: false }) + + getActionStatusSpy.mockResolvedValue(receipt('b'.repeat(32))) + const own = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(1500) + await expect(own).resolves.toMatchObject({ ok: true }) + }) }) describe('startUpdatePoller', () => { diff --git a/apps/desktop/src/store/updates.ts b/apps/desktop/src/store/updates.ts index bdcd35b8cb95..b78ecf5ffa7d 100644 --- a/apps/desktop/src/store/updates.ts +++ b/apps/desktop/src/store/updates.ts @@ -22,7 +22,7 @@ import { $connectionsRegistry, refreshConnectionsRegistry } from '@/store/connec import { reconnectGateway } from '@/store/gateway-reconnect' import { dismissNotification, notify } from '@/store/notifications' import { $connection } from '@/store/session' -import type { BackendUpdateCheckResponse } from '@/types/hermes' +import type { BackendUpdateCheckResponse, UpdateReceiptSummary } from '@/types/hermes' /** Keyed per retired-channel revision: a new retirement (or a revision bump on * the same channel) re-shows the notice, a plain re-check never does. */ @@ -747,7 +747,15 @@ const BACKEND_ACTION_POLL_MS = 1500 const BACKEND_ACTION_MAX_MS = 6 * 60 * 1000 const BACKEND_RETURN_MAX_MS = 4 * 60 * 1000 -function finishBackendApply(returned: boolean): DesktopUpdateApplyResult { +// C3: a committed update succeeds even when post-commit steps are still owed. +// Name every owed step (and its reason) instead of reporting plain success. +function owedSteps(receipt: UpdateReceiptSummary | undefined): string { + return [...(receipt?.followups ?? []), ...(receipt?.user_action ? [receipt.user_action] : [])] + .map(step => (step.reason ? `${step.step} (${step.reason})` : step.step)) + .join('; ') +} + +function finishBackendApply(returned: boolean, receipt?: UpdateReceiptSummary): DesktopUpdateApplyResult { if (returned) { $backendUpdateApply.set(IDLE) setUpdateOverlayOpen(false) @@ -765,6 +773,14 @@ function finishBackendApply(returned: boolean): DesktopUpdateApplyResult { // affordance in remote mode targets the backend, so nothing ever told // them the app itself was stale). Nudge with a one-click client update. void maybeNudgeClientAfterBackendUpdate() + const owed = owedSteps(receipt) + + if (owed) { + const message = translateNow('updates.applyStatus.owed', owed) + notify({ durationMs: 0, kind: 'warning', message }) + + return { ok: true, message } + } return { ok: true, message: 'Backend update applied.' } } @@ -813,7 +829,11 @@ function completedAfterRestart( * run started at-or-after we kicked the update off counts — an older * receipt describes a previous update, and a still-running one proves * nothing yet. The 60s slack absorbs client/backend clock skew. */ -function receiptProvesOutcome(status: Awaited>, applyStartedAtMs: number): boolean { +function receiptProvesOutcome( + status: Awaited>, + applyStartedAtMs: number, + actionId: string | undefined +): boolean { const receipt = status.receipt if (!receipt || !receipt.finished_at || !receipt.started_at) { @@ -824,6 +844,12 @@ function receiptProvesOutcome(status: Awaited return false } + // A backend that records the writing action (null for a CLI run) proves + // identity directly: another action's success cannot certify this one. + if (actionId && receipt.action_id !== undefined) { + return receipt.action_id === actionId + } + const startedMs = Date.parse(receipt.started_at) return Number.isFinite(startedMs) && startedMs >= applyStartedAtMs - 60_000 @@ -929,15 +955,15 @@ async function runBackendUpdate(): Promise { } if (last.exit_code === 0 || (last.exit_code === null && completedAfterRestart(last, started.action_id))) { - return finishBackendApply(true) + return finishBackendApply(true, last.receipt) } // #91277 bullet 3: the backend now attaches the durable update // receipt to the status. A receipt whose run STARTED after we kicked // this update off is authoritative — read its outcome instead of // inferring from log markers or timing out across the restart gap. - if (last.exit_code === null && receiptProvesOutcome(last, applyStartedAtMs)) { - return finishBackendApply(last.receipt!.outcome === 'success') + if (last.exit_code === null && receiptProvesOutcome(last, applyStartedAtMs, started.action_id)) { + return finishBackendApply(last.receipt!.outcome === 'success', last.receipt) } if (!started.action_id && last.exit_code === null) { diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index 69b4cba8a977..81647d34e625 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -1557,6 +1557,11 @@ export interface UpdateReceiptSummary { post_sha: string | null post_version: string | null fleet_states: string[] + /** Post-commit steps a committed (successful) update still owes. */ + followups?: Array<{ step: string; reason: string }> + user_action?: { step: string; reason: string } | null + /** Dashboard action that wrote the receipt; null for a CLI-run update. */ + action_id?: string | null } export interface ActionStatusResponse { diff --git a/hermes_cli/web_routers/actions.py b/hermes_cli/web_routers/actions.py index f6c0845f2ea1..050ee2dfd1d7 100644 --- a/hermes_cli/web_routers/actions.py +++ b/hermes_cli/web_routers/actions.py @@ -450,6 +450,11 @@ def _latest_update_receipt_summary(action_id: Optional[str] = None) -> Optional[ "pre_sha": (receipt.get("pre_update") or {}).get("sha"), "post_sha": post.get("sha"), "post_version": post.get("version"), "fleet_states": sorted({str(e.get("state")) for e in receipt.get("fleet") or [] if isinstance(e, dict)}), + # C3: a committed run is ``success`` even while post-commit steps are owed; carry them + # so a client reports the owed step and its remedy instead of plain success. + "followups": [{"step": str(f.get("step")), "reason": str(f.get("reason") or "")} + for f in receipt.get("followups") or [] if isinstance(f, dict)], + "user_action": receipt.get("user_action") if isinstance(receipt.get("user_action"), dict) else None, } except Exception: return None diff --git a/locales/_keys.desktop.json b/locales/_keys.desktop.json index b0babc71d73a..67e80ac0a435 100644 --- a/locales/_keys.desktop.json +++ b/locales/_keys.desktop.json @@ -4697,6 +4697,7 @@ "updates.applyStatus.failed", "updates.applyStatus.noReturn", "updates.applyStatus.notAvailable", + "updates.applyStatus.owed", "updates.applyStatus.preparing", "updates.applyStatus.pulling", "updates.applyStatus.restarting", diff --git a/tests/hermes_cli/test_web_server_update_status.py b/tests/hermes_cli/test_web_server_update_status.py index 132e486291b7..e42a65d4d8b1 100644 --- a/tests/hermes_cli/test_web_server_update_status.py +++ b/tests/hermes_cli/test_web_server_update_status.py @@ -119,3 +119,35 @@ def test_root_latest_receipt_of_another_profile_never_certifies_this_action(self # The attached receipt names its writer: B's own when it exists, else A's (never B's). assert data["receipt"]["action_id"] == (b_id if b_outcome else a_id) assert data["receipt"]["outcome"] == (b_outcome or "success") + + @pytest.mark.parametrize("debt", ["none", "followups", "user_action"]) + def test_status_receipt_carries_owed_steps_of_a_committed_update(self, monkeypatch, tmp_path, debt): + # C3: owed post-commit steps keep the run a success (exit 0); the summary must still name + # them so the dashboard and Desktop report the debt instead of plain success. + from hermes_cli import update_receipt + + root, action_id = tmp_path / "root", "c" * 32 + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "coder")) + monkeypatch.setenv("HERMES_ACTION_ID", action_id) + update_receipt.begin_update_receipt() + if debt == "followups": + update_receipt.record_followup("dependencies", "synthetic selected-interpreter sync failed") + update_receipt.record_followup("config_migration", "disk full") + elif debt == "user_action": + update_receipt.record_user_action("autostash", "re-apply the parked stash") + update_receipt.finalize_update_receipt("success") + (tmp_path / "hermes-update.log").write_text( + f"=== hermes-update started 2026-08-17 11:19:34 {action_id} ===\n", encoding="utf-8") + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + + data = self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + receipt = data["receipt"] + assert [f["step"] for f in receipt["followups"]] == ( + ["dependencies", "config_migration"] if debt == "followups" else []) + assert receipt["user_action"] == ( + {"step": "autostash", "reason": "re-apply the parked stash"} if debt == "user_action" else None) + # Debt never changes the C3 exit mapping: owed follow-ups stay exit 0, a user action is partial. + assert data["exit_code"] == (1 if debt == "user_action" else 0) diff --git a/web/src/contexts/SystemActions.tsx b/web/src/contexts/SystemActions.tsx index db92b3651530..6f228299d522 100644 --- a/web/src/contexts/SystemActions.tsx +++ b/web/src/contexts/SystemActions.tsx @@ -9,6 +9,13 @@ import { type SystemAction, } from "./system-actions-context"; +// hermes-update status only: the receipt summary's owed post-commit steps (a committed update is +// exit 0 even while they are owed). +interface ReceiptDebt { + followups?: { step: string; reason: string }[]; + user_action?: { step: string; reason: string } | null; +} + const ACTION_NAMES: Record = { restart: "gateway-restart", update: "hermes-update", @@ -51,12 +58,20 @@ export function SystemActionsProvider({ ? sharedGatewayProfiles(await api.getStatus().catch(() => null)) : null; if (cancelled) return; + // C3: exit 0 with owed post-commit steps is still a success, but name what is owed. + const receipt = (resp as ActionStatusResponse & { receipt?: ReceiptDebt }).receipt; + const owed = [ + ...(receipt?.followups ?? []), + ...(receipt?.user_action ? [receipt.user_action] : []), + ].map((step) => step.step); setToast({ - type: ok ? "success" : "error", + type: ok && !owed.length ? "success" : "error", message: ok ? shared ? sharedGatewayRestartedMessage(shared.length) - : t.status.actionFinished + : owed.length + ? `${t.status.actionFinishedOwed}: ${owed.join(", ")}` + : t.status.actionFinished : `${t.status.actionFailed} (exit ${resp.exit_code ?? "?"})`, }); return; @@ -71,7 +86,7 @@ export function SystemActionsProvider({ return () => { cancelled = true; }; - }, [activeAction, t.status.actionFinished, t.status.actionFailed]); + }, [activeAction, t.status.actionFinished, t.status.actionFinishedOwed, t.status.actionFailed]); const runAction = useCallback( async (action: SystemAction) => { diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 28684fc45926..edee56dc8373 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -128,6 +128,7 @@ export const en: Translations = { status: { actionFailed: "Action failed", actionFinished: "Finished", + actionFinishedOwed: "Updated, but still owed (re-run `hermes update` to finish)", actions: "Actions", agent: "Agent", activeSessions: "Active Sessions", diff --git a/web/src/i18n/types.ts b/web/src/i18n/types.ts index 5fe858198831..289df1e09583 100644 --- a/web/src/i18n/types.ts +++ b/web/src/i18n/types.ts @@ -145,6 +145,7 @@ export interface Translations { status: { actionFailed: string; actionFinished: string; + actionFinishedOwed: string; actions: string; agent: string; connected: string; From ec8e0cf829b4ed59eb796019d71e838ed3a41eb9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:13:19 -0700 Subject: [PATCH 50/62] fix: dashboard owed-update toast string exists in every web locale `actionFinishedOwed` was added to the English catalog and the shared type only, so `tsc -b` (what the Docker image and nix builds run) failed on the 15 locales typed against the full shape, and Arabic fell back to English. Adds the translated string to all 16 catalogs. --- web/src/i18n/af.ts | 1 + web/src/i18n/ar.ts | 1 + web/src/i18n/de.ts | 1 + web/src/i18n/es.ts | 1 + web/src/i18n/fr.ts | 1 + web/src/i18n/ga.ts | 1 + web/src/i18n/hu.ts | 1 + web/src/i18n/it.ts | 1 + web/src/i18n/ja.ts | 1 + web/src/i18n/ko.ts | 1 + web/src/i18n/pt.ts | 1 + web/src/i18n/ru.ts | 1 + web/src/i18n/tr.ts | 1 + web/src/i18n/uk.ts | 1 + web/src/i18n/zh-hant.ts | 1 + web/src/i18n/zh.ts | 1 + 16 files changed, 16 insertions(+) diff --git a/web/src/i18n/af.ts b/web/src/i18n/af.ts index d84e86093b75..72d67ccaa107 100644 --- a/web/src/i18n/af.ts +++ b/web/src/i18n/af.ts @@ -95,6 +95,7 @@ export const af: Translations = { status: { actionFailed: "Aksie het misluk", actionFinished: "Voltooi", + actionFinishedOwed: "Bygewerk, maar nog uitstaande (voer `hermes update` weer uit om klaar te maak)", actions: "Aksies", agent: "Agent", activeSessions: "Aktiewe Sessies", diff --git a/web/src/i18n/ar.ts b/web/src/i18n/ar.ts index 4c84eff15d93..8db09d72c4a3 100644 --- a/web/src/i18n/ar.ts +++ b/web/src/i18n/ar.ts @@ -95,6 +95,7 @@ export const ar = defineLocale({ status: { actionFailed: "فشلت الإجراء", actionFinished: "انتهى", + actionFinishedOwed: "تم التحديث، لكن لا تزال هناك خطوات معلّقة (أعد تشغيل `hermes update` لإكمالها)", actions: "إجراءات", agent: "العامل", activeSessions: "الجلسات النشطة", diff --git a/web/src/i18n/de.ts b/web/src/i18n/de.ts index 1d10a98500ec..2acc8cba6dbc 100644 --- a/web/src/i18n/de.ts +++ b/web/src/i18n/de.ts @@ -95,6 +95,7 @@ export const de: Translations = { status: { actionFailed: "Aktion fehlgeschlagen", actionFinished: "Abgeschlossen", + actionFinishedOwed: "Aktualisiert, aber noch offen (`hermes update` erneut ausführen, um abzuschließen)", actions: "Aktionen", agent: "Agent", activeSessions: "Aktive Sitzungen", diff --git a/web/src/i18n/es.ts b/web/src/i18n/es.ts index 86a81d004723..f816a2e8808a 100644 --- a/web/src/i18n/es.ts +++ b/web/src/i18n/es.ts @@ -95,6 +95,7 @@ export const es: Translations = { status: { actionFailed: "Acción fallida", actionFinished: "Finalizado", + actionFinishedOwed: "Actualizado, pero aún pendiente (vuelve a ejecutar `hermes update` para terminar)", actions: "Acciones", agent: "Agente", activeSessions: "Sesiones activas", diff --git a/web/src/i18n/fr.ts b/web/src/i18n/fr.ts index b51154322ffd..5b9db2f76665 100644 --- a/web/src/i18n/fr.ts +++ b/web/src/i18n/fr.ts @@ -95,6 +95,7 @@ export const fr: Translations = { status: { actionFailed: "Action échouée", actionFinished: "Terminé", + actionFinishedOwed: "Mis à jour, mais encore en attente (relancez `hermes update` pour terminer)", actions: "Actions", agent: "Agent", activeSessions: "Sessions actives", diff --git a/web/src/i18n/ga.ts b/web/src/i18n/ga.ts index 131970fb5912..6896fd790a7f 100644 --- a/web/src/i18n/ga.ts +++ b/web/src/i18n/ga.ts @@ -95,6 +95,7 @@ export const ga: Translations = { status: { actionFailed: "Theip ar an ngníomh", actionFinished: "Críochnaithe", + actionFinishedOwed: "Nuashonraithe, ach fós le críochnú (rith `hermes update` arís chun é a chríochnú)", actions: "Gníomhartha", agent: "Agent", activeSessions: "Seisiúin ghníomhacha", diff --git a/web/src/i18n/hu.ts b/web/src/i18n/hu.ts index 6259ad8d59b3..d8b585732908 100644 --- a/web/src/i18n/hu.ts +++ b/web/src/i18n/hu.ts @@ -95,6 +95,7 @@ export const hu: Translations = { status: { actionFailed: "Művelet sikertelen", actionFinished: "Befejezve", + actionFinishedOwed: "Frissítve, de még függőben (futtasd újra a `hermes update` parancsot a befejezéshez)", actions: "Műveletek", agent: "Ügynök", activeSessions: "Aktív munkamenetek", diff --git a/web/src/i18n/it.ts b/web/src/i18n/it.ts index c11996f5f471..962668ebcc06 100644 --- a/web/src/i18n/it.ts +++ b/web/src/i18n/it.ts @@ -95,6 +95,7 @@ export const it: Translations = { status: { actionFailed: "Azione non riuscita", actionFinished: "Completata", + actionFinishedOwed: "Aggiornato, ma ancora in sospeso (riesegui `hermes update` per completare)", actions: "Azioni", agent: "Agente", activeSessions: "Sessioni attive", diff --git a/web/src/i18n/ja.ts b/web/src/i18n/ja.ts index 41d73a82fc87..3cb96bf9b9ff 100644 --- a/web/src/i18n/ja.ts +++ b/web/src/i18n/ja.ts @@ -95,6 +95,7 @@ export const ja: Translations = { status: { actionFailed: "アクションが失敗しました", actionFinished: "完了", + actionFinishedOwed: "更新済みですが未完了の処理があります（完了するには `hermes update` を再実行してください）", actions: "アクション", agent: "エージェント", activeSessions: "アクティブなセッション", diff --git a/web/src/i18n/ko.ts b/web/src/i18n/ko.ts index d1f2a179c205..5d2e99113e6e 100644 --- a/web/src/i18n/ko.ts +++ b/web/src/i18n/ko.ts @@ -95,6 +95,7 @@ export const ko: Translations = { status: { actionFailed: "작업 실패", actionFinished: "완료됨", + actionFinishedOwed: "업데이트되었지만 아직 남은 작업이 있습니다 (완료하려면 `hermes update`를 다시 실행하세요)", actions: "작업", agent: "에이전트", activeSessions: "활성 세션", diff --git a/web/src/i18n/pt.ts b/web/src/i18n/pt.ts index e88a896357af..c8e02037fef2 100644 --- a/web/src/i18n/pt.ts +++ b/web/src/i18n/pt.ts @@ -95,6 +95,7 @@ export const pt: Translations = { status: { actionFailed: "Ação falhou", actionFinished: "Concluído", + actionFinishedOwed: "Atualizado, mas ainda pendente (execute `hermes update` novamente para concluir)", actions: "Ações", agent: "Agente", activeSessions: "Sessões ativas", diff --git a/web/src/i18n/ru.ts b/web/src/i18n/ru.ts index 67357dae064c..2df6a64b5773 100644 --- a/web/src/i18n/ru.ts +++ b/web/src/i18n/ru.ts @@ -95,6 +95,7 @@ export const ru: Translations = { status: { actionFailed: "Ошибка действия", actionFinished: "Завершено", + actionFinishedOwed: "Обновлено, но работа ещё не завершена (запустите `hermes update` снова, чтобы закончить)", actions: "Действия", agent: "Агент", activeSessions: "Активные сессии", diff --git a/web/src/i18n/tr.ts b/web/src/i18n/tr.ts index 80c2812cf2c7..e6034c59c668 100644 --- a/web/src/i18n/tr.ts +++ b/web/src/i18n/tr.ts @@ -95,6 +95,7 @@ export const tr: Translations = { status: { actionFailed: "İşlem başarısız", actionFinished: "Tamamlandı", + actionFinishedOwed: "Güncellendi, ancak bekleyen adımlar var (bitirmek için `hermes update` komutunu yeniden çalıştırın)", actions: "İşlemler", agent: "Agent", activeSessions: "Aktif Oturumlar", diff --git a/web/src/i18n/uk.ts b/web/src/i18n/uk.ts index 60cc1eae4f30..1c21a45b2d0c 100644 --- a/web/src/i18n/uk.ts +++ b/web/src/i18n/uk.ts @@ -95,6 +95,7 @@ export const uk: Translations = { status: { actionFailed: "Дія не вдалася", actionFinished: "Завершено", + actionFinishedOwed: "Оновлено, але роботу ще не завершено (запустіть `hermes update` знову, щоб завершити)", actions: "Дії", agent: "Агент", activeSessions: "Активні сесії", diff --git a/web/src/i18n/zh-hant.ts b/web/src/i18n/zh-hant.ts index 3ff60ce22a7e..8fb2709a9d3d 100644 --- a/web/src/i18n/zh-hant.ts +++ b/web/src/i18n/zh-hant.ts @@ -95,6 +95,7 @@ export const zhHant: Translations = { status: { actionFailed: "動作失敗", actionFinished: "已完成", + actionFinishedOwed: "已更新，但仍有待完成的步驟（重新執行 `hermes update` 以完成）", actions: "動作", agent: "代理", activeSessions: "使用中工作階段", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index a156664aa4c7..d75e785464a7 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -94,6 +94,7 @@ export const zh: Translations = { status: { actionFailed: "操作失败", actionFinished: "已完成", + actionFinishedOwed: "已更新，但仍有待完成的步骤（重新运行 `hermes update` 以完成）", actions: "操作", agent: "代理", activeSessions: "活跃会话", From 4150ed1d61349fce876cfe9f2a32f67cb4209634 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:05:21 -0700 Subject: [PATCH 51/62] test(update): the utf8-children test stubs the treeless-checkout conversion main's convert_treeless_checkout_first (1c97b57a936) runs in update_completion._prepare before the completion children, and its git probe went through the Popen this test replaces to capture the children's argv, so it was counted as a third spawn. --- tests/hermes_cli/test_update_post_commit_followups.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py index b985bc0c3dc2..43c7d3164314 100644 --- a/tests/hermes_cli/test_update_post_commit_followups.py +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -293,7 +293,7 @@ def test_both_completion_children_run_in_utf8_mode(tmp_path, monkeypatch): into a pipe: each argv carries -X utf8 (win-utf8 review; the bootstrap child got it first).""" import pm from pm import client, environments, receipt - from hermes_cli import update_completion, venv_sync + from hermes_cli import gitlock, update_completion, venv_sync calls = [] @@ -303,9 +303,10 @@ def popen(command, **kwargs): monkeypatch.setattr(subprocess, "call", lambda command, **kw: calls.append(command) or 0) monkeypatch.setattr(subprocess, "Popen", popen) + # The treeless-checkout conversion's git probe would be captured as a third spawn. for module, name in ((venv_sync, "refuse_foreign_owned_venv"), (venv_sync, "arm_completion"), (venv_sync, "collect_superseded_generations"), (client, "ensure_tools_for_sync"), - (pm, "sync_venv")): + (pm, "sync_venv"), (gitlock, "convert_treeless_checkout_first")): monkeypatch.setattr(module, name, lambda *a, **k: None) monkeypatch.setattr(receipt, "last_for_update", lambda *a, **k: None) monkeypatch.setattr(environments, "project_python", lambda root: "python") From 8494c2859397d56f939c67ade5c0f07fa61e2fb6 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:20:24 -0700 Subject: [PATCH 52/62] fix: a named profile's updater and pm receipts see each other again C3 moved update receipts to the ROOT store (get_default_hermes_root) so the Desktop, the SSH observer and the hand-off scripts find a sticky profile's run. pm/receipt.py (byte-identical by ruling) still writes get_hermes_home(), so under HERMES_HOME=/profiles/foo the two stores split: `hermes -p foo pm status` lost update receipts, read_latest_receipt() consumers (fleet checks, serve obligations, dashboard) lost the profile's pm receipts, and receipts written to the profile store before the move were never read. Keep the root writer and add the profile store as a second store when it differs: latest.json is mirrored into it (pm status reads only that one), read_latest_receipt() takes the newest pointer of both (root on a tie, i.e. last writer wins as when they shared one folder) and read_receipt_for_action searches both archives. Default homes resolve to one store: unchanged. --- hermes_cli/update_receipt.py | 40 ++++++++++--- .../test_update_receipt_profile_store.py | 59 +++++++++++++++++++ 2 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 tests/hermes_cli/test_update_receipt_profile_store.py diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index 8e7596500a66..d30ec60bfbd6 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -235,6 +235,26 @@ def _receipt_dir() -> Path: return get_default_hermes_root() / "logs" / "update_receipts" +def _receipt_dirs() -> list[Path]: + """The root store, then the profile's own when a named profile makes them differ. ``pm/`` + writes its sync receipts to ``get_hermes_home()`` and receipts from before the root move live + there too, so readers take both and writers mirror ``latest.json`` into it (``hermes -p + pm status`` reads only that one).""" + from hermes_constants import get_hermes_home + + root, profile = _receipt_dir(), get_hermes_home() / "logs" / "update_receipts" + return [root] if profile.resolve() == root.resolve() else [root, profile] + + +def _write_latest(payload: bytes) -> None: + """Replace ``latest.json`` in every store a reader of this home consults.""" + from hermes_cli.runtime_state import _atomic_bytes + + for directory in _receipt_dirs(): + directory.mkdir(parents=True, exist_ok=True) + _atomic_bytes(directory / "latest.json", payload) + + def _run_file(directory: Path, data: dict[str, Any]) -> Path: """One archive file per run, named at begin so the running and terminal records coincide.""" stamp = re.sub(r"[^0-9]", "", str(data.get("started_at") or ""))[:14] or time.strftime("%Y%m%d%H%M%S") @@ -261,7 +281,7 @@ def _persist_running(data: dict[str, Any]) -> None: "writer_create_time": _process_create_time(os.getpid())}, indent=2, default=str) + "\n").encode("utf-8") _atomic_bytes(path, payload) - _atomic_bytes(directory / "latest.json", payload) + _write_latest(payload) def _owner_alive(record: dict[str, Any]) -> bool: @@ -305,7 +325,7 @@ def reconcile_interrupted_runs() -> list[dict[str, Any]]: payload = (json.dumps(record, indent=2, default=str) + "\n").encode("utf-8") _atomic_bytes(path, payload) if latest.get("update_id") == record.get("update_id"): - _atomic_bytes(directory / "latest.json", payload) + _write_latest(payload) interrupted.append(record) for record in interrupted: print(f"⚠ The previous update ({record.get('started_at')}) was interrupted before it finished " @@ -420,7 +440,7 @@ def amend_terminal_followup(update_id: str, step: str, reason: str) -> None: payload = (json.dumps(record, indent=2, default=str) + "\n").encode("utf-8") _atomic_bytes(path, payload) if (read_latest_receipt() or {}).get("update_id") == update_id: - _atomic_bytes(directory / "latest.json", payload) + _write_latest(payload) def _record(method: str, what: str, *args: Any, **kwargs: Any) -> None: @@ -554,7 +574,7 @@ def finalize_update_receipt(outcome: str, fleet: list | None = None, stop_reason payload = (json.dumps(receipt.data, indent=2, default=str) + "\n").encode("utf-8") _atomic_bytes(path, payload) with suppress(Exception): # stable pointer for the dashboard/desktop - _atomic_bytes(directory / "latest.json", payload) + _write_latest(payload) _prune_old_receipts(directory) _publish_shared_metrics(receipt.data) return path @@ -731,7 +751,7 @@ def settle_latest_receipt_fleet(fleet: list[dict[str, Any]], *, discharges) -> b receipt["gateway_restart"] = gateway_restart if not discharges(receipt): return False - path.write_text(json.dumps(receipt, indent=2, default=str), encoding="utf-8") + _write_latest((json.dumps(receipt, indent=2, default=str) + "\n").encode("utf-8")) return True except Exception as exc: logger.debug("Could not settle latest update receipt from the live fleet: %s", exc) @@ -741,9 +761,12 @@ def settle_latest_receipt_fleet(fleet: list[dict[str, Any]], *, discharges) -> b def read_latest_receipt() -> Optional[dict[str, Any]]: """Read the most recent update receipt, or None. Never raises.""" with suppress(Exception): - path = _receipt_dir() / "latest.json" - if not path.is_file(): + # The newest pointer wins (the root on a tie): the last writer of either store, exactly + # as when updater and pm shared one folder. + points = [d / "latest.json" for d in _receipt_dirs() if (d / "latest.json").is_file()] + if not points: return None + path = max(points, key=lambda p: p.stat().st_mtime) payload = json.loads(path.read_text(encoding="utf-8-sig")) return payload if isinstance(payload, dict) else None return None @@ -756,7 +779,8 @@ def read_receipt_for_action(action_id: str) -> Optional[dict[str, Any]]: if latest and latest.get("action_id") == action_id: return latest with suppress(Exception): - for path in sorted(_receipt_dir().glob("update_*.json"), key=lambda p: p.stat().st_mtime, reverse=True): + archives = (p for d in _receipt_dirs() for p in d.glob("update_*.json")) + for path in sorted(archives, key=lambda p: p.stat().st_mtime, reverse=True): with suppress(Exception): payload = json.loads(path.read_text(encoding="utf-8-sig")) if isinstance(payload, dict) and payload.get("action_id") == action_id: diff --git a/tests/hermes_cli/test_update_receipt_profile_store.py b/tests/hermes_cli/test_update_receipt_profile_store.py new file mode 100644 index 000000000000..b0cab31aa271 --- /dev/null +++ b/tests/hermes_cli/test_update_receipt_profile_store.py @@ -0,0 +1,59 @@ +"""Under a named profile the updater (root store) and pm (profile store) still see each other's receipts.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import textwrap + +ROOT = Path(__file__).resolve().parents[2] + +_PROBE = textwrap.dedent(""" + import json, sys, time + from pathlib import Path + sys.path.insert(0, sys.argv[1]) + from hermes_cli import update_receipt as receipts + from pm import receipt as pm_receipt + + old = Path(sys.argv[2]) / 'logs' / 'update_receipts' + old.mkdir(parents=True) + (old / 'update_20260101_000000_1_old.json').write_text( + json.dumps({'update_id': 'old', 'action_id': 'a' * 32, 'outcome': 'success'}), encoding='utf-8') + receipts.begin_update_receipt() + receipts.finalize_update_receipt('success') + update_id = receipts.read_latest_receipt()['update_id'] + pm_sees_update = (pm_receipt.latest() or {}).get('update_id') + time.sleep(0.05) + pm_receipt.begin('sync') + pm_receipt.finalize('ok') + Path('result.json').write_text(json.dumps({ + 'update_id': update_id, 'pm_sees_update': pm_sees_update, + 'updater_sees_pm': (receipts.read_latest_receipt() or {}).get('kind'), + 'old_archive': (receipts.read_receipt_for_action('a' * 32) or {}).get('update_id'), + 'root_archives': [p.name for p in receipts._receipt_dir().glob('update_*.json')], + })) +""") + + +def test_named_profile_sees_update_and_pm_receipts_in_both_directions(tmp_path): + home = tmp_path / "account" + profile = home / ".hermes" / "profiles" / "foo" + profile.mkdir(parents=True) + script = tmp_path / "probe.py" + script.write_text(_PROBE, encoding="utf-8") + env = {key: value for key, value in os.environ.items() if not key.startswith(("HERMES_", "PYTEST_"))} + env.update(HOME=str(home), USERPROFILE=str(home), HERMES_HOME=str(profile), + HERMES_RUNTIME_DIR=str(tmp_path / "tools"), PYTHONDONTWRITEBYTECODE="1", + PYTHONPATH=os.pathsep.join([str(ROOT), *filter(None, sys.path)])) + done = subprocess.run([sys.executable, str(script), str(ROOT), str(profile)], env=env, cwd=tmp_path, + text=True, capture_output=True, timeout=60) + assert done.returncode == 0, done.stdout + done.stderr + result = json.loads((tmp_path / "result.json").read_text(encoding="utf-8")) + + # The updater still writes the root store the Desktop and hand-off scripts read. + assert any(result["update_id"] in name for name in result["root_archives"]) + assert result["pm_sees_update"] == result["update_id"] # `hermes -p foo pm status` + assert result["updater_sees_pm"] == "sync" # read_latest_receipt() consumers + assert result["old_archive"] == "old" # receipts written to the profile store before the move From 2715cff93c10246ff0bbab54c3c68887c46ae289 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:27:19 -0700 Subject: [PATCH 53/62] fix: a restarted dashboard keeps its update action identity past the log tail After a dashboard restart the only record of which update THIS dashboard started was the `=== hermes-update started ... ===` header, read from a 2,000-line tail. A build that printed more than 2,000 lines (or a rotated log) hid it: spawned_action_id became None, the matching root completion marker and the matching archived success receipt were both refused, and the route answered running=false/exit_code=null, which the web poller shows as "Action failed (exit ?)". _spawn_hermes_action now also writes the id to a `.action_id` sidecar (atomic) next to the log; the status route falls back to it when no header is visible. A visible header still wins (a dashboard predating the sidecar wrote only the header, and the newest header is the latest spawn). The identity selects B's own archived receipt even when another profile's run is latest; failed/unfinished actions still report exit null. --- hermes_cli/web_routers/actions.py | 12 ++++- hermes_cli/web_server_gateway.py | 5 +++ .../test_web_server_update_status.py | 44 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/hermes_cli/web_routers/actions.py b/hermes_cli/web_routers/actions.py index 050ee2dfd1d7..f0c60579a025 100644 --- a/hermes_cli/web_routers/actions.py +++ b/hermes_cli/web_routers/actions.py @@ -147,6 +147,16 @@ def _latest_spawned_update_action_id(lines: List[str]) -> Optional[str]: return None +def _persisted_action_id(log_dir: Path, name: str) -> Optional[str]: + """Action id of the latest spawn of ``name`` from its sidecar (survives the log's tail + bound and rotation). A visible start header wins: a dashboard predating the sidecar wrote + only the header, and the newest header IS the latest spawn.""" + with contextlib.suppress(OSError, UnicodeDecodeError): + value = (log_dir / f"{name}.action_id").read_text(encoding="utf-8-sig").strip() + return value if re.fullmatch(r"[0-9a-f]{32}", value) else None + return None + + @router.post("/api/gateway/restart") async def restart_gateway(profile: Optional[str] = None): """Kick off a ``hermes gateway restart`` in the background.""" @@ -374,7 +384,7 @@ async def get_action_status(name: str, lines: int = 200): durable_update_action_id = _durable_completed_update_action_id( _tail_lines(get_default_hermes_root() / "logs" / "update.log", 2000)) spawned_action_id = (_latest_spawned_update_action_id(_tail_lines(log_dir / log_file_name, 2000)) - or _ACTION_IDS.get(name)) + or _persisted_action_id(log_dir, name) or _ACTION_IDS.get(name)) if durable_update_action_id != spawned_action_id: # The root log is shared by every profile: another profile's (or an older) run's # completion never certifies the action this dashboard started. diff --git a/hermes_cli/web_server_gateway.py b/hermes_cli/web_server_gateway.py index 9f4b5bb0c0ad..2aa9d6a61460 100644 --- a/hermes_cli/web_server_gateway.py +++ b/hermes_cli/web_server_gateway.py @@ -462,6 +462,11 @@ def _spawn_hermes_action( action_id = (env_overrides or {}).get("HERMES_ACTION_ID") stamp = time.strftime('%Y-%m-%d %H:%M:%S') + (f" {action_id}" if action_id else "") log_file.write(f"\n=== {name} started {stamp} ===\n".encode()) + if action_id: + # ...and a sidecar keeps it once build output pushes the header past the status route's + # bounded tail, or the log rotates: a restarted dashboard still knows which action it ran. + from hermes_cli.runtime_state import _atomic_bytes + _atomic_bytes(_ACTION_LOG_DIR / f"{name}.action_id", action_id.encode("ascii")) from hermes_cli._launchers import runtime_command cmd = runtime_command(PROJECT_ROOT, subcommand) diff --git a/tests/hermes_cli/test_web_server_update_status.py b/tests/hermes_cli/test_web_server_update_status.py index e42a65d4d8b1..fdab5db065cf 100644 --- a/tests/hermes_cli/test_web_server_update_status.py +++ b/tests/hermes_cli/test_web_server_update_status.py @@ -87,6 +87,50 @@ def status_after_root_completion(action_id): assert own["exit_code"] == 0 assert own["action_id"] == b_id + @pytest.mark.parametrize("log_shape,b_outcome", [ + ("short", "success"), ("long", "success"), ("rotated", "success"), ("long", "failed"), ("long", None)]) + def test_restarted_dashboard_keeps_its_action_identity_past_the_log_tail( + self, monkeypatch, tmp_path, log_shape, b_outcome): + # The status route reads at most 2,000 log lines: build output (or rotation) pushing the + # start header out must not turn B's own success into exit null ("Action failed (exit ?)"), + # nor let profile A's newer latest receipt answer for B. + from hermes_cli import update_receipt + + root, b_id, a_id = tmp_path / "root", "b" * 32, "a" * 32 + (root / "logs").mkdir(parents=True) + b_logs = tmp_path / "b-logs" + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", b_logs) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + monkeypatch.setattr(_web_server_gateway.subprocess, "Popen", lambda *a, **kw: types.SimpleNamespace(pid=7)) + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + _web_server_gateway._spawn_hermes_action(["update"], "hermes-update", env_overrides={"HERMES_ACTION_ID": b_id}) + for profile, action_id, outcome in (("b", b_id, b_outcome), ("a", a_id, "success")): + if outcome: + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / profile)) + monkeypatch.setenv("HERMES_ACTION_ID", action_id) + update_receipt.begin_update_receipt() + update_receipt.finalize_update_receipt(outcome) + monkeypatch.delenv("HERMES_ACTION_ID") + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + log = b_logs / "hermes-update.log" + if log_shape == "long": + with log.open("a", encoding="utf-8") as handle: + handle.write("build output\n" * 2001) + elif log_shape == "rotated": + log.unlink() + if b_outcome == "success": + (root / "logs" / "update.log").write_text(f"=== hermes-update completed {b_id} ===\n", encoding="utf-8") + _web_server_gateway._ACTION_PROCS.clear() # dashboard restarted: in-memory registries lost + _web_server_gateway._ACTION_IDS.clear() + + data = self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + assert data["running"] is False + assert data["exit_code"] == (0 if b_outcome == "success" else None) + assert data.get("action_id") == (b_id if b_outcome == "success" else None) + assert data["receipt"]["action_id"] == (b_id if b_outcome else a_id) + @pytest.mark.parametrize("b_outcome", [None, "failed", "success"]) def test_root_latest_receipt_of_another_profile_never_certifies_this_action(self, monkeypatch, tmp_path, b_outcome): # The receipt store is root-wide too: A's later success in latest.json must not become B's From 01d789b09acc185db05937da6ad743350fed0fec Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:30:57 -0700 Subject: [PATCH 54/62] fix: managed SSH results name owed work on restore-failed and partial updates managedUpdateMessage returned a generic update/restore message before reading the receipt's debt whenever the combined outcome was not `updated`. Two reachable outcomes lost necessary recovery data from the displayed result and the Update-all row: a committed success whose profile restoration failed (owed follow-ups dropped) and a `partial` run whose record_user_action carries the exact parked-local-change instruction (dropped; only "the remote update failed" shown). Compose the debt with every verdict: owed steps are listed for any terminal receipt, `owed` is returned, follow-ups get the rerun advice by name, and a user action is shown verbatim (never as "re-run hermes update", which does not restore a parked stash). updateOk/restoreOk stay independent. --- .../electron/managed-ssh-update.test.ts | 46 +++++++++++++++++-- apps/desktop/electron/managed-ssh-update.ts | 35 ++++++++------ 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/apps/desktop/electron/managed-ssh-update.test.ts b/apps/desktop/electron/managed-ssh-update.test.ts index 8196caf91fd4..348e83e1c7f4 100644 --- a/apps/desktop/electron/managed-ssh-update.test.ts +++ b/apps/desktop/electron/managed-ssh-update.test.ts @@ -512,18 +512,22 @@ test.runIf(process.platform !== 'win32')( async () => { const home = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-owed-')) - const run = async (followups: unknown[]) => { + const run = async ( + followups: unknown[], + { exitCode = 0, outcome = 'success', restoreFails = false, userAction = null as unknown } = {} + ) => { const receipts = path.join(home, 'logs', 'update_receipts') await mkdir(receipts, { recursive: true }) - await writeFile(path.join(home, `.update_exit_code.${CORRELATION}`), '0') + await writeFile(path.join(home, `.update_exit_code.${CORRELATION}`), String(exitCode)) await writeFile( path.join(receipts, 'update_x.json'), JSON.stringify({ correlation_id: CORRELATION, - outcome: 'success', + outcome, started_at: '2026-08-23T00:00:00Z', finished_at: '2026-08-23T00:01:00Z', - followups + followups, + user_action: userAction }) ) @@ -543,7 +547,11 @@ test.runIf(process.platform !== 'win32')( updateRemote: async () => ({ exitCode: observed.exitCode!, receipt: observed.receipt! }), awaitRestoreClearance: async () => {}, closeTransports: async () => {}, - restoreScope: async () => {}, + restoreScope: async () => { + if (restoreFails) { + throw new Error('restore refused') + } + }, releaseGate: () => {} }) } @@ -564,6 +572,34 @@ test.runIf(process.platform !== 'win32')( assert.equal(clean.ok, true) assert.equal(clean.owed, undefined) assert.equal(clean.message, 'Remote Hermes updated and every managed SSH profile is ready.') + + // The debt survives a failed restoration: update and restore verdicts stay independent. + const unrestored = await run([{ step: 'dependencies', reason: 'sync failed' }], { restoreFails: true }) + + assert.equal(unrestored.updateOk, true) + assert.equal(unrestored.restoreOk, false) + assert.deepEqual(unrestored.owed, [{ step: 'dependencies', reason: 'sync failed' }]) + assert.match( + managedSshUpdateAllRow({ id: 'home' }, unrestored).detail || '', + /still owed: dependencies \(sync failed\)/ + ) + + // record_user_action turns a success into partial/exit 1: its exact manual instruction is shown, + // and the rerun advice is never offered for a parked stash. + const instruction = 'Run `git stash apply stash@{0}` in /opt/hermes to restore your local changes.' + const partial = await run([], { + exitCode: 1, + outcome: 'partial', + userAction: { step: 'local_changes', reason: instruction } + }) + + assert.equal(partial.updateOk, false) + assert.equal(partial.restoreOk, true) + assert.deepEqual(partial.owed, [{ step: 'local_changes', reason: instruction }]) + const detail = managedSshUpdateAllRow({ id: 'home' }, partial).detail || '' + assert.ok(detail.includes(instruction), detail) + assert.match(detail, /local_changes/) + assert.doesNotMatch(detail, /Re-run `hermes update`/) } finally { await rm(home, { force: true, recursive: true }) } diff --git a/apps/desktop/electron/managed-ssh-update.ts b/apps/desktop/electron/managed-ssh-update.ts index 3e9b1bd07aaa..808ee04525c3 100644 --- a/apps/desktop/electron/managed-ssh-update.ts +++ b/apps/desktop/electron/managed-ssh-update.ts @@ -506,33 +506,42 @@ function withReceiptDebt(receipt: any): ManagedUpdateReceiptSummary { } // C3: a committed update is a success even when post-commit steps are owed; -// name each one and its remedy instead of claiming everything is ready. +// name each one and its remedy instead of claiming everything is ready. The +// debt rides on every terminal receipt, so it is composed with the update and +// restoration verdicts, never only on success: a committed run whose profile +// restore failed, and a partial run that parked local changes, still owe it. function managedUpdateMessage( outcome: ManagedUpdateOutcome, restoreOk: boolean, receipt: ManagedUpdateReceiptSummary | undefined ): { owed?: ManagedUpdateOwedStep[]; message: string } { - if (outcome !== 'updated') { - return { - message: restoreOk - ? 'The remote update failed, but every managed SSH profile was restored.' - : 'The remote update transaction could not restore every managed SSH profile.' - } - } - const followups = receipt?.followups ?? [] const userAction = receipt?.userAction const owed = [...followups, ...(userAction ? [userAction] : [])] + const status = + outcome === 'updated' + ? 'Remote Hermes updated' + : restoreOk + ? 'The remote update failed, but every managed SSH profile was restored.' + : 'The remote update transaction could not restore every managed SSH profile.' + if (!owed.length) { - return { message: 'Remote Hermes updated and every managed SSH profile is ready.' } + return { message: outcome === 'updated' ? `${status} and every managed SSH profile is ready.` : status } } const message = - 'Remote Hermes updated, but these steps are still owed: ' + - owed.map(step => (step.reason ? `${step.step} (${step.reason})` : step.step)).join('; ') + + (outcome === 'updated' ? `${status}, but these steps are still owed: ` : `${status} These steps are still owed: `) + + [ + ...followups.map(step => (step.reason ? `${step.step} (${step.reason})` : step.step)), + ...(userAction ? [userAction.step] : []) + ].join('; ') + '.' + - (followups.length ? ' Re-run `hermes update` on the remote to finish them.' : '') + + // Only follow-ups are finished by a rerun; a user action (e.g. a parked + // stash) is the producer's exact manual instruction, shown verbatim. + (followups.length + ? ` Re-run \`hermes update\` on the remote to finish ${followups.map(step => step.step).join(', ')}.` + : '') + (userAction?.reason ? ` ${userAction.reason}` : '') return { owed, message } From d09626bdd9c45435bba74c895c7ee614132f73b7 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:35:20 -0700 Subject: [PATCH 55/62] fix: a post-commit follow-up is owed once, through one owe_followup helper "record the follow-up, then amend the closed receipt" was hand-written at six sites (three Windows-resume sites, the historical-takeover resume, gateway_restart, channel_adoption). Two of them -- _complete_source_update's Windows resume and channel_adoption -- amended without checking that the run's receipt was already closed. On an open run record_followup has just persisted the follow-up into the running record, so the amend appended a second copy to that durable record and to latest.json (what an interrupted run leaves behind). update_receipt.owe_followup(update_id, step, reason) records the step and amends the archive only once the run is closed; every site now calls it. --- hermes_cli/update_cmd.py | 21 +++++------ hermes_cli/update_completion.py | 6 ++-- hermes_cli/update_finish.py | 10 ++---- hermes_cli/update_receipt.py | 9 +++++ .../test_update_receipt_owe_followup.py | 36 +++++++++++++++++++ 5 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 tests/hermes_cli/test_update_receipt_owe_followup.py diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 6d0929508622..c45e99ffc96c 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -748,12 +748,10 @@ def _resume_paused_gateways_at_exit(token: dict | None, request: dict | None) -> try: _m()._resume_windows_gateways_after_update(token) except Exception as exc: # health: allow BLE001 -- a restart failure is owed, not the update's status - from hermes_cli.update_receipt import _current, amend_terminal_followup, record_followup + from hermes_cli.update_receipt import owe_followup - reason = f"Windows gateway recovery failed: {exc}" - record_followup("windows_resume", reason) - if _current.get() is None and request: # the run already finalized its receipt - amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", reason) + owe_followup(request["receipt"]["update_id"] if request else None, "windows_resume", + f"Windows gateway recovery failed: {exc}") def _complete_source_update(request: dict | None) -> None: @@ -812,11 +810,9 @@ def _complete_source_update(request: dict | None) -> None: try: _m()._resume_windows_gateways_after_update(token) except Exception as exc: # health: allow BLE001 -- the code is committed (C3) - from hermes_cli.update_receipt import amend_terminal_followup, record_followup + from hermes_cli.update_receipt import owe_followup - reason = f"Windows gateway recovery failed: {exc}" - record_followup("windows_resume", reason) - amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", reason) + owe_followup(request["receipt"]["update_id"], "windows_resume", f"Windows gateway recovery failed: {exc}") _settle_windows_resume(request) if result["exit_code"]: raise SystemExit(result["exit_code"]) @@ -826,11 +822,10 @@ def _complete_source_update(request: dict | None) -> None: if adopt_retired_channel(request): print(f"→ Source subscription moved to {request['channel_retirement']['destination']}") except Exception as exc: # health: allow BLE001 -- the code is committed (C3); the next update re-adopts - from hermes_cli.update_receipt import amend_terminal_followup, record_followup + from hermes_cli.update_receipt import owe_followup - reason = str(exc) or type(exc).__name__ - record_followup("channel_adoption", reason, retry="the next `hermes update` adopts it again") - amend_terminal_followup(request["receipt"]["update_id"], "channel_adoption", reason) + owe_followup(request["receipt"]["update_id"], "channel_adoption", str(exc) or type(exc).__name__, + retry="the next `hermes update` adopts it again") def _reconcile_diverged_checkout(git_cmd, branch: str, pre_pull_sha, *, target_ref=None) -> None: diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index c209e851a8e4..b4631bdad019 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -462,10 +462,8 @@ def _finish(request: dict, result_path: Path) -> int: from hermes_cli.update_cmd import _resume_windows_gateways_after_update _resume_windows_gateways_after_update(request["windows_resume"]) except Exception as exc: - step_reason = f"Windows gateway recovery failed: {exc}" - update_receipt.record_followup("windows_resume", step_reason) - if update_receipt._current.get() is None: # verification already finalized this run - update_receipt.amend_terminal_followup(request["receipt"]["update_id"], "windows_resume", step_reason) + update_receipt.owe_followup(request["receipt"]["update_id"], "windows_resume", + f"Windows gateway recovery failed: {exc}") update_receipt.finalize_pending_update_receipt(code, reason) terminal_receipt = _read_terminal_receipt(request) if terminal_receipt and terminal_receipt.get("outcome") == "success": diff --git a/hermes_cli/update_finish.py b/hermes_cli/update_finish.py index beb386eb45d5..2014b0a1b6cb 100644 --- a/hermes_cli/update_finish.py +++ b/hermes_cli/update_finish.py @@ -63,10 +63,7 @@ def finish_update(*, root, assume_yes, gateway_mode, pre_update_snapshot_id, _verify_fleet_after_update(restarted, _pre_update_plan=plan, _windows_gateway_resume=windows_resume, update_complete=bool(runtime_safe) and not tail_owed) except (Exception, SystemExit) as exc: # health: allow BLE001 -- the code is committed (C3): the fleet obligation stays armed - reason = str(exc) or type(exc).__name__ - record_followup("gateway_restart", reason) - if update_receipt._current.get() is None and update_id: # verification already finalized the run - update_receipt.amend_terminal_followup(update_id, "gateway_restart", reason) + update_receipt.owe_followup(update_id, "gateway_restart", str(exc) or type(exc).__name__) def _restore_plan(data): @@ -187,10 +184,7 @@ def main(context: Path, result: Path) -> int: resume(token) except Exception as exc: # noqa: BLE001 — a restart failure is owed, never the exit status (C3) attempted = True - reason = f"Windows gateway recovery failed: {exc}" - update_receipt.record_followup("windows_resume", reason) - if update_receipt._current.get() is None: # the run already finalized its receipt - update_receipt.amend_terminal_followup(request["update_id"], "windows_resume", reason) + update_receipt.owe_followup(request["update_id"], "windows_resume", f"Windows gateway recovery failed: {exc}") handled = cli_started or (begun and update_receipt._current.get() is None) written = update_receipt.finalize_pending_update_receipt(code, "historical takeover completion") result.write_text(json.dumps({ diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index d30ec60bfbd6..f107a8ff8a24 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -443,6 +443,15 @@ def amend_terminal_followup(update_id: str, step: str, reason: str) -> None: _write_latest(payload) +def owe_followup(update_id: Optional[str], step: str, reason: str, **retry: str) -> None: + """A post-commit step failed: record it on the run while its receipt is open, else amend the + run's closed receipt. Amending an OPEN run would append a second copy beside the one + ``record_followup`` just persisted. Never raises.""" + record_followup(step, reason, **retry) + if _current.get() is None and update_id: + amend_terminal_followup(update_id, step, reason) + + def _record(method: str, what: str, *args: Any, **kwargs: Any) -> None: """Invoke ``method`` on the active receipt; no-op when none, never raises. diff --git a/tests/hermes_cli/test_update_receipt_owe_followup.py b/tests/hermes_cli/test_update_receipt_owe_followup.py new file mode 100644 index 000000000000..381e4bb8f660 --- /dev/null +++ b/tests/hermes_cli/test_update_receipt_owe_followup.py @@ -0,0 +1,36 @@ +"""A post-commit follow-up lands on the run exactly once, whether its receipt is still open or closed.""" +from __future__ import annotations + +import json + +from hermes_cli import update_receipt + + +def _steps(path): + return [f["step"] for f in json.loads(path.read_text(encoding="utf-8")).get("followups") or []] + + +def test_follow_up_on_an_open_run_is_recorded_once_not_amended_again(_isolate_hermes_home): + update_receipt.begin_update_receipt() + run = update_receipt._current.get().data + archive = update_receipt._run_file(update_receipt._receipt_dir(), run) + + update_receipt.owe_followup(run["update_id"], "channel_adoption", "remote refused", + retry="the next `hermes update` adopts it again") + + # The durable running record (what an interrupted run leaves behind) owes it once. + assert _steps(archive) == ["channel_adoption"] + update_receipt.finalize_update_receipt("success") + assert _steps(archive) == ["channel_adoption"] + + +def test_follow_up_after_the_run_closed_amends_its_terminal_receipt(_isolate_hermes_home): + update_receipt.begin_update_receipt() + run = dict(update_receipt._current.get().data) + update_receipt.finalize_update_receipt("success") + + update_receipt.owe_followup(run["update_id"], "windows_resume", "Windows gateway recovery failed: boom") + + assert _steps(update_receipt._run_file(update_receipt._receipt_dir(), run)) == ["windows_resume"] + assert [f["step"] for f in update_receipt.read_latest_receipt()["followups"]] == ["windows_resume"] + assert update_receipt.read_latest_receipt()["outcome"] == "success" From c8d2025392add709d101cb7b559f18c8b28f4f38 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:35:58 -0700 Subject: [PATCH 56/62] fix: the dashboard reads update.log through the one log-path resolver The status route rebuilt /logs/update.log by hand while hermes_cli.logs.log_file_path("update") already owns that placement (root home for the update tee). One resolver, so a future move cannot split the writer and this reader. --- hermes_cli/web_routers/actions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hermes_cli/web_routers/actions.py b/hermes_cli/web_routers/actions.py index f0c60579a025..661379ac1cd5 100644 --- a/hermes_cli/web_routers/actions.py +++ b/hermes_cli/web_routers/actions.py @@ -379,10 +379,9 @@ async def get_action_status(name: str, lines: int = 200): if name == "hermes-update": # ``hermes update`` mirrors to the ROOT home's update.log (main_dashboard), never this # dashboard's profile home: read it where it is written. - from hermes_constants import get_default_hermes_root + from hermes_cli.logs import log_file_path - durable_update_action_id = _durable_completed_update_action_id( - _tail_lines(get_default_hermes_root() / "logs" / "update.log", 2000)) + durable_update_action_id = _durable_completed_update_action_id(_tail_lines(log_file_path("update"), 2000)) spawned_action_id = (_latest_spawned_update_action_id(_tail_lines(log_dir / log_file_name, 2000)) or _persisted_action_id(log_dir, name) or _ACTION_IDS.get(name)) if durable_update_action_id != spawned_action_id: From 8fc2475b8af581ada203c85f6e5a214bc0db4548 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:01:06 -0700 Subject: [PATCH 57/62] fix: Desktop backend update names owed work on partial and receipt-proven runs The backend-apply store consulted the receipt's owed steps only on exit 0. A partial run (record_user_action turns a committed success into partial / exit 1, e.g. a stash whose restore conflicted) fell through to "Backend update failed." and a receipt-proven partial after the restart to "did not come back online", dropping the exact recovery instruction; and the success branch read followups from whatever receipt the status route attached, which is the latest receipt when this action has none. Add @hermes/shared updateDebt(receipt, actionId): the debt of a committed (success|partial) run, only when the receipt's action_id is the id the start request returned. The store composes it on success and failure: follow-ups keep the rerun remedy, a user action shows the producer's instruction verbatim (a rerun does not restore a parked stash). The existing owed-success spec's receipt gains the matching action_id the real route emits. --- apps/desktop/src/store/updates.test.ts | 44 +++++++++++++- apps/desktop/src/store/updates.ts | 82 ++++++++++++++++---------- apps/shared/src/index.ts | 1 + apps/shared/src/update-debt.ts | 49 +++++++++++++++ 4 files changed, 144 insertions(+), 32 deletions(-) create mode 100644 apps/shared/src/update-debt.ts diff --git a/apps/desktop/src/store/updates.test.ts b/apps/desktop/src/store/updates.test.ts index a0cb66cb6375..5b8811c54ea4 100644 --- a/apps/desktop/src/store/updates.test.ts +++ b/apps/desktop/src/store/updates.test.ts @@ -1496,7 +1496,12 @@ describe('applyBackendUpdate recovery', () => { name: 'hermes-update', pid: null, running: false, - receipt: { outcome: 'success', followups: [{ step: 'dependencies', reason: 'sync failed' }], user_action: null } + receipt: { + action_id: 'e'.repeat(32), + outcome: 'success', + followups: [{ step: 'dependencies', reason: 'sync failed' }], + user_action: null + } }) notifySpy.mockClear() @@ -1511,6 +1516,43 @@ describe('applyBackendUpdate recovery', () => { ) }) + it("names a partial update's owed user action verbatim, only from this action's own receipt", async () => { + const instruction = 'Your local changes are parked in stash@{0}; run `git stash pop` in ~/.hermes/hermes-agent.' + + const status = (actionId: string) => ({ + exit_code: 1, + lines: ['update parked local changes'], + name: 'hermes-update', + pid: null, + running: false, + receipt: { + action_id: actionId, + outcome: 'partial', + followups: [], + user_action: { step: 'local_changes', reason: instruction } + } + }) + + updateHermesSpy.mockResolvedValue({ action_id: 'c'.repeat(32), ok: true, name: 'hermes-update', pid: 1 }) + getActionStatusSpy.mockResolvedValue(status('c'.repeat(32))) + const own = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(1500) + const result = await own + + // exit 1 stays a failed apply, but the committed run's debt is named, never as "re-run hermes update". + expect(result).toMatchObject({ ok: false, error: 'apply-failed' }) + expect(result.message).toContain(`local_changes: ${instruction}`) + expect(result.message).not.toMatch(/Re-run `hermes update`/) + expect($backendUpdateApply.get()).toMatchObject({ stage: 'error', message: result.message }) + + // The status route attaches the latest receipt when this action has none: another run's debt is not ours. + getActionStatusSpy.mockResolvedValue(status('d'.repeat(32))) + const foreign = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(1500) + + expect((await foreign).message).not.toContain('local_changes') + }) + it("never lets another action's receipt certify this update", async () => { const receipt = (actionId: string) => ({ exit_code: null, diff --git a/apps/desktop/src/store/updates.ts b/apps/desktop/src/store/updates.ts index b78ecf5ffa7d..4b39552ae50e 100644 --- a/apps/desktop/src/store/updates.ts +++ b/apps/desktop/src/store/updates.ts @@ -3,6 +3,7 @@ * surfaces it as an ambient pill, and orchestrates the apply flow. */ +import { updateDebt } from '@hermes/shared' import { atom } from 'nanostores' import { connectionScoped, profileScoped } from '@/api/client' @@ -747,15 +748,43 @@ const BACKEND_ACTION_POLL_MS = 1500 const BACKEND_ACTION_MAX_MS = 6 * 60 * 1000 const BACKEND_RETURN_MAX_MS = 4 * 60 * 1000 -// C3: a committed update succeeds even when post-commit steps are still owed. -// Name every owed step (and its reason) instead of reporting plain success. -function owedSteps(receipt: UpdateReceiptSummary | undefined): string { - return [...(receipt?.followups ?? []), ...(receipt?.user_action ? [receipt.user_action] : [])] - .map(step => (step.reason ? `${step.step} (${step.reason})` : step.step)) - .join('; ') +// C3: a committed update succeeds even when post-commit steps are still owed, +// and a partial one (committed, but the user must act) still owes them. Name +// every owed step of THIS action's committed run whatever the exit: follow-ups +// with the rerun remedy, a user action with the producer's own instruction, +// verbatim (a rerun does not restore a parked stash). +function owedMessage(receipt: UpdateReceiptSummary | undefined, actionId: string | undefined): string | null { + const debt = updateDebt(receipt, actionId) + + if (!debt) { + return null + } + + return [debt.followups && translateNow('updates.applyStatus.owed', debt.followups), debt.userAction] + .filter(Boolean) + .join(' ') +} + +// A failed apply whose receipt still proves a committed partial run (exit 1) +// names what that run owes instead of only "failed" / "did not come back". +function failBackendApply( + fallback: { shown: string; result: string }, + status: { receipt?: UpdateReceiptSummary } | null, + actionId: string | undefined +): DesktopUpdateApplyResult { + const owed = owedMessage(status?.receipt, actionId) + const message = owed ? `${translateNow('updates.applyStatus.failed')} ${owed}` : fallback.shown + + $backendUpdateApply.set({ ...$backendUpdateApply.get(), applying: false, stage: 'error', error: 'apply-failed', message }) + + return { ok: false, error: 'apply-failed', message: owed ? message : fallback.result } } -function finishBackendApply(returned: boolean, receipt?: UpdateReceiptSummary): DesktopUpdateApplyResult { +function finishBackendApply( + returned: boolean, + receipt?: UpdateReceiptSummary, + actionId?: string +): DesktopUpdateApplyResult { if (returned) { $backendUpdateApply.set(IDLE) setUpdateOverlayOpen(false) @@ -773,27 +802,22 @@ function finishBackendApply(returned: boolean, receipt?: UpdateReceiptSummary): // affordance in remote mode targets the backend, so nothing ever told // them the app itself was stale). Nudge with a one-click client update. void maybeNudgeClientAfterBackendUpdate() - const owed = owedSteps(receipt) + const owed = owedMessage(receipt, actionId) if (owed) { - const message = translateNow('updates.applyStatus.owed', owed) - notify({ durationMs: 0, kind: 'warning', message }) + notify({ durationMs: 0, kind: 'warning', message: owed }) - return { ok: true, message } + return { ok: true, message: owed } } return { ok: true, message: 'Backend update applied.' } } - $backendUpdateApply.set({ - ...$backendUpdateApply.get(), - applying: false, - stage: 'error', - error: 'apply-failed', - message: translateNow('updates.applyStatus.noReturn') - }) - - return { ok: false, error: 'apply-failed', message: 'Backend did not come back online.' } + return failBackendApply( + { shown: translateNow('updates.applyStatus.noReturn'), result: 'Backend did not come back online.' }, + { receipt }, + actionId + ) } function ingestBackendActionStatus(status: Awaited>): void { @@ -955,7 +979,7 @@ async function runBackendUpdate(): Promise { } if (last.exit_code === 0 || (last.exit_code === null && completedAfterRestart(last, started.action_id))) { - return finishBackendApply(true, last.receipt) + return finishBackendApply(true, last.receipt, started.action_id) } // #91277 bullet 3: the backend now attaches the durable update @@ -963,7 +987,7 @@ async function runBackendUpdate(): Promise { // this update off is authoritative — read its outcome instead of // inferring from log markers or timing out across the restart gap. if (last.exit_code === null && receiptProvesOutcome(last, applyStartedAtMs, started.action_id)) { - return finishBackendApply(last.receipt!.outcome === 'success', last.receipt) + return finishBackendApply(last.receipt!.outcome === 'success', last.receipt, started.action_id) } if (!started.action_id && last.exit_code === null) { @@ -983,15 +1007,11 @@ async function runBackendUpdate(): Promise { } } - $backendUpdateApply.set({ - ...$backendUpdateApply.get(), - applying: false, - stage: 'error', - error: 'apply-failed', - message: translateNow('updates.applyStatus.failed') - }) - - return { ok: false, error: 'apply-failed', message: 'Backend update failed.' } + return failBackendApply( + { shown: translateNow('updates.applyStatus.failed'), result: 'Backend update failed.' }, + last, + started.action_id + ) } catch (error) { const message = error instanceof Error ? error.message : String(error) $backendUpdateApply.set({ diff --git a/apps/shared/src/index.ts b/apps/shared/src/index.ts index 00144fdb1443..b67989b3607a 100644 --- a/apps/shared/src/index.ts +++ b/apps/shared/src/index.ts @@ -174,6 +174,7 @@ export { WINDOWS_GLASS_MIN_BUILD, type WindowsBackgroundMaterial } from './translucency' +export { type UpdateDebt, updateDebt, type UpdateDebtReceipt, type UpdateDebtStep } from './update-debt' export { buildHermesWebSocketUrl, type GatewayAuthMode, diff --git a/apps/shared/src/update-debt.ts b/apps/shared/src/update-debt.ts new file mode 100644 index 000000000000..3e03b162bd11 --- /dev/null +++ b/apps/shared/src/update-debt.ts @@ -0,0 +1,49 @@ +// Owed post-commit work of a dashboard-started `hermes update`, read from the +// receipt summary the action status route attaches. Shared by the web +// dashboard toast and the Desktop backend-apply store so both surfaces name +// the same debt with the same rules as the managed SSH result. + +export interface UpdateDebtStep { + step: string + reason?: string | null +} + +export interface UpdateDebtReceipt { + outcome?: string | null + action_id?: string | null + followups?: UpdateDebtStep[] | null + user_action?: UpdateDebtStep | null +} + +export interface UpdateDebt { + /** Follow-ups a rerun of `hermes update` finishes, as `step (reason); step`; '' when none. */ + followups: string + /** What only the user can do (e.g. re-apply a parked stash) as `step: `; + * '' when none. Never phrase this as "re-run hermes update": a rerun does not restore it. */ + userAction: string +} + +/** The debt of THIS action's committed run, or null when nothing is owed. + * + * Committed is `success` or `partial` (record_user_action turns a committed success into + * partial / exit 1), so the debt is read for every committed outcome, not only exit 0. The + * status route attaches the latest receipt when this action has none, so a receipt is only + * this action's when its `action_id` matches the id the start request returned. */ +export function updateDebt(receipt: UpdateDebtReceipt | null | undefined, actionId: string | undefined): UpdateDebt | null { + if (!receipt || !actionId || receipt.action_id !== actionId) { + return null + } + + if (receipt.outcome !== 'success' && receipt.outcome !== 'partial') { + return null + } + + const followups = (receipt.followups ?? []) + .map(step => (step.reason ? `${step.step} (${step.reason})` : step.step)) + .join('; ') + + const action = receipt.user_action + const userAction = action ? (action.reason ? `${action.step}: ${action.reason}` : action.step) : '' + + return followups || userAction ? { followups, userAction } : null +} From b299cf32e88828bf362039425ca91b9c84cf57e9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:01:37 -0700 Subject: [PATCH 58/62] fix: dashboard update toast names a partial run's owed work, only from its own receipt For a partial run (committed, exit 1 after record_user_action) the toast said only "Action failed (exit 1)" and dropped the receipt's recovery instruction; on exit 0 it named follow-ups from whatever receipt the status route attached, which is the latest one when this action has none. Keep the update POST's action_id and read the debt through the shared updateDebt (committed success|partial, matching action_id only). The toast composes it with every verdict: follow-ups keep the rerun remedy, a user action shows the producer's own instruction verbatim (never "re-run hermes update", which does not restore a parked stash). --- web/src/contexts/SystemActions.test.tsx | 100 ++++++++++++++++++++++++ web/src/contexts/SystemActions.tsx | 55 ++++++++----- 2 files changed, 134 insertions(+), 21 deletions(-) create mode 100644 web/src/contexts/SystemActions.test.tsx diff --git a/web/src/contexts/SystemActions.test.tsx b/web/src/contexts/SystemActions.test.tsx new file mode 100644 index 000000000000..dc6bf2ac2a00 --- /dev/null +++ b/web/src/contexts/SystemActions.test.tsx @@ -0,0 +1,100 @@ +// @vitest-environment jsdom +// The update toast names what a committed update still owes for every committed outcome: a +// partial run (exit 1 after record_user_action) shows the producer's own instruction, and only +// this action's receipt may name debt (the status route attaches the latest receipt otherwise). + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; + +const apiMocks = vi.hoisted(() => ({ + getActionStatus: vi.fn(), + getStatus: vi.fn(), + restartGateway: vi.fn(), + updateHermes: vi.fn(), +})); + +vi.mock("@/lib/api", () => ({ api: apiMocks })); + +import { I18nProvider } from "@/i18n"; +import { SystemActionsProvider } from "./SystemActions"; +import { useSystemActions } from "./useSystemActions"; + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; + +const INSTRUCTION = + "Your local changes are parked in stash@{0}; run `git stash pop` in ~/.hermes/hermes-agent."; + +let container: HTMLDivElement; +let root: Root; + +function Probe() { + const { runAction } = useSystemActions(); + return