From f97b0f31d4729a5c928c7ebcb1b91b18a2a8d7b3 Mon Sep 17 00:00:00 2001 From: Luke The Dev Date: Tue, 14 Jul 2026 20:12:39 +1000 Subject: [PATCH] fix(#41662): keep Windows gateway alive with windowless watchdog Add a profile-scoped Windows watchdog that preserves cron continuity after a hard gateway crash while honoring intentional stops and uninstalls. - Schedule periodic ticks through Task Scheduler XML + wscript/VBS, never cmd.exe, preserving the existing windowless launcher invariant. - Use IgnoreNew plus a waiting tick launcher to prevent overlapping checks. - Start a hidden long-lived watchdog loop from the Startup-folder fallback, covering machines where Scheduled Task installation is unavailable. - Preserve HERMES_HOME, VIRTUAL_ENV, PYTHONPATH, and detached-launch env parity. - Honor gateway_state=stopped, recheck liveness before spawning, and exit the fallback loop when gateway persistence is uninstalled. - Remove watchdog tasks/scripts during uninstall and report watchdog state in normal/deep status output. - Add behavior tests for crash respawn, healthy/no-op, planned stop, uninstall, VBS/XML launcher behavior, fallback installation, cleanup, and status. The stale execute-code RPC commits from the original branch are intentionally excluded; current main already contains that functionality. --- hermes_cli/gateway_watchdog.py | 227 +++++++++++++ hermes_cli/gateway_windows.py | 389 ++++++++++++++++++++-- tests/hermes_cli/test_gateway_watchdog.py | 265 +++++++++++++++ 3 files changed, 854 insertions(+), 27 deletions(-) create mode 100644 hermes_cli/gateway_watchdog.py create mode 100644 tests/hermes_cli/test_gateway_watchdog.py diff --git a/hermes_cli/gateway_watchdog.py b/hermes_cli/gateway_watchdog.py new file mode 100644 index 000000000000..55e998b80548 --- /dev/null +++ b/hermes_cli/gateway_watchdog.py @@ -0,0 +1,227 @@ +"""Windows gateway watchdog for crash recovery and cron continuity. + +The Scheduled Task runs one hidden tick every two minutes. Machines that cannot +create Scheduled Tasks launch the same module as a hidden login-time loop. The +watchdog never treats an intentional ``gateway stop`` as a crash and exits once +the gateway service is uninstalled. +""" + +from __future__ import annotations + +import argparse +import datetime +import os +import sys +import time +import traceback +from pathlib import Path + + +_LOG_MAX_LINES = 1000 +_DEFAULT_INTERVAL_SECONDS = 120.0 + + +def _resolve_log_path() -> Path: + """Return the watchdog log path under the current HERMES_HOME. + + Imported lazily so a missing/broken import doesn't crash the watchdog. + Falls back to ~/.hermes/logs/ if HERMES_HOME is unavailable. + """ + try: + from hermes_constants import get_hermes_home + home = get_hermes_home() + except Exception: + home = Path(os.environ.get("HERMES_HOME") or Path.home() / ".hermes") + log_dir = Path(home) / "logs" + return log_dir / "gateway-watchdog.log" + + +def _log(msg: str) -> None: + """Append a timestamped line to the watchdog log (best-effort). + + A logging failure must never crash the watchdog. + """ + try: + log_path = _resolve_log_path() + log_path.parent.mkdir(parents=True, exist_ok=True) + ts = datetime.datetime.now(datetime.timezone.utc).isoformat( + timespec="seconds" + ) + with open(log_path, "a", encoding="utf-8") as fh: + fh.write(f"[{ts}] {msg}\n") + except Exception: + pass + + +def _truncate_log() -> None: + """Keep the watchdog log bounded (last _LOG_MAX_LINES lines). + + Cheap per-invocation maintenance — file is small and reads sequentially. + """ + try: + log_path = _resolve_log_path() + if not log_path.exists(): + return + with open(log_path, "r", encoding="utf-8", errors="replace") as fh: + lines = fh.readlines() + if len(lines) <= _LOG_MAX_LINES: + return + tail = lines[-_LOG_MAX_LINES :] + tmp = log_path.with_suffix(log_path.suffix + ".tmp") + with open(tmp, "w", encoding="utf-8") as fh: + fh.writelines(tail) + tmp.replace(log_path) + except Exception: + pass + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse CLI arguments.""" + parser = argparse.ArgumentParser( + prog="hermes_cli.gateway_watchdog", + description="Windows watchdog for automatic gateway respawn on crash.", + ) + parser.add_argument( + "--loop", + action="store_true", + help="Run continuously for the Startup-folder fallback.", + ) + parser.add_argument( + "--interval", + type=float, + default=_DEFAULT_INTERVAL_SECONDS, + help="Seconds between checks in loop mode.", + ) + return parser.parse_args(argv) + + +def _gateway_is_alive() -> tuple[bool, int | None]: + """Probe whether a gateway is running for the current HERMES_HOME. + + Returns (alive, pid). When alive is True, pid is the running gateway PID. + When alive is False, pid is None. + + Conservative on errors: if we can't import probes, assume gateway is alive + (don't force-respawn on import failures). + """ + try: + from gateway.status import get_running_pid + except Exception as exc: + _log(f"probe import failure: {exc!r}") + return (True, None) + + try: + pid = get_running_pid(cleanup_stale=False) + except Exception as exc: + _log(f"get_running_pid raised: {exc!r}") + return (True, None) + return (pid is not None, pid) + + +def _respawn() -> int | None: + """Spawn a fresh detached gateway using CREATE_BREAKAWAY_FROM_JOB. + + Return the PID on success, None on failure. + + The respawn uses gateway_windows._spawn_detached() which applies + CREATE_BREAKAWAY_FROM_JOB so the new gateway survives if the watchdog + or its parent job object dies. + """ + try: + from hermes_cli import gateway_windows + except Exception as exc: + _log(f"gateway_windows import failure: {exc!r}") + return None + try: + return gateway_windows._spawn_detached() + except Exception as exc: + _log(f"_spawn_detached raised: {exc!r}\n{traceback.format_exc()}") + return None + + +def _service_is_installed() -> bool: + """Return whether this profile still has gateway persistence installed.""" + try: + from hermes_cli import gateway_windows + + return bool(gateway_windows.is_installed()) + except Exception as exc: + _log(f"service install probe failed: {exc!r}") + return False + + +def _was_intentionally_stopped() -> bool: + """Return whether the operator explicitly stopped the gateway.""" + try: + from gateway.status import read_runtime_status + + status = read_runtime_status() or {} + except Exception as exc: + _log(f"runtime status probe failed: {exc!r}") + return True + return status.get("gateway_state") == "stopped" + + +def _run_once() -> bool: + """Run one health check. + + Returns ``False`` only when the service is no longer installed, which tells + the Startup-folder loop to exit. All other outcomes keep the loop alive. + """ + if not _service_is_installed(): + _log("gateway service is not installed; watchdog exiting") + return False + + alive, _pid = _gateway_is_alive() + if alive: + return True + if _was_intentionally_stopped(): + return True + + # Recheck immediately before spawning to close the common manual-start race. + alive, _pid = _gateway_is_alive() + if alive: + return True + + _log("gateway is down; respawning") + new_pid = _respawn() + if new_pid is None: + _log("respawn failed; will retry on next tick") + else: + _log(f"respawned gateway pid={new_pid}") + return True + + +def main(argv: list[str] | None = None) -> int: + """Entrypoint for pythonw -m hermes_cli.gateway_watchdog. + + Always returns 0 so the schtasks parent never marks the task as failed. + """ + try: + args = _parse_args(argv) + except SystemExit: + # argparse exits for --help or bad args. + # Return 0 so the schtasks parent does not mark the task as failed. + return 0 + except Exception as exc: + _log(f"argparse raised: {exc!r}") + return 0 + + _truncate_log() + + interval = max(float(args.interval), 1.0) + while True: + try: + keep_running = _run_once() + except Exception as exc: + _log(f"watchdog uncaught: {exc!r}\n{traceback.format_exc()}") + keep_running = True + if not args.loop or not keep_running: + break + time.sleep(interval) + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hermes_cli/gateway_windows.py b/hermes_cli/gateway_windows.py index 55ed976433da..1ab8728f08b4 100644 --- a/hermes_cli/gateway_windows.py +++ b/hermes_cli/gateway_windows.py @@ -297,6 +297,21 @@ def get_task_name() -> str: return _TASK_NAME_DEFAULT return f"{_TASK_NAME_DEFAULT}_{suffix}" +def _get_watchdog_task_name() -> str: + """Per-profile name for the gateway watchdog task (respawns on crash). + + Default: ``Hermes_Gateway_Watchdog`` + Named profile X: ``Hermes_Gateway_Watchdog_`` + """ + _assert_windows() + from hermes_cli.gateway import _profile_suffix + + suffix = _profile_suffix() + if not suffix: + return f"{_TASK_NAME_DEFAULT}_Watchdog" + return f"{_TASK_NAME_DEFAULT}_Watchdog_{suffix}" + + def _sanitize_filename(value: str) -> str: """Remove characters illegal in Windows filenames.""" @@ -317,6 +332,26 @@ def get_task_script_path() -> Path: script_dir.mkdir(parents=True, exist_ok=True) return script_dir / f"{_sanitize_filename(get_task_name())}.cmd" +def _get_watchdog_script_path() -> Path: + """The generated ``gateway-watchdog.cmd`` wrapper for the watchdog task. + + Lives under ``/gateway-service/watchdog-.cmd`` + so it stays scoped per profile and doesn't conflict with the main gateway task. + """ + _assert_windows() + from hermes_cli.config import get_hermes_home + + script_dir = Path(get_hermes_home()) / "gateway-service" + script_dir.mkdir(parents=True, exist_ok=True) + return script_dir / f"watchdog-{_sanitize_filename(_get_watchdog_task_name())}.cmd" + + +def _get_watchdog_loop_vbs_path() -> Path: + """Return the hidden loop launcher used by the Startup-folder fallback.""" + script_path = _get_watchdog_script_path() + return script_path.with_name(f"{script_path.stem}.loop.vbs") + + def _startup_dir() -> Path: appdata = os.environ.get("APPDATA", "").strip() @@ -425,6 +460,43 @@ def _build_gateway_cmd_script( lines.append("exit /b 0") return "\r\n".join(lines) + "\r\n" +def _build_watchdog_cmd_script( + python_path: str, + working_dir: str, + hermes_home: str, + profile_arg: str, +) -> str: + """Build the ``gateway-watchdog.cmd`` wrapper content (CRLF-terminated). + + The watchdog script: + - cd's into the stable working directory + - exports HERMES_HOME, PYTHONIOENCODING, VIRTUAL_ENV + - invokes ``pythonw -m hermes_cli.gateway_watchdog [--profile X]`` + - runs every few minutes (1-5 min), checks if gateway is alive + - silently respawns gateway if it has crashed + - outputs to NUL (watchdog module logs to gateway-watchdog.log instead) + + Uses pythonw.exe (GUI subsystem) to avoid console flashes. + """ + lines = ["@echo off", f"rem {_TASK_DESCRIPTION} (Watchdog)"] + lines.append(f"cd /d {_quote_cmd_script_arg(working_dir)}") + lines.append(f'set "HERMES_HOME={hermes_home}"') + lines.append('set "PYTHONIOENCODING=utf-8"') + lines.append('set "HERMES_GATEWAY_DETACHED=1"') + pythonw_path, venv_dir, extra_pythonpath = _resolve_detached_python(python_path) + lines.append(f'set "VIRTUAL_ENV={_preserve_hermes_home_path(venv_dir)}"') + pythonpath_entries = [ + _preserve_hermes_home_path(Path(__file__).resolve().parent.parent), + *[_preserve_hermes_home_path(entry) for entry in extra_pythonpath], + ] + lines.append(f'set "PYTHONPATH={";".join([*pythonpath_entries, "%PYTHONPATH%"])}"') + prog_args = [pythonw_path, "-m", "hermes_cli.gateway_watchdog"] + # Redirect stdout/stderr to NUL; the watchdog module writes its own log. + lines.append(" ".join(_quote_cmd_script_arg(a) for a in prog_args) + " >NUL 2>&1") + lines.append("exit /b 0") + return "\r\n".join(lines) + "\r\n" + + def _quote_vbs_string(value: str) -> str: """Quote a value as a VBScript double-quoted string literal. @@ -500,7 +572,52 @@ def _build_gateway_vbs_script( return "\r\n".join(lines) + "\r\n" -def _build_startup_launcher(script_path: Path) -> str: +def _build_watchdog_vbs_script( + python_path: str, + working_dir: str, + hermes_home: str, + *, + loop: bool = False, +) -> str: + """Build a console-less watchdog launcher with gateway environment parity.""" + pythonw_path, venv_dir, extra_pythonpath = _resolve_detached_python(python_path) + prog_args = [pythonw_path, "-m", "hermes_cli.gateway_watchdog"] + if loop: + prog_args.append("--loop") + command_line = subprocess.list2cmdline(prog_args) + repo_root = _preserve_hermes_home_path(Path(__file__).resolve().parent.parent) + static_pythonpath = os.pathsep.join( + [repo_root, *[_preserve_hermes_home_path(entry) for entry in extra_pythonpath]] + ) + lines = [ + f"' {_TASK_DESCRIPTION} (Watchdog)", + "Option Explicit", + "Dim sh, env, existing_pp", + 'Set sh = CreateObject("WScript.Shell")', + 'Set env = sh.Environment("PROCESS")', + f"env.Item({_quote_vbs_string('HERMES_HOME')}) = {_quote_vbs_string(hermes_home)}", + f"env.Item({_quote_vbs_string('PYTHONIOENCODING')}) = {_quote_vbs_string('utf-8')}", + f"env.Item({_quote_vbs_string('HERMES_GATEWAY_DETACHED')}) = {_quote_vbs_string('1')}", + f"env.Item({_quote_vbs_string('VIRTUAL_ENV')}) = {_quote_vbs_string(_preserve_hermes_home_path(venv_dir))}", + f"existing_pp = env.Item({_quote_vbs_string('PYTHONPATH')})", + "If Len(existing_pp) > 0 Then", + f" env.Item({_quote_vbs_string('PYTHONPATH')}) = {_quote_vbs_string(static_pythonpath + os.pathsep)} & existing_pp", + "Else", + f" env.Item({_quote_vbs_string('PYTHONPATH')}) = {_quote_vbs_string(static_pythonpath)}", + "End If", + f"sh.CurrentDirectory = {_quote_vbs_string(working_dir)}", + # A Scheduled Task tick waits for its short-lived watchdog process. + # This lets IgnoreNew prevent overlap. + # The Startup fallback's long-lived loop detaches from the login launcher. + f"sh.Run {_quote_vbs_string(command_line)}, 0, {'False' if loop else 'True'}", + ] + return "\r\n".join(lines) + "\r\n" + + +def _build_startup_launcher( + script_path: Path, + watchdog_loop_path: Path | None = None, +) -> str: """The tiny .vbs that goes in the Startup folder and chains hidden. Defense-in-depth: bail out silently if the target script is gone. Test @@ -521,6 +638,17 @@ def _build_startup_launcher(script_path: Path) -> str: 'Set sh = CreateObject("WScript.Shell")', f"sh.Run {_quote_vbs_string(command)}, 0, False", ] + if watchdog_loop_path is not None: + watchdog_target = str(watchdog_loop_path) + watchdog_command = subprocess.list2cmdline(["wscript.exe", watchdog_target]) + lines.extend( + [ + f"target = {_quote_vbs_string(watchdog_target)}", + "If fso.FileExists(target) Then", + f" sh.Run {_quote_vbs_string(watchdog_command)}, 0, False", + "End If", + ] + ) return "\r\n".join(lines) + "\r\n" @@ -557,6 +685,49 @@ def _write_task_script() -> Path: return script_path +def _write_watchdog_script() -> Path: + """Generate and write the gateway-watchdog.cmd wrapper. Return its absolute path.""" + _assert_windows() + # Local imports to avoid circular-init at module load time. + from hermes_cli.config import get_hermes_home + from hermes_cli.gateway import ( + PROJECT_ROOT, + get_python_path, + ) + + python_path = _preserve_hermes_home_path(get_python_path()) + working_dir = _stable_gateway_working_dir(PROJECT_ROOT) + hermes_home = str(Path(get_hermes_home())) + + content = _build_watchdog_cmd_script(python_path, working_dir, hermes_home, "") + script_path = _get_watchdog_script_path() + tmp = script_path.with_suffix(".tmp") + tmp.write_text(content, encoding="utf-8", newline="") + tmp.replace(script_path) + + vbs_content = _build_watchdog_vbs_script( + python_path, + working_dir, + hermes_home, + ) + vbs_path = script_path.with_suffix(".vbs") + vbs_tmp = vbs_path.with_name(vbs_path.name + ".tmp") + vbs_tmp.write_text(vbs_content, encoding="utf-8", newline="") + vbs_tmp.replace(vbs_path) + + loop_content = _build_watchdog_vbs_script( + python_path, + working_dir, + hermes_home, + loop=True, + ) + loop_path = _get_watchdog_loop_vbs_path() + loop_tmp = loop_path.with_name(loop_path.name + ".tmp") + loop_tmp.write_text(loop_content, encoding="utf-8", newline="") + loop_tmp.replace(loop_path) + return script_path + + # --------------------------------------------------------------------------- # Install / uninstall # --------------------------------------------------------------------------- @@ -640,6 +811,68 @@ def _write_scheduled_task_xml(task_name: str, launcher_path: Path, user: str | N return xml_path +def _build_watchdog_scheduled_task_xml( + task_name: str, + launcher_path: Path, + user: str | None, +) -> str: + """Render a hidden periodic watchdog task using the VBS/wscript path.""" + user_principal = f"\n {escape(user)}" if user else "" + return f""" + + + {escape(_TASK_DESCRIPTION)} watchdog + + + + 2000-01-01T00:00:00 + true + + PT2M + false + + + + + {user_principal} + InteractiveToken + LeastPrivilege + + + + IgnoreNew + false + false + true + true + true + true + PT1M + + + + wscript.exe + //B //Nologo "{escape(str(launcher_path))}" + + + +""" + + +def _write_watchdog_scheduled_task_xml( + task_name: str, + launcher_path: Path, + user: str | None, +) -> Path: + xml_path = launcher_path.with_suffix(".task.xml") + xml_path.write_text( + _build_watchdog_scheduled_task_xml(task_name, launcher_path, user), + encoding="utf-16", + newline="", + ) + return xml_path + + def _install_scheduled_task(task_name: str, script_path: Path) -> tuple[bool, str]: """Create or replace the Scheduled Task. Returns (success, detail). @@ -689,7 +922,15 @@ def _install_startup_entry(script_path: Path) -> Path: entry = get_startup_entry_path() entry.parent.mkdir(parents=True, exist_ok=True) tmp = entry.with_suffix(".tmp") - tmp.write_text(_build_startup_launcher(script_path), encoding="utf-8", newline="") + watchdog_loop_path = _get_watchdog_loop_vbs_path() + tmp.write_text( + _build_startup_launcher( + script_path, + watchdog_loop_path if watchdog_loop_path.exists() else None, + ), + encoding="utf-8", + newline="", + ) tmp.replace(entry) legacy_entry = _legacy_startup_entry_path() try: @@ -700,6 +941,57 @@ def _install_startup_entry(script_path: Path) -> Path: return entry +def _install_watchdog_task() -> tuple[bool, str]: + """Install the watchdog Scheduled Task (respawns gateway if crashed). + + Returns (success, detail). + + The watchdog runs every 2 minutes and checks if the gateway is alive. + If the gateway is down, it respawns it. This is a best-effort mechanism + to keep cron jobs running even after gateway crashes (addresses #41662). + """ + watchdog_task_name = _get_watchdog_task_name() + watchdog_script_path = _write_watchdog_script() + + # Delete the task first to avoid stale trigger/action settings. + delete_code, delete_out, delete_err = _exec_schtasks(["/Delete", "/F", "/TN", watchdog_task_name]) + delete_detail = (delete_err or delete_out or "").strip() + if delete_code != 0 and delete_detail and "cannot find" not in delete_detail.lower(): + if _is_access_denied(delete_detail): + return (False, f"schtasks /Delete (watchdog) failed: {delete_detail}") + + user = _resolve_task_user() + launcher_path = watchdog_script_path.with_suffix(".vbs") + xml_path = _write_watchdog_scheduled_task_xml( + watchdog_task_name, + launcher_path, + user, + ) + base = ["/Create", "/F", "/TN", watchdog_task_name, "/XML", str(xml_path)] + variants = [[*base, "/RU", user, "/NP", "/IT"]] if user else [] + variants.append(base) + last_code = 1 + last_err = "" + try: + for argv in variants: + code, out, err = _exec_schtasks(argv) + if code == 0: + return ( + True, + f"Created Scheduled Task {watchdog_task_name!r} (runs every 2 minutes)", + ) + last_code, last_err = code, (err or out or "") + finally: + try: + xml_path.unlink(missing_ok=True) + except OSError: + pass + return ( + False, + f"schtasks /Create (watchdog) failed (code {last_code}): {last_err.strip()}", + ) + + def _derive_venv_pythonw(python_exe: str) -> str: """Given a ``python.exe`` path, return the sibling ``pythonw.exe`` if present. @@ -997,9 +1289,11 @@ def _prompt_install_choices( def _install_startup_fallback(script_path: Path, start_now: bool, detail: str) -> None: """Install the Startup-folder fallback and optionally start once.""" print(f"↻ Scheduled Task install blocked ({detail.splitlines()[0]}) — using Startup folder fallback") + _write_watchdog_script() entry = _install_startup_entry(script_path) print(f"✓ Installed Windows login item: {entry}") print(f" Task script: {script_path}") + print("✓ Installed hidden gateway watchdog through the login item.") # Re-running `hermes -p gateway install` must be safe. # Startup-folder fallback only installs login persistence. Starting is @@ -1092,6 +1386,17 @@ def install( else: print("ℹ Gateway not started now.") print(" Start manually with: hermes gateway start") + + # Install the watchdog task to respawn the gateway after a crash (#41662). + watchdog_ok, watchdog_detail = _install_watchdog_task() + if watchdog_ok: + print(f"✓ {watchdog_detail}") + print("ℹ Cron jobs will continue even if the gateway crashes (watchdog respawns it).") + else: + print(f"⚠ Watchdog installation failed: {watchdog_detail}") + print(" The gateway will auto-start on login, but cron jobs may stop if it crashes.") + print(" (This is not critical; the main gateway task is working.)") + _print_next_steps() return @@ -1118,29 +1423,7 @@ def install( # schtasks create didn't work. See if it's a "fall back to startup" case. if _should_fall_back(1, detail): - print(f"↻ Scheduled Task install blocked ({detail.splitlines()[0]}) — using Startup folder fallback") - entry = _install_startup_entry(script_path) - print(f"✓ Installed Windows login item: {entry}") - print(f" Task script: {script_path}") - - # Re-running `hermes -p gateway install` must be safe. - # Startup-folder fallback only installs login persistence. Starting is - # controlled by the pre-UAC start_now answer so all user decisions happen - # before any elevation prompt. - from hermes_cli.gateway import find_gateway_pids, _profile_arg - - running_pids = list(find_gateway_pids()) - if running_pids: - print(f"✓ Gateway already running (PID: {', '.join(map(str, running_pids))})") - elif start_now: - pid = _spawn_detached() - _report_gateway_start(f"direct spawn (PID {pid})") - else: - profile_arg = _profile_arg() - start_cmd = f"hermes {profile_arg} gateway start" if profile_arg else "hermes gateway start" - print("ℹ Startup fallback installed; gateway not started now.") - print(f" Start manually with: {start_cmd}") - _print_next_steps() + _install_startup_fallback(script_path, start_now, detail) return # Unknown schtasks error — surface it and bail. @@ -1194,6 +1477,10 @@ def uninstall() -> None: vbs_script_path = script_path.with_suffix(".vbs") startup_entry = get_startup_entry_path() legacy_startup_entry = _legacy_startup_entry_path() + watchdog_task_name = _get_watchdog_task_name() + watchdog_script_path = _get_watchdog_script_path() + watchdog_vbs_path = watchdog_script_path.with_suffix(".vbs") + watchdog_loop_vbs_path = _get_watchdog_loop_vbs_path() scheduled_task_removed = False if is_task_registered(): @@ -1218,11 +1505,31 @@ def uninstall() -> None: else: print(f"⚠ schtasks /Delete returned code {code}: {detail}") + watchdog_task_removed = False + if is_watchdog_task_registered(): + code, _out, err = _exec_schtasks( + ["/Delete", "/F", "/TN", watchdog_task_name] + ) + detail = err.strip() + if code == 0: + watchdog_task_removed = True + print(f"✓ Removed Scheduled Task {watchdog_task_name!r}") + elif _is_access_denied(detail) and not _is_running_as_admin(): + print( + "⚠ Watchdog Scheduled Task needs administrator approval " + f"to remove: {detail or 'access denied'}" + ) + else: + print(f"⚠ watchdog schtasks /Delete returned code {code}: {detail}") + for path, label in [ (startup_entry, "Windows login item"), (legacy_startup_entry, "legacy Windows login item"), (script_path, "Task script"), (vbs_script_path, "Task launcher"), + (watchdog_script_path, "Watchdog task script"), + (watchdog_vbs_path, "Watchdog task launcher"), + (watchdog_loop_vbs_path, "Watchdog loop launcher"), ]: try: path.unlink() @@ -1232,6 +1539,8 @@ def uninstall() -> None: if is_task_registered() and not scheduled_task_removed: print(f"⚠ Scheduled Task still registered: {task_name}") + if is_watchdog_task_registered() and not watchdog_task_removed: + print(f"⚠ Scheduled Task still registered: {watchdog_task_name}") # --------------------------------------------------------------------------- @@ -1243,6 +1552,14 @@ def is_task_registered() -> bool: return code == 0 +def is_watchdog_task_registered() -> bool: + """Return whether the periodic watchdog task exists for this profile.""" + code, _out, _err = _exec_schtasks( + ["/Query", "/TN", _get_watchdog_task_name()] + ) + return code == 0 + + def is_startup_entry_installed() -> bool: return get_startup_entry_path().exists() or _legacy_startup_entry_path().exists() @@ -1252,9 +1569,11 @@ def is_installed() -> bool: return is_task_registered() or is_startup_entry_installed() -def query_task_status() -> dict[str, str]: +def query_task_status(task_name: str | None = None) -> dict[str, str]: """Parse ``schtasks /Query /V /FO LIST`` and pull the interesting keys.""" - code, out, err = _exec_schtasks(["/Query", "/TN", get_task_name(), "/V", "/FO", "LIST"]) + code, out, err = _exec_schtasks( + ["/Query", "/TN", task_name or get_task_name(), "/V", "/FO", "LIST"] + ) if code != 0: return {} info: dict[str, str] = {} @@ -1420,6 +1739,7 @@ def status(deep: bool = False) -> None: task_name = get_task_name() task_installed = is_task_registered() startup_installed = is_startup_entry_installed() + watchdog_task_installed = is_watchdog_task_registered() pids = _gateway_pids() if task_installed: @@ -1437,6 +1757,19 @@ def status(deep: bool = False) -> None: else: print("✗ Gateway service not installed") + if watchdog_task_installed: + watchdog_task_name = _get_watchdog_task_name() + print(f"✓ Watchdog Scheduled Task registered: {watchdog_task_name}") + watchdog_info = query_task_status(watchdog_task_name) + if watchdog_info: + for key in ("status", "last run time", "last run result"): + if key in watchdog_info: + print(f" Watchdog {key.title()}: {watchdog_info[key]}") + elif startup_installed and _get_watchdog_loop_vbs_path().exists(): + print("✓ Hidden watchdog supervisor installed through the login item") + else: + print("✗ Gateway watchdog not installed") + if pids: print(f"✓ Gateway process running (PID: {', '.join(map(str, pids))})") else: @@ -1447,6 +1780,8 @@ def status(deep: bool = False) -> None: print(f" Task name: {task_name}") print(f" Task script: {get_task_script_path()}") print(f" Startup entry: {get_startup_entry_path()}") + print(f" Watchdog script: {_get_watchdog_script_path()}") + print(f" Watchdog loop: {_get_watchdog_loop_vbs_path()}") # Surface the per-probe truth so the user can see *which* signal # is lying when the high-level summary disagrees with reality. _print_deep_probes() diff --git a/tests/hermes_cli/test_gateway_watchdog.py b/tests/hermes_cli/test_gateway_watchdog.py new file mode 100644 index 000000000000..50a16eb6b0b8 --- /dev/null +++ b/tests/hermes_cli/test_gateway_watchdog.py @@ -0,0 +1,265 @@ +"""Behavior tests for the Windows gateway watchdog (issue #41662).""" + +from pathlib import Path + +from hermes_cli import gateway_watchdog, gateway_windows + + +def test_watchdog_noops_when_gateway_is_running(monkeypatch): + spawned = [] + monkeypatch.setattr(gateway_watchdog, "_service_is_installed", lambda: True) + monkeypatch.setattr(gateway_watchdog, "_gateway_is_alive", lambda: (True, 42)) + monkeypatch.setattr(gateway_watchdog, "_respawn", lambda: spawned.append(True)) + + assert gateway_watchdog.main([]) == 0 + assert spawned == [] + + +def test_watchdog_respawns_when_gateway_crashed(monkeypatch): + probes = iter([(False, None), (False, None)]) + spawned = [] + monkeypatch.setattr(gateway_watchdog, "_service_is_installed", lambda: True) + monkeypatch.setattr(gateway_watchdog, "_gateway_is_alive", lambda: next(probes)) + monkeypatch.setattr(gateway_watchdog, "_was_intentionally_stopped", lambda: False) + monkeypatch.setattr( + gateway_watchdog, + "_respawn", + lambda: spawned.append(True) or 12345, + ) + monkeypatch.setattr(gateway_watchdog, "_log", lambda _message: None) + + assert gateway_watchdog.main([]) == 0 + assert spawned == [True] + + +def test_watchdog_honors_intentional_stop(monkeypatch): + spawned = [] + monkeypatch.setattr(gateway_watchdog, "_service_is_installed", lambda: True) + monkeypatch.setattr(gateway_watchdog, "_gateway_is_alive", lambda: (False, None)) + monkeypatch.setattr(gateway_watchdog, "_was_intentionally_stopped", lambda: True) + monkeypatch.setattr(gateway_watchdog, "_respawn", lambda: spawned.append(True)) + + assert gateway_watchdog.main([]) == 0 + assert spawned == [] + + +def test_watchdog_loop_exits_after_uninstall(monkeypatch): + slept = [] + monkeypatch.setattr(gateway_watchdog, "_service_is_installed", lambda: False) + monkeypatch.setattr(gateway_watchdog, "_log", lambda _message: None) + monkeypatch.setattr(gateway_watchdog.time, "sleep", lambda delay: slept.append(delay)) + + assert gateway_watchdog.main(["--loop", "--interval", "1"]) == 0 + assert slept == [] + + +def test_watchdog_task_xml_is_periodic_windowless_and_single_flight(): + xml = gateway_windows._build_watchdog_scheduled_task_xml( + "Hermes_Gateway_Watchdog", + Path(r"C:\Hermes\watchdog.vbs"), + r"DOMAIN\alice", + ) + + assert "PT2M" in xml + assert "IgnoreNew" in xml + assert "wscript.exe" in xml + assert "watchdog.vbs" in xml + assert "cmd.exe" not in xml + + +def test_watchdog_vbs_waits_for_ticks_but_detaches_startup_loop(monkeypatch): + monkeypatch.setattr( + gateway_windows, + "_resolve_detached_python", + lambda _exe: (r"C:\Python\pythonw.exe", Path(r"C:\venv"), []), + ) + tick = gateway_windows._build_watchdog_vbs_script( + r"C:\Python\python.exe", + r"C:\Hermes", + r"C:\HermesHome", + ) + loop = gateway_windows._build_watchdog_vbs_script( + r"C:\Python\python.exe", + r"C:\Hermes", + r"C:\HermesHome", + loop=True, + ) + + assert "cmd.exe" not in tick + assert ", 0, True" in tick + assert "--loop" not in tick + assert ", 0, False" in loop + assert "--loop" in loop + + +def test_startup_launcher_starts_gateway_then_hidden_watchdog_loop(): + launcher = gateway_windows._build_startup_launcher( + Path(r"C:\Hermes\gateway.cmd"), + Path(r"C:\Hermes\watchdog.loop.vbs"), + ) + + gateway_pos = launcher.index("gateway.vbs") + watchdog_pos = launcher.index("watchdog.loop.vbs") + assert gateway_pos < watchdog_pos + assert launcher.count(", 0, False") == 2 + + +def test_startup_fallback_writes_and_installs_watchdog(monkeypatch, tmp_path): + gateway_script = tmp_path / "gateway.cmd" + startup_entry = tmp_path / "startup.vbs" + calls = [] + monkeypatch.setattr( + gateway_windows, + "_write_watchdog_script", + lambda: calls.append("write_watchdog") or (tmp_path / "watchdog.cmd"), + ) + monkeypatch.setattr( + gateway_windows, + "_install_startup_entry", + lambda path: calls.append(("install_startup", path)) or startup_entry, + ) + monkeypatch.setattr( + gateway_windows, + "_get_watchdog_loop_vbs_path", + lambda: tmp_path / "watchdog.loop.vbs", + ) + monkeypatch.setattr( + "hermes_cli.gateway.find_gateway_pids", + lambda: [99], + ) + monkeypatch.setattr( + gateway_windows, + "_print_next_steps", + lambda: calls.append("next_steps"), + ) + + gateway_windows._install_startup_fallback( + gateway_script, + start_now=False, + detail="access denied", + ) + + assert calls == [ + "write_watchdog", + ("install_startup", gateway_script), + "next_steps", + ] + + +def test_uninstall_removes_watchdog_task_and_artifacts(monkeypatch, tmp_path): + gateway_script = tmp_path / "gateway.cmd" + watchdog_script = tmp_path / "watchdog.cmd" + startup_entry = tmp_path / "startup.vbs" + legacy_entry = tmp_path / "startup.cmd" + watchdog_loop = tmp_path / "watchdog.loop.vbs" + for path in ( + gateway_script, + gateway_script.with_suffix(".vbs"), + watchdog_script, + watchdog_script.with_suffix(".vbs"), + watchdog_loop, + startup_entry, + legacy_entry, + ): + path.write_text("x", encoding="utf-8") + + calls = [] + monkeypatch.setattr(gateway_windows, "_assert_windows", lambda: None) + monkeypatch.setattr(gateway_windows, "get_task_name", lambda: "Gateway") + monkeypatch.setattr( + gateway_windows, + "_get_watchdog_task_name", + lambda: "Gateway_Watchdog", + ) + monkeypatch.setattr( + gateway_windows, + "get_task_script_path", + lambda: gateway_script, + ) + monkeypatch.setattr( + gateway_windows, + "_get_watchdog_script_path", + lambda: watchdog_script, + ) + monkeypatch.setattr( + gateway_windows, + "_get_watchdog_loop_vbs_path", + lambda: watchdog_loop, + ) + monkeypatch.setattr( + gateway_windows, + "get_startup_entry_path", + lambda: startup_entry, + ) + monkeypatch.setattr( + gateway_windows, + "_legacy_startup_entry_path", + lambda: legacy_entry, + ) + registered = {"gateway": True, "watchdog": True} + monkeypatch.setattr( + gateway_windows, + "is_task_registered", + lambda: registered["gateway"], + ) + monkeypatch.setattr( + gateway_windows, + "is_watchdog_task_registered", + lambda: registered["watchdog"], + ) + + def fake_schtasks(argv): + calls.append(argv) + if argv[-1] == "Gateway": + registered["gateway"] = False + if argv[-1] == "Gateway_Watchdog": + registered["watchdog"] = False + return (0, "", "") + + monkeypatch.setattr(gateway_windows, "_exec_schtasks", fake_schtasks) + + gateway_windows.uninstall() + + assert ["/Delete", "/F", "/TN", "Gateway"] in calls + assert ["/Delete", "/F", "/TN", "Gateway_Watchdog"] in calls + assert not gateway_script.exists() + assert not gateway_script.with_suffix(".vbs").exists() + assert not watchdog_script.exists() + assert not watchdog_script.with_suffix(".vbs").exists() + assert not watchdog_loop.exists() + assert not startup_entry.exists() + assert not legacy_entry.exists() + + +def test_status_reports_periodic_watchdog(monkeypatch, capsys): + monkeypatch.setattr(gateway_windows, "_assert_windows", lambda: None) + monkeypatch.setattr(gateway_windows, "get_task_name", lambda: "Gateway") + monkeypatch.setattr( + gateway_windows, + "_get_watchdog_task_name", + lambda: "Gateway_Watchdog", + ) + monkeypatch.setattr(gateway_windows, "is_task_registered", lambda: True) + monkeypatch.setattr( + gateway_windows, + "is_watchdog_task_registered", + lambda: True, + ) + monkeypatch.setattr( + gateway_windows, + "is_startup_entry_installed", + lambda: False, + ) + monkeypatch.setattr(gateway_windows, "_gateway_pids", lambda: [321]) + monkeypatch.setattr( + gateway_windows, + "query_task_status", + lambda task_name=None: {"status": "Ready"}, + ) + + gateway_windows.status() + + output = capsys.readouterr().out + assert "Scheduled Task registered: Gateway" in output + assert "Watchdog Scheduled Task registered: Gateway_Watchdog" in output + assert "Gateway process running (PID: 321)" in output