fix(gateway): startup-liveness watchdog for pre-event-loop deadlocks (OOF-298)

cli.py

@@ -20919,6 +20919,13 @@ def main(
     # Handle gateway mode (messaging + cron)
     if gateway:
         import asyncio
+        # Startup-liveness watchdog (OOF-298): this legacy entry point must
+        # be covered too — arm before importing the gateway graph.
+        try:
+            from hermes_startup_watchdog import arm_startup_watchdog
+            arm_startup_watchdog()
+        except Exception:
+            pass
         from gateway.run import start_gateway
         print("Starting Hermes Gateway (messaging platforms)...")
         asyncio.run(start_gateway())

gateway/run.py

@@ -12442,6 +12442,20 @@ async def start(self) -> bool:
             self._gateway_loop = None
         if self._gateway_loop is not None:
             self._start_loop_liveness_guards(self._gateway_loop)
+            # The event loop is confirmed live: the startup-liveness
+            # watchdog's job is done and the loop-liveness watchdog (armed
+            # just above) takes over from here (OOF-298). Disarm even when
+            # the loop guards are config-disabled — the startup watchdog
+            # only covers the pre-loop window, never adapter connects or
+            # steady-state. Deliberately inside the loop-confirmed branch:
+            # if the loop somehow isn't live, startup has NOT reached the
+            # milestone and the watchdog must stay armed.
+            try:
+                from gateway.startup_watchdog import disarm_startup_watchdog
+
+                disarm_startup_watchdog()
+            except Exception:
+                logger.debug("Startup watchdog disarm failed", exc_info=True)
         logger.info("Session storage: %s", self.config.sessions_dir)
 
         # Sanity-check that systemd's TimeoutStopSec covers our drain
@@ -30961,6 +30975,19 @@ def main():
     except Exception:
         pass
 
+    # Startup-liveness watchdog (OOF-298): armed before config load, DB
+    # opens, and the rest of pre-loop startup so a deadlock in that window
+    # still gets the process respawned by the service supervisor instead of
+    # wedging as a live-PID zombie. (Import-time coverage for the standard
+    # ``hermes gateway run`` path is provided even earlier, by the argv
+    # fast-path in hermes_cli.main.) Disarmed by GatewayRunner once the
+    # event loop is confirmed live.
+    try:
+        from gateway.startup_watchdog import arm_startup_watchdog
+        arm_startup_watchdog()
+    except Exception:
+        pass
+
     # Force UTF-8 stdio on Windows — gateway logs and startup banner would
     # otherwise UnicodeEncodeError on cp1252 consoles.  No-op on POSIX.
     try:

gateway/startup_watchdog.py

@@ -0,0 +1,42 @@
+"""Compatibility shim — the real implementation is ``hermes_startup_watchdog``.
+
+The startup-liveness watchdog (OOF-298) must be armable *before* the
+``gateway`` package is imported: ``gateway/__init__`` eagerly pulls in the
+config/session/delivery graph, and an import-time deadlock is squarely inside
+the watchdog's coverage mandate. The implementation therefore lives at the
+repository top level as a stdlib-only module.
+
+This shim keeps the intuitive ``gateway.startup_watchdog`` import path
+working for code that runs after the package is loaded (the disarm site in
+``gateway.run``, tests, operators poking at a REPL).
+"""
+
+from hermes_startup_watchdog import (  # noqa: F401
+    DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S,
+    ENV_STARTUP_WATCHDOG,
+    ENV_STARTUP_WATCHDOG_TIMEOUT_S,
+    SERVICE_RESTART_EXIT_CODE,
+    StartupWatchdogHandle,
+    arm_startup_watchdog,
+    disarm_startup_watchdog,
+    get_startup_watchdog_dump_path,
+    kick_startup_watchdog,
+    report_startup_progress,
+    resolve_startup_watchdog_timeout,
+    startup_watchdog_disabled,
+)
+
+__all__ = [
+    "DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S",
+    "ENV_STARTUP_WATCHDOG",
+    "ENV_STARTUP_WATCHDOG_TIMEOUT_S",
+    "SERVICE_RESTART_EXIT_CODE",
+    "StartupWatchdogHandle",
+    "arm_startup_watchdog",
+    "disarm_startup_watchdog",
+    "get_startup_watchdog_dump_path",
+    "kick_startup_watchdog",
+    "report_startup_progress",
+    "resolve_startup_watchdog_timeout",
+    "startup_watchdog_disabled",
+]

hermes_cli/gateway.py

@@ -5739,6 +5739,19 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo
     _guard_existing_gateway_process_conflict(replace=replace)
     sys.path.insert(0, str(PROJECT_ROOT))
 
+    # Startup-liveness watchdog (OOF-298), idempotent backstop: normal
+    # ``hermes gateway run`` invocations already armed in hermes_cli.main's
+    # argv fast-path (before the heavy import graph), but programmatic
+    # callers can enter run_gateway() directly. Placed after the
+    # process-conflict guards: a --replace loser exiting above must not have
+    # armed a watchdog first. Disarmed by GatewayRunner once the event loop
+    # is confirmed live.
+    try:
+        from hermes_startup_watchdog import arm_startup_watchdog
+        arm_startup_watchdog()
+    except Exception:
+        pass
+
     # Detached Windows gateway runs must ignore console-control broadcasts
     # from sibling CLI processes, but foreground `hermes gateway run` still
     # needs to obey the banner's "Press Ctrl+C to stop" contract.
@@ -5921,6 +5934,15 @@ def _atexit_hook() -> None:
                 _storm.window_s,
                 _storm.backoff_s,
             )
+            # The backoff sleep is intentional idle time — tell the startup
+            # watchdog (OOF-298) so it isn't mistaken for a parked deadlock
+            # and hard-exited mid-backoff (which would defeat the breaker).
+            try:
+                from gateway.startup_watchdog import kick_startup_watchdog
+
+                kick_startup_watchdog(extra_s=_storm.backoff_s)
+            except Exception:
+                pass
             _time.sleep(_storm.backoff_s)
     except Exception as _be:
         logger.debug("respawn-storm breaker check failed (non-fatal): %s", _be)

hermes_cli/main.py

@@ -102,6 +102,24 @@
 except Exception:
     pass
 
+# Startup-liveness watchdog (OOF-298): for gateway runs, arm BEFORE the heavy
+# module-level import graph below — an import-time deadlock (native-extension
+# init, contended import lock) is exactly the "wedged before the event loop,
+# no logs, live PID" class this watchdog exists for. ``hermes_startup_watchdog``
+# is stdlib-only, so importing it here cannot itself wedge on application
+# code. argv sniffing is deliberately crude: over-arming is harmless (any
+# non-gateway command either exits well inside the deadline or... should be
+# covered anyway if it wedges), while under-arming recreates OOF-298.
+# GatewayRunner disarms once the event loop is confirmed live.
+if "gateway" in sys.argv[1:] and "run" in sys.argv[1:]:
+    try:
+        from hermes_startup_watchdog import arm_startup_watchdog as _arm_sw
+
+        _arm_sw()
+        del _arm_sw
+    except Exception:
+        pass
+
 
 def _exit_after_oneshot(rc: object) -> None:
     """Exit one-shot mode without letting late native finalizers change rc.