From 76523693df33d510bf9ba0915434ca927c3e290b Mon Sep 17 00:00:00 2001
From: yayu <yayu@yayuMacStudio.local>
Date: Sun, 31 May 2026 10:28:26 +0800
Subject: [PATCH] fix(gateway): add hard-exit watchdog to prevent drain hang on
 stop/restart
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When the gateway is stopped or restarted while sessions are active,
_stop_impl() may hang indefinitely if zombie agent threads cannot be
interrupted — causing launchd/systemd to escalate to SIGKILL, which
bypasses log flushing and DB close.

Two layered watchdogs are added:

**Layer 1 — Unconditional stop watchdog** (fires at shutdown entry):
Daemon thread that calls os._exit(0) after drain_timeout + 20s,
guaranteeing the process exits regardless of where the shutdown sequence
gets stuck (drain, interrupt wait, adapter.disconnect, DB close).
Covers the blind spot of the per-zombie watchdog when _running_agents is
cleared before blocking threads finish.

**Layer 2 — Per-zombie watchdog** (fires only when zombie sessions remain
after the interrupt + 5s wait):
Daemon thread that calls os._exit(0) after 10s, allowing the rest of
_stop_impl() to continue (log flush, DB close) while guaranteeing exit
even if adapter.disconnect() hangs.

os._exit(0) is intentional: sys.exit() raises SystemExit which asyncio
and atexit machinery may catch or defer. os._exit() is the correct
last-resort in a daemon/service context.

Observed on: macOS launchd, Linux systemd. Triggered by: blocking
subprocess in terminal() tool, synchronous network call that ignores
asyncio cancellation.
---
 gateway/run.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/gateway/run.py b/gateway/run.py
index 09f6f990bc7f..38ca23b47939 100644
--- a/gateway/run.py
+++ b/gateway/run.py
@@ -6059,6 +6059,30 @@ def _kill_tool_subprocesses(phase: str) -> None:
             def _phase_elapsed() -> float:
                 return time.monotonic() - _stop_started_at
 
+            # Unconditional hard-exit watchdog: regardless of where the shutdown
+            # sequence gets stuck (drain, interrupt, adapter.disconnect, DB close),
+            # this daemon thread guarantees the process exits within
+            # (drain_timeout + 20s).  This is a belt-and-suspenders guard on top
+            # of the per-zombie watchdog below — the per-zombie guard has a blind
+            # spot when _running_agents is cleared before blocking threads finish.
+            _watchdog_delay = self._restart_drain_timeout + 20.0
+            import threading as _threading_wd
+            def _unconditional_exit_watchdog(delay: float) -> None:
+                import time as _time, os as _os
+                _time.sleep(delay)
+                _os._exit(0)  # noqa: SIM115 — intentional last-resort hard exit
+            _wd_thread = _threading_wd.Thread(
+                target=_unconditional_exit_watchdog,
+                args=(_watchdog_delay,),
+                daemon=True,
+                name="gateway-stop-watchdog",
+            )
+            _wd_thread.start()
+            logger.info(
+                "Shutdown watchdog started: process will os._exit(0) in %.0fs if still alive",
+                _watchdog_delay,
+            )
+
             self._running = False
             self._draining = True
 
@@ -6163,6 +6187,28 @@ def _phase_elapsed() -> float:
                     self._update_runtime_status("draining")
                     await asyncio.sleep(0.1)
 
+                if self._running_agents:
+                    # Zombie sessions whose threads cannot be interrupted.
+                    # Schedule a hard os._exit() 10s from now so launchd /
+                    # systemd does NOT need to SIGKILL us — letting the rest
+                    # of _stop_impl() run for log flushing and DB close, but
+                    # guaranteeing we exit even if an adapter.disconnect() hangs.
+                    _hard_exit_delay = 10.0
+                    logger.warning(
+                        "Gateway drain: %d zombie session(s) remain after interrupt; "
+                        "scheduling os._exit(0) in %.0fs as last-resort watchdog.",
+                        len(self._running_agents),
+                        _hard_exit_delay,
+                    )
+                    import threading as _threading
+                    def _hard_exit_watchdog(delay: float) -> None:
+                        import time as _time
+                        _time.sleep(delay)
+                        import os as _os
+                        _os._exit(0)  # noqa: SIM115 — intentional hard exit after zombie drain
+                    _t = _threading.Thread(target=_hard_exit_watchdog, args=(_hard_exit_delay,), daemon=True)
+                    _t.start()
+
                 # Kill lingering tool subprocesses NOW, before we spend more
                 # budget on adapter disconnect / session DB close.  Under
                 # systemd (TimeoutStopSec bounded by drain_timeout+headroom),