From a2cb7a051a32cb28190a457f6c31d45ce0d06f08 Mon Sep 17 00:00:00 2001
From: RelaxJonh <92573950+RelaxJonh@users.noreply.github.com>
Date: Sat, 1 Aug 2026 20:13:25 +0700
Subject: [PATCH] fix: reap stale background processes to prevent gateway
 starvation (#76115)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add a background sweep thread to ProcessRegistry that kills running
background processes exceeding MAX_ACTIVE_PROCESS_AGE (default 24h).

Without this, abandoned or stuck tool subprocesses (e.g. pnpm build)
run indefinitely, consuming unbounded memory until the gateway cgroup
hits MemoryHigh — starving the asyncio event loop and causing all
platforms and crons to time out.

Changes:
- Add STALE_SWEEP_INTERVAL constant (300s / 5 min)
- Add _stale_sweep_loop() daemon thread started at registry init
- Add _reap_stale_running() method using existing kill_process()
  which handles process-tree teardown (SIGKILL/SIGTERM POSIX,
  taskkill /T /F Windows)

Fixes NousResearch/hermes-agent#76115
---
 tools/process_registry.py | 65 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/tools/process_registry.py b/tools/process_registry.py
index 1cfc7ee7d0cc..0e79f9ec55d0 100644
--- a/tools/process_registry.py
+++ b/tools/process_registry.py
@@ -60,6 +60,7 @@
 FINISHED_TTL_SECONDS = 1800     # Keep finished processes for 30 minutes
 MAX_PROCESSES = 64              # Max concurrent tracked processes (LRU pruning)
 MAX_ACTIVE_PROCESS_AGE = 86400  # 24h default — see session_reset.bg_process_max_age_hours (#29177)
+STALE_SWEEP_INTERVAL = 300     # Check for stale background processes every 5 minutes
 
 # Watch pattern rate limiting — PER SESSION.
 # Hard rule: at most ONE watch-match notification every WATCH_MIN_INTERVAL_SECONDS.
@@ -214,6 +215,17 @@ def __init__(self):
         # UI view is dropped (the user can reopen it from the status stack).
         self.on_close = None
 
+        # Background sweep thread — kills running processes that exceed
+        # MAX_ACTIVE_PROCESS_AGE to prevent leaked subprocesses from consuming
+        # unbounded resources (issue #76115).
+        self._stale_sweep_stop = threading.Event()
+        self._stale_sweep_thread = threading.Thread(
+            target=self._stale_sweep_loop,
+            daemon=True,
+            name="process-stale-sweep",
+        )
+        self._stale_sweep_thread.start()
+
     @staticmethod
     def _clean_shell_noise(text: str) -> str:
         """Strip shell startup warnings from the beginning of output."""
@@ -1948,6 +1960,59 @@ def kill_all(self, task_id: str = None) -> int:
                 killed += 1
         return killed
 
+    # ----- Stale-process sweep (issue #76115) -----
+
+    def _stale_sweep_loop(self) -> None:
+        """Background daemon that kills long-running background processes.
+
+        Prevents leaked subprocesses (e.g. abandoned builds) from consuming
+        unbounded memory and starving the gateway event loop.  Runs every
+        STALE_SWEEP_INTERVAL seconds and kills any running process that
+        exceeds MAX_ACTIVE_PROCESS_AGE.
+        """
+        while not self._stale_sweep_stop.is_set():
+            try:
+                self._stale_sweep_stop.wait(STALE_SWEEP_INTERVAL)
+                if self._stale_sweep_stop.is_set():
+                    break
+                self._reap_stale_running()
+            except Exception:
+                logger.debug("Stale-process sweep iteration failed", exc_info=True)
+
+    def _reap_stale_running(self, max_age: Optional[float] = None) -> int:
+        """Kill running processes that exceed *max_age* seconds.
+
+        Returns the number of processes killed.  Uses the existing
+        ``kill_process`` infrastructure which handles process-tree teardown
+        (SIGKILL/SIGTERM on POSIX, taskkill /T /F on Windows) and moves
+        the session to the finished dict.
+        """
+        if max_age is None:
+            max_age = float(MAX_ACTIVE_PROCESS_AGE)
+        now = time.time()
+        with self._lock:
+            stale_ids = [
+                s.id for s in self._running.values()
+                if not s.exited and (now - s.started_at) > max_age
+            ]
+
+        killed = 0
+        for sid in stale_ids:
+            age_hours = None
+            with self._lock:
+                s = self._running.get(sid)
+                if s:
+                    age_hours = (now - s.started_at) / 3600
+            logger.warning(
+                "Reaping stale background process %s (age %.1fh exceeds max %dh): %s",
+                sid, age_hours or 0, int(max_age // 3600),
+                s.command if s else "<unknown>",
+            )
+            result = self.kill_process(sid, source="stale_sweep", consume_output=False)
+            if result.get("status") in {"killed", "already_exited"}:
+                killed += 1
+        return killed
+
     # ----- Cleanup / Pruning -----
 
     def _prune_if_needed(self):