From 087e9c3b2f9ce9cce82cd826341096a109aae241 Mon Sep 17 00:00:00 2001
From: Charan Jagwani <cjagwani@nvidia.com>
Date: Tue, 12 May 2026 07:31:42 -0700
Subject: [PATCH 1/2] fix(sandbox): auto-respawn gateway when it exits
 unexpectedly (#2757)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The sandbox entrypoint (scripts/nemoclaw-start.sh) ends with
`wait "$GATEWAY_PID"`. When the gateway process dies, that wait
unblocks, PID 1 exits, and Docker reaps the entire sandbox container.
NemoClaw also doesn't pass any --restart policy when OpenShell creates
the sandbox, so no automatic recovery happens — users have to run
`nemoclaw connect` to bring the sandbox back.

Wrap the terminal wait in a respawn loop on both branches (non-root
and root/step-down). The loop:
  - exits cleanly on rc=0 (graceful gateway shutdown) so SIGTERM /
    SIGINT shutdown via cleanup_on_signal is unaffected;
  - sleeps 2s and relaunches the gateway on any non-zero exit (kill,
    OOM, crash);
  - tracks respawn count in a 60s sliding window and logs a CRITICAL
    line after 5 respawns to surface a crashing gateway rather than
    silently masking it;
  - updates SANDBOX_WAIT_PID and appends to SANDBOX_CHILD_PIDS so the
    existing signal-cleanup path sees the new pid.

Closes #2757.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Signed-off-by: Charan Jagwani <cjagwani@nvidia.com>
---
 scripts/nemoclaw-start.sh | 61 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 58 insertions(+), 3 deletions(-)

diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh
index b38c093ba98..2708d1b63c1 100755
--- a/scripts/nemoclaw-start.sh
+++ b/scripts/nemoclaw-start.sh
@@ -2018,8 +2018,35 @@ if [ "$(id -u)" -ne 0 ]; then
   trap cleanup_on_signal SIGTERM SIGINT
   print_dashboard_urls
 
-  wait "$GATEWAY_PID"
-  exit $?
+  # Auto-respawn gateway on unexpected death (NVIDIA/NemoClaw#2757). Without
+  # this loop, gateway death unblocks `wait` → PID 1 exits → Docker reaps the
+  # whole sandbox container, forcing users to run `nemoclaw connect` to recover.
+  RESPAWN_COUNT=0
+  RESPAWN_WINDOW_START=$(date +%s)
+  while :; do
+    wait "$GATEWAY_PID"
+    RC=$?
+    if [ "$RC" -eq 0 ]; then
+      exit 0
+    fi
+    NOW=$(date +%s)
+    if [ $((NOW - RESPAWN_WINDOW_START)) -gt 60 ]; then
+      RESPAWN_COUNT=0
+      RESPAWN_WINDOW_START=$NOW
+    fi
+    RESPAWN_COUNT=$((RESPAWN_COUNT + 1))
+    if [ "$RESPAWN_COUNT" -ge 5 ]; then
+      echo "[gateway] CRITICAL: $RESPAWN_COUNT respawns in <60s — gateway likely unstable; check /tmp/gateway.log" >&2
+    fi
+    echo "[gateway] pid $GATEWAY_PID exited (rc=$RC); respawning (#$RESPAWN_COUNT in window) in 2s" >&2
+    sleep 2
+    nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 &
+    GATEWAY_PID=$!
+    # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
+    SANDBOX_WAIT_PID="$GATEWAY_PID"
+    SANDBOX_CHILD_PIDS+=("$GATEWAY_PID")
+    echo "[gateway] respawned (pid $GATEWAY_PID)" >&2
+  done
 fi
 
 # ── Root path (full privilege separation via setpriv) ──────────
@@ -2210,4 +2237,32 @@ print_dashboard_urls
 
 # Keep container running by waiting on the gateway process.
 # This script is PID 1 (ENTRYPOINT); if it exits, Docker kills all children.
-wait "$GATEWAY_PID"
+# Auto-respawn gateway on unexpected death (NVIDIA/NemoClaw#2757). Without
+# this loop, gateway death unblocks `wait` → PID 1 exits → Docker reaps the
+# whole sandbox container, forcing users to run `nemoclaw connect` to recover.
+RESPAWN_COUNT=0
+RESPAWN_WINDOW_START=$(date +%s)
+while :; do
+  wait "$GATEWAY_PID"
+  RC=$?
+  if [ "$RC" -eq 0 ]; then
+    exit 0
+  fi
+  NOW=$(date +%s)
+  if [ $((NOW - RESPAWN_WINDOW_START)) -gt 60 ]; then
+    RESPAWN_COUNT=0
+    RESPAWN_WINDOW_START=$NOW
+  fi
+  RESPAWN_COUNT=$((RESPAWN_COUNT + 1))
+  if [ "$RESPAWN_COUNT" -ge 5 ]; then
+    echo "[gateway] CRITICAL: $RESPAWN_COUNT respawns in <60s — gateway likely unstable; check /tmp/gateway.log" >&2
+  fi
+  echo "[gateway] pid $GATEWAY_PID exited (rc=$RC); respawning (#$RESPAWN_COUNT in window) in 2s" >&2
+  sleep 2
+  nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 &
+  GATEWAY_PID=$!
+  # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
+  SANDBOX_WAIT_PID="$GATEWAY_PID"
+  SANDBOX_CHILD_PIDS+=("$GATEWAY_PID")
+  echo "[gateway] respawned (pid $GATEWAY_PID)" >&2
+done

From d42db5b3bd3ba74d63f5800202272ae9d24dd916 Mon Sep 17 00:00:00 2001
From: Charan Jagwani <cjagwani@nvidia.com>
Date: Tue, 12 May 2026 08:01:51 -0700
Subject: [PATCH 2/2] fix(sandbox): address CodeRabbit feedback on #3409
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two fixes from CodeRabbit review of the gateway respawn loop:

1. Guard `wait` from errexit. scripts/nemoclaw-start.sh runs under
   `set -euo pipefail` (line 33), so a bare `wait "$GATEWAY_PID"`
   returning non-zero (e.g. 137 from kill -9) terminates PID 1 before
   the respawn logic ever runs — defeating the fix entirely. Replaced
   with `RC=0; wait "$GATEWAY_PID" || RC=$?` in both branches so
   errexit can't fire on the wait.

2. Implement a true sliding 60s window for the crash-loop alarm. The
   previous logic anchored the window at startup/reset, so bursts of
   five crashes spanning the boundary (e.g. at 9s/22s/35s/49s/62s)
   never triggered the alarm even though all five happened within
   53s of each other. Now tracks individual timestamps in
   RESPAWN_TIMES, prunes anything older than 60s each iteration,
   counts the remainder.

Re-verified in a synthetic Docker harness with `set -euo pipefail`:
respawn fires correctly on kill -9, and CRITICAL alarm fires on the
5th kill when spaced ~13s apart over a 53s span — the exact case the
old fixed-window logic missed.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Signed-off-by: Charan Jagwani <cjagwani@nvidia.com>
---
 scripts/nemoclaw-start.sh | 58 ++++++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 22 deletions(-)

diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh
index 2708d1b63c1..344be85f416 100755
--- a/scripts/nemoclaw-start.sh
+++ b/scripts/nemoclaw-start.sh
@@ -2021,24 +2021,31 @@ if [ "$(id -u)" -ne 0 ]; then
   # Auto-respawn gateway on unexpected death (NVIDIA/NemoClaw#2757). Without
   # this loop, gateway death unblocks `wait` → PID 1 exits → Docker reaps the
   # whole sandbox container, forcing users to run `nemoclaw connect` to recover.
-  RESPAWN_COUNT=0
-  RESPAWN_WINDOW_START=$(date +%s)
+  # RESPAWN_TIMES is a true sliding 60s window of crash timestamps; entries
+  # older than the cutoff are pruned each iteration so bursts spanning a
+  # window boundary still trigger the >=5 alarm.
+  RESPAWN_TIMES=()
   while :; do
-    wait "$GATEWAY_PID"
-    RC=$?
+    # `wait` must be guarded with `|| RC=$?` because errexit (set -e on
+    # line 33) would otherwise exit PID 1 the instant the gateway returns
+    # non-zero, defeating the respawn loop entirely.
+    RC=0
+    wait "$GATEWAY_PID" || RC=$?
     if [ "$RC" -eq 0 ]; then
       exit 0
     fi
     NOW=$(date +%s)
-    if [ $((NOW - RESPAWN_WINDOW_START)) -gt 60 ]; then
-      RESPAWN_COUNT=0
-      RESPAWN_WINDOW_START=$NOW
-    fi
-    RESPAWN_COUNT=$((RESPAWN_COUNT + 1))
+    RESPAWN_TIMES+=("$NOW")
+    _PRUNED=()
+    for _t in "${RESPAWN_TIMES[@]+"${RESPAWN_TIMES[@]}"}"; do
+      [ $((NOW - _t)) -le 60 ] && _PRUNED+=("$_t")
+    done
+    RESPAWN_TIMES=("${_PRUNED[@]+"${_PRUNED[@]}"}")
+    RESPAWN_COUNT=${#RESPAWN_TIMES[@]}
     if [ "$RESPAWN_COUNT" -ge 5 ]; then
-      echo "[gateway] CRITICAL: $RESPAWN_COUNT respawns in <60s — gateway likely unstable; check /tmp/gateway.log" >&2
+      echo "[gateway] CRITICAL: $RESPAWN_COUNT respawns in 60s window — gateway likely unstable; check /tmp/gateway.log" >&2
     fi
-    echo "[gateway] pid $GATEWAY_PID exited (rc=$RC); respawning (#$RESPAWN_COUNT in window) in 2s" >&2
+    echo "[gateway] pid $GATEWAY_PID exited (rc=$RC); respawning (#$RESPAWN_COUNT in 60s window) in 2s" >&2
     sleep 2
     nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 &
     GATEWAY_PID=$!
@@ -2240,24 +2247,31 @@ print_dashboard_urls
 # Auto-respawn gateway on unexpected death (NVIDIA/NemoClaw#2757). Without
 # this loop, gateway death unblocks `wait` → PID 1 exits → Docker reaps the
 # whole sandbox container, forcing users to run `nemoclaw connect` to recover.
-RESPAWN_COUNT=0
-RESPAWN_WINDOW_START=$(date +%s)
+# RESPAWN_TIMES is a true sliding 60s window of crash timestamps; entries
+# older than the cutoff are pruned each iteration so bursts spanning a
+# window boundary still trigger the >=5 alarm.
+RESPAWN_TIMES=()
 while :; do
-  wait "$GATEWAY_PID"
-  RC=$?
+  # `wait` must be guarded with `|| RC=$?` because errexit (set -e on
+  # line 33) would otherwise exit PID 1 the instant the gateway returns
+  # non-zero, defeating the respawn loop entirely.
+  RC=0
+  wait "$GATEWAY_PID" || RC=$?
   if [ "$RC" -eq 0 ]; then
     exit 0
   fi
   NOW=$(date +%s)
-  if [ $((NOW - RESPAWN_WINDOW_START)) -gt 60 ]; then
-    RESPAWN_COUNT=0
-    RESPAWN_WINDOW_START=$NOW
-  fi
-  RESPAWN_COUNT=$((RESPAWN_COUNT + 1))
+  RESPAWN_TIMES+=("$NOW")
+  _PRUNED=()
+  for _t in "${RESPAWN_TIMES[@]+"${RESPAWN_TIMES[@]}"}"; do
+    [ $((NOW - _t)) -le 60 ] && _PRUNED+=("$_t")
+  done
+  RESPAWN_TIMES=("${_PRUNED[@]+"${_PRUNED[@]}"}")
+  RESPAWN_COUNT=${#RESPAWN_TIMES[@]}
   if [ "$RESPAWN_COUNT" -ge 5 ]; then
-    echo "[gateway] CRITICAL: $RESPAWN_COUNT respawns in <60s — gateway likely unstable; check /tmp/gateway.log" >&2
+    echo "[gateway] CRITICAL: $RESPAWN_COUNT respawns in 60s window — gateway likely unstable; check /tmp/gateway.log" >&2
   fi
-  echo "[gateway] pid $GATEWAY_PID exited (rc=$RC); respawning (#$RESPAWN_COUNT in window) in 2s" >&2
+  echo "[gateway] pid $GATEWAY_PID exited (rc=$RC); respawning (#$RESPAWN_COUNT in 60s window) in 2s" >&2
   sleep 2
   nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 &
   GATEWAY_PID=$!