fix(sandbox): pin gateway.reload=hot and add gateway serving watchdog (#4710)

Dockerfile

@@ -1061,10 +1061,15 @@ RUN set -eu; \
 #           host-side delivery-chain monitoring (verify-deployment.ts, host
 #           port forward, sandbox status).
 #
-# The process pattern matches both `openclaw gateway run` (the launcher
-# command nemoclaw-start runs) and `openclaw-gateway` (the re-execed
-# binary form OpenClaw switches into after startup). This is the same
-# variant set the host-side gateway-stop script in services.ts matches.
+# The process pattern matches `openclaw gateway run` (the launcher
+# command nemoclaw-start runs), `openclaw-gateway` (the re-execed
+# binary form OpenClaw switches into after startup), and a cmdline of
+# exactly `openclaw` (some OpenClaw builds truncate the rewritten
+# process title to the bare binary name — observed in #4710). The bare
+# form is anchored (^openclaw$) so an agent running ordinary `openclaw
+# <subcommand>` CLI invocations can never satisfy the liveness probe.
+# This is the same variant set the host-side recovery script in
+# src/lib/agent/runtime.ts kills before relaunching.
 #
 # pgrep uses --ignore-ancestors so it cannot self-match the healthcheck
 # shell that Docker spawns to run this CMD — that shell's argv contains
@@ -1082,7 +1087,7 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=45s --retries=3 \
         if [ "$rc" = 0 ]; then exit 0; fi; \
         if [ "$rc" != 7 ]; then exit 1; fi; \
         [ -f /tmp/nemoclaw-gateway-local ] || exit 0; \
-        pgrep --ignore-ancestors -f 'openclaw[ -]gateway' > /dev/null 2>&1 || exit 1; \
+        pgrep --ignore-ancestors -f '^openclaw$|openclaw[ -]gateway' > /dev/null 2>&1 || exit 1; \
         [ -s /tmp/gateway.log ]
 
 # Entrypoint runs as root to start the gateway as the gateway user,

ci/test-file-size-budget.json

@@ -8,7 +8,7 @@
     "test/channels-add-preset.test.ts": 1872,
     "test/generate-openclaw-config.test.ts": 1990,
     "test/install-preflight.test.ts": 4207,
-    "test/nemoclaw-start.test.ts": 5231,
+    "test/nemoclaw-start.test.ts": 5237,
     "test/onboard-messaging.test.ts": 2094,
     "test/onboard-selection.test.ts": 6891,
     "test/onboard.test.ts": 4775,

docs/reference/troubleshooting.mdx

@@ -364,6 +364,30 @@ $$nemoclaw <name> connect
 
 Run `$$nemoclaw status` for a broader gateway health report.
 
+### Sandbox container reports `(unhealthy)` while the agent gateway process is still alive
+
+The in-sandbox OpenClaw gateway can drop its HTTP listener while its process stays alive.
+A restart-class configuration change makes the gateway restart itself in place, and if that restart fails the process parks with no listener (`/tmp/gateway.log` shows `gateway startup failed: ... Process will stay alive`).
+Docker then marks the container `(unhealthy)` even though `pgrep` still finds the gateway.
+
+NemoClaw prevents and self-heals this:
+
+- The generated sandbox config pins `gateway.reload.mode` to `hot`, so configuration changes never make the gateway restart itself out from under the sandbox supervisor.
+- A serving watchdog inside the sandbox kills a gateway that stops listening after it has served, and the supervisor relaunches it (look for `[gateway-watchdog]` lines in `$$nemoclaw <name> logs`).
+
+Because of the `hot` pin, restart-class configuration changes made inside the sandbox — for example `openclaw plugins install` — log `config reload requires gateway restart; hot mode ignoring` and do not take effect until the gateway restarts.
+Apply them with a supervised restart:
+
+```bash
+$$nemoclaw <name> recover
+```
+
+or rebuild the sandbox for changes that affect provisioning:
+
+```bash
+$$nemoclaw <name> rebuild --yes
+```
+
 ### Invalid sandbox name
 
 Sandbox names must be lowercase, start with a letter, contain only letters, numbers, and internal hyphens, and end with a letter or number.

scripts/generate-openclaw-config.mts

@@ -935,6 +935,19 @@ export function buildConfig(env: Env = process.env): JsonObject {
       },
       trustedProxies: ["127.0.0.1", "::1"],
       auth: { token: "" },
+      // Restart-class config changes (plugins.installs, models.pricing,
+      // unrecognized keys, ...) must not let the gateway SIGUSR1-restart
+      // itself: in containers the in-process restart path can fail and park
+      // the process alive with no HTTP listener, which the PID-wait respawn
+      // loop in nemoclaw-start.sh cannot observe (#4710). Hot mode makes the
+      // gateway ignore plan-driven restarts; NemoClaw applies restart-class
+      // changes through sandbox rebuild or `nemoclaw <name> recover` instead.
+      // Removal condition (also for the serving watchdog in
+      // nemoclaw-start.sh): once the pinned OpenClaw release exits non-zero
+      // when a failed in-process restart cannot re-bind its listener — so the
+      // respawn loop sees the death — this pin can revert to the default
+      // reload mode after a wedge drill proves no regression.
+      reload: { mode: "hot" },
     },
   };
 

scripts/nemoclaw-start.sh

@@ -3275,6 +3275,147 @@ start_plugin_registry_refresh() {
   PLUGIN_REFRESH_PID=$!
 }
 
+# Watchdog for the in-container gateway HTTP listener (#4710). OpenClaw's
+# config reloader can SIGUSR1-restart the gateway in-process; in containers a
+# failed restart parks the process alive with its listener closed ("gateway
+# startup failed: ... Process will stay alive"). The #2757 respawn loop only
+# observes process exit, so an alive-but-deaf gateway would stay wedged until
+# a human runs `nemoclaw <sandbox> recover`. This watchdog probes the local
+# health endpoint and — once it has seen a listener at least once — kills the
+# gateway after sustained connection-refused so the respawn loop relaunches
+# it. Only curl exit 7 counts as "listener gone": 200/401 mean serving, and
+# timeout / HTTP-error outcomes (curl 28/22) mean a listener exists and remain
+# the Docker HEALTHCHECK's responsibility. Arming only after the first
+# non-refused probe means a slow first boot is never killed; failed first
+# boots stay the respawn loop's and HEALTHCHECK's job.
+GATEWAY_PID_FILE=/tmp/nemoclaw-gateway.pid
+
+# Record the live gateway PID where the watchdog (a separate PID 1 child whose
+# copy of $GATEWAY_PID goes stale after a respawn) can re-read it each cycle.
+# The pidfile lives in sticky, world-writable /tmp, so in root mode it must
+# never be written through a path a sandbox process could have planted:
+# unlink-then-open is symlink-raceable (root would follow an attacker symlink
+# and gain an arbitrary write/chmod). Write to a mktemp-owned file and
+# atomically rename it into place — rename(2) replaces a planted symlink as a
+# directory entry instead of following it, and the result is owned by PID 1's
+# uid so the sandbox user can neither modify nor replace it (it can at worst
+# deny its own sandbox's self-healing, e.g. by pre-creating a directory).
+# Best-effort: a write failure must never block startup or respawn.
+record_gateway_pid() {
+  local tmp
+  tmp="$(mktemp "${GATEWAY_PID_FILE}.XXXXXX" 2>/dev/null)" || return 0
+  if ! printf '%s\n' "$1" >"$tmp" 2>/dev/null; then
+    rm -f "$tmp" 2>/dev/null || true
+    return 0
+  fi
+  chmod 644 "$tmp" 2>/dev/null || true
+  mv -f "$tmp" "$GATEWAY_PID_FILE" 2>/dev/null || rm -f "$tmp" 2>/dev/null || true
+}
+
+# PID-reuse / tamper defense: only kill a process whose cmdline still looks
+# like the OpenClaw gateway. Same pattern family as the host-side recovery
+# script (src/lib/agent/runtime.ts): matches the launch argv
+# ("... openclaw gateway run --port N") and the rewritten process titles
+# ("openclaw-gateway", bare "openclaw").
+gateway_pid_is_openclaw_gateway() {
+  # _NEMOCLAW_PROC_ROOT is a test seam (unit tests also run on macOS, which
+  # has no /proc). Production always uses /proc: the watchdog inherits PID 1's
+  # environment, which the sandbox user cannot influence.
+  local cmdline
+  cmdline="$(tr '\0' ' ' <"${_NEMOCLAW_PROC_ROOT:-/proc}/$1/cmdline" 2>/dev/null)" || return 1
+  cmdline="${cmdline%"${cmdline##*[![:space:]]}"}"
+  [ -n "$cmdline" ] || return 1
+  printf '%s' "$cmdline" | grep -qE 'openclaw([ -]gateway| gateway run|$)'
+}
+
+start_gateway_serving_watchdog() {
+  (
+    local interval refused_threshold armed=0 refused_streak=0 pid last_pid="" rc msg
+    interval="${NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS:-30}"
+    refused_threshold="${NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD:-4}"
+    # Both knobs must be positive integers: a zero/garbage interval would
+    # busy-loop the probe, and a zero threshold would kill on the first
+    # refusal. Fall back to the defaults rather than trusting bad input.
+    case "$interval" in
+      [1-9] | [1-9][0-9]*) ;;
+      *)
+        echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS='${interval}'; defaulting to 30" >&2
+        interval=30
+        ;;
+    esac
+    case "$refused_threshold" in
+      [1-9] | [1-9][0-9]*) ;;
+      *)
+        echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD='${refused_threshold}'; defaulting to 4" >&2
+        refused_threshold=4
+        ;;
+    esac
+    [ -n "${_DASHBOARD_PORT:-}" ] || exit 0
+    while :; do
+      sleep "$interval"
+      pid="$(cat "$GATEWAY_PID_FILE" 2>/dev/null)" || pid=""
+      case "$pid" in
+        '' | *[!0-9]*)
+          last_pid=""
+          armed=0
+          refused_streak=0
+          continue
+          ;;
+      esac
+      # A respawned gateway must earn its own armed state — never inherit
+      # the previous PID's serve history, or a booting replacement could be
+      # killed for refusals that belong to its predecessor.
+      if [ "$pid" != "$last_pid" ]; then
+        last_pid="$pid"
+        armed=0
+        refused_streak=0
+      fi
+      if ! kill -0 "$pid" 2>/dev/null; then
+        # Process exit is the respawn loop's signal, not ours.
+        last_pid=""
+        armed=0
+        refused_streak=0
+        continue
+      fi
+      rc=0
+      curl -s -o /dev/null --max-time 5 "http://127.0.0.1:${_DASHBOARD_PORT}/health" 2>/dev/null || rc=$?
+      if [ "$rc" -ne 7 ]; then
+        armed=1
+        refused_streak=0
+        continue
+      fi
+      [ "$armed" -eq 1 ] || continue
+      refused_streak=$((refused_streak + 1))
+      if [ "$refused_streak" -lt "$refused_threshold" ]; then
+        echo "[gateway-watchdog] gateway pid $pid alive but port ${_DASHBOARD_PORT} refused connection ($refused_streak/$refused_threshold) (#4710)" >&2
+        continue
+      fi
+      if ! gateway_pid_is_openclaw_gateway "$pid"; then
+        echo "[gateway-watchdog] pid $pid no longer looks like the openclaw gateway; not killing (#4710)" >&2
+        armed=0
+        refused_streak=0
+        continue
+      fi
+      msg="[gateway-watchdog] CRITICAL: gateway pid $pid is alive but dropped its HTTP listener on port ${_DASHBOARD_PORT} ($refused_streak consecutive refused probes); killing it so the respawn loop can relaunch (#4710)"
+      echo "$msg" >&2
+      # _NEMOCLAW_GATEWAY_LOG is a test seam; production always appends to
+      # /tmp/gateway.log alongside the gateway's own output.
+      echo "$msg" >>"${_NEMOCLAW_GATEWAY_LOG:-/tmp/gateway.log}" 2>/dev/null || true
+      kill -TERM "$pid" 2>/dev/null || true
+      for _ in 1 2 3 4 5 6 7 8 9 10; do
+        kill -0 "$pid" 2>/dev/null || break
+        sleep 1
+      done
+      if kill -0 "$pid" 2>/dev/null; then
+        kill -KILL "$pid" 2>/dev/null || true
+      fi
+      armed=0
+      refused_streak=0
+    done
+  ) &
+  GATEWAY_WATCHDOG_PID=$!
+}
+
 # ── Main ─────────────────────────────────────────────────────────
 
 # Migrate legacy symlink layout before anything else reads .openclaw
@@ -3388,6 +3529,7 @@ if [ "$(id -u)" -ne 0 ]; then
   mark_in_container_gateway
   nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 &
   GATEWAY_PID=$!
+  record_gateway_pid "$GATEWAY_PID"
   echo "[gateway] openclaw gateway launched (pid $GATEWAY_PID)" >&2
   # Diagnostic: mirror gateway log to PID 1's stderr — see root-mode block
   # below for rationale (NVIDIA/NemoClaw#2484).
@@ -3397,6 +3539,7 @@ if [ "$(id -u)" -ne 0 ]; then
   start_persistent_gateway_log_mirror || exit 1
   start_auto_pair
   start_plugin_registry_refresh
+  start_gateway_serving_watchdog
   # NOTE: PIDs are collected after launch; a signal arriving between trap
   # registration and the final append is a small race window (same as before
   # the shared-library refactor). Acceptable for entrypoint-level cleanup.
@@ -3405,6 +3548,7 @@ if [ "$(id -u)" -ne 0 ]; then
   [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID")
   [ -n "${GATEWAY_LOG_PERSIST_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_PERSIST_PID")
   [ -n "${PLUGIN_REFRESH_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$PLUGIN_REFRESH_PID")
+  [ -n "${GATEWAY_WATCHDOG_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_WATCHDOG_PID")
   # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
   SANDBOX_WAIT_PID="$GATEWAY_PID"
   trap cleanup_on_signal SIGTERM SIGINT
@@ -3441,6 +3585,7 @@ if [ "$(id -u)" -ne 0 ]; then
     sleep 2
     nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 &
     GATEWAY_PID=$!
+    record_gateway_pid "$GATEWAY_PID"
     # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
     SANDBOX_WAIT_PID="$GATEWAY_PID"
     SANDBOX_CHILD_PIDS+=("$GATEWAY_PID")
@@ -3613,6 +3758,7 @@ validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON
 mark_in_container_gateway
 nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 &
 GATEWAY_PID=$!
+record_gateway_pid "$GATEWAY_PID"
 echo "[gateway] openclaw gateway launched as 'gateway' user (pid $GATEWAY_PID)" >&2
 
 # Diagnostic: mirror gateway log to PID 1's stderr so its content surfaces in
@@ -3652,6 +3798,8 @@ start_auto_pair
 # proves /nemoclaw registration without the refresh.
 start_plugin_registry_refresh
 
+start_gateway_serving_watchdog
+
 # NOTE: PIDs are collected after launch; a signal arriving between trap
 # registration and the final append is a small race window (same as before
 # the shared-library refactor). Acceptable for entrypoint-level cleanup.
@@ -3660,6 +3808,7 @@ SANDBOX_CHILD_PIDS=("$GATEWAY_PID")
 [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID")
 [ -n "${GATEWAY_LOG_PERSIST_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_PERSIST_PID")
 [ -n "${PLUGIN_REFRESH_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$PLUGIN_REFRESH_PID")
+[ -n "${GATEWAY_WATCHDOG_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_WATCHDOG_PID")
 # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
 SANDBOX_WAIT_PID="$GATEWAY_PID"
 trap cleanup_on_signal SIGTERM SIGINT
@@ -3698,6 +3847,7 @@ while :; do
   sleep 2
   nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 &
   GATEWAY_PID=$!
+  record_gateway_pid "$GATEWAY_PID"
   # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
   SANDBOX_WAIT_PID="$GATEWAY_PID"
   SANDBOX_CHILD_PIDS+=("$GATEWAY_PID")

src/lib/state/openclaw-config-merge.test.ts

@@ -58,6 +58,28 @@ describe("mergeOpenClawRestoredConfig", () => {
     expect((merged as { channels: Record<string, unknown> }).channels.slack).toBeUndefined();
   });
 
+  it("keeps the rebuilt gateway section — including the reload pin — over the backup's (#4710)", () => {
+    // gateway.reload.mode="hot" is what keeps the in-sandbox gateway from
+    // SIGUSR1-restarting itself out from under the nemoclaw-start respawn
+    // loop. A backup taken before the pin existed (or carrying a different
+    // mode) must not reintroduce restart-mode reloads on restore.
+    const merged = mergeOpenClawRestoredConfig(
+      {
+        gateway: {
+          auth: { token: "stale-token" },
+          reload: { mode: "hybrid" },
+          controlUi: { allowInsecureAuth: true },
+        },
+      },
+      { gateway: { auth: { token: "fresh-token" }, reload: { mode: "hot" } } },
+    ) as { gateway: unknown };
+
+    expect(merged.gateway).toEqual({
+      auth: { token: "fresh-token" },
+      reload: { mode: "hot" },
+    });
+  });
+
   it("does not resurrect managed channels when the rebuilt config omits channels", () => {
     const merged = mergeOpenClawRestoredConfig(
       {