fix(sandbox): pin gateway.reload=hot and add gateway serving watchdog (#4710)

ci/test-file-size-budget.json

@@ -8,7 +8,7 @@
     "test/channels-add-preset.test.ts": 1871,
     "test/generate-openclaw-config.test.ts": 1984,
     "test/install-preflight.test.ts": 4207,
-    "test/nemoclaw-start.test.ts": 5160,
+    "test/nemoclaw-start.test.ts": 5043,
     "test/onboard-messaging.test.ts": 2062,
     "test/onboard-selection.test.ts": 6888,
     "test/onboard.test.ts": 4774,

docs/reference/troubleshooting.mdx

@@ -392,6 +392,30 @@ $$nemoclaw <name> connect
 
 Run `$$nemoclaw status` for a broader gateway health report.
 
+### Sandbox container reports `(unhealthy)` while the agent gateway process is still alive
+
+The in-sandbox OpenClaw gateway can drop its HTTP listener while its process stays alive.
+A restart-class configuration change makes the gateway restart itself in place, and if that restart fails the process parks with no listener (`/tmp/gateway.log` shows `gateway startup failed: ... Process will stay alive`).
+Docker then marks the container `(unhealthy)` even though `pgrep` still finds the gateway.
+
+NemoClaw prevents and self-heals this:
+
+- The generated sandbox config pins `gateway.reload.mode` to `hot`, so configuration changes never make the gateway restart itself out from under the sandbox supervisor.
+- A serving watchdog inside the sandbox kills a gateway that stops listening after it has served, and the supervisor relaunches it (look for `[gateway-watchdog]` lines in `$$nemoclaw <name> logs`).
+
+Because of the `hot` pin, restart-class configuration changes made inside the sandbox — for example `openclaw plugins install` — log `config reload requires gateway restart; hot mode ignoring` and do not take effect until the gateway restarts.
+Apply them with a supervised restart:
+
+```bash
+$$nemoclaw <name> recover
+```
+
+or rebuild the sandbox for changes that affect provisioning:
+
+```bash
+$$nemoclaw <name> rebuild --yes
+```
+
 ### Invalid sandbox name
 
 Sandbox names must be lowercase, start with a letter, contain only letters, numbers, and internal hyphens, and end with a letter or number.

scripts/generate-openclaw-config.mts

@@ -1231,6 +1231,19 @@ export function buildConfig(env: Env = process.env): JsonObject {
       },
       trustedProxies: ["127.0.0.1", "::1"],
       auth: { token: "" },
+      // Restart-class config changes (plugins.installs, models.pricing,
+      // unrecognized keys, ...) must not let the gateway SIGUSR1-restart
+      // itself: in containers the in-process restart path can fail and park
+      // the process alive with no HTTP listener, which the PID-wait respawn
+      // loop in nemoclaw-start.sh cannot observe (#4710). Hot mode makes the
+      // gateway ignore plan-driven restarts; NemoClaw applies restart-class
+      // changes through sandbox rebuild or `nemoclaw <name> recover` instead.
+      // Removal condition (also for the serving watchdog in
+      // nemoclaw-start.sh): once the pinned OpenClaw release exits non-zero
+      // when a failed in-process restart cannot re-bind its listener — so the
+      // respawn loop sees the death — this pin can revert to the default
+      // reload mode after a wedge drill proves no regression.
+      reload: { mode: "hot" },
     },
   };
 

scripts/nemoclaw-start.sh

@@ -244,6 +244,10 @@ NEMOCLAW_CMD=("$@")
 # exists if-and-only-if this container is about to start the gateway. Both the
 # root and non-root entrypoint paths call `mark_in_container_gateway` directly
 # before their `openclaw gateway run` invocation.
+# Internal test seam shared by the PID writer and watchdog. This is deliberately
+# not documented as a public env API; production always keeps the default path.
+GATEWAY_PID_FILE=/tmp/nemoclaw-gateway.pid
+
 # Best-effort: a write failure must never block startup.
 mark_in_container_gateway() {
   _nemoclaw_safe_create_tmp_file /tmp/nemoclaw-gateway-local 600 "" best-effort 2>/dev/null || true
@@ -256,7 +260,7 @@ mark_in_container_gateway() {
 # is tracked and a window where the gateway is down reads as unhealthy.
 # Best-effort: a write failure must never block startup.
 record_gateway_pid() {
-  printf '%s\n' "${1:-}" | _nemoclaw_safe_replace_tmp_file /tmp/nemoclaw-gateway.pid 600 "" best-effort 2>/dev/null || true
+  printf '%s\n' "${1:-}" | _nemoclaw_safe_replace_tmp_file "$GATEWAY_PID_FILE" 600 "" best-effort 2>/dev/null || true
 }
 
 _chat_ui_url_port() {
@@ -3696,6 +3700,124 @@ start_plugin_registry_refresh() {
   PLUGIN_REFRESH_PID=$!
 }
 
+# Watchdog for the in-container gateway HTTP listener (#4710). OpenClaw's
+# config reloader can SIGUSR1-restart the gateway in-process; in containers a
+# failed restart parks the process alive with its listener closed ("gateway
+# startup failed: ... Process will stay alive"). The #2757 respawn loop only
+# observes process exit, so an alive-but-deaf gateway would stay wedged until
+# a human runs `nemoclaw <sandbox> recover`. This watchdog probes the local
+# health endpoint and — once it has seen a listener at least once — kills the
+# gateway after sustained connection-refused so the respawn loop relaunches
+# it. Only curl exit 7 counts as "listener gone": 200/401 mean serving, and
+# timeout / HTTP-error outcomes (curl 28/22) mean a listener exists and remain
+# the Docker HEALTHCHECK's responsibility. Arming only after the first
+# non-refused probe means a slow first boot is never killed; failed first
+# boots stay the respawn loop's and HEALTHCHECK's job.
+
+# PID-reuse / tamper defense: only kill a process whose cmdline still looks
+# like the OpenClaw gateway. Same pattern family as the host-side recovery
+# script (src/lib/agent/runtime.ts): matches the launch argv
+# ("... openclaw gateway run --port N") and the rewritten process titles
+# ("openclaw-gateway", bare "openclaw").
+gateway_pid_is_openclaw_gateway() {
+  # _NEMOCLAW_PROC_ROOT is a test seam (unit tests also run on macOS, which
+  # has no /proc). Production always uses /proc: the watchdog inherits PID 1's
+  # environment, which the sandbox user cannot influence.
+  local cmdline
+  cmdline="$(tr '\0' ' ' <"${_NEMOCLAW_PROC_ROOT:-/proc}/$1/cmdline" 2>/dev/null)" || return 1
+  cmdline="${cmdline%"${cmdline##*[![:space:]]}"}"
+  [ -n "$cmdline" ] || return 1
+  printf '%s' "$cmdline" | grep -qE 'openclaw([ -]gateway| gateway run|$)'
+}
+
+start_gateway_serving_watchdog() {
+  (
+    local interval refused_threshold armed=0 refused_streak=0 pid last_pid="" rc msg
+    interval="${NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS:-30}"
+    refused_threshold="${NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD:-4}"
+    # Both knobs must be positive integers: a zero/garbage interval would
+    # busy-loop the probe, and a zero threshold would kill on the first
+    # refusal. Fall back to the defaults rather than trusting bad input.
+    case "$interval" in
+      [1-9] | [1-9][0-9]*) ;;
+      *)
+        echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS='${interval}'; defaulting to 30" >&2
+        interval=30
+        ;;
+    esac
+    case "$refused_threshold" in
+      [1-9] | [1-9][0-9]*) ;;
+      *)
+        echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD='${refused_threshold}'; defaulting to 4" >&2
+        refused_threshold=4
+        ;;
+    esac
+    [ -n "${_DASHBOARD_PORT:-}" ] || exit 0
+    while :; do
+      sleep "$interval"
+      pid="$(cat "$GATEWAY_PID_FILE" 2>/dev/null)" || pid=""
+      case "$pid" in
+        '' | *[!0-9]*)
+          last_pid=""
+          armed=0
+          refused_streak=0
+          continue
+          ;;
+      esac
+      # A respawned gateway must earn its own armed state — never inherit
+      # the previous PID's serve history, or a booting replacement could be
+      # killed for refusals that belong to its predecessor.
+      if [ "$pid" != "$last_pid" ]; then
+        last_pid="$pid"
+        armed=0
+        refused_streak=0
+      fi
+      if ! kill -0 "$pid" 2>/dev/null; then
+        # Process exit is the respawn loop's signal, not ours.
+        last_pid=""
+        armed=0
+        refused_streak=0
+        continue
+      fi
+      rc=0
+      curl -s -o /dev/null --max-time 5 "http://127.0.0.1:${_DASHBOARD_PORT}/health" 2>/dev/null || rc=$?
+      if [ "$rc" -ne 7 ]; then
+        armed=1
+        refused_streak=0
+        continue
+      fi
+      [ "$armed" -eq 1 ] || continue
+      refused_streak=$((refused_streak + 1))
+      if [ "$refused_streak" -lt "$refused_threshold" ]; then
+        echo "[gateway-watchdog] gateway pid $pid alive but port ${_DASHBOARD_PORT} refused connection ($refused_streak/$refused_threshold) (#4710)" >&2
+        continue
+      fi
+      if ! gateway_pid_is_openclaw_gateway "$pid"; then
+        echo "[gateway-watchdog] pid $pid no longer looks like the openclaw gateway; not killing (#4710)" >&2
+        armed=0
+        refused_streak=0
+        continue
+      fi
+      msg="[gateway-watchdog] CRITICAL: gateway pid $pid is alive but dropped its HTTP listener on port ${_DASHBOARD_PORT} ($refused_streak consecutive refused probes); killing it so the respawn loop can relaunch (#4710)"
+      echo "$msg" >&2
+      # _NEMOCLAW_GATEWAY_LOG is a test seam; production always appends to
+      # /tmp/gateway.log alongside the gateway's own output.
+      echo "$msg" >>"${_NEMOCLAW_GATEWAY_LOG:-/tmp/gateway.log}" 2>/dev/null || true
+      kill -TERM "$pid" 2>/dev/null || true
+      for _ in 1 2 3 4 5 6 7 8 9 10; do
+        kill -0 "$pid" 2>/dev/null || break
+        sleep 1
+      done
+      if kill -0 "$pid" 2>/dev/null; then
+        kill -KILL "$pid" 2>/dev/null || true
+      fi
+      armed=0
+      refused_streak=0
+    done
+  ) &
+  GATEWAY_WATCHDOG_PID=$!
+}
+
 # ── Main ─────────────────────────────────────────────────────────
 
 # Migrate legacy symlink layout before anything else reads .openclaw
@@ -3816,6 +3938,7 @@ if [ "$(id -u)" -ne 0 ]; then
   start_persistent_gateway_log_mirror || exit 1
   start_auto_pair
   start_plugin_registry_refresh
+  start_gateway_serving_watchdog
   # NOTE: PIDs are collected after launch; a signal arriving between trap
   # registration and the final append is a small race window (same as before
   # the shared-library refactor). Acceptable for entrypoint-level cleanup.
@@ -3824,6 +3947,7 @@ if [ "$(id -u)" -ne 0 ]; then
   [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID")
   [ -n "${GATEWAY_LOG_PERSIST_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_PERSIST_PID")
   [ -n "${PLUGIN_REFRESH_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$PLUGIN_REFRESH_PID")
+  [ -n "${GATEWAY_WATCHDOG_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_WATCHDOG_PID")
   # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
   SANDBOX_WAIT_PID="$GATEWAY_PID"
   trap cleanup_on_signal SIGTERM SIGINT
@@ -4066,6 +4190,8 @@ start_auto_pair
 # proves /nemoclaw registration without the refresh.
 start_plugin_registry_refresh
 
+start_gateway_serving_watchdog
+
 # NOTE: PIDs are collected after launch; a signal arriving between trap
 # registration and the final append is a small race window (same as before
 # the shared-library refactor). Acceptable for entrypoint-level cleanup.
@@ -4074,6 +4200,7 @@ SANDBOX_CHILD_PIDS=("$GATEWAY_PID")
 [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID")
 [ -n "${GATEWAY_LOG_PERSIST_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_PERSIST_PID")
 [ -n "${PLUGIN_REFRESH_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$PLUGIN_REFRESH_PID")
+[ -n "${GATEWAY_WATCHDOG_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_WATCHDOG_PID")
 # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
 SANDBOX_WAIT_PID="$GATEWAY_PID"
 trap cleanup_on_signal SIGTERM SIGINT

src/lib/state/openclaw-config-merge.test.ts

@@ -58,6 +58,28 @@ describe("mergeOpenClawRestoredConfig", () => {
     expect((merged as { channels: Record<string, unknown> }).channels.slack).toBeUndefined();
   });
 
+  it("keeps the rebuilt gateway section — including the reload pin — over the backup's (#4710)", () => {
+    // gateway.reload.mode="hot" is what keeps the in-sandbox gateway from
+    // SIGUSR1-restarting itself out from under the nemoclaw-start respawn
+    // loop. A backup taken before the pin existed (or carrying a different
+    // mode) must not reintroduce restart-mode reloads on restore.
+    const merged = mergeOpenClawRestoredConfig(
+      {
+        gateway: {
+          auth: { token: "stale-token" },
+          reload: { mode: "hybrid" },
+          controlUi: { allowInsecureAuth: true },
+        },
+      },
+      { gateway: { auth: { token: "fresh-token" }, reload: { mode: "hot" } } },
+    ) as { gateway: unknown };
+
+    expect(merged.gateway).toEqual({
+      auth: { token: "fresh-token" },
+      reload: { mode: "hot" },
+    });
+  });
+
   it("does not resurrect managed channels when the rebuilt config omits channels", () => {
     const merged = mergeOpenClawRestoredConfig(
       {

test/gateway-pid-recording.test.ts

@@ -40,13 +40,11 @@ describe("nemoclaw-start gateway PID recording for HEALTHCHECK (#4952)", () => {
     const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-pid-"));
     try {
       const pidPath = path.join(tmp, "nemoclaw-gateway.pid");
-      const fn = extractFunction(src, "record_gateway_pid").replaceAll(
-        "/tmp/nemoclaw-gateway.pid",
-        pidPath,
-      );
+      const fn = extractFunction(src, "record_gateway_pid");
       const script = [
         "set -euo pipefail",
         safeTmpHelpers(src),
+        `GATEWAY_PID_FILE=${JSON.stringify(pidPath)}`,
         fn,
         'record_gateway_pid "12345"',
       ].join("\n");
@@ -65,13 +63,11 @@ describe("nemoclaw-start gateway PID recording for HEALTHCHECK (#4952)", () => {
     // The writer must swallow errors: a failed write must never abort the
     // entrypoint. Point it at an unwritable path and assert success.
     const src = fs.readFileSync(START_SCRIPT, "utf-8");
-    const fn = extractFunction(src, "record_gateway_pid").replaceAll(
-      "/tmp/nemoclaw-gateway.pid",
-      "/nonexistent-dir/nemoclaw-gateway.pid",
-    );
+    const fn = extractFunction(src, "record_gateway_pid");
     const script = [
       "set -euo pipefail",
       safeTmpHelpers(src),
+      "GATEWAY_PID_FILE=/nonexistent-dir/nemoclaw-gateway.pid",
       fn,
       'record_gateway_pid "12345"',
     ].join("\n");