fix(sandbox): match re-execed plain-openclaw gateway argv in HEALTHCHECK (#4952)

Dockerfile

@@ -1061,10 +1061,25 @@ RUN set -eu; \
 #           host-side delivery-chain monitoring (verify-deployment.ts, host
 #           port forward, sandbox status).
 #
-# The process pattern matches both `openclaw gateway run` (the launcher
-# command nemoclaw-start runs) and `openclaw-gateway` (the re-execed
-# binary form OpenClaw switches into after startup). This is the same
-# variant set the host-side gateway-stop script in services.ts matches.
+# The pgrep pattern matches both `openclaw gateway run` (the launcher
+# command nemoclaw-start runs) and `openclaw-gateway` (the older re-execed
+# binary form). Recent OpenClaw (v0.0.44 / 2026.5.18+) re-execs the
+# long-running gateway into a process whose argv is plain `openclaw` with
+# no `gateway` token at all (#4952), which that pattern cannot see — so on a
+# marker-present container whose in-container curl probe failed, the stale
+# pattern reported a live gateway as permanently unhealthy.
+#
+# When the pattern misses, fall back to the gateway PID that nemoclaw-start
+# recorded in /tmp/nemoclaw-gateway.pid (record_gateway_pid, written for both
+# the root and non-root launch paths and refreshed on every respawn) and
+# confirm THAT pid is still a live `openclaw` process. This deliberately does
+# not match any process merely named `openclaw`: a bare `pgrep -x openclaw`
+# would keep Docker healthy when the real gateway has died but an unrelated
+# `openclaw` one-shot (e.g. `openclaw agent ...`) happens to be running,
+# defeating restart/self-healing. The recorded-pid check is gateway-specific
+# and survives PID reuse via the comm prefix guard. `ps -o comm=` reads the
+# (15-char) process name, which is `openclaw` for the re-execed gateway and
+# `openclaw-gatewa(y)` for the legacy form — both match `openclaw*`.
 #
 # pgrep uses --ignore-ancestors so it cannot self-match the healthcheck
 # shell that Docker spawns to run this CMD — that shell's argv contains
@@ -1082,7 +1097,11 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=45s --retries=3 \
         if [ "$rc" = 0 ]; then exit 0; fi; \
         if [ "$rc" != 7 ]; then exit 1; fi; \
         [ -f /tmp/nemoclaw-gateway-local ] || exit 0; \
-        pgrep --ignore-ancestors -f 'openclaw[ -]gateway' > /dev/null 2>&1 || exit 1; \
+        if ! pgrep --ignore-ancestors -f 'openclaw[ -]gateway' > /dev/null 2>&1; then \
+            gwpid="$(cat /tmp/nemoclaw-gateway.pid 2>/dev/null)"; \
+            case "${gwpid:-x}" in *[!0-9]*) exit 1 ;; esac; \
+            case "$(ps -p "$gwpid" -o comm= 2>/dev/null)" in openclaw*) ;; *) exit 1 ;; esac; \
+        fi; \
         [ -s /tmp/gateway.log ]
 
 # Entrypoint runs as root to start the gateway as the gateway user,

scripts/nemoclaw-start.sh

@@ -218,6 +218,16 @@ mark_in_container_gateway() {
   : >/tmp/nemoclaw-gateway-local 2>/dev/null || true
 }
 
+# Record the PID of the live in-container gateway so the Docker HEALTHCHECK
+# can confirm the actual gateway process (not merely *some* `openclaw`
+# process) is still alive when the in-container curl probe cannot reach the
+# dashboard port (#4952). Refreshed on every (re)launch so a respawned gateway
+# is tracked and a window where the gateway is down reads as unhealthy.
+# Best-effort: a write failure must never block startup.
+record_gateway_pid() {
+  printf '%s\n' "${1:-}" >/tmp/nemoclaw-gateway.pid 2>/dev/null || true
+}
+
 _chat_ui_url_port() {
   [ -n "${CHAT_UI_URL:-}" ] || return 1
   python3 - "$CHAT_UI_URL" <<'PYPORT'
@@ -3434,6 +3444,7 @@ if [ "$(id -u)" -ne 0 ]; then
   mark_in_container_gateway
   nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 &
   GATEWAY_PID=$!
+  record_gateway_pid "$GATEWAY_PID"
   echo "[gateway] openclaw gateway launched (pid $GATEWAY_PID)" >&2
   # Diagnostic: mirror gateway log to PID 1's stderr — see root-mode block
   # below for rationale (NVIDIA/NemoClaw#2484).
@@ -3487,6 +3498,7 @@ if [ "$(id -u)" -ne 0 ]; then
     sleep 2
     nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 &
     GATEWAY_PID=$!
+    record_gateway_pid "$GATEWAY_PID"
     # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
     SANDBOX_WAIT_PID="$GATEWAY_PID"
     SANDBOX_CHILD_PIDS+=("$GATEWAY_PID")
@@ -3659,6 +3671,7 @@ validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON
 mark_in_container_gateway
 nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 &
 GATEWAY_PID=$!
+record_gateway_pid "$GATEWAY_PID"
 echo "[gateway] openclaw gateway launched as 'gateway' user (pid $GATEWAY_PID)" >&2
 
 # Diagnostic: mirror gateway log to PID 1's stderr so its content surfaces in
@@ -3744,6 +3757,7 @@ while :; do
   sleep 2
   nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 &
   GATEWAY_PID=$!
+  record_gateway_pid "$GATEWAY_PID"
   # shellcheck disable=SC2034  # read by cleanup_on_signal from sandbox-init.sh
   SANDBOX_WAIT_PID="$GATEWAY_PID"
   SANDBOX_CHILD_PIDS+=("$GATEWAY_PID")

test/gateway-pid-recording.test.ts

@@ -0,0 +1,63 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Producer-side coverage for the #4952 HEALTHCHECK fix: nemoclaw-start must
+// record the live gateway PID in /tmp/nemoclaw-gateway.pid so the Docker
+// HEALTHCHECK can confirm the actual gateway process is alive (not merely some
+// process named `openclaw`) when the in-container curl probe cannot reach the
+// dashboard port. The consumer side (the HEALTHCHECK reading this file) is
+// covered in test/sandbox-provisioning.test.ts.
+
+import { spawnSync } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+
+const START_SCRIPT = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh");
+
+/** Slice a brace-balanced (no nested braces) shell function out of the source. */
+function extractFunction(src: string, name: string): string {
+  const start = src.indexOf(`${name}() {`);
+  if (start === -1) throw new Error(`Expected ${name} in scripts/nemoclaw-start.sh`);
+  const end = src.indexOf("\n}", start);
+  if (end === -1) throw new Error(`Expected closing brace for ${name}`);
+  return src.slice(start, end + 2);
+}
+
+describe("nemoclaw-start gateway PID recording for HEALTHCHECK (#4952)", () => {
+  it("record_gateway_pid writes the gateway PID to the file the HEALTHCHECK reads", () => {
+    const src = fs.readFileSync(START_SCRIPT, "utf-8");
+    const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-pid-"));
+    try {
+      const pidPath = path.join(tmp, "nemoclaw-gateway.pid");
+      const fn = extractFunction(src, "record_gateway_pid").replaceAll(
+        "/tmp/nemoclaw-gateway.pid",
+        pidPath,
+      );
+      const script = ["set -euo pipefail", fn, 'record_gateway_pid "12345"'].join("\n");
+
+      const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 });
+
+      expect(result.status).toBe(0);
+      expect(fs.readFileSync(pidPath, "utf-8").trim()).toBe("12345");
+    } finally {
+      fs.rmSync(tmp, { recursive: true, force: true });
+    }
+  });
+
+  it("never fails startup when the PID file cannot be written (best-effort)", () => {
+    // The writer must swallow errors: a failed write must never abort the
+    // entrypoint. Point it at an unwritable path and assert success.
+    const src = fs.readFileSync(START_SCRIPT, "utf-8");
+    const fn = extractFunction(src, "record_gateway_pid").replaceAll(
+      "/tmp/nemoclaw-gateway.pid",
+      "/nonexistent-dir/nemoclaw-gateway.pid",
+    );
+    const script = ["set -euo pipefail", fn, 'record_gateway_pid "12345"'].join("\n");
+
+    const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 });
+
+    expect(result.status).toBe(0);
+  });
+});

test/nemoclaw-start.test.ts

@@ -2309,7 +2309,7 @@ describe("nemoclaw-start gateway launch signal handling", () => {
         "start_persistent_gateway_log_mirror() { sleep 30 & GATEWAY_LOG_PERSIST_PID=$!; }",
         "start_auto_pair() { sleep 30 & AUTO_PAIR_PID=$!; }",
         "start_plugin_registry_refresh() { :; }",
-        "cleanup_on_signal() { :; }",
+        "cleanup_on_signal() { :; }; record_gateway_pid() { :; }", // record_gateway_pid: #4952
         extractShellFunctionFromSource(src, "mark_in_container_gateway").replaceAll(
           "/tmp/nemoclaw-gateway-local",
           markerPath,

test/sandbox-provisioning.test.ts

@@ -544,6 +544,193 @@ describe("sandbox provisioning: image health checks (#1430)", () => {
         expect(probe.calls).not.toContain("pgrep");
       });
     });
+
+    // #4952: recent OpenClaw (v0.0.44 / 2026.5.18+) re-execs the long-running
+    // gateway into a process whose argv is plain `openclaw` — no `gateway`
+    // token at all (see the gateway_pid() helper in
+    // test/e2e/test-issue-2478-crash-loop-recovery.sh). The in-container curl
+    // probe fails (connection refused, exit 7) on runtime shapes where the
+    // dashboard port lives outside this namespace, so the healthcheck falls
+    // back to the in-container gateway-liveness check. A pgrep that only
+    // matches `openclaw[ -]gateway` cannot see the re-execed plain-`openclaw`
+    // process, so the marker-present container is reported permanently
+    // unhealthy even though the gateway is alive and serving.
+    //
+    // The fallback must stay gateway-specific: matching *any* process named
+    // `openclaw` would keep Docker green when the real gateway has died but an
+    // unrelated `openclaw` one-shot (e.g. `openclaw agent ...`) is running,
+    // defeating restart/self-healing. So nemoclaw-start records the live
+    // gateway PID in /tmp/nemoclaw-gateway.pid and the fallback confirms THAT
+    // pid is still a live `openclaw` process.
+    //
+    // Unlike runProductionHealthProbe above (which forces pgrep's exit code
+    // and therefore can never exercise the pattern), this drives a pgrep mock
+    // that matches its pattern against a simulated process table and a ps mock
+    // that resolves the recorded PID — so the probe's outcome depends on the
+    // real argv shape AND on whether the recorded gateway PID is alive.
+    describe("matches the re-execed plain-`openclaw` gateway argv (#4952)", () => {
+      // procTable entries are `comm|args`: `pgrep -f PAT` matches PAT (ERE)
+      // against args; bare `pgrep PAT` matches PAT (ERE) against comm.
+      // psTable maps a recorded PID to the `comm` that `ps -p <pid> -o comm=`
+      // returns (an absent PID models a dead/reused process: empty output).
+      function runHealthProbe({
+        procTable = [],
+        gatewayPid = null,
+        psTable = {},
+        curlExit = 7,
+        gatewayLog = "gateway log line\n",
+      }: {
+        procTable?: string[];
+        gatewayPid?: string | null;
+        psTable?: Record<string, string>;
+        curlExit?: number;
+        gatewayLog?: string;
+      }) {
+        const dockerfile = fs.readFileSync(DOCKERFILE, "utf-8");
+        const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-health-argv-"));
+        const logPath = path.join(tmp, "gateway.log");
+        const markerPath = path.join(tmp, "nemoclaw-gateway-local");
+        const pidPath = path.join(tmp, "nemoclaw-gateway.pid");
+        const command = dockerHealthCommandBetween(
+          dockerfile,
+          "# Health check: poll the gateway's /health endpoint",
+          "# Entrypoint runs as root",
+        )
+          .replaceAll("/tmp/gateway.log", logPath)
+          .replaceAll("/tmp/nemoclaw-gateway-local", markerPath)
+          .replaceAll("/tmp/nemoclaw-gateway.pid", pidPath);
+
+        // Gateway is up and the marker is present: this container runs the
+        // in-container gateway, so the liveness fallback is meaningful.
+        if (gatewayLog !== "") {
+          fs.writeFileSync(logPath, gatewayLog);
+        }
+        fs.writeFileSync(markerPath, "");
+        if (gatewayPid !== null) {
+          fs.writeFileSync(pidPath, `${gatewayPid}\n`);
+        }
+
+        const pgrepMock = [
+          "pgrep() {",
+          '  printf "pgrep %s\\n" "$*" >> "$call_log";',
+          "  local use_f=0 exact=0 pat='';",
+          '  for a in "$@"; do',
+          '    case "$a" in',
+          "      --ignore-ancestors) ;;",
+          "      -f) use_f=1 ;;",
+          "      -x) exact=1 ;;",
+          "      -*) ;;",
+          '      *) pat="$a" ;;',
+          "    esac;",
+          "  done;",
+          '  local found=1 oldifs="$IFS" line comm args;',
+          "  IFS=$'\\n';",
+          "  for line in $FAKE_PROCS; do",
+          '    [ -n "$line" ] || continue;',
+          '    comm="${line%%|*}"; args="${line#*|}";',
+          '    if [ "$use_f" = 1 ]; then',
+          '      printf "%s" "$args" | grep -Eq "$pat" && { found=0; break; };',
+          '    elif [ "$exact" = 1 ]; then',
+          '      [ "$comm" = "$pat" ] && { found=0; break; };',
+          "    else",
+          '      printf "%s" "$comm" | grep -Eq "$pat" && { found=0; break; };',
+          "    fi;",
+          "  done;",
+          '  IFS="$oldifs"; return $found;',
+          "}",
+        ].join("\n");
+
+        // `ps -p <pid> -o comm=` → the recorded process name, empty when the
+        // PID is not in the table (dead/reused).
+        const psMock = [
+          "ps() {",
+          '  printf "ps %s\\n" "$*" >> "$call_log";',
+          '  local pid="" prev="";',
+          '  for a in "$@"; do [ "$prev" = "-p" ] && pid="$a"; prev="$a"; done;',
+          "  local line oldifs=\"$IFS\"; IFS=$'\\n';",
+          "  for line in $PS_TABLE; do",
+          '    [ -n "$line" ] || continue;',
+          '    if [ "${line%%=*}" = "$pid" ]; then printf "%s\\n" "${line#*=}"; IFS="$oldifs"; return 0; fi;',
+          "  done;",
+          '  IFS="$oldifs"; return 1;',
+          "}",
+        ].join("\n");
+
+        const psEnv = Object.entries(psTable)
+          .map(([pid, comm]) => `${pid}=${comm}`)
+          .join("\n");
+
+        try {
+          return runLoggedDockerShell(
+            command,
+            tmp,
+            [
+              `curl() { printf "curl %s\\n" "$*" >> "$call_log"; return ${curlExit}; }`,
+              pgrepMock,
+              psMock,
+            ],
+            { FAKE_PROCS: procTable.join("\n"), PS_TABLE: psEnv },
+          );
+        } finally {
+          fs.rmSync(tmp, { recursive: true, force: true });
+        }
+      }
+
+      it("reports healthy when the live gateway re-execed to a plain `openclaw` argv", () => {
+        // The gateway carries no `gateway` token in its argv (the #4952
+        // shape); liveness is proven by the recorded PID resolving to an
+        // `openclaw` process.
+        const probe = runHealthProbe({
+          procTable: ["openclaw|openclaw"],
+          gatewayPid: "4242",
+          psTable: { "4242": "openclaw" },
+        });
+        expect(probe.result.status).toBe(0);
+      });
+
+      it("still reports healthy for the launcher-form `openclaw gateway run` argv", () => {
+        // pgrep matches the gateway-token form directly; no PID lookup needed.
+        const probe = runHealthProbe({ procTable: ["openclaw|openclaw gateway run --port 18789"] });
+        expect(probe.result.status).toBe(0);
+      });
+
+      it("still reports healthy for the legacy re-execed `openclaw-gateway` argv", () => {
+        const probe = runHealthProbe({
+          procTable: ["openclaw-gateway|openclaw-gateway --port 18789"],
+        });
+        expect(probe.result.status).toBe(0);
+      });
+
+      it("reports unhealthy when no openclaw process is alive and no gateway PID was recorded", () => {
+        const probe = runHealthProbe({
+          procTable: ["bash|bash /usr/local/bin/nemoclaw-start"],
+          gatewayPid: null,
+        });
+        expect(probe.result.status).toBe(1);
+      });
+
+      // The tightening that closes the self-healing gap: an unrelated
+      // `openclaw` one-shot is running (a bare `pgrep -x openclaw` would have
+      // matched it and falsely reported healthy), but the recorded gateway PID
+      // is dead. The container must report unhealthy so Docker restarts it.
+      it("reports unhealthy when the recorded gateway PID is dead even if a non-gateway `openclaw` process exists", () => {
+        const probe = runHealthProbe({
+          procTable: ["openclaw|openclaw agent run-task"],
+          gatewayPid: "9999",
+          psTable: {}, // 9999 is gone
+        });
+        expect(probe.result.status).toBe(1);
+      });
+
+      it("reports unhealthy when the recorded gateway PID was reused by a non-openclaw process", () => {
+        const probe = runHealthProbe({
+          procTable: ["bash|bash"],
+          gatewayPid: "4242",
+          psTable: { "4242": "bash" }, // PID reuse
+        });
+        expect(probe.result.status).toBe(1);
+      });
+    });
   });
 
   it.each([