diff --git a/Dockerfile b/Dockerfile index aa9fdf4d26c..294105608f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1061,10 +1061,25 @@ RUN set -eu; \ # host-side delivery-chain monitoring (verify-deployment.ts, host # port forward, sandbox status). # -# The process pattern matches both `openclaw gateway run` (the launcher -# command nemoclaw-start runs) and `openclaw-gateway` (the re-execed -# binary form OpenClaw switches into after startup). This is the same -# variant set the host-side gateway-stop script in services.ts matches. +# The pgrep pattern matches both `openclaw gateway run` (the launcher +# command nemoclaw-start runs) and `openclaw-gateway` (the older re-execed +# binary form). Recent OpenClaw (v0.0.44 / 2026.5.18+) re-execs the +# long-running gateway into a process whose argv is plain `openclaw` with +# no `gateway` token at all (#4952), which that pattern cannot see — so on a +# marker-present container whose in-container curl probe failed, the stale +# pattern reported a live gateway as permanently unhealthy. +# +# When the pattern misses, fall back to the gateway PID that nemoclaw-start +# recorded in /tmp/nemoclaw-gateway.pid (record_gateway_pid, written for both +# the root and non-root launch paths and refreshed on every respawn) and +# confirm THAT pid is still a live `openclaw` process. This deliberately does +# not match any process merely named `openclaw`: a bare `pgrep -x openclaw` +# would keep Docker healthy when the real gateway has died but an unrelated +# `openclaw` one-shot (e.g. `openclaw agent ...`) happens to be running, +# defeating restart/self-healing. The recorded-pid check is gateway-specific +# and survives PID reuse via the comm prefix guard. `ps -o comm=` reads the +# (15-char) process name, which is `openclaw` for the re-execed gateway and +# `openclaw-gatewa(y)` for the legacy form — both match `openclaw*`. # # pgrep uses --ignore-ancestors so it cannot self-match the healthcheck # shell that Docker spawns to run this CMD — that shell's argv contains @@ -1082,7 +1097,11 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=45s --retries=3 \ if [ "$rc" = 0 ]; then exit 0; fi; \ if [ "$rc" != 7 ]; then exit 1; fi; \ [ -f /tmp/nemoclaw-gateway-local ] || exit 0; \ - pgrep --ignore-ancestors -f 'openclaw[ -]gateway' > /dev/null 2>&1 || exit 1; \ + if ! pgrep --ignore-ancestors -f 'openclaw[ -]gateway' > /dev/null 2>&1; then \ + gwpid="$(cat /tmp/nemoclaw-gateway.pid 2>/dev/null)"; \ + case "${gwpid:-x}" in *[!0-9]*) exit 1 ;; esac; \ + case "$(ps -p "$gwpid" -o comm= 2>/dev/null)" in openclaw*) ;; *) exit 1 ;; esac; \ + fi; \ [ -s /tmp/gateway.log ] # Entrypoint runs as root to start the gateway as the gateway user, diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index d3dce6c487e..17ac1fb05b1 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -218,6 +218,16 @@ mark_in_container_gateway() { : >/tmp/nemoclaw-gateway-local 2>/dev/null || true } +# Record the PID of the live in-container gateway so the Docker HEALTHCHECK +# can confirm the actual gateway process (not merely *some* `openclaw` +# process) is still alive when the in-container curl probe cannot reach the +# dashboard port (#4952). Refreshed on every (re)launch so a respawned gateway +# is tracked and a window where the gateway is down reads as unhealthy. +# Best-effort: a write failure must never block startup. +record_gateway_pid() { + printf '%s\n' "${1:-}" >/tmp/nemoclaw-gateway.pid 2>/dev/null || true +} + _chat_ui_url_port() { [ -n "${CHAT_UI_URL:-}" ] || return 1 python3 - "$CHAT_UI_URL" <<'PYPORT' @@ -3434,6 +3444,7 @@ if [ "$(id -u)" -ne 0 ]; then mark_in_container_gateway nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 & GATEWAY_PID=$! + record_gateway_pid "$GATEWAY_PID" echo "[gateway] openclaw gateway launched (pid $GATEWAY_PID)" >&2 # Diagnostic: mirror gateway log to PID 1's stderr — see root-mode block # below for rationale (NVIDIA/NemoClaw#2484). @@ -3487,6 +3498,7 @@ if [ "$(id -u)" -ne 0 ]; then sleep 2 nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 & GATEWAY_PID=$! + record_gateway_pid "$GATEWAY_PID" # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" SANDBOX_CHILD_PIDS+=("$GATEWAY_PID") @@ -3659,6 +3671,7 @@ validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON mark_in_container_gateway nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 & GATEWAY_PID=$! +record_gateway_pid "$GATEWAY_PID" echo "[gateway] openclaw gateway launched as 'gateway' user (pid $GATEWAY_PID)" >&2 # Diagnostic: mirror gateway log to PID 1's stderr so its content surfaces in @@ -3744,6 +3757,7 @@ while :; do sleep 2 nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >>/tmp/gateway.log 2>&1 & GATEWAY_PID=$! + record_gateway_pid "$GATEWAY_PID" # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" SANDBOX_CHILD_PIDS+=("$GATEWAY_PID") diff --git a/test/gateway-pid-recording.test.ts b/test/gateway-pid-recording.test.ts new file mode 100644 index 00000000000..fbdd3066600 --- /dev/null +++ b/test/gateway-pid-recording.test.ts @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Producer-side coverage for the #4952 HEALTHCHECK fix: nemoclaw-start must +// record the live gateway PID in /tmp/nemoclaw-gateway.pid so the Docker +// HEALTHCHECK can confirm the actual gateway process is alive (not merely some +// process named `openclaw`) when the in-container curl probe cannot reach the +// dashboard port. The consumer side (the HEALTHCHECK reading this file) is +// covered in test/sandbox-provisioning.test.ts. + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const START_SCRIPT = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh"); + +/** Slice a brace-balanced (no nested braces) shell function out of the source. */ +function extractFunction(src: string, name: string): string { + const start = src.indexOf(`${name}() {`); + if (start === -1) throw new Error(`Expected ${name} in scripts/nemoclaw-start.sh`); + const end = src.indexOf("\n}", start); + if (end === -1) throw new Error(`Expected closing brace for ${name}`); + return src.slice(start, end + 2); +} + +describe("nemoclaw-start gateway PID recording for HEALTHCHECK (#4952)", () => { + it("record_gateway_pid writes the gateway PID to the file the HEALTHCHECK reads", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-pid-")); + try { + const pidPath = path.join(tmp, "nemoclaw-gateway.pid"); + const fn = extractFunction(src, "record_gateway_pid").replaceAll( + "/tmp/nemoclaw-gateway.pid", + pidPath, + ); + const script = ["set -euo pipefail", fn, 'record_gateway_pid "12345"'].join("\n"); + + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + + expect(result.status).toBe(0); + expect(fs.readFileSync(pidPath, "utf-8").trim()).toBe("12345"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("never fails startup when the PID file cannot be written (best-effort)", () => { + // The writer must swallow errors: a failed write must never abort the + // entrypoint. Point it at an unwritable path and assert success. + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const fn = extractFunction(src, "record_gateway_pid").replaceAll( + "/tmp/nemoclaw-gateway.pid", + "/nonexistent-dir/nemoclaw-gateway.pid", + ); + const script = ["set -euo pipefail", fn, 'record_gateway_pid "12345"'].join("\n"); + + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + + expect(result.status).toBe(0); + }); +}); diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index 10244304836..c2a7a293152 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -2309,7 +2309,7 @@ describe("nemoclaw-start gateway launch signal handling", () => { "start_persistent_gateway_log_mirror() { sleep 30 & GATEWAY_LOG_PERSIST_PID=$!; }", "start_auto_pair() { sleep 30 & AUTO_PAIR_PID=$!; }", "start_plugin_registry_refresh() { :; }", - "cleanup_on_signal() { :; }", + "cleanup_on_signal() { :; }; record_gateway_pid() { :; }", // record_gateway_pid: #4952 extractShellFunctionFromSource(src, "mark_in_container_gateway").replaceAll( "/tmp/nemoclaw-gateway-local", markerPath, diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index ff43de853d5..3f0290d9cf8 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -544,6 +544,193 @@ describe("sandbox provisioning: image health checks (#1430)", () => { expect(probe.calls).not.toContain("pgrep"); }); }); + + // #4952: recent OpenClaw (v0.0.44 / 2026.5.18+) re-execs the long-running + // gateway into a process whose argv is plain `openclaw` — no `gateway` + // token at all (see the gateway_pid() helper in + // test/e2e/test-issue-2478-crash-loop-recovery.sh). The in-container curl + // probe fails (connection refused, exit 7) on runtime shapes where the + // dashboard port lives outside this namespace, so the healthcheck falls + // back to the in-container gateway-liveness check. A pgrep that only + // matches `openclaw[ -]gateway` cannot see the re-execed plain-`openclaw` + // process, so the marker-present container is reported permanently + // unhealthy even though the gateway is alive and serving. + // + // The fallback must stay gateway-specific: matching *any* process named + // `openclaw` would keep Docker green when the real gateway has died but an + // unrelated `openclaw` one-shot (e.g. `openclaw agent ...`) is running, + // defeating restart/self-healing. So nemoclaw-start records the live + // gateway PID in /tmp/nemoclaw-gateway.pid and the fallback confirms THAT + // pid is still a live `openclaw` process. + // + // Unlike runProductionHealthProbe above (which forces pgrep's exit code + // and therefore can never exercise the pattern), this drives a pgrep mock + // that matches its pattern against a simulated process table and a ps mock + // that resolves the recorded PID — so the probe's outcome depends on the + // real argv shape AND on whether the recorded gateway PID is alive. + describe("matches the re-execed plain-`openclaw` gateway argv (#4952)", () => { + // procTable entries are `comm|args`: `pgrep -f PAT` matches PAT (ERE) + // against args; bare `pgrep PAT` matches PAT (ERE) against comm. + // psTable maps a recorded PID to the `comm` that `ps -p -o comm=` + // returns (an absent PID models a dead/reused process: empty output). + function runHealthProbe({ + procTable = [], + gatewayPid = null, + psTable = {}, + curlExit = 7, + gatewayLog = "gateway log line\n", + }: { + procTable?: string[]; + gatewayPid?: string | null; + psTable?: Record; + curlExit?: number; + gatewayLog?: string; + }) { + const dockerfile = fs.readFileSync(DOCKERFILE, "utf-8"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-health-argv-")); + const logPath = path.join(tmp, "gateway.log"); + const markerPath = path.join(tmp, "nemoclaw-gateway-local"); + const pidPath = path.join(tmp, "nemoclaw-gateway.pid"); + const command = dockerHealthCommandBetween( + dockerfile, + "# Health check: poll the gateway's /health endpoint", + "# Entrypoint runs as root", + ) + .replaceAll("/tmp/gateway.log", logPath) + .replaceAll("/tmp/nemoclaw-gateway-local", markerPath) + .replaceAll("/tmp/nemoclaw-gateway.pid", pidPath); + + // Gateway is up and the marker is present: this container runs the + // in-container gateway, so the liveness fallback is meaningful. + if (gatewayLog !== "") { + fs.writeFileSync(logPath, gatewayLog); + } + fs.writeFileSync(markerPath, ""); + if (gatewayPid !== null) { + fs.writeFileSync(pidPath, `${gatewayPid}\n`); + } + + const pgrepMock = [ + "pgrep() {", + ' printf "pgrep %s\\n" "$*" >> "$call_log";', + " local use_f=0 exact=0 pat='';", + ' for a in "$@"; do', + ' case "$a" in', + " --ignore-ancestors) ;;", + " -f) use_f=1 ;;", + " -x) exact=1 ;;", + " -*) ;;", + ' *) pat="$a" ;;', + " esac;", + " done;", + ' local found=1 oldifs="$IFS" line comm args;', + " IFS=$'\\n';", + " for line in $FAKE_PROCS; do", + ' [ -n "$line" ] || continue;', + ' comm="${line%%|*}"; args="${line#*|}";', + ' if [ "$use_f" = 1 ]; then', + ' printf "%s" "$args" | grep -Eq "$pat" && { found=0; break; };', + ' elif [ "$exact" = 1 ]; then', + ' [ "$comm" = "$pat" ] && { found=0; break; };', + " else", + ' printf "%s" "$comm" | grep -Eq "$pat" && { found=0; break; };', + " fi;", + " done;", + ' IFS="$oldifs"; return $found;', + "}", + ].join("\n"); + + // `ps -p -o comm=` → the recorded process name, empty when the + // PID is not in the table (dead/reused). + const psMock = [ + "ps() {", + ' printf "ps %s\\n" "$*" >> "$call_log";', + ' local pid="" prev="";', + ' for a in "$@"; do [ "$prev" = "-p" ] && pid="$a"; prev="$a"; done;', + " local line oldifs=\"$IFS\"; IFS=$'\\n';", + " for line in $PS_TABLE; do", + ' [ -n "$line" ] || continue;', + ' if [ "${line%%=*}" = "$pid" ]; then printf "%s\\n" "${line#*=}"; IFS="$oldifs"; return 0; fi;', + " done;", + ' IFS="$oldifs"; return 1;', + "}", + ].join("\n"); + + const psEnv = Object.entries(psTable) + .map(([pid, comm]) => `${pid}=${comm}`) + .join("\n"); + + try { + return runLoggedDockerShell( + command, + tmp, + [ + `curl() { printf "curl %s\\n" "$*" >> "$call_log"; return ${curlExit}; }`, + pgrepMock, + psMock, + ], + { FAKE_PROCS: procTable.join("\n"), PS_TABLE: psEnv }, + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + } + + it("reports healthy when the live gateway re-execed to a plain `openclaw` argv", () => { + // The gateway carries no `gateway` token in its argv (the #4952 + // shape); liveness is proven by the recorded PID resolving to an + // `openclaw` process. + const probe = runHealthProbe({ + procTable: ["openclaw|openclaw"], + gatewayPid: "4242", + psTable: { "4242": "openclaw" }, + }); + expect(probe.result.status).toBe(0); + }); + + it("still reports healthy for the launcher-form `openclaw gateway run` argv", () => { + // pgrep matches the gateway-token form directly; no PID lookup needed. + const probe = runHealthProbe({ procTable: ["openclaw|openclaw gateway run --port 18789"] }); + expect(probe.result.status).toBe(0); + }); + + it("still reports healthy for the legacy re-execed `openclaw-gateway` argv", () => { + const probe = runHealthProbe({ + procTable: ["openclaw-gateway|openclaw-gateway --port 18789"], + }); + expect(probe.result.status).toBe(0); + }); + + it("reports unhealthy when no openclaw process is alive and no gateway PID was recorded", () => { + const probe = runHealthProbe({ + procTable: ["bash|bash /usr/local/bin/nemoclaw-start"], + gatewayPid: null, + }); + expect(probe.result.status).toBe(1); + }); + + // The tightening that closes the self-healing gap: an unrelated + // `openclaw` one-shot is running (a bare `pgrep -x openclaw` would have + // matched it and falsely reported healthy), but the recorded gateway PID + // is dead. The container must report unhealthy so Docker restarts it. + it("reports unhealthy when the recorded gateway PID is dead even if a non-gateway `openclaw` process exists", () => { + const probe = runHealthProbe({ + procTable: ["openclaw|openclaw agent run-task"], + gatewayPid: "9999", + psTable: {}, // 9999 is gone + }); + expect(probe.result.status).toBe(1); + }); + + it("reports unhealthy when the recorded gateway PID was reused by a non-openclaw process", () => { + const probe = runHealthProbe({ + procTable: ["bash|bash"], + gatewayPid: "4242", + psTable: { "4242": "bash" }, // PID reuse + }); + expect(probe.result.status).toBe(1); + }); + }); }); it.each([