diff --git a/docs/reference/architecture.mdx b/docs/reference/architecture.mdx index 515686c5a0..1ba1e54509 100644 --- a/docs/reference/architecture.mdx +++ b/docs/reference/architecture.mdx @@ -87,7 +87,8 @@ Per-user units, partial units, and user-manager or bus outages do not take over That compatibility fallback remains until supported upgrade paths no longer include pre-service OpenShell installs and the package-managed handoff has direct nightly coverage. On Apple Silicon macOS, NemoClaw starts the OpenShell Docker-driver gateway and creates the sandbox as a Docker container. In both Docker-driver modes, the sandbox is a Docker container, not a Kubernetes pod. -The in-container `/tmp/nemoclaw-gateway-local` marker is written only by entrypoint paths that actually launch an in-container gateway. +Entrypoint supervisors create the in-container `/tmp/nemoclaw-gateway-local` marker only when they actually launch an in-container gateway, and they normally keep it present while that supervisor is active. +On normal exits, handled `SIGTERM`/`SIGINT`, startup failures, and shell `errexit` termination through the `EXIT` trap, the supervisor removes the marker on a best-effort basis so the Docker health check does not keep trusting a stale gateway PID. Terminal runtimes may not write it. NemoClaw does not treat sandbox environment hints such as `OPENSHELL_DRIVERS` as authoritative for gateway ownership. Legacy non-Docker-driver installs still use the k3s-based gateway path. diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index 9fdb9d53b7..72fa2a46d1 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -350,6 +350,22 @@ mark_in_container_gateway() { _nemoclaw_safe_create_tmp_file /tmp/nemoclaw-gateway-local 600 "" best-effort 2>/dev/null || true } +# Drop the in-container gateway marker (#4952). The HEALTHCHECK's pidfile +# fallback trusts /tmp/nemoclaw-gateway.pid, which is refreshed *only* by +# record_gateway_pid inside this supervisor's launch/respawn paths. On +# OpenShell docker-driver sandboxes this script is NOT PID 1 -- OpenShell's +# `sleep infinity` keeps the container alive as a sibling -- so when the +# supervise loop exits, the container lives on but nothing refreshes the +# pidfile. The marker would otherwise stay in place, leaving the healthcheck +# trusting a stale PID forever (permanent false `unhealthy`). Tying marker +# removal to supervisor exit completes the #4710 marker semantics: the marker +# means "a supervisor is actively managing the gateway and keeping the pidfile +# fresh". Once it is gone the healthcheck takes the marker-absent -> healthy +# branch (#4503) instead. Best-effort: failure must never block teardown. +clear_in_container_gateway_marker() { + rm -f /tmp/nemoclaw-gateway-local 2>/dev/null || true +} + # Record the PID/starttime identity of the live in-container gateway so the # Docker HEALTHCHECK # can confirm the actual gateway process (not merely *some* `openclaw` @@ -4049,7 +4065,9 @@ GUARDENVEOF # cleanup_on_signal is provided by sandbox-init.sh. It reads # SANDBOX_CHILD_PIDS (array of all PIDs) and SANDBOX_WAIT_PID (the # primary process whose exit status is returned). -# Each code path below sets these before registering the trap. +# Each code path arms the trap before launching the gateway. These values are +# populated as children start; cleanup refreshes and validates them before +# signaling anything. # Keep per-user rc files out of runtime proxy wiring. Older images and prior # entrypoint versions wrote a two-line shim into .bashrc/.profile; remove that @@ -4872,7 +4890,24 @@ wait_for_openclaw_gateway_internal() { return 1 } +arm_openclaw_gateway_supervisor_cleanup() { + # Bash does not run an EXIT trap when an untrapped SIGTERM/SIGINT terminates + # the shell, so both traps must be live before the marker is written. + trap cleanup_openclaw_on_signal SIGTERM SIGINT + trap clear_in_container_gateway_marker EXIT +} + launch_openclaw_gateway() { + # Drop the gateway marker whenever this supervisor exits -- clean gateway + # exit (`exit 0` below), a forwarded signal (cleanup_openclaw_on_signal ends + # in `cleanup_on_signal` -> `exit`), or errexit. This is the #4952 fix: on + # docker-driver sandboxes this script is not PID 1, so it can exit while the + # container lives on; a surviving marker would leave the HEALTHCHECK trusting + # a stale pidfile. Arm this before marking so early launch failures cannot + # leave the marker behind. The marker is re-dropped at each launch + # (mark_in_container_gateway), so the respawn loop -- which never exits the + # script -- keeps it in place. + arm_openclaw_gateway_supervisor_cleanup mark_in_container_gateway nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" sh -c \ 'umask 0007; exec "$@" >>/tmp/gateway.log 2>&1' sh \ @@ -4894,6 +4929,16 @@ launch_openclaw_gateway() { echo "[gateway] openclaw gateway launched as 'gateway' user (pid $GATEWAY_PID)" >&2 } +launch_openclaw_gateway_non_root() { + arm_openclaw_gateway_supervisor_cleanup + mark_in_container_gateway + nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 & + GATEWAY_PID=$! + capture_openclaw_pid_start_identity "$GATEWAY_PID" GATEWAY_PID_START_IDENTITY || exit 1 + record_gateway_pid "$GATEWAY_PID" "$GATEWAY_PID_START_IDENTITY" + echo "[gateway] openclaw gateway launched (pid $GATEWAY_PID)" >&2 +} + openclaw_supervised_aux_pid_is_live() { local pid="$1" local expected_identity="$2" @@ -5492,12 +5537,7 @@ if [ "$(id -u)" -ne 0 ]; then # to healthy — see the mark_in_container_gateway comment near the top of this # file for the #4710 rationale (why the marker is tied to the launch site # rather than an env-var conditional at startup). - mark_in_container_gateway - nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 & - GATEWAY_PID=$! - capture_openclaw_pid_start_identity "$GATEWAY_PID" GATEWAY_PID_START_IDENTITY || exit 1 - record_gateway_pid "$GATEWAY_PID" "$GATEWAY_PID_START_IDENTITY" - echo "[gateway] openclaw gateway launched (pid $GATEWAY_PID)" >&2 + launch_openclaw_gateway_non_root # Diagnostic: mirror gateway log to PID 1's stderr — see root-mode block # below for rationale (NVIDIA/NemoClaw#2484). { tail -n +1 -F /tmp/gateway.log 2>/dev/null | sed -u 's/^/[gateway-log:] /' >&2; } & @@ -5515,7 +5555,6 @@ if [ "$(id -u)" -ne 0 ]; then refresh_openclaw_supervised_child_pids # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" - trap cleanup_openclaw_on_signal SIGTERM SIGINT print_dashboard_urls # Auto-respawn gateway on unexpected death (NVIDIA/NemoClaw#2757). Without @@ -5720,6 +5759,7 @@ validate_nemoclaw_tmp_permissions # Marking, privilege step-down, log redirection, and PID recording are kept in # one reusable launch primitive so PID 1 owns initial start, crash respawn, and # host-requested restart identically. +# The launch primitive arms signal and EXIT cleanup before writing the marker. launch_openclaw_gateway # Diagnostic: mirror gateway log to PID 1's stderr so its content surfaces in @@ -5769,7 +5809,6 @@ start_gateway_serving_watchdog refresh_openclaw_supervised_child_pids # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" -trap cleanup_openclaw_on_signal SIGTERM SIGINT if ! gateway_control_init; then echo "[gateway-control] privileged gateway control unavailable" >&2 fi diff --git a/test/nemoclaw-start-gateway-health.test.ts b/test/nemoclaw-start-gateway-health.test.ts index 488349ec43..9ca9783d26 100644 --- a/test/nemoclaw-start-gateway-health.test.ts +++ b/test/nemoclaw-start-gateway-health.test.ts @@ -47,6 +47,10 @@ function extractShellFunction(src: string, name: string): string { return `${name}() {${body.slice(0, closing?.index ?? 0)}\n}`; } +function gatewayMarkerFunction(src: string, name: string, markerPath: string): string { + return extractShellFunction(src, name).replaceAll("/tmp/nemoclaw-gateway-local", markerPath); +} + function extractGatewayLogAppendFunction(src: string, gatewayLog: string): string { const functionSource = extractShellFunction(src, "append_openclaw_gateway_log_line"); const marker = ' local log_file="/tmp/gateway.log"'; @@ -101,6 +105,7 @@ function watchdogFunctions(gatewayLog: string): string { function rootGatewayLifecycleFunctions(src: string, gatewayLog: string): string { return [ pidIdentityFunctions(src), + extractShellFunction(src, "arm_openclaw_gateway_supervisor_cleanup"), extractShellFunction(src, "launch_openclaw_gateway").replaceAll("/tmp/gateway.log", gatewayLog), extractShellFunction(src, "openclaw_supervised_aux_pid_is_live"), extractShellFunction(src, "stop_openclaw_supervised_gateway"), @@ -111,6 +116,18 @@ function rootGatewayLifecycleFunctions(src: string, gatewayLog: string): string ].join("\n"); } +function gatewayLaunchBlock(src: string, kind: "non-root" | "root", gatewayLog: string): string { + const startMarker = + kind === "non-root" + ? "# Start gateway in background, auto-pair, then wait" + : "# Start the gateway as the 'gateway' user."; + const start = src.indexOf(startMarker); + const end = src.indexOf('SANDBOX_WAIT_PID="$GATEWAY_PID"', start); + expect(start, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe(-1); + expect(end, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe(-1); + return src.slice(start, src.indexOf("\n", end)).replaceAll("/tmp/gateway.log", gatewayLog); +} + // Drive the watchdog end-to-end against a real background process standing in // for the gateway. `curlPlan` is the sequence of curl exit codes the stubbed // probe returns, one per watchdog cycle; the last entry repeats forever. @@ -994,12 +1011,8 @@ describe("healthcheck marker (#4503, #4710)", () => { }); }); -// Behavioral wiring coverage: run the real launch block of each entrypoint -// mode with the real marker/pidfile/watchdog helpers and assert their -// runtime effects. This replaces source-text assertions (banned by -// ci/source-shape-test-budget.json) and locks the #4748 regression -// behaviorally: OPENSHELL_DRIVERS is exported during the run and must have -// no influence on whether the marker is dropped. +// Run both real launch paths with their marker, pidfile, and watchdog helpers. +// This behaviorally covers the driver-env marker regression (#4748). describe("gateway launch wiring (#4710)", () => { it("exits PID 1 without signaling when gateway identity capture fails", () => { const src = fs.readFileSync(START_SCRIPT, "utf-8"); @@ -1021,10 +1034,12 @@ describe("gateway launch wiring (#4710)", () => { "GATEWAY_PID=0", "GATEWAY_PID_START_IDENTITY=", "mark_in_container_gateway() { :; }", + "clear_in_container_gateway_marker() { :; }\ncleanup_openclaw_on_signal() { :; }", "capture_openclaw_pid_start_identity() { return 1; }", 'clear_gateway_pid_record() { printf "clear\\n" >>"$EVENT_LOG"; }', 'kill() { printf "unexpected-kill:%s\\n" "$*" >>"$EVENT_LOG"; }', 'wait() { printf "unexpected-wait:%s\\n" "$*" >>"$EVENT_LOG"; }', + extractShellFunction(src, "arm_openclaw_gateway_supervisor_cleanup"), launch, "launch_openclaw_gateway", ].join("\n"), @@ -1038,20 +1053,6 @@ describe("gateway launch wiring (#4710)", () => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); - function launchBlock(src: string, kind: "non-root" | "root"): string { - const startMarker = - kind === "non-root" - ? "# Start gateway in background, auto-pair, then wait" - : "# Start the gateway as the 'gateway' user."; - const start = src.indexOf(startMarker); - const trap = src.indexOf("trap cleanup_openclaw_on_signal SIGTERM SIGINT", start); - expect(start, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe( - -1, - ); - expect(trap, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe(-1); - return src.slice(start, src.indexOf("\n", trap)); - } - function runLaunchWiring(kind: "non-root" | "root") { const src = fs.readFileSync(START_SCRIPT, "utf-8"); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `nemoclaw-launch-wiring-${kind}-`)); @@ -1074,9 +1075,11 @@ describe("gateway launch wiring (#4710)", () => { const realFunctions = [ safeTmpHelpers(src), - extractShellFunction(src, "mark_in_container_gateway").replaceAll( - "/tmp/nemoclaw-gateway-local", - markerPath, + gatewayMarkerFunction(src, "mark_in_container_gateway", markerPath), + gatewayMarkerFunction(src, "clear_in_container_gateway_marker", markerPath), + extractShellFunction(src, "launch_openclaw_gateway_non_root").replaceAll( + "/tmp/gateway.log", + gatewayLog, ), extractShellFunction(src, "record_gateway_pid"), extractShellFunction(src, "gateway_pid_is_openclaw_gateway"), @@ -1106,7 +1109,8 @@ describe("gateway launch wiring (#4710)", () => { "STEP_DOWN_PREFIX_SANDBOX=(gosu sandbox)", "STEP_DOWN_PREFIX_GATEWAY=(gosu gateway)", realFunctions, - launchBlock(src, kind).replaceAll("/tmp/gateway.log", gatewayLog), + gatewayLaunchBlock(src, kind, gatewayLog), + `if [ -f ${JSON.stringify(markerPath)} ]; then printf "MARKER_PRESENT=1\\n"; fi`, `for _ in $(command seq 1 100); do [ -s ${JSON.stringify(openclawLog)} ] && break; command sleep 0.1; done`, 'printf "GATEWAY_PID=%s\\n" "$GATEWAY_PID"', 'printf "WATCHDOG_PID=%s\\n" "${GATEWAY_WATCHDOG_PID:-}"', @@ -1124,20 +1128,31 @@ describe("gateway launch wiring (#4710)", () => { const watchdogPid = stdout.match(/^WATCHDOG_PID=(\d+)$/m)?.[1]; const childPids = (stdout.match(/^CHILD_PIDS=(.+)$/m)?.[1] ?? "").split(/\s+/); const pidFileContent = readFileIfPresent(pidFile)?.trim() ?? null; + const markerPresent = stdout.includes("MARKER_PRESENT=1"); const markerExists = readFileIfPresent(markerPath) !== null; fs.rmSync(tmpDir, { recursive: true, force: true }); - return { result, stdout, gatewayPid, watchdogPid, childPids, pidFileContent, markerExists }; + return { + result, + stdout, + gatewayPid, + watchdogPid, + childPids, + pidFileContent, + markerPresent, + markerExists, + }; } it.each([ "non-root", "root", - ] as const)("%s launch drops the marker, records the gateway PID, and starts the tracked watchdog", (kind) => { + ] as const)("%s launch clears the marker on supervisor exit after recording the gateway PID", (kind) => { const run = runLaunchWiring(kind); expect(run.result.status, `script failed: ${run.result.stderr}`).toBe(0); - // Marker dropped by the launch site, even with OPENSHELL_DRIVERS=docker - // exported — env hints must not gate it (#4748 was a no-op for this). - expect(run.markerExists).toBe(true); + expect(run.markerPresent).toBe(true); + // The supervisor EXIT trap clears the in-container marker when this fixture + // exits, returning healthchecks to the marker-absent branch (#4952). + expect(run.markerExists).toBe(false); // The watchdog reads the gateway PID from the pidfile each cycle. expect(run.gatewayPid).toBeDefined(); expect(run.pidFileContent?.split(" ")[0]).toBe(run.gatewayPid); @@ -1356,27 +1371,11 @@ describe("respawn loop pidfile refresh (#4710)", () => { }); }); -// Launch-path signal handling and child-PID tracking for both entrypoint -// modes. Moved from test/nemoclaw-start.test.ts so the legacy file stays -// under its ratcheted size budget; this file owns gateway-launch coverage. +// Launch-path signal handling and child-PID tracking for both entrypoint modes. +// This file owns gateway launch coverage to keep the legacy test within budget. describe("nemoclaw-start gateway launch signal handling", () => { const src = fs.readFileSync(START_SCRIPT, "utf-8"); - function launchBlock(kind: "non-root" | "root", gatewayLog: string): string { - const startMarker = - kind === "non-root" - ? "# Start gateway in background, auto-pair, then wait" - : "# Start the gateway as the 'gateway' user."; - const start = src.indexOf(startMarker); - const trap = src.indexOf("trap cleanup_openclaw_on_signal SIGTERM SIGINT", start); - expect(start, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe( - -1, - ); - expect(trap, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe(-1); - const lineEnd = src.indexOf("\n", trap); - return src.slice(start, lineEnd).replaceAll("/tmp/gateway.log", gatewayLog); - } - function runLaunchBlock(kind: "non-root" | "root") { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `nemoclaw-launch-${kind}-`)); const fakeBin = path.join(tmpDir, "bin"); @@ -1417,20 +1416,20 @@ describe("nemoclaw-start gateway launch signal handling", () => { "start_plugin_registry_refresh() { :; }", "cleanup_on_signal() { :; }", safeTmpHelpers(src), - extractShellFunction(src, "mark_in_container_gateway").replaceAll( - "/tmp/nemoclaw-gateway-local", - markerPath, - ), - // #4710: the launch block also records the gateway PID for the - // serving watchdog and starts the watchdog alongside the other - // background services. Stub both — watchdog behavior has its own - // suite in test/nemoclaw-start-gateway-health.test.ts. + gatewayMarkerFunction(src, "mark_in_container_gateway", markerPath), + gatewayMarkerFunction(src, "clear_in_container_gateway_marker", markerPath), + // Stub PID recording and the serving watchdog; each has focused tests + // elsewhere in this suite (#4710). "record_gateway_pid() { :; }", 'start_gateway_serving_watchdog() { sleep 30 & GATEWAY_WATCHDOG_PID=$!; capture_openclaw_pid_start_identity "$GATEWAY_WATCHDOG_PID" GATEWAY_WATCHDOG_PID_START_IDENTITY; }', + extractShellFunction(src, "launch_openclaw_gateway_non_root").replaceAll( + "/tmp/gateway.log", + gatewayLog, + ), rootGatewayLifecycleFunctions(src, gatewayLog), "STEP_DOWN_PREFIX_SANDBOX=(gosu sandbox)", "STEP_DOWN_PREFIX_GATEWAY=(gosu gateway)", - launchBlock(kind, gatewayLog), + gatewayLaunchBlock(src, kind, gatewayLog), kind === "root" ? `for _ in ${waitForLaunchLogIterations}; do [ -s ${JSON.stringify(gosuLog)} ] && [ -s ${JSON.stringify(openclawLog)} ] && break; sleep 0.1; done` : `for _ in ${waitForLaunchLogIterations}; do [ -s ${JSON.stringify(openclawLog)} ] && break; sleep 0.1; done`, @@ -1457,7 +1456,7 @@ describe("nemoclaw-start gateway launch signal handling", () => { it("registers child PIDs, redirects gateway output, and traps signals in non-root mode", () => { const { result, openclaw, gateway } = runLaunchBlock("non-root"); - expect(result.status).toBe(0); + expect(result.status, result.stderr).toBe(0); expect(openclaw).toContain("gateway run --port 19000"); expect(openclaw).toContain("marker=present"); expect(openclaw).not.toContain("marker=absent"); diff --git a/test/nemoclaw-start-gateway-marker.test.ts b/test/nemoclaw-start-gateway-marker.test.ts index 2540a5c7f2..e60d253905 100644 --- a/test/nemoclaw-start-gateway-marker.test.ts +++ b/test/nemoclaw-start-gateway-marker.test.ts @@ -261,4 +261,344 @@ describe("nemoclaw-start in-container gateway healthcheck marker (#4503, #4710)" fs.rmSync(tmpDir, { recursive: true, force: true }); } }); + + // #4952: the HEALTHCHECK's pidfile fallback trusts /tmp/nemoclaw-gateway.pid, + // which only this supervisor refreshes. On docker-driver sandboxes the script + // is not PID 1 (OpenShell's `sleep infinity` keeps the container alive), so + // the supervisor can exit while the container lives on. If the marker + // survived that exit, the healthcheck would trust a stale PID forever and + // report a working sandbox as permanently unhealthy. The fix drops the marker + // on every supervisor exit via a `trap clear_in_container_gateway_marker + // EXIT`, so the healthcheck then takes the marker-absent -> healthy branch + // (#4503). The marker is re-dropped at each launch, so the respawn loop (which + // never exits the script) keeps it in place. + + it("clear_in_container_gateway_marker removes the marker and is a no-op when absent (#4952)", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-clear-")); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const clearFn = extractShellFunctionFromSource( + src, + "clear_in_container_gateway_marker", + ).replaceAll("/tmp/nemoclaw-gateway-local", markerPath); + + try { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + clearFn, + // No-op when the marker is absent: must succeed, not error. + "clear_in_container_gateway_marker", + `[ -e ${JSON.stringify(markerPath)} ] && echo UNEXPECTED_PRESENT || echo ABSENT_OK`, + // Now create it and confirm the helper removes it. + `: > ${JSON.stringify(markerPath)}`, + "clear_in_container_gateway_marker", + `[ -e ${JSON.stringify(markerPath)} ] && echo STILL_PRESENT || echo REMOVED`, + ].join("\n"); + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("ABSENT_OK"); + expect(result.stdout).toContain("REMOVED"); + expect(result.stdout).not.toContain("UNEXPECTED_PRESENT"); + expect(result.stdout).not.toContain("STILL_PRESENT"); + expect(fs.existsSync(markerPath)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it.each([ + { label: "root launch helper", launchFunction: "launch_openclaw_gateway" }, + { label: "non-root launch helper", launchFunction: "launch_openclaw_gateway_non_root" }, + ])("clears the marker when $label exits before recording PID identity (#4952)", ({ + launchFunction, + }) => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-early-exit-")); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + + try { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + safeTmpHelpers(src), + extractShellFunctionFromSource(src, "mark_in_container_gateway").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ), + extractShellFunctionFromSource(src, "clear_in_container_gateway_marker").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ), + extractShellFunctionFromSource(src, "arm_openclaw_gateway_supervisor_cleanup"), + extractShellFunctionFromSource(src, launchFunction), + "cleanup_openclaw_on_signal() { exit 143; }", + "STEP_DOWN_PREFIX_GATEWAY=(env)", + "OPENCLAW=/bin/true", + "_DASHBOARD_PORT=18789", + "GATEWAY_PID=0", + "GATEWAY_PID_START_IDENTITY=", + "GATEWAY_PID_FILE=", + "capture_openclaw_pid_start_identity() { return 1; }", + "record_gateway_pid() { :; }", + "clear_gateway_pid_record() { :; }", + launchFunction, + ].join("\n"); + const result = spawnSync("bash", ["-c", script], { + encoding: "utf-8", + timeout: 5000, + }); + expect(result.status).toBe(1); + expect(fs.existsSync(markerPath)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it.each([ + { label: "non-root", signal: "TERM", exitCode: 143 }, + { label: "non-root", signal: "INT", exitCode: 130 }, + { label: "root", signal: "TERM", exitCode: 143 }, + { label: "root", signal: "INT", exitCode: 130 }, + ])("arms $signal cleanup before the $label marker write (#4952)", ({ + label, + signal, + exitCode, + }) => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-early-signal-")); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const clearFn = extractShellFunctionFromSource( + src, + "clear_in_container_gateway_marker", + ).replaceAll("/tmp/nemoclaw-gateway-local", markerPath); + const launchFunction = + label === "root" ? "launch_openclaw_gateway" : "launch_openclaw_gateway_non_root"; + + try { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + clearFn, + extractShellFunctionFromSource(src, "arm_openclaw_gateway_supervisor_cleanup"), + extractShellFunctionFromSource(src, launchFunction), + `cleanup_openclaw_on_signal() { exit ${exitCode}; }`, + `mark_in_container_gateway() { : > ${JSON.stringify(markerPath)}; kill -${signal} $$; }`, + "STEP_DOWN_PREFIX_GATEWAY=(env)", + "OPENCLAW=/bin/true", + "_DASHBOARD_PORT=18789", + launchFunction, + ].join("\n"); + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + expect(result.status, result.stderr).toBe(exitCode); + expect(fs.existsSync(markerPath)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + // Exercises the real exit-trap wiring, not just the helper: registers the + // same `trap clear_in_container_gateway_marker EXIT` the supervisor installs, + // drops the marker via mark_in_container_gateway, then lets the shell reach a + // clean `exit 0`. The marker must be gone once the process exits, which is + // exactly the state that flips the healthcheck back to the marker-absent + // healthy branch. + it("drops the marker when the supervisor reaches a clean exit (#4952)", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-exit-")); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const markFn = extractShellFunctionFromSource(src, "mark_in_container_gateway").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ); + const clearFn = extractShellFunctionFromSource( + src, + "clear_in_container_gateway_marker", + ).replaceAll("/tmp/nemoclaw-gateway-local", markerPath); + const armCleanup = extractShellFunctionFromSource( + src, + "arm_openclaw_gateway_supervisor_cleanup", + ); + + try { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + safeTmpHelpers(src), + markFn, + clearFn, + armCleanup, + "cleanup_openclaw_on_signal() { :; }", + "arm_openclaw_gateway_supervisor_cleanup", + "mark_in_container_gateway", + `[ -e ${JSON.stringify(markerPath)} ] && echo MARKER_PRESENT_BEFORE_EXIT`, + "exit 0", + ].join("\n"); + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + expect(result.status).toBe(0); + // The marker existed while the supervisor was running... + expect(result.stdout).toContain("MARKER_PRESENT_BEFORE_EXIT"); + // ...and is gone the moment it exits. + expect(fs.existsSync(markerPath)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("drops the marker when the supervisor exits through errexit (#4952)", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-errexit-")); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const markFn = extractShellFunctionFromSource(src, "mark_in_container_gateway").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ); + const clearFn = extractShellFunctionFromSource( + src, + "clear_in_container_gateway_marker", + ).replaceAll("/tmp/nemoclaw-gateway-local", markerPath); + const armCleanup = extractShellFunctionFromSource( + src, + "arm_openclaw_gateway_supervisor_cleanup", + ); + + try { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + safeTmpHelpers(src), + markFn, + clearFn, + armCleanup, + "cleanup_openclaw_on_signal() { :; }", + "arm_openclaw_gateway_supervisor_cleanup", + "mark_in_container_gateway", + `[ -e ${JSON.stringify(markerPath)} ] && echo MARKER_PRESENT_BEFORE_ERREXIT`, + "false", + "echo UNREACHABLE", + ].join("\n"); + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + expect(result.status, result.stderr).toBe(1); + expect(result.stdout).toContain("MARKER_PRESENT_BEFORE_ERREXIT"); + expect(result.stdout).not.toContain("UNREACHABLE"); + expect(fs.existsSync(markerPath)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + // Signal path: cleanup_openclaw_on_signal delegates to cleanup_on_signal + // (shared from sandbox-init.sh), which ends in `exit`, so the EXIT trap fires + // for SIGTERM/SIGINT teardown too. The marker must be cleared on a forwarded + // signal, not only on a clean gateway exit. + // Run synchronously: a backgrounded coroutine delivers SIGTERM to the script + // itself while it blocks in `wait`, mirroring the supervise loop being + // signalled. This avoids cross-process timing races. + it("drops the marker when the supervisor is terminated by a signal (#4952)", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-signal-")); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const markFn = extractShellFunctionFromSource(src, "mark_in_container_gateway").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ); + const clearFn = extractShellFunctionFromSource( + src, + "clear_in_container_gateway_marker", + ).replaceAll("/tmp/nemoclaw-gateway-local", markerPath); + const armCleanup = extractShellFunctionFromSource( + src, + "arm_openclaw_gateway_supervisor_cleanup", + ); + + try { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + safeTmpHelpers(src), + markFn, + clearFn, + // Minimal stand-ins for the production signal path: the OpenClaw wrapper + // delegates to the shared cleanup helper, which ends in `exit` and + // triggers the EXIT trap where marker cleanup lives. + "cleanup_on_signal() { exit 143; }", + "cleanup_openclaw_on_signal() { cleanup_on_signal; }", + armCleanup, + "arm_openclaw_gateway_supervisor_cleanup", + "mark_in_container_gateway", + `[ -e ${JSON.stringify(markerPath)} ] && echo MARKER_PRESENT_BEFORE_SIGNAL`, + // Deliver SIGTERM to ourselves while we block in `wait`, the same shape + // as the supervise loop being signalled mid-wait. Background stdio is + // redirected so spawnSync isn't held open by an inherited pipe after we + // exit. + "( sleep 0.2; kill -TERM $$ ) >/dev/null 2>&1 &", + "sleep 5 >/dev/null 2>&1 &", + "BLOCK_PID=$!", + "wait $BLOCK_PID", + ].join("\n"); + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 8000 }); + // The script exits via the SIGTERM trap -> cleanup_openclaw_on_signal -> + // cleanup_on_signal -> exit 143. + expect(result.status).toBe(143); + expect(result.stdout).toContain("MARKER_PRESENT_BEFORE_SIGNAL"); + // The EXIT trap fired on the signal teardown and cleared the marker. + expect(fs.existsSync(markerPath)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, 15000); + + // Restart semantics: while the supervisor is alive and respawning the gateway + // (the script never exits), the marker must stay in place so the #4952 + // pidfile fallback keeps probing the live gateway. Only a supervisor *exit* + // clears it. Here the EXIT trap is armed but the script keeps running, and a + // re-launch re-drops the marker idempotently — the marker is present + // throughout. + it("keeps the marker in place across respawns while the supervisor runs (#4952)", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-respawn-")); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const markFn = extractShellFunctionFromSource(src, "mark_in_container_gateway").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ); + const clearFn = extractShellFunctionFromSource( + src, + "clear_in_container_gateway_marker", + ).replaceAll("/tmp/nemoclaw-gateway-local", markerPath); + const armCleanup = extractShellFunctionFromSource( + src, + "arm_openclaw_gateway_supervisor_cleanup", + ); + + try { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + safeTmpHelpers(src), + markFn, + clearFn, + armCleanup, + "cleanup_openclaw_on_signal() { :; }", + "arm_openclaw_gateway_supervisor_cleanup", + // Initial launch. + "mark_in_container_gateway", + `[ -e ${JSON.stringify(markerPath)} ] && echo AFTER_LAUNCH`, + // Simulate a respawn iteration: the loop body re-marks before relaunch + // and the script does NOT exit between iterations. + "mark_in_container_gateway", + `[ -e ${JSON.stringify(markerPath)} ] && echo AFTER_RESPAWN`, + // The script is still running here — the EXIT trap has not fired. + "exit 0", + ].join("\n"); + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("AFTER_LAUNCH"); + expect(result.stdout).toContain("AFTER_RESPAWN"); + // Once the script finally exits, the trap clears it. + expect(fs.existsSync(markerPath)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); });