From 4f37e0b0933b9ee43a4b12469eb4e337c9e46567 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 1 Jul 2026 16:59:34 +0000 Subject: [PATCH 1/3] fix(sandbox): respawn gateway after watchdog SIGTERM instead of exiting PID 1 The serving watchdog SIGTERMs a gateway that dropped its HTTP listener so the respawn loop can relaunch it, but OpenClaw exits 0 on a graceful SIGTERM. Both respawn loops treat a clean rc=0 exit as an operator-requested shutdown and exit PID 1, tearing down the whole sandbox instead of respawning it. Record an identity-scoped kill marker before the watchdog SIGTERM and consume it in both respawn loops, so a watchdog-induced clean exit respawns while a genuine operator clean exit still stops the sandbox. The marker is scoped to the exact pid and start identity being killed and consumed once, so a stale marker cannot force an unwanted respawn. Co-Authored-By: Claude Signed-off-by: Tinson Lai --- scripts/nemoclaw-start.sh | 21 ++++- test/gateway-watchdog-kill-marker.test.ts | 99 +++++++++++++++++++++++ 2 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 test/gateway-watchdog-kill-marker.test.ts diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index d5cb5687799..a0b3ba0ac56 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -254,6 +254,7 @@ NEMOCLAW_CMD=("$@") # Internal test seam shared by the PID writer and watchdog. This is deliberately # not documented as a public env API; production always keeps the default path. GATEWAY_PID_FILE=/tmp/nemoclaw-gateway.pid +GATEWAY_WATCHDOG_KILL_FILE="${_NEMOCLAW_GATEWAY_WATCHDOG_KILL_FILE:-/tmp/nemoclaw-gateway-watchdog-kill}" # A numeric PID is not a process identity: Linux may reuse it immediately # after the child is reaped. Capture `/proc//stat` field 22 (starttime) @@ -360,6 +361,19 @@ clear_gateway_pid_record() { printf '' | _nemoclaw_safe_replace_tmp_file "$GATEWAY_PID_FILE" 600 "" best-effort 2>/dev/null || true } +record_gateway_watchdog_kill() { + printf '%s\n' "${1:-}" \ + | _nemoclaw_safe_replace_tmp_file "$GATEWAY_WATCHDOG_KILL_FILE" 600 "" best-effort 2>/dev/null || true +} + +consume_gateway_watchdog_kill() { + local expected="$1" marked="" + [ -f "$GATEWAY_WATCHDOG_KILL_FILE" ] || return 1 + IFS= read -r marked <"$GATEWAY_WATCHDOG_KILL_FILE" 2>/dev/null || true + rm -f "$GATEWAY_WATCHDOG_KILL_FILE" 2>/dev/null || true + [ -n "$marked" ] && [ "$marked" = "$expected" ] +} + _chat_ui_url_port() { [ -n "${CHAT_UI_URL:-}" ] || return 1 python3 - "$CHAT_UI_URL" <<'PYPORT' @@ -4122,6 +4136,7 @@ start_gateway_serving_watchdog() { # _NEMOCLAW_GATEWAY_LOG is a test seam; production always appends to # /tmp/gateway.log alongside the gateway's own output. echo "$msg" >>"${_NEMOCLAW_GATEWAY_LOG:-/tmp/gateway.log}" 2>/dev/null || true + record_gateway_watchdog_kill "$tracked_identity" kill -TERM "$pid" 2>/dev/null || true for _ in 1 2 3 4 5 6 7 8 9 10; do openclaw_supervised_pid_is_live "$pid" "$start_identity" || break @@ -4799,7 +4814,8 @@ if [ "$(id -u)" -ne 0 ]; then EXITED_GATEWAY_PID="$GATEWAY_PID" wait "$EXITED_GATEWAY_PID" || RC=$? mark_openclaw_gateway_stopped - if [ "$RC" -eq 0 ]; then + if [ "$RC" -eq 0 ] \ + && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${GATEWAY_PID_START_IDENTITY}"; then exit 0 fi NOW=$(date +%s) @@ -5082,7 +5098,8 @@ while :; do handle_openclaw_gateway_control_request || true continue fi - if [ "$RC" -eq 0 ]; then + if [ "$RC" -eq 0 ] \ + && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${GATEWAY_PID_START_IDENTITY}"; then exit 0 fi NOW=$(date +%s) diff --git a/test/gateway-watchdog-kill-marker.test.ts b/test/gateway-watchdog-kill-marker.test.ts new file mode 100644 index 00000000000..8f99b269392 --- /dev/null +++ b/test/gateway-watchdog-kill-marker.test.ts @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const START_SCRIPT = path.resolve(HERE, "..", "scripts", "nemoclaw-start.sh"); + +function requireNonNegative(value: number, message: string): number { + return value >= 0 + ? value + : (() => { + throw new Error(message); + })(); +} + +function extractShellFunction(scriptPath: string, name: string): string { + const body = readFileSync(scriptPath, "utf8"); + const startMarker = `${name}() {`; + const start = requireNonNegative( + body.indexOf(startMarker), + `function ${name} not found in ${scriptPath}`, + ); + const lines = body.slice(start).split("\n"); + const endIndex = requireNonNegative( + lines.findIndex((line, index) => index > 0 && line === "}"), + `function ${name} missing closing brace in ${scriptPath}`, + ); + return lines.slice(0, endIndex + 1).join("\n"); +} + +function runMarkerScenario(scenario: string): { status: number; stdout: string } { + const helper = extractShellFunction(START_SCRIPT, "_nemoclaw_safe_replace_tmp_file"); + const record = extractShellFunction(START_SCRIPT, "record_gateway_watchdog_kill"); + const consume = extractShellFunction(START_SCRIPT, "consume_gateway_watchdog_kill"); + const harness = ` +${helper} +${record} +${consume} +GATEWAY_WATCHDOG_KILL_FILE="$(mktemp -u "\${TMPDIR:-/tmp}/nemoclaw-wd-kill.XXXXXX")" +${scenario} +`; + const result = spawnSync("bash", ["-c", harness], { encoding: "utf-8", timeout: 10_000 }); + return { status: result.status ?? -1, stdout: (result.stdout ?? "").trim() }; +} + +describe("gateway watchdog kill marker", () => { + it("respawns (match) when the consumed identity equals the recorded one", () => { + const { status } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "123:456"`, + ); + expect(status).toBe(0); + }); + + it("does not match a different gateway identity", () => { + const { status } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "999:000"`, + ); + expect(status).toBe(1); + }); + + it("does not match when no marker was recorded", () => { + const { status } = runMarkerScenario(`consume_gateway_watchdog_kill "123:456"`); + expect(status).toBe(1); + }); + + it("does not match an empty recorded identity", () => { + const { status } = runMarkerScenario( + `record_gateway_watchdog_kill ""\nconsume_gateway_watchdog_kill "123:456"`, + ); + expect(status).toBe(1); + }); + + it("matches only once — a second consume of the same identity misses", () => { + const { status } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "123:456"`, + ); + expect(status).toBe(1); + }); + + it("clears the marker on a matching consume", () => { + const { stdout } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "123:456"\ntest -f "$GATEWAY_WATCHDOG_KILL_FILE" && echo PRESENT || echo ABSENT`, + ); + expect(stdout).toBe("ABSENT"); + }); + + it("clears a stale marker even when the identity does not match", () => { + const { stdout } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "999:000"\ntest -f "$GATEWAY_WATCHDOG_KILL_FILE" && echo PRESENT || echo ABSENT`, + ); + expect(stdout).toBe("ABSENT"); + }); +}); From 6c99e767019a500e9cd996f5c727dd4b9316a67d Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 1 Jul 2026 17:28:24 +0000 Subject: [PATCH 2/3] fix(sandbox): snapshot gateway identity before consuming watchdog kill marker mark_openclaw_gateway_stopped clears GATEWAY_PID_START_IDENTITY, and both respawn loops ran it (directly, and via openclaw_reap_exited_gateway) before consume_gateway_watchdog_kill. The consume argument therefore degraded to "${pid}:" with an empty identity and never matched the watchdog's "${pid}:${start_identity}" record, so a watchdog-induced clean exit still fell through to exit 0 and tore down the sandbox. Snapshot the start identity into EXITED_GATEWAY_START_IDENTITY alongside EXITED_GATEWAY_PID before the stop clears it, and consume with the snapshot in both loops. Add regression coverage for the snapshot-before-clear contract. Co-Authored-By: Claude Signed-off-by: Tinson Lai --- scripts/nemoclaw-start.sh | 6 +++-- test/gateway-watchdog-kill-marker.test.ts | 27 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index a0b3ba0ac56..7bfb4230e58 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -4812,10 +4812,11 @@ if [ "$(id -u)" -ne 0 ]; then # non-zero, defeating the respawn loop entirely. RC=0 EXITED_GATEWAY_PID="$GATEWAY_PID" + EXITED_GATEWAY_START_IDENTITY="$GATEWAY_PID_START_IDENTITY" wait "$EXITED_GATEWAY_PID" || RC=$? mark_openclaw_gateway_stopped if [ "$RC" -eq 0 ] \ - && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${GATEWAY_PID_START_IDENTITY}"; then + && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${EXITED_GATEWAY_START_IDENTITY}"; then exit 0 fi NOW=$(date +%s) @@ -5084,6 +5085,7 @@ while :; do fi EXITED_GATEWAY_PID="$GATEWAY_PID" + EXITED_GATEWAY_START_IDENTITY="$GATEWAY_PID_START_IDENTITY" REAP_STATUS=0 openclaw_reap_exited_gateway || REAP_STATUS=$? if [ "$REAP_STATUS" -eq 3 ]; then @@ -5099,7 +5101,7 @@ while :; do continue fi if [ "$RC" -eq 0 ] \ - && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${GATEWAY_PID_START_IDENTITY}"; then + && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${EXITED_GATEWAY_START_IDENTITY}"; then exit 0 fi NOW=$(date +%s) diff --git a/test/gateway-watchdog-kill-marker.test.ts b/test/gateway-watchdog-kill-marker.test.ts index 8f99b269392..2108ed61d74 100644 --- a/test/gateway-watchdog-kill-marker.test.ts +++ b/test/gateway-watchdog-kill-marker.test.ts @@ -96,4 +96,31 @@ describe("gateway watchdog kill marker", () => { ); expect(stdout).toBe("ABSENT"); }); + + it("respawns when the loop snapshots the start identity before it is cleared", () => { + const { status } = runMarkerScenario( + [ + `GATEWAY_PID_START_IDENTITY="456"`, + `EXITED_GATEWAY_PID="123"`, + `EXITED_GATEWAY_START_IDENTITY="$GATEWAY_PID_START_IDENTITY"`, + `record_gateway_watchdog_kill "123:456"`, + `GATEWAY_PID_START_IDENTITY=""`, + `consume_gateway_watchdog_kill "\${EXITED_GATEWAY_PID}:\${EXITED_GATEWAY_START_IDENTITY}"`, + ].join("\n"), + ); + expect(status).toBe(0); + }); + + it("misses the marker when the loop reads the identity after it is cleared", () => { + const { status } = runMarkerScenario( + [ + `GATEWAY_PID_START_IDENTITY="456"`, + `EXITED_GATEWAY_PID="123"`, + `record_gateway_watchdog_kill "123:456"`, + `GATEWAY_PID_START_IDENTITY=""`, + `consume_gateway_watchdog_kill "\${EXITED_GATEWAY_PID}:\${GATEWAY_PID_START_IDENTITY}"`, + ].join("\n"), + ); + expect(status).toBe(1); + }); }); From 233299e3aca0527be89b570c829dadef915d9ea1 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 1 Jul 2026 17:50:39 +0000 Subject: [PATCH 3/3] test(sandbox): exercise the real respawn loop for the watchdog kill marker The prior ordering tests hand-wrote the loop's assignment order inline, so they would keep passing even if the production snapshot-before-clear ordering regressed. Extract the non-root respawn loop's critical section straight from scripts/nemoclaw-start.sh and run it with wait/mark_openclaw_gateway_stopped stubbed to model a clean exit and the identity clear, so a moved snapshot or wrong variable makes the tests fail. Cover respawn on a matching watchdog marker and teardown on both a no-marker operator exit and a non-matching marker. Co-Authored-By: Claude Signed-off-by: Tinson Lai --- test/gateway-watchdog-kill-marker.test.ts | 71 +++++++++++++++-------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/test/gateway-watchdog-kill-marker.test.ts b/test/gateway-watchdog-kill-marker.test.ts index 2108ed61d74..9b872bd6e37 100644 --- a/test/gateway-watchdog-kill-marker.test.ts +++ b/test/gateway-watchdog-kill-marker.test.ts @@ -49,6 +49,43 @@ ${scenario} return { status: result.status ?? -1, stdout: (result.stdout ?? "").trim() }; } +function extractRespawnCriticalSection(scriptPath: string): string { + const body = readFileSync(scriptPath, "utf8"); + const startMarker = ' RC=0\n EXITED_GATEWAY_PID="$GATEWAY_PID"'; + const start = requireNonNegative( + body.indexOf(startMarker), + `non-root respawn critical section not found in ${scriptPath}`, + ); + const endMarker = " NOW=$(date +%s)"; + const end = requireNonNegative( + body.indexOf(endMarker, start), + `non-root respawn critical section end not found in ${scriptPath}`, + ); + return body.slice(start, end).trimEnd(); +} + +function runRespawnCriticalSection(setup: string): { status: number; stdout: string } { + const helper = extractShellFunction(START_SCRIPT, "_nemoclaw_safe_replace_tmp_file"); + const record = extractShellFunction(START_SCRIPT, "record_gateway_watchdog_kill"); + const consume = extractShellFunction(START_SCRIPT, "consume_gateway_watchdog_kill"); + const section = extractRespawnCriticalSection(START_SCRIPT); + const harness = ` +${helper} +${record} +${consume} +wait() { return "\${STUB_WAIT_RC:-0}"; } +mark_openclaw_gateway_stopped() { GATEWAY_PID=0; GATEWAY_PID_START_IDENTITY=""; } +GATEWAY_WATCHDOG_KILL_FILE="$(mktemp -u "\${TMPDIR:-/tmp}/nemoclaw-wd-kill.XXXXXX")" +GATEWAY_PID="123" +GATEWAY_PID_START_IDENTITY="456" +${setup} +${section} +echo RESPAWN +`; + const result = spawnSync("bash", ["-c", harness], { encoding: "utf-8", timeout: 10_000 }); + return { status: result.status ?? -1, stdout: (result.stdout ?? "").trim() }; +} + describe("gateway watchdog kill marker", () => { it("respawns (match) when the consumed identity equals the recorded one", () => { const { status } = runMarkerScenario( @@ -97,30 +134,18 @@ describe("gateway watchdog kill marker", () => { expect(stdout).toBe("ABSENT"); }); - it("respawns when the loop snapshots the start identity before it is cleared", () => { - const { status } = runMarkerScenario( - [ - `GATEWAY_PID_START_IDENTITY="456"`, - `EXITED_GATEWAY_PID="123"`, - `EXITED_GATEWAY_START_IDENTITY="$GATEWAY_PID_START_IDENTITY"`, - `record_gateway_watchdog_kill "123:456"`, - `GATEWAY_PID_START_IDENTITY=""`, - `consume_gateway_watchdog_kill "\${EXITED_GATEWAY_PID}:\${EXITED_GATEWAY_START_IDENTITY}"`, - ].join("\n"), - ); - expect(status).toBe(0); + it("respawns via the real loop when the watchdog recorded the exiting identity", () => { + const { stdout } = runRespawnCriticalSection(`record_gateway_watchdog_kill "123:456"`); + expect(stdout).toBe("RESPAWN"); }); - it("misses the marker when the loop reads the identity after it is cleared", () => { - const { status } = runMarkerScenario( - [ - `GATEWAY_PID_START_IDENTITY="456"`, - `EXITED_GATEWAY_PID="123"`, - `record_gateway_watchdog_kill "123:456"`, - `GATEWAY_PID_START_IDENTITY=""`, - `consume_gateway_watchdog_kill "\${EXITED_GATEWAY_PID}:\${GATEWAY_PID_START_IDENTITY}"`, - ].join("\n"), - ); - expect(status).toBe(1); + it("tears down via the real loop on a genuine operator clean exit with no marker", () => { + const { stdout } = runRespawnCriticalSection(""); + expect(stdout).toBe(""); + }); + + it("tears down via the real loop when the marker identity does not match the exit", () => { + const { stdout } = runRespawnCriticalSection(`record_gateway_watchdog_kill "123:999"`); + expect(stdout).toBe(""); }); });