diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index d5cb5687799..7bfb4230e58 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -254,6 +254,7 @@ NEMOCLAW_CMD=("$@") # Internal test seam shared by the PID writer and watchdog. This is deliberately # not documented as a public env API; production always keeps the default path. GATEWAY_PID_FILE=/tmp/nemoclaw-gateway.pid +GATEWAY_WATCHDOG_KILL_FILE="${_NEMOCLAW_GATEWAY_WATCHDOG_KILL_FILE:-/tmp/nemoclaw-gateway-watchdog-kill}" # A numeric PID is not a process identity: Linux may reuse it immediately # after the child is reaped. Capture `/proc//stat` field 22 (starttime) @@ -360,6 +361,19 @@ clear_gateway_pid_record() { printf '' | _nemoclaw_safe_replace_tmp_file "$GATEWAY_PID_FILE" 600 "" best-effort 2>/dev/null || true } +record_gateway_watchdog_kill() { + printf '%s\n' "${1:-}" \ + | _nemoclaw_safe_replace_tmp_file "$GATEWAY_WATCHDOG_KILL_FILE" 600 "" best-effort 2>/dev/null || true +} + +consume_gateway_watchdog_kill() { + local expected="$1" marked="" + [ -f "$GATEWAY_WATCHDOG_KILL_FILE" ] || return 1 + IFS= read -r marked <"$GATEWAY_WATCHDOG_KILL_FILE" 2>/dev/null || true + rm -f "$GATEWAY_WATCHDOG_KILL_FILE" 2>/dev/null || true + [ -n "$marked" ] && [ "$marked" = "$expected" ] +} + _chat_ui_url_port() { [ -n "${CHAT_UI_URL:-}" ] || return 1 python3 - "$CHAT_UI_URL" <<'PYPORT' @@ -4122,6 +4136,7 @@ start_gateway_serving_watchdog() { # _NEMOCLAW_GATEWAY_LOG is a test seam; production always appends to # /tmp/gateway.log alongside the gateway's own output. echo "$msg" >>"${_NEMOCLAW_GATEWAY_LOG:-/tmp/gateway.log}" 2>/dev/null || true + record_gateway_watchdog_kill "$tracked_identity" kill -TERM "$pid" 2>/dev/null || true for _ in 1 2 3 4 5 6 7 8 9 10; do openclaw_supervised_pid_is_live "$pid" "$start_identity" || break @@ -4797,9 +4812,11 @@ if [ "$(id -u)" -ne 0 ]; then # non-zero, defeating the respawn loop entirely. RC=0 EXITED_GATEWAY_PID="$GATEWAY_PID" + EXITED_GATEWAY_START_IDENTITY="$GATEWAY_PID_START_IDENTITY" wait "$EXITED_GATEWAY_PID" || RC=$? mark_openclaw_gateway_stopped - if [ "$RC" -eq 0 ]; then + if [ "$RC" -eq 0 ] \ + && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${EXITED_GATEWAY_START_IDENTITY}"; then exit 0 fi NOW=$(date +%s) @@ -5068,6 +5085,7 @@ while :; do fi EXITED_GATEWAY_PID="$GATEWAY_PID" + EXITED_GATEWAY_START_IDENTITY="$GATEWAY_PID_START_IDENTITY" REAP_STATUS=0 openclaw_reap_exited_gateway || REAP_STATUS=$? if [ "$REAP_STATUS" -eq 3 ]; then @@ -5082,7 +5100,8 @@ while :; do handle_openclaw_gateway_control_request || true continue fi - if [ "$RC" -eq 0 ]; then + if [ "$RC" -eq 0 ] \ + && ! consume_gateway_watchdog_kill "${EXITED_GATEWAY_PID}:${EXITED_GATEWAY_START_IDENTITY}"; then exit 0 fi NOW=$(date +%s) diff --git a/test/gateway-watchdog-kill-marker.test.ts b/test/gateway-watchdog-kill-marker.test.ts new file mode 100644 index 00000000000..9b872bd6e37 --- /dev/null +++ b/test/gateway-watchdog-kill-marker.test.ts @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const START_SCRIPT = path.resolve(HERE, "..", "scripts", "nemoclaw-start.sh"); + +function requireNonNegative(value: number, message: string): number { + return value >= 0 + ? value + : (() => { + throw new Error(message); + })(); +} + +function extractShellFunction(scriptPath: string, name: string): string { + const body = readFileSync(scriptPath, "utf8"); + const startMarker = `${name}() {`; + const start = requireNonNegative( + body.indexOf(startMarker), + `function ${name} not found in ${scriptPath}`, + ); + const lines = body.slice(start).split("\n"); + const endIndex = requireNonNegative( + lines.findIndex((line, index) => index > 0 && line === "}"), + `function ${name} missing closing brace in ${scriptPath}`, + ); + return lines.slice(0, endIndex + 1).join("\n"); +} + +function runMarkerScenario(scenario: string): { status: number; stdout: string } { + const helper = extractShellFunction(START_SCRIPT, "_nemoclaw_safe_replace_tmp_file"); + const record = extractShellFunction(START_SCRIPT, "record_gateway_watchdog_kill"); + const consume = extractShellFunction(START_SCRIPT, "consume_gateway_watchdog_kill"); + const harness = ` +${helper} +${record} +${consume} +GATEWAY_WATCHDOG_KILL_FILE="$(mktemp -u "\${TMPDIR:-/tmp}/nemoclaw-wd-kill.XXXXXX")" +${scenario} +`; + const result = spawnSync("bash", ["-c", harness], { encoding: "utf-8", timeout: 10_000 }); + return { status: result.status ?? -1, stdout: (result.stdout ?? "").trim() }; +} + +function extractRespawnCriticalSection(scriptPath: string): string { + const body = readFileSync(scriptPath, "utf8"); + const startMarker = ' RC=0\n EXITED_GATEWAY_PID="$GATEWAY_PID"'; + const start = requireNonNegative( + body.indexOf(startMarker), + `non-root respawn critical section not found in ${scriptPath}`, + ); + const endMarker = " NOW=$(date +%s)"; + const end = requireNonNegative( + body.indexOf(endMarker, start), + `non-root respawn critical section end not found in ${scriptPath}`, + ); + return body.slice(start, end).trimEnd(); +} + +function runRespawnCriticalSection(setup: string): { status: number; stdout: string } { + const helper = extractShellFunction(START_SCRIPT, "_nemoclaw_safe_replace_tmp_file"); + const record = extractShellFunction(START_SCRIPT, "record_gateway_watchdog_kill"); + const consume = extractShellFunction(START_SCRIPT, "consume_gateway_watchdog_kill"); + const section = extractRespawnCriticalSection(START_SCRIPT); + const harness = ` +${helper} +${record} +${consume} +wait() { return "\${STUB_WAIT_RC:-0}"; } +mark_openclaw_gateway_stopped() { GATEWAY_PID=0; GATEWAY_PID_START_IDENTITY=""; } +GATEWAY_WATCHDOG_KILL_FILE="$(mktemp -u "\${TMPDIR:-/tmp}/nemoclaw-wd-kill.XXXXXX")" +GATEWAY_PID="123" +GATEWAY_PID_START_IDENTITY="456" +${setup} +${section} +echo RESPAWN +`; + const result = spawnSync("bash", ["-c", harness], { encoding: "utf-8", timeout: 10_000 }); + return { status: result.status ?? -1, stdout: (result.stdout ?? "").trim() }; +} + +describe("gateway watchdog kill marker", () => { + it("respawns (match) when the consumed identity equals the recorded one", () => { + const { status } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "123:456"`, + ); + expect(status).toBe(0); + }); + + it("does not match a different gateway identity", () => { + const { status } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "999:000"`, + ); + expect(status).toBe(1); + }); + + it("does not match when no marker was recorded", () => { + const { status } = runMarkerScenario(`consume_gateway_watchdog_kill "123:456"`); + expect(status).toBe(1); + }); + + it("does not match an empty recorded identity", () => { + const { status } = runMarkerScenario( + `record_gateway_watchdog_kill ""\nconsume_gateway_watchdog_kill "123:456"`, + ); + expect(status).toBe(1); + }); + + it("matches only once — a second consume of the same identity misses", () => { + const { status } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "123:456"`, + ); + expect(status).toBe(1); + }); + + it("clears the marker on a matching consume", () => { + const { stdout } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "123:456"\ntest -f "$GATEWAY_WATCHDOG_KILL_FILE" && echo PRESENT || echo ABSENT`, + ); + expect(stdout).toBe("ABSENT"); + }); + + it("clears a stale marker even when the identity does not match", () => { + const { stdout } = runMarkerScenario( + `record_gateway_watchdog_kill "123:456"\nconsume_gateway_watchdog_kill "999:000"\ntest -f "$GATEWAY_WATCHDOG_KILL_FILE" && echo PRESENT || echo ABSENT`, + ); + expect(stdout).toBe("ABSENT"); + }); + + it("respawns via the real loop when the watchdog recorded the exiting identity", () => { + const { stdout } = runRespawnCriticalSection(`record_gateway_watchdog_kill "123:456"`); + expect(stdout).toBe("RESPAWN"); + }); + + it("tears down via the real loop on a genuine operator clean exit with no marker", () => { + const { stdout } = runRespawnCriticalSection(""); + expect(stdout).toBe(""); + }); + + it("tears down via the real loop when the marker identity does not match the exit", () => { + const { stdout } = runRespawnCriticalSection(`record_gateway_watchdog_kill "123:999"`); + expect(stdout).toBe(""); + }); +});