diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 9071c1bae1..c3c785f74a 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -8,7 +8,7 @@ "test/channels-add-preset.test.ts": 1871, "test/generate-openclaw-config.test.ts": 1984, "test/install-preflight.test.ts": 4207, - "test/nemoclaw-start.test.ts": 5160, + "test/nemoclaw-start.test.ts": 5043, "test/onboard-messaging.test.ts": 2062, "test/onboard-selection.test.ts": 6888, "test/onboard.test.ts": 4774, diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 27367e32a7..6ceac31cc0 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -392,6 +392,30 @@ $$nemoclaw connect Run `$$nemoclaw status` for a broader gateway health report. +### Sandbox container reports `(unhealthy)` while the agent gateway process is still alive + +The in-sandbox OpenClaw gateway can drop its HTTP listener while its process stays alive. +A restart-class configuration change makes the gateway restart itself in place, and if that restart fails the process parks with no listener (`/tmp/gateway.log` shows `gateway startup failed: ... Process will stay alive`). +Docker then marks the container `(unhealthy)` even though `pgrep` still finds the gateway. + +NemoClaw prevents and self-heals this: + +- The generated sandbox config pins `gateway.reload.mode` to `hot`, so configuration changes never make the gateway restart itself out from under the sandbox supervisor. +- A serving watchdog inside the sandbox kills a gateway that stops listening after it has served, and the supervisor relaunches it (look for `[gateway-watchdog]` lines in `$$nemoclaw logs`). + +Because of the `hot` pin, restart-class configuration changes made inside the sandbox — for example `openclaw plugins install` — log `config reload requires gateway restart; hot mode ignoring` and do not take effect until the gateway restarts. +Apply them with a supervised restart: + +```bash +$$nemoclaw recover +``` + +or rebuild the sandbox for changes that affect provisioning: + +```bash +$$nemoclaw rebuild --yes +``` + ### Invalid sandbox name Sandbox names must be lowercase, start with a letter, contain only letters, numbers, and internal hyphens, and end with a letter or number. diff --git a/scripts/generate-openclaw-config.mts b/scripts/generate-openclaw-config.mts index 6f1b698afe..258ceef613 100755 --- a/scripts/generate-openclaw-config.mts +++ b/scripts/generate-openclaw-config.mts @@ -1231,6 +1231,19 @@ export function buildConfig(env: Env = process.env): JsonObject { }, trustedProxies: ["127.0.0.1", "::1"], auth: { token: "" }, + // Restart-class config changes (plugins.installs, models.pricing, + // unrecognized keys, ...) must not let the gateway SIGUSR1-restart + // itself: in containers the in-process restart path can fail and park + // the process alive with no HTTP listener, which the PID-wait respawn + // loop in nemoclaw-start.sh cannot observe (#4710). Hot mode makes the + // gateway ignore plan-driven restarts; NemoClaw applies restart-class + // changes through sandbox rebuild or `nemoclaw recover` instead. + // Removal condition (also for the serving watchdog in + // nemoclaw-start.sh): once the pinned OpenClaw release exits non-zero + // when a failed in-process restart cannot re-bind its listener — so the + // respawn loop sees the death — this pin can revert to the default + // reload mode after a wedge drill proves no regression. + reload: { mode: "hot" }, }, }; diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index 5412e2cda4..a566b3556a 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -244,6 +244,10 @@ NEMOCLAW_CMD=("$@") # exists if-and-only-if this container is about to start the gateway. Both the # root and non-root entrypoint paths call `mark_in_container_gateway` directly # before their `openclaw gateway run` invocation. +# Internal test seam shared by the PID writer and watchdog. This is deliberately +# not documented as a public env API; production always keeps the default path. +GATEWAY_PID_FILE=/tmp/nemoclaw-gateway.pid + # Best-effort: a write failure must never block startup. mark_in_container_gateway() { _nemoclaw_safe_create_tmp_file /tmp/nemoclaw-gateway-local 600 "" best-effort 2>/dev/null || true @@ -256,7 +260,7 @@ mark_in_container_gateway() { # is tracked and a window where the gateway is down reads as unhealthy. # Best-effort: a write failure must never block startup. record_gateway_pid() { - printf '%s\n' "${1:-}" | _nemoclaw_safe_replace_tmp_file /tmp/nemoclaw-gateway.pid 600 "" best-effort 2>/dev/null || true + printf '%s\n' "${1:-}" | _nemoclaw_safe_replace_tmp_file "$GATEWAY_PID_FILE" 600 "" best-effort 2>/dev/null || true } _chat_ui_url_port() { @@ -3696,6 +3700,124 @@ start_plugin_registry_refresh() { PLUGIN_REFRESH_PID=$! } +# Watchdog for the in-container gateway HTTP listener (#4710). OpenClaw's +# config reloader can SIGUSR1-restart the gateway in-process; in containers a +# failed restart parks the process alive with its listener closed ("gateway +# startup failed: ... Process will stay alive"). The #2757 respawn loop only +# observes process exit, so an alive-but-deaf gateway would stay wedged until +# a human runs `nemoclaw recover`. This watchdog probes the local +# health endpoint and — once it has seen a listener at least once — kills the +# gateway after sustained connection-refused so the respawn loop relaunches +# it. Only curl exit 7 counts as "listener gone": 200/401 mean serving, and +# timeout / HTTP-error outcomes (curl 28/22) mean a listener exists and remain +# the Docker HEALTHCHECK's responsibility. Arming only after the first +# non-refused probe means a slow first boot is never killed; failed first +# boots stay the respawn loop's and HEALTHCHECK's job. + +# PID-reuse / tamper defense: only kill a process whose cmdline still looks +# like the OpenClaw gateway. Same pattern family as the host-side recovery +# script (src/lib/agent/runtime.ts): matches the launch argv +# ("... openclaw gateway run --port N") and the rewritten process titles +# ("openclaw-gateway", bare "openclaw"). +gateway_pid_is_openclaw_gateway() { + # _NEMOCLAW_PROC_ROOT is a test seam (unit tests also run on macOS, which + # has no /proc). Production always uses /proc: the watchdog inherits PID 1's + # environment, which the sandbox user cannot influence. + local cmdline + cmdline="$(tr '\0' ' ' <"${_NEMOCLAW_PROC_ROOT:-/proc}/$1/cmdline" 2>/dev/null)" || return 1 + cmdline="${cmdline%"${cmdline##*[![:space:]]}"}" + [ -n "$cmdline" ] || return 1 + printf '%s' "$cmdline" | grep -qE 'openclaw([ -]gateway| gateway run|$)' +} + +start_gateway_serving_watchdog() { + ( + local interval refused_threshold armed=0 refused_streak=0 pid last_pid="" rc msg + interval="${NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS:-30}" + refused_threshold="${NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD:-4}" + # Both knobs must be positive integers: a zero/garbage interval would + # busy-loop the probe, and a zero threshold would kill on the first + # refusal. Fall back to the defaults rather than trusting bad input. + case "$interval" in + [1-9] | [1-9][0-9]*) ;; + *) + echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS='${interval}'; defaulting to 30" >&2 + interval=30 + ;; + esac + case "$refused_threshold" in + [1-9] | [1-9][0-9]*) ;; + *) + echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD='${refused_threshold}'; defaulting to 4" >&2 + refused_threshold=4 + ;; + esac + [ -n "${_DASHBOARD_PORT:-}" ] || exit 0 + while :; do + sleep "$interval" + pid="$(cat "$GATEWAY_PID_FILE" 2>/dev/null)" || pid="" + case "$pid" in + '' | *[!0-9]*) + last_pid="" + armed=0 + refused_streak=0 + continue + ;; + esac + # A respawned gateway must earn its own armed state — never inherit + # the previous PID's serve history, or a booting replacement could be + # killed for refusals that belong to its predecessor. + if [ "$pid" != "$last_pid" ]; then + last_pid="$pid" + armed=0 + refused_streak=0 + fi + if ! kill -0 "$pid" 2>/dev/null; then + # Process exit is the respawn loop's signal, not ours. + last_pid="" + armed=0 + refused_streak=0 + continue + fi + rc=0 + curl -s -o /dev/null --max-time 5 "http://127.0.0.1:${_DASHBOARD_PORT}/health" 2>/dev/null || rc=$? + if [ "$rc" -ne 7 ]; then + armed=1 + refused_streak=0 + continue + fi + [ "$armed" -eq 1 ] || continue + refused_streak=$((refused_streak + 1)) + if [ "$refused_streak" -lt "$refused_threshold" ]; then + echo "[gateway-watchdog] gateway pid $pid alive but port ${_DASHBOARD_PORT} refused connection ($refused_streak/$refused_threshold) (#4710)" >&2 + continue + fi + if ! gateway_pid_is_openclaw_gateway "$pid"; then + echo "[gateway-watchdog] pid $pid no longer looks like the openclaw gateway; not killing (#4710)" >&2 + armed=0 + refused_streak=0 + continue + fi + msg="[gateway-watchdog] CRITICAL: gateway pid $pid is alive but dropped its HTTP listener on port ${_DASHBOARD_PORT} ($refused_streak consecutive refused probes); killing it so the respawn loop can relaunch (#4710)" + echo "$msg" >&2 + # _NEMOCLAW_GATEWAY_LOG is a test seam; production always appends to + # /tmp/gateway.log alongside the gateway's own output. + echo "$msg" >>"${_NEMOCLAW_GATEWAY_LOG:-/tmp/gateway.log}" 2>/dev/null || true + kill -TERM "$pid" 2>/dev/null || true + for _ in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + if kill -0 "$pid" 2>/dev/null; then + kill -KILL "$pid" 2>/dev/null || true + fi + armed=0 + refused_streak=0 + done + ) & + GATEWAY_WATCHDOG_PID=$! +} + # ── Main ───────────────────────────────────────────────────────── # Migrate legacy symlink layout before anything else reads .openclaw @@ -3816,6 +3938,7 @@ if [ "$(id -u)" -ne 0 ]; then start_persistent_gateway_log_mirror || exit 1 start_auto_pair start_plugin_registry_refresh + start_gateway_serving_watchdog # NOTE: PIDs are collected after launch; a signal arriving between trap # registration and the final append is a small race window (same as before # the shared-library refactor). Acceptable for entrypoint-level cleanup. @@ -3824,6 +3947,7 @@ if [ "$(id -u)" -ne 0 ]; then [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID") [ -n "${GATEWAY_LOG_PERSIST_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_PERSIST_PID") [ -n "${PLUGIN_REFRESH_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$PLUGIN_REFRESH_PID") + [ -n "${GATEWAY_WATCHDOG_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_WATCHDOG_PID") # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" trap cleanup_on_signal SIGTERM SIGINT @@ -4066,6 +4190,8 @@ start_auto_pair # proves /nemoclaw registration without the refresh. start_plugin_registry_refresh +start_gateway_serving_watchdog + # NOTE: PIDs are collected after launch; a signal arriving between trap # registration and the final append is a small race window (same as before # the shared-library refactor). Acceptable for entrypoint-level cleanup. @@ -4074,6 +4200,7 @@ SANDBOX_CHILD_PIDS=("$GATEWAY_PID") [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID") [ -n "${GATEWAY_LOG_PERSIST_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_PERSIST_PID") [ -n "${PLUGIN_REFRESH_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$PLUGIN_REFRESH_PID") +[ -n "${GATEWAY_WATCHDOG_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_WATCHDOG_PID") # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" trap cleanup_on_signal SIGTERM SIGINT diff --git a/src/lib/state/openclaw-config-merge.test.ts b/src/lib/state/openclaw-config-merge.test.ts index 7652019284..85bdad160c 100644 --- a/src/lib/state/openclaw-config-merge.test.ts +++ b/src/lib/state/openclaw-config-merge.test.ts @@ -58,6 +58,28 @@ describe("mergeOpenClawRestoredConfig", () => { expect((merged as { channels: Record }).channels.slack).toBeUndefined(); }); + it("keeps the rebuilt gateway section — including the reload pin — over the backup's (#4710)", () => { + // gateway.reload.mode="hot" is what keeps the in-sandbox gateway from + // SIGUSR1-restarting itself out from under the nemoclaw-start respawn + // loop. A backup taken before the pin existed (or carrying a different + // mode) must not reintroduce restart-mode reloads on restore. + const merged = mergeOpenClawRestoredConfig( + { + gateway: { + auth: { token: "stale-token" }, + reload: { mode: "hybrid" }, + controlUi: { allowInsecureAuth: true }, + }, + }, + { gateway: { auth: { token: "fresh-token" }, reload: { mode: "hot" } } }, + ) as { gateway: unknown }; + + expect(merged.gateway).toEqual({ + auth: { token: "fresh-token" }, + reload: { mode: "hot" }, + }); + }); + it("does not resurrect managed channels when the rebuilt config omits channels", () => { const merged = mergeOpenClawRestoredConfig( { diff --git a/test/gateway-pid-recording.test.ts b/test/gateway-pid-recording.test.ts index e1ad64bdf0..3ec77349b3 100644 --- a/test/gateway-pid-recording.test.ts +++ b/test/gateway-pid-recording.test.ts @@ -40,13 +40,11 @@ describe("nemoclaw-start gateway PID recording for HEALTHCHECK (#4952)", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-pid-")); try { const pidPath = path.join(tmp, "nemoclaw-gateway.pid"); - const fn = extractFunction(src, "record_gateway_pid").replaceAll( - "/tmp/nemoclaw-gateway.pid", - pidPath, - ); + const fn = extractFunction(src, "record_gateway_pid"); const script = [ "set -euo pipefail", safeTmpHelpers(src), + `GATEWAY_PID_FILE=${JSON.stringify(pidPath)}`, fn, 'record_gateway_pid "12345"', ].join("\n"); @@ -65,13 +63,11 @@ describe("nemoclaw-start gateway PID recording for HEALTHCHECK (#4952)", () => { // The writer must swallow errors: a failed write must never abort the // entrypoint. Point it at an unwritable path and assert success. const src = fs.readFileSync(START_SCRIPT, "utf-8"); - const fn = extractFunction(src, "record_gateway_pid").replaceAll( - "/tmp/nemoclaw-gateway.pid", - "/nonexistent-dir/nemoclaw-gateway.pid", - ); + const fn = extractFunction(src, "record_gateway_pid"); const script = [ "set -euo pipefail", safeTmpHelpers(src), + "GATEWAY_PID_FILE=/nonexistent-dir/nemoclaw-gateway.pid", fn, 'record_gateway_pid "12345"', ].join("\n"); diff --git a/test/generate-openclaw-config-reload.test.ts b/test/generate-openclaw-config-reload.test.ts new file mode 100644 index 0000000000..99e06dcb06 --- /dev/null +++ b/test/generate-openclaw-config-reload.test.ts @@ -0,0 +1,114 @@ +// @ts-nocheck +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Tests for the gateway.reload pin in scripts/generate-openclaw-config.mts +// (#4710). The in-sandbox OpenClaw gateway must run with reload mode "hot": +// in the default "hybrid" mode a restart-class config change makes the +// gateway SIGUSR1-restart itself in-process, and a failed restart parks the +// process alive with no HTTP listener — invisible to the PID-wait respawn +// loop in nemoclaw-start.sh. Split out of test/generate-openclaw-config.test.ts, +// which is at its size budget (ci/test-file-size-budget.json). + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildConfig, main } from "../scripts/generate-openclaw-config.mts"; + +/** Minimal env vars required for a valid config generation run. */ +const BASE_ENV: Record = { + NEMOCLAW_MODEL: "test-model", + NEMOCLAW_PROVIDER_KEY: "test-provider", + NEMOCLAW_PRIMARY_MODEL_REF: "test-ref", + CHAT_UI_URL: "http://127.0.0.1:18789", + NEMOCLAW_INFERENCE_BASE_URL: "http://localhost:8080", + NEMOCLAW_INFERENCE_API: "openai", + NEMOCLAW_INFERENCE_COMPAT_B64: Buffer.from("{}").toString("base64"), + NEMOCLAW_PROXY_HOST: "10.200.0.1", + NEMOCLAW_PROXY_PORT: "3128", + NEMOCLAW_CONTEXT_WINDOW: "131072", + NEMOCLAW_MAX_TOKENS: "4096", + NEMOCLAW_REASONING: "false", + NEMOCLAW_AGENT_TIMEOUT: "600", +}; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-config-reload-")); +}); + +afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +function withConfigEnv(envOverrides: Record, fn: () => T): T { + const originalEnv = { ...process.env }; + for (const key of Object.keys(process.env).filter( + (key) => key.startsWith("NEMOCLAW_") || key === "CHAT_UI_URL", + )) { + delete process.env[key]; + } + Object.assign(process.env, BASE_ENV, envOverrides, { HOME: tmpDir }); + try { + return fn(); + } finally { + for (const key of Object.keys(process.env).filter((key) => !(key in originalEnv))) { + delete process.env[key]; + } + Object.assign(process.env, originalEnv); + } +} + +function buildConfigDirect(envOverrides: Record = {}): any { + return withConfigEnv(envOverrides, () => buildConfig()); +} + +describe("gateway.reload pin (#4710)", () => { + it("pins gateway.reload.mode to hot in the generated config", () => { + const config = buildConfigDirect(); + expect(config.gateway.reload).toEqual({ mode: "hot" }); + }); + + it("keeps the pin across unrelated env permutations", () => { + const permutations: Record[] = [ + { NEMOCLAW_WEB_SEARCH_ENABLED: "1" }, + { NEMOCLAW_OPENCLAW_MANAGED_PROXY: "0" }, + { NEMOCLAW_AGENT_HEARTBEAT_EVERY: "5m" }, + { CHAT_UI_URL: "http://127.0.0.1:18792" }, + ]; + for (const overrides of permutations) { + const config = buildConfigDirect(overrides); + expect(config.gateway.reload, JSON.stringify(overrides)).toEqual({ mode: "hot" }); + } + }); + + // Generous timeout: main() does real file I/O and the suite shares a + // worker pool with heavier integration files. + it("re-pins hot mode when an existing config carries a different reload mode", { + timeout: 20000, + }, () => { + // preserveExistingPluginInstalls() merges plugin install records from an + // existing openclaw.json into the regenerated config; the gateway block + // (including reload) must come from the generator, not the old file. + const configDir = path.join(tmpDir, ".openclaw"); + fs.mkdirSync(configDir, { recursive: true }); + const configPath = path.join(configDir, "openclaw.json"); + fs.writeFileSync( + configPath, + JSON.stringify({ + gateway: { reload: { mode: "hybrid" }, auth: { token: "stale" } }, + plugins: { installs: { "custom-plugin": { origin: "npm" } } }, + }), + ); + + withConfigEnv({}, () => main()); + + const written = JSON.parse(fs.readFileSync(configPath, "utf-8")); + expect(written.gateway.reload).toEqual({ mode: "hot" }); + // The plugin-install carryover still works alongside the pin. + expect(written.plugins.installs["custom-plugin"]).toEqual({ origin: "npm" }); + }); +}); diff --git a/test/nemoclaw-start-gateway-health.test.ts b/test/nemoclaw-start-gateway-health.test.ts new file mode 100644 index 0000000000..9b2d7ac2be --- /dev/null +++ b/test/nemoclaw-start-gateway-health.test.ts @@ -0,0 +1,874 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Gateway-health coverage for scripts/nemoclaw-start.sh (#4503, #4710): +// the Docker HEALTHCHECK marker invariants and the gateway serving watchdog. +// The OpenClaw gateway can drop its HTTP listener while the process stays +// alive (failed in-process SIGUSR1 restart); the #2757 respawn loop only sees +// process exit, so the watchdog must kill an alive-but-deaf gateway to hand +// recovery back to the respawn loop. Marker tests are split from +// test/nemoclaw-start.test.ts, which is at its size budget +// (ci/test-file-size-budget.json). + +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { describe, expect, it } from "vitest"; + +const START_SCRIPT = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh"); + +// Read a file that may legitimately be absent without a check-then-read +// race (CodeQL js/file-system-race): attempt the read and treat a missing +// file as null. +function readFileIfPresent(filePath: string): string | null { + try { + return fs.readFileSync(filePath, "utf-8"); + } catch { + return null; + } +} + +function extractShellFunction(src: string, name: string): string { + const header = `${name}() {`; + const start = src.indexOf(header); + expect(start, `Expected ${name} in scripts/nemoclaw-start.sh`).not.toBe(-1); + const bodyStart = start + header.length; + const body = src.slice(bodyStart); + const closing = body.match(/^}$/m); + expect(closing, `Expected closing brace for ${name} in scripts/nemoclaw-start.sh`).not.toBeNull(); + return `${name}() {${body.slice(0, closing?.index ?? 0)}\n}`; +} + +function safeTmpHelpers(src: string): string { + const start = src.indexOf("_nemoclaw_safe_replace_tmp_file() {"); + const end = src.indexOf("_START_LOG=", Math.max(start, 0)); + expect(start, "Expected safe temp helpers in scripts/nemoclaw-start.sh").not.toBe(-1); + expect(end, "Expected safe temp helpers in scripts/nemoclaw-start.sh").toBeGreaterThan(start); + return src.slice(start, end); +} + +function watchdogFunctions(): string { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + return [ + safeTmpHelpers(src), + extractShellFunction(src, "record_gateway_pid"), + extractShellFunction(src, "gateway_pid_is_openclaw_gateway"), + extractShellFunction(src, "start_gateway_serving_watchdog"), + ].join("\n"); +} + +// Drive the watchdog end-to-end against a real background process standing in +// for the gateway. `curlPlan` is the sequence of curl exit codes the stubbed +// probe returns, one per watchdog cycle; the last entry repeats forever. +// The proc fixture under _NEMOCLAW_PROC_ROOT controls what the PID-identity +// check sees for the fake gateway. +function runWatchdog(opts: { + curlPlan: number[]; + cmdline?: string; + env?: Record; + // How long to let the watchdog run when no kill is expected (seconds). + settleSeconds?: number; + expectKill: boolean; +}): { + result: ReturnType; + fakeAlive: boolean; + wedgeLog: string; + tmpDir: string; +} { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-watchdog-")); + const planFile = path.join(tmpDir, "curl-plan.txt"); + const wedgeLogFile = path.join(tmpDir, "gateway.log"); + const pidFile = path.join(tmpDir, "gateway.pid"); + const procRoot = path.join(tmpDir, "proc"); + fs.writeFileSync(planFile, `${opts.curlPlan.join("\n")}\n`); + + const settle = opts.settleSeconds ?? 0.5; + const wrapper = [ + "#!/usr/bin/env bash", + "set -o pipefail", + `GATEWAY_PID_FILE=${JSON.stringify(pidFile)}`, + "_DASHBOARD_PORT=18789", + `_NEMOCLAW_PROC_ROOT=${JSON.stringify(procRoot)}`, + `_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(wedgeLogFile)}`, + // Throttle rather than no-op so the spinning loop stays cheap but the + // test still completes in well under a second per cycle. + "sleep() { command sleep 0.01; }", + // curl stub: pop the next exit code off the plan; keep the last one. + `_CURL_PLAN=${JSON.stringify(planFile)}`, + "curl() {", + " local next rest", + ' next="$(head -n1 "$_CURL_PLAN" 2>/dev/null)"', + ' [ -n "$next" ] || next=0', + ' rest="$(tail -n +2 "$_CURL_PLAN" 2>/dev/null)"', + ' if [ -n "$rest" ]; then printf "%s\\n" "$rest" >"$_CURL_PLAN"; fi', + ' return "$next"', + "}", + // A real process stands in for the gateway so kill -0 / kill -TERM are + // exercised for real; its claimed cmdline comes from the proc fixture. + "command sleep 60 &", + "FAKE_GATEWAY_PID=$!", + `mkdir -p ${JSON.stringify(procRoot)}/$FAKE_GATEWAY_PID`, + `printf '%s' ${JSON.stringify(opts.cmdline ?? "openclaw-gateway")} >${JSON.stringify(procRoot)}/$FAKE_GATEWAY_PID/cmdline`, + watchdogFunctions(), + 'record_gateway_pid "$FAKE_GATEWAY_PID"', + "start_gateway_serving_watchdog", + 'printf "WATCHDOG_PID=%s\\n" "$GATEWAY_WATCHDOG_PID"', + ...(opts.expectKill + ? [ + // Poll until the watchdog kills the fake gateway (or time out). + "for _ in $(command seq 1 300); do", + ' kill -0 "$FAKE_GATEWAY_PID" 2>/dev/null || break', + " command sleep 0.02", + "done", + ] + : [`command sleep ${settle}`]), + 'if kill -0 "$FAKE_GATEWAY_PID" 2>/dev/null; then printf "FAKE_ALIVE=1\\n"; else printf "FAKE_ALIVE=0\\n"; fi', + // Disown before killing: bash's asynchronous job-termination report + // includes the full job command text (the watchdog subshell body), which + // would pollute stderr assertions. + "disown -a 2>/dev/null || true", + 'kill -KILL "$GATEWAY_WATCHDOG_PID" 2>/dev/null || true', + 'kill -KILL "$FAKE_GATEWAY_PID" 2>/dev/null || true', + "command sleep 0.05", + ].join("\n"); + + const script = path.join(tmpDir, "run.sh"); + fs.writeFileSync(script, wrapper, { mode: 0o755 }); + + const result = spawnSync("bash", [script], { + encoding: "utf-8", + timeout: 30000, + env: { ...process.env, ...(opts.env ?? {}) }, + }); + + const stdout = typeof result.stdout === "string" ? result.stdout : ""; + const fakeAlive = /^FAKE_ALIVE=1$/m.test(stdout); + const wedgeLog = readFileIfPresent(wedgeLogFile) ?? ""; + return { result, fakeAlive, wedgeLog, tmpDir }; +} + +describe("gateway serving watchdog (#4710)", () => { + it("kills an alive-but-deaf gateway after sustained connection-refused and logs CRITICAL", () => { + const { result, fakeAlive, wedgeLog, tmpDir } = runWatchdog({ + curlPlan: [0, 7, 7, 7, 7], + expectKill: true, + }); + try { + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + expect(fakeAlive).toBe(false); + expect(result.stderr).toContain("dropped its HTTP listener on port 18789"); + expect(wedgeLog).toContain("[gateway-watchdog] CRITICAL"); + expect(wedgeLog).toContain("dropped its HTTP listener on port 18789"); + expect(wedgeLog).toContain("(#4710)"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("never arms — and never kills — when the gateway has not served yet", () => { + // A gateway that is still booting (or failed to boot) refuses from the + // start; that case belongs to the respawn loop and the Docker + // HEALTHCHECK, not the watchdog. + const { result, fakeAlive, wedgeLog, tmpDir } = runWatchdog({ + curlPlan: [7], + expectKill: false, + }); + try { + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + expect(fakeAlive).toBe(true); + expect(result.stderr).not.toContain("dropped its HTTP listener on port 18789"); + expect(wedgeLog).toBe(""); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("resets the refused streak when a probe succeeds again", () => { + // Three refusals (below the threshold of four), recovery, three more — + // the streak must reset at each success and the gateway must survive. + const { result, fakeAlive, tmpDir } = runWatchdog({ + curlPlan: [0, 7, 7, 7, 0, 7, 7, 7, 0], + expectKill: false, + }); + try { + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + expect(fakeAlive).toBe(true); + expect(result.stderr).not.toContain("dropped its HTTP listener on port 18789"); + expect(result.stderr).toContain("refused connection (1/4)"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("treats curl timeout and HTTP-error outcomes as listener-present", () => { + // curl 28 (timeout) and 22 (HTTP error) prove a listener exists; they + // arm the watchdog but never count toward the refused streak — a wedged + // listener that still accepts connections stays the HEALTHCHECK's call. + const { result, fakeAlive, tmpDir } = runWatchdog({ + curlPlan: [28, 22, 28, 22, 28, 22, 28, 22], + expectKill: false, + }); + try { + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + expect(fakeAlive).toBe(true); + expect(result.stderr).not.toContain("dropped its HTTP listener on port 18789"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("does not kill a PID whose cmdline no longer looks like the gateway", () => { + const { result, fakeAlive, tmpDir } = runWatchdog({ + curlPlan: [0, 7, 7, 7, 7], + cmdline: "vim notes.txt", + expectKill: false, + settleSeconds: 0.8, + }); + try { + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + expect(fakeAlive).toBe(true); + expect(result.stderr).toContain("no longer looks like the openclaw gateway"); + expect(result.stderr).not.toContain("dropped its HTTP listener on port 18789"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("honors the refused-threshold env override", () => { + const { result, fakeAlive, tmpDir } = runWatchdog({ + curlPlan: [0, 7, 7], + env: { NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD: "2" }, + expectKill: true, + }); + try { + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + expect(fakeAlive).toBe(false); + expect(result.stderr).toContain("2 consecutive refused probes"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("falls back to defaults when the env knobs are not positive integers", () => { + // A zero/garbage interval would busy-loop the probe; a zero threshold + // would kill on the first refusal. Both must be rejected with a warning + // while the watchdog keeps working on the defaults. + const { result, fakeAlive, tmpDir } = runWatchdog({ + curlPlan: [0, 7, 7, 7, 7], + env: { + NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS: "0", + NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD: "banana", + }, + expectKill: true, + }); + try { + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + expect(result.stderr).toContain( + "invalid NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS='0'; defaulting to 30", + ); + expect(result.stderr).toContain( + "invalid NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD='banana'; defaulting to 4", + ); + // Default threshold of 4 still applies. + expect(fakeAlive).toBe(false); + expect(result.stderr).toContain("4 consecutive refused probes"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("does not inherit the armed state when the pidfile switches to a new gateway PID", () => { + // A fast respawn can replace the pidfile between probes without the + // watchdog ever observing the old PID as dead. The new gateway must earn + // its own armed state — otherwise its boot-time refusals would count + // against the predecessor's serve history and it could be killed while + // still starting up. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-watchdog-swap-")); + try { + const planFile = path.join(tmpDir, "curl-plan.txt"); + const probeLog = path.join(tmpDir, "probes.log"); + const pidFile = path.join(tmpDir, "gateway.pid"); + const procRoot = path.join(tmpDir, "proc"); + // First probe arms on gateway A; everything after refuses. + fs.writeFileSync(planFile, "0\n7\n"); + + const wrapper = [ + "#!/usr/bin/env bash", + "set -o pipefail", + `GATEWAY_PID_FILE=${JSON.stringify(pidFile)}`, + "_DASHBOARD_PORT=18789", + `_NEMOCLAW_PROC_ROOT=${JSON.stringify(procRoot)}`, + `_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(path.join(tmpDir, "gateway.log"))}`, + // A low threshold makes an inherited armed state lethal within a few + // cycles, so survival proves the per-PID reset. + "export NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD=2", + "sleep() { command sleep 0.01; }", + `_CURL_PLAN=${JSON.stringify(planFile)}`, + "curl() {", + " local next rest", + ' next="$(head -n1 "$_CURL_PLAN" 2>/dev/null)"', + ' [ -n "$next" ] || next=0', + ' rest="$(tail -n +2 "$_CURL_PLAN" 2>/dev/null)"', + ' if [ -n "$rest" ]; then printf "%s\\n" "$rest" >"$_CURL_PLAN"; fi', + ` printf 'probe\\n' >> ${JSON.stringify(probeLog)}`, + ' return "$next"', + "}", + "command sleep 60 &", + "GATEWAY_A=$!", + "command sleep 60 &", + "GATEWAY_B=$!", + `mkdir -p ${JSON.stringify(procRoot)}/$GATEWAY_A ${JSON.stringify(procRoot)}/$GATEWAY_B`, + `printf 'openclaw-gateway' >${JSON.stringify(procRoot)}/$GATEWAY_A/cmdline`, + `printf 'openclaw-gateway' >${JSON.stringify(procRoot)}/$GATEWAY_B/cmdline`, + watchdogFunctions(), + 'record_gateway_pid "$GATEWAY_A"', + "start_gateway_serving_watchdog", + // Wait until gateway A has been probed (and armed via the plan's 0), + // then swap the pidfile to gateway B while refusals continue. + `for _ in $(command seq 1 200); do [ -s ${JSON.stringify(probeLog)} ] && break; command sleep 0.02; done`, + 'record_gateway_pid "$GATEWAY_B"', + "command sleep 0.6", + 'if kill -0 "$GATEWAY_B" 2>/dev/null; then printf "B_ALIVE=1\\n"; else printf "B_ALIVE=0\\n"; fi', + "disown -a 2>/dev/null || true", + 'kill -KILL "$GATEWAY_WATCHDOG_PID" "$GATEWAY_A" "$GATEWAY_B" 2>/dev/null || true', + "command sleep 0.05", + ].join("\n"); + + const script = path.join(tmpDir, "run.sh"); + fs.writeFileSync(script, wrapper, { mode: 0o755 }); + const result = spawnSync("bash", [script], { encoding: "utf-8", timeout: 30000 }); + + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + const stdout = typeof result.stdout === "string" ? result.stdout : ""; + // Without the per-PID reset, B inherits armed=1 and dies after two + // refused probes (threshold 2, 10ms cycles) well inside the 600ms + // observation window. + expect(stdout).toContain("B_ALIVE=1"); + expect(result.stderr).not.toContain("dropped its HTTP listener on port 18789"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); + +describe("record_gateway_pid", () => { + it("replaces a planted symlink without writing through it (#4710 pidfile race)", () => { + // In root mode the pidfile lives in sticky /tmp; a sandbox process can + // plant a symlink at that path between respawns. The update must replace + // the symlink as a directory entry (atomic rename), never open it. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-watchdog-pid-symlink-")); + try { + const pidFile = path.join(tmpDir, "gateway.pid"); + const sensitiveTarget = path.join(tmpDir, "sensitive.txt"); + fs.writeFileSync(sensitiveTarget, "do not touch", { mode: 0o600 }); + fs.symlinkSync(sensitiveTarget, pidFile); + + const script = path.join(tmpDir, "run.sh"); + fs.writeFileSync( + script, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `GATEWAY_PID_FILE=${JSON.stringify(pidFile)}`, + safeTmpHelpers(fs.readFileSync(START_SCRIPT, "utf-8")), + extractShellFunction(fs.readFileSync(START_SCRIPT, "utf-8"), "record_gateway_pid"), + "record_gateway_pid 4242", + ].join("\n"), + { mode: 0o755 }, + ); + + const result = spawnSync("bash", [script], { encoding: "utf-8", timeout: 5000 }); + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + // O_NOFOLLOW makes a single open both the not-a-symlink assertion and + // the content read — no check-then-use window. + const fd = fs.openSync(pidFile, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + expect(fs.readFileSync(fd, "utf-8")).toBe("4242\n"); + } finally { + fs.closeSync(fd); + } + // The symlink target was never opened, written, or chmod-ed. + expect(fs.readFileSync(sensitiveTarget, "utf-8")).toBe("do not touch"); + expect((fs.statSync(sensitiveTarget).mode & 0o777).toString(8)).toBe("600"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("writes the pidfile with 600 permissions, replacing any preexisting file", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-watchdog-pid-")); + try { + const pidFile = path.join(tmpDir, "gateway.pid"); + // Adversarial preexisting file: wrong content, restrictive mode. + fs.writeFileSync(pidFile, "99999", { mode: 0o600 }); + + const script = path.join(tmpDir, "run.sh"); + fs.writeFileSync( + script, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `GATEWAY_PID_FILE=${JSON.stringify(pidFile)}`, + safeTmpHelpers(fs.readFileSync(START_SCRIPT, "utf-8")), + extractShellFunction(fs.readFileSync(START_SCRIPT, "utf-8"), "record_gateway_pid"), + "record_gateway_pid 4242", + ].join("\n"), + { mode: 0o755 }, + ); + + const result = spawnSync("bash", [script], { encoding: "utf-8", timeout: 5000 }); + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + expect(fs.readFileSync(pidFile, "utf-8")).toBe("4242\n"); + expect((fs.statSync(pidFile).mode & 0o777).toString(8)).toBe("600"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); + +describe("gateway_pid_is_openclaw_gateway", () => { + function checkCmdline(rawCmdline: Buffer | null): number | null { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-watchdog-cmdline-")); + try { + const procRoot = path.join(tmpDir, "proc"); + for (const cmdline of rawCmdline === null ? [] : [rawCmdline]) { + fs.mkdirSync(path.join(procRoot, "4242"), { recursive: true }); + fs.writeFileSync(path.join(procRoot, "4242", "cmdline"), cmdline); + } + const script = path.join(tmpDir, "run.sh"); + fs.writeFileSync( + script, + [ + "#!/usr/bin/env bash", + `_NEMOCLAW_PROC_ROOT=${JSON.stringify(procRoot)}`, + extractShellFunction( + fs.readFileSync(START_SCRIPT, "utf-8"), + "gateway_pid_is_openclaw_gateway", + ), + "gateway_pid_is_openclaw_gateway 4242", + ].join("\n"), + { mode: 0o755 }, + ); + return spawnSync("bash", [script], { encoding: "utf-8", timeout: 5000 }).status; + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + } + + const nulArgv = (...argv: string[]): Buffer => Buffer.from(`${argv.join("\u0000")}\u0000`); + + it("matches the launch argv and both rewritten process-title forms", () => { + // Launch argv as /proc presents it: NUL-separated. + expect( + checkCmdline(nulArgv("node", "/usr/local/bin/openclaw", "gateway", "run", "--port", "18789")), + ).toBe(0); + // Rewritten titles observed across OpenClaw builds (#4710). + expect(checkCmdline(nulArgv("openclaw-gateway"))).toBe(0); + expect(checkCmdline(nulArgv("openclaw"))).toBe(0); + }); + + it("rejects reused PIDs, empty cmdlines, and missing proc entries", () => { + expect(checkCmdline(nulArgv("vim", "notes.txt"))).not.toBe(0); + expect(checkCmdline(nulArgv("sleep", "60"))).not.toBe(0); + expect(checkCmdline(Buffer.from(""))).not.toBe(0); + expect(checkCmdline(null)).not.toBe(0); + }); +}); + +describe("healthcheck marker (#4503, #4710)", () => { + // Behavioral test of the marker function: confirms the helper itself writes + // an empty file at the target path and is a no-op when the path is already + // present (idempotent restart-loop semantics). The launch-wiring suite + // below proves the marker is dropped by the launch path itself (and only + // there), independent of env hints like OPENSHELL_DRIVERS. + it("mark_in_container_gateway writes the marker file idempotently (#4710)", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gw-marker-")); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const fnSrc = extractShellFunction(src, "mark_in_container_gateway").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ); + + try { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + safeTmpHelpers(src), + fnSrc, + "mark_in_container_gateway", + "mark_in_container_gateway", // second call must be a no-op + ].join("\n"); + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + expect(result.status).toBe(0); + // statSync throws when the marker is missing, so this single call + // asserts both existence and emptiness (`:` redirected, not appended). + expect(fs.statSync(markerPath).size).toBe(0); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); + +// Behavioral wiring coverage: run the real launch block of each entrypoint +// mode with the real marker/pidfile/watchdog helpers and assert their +// runtime effects. This replaces source-text assertions (banned by +// ci/source-shape-test-budget.json) and locks the #4748 regression +// behaviorally: OPENSHELL_DRIVERS is exported during the run and must have +// no influence on whether the marker is dropped. +describe("gateway launch wiring (#4710)", () => { + function launchBlock(src: string, kind: "non-root" | "root"): string { + const startMarker = + kind === "non-root" + ? "# Start gateway in background, auto-pair, then wait" + : "# Start the gateway as the 'gateway' user."; + const start = src.indexOf(startMarker); + const trap = src.indexOf("trap cleanup_on_signal SIGTERM SIGINT", start); + expect(start, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe( + -1, + ); + expect(trap, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe(-1); + return src.slice(start, src.indexOf("\n", trap)); + } + + function runLaunchWiring(kind: "non-root" | "root") { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `nemoclaw-launch-wiring-${kind}-`)); + const fakeBin = path.join(tmpDir, "bin"); + const openclawLog = path.join(tmpDir, "openclaw.log"); + const gatewayLog = path.join(tmpDir, "gateway.log"); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const pidFile = path.join(tmpDir, "gateway.pid"); + const scriptPath = path.join(tmpDir, "run.sh"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "openclaw"), + `#!/usr/bin/env bash\nprintf '%s\\n' "$*" >> ${JSON.stringify(openclawLog)}\nexec sleep 30\n`, + { mode: 0o755 }, + ); + fs.writeFileSync(path.join(fakeBin, "gosu"), `#!/usr/bin/env bash\nshift\nexec "$@"\n`, { + mode: 0o755, + }); + fs.writeFileSync(gatewayLog, "gateway booting\n"); + + const realFunctions = [ + safeTmpHelpers(src), + extractShellFunction(src, "mark_in_container_gateway").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ), + extractShellFunction(src, "record_gateway_pid"), + extractShellFunction(src, "gateway_pid_is_openclaw_gateway"), + extractShellFunction(src, "start_gateway_serving_watchdog"), + ].join("\n"); + + fs.writeFileSync( + scriptPath, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `export PATH=${JSON.stringify(`${fakeBin}:${process.env.PATH || ""}`)}`, + `OPENCLAW=${JSON.stringify(path.join(fakeBin, "openclaw"))}`, + '_DASHBOARD_PORT="19000"', + // #4748 regression lock: the env hint must have NO influence on the + // marker — it is dropped because this block launches the gateway. + "export OPENSHELL_DRIVERS=docker", + `GATEWAY_PID_FILE=${JSON.stringify(pidFile)}`, + // Keep the watchdog idle for the duration of the test run. + "export NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS=300", + "start_persistent_gateway_log_mirror() { command sleep 30 & GATEWAY_LOG_PERSIST_PID=$!; }", + "start_auto_pair() { command sleep 30 & AUTO_PAIR_PID=$!; }", + "start_plugin_registry_refresh() { :; }", + "cleanup_on_signal() { :; }", + "STEP_DOWN_PREFIX_SANDBOX=(gosu sandbox)", + "STEP_DOWN_PREFIX_GATEWAY=(gosu gateway)", + realFunctions, + launchBlock(src, kind).replaceAll("/tmp/gateway.log", gatewayLog), + `for _ in $(command seq 1 100); do [ -s ${JSON.stringify(openclawLog)} ] && break; command sleep 0.1; done`, + 'printf "GATEWAY_PID=%s\\n" "$GATEWAY_PID"', + 'printf "WATCHDOG_PID=%s\\n" "${GATEWAY_WATCHDOG_PID:-}"', + 'printf "CHILD_PIDS=%s\\n" "${SANDBOX_CHILD_PIDS[*]}"', + 'if [ -n "${GATEWAY_WATCHDOG_PID:-}" ] && kill -0 "$GATEWAY_WATCHDOG_PID" 2>/dev/null; then printf "WATCHDOG_ALIVE=1\\n"; fi', + "disown -a 2>/dev/null || true", + 'for pid in "${SANDBOX_CHILD_PIDS[@]}"; do pkill -P "$pid" 2>/dev/null || true; kill -9 "$pid" 2>/dev/null || true; done', + ].join("\n"), + { mode: 0o700 }, + ); + + const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 15_000 }); + const stdout = typeof result.stdout === "string" ? result.stdout : ""; + const gatewayPid = stdout.match(/^GATEWAY_PID=(\d+)$/m)?.[1]; + const watchdogPid = stdout.match(/^WATCHDOG_PID=(\d+)$/m)?.[1]; + const childPids = (stdout.match(/^CHILD_PIDS=(.+)$/m)?.[1] ?? "").split(/\s+/); + const pidFileContent = readFileIfPresent(pidFile)?.trim() ?? null; + const markerExists = readFileIfPresent(markerPath) !== null; + fs.rmSync(tmpDir, { recursive: true, force: true }); + return { result, stdout, gatewayPid, watchdogPid, childPids, pidFileContent, markerExists }; + } + + it.each([ + "non-root", + "root", + ] as const)("%s launch drops the marker, records the gateway PID, and starts the tracked watchdog", (kind) => { + const run = runLaunchWiring(kind); + expect(run.result.status, `script failed: ${run.result.stderr}`).toBe(0); + // Marker dropped by the launch site, even with OPENSHELL_DRIVERS=docker + // exported — env hints must not gate it (#4748 was a no-op for this). + expect(run.markerExists).toBe(true); + // The watchdog reads the gateway PID from the pidfile each cycle. + expect(run.gatewayPid).toBeDefined(); + expect(run.pidFileContent).toBe(run.gatewayPid); + // The watchdog runs and is registered for SIGTERM cleanup. + expect(run.watchdogPid).toBeDefined(); + expect(run.stdout).toContain("WATCHDOG_ALIVE=1"); + expect(run.childPids).toContain(run.watchdogPid); + expect(run.childPids).toContain(run.gatewayPid); + }); +}); + +// The respawn loop reassigns GATEWAY_PID when it relaunches a dead gateway; +// it must refresh the pidfile too, or the watchdog would keep reading the +// dead PID and go inert for the rest of the sandbox's life. +describe("respawn loop pidfile refresh (#4710)", () => { + function respawnLoop(src: string, kind: "non-root" | "root"): string { + const first = src.indexOf("RESPAWN_TIMES=()"); + const start = kind === "non-root" ? first : src.indexOf("RESPAWN_TIMES=()", first + 1); + expect(start, `Expected ${kind} respawn loop in scripts/nemoclaw-start.sh`).not.toBe(-1); + const endToken = kind === "non-root" ? "\n done" : "\ndone"; + const end = src.indexOf(endToken, start); + expect(end, `Expected ${kind} respawn loop terminator in scripts/nemoclaw-start.sh`).not.toBe( + -1, + ); + return src.slice(start, end + endToken.length); + } + + it.each([ + "non-root", + "root", + ] as const)("%s respawn records the relaunched gateway PID in the pidfile", (kind) => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `nemoclaw-respawn-${kind}-`)); + const fakeBin = path.join(tmpDir, "bin"); + const openclawLog = path.join(tmpDir, "openclaw.log"); + const gatewayLog = path.join(tmpDir, "gateway.log"); + const pidFile = path.join(tmpDir, "gateway.pid"); + const initialPidFile = path.join(tmpDir, "initial.pid"); + const scriptPath = path.join(tmpDir, "run.sh"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "openclaw"), + `#!/usr/bin/env bash\nprintf '%s\\n' "$*" >> ${JSON.stringify(openclawLog)}\nexec sleep 30\n`, + { mode: 0o755 }, + ); + fs.writeFileSync(path.join(fakeBin, "gosu"), `#!/usr/bin/env bash\nshift\nexec "$@"\n`, { + mode: 0o755, + }); + + fs.writeFileSync( + scriptPath, + [ + "#!/usr/bin/env bash", + "set -o pipefail", + `export PATH=${JSON.stringify(`${fakeBin}:${process.env.PATH || ""}`)}`, + `OPENCLAW=${JSON.stringify(path.join(fakeBin, "openclaw"))}`, + '_DASHBOARD_PORT="19000"', + `GATEWAY_PID_FILE=${JSON.stringify(pidFile)}`, + "STEP_DOWN_PREFIX_GATEWAY=(gosu gateway)", + // The loop sleeps 2s between respawns; keep the test fast. + "sleep() { command sleep 0.05; }", + safeTmpHelpers(src), + extractShellFunction(src, "record_gateway_pid"), + "SANDBOX_CHILD_PIDS=()", + "SANDBOX_WAIT_PID=", + "(", + // A gateway that dies immediately with a non-zero status drives + // exactly one respawn iteration. + ' bash -c "exit 7" &', + " GATEWAY_PID=$!", + ' record_gateway_pid "$GATEWAY_PID"', + ` printf '%s' "$GATEWAY_PID" > ${JSON.stringify(initialPidFile)}`, + respawnLoop(src, kind).replaceAll("/tmp/gateway.log", gatewayLog), + ") &", + "LOOP_PID=$!", + 'INITIAL=""; CURRENT=""', + "for _ in $(command seq 1 200); do", + ` INITIAL="$(cat ${JSON.stringify(initialPidFile)} 2>/dev/null || true)"`, + ` CURRENT="$(cat ${JSON.stringify(pidFile)} 2>/dev/null || true)"`, + ' if [ -n "$INITIAL" ] && [ -n "$CURRENT" ] && [ "$CURRENT" != "$INITIAL" ]; then break; fi', + " command sleep 0.05", + "done", + // The pidfile is refreshed at spawn time; give the respawned stub a + // moment to actually execute and write its argv log before cleanup. + `for _ in $(command seq 1 100); do [ -s ${JSON.stringify(openclawLog)} ] && break; command sleep 0.05; done`, + 'printf "INITIAL=%s\\n" "$INITIAL"', + 'printf "CURRENT=%s\\n" "$CURRENT"', + 'if [ -n "$CURRENT" ] && kill -0 "$CURRENT" 2>/dev/null; then printf "RESPAWNED_ALIVE=1\\n"; fi', + "disown -a 2>/dev/null || true", + // Kill the loop before its gateway so it cannot respawn again. + 'kill -9 "$LOOP_PID" 2>/dev/null || true', + 'pkill -P "$LOOP_PID" 2>/dev/null || true', + '[ -n "$CURRENT" ] && kill -9 "$CURRENT" 2>/dev/null || true', + "exit 0", + ].join("\n"), + { mode: 0o700 }, + ); + + try { + const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 20_000 }); + const stdout = typeof result.stdout === "string" ? result.stdout : ""; + expect(result.status, `script failed: ${result.stderr}`).toBe(0); + const initial = stdout.match(/^INITIAL=(\d+)$/m)?.[1]; + const current = stdout.match(/^CURRENT=(\d+)$/m)?.[1]; + expect(initial, `no initial pid in: ${stdout}`).toBeDefined(); + expect(current, `no current pid in: ${stdout}`).toBeDefined(); + expect(current).not.toBe(initial); + expect(stdout).toContain("RESPAWNED_ALIVE=1"); + expect(fs.readFileSync(openclawLog, "utf-8")).toContain("gateway run --port 19000"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); + +// Launch-path signal handling and child-PID tracking for both entrypoint +// modes. Moved from test/nemoclaw-start.test.ts so the legacy file stays +// under its ratcheted size budget; this file owns gateway-launch coverage. +describe("nemoclaw-start gateway launch signal handling", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + + function launchBlock(kind: "non-root" | "root", gatewayLog: string): string { + const startMarker = + kind === "non-root" + ? "# Start gateway in background, auto-pair, then wait" + : "# Start the gateway as the 'gateway' user."; + const start = src.indexOf(startMarker); + const trap = src.indexOf("trap cleanup_on_signal SIGTERM SIGINT", start); + expect(start, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe( + -1, + ); + expect(trap, `Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`).not.toBe(-1); + const lineEnd = src.indexOf("\n", trap); + return src.slice(start, lineEnd).replaceAll("/tmp/gateway.log", gatewayLog); + } + + function runLaunchBlock(kind: "non-root" | "root") { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `nemoclaw-launch-${kind}-`)); + const fakeBin = path.join(tmpDir, "bin"); + const openclawLog = path.join(tmpDir, "openclaw.log"); + const gosuLog = path.join(tmpDir, "gosu.log"); + const gatewayLog = path.join(tmpDir, "gateway.log"); + const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); + const scriptPath = path.join(tmpDir, "run.sh"); + const waitForLaunchLogIterations = Array.from({ length: 100 }, (_, i) => String(i + 1)).join( + " ", + ); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "openclaw"), + `#!/usr/bin/env bash\nprintf '%s\\n' "$*" >> ${JSON.stringify(openclawLog)}\nif [ -f ${JSON.stringify(markerPath)} ]; then printf 'marker=present\\n' >> ${JSON.stringify(openclawLog)}; else printf 'marker=absent\\n' >> ${JSON.stringify(openclawLog)}; fi\nprintf 'state=%s oauth=%s home=%s config=%s\\n' "$OPENCLAW_STATE_DIR" "$OPENCLAW_OAUTH_DIR" "$OPENCLAW_HOME" "$OPENCLAW_CONFIG_PATH" >> ${JSON.stringify(openclawLog)}\nprintf 'gateway stdout marker\\n'\nprintf 'gateway stderr marker\\n' >&2\nexec sleep 30\n`, + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(fakeBin, "gosu"), + `#!/usr/bin/env bash\nprintf 'user=%s args=%s\\n' "$1" "${"$*"}" >> ${JSON.stringify(gosuLog)}\nshift\nexec "$@"\n`, + { mode: 0o755 }, + ); + fs.writeFileSync(gatewayLog, "gateway booting\n"); + fs.writeFileSync( + scriptPath, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `export PATH=${JSON.stringify(`${fakeBin}:${process.env.PATH || ""}`)}`, + `OPENCLAW=${JSON.stringify(path.join(fakeBin, "openclaw"))}`, + "export OPENCLAW_HOME=/sandbox", + "export OPENCLAW_STATE_DIR=/sandbox/.openclaw", + "export OPENCLAW_CONFIG_PATH=/sandbox/.openclaw/openclaw.json", + "export OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials", + '_DASHBOARD_PORT="19000"', + "start_persistent_gateway_log_mirror() { sleep 30 & GATEWAY_LOG_PERSIST_PID=$!; }", + "start_auto_pair() { sleep 30 & AUTO_PAIR_PID=$!; }", + "start_plugin_registry_refresh() { :; }", + "cleanup_on_signal() { :; }", + safeTmpHelpers(src), + extractShellFunction(src, "mark_in_container_gateway").replaceAll( + "/tmp/nemoclaw-gateway-local", + markerPath, + ), + // #4710: the launch block also records the gateway PID for the + // serving watchdog and starts the watchdog alongside the other + // background services. Stub both — watchdog behavior has its own + // suite in test/nemoclaw-start-gateway-health.test.ts. + "record_gateway_pid() { :; }", + "start_gateway_serving_watchdog() { :; }", + "STEP_DOWN_PREFIX_SANDBOX=(gosu sandbox)", + "STEP_DOWN_PREFIX_GATEWAY=(gosu gateway)", + launchBlock(kind, gatewayLog), + kind === "root" + ? `for _ in ${waitForLaunchLogIterations}; do [ -s ${JSON.stringify(gosuLog)} ] && [ -s ${JSON.stringify(openclawLog)} ] && break; sleep 0.1; done` + : `for _ in ${waitForLaunchLogIterations}; do [ -s ${JSON.stringify(openclawLog)} ] && break; sleep 0.1; done`, + 'printf "GATEWAY_PID=%s\\n" "$GATEWAY_PID"', + 'printf "AUTO_PAIR_PID=%s\\n" "${AUTO_PAIR_PID:-}"', + 'printf "TAIL_PID=%s\\n" "${GATEWAY_LOG_TAIL_PID:-}"', + 'printf "PERSIST_PID=%s\\n" "${GATEWAY_LOG_PERSIST_PID:-}"', + 'printf "WAIT_PID=%s\\n" "$SANDBOX_WAIT_PID"', + 'printf "CHILD_PIDS=%s\\n" "${SANDBOX_CHILD_PIDS[*]}"', + "trap -p SIGTERM", + 'for pid in "${SANDBOX_CHILD_PIDS[@]}"; do pkill -P "$pid" 2>/dev/null || true; kill "$pid" 2>/dev/null || true; done', + 'for pid in "${SANDBOX_CHILD_PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done', + ].join("\n"), + { mode: 0o700 }, + ); + + const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 15_000 }); + const openclaw = readFileIfPresent(openclawLog) ?? ""; + const gosu = readFileIfPresent(gosuLog) ?? ""; + const gateway = readFileIfPresent(gatewayLog) ?? ""; + fs.rmSync(tmpDir, { recursive: true, force: true }); + return { result, openclaw, gosu, gateway }; + } + + it("registers child PIDs, redirects gateway output, and traps signals in non-root mode", () => { + const { result, openclaw, gateway } = runLaunchBlock("non-root"); + expect(result.status).toBe(0); + expect(openclaw).toContain("gateway run --port 19000"); + expect(openclaw).toContain("marker=present"); + expect(openclaw).not.toContain("marker=absent"); + expect(openclaw).toContain( + "state=/sandbox/.openclaw oauth=/sandbox/.openclaw/credentials home=/sandbox config=/sandbox/.openclaw/openclaw.json", + ); + expect(gateway).toContain("gateway stdout marker"); + expect(gateway).toContain("gateway stderr marker"); + expect(result.stdout).not.toContain("gateway stdout marker"); + const stdout = result.stdout; + const gatewayPid = stdout.match(/GATEWAY_PID=(\d+)/)?.[1]; + expect(gatewayPid).toBeTruthy(); + expect(stdout).toContain(`WAIT_PID=${gatewayPid}`); + expect(stdout).toContain(`CHILD_PIDS=${gatewayPid}`); + expect(stdout).toMatch(/AUTO_PAIR_PID=\d+/); + expect(stdout).toMatch(/TAIL_PID=\d+/); + expect(stdout).toMatch(/PERSIST_PID=\d+/); + expect(stdout).toContain("cleanup_on_signal"); + }); + + it("launches the root gateway through gosu with the configured port and tracks child PIDs", () => { + const { result, openclaw, gosu } = runLaunchBlock("root"); + expect(result.status).toBe(0); + expect(gosu).toContain("user=gateway"); + expect(gosu).toContain("gateway run --port 19000"); + expect(openclaw).toContain("marker=present"); + expect(openclaw).not.toContain("marker=absent"); + expect(openclaw).toContain( + "state=/sandbox/.openclaw oauth=/sandbox/.openclaw/credentials home=/sandbox config=/sandbox/.openclaw/openclaw.json", + ); + const gatewayPid = result.stdout.match(/GATEWAY_PID=(\d+)/)?.[1]; + expect(gatewayPid).toBeTruthy(); + expect(result.stdout).toContain(`WAIT_PID=${gatewayPid}`); + expect(result.stdout).toContain(`CHILD_PIDS=${gatewayPid}`); + expect(result.stdout).toMatch(/AUTO_PAIR_PID=\d+/); + expect(result.stdout).toMatch(/TAIL_PID=\d+/); + expect(result.stdout).toMatch(/PERSIST_PID=\d+/); + expect(result.stdout).toContain("cleanup_on_signal"); + }); +}); diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index fb1fb1d4f8..214cffd954 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -6,6 +6,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import * as ts from "typescript"; import { describe, expect, it } from "vitest"; const START_SCRIPT = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh"); @@ -141,10 +142,20 @@ function startScriptHeredoc(src: string, marker: string): string { ? ["telegram", "telegram-diagnostics.ts"] : undefined; expect(channelPreload).toBeTruthy(); - return fs.readFileSync( - path.join(CHANNEL_RUNTIME_SCRIPTS, channelPreload[0], "runtime", channelPreload[1]), - "utf-8", + const preloadPath = path.join( + CHANNEL_RUNTIME_SCRIPTS, + channelPreload[0], + "runtime", + channelPreload[1], ); + const preloadSource = fs.readFileSync(preloadPath, "utf-8"); + if (!preloadPath.endsWith(".ts")) return preloadSource; + return ts.transpileModule(preloadSource, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2022, + }, + }).outputText; } function trustedApprovalPolicyFile(): string { @@ -2366,134 +2377,6 @@ exit 2 }, 30_000); }); -describe("nemoclaw-start gateway launch signal handling", () => { - const src = fs.readFileSync(START_SCRIPT, "utf-8"); - - function launchBlock(kind: "non-root" | "root", gatewayLog: string): string { - const startMarker = - kind === "non-root" - ? "# Start gateway in background, auto-pair, then wait" - : "# Start the gateway as the 'gateway' user."; - const start = src.indexOf(startMarker); - const trap = src.indexOf("trap cleanup_on_signal SIGTERM SIGINT", start); - if (start === -1 || trap === -1) { - throw new Error(`Expected ${kind} gateway launch block in scripts/nemoclaw-start.sh`); - } - const lineEnd = src.indexOf("\n", trap); - return src.slice(start, lineEnd).replaceAll("/tmp/gateway.log", gatewayLog); - } - - function runLaunchBlock(kind: "non-root" | "root") { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `nemoclaw-launch-${kind}-`)); - const fakeBin = path.join(tmpDir, "bin"); - const openclawLog = path.join(tmpDir, "openclaw.log"); - const gosuLog = path.join(tmpDir, "gosu.log"); - const gatewayLog = path.join(tmpDir, "gateway.log"); - const markerPath = path.join(tmpDir, "nemoclaw-gateway-local"); - const scriptPath = path.join(tmpDir, "run.sh"); - const waitForLaunchLogIterations = Array.from({ length: 100 }, (_, i) => String(i + 1)).join( - " ", - ); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "openclaw"), - `#!/usr/bin/env bash\nprintf '%s\\n' "$*" >> ${JSON.stringify(openclawLog)}\nif [ -f ${JSON.stringify(markerPath)} ]; then printf 'marker=present\\n' >> ${JSON.stringify(openclawLog)}; else printf 'marker=absent\\n' >> ${JSON.stringify(openclawLog)}; fi\nprintf 'state=%s oauth=%s home=%s config=%s\\n' "$OPENCLAW_STATE_DIR" "$OPENCLAW_OAUTH_DIR" "$OPENCLAW_HOME" "$OPENCLAW_CONFIG_PATH" >> ${JSON.stringify(openclawLog)}\nprintf 'gateway stdout marker\\n'\nprintf 'gateway stderr marker\\n' >&2\nexec sleep 30\n`, - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(fakeBin, "gosu"), - `#!/usr/bin/env bash\nprintf 'user=%s args=%s\\n' "$1" "${"$*"}" >> ${JSON.stringify(gosuLog)}\nshift\nexec "$@"\n`, - { mode: 0o755 }, - ); - fs.writeFileSync(gatewayLog, "gateway booting\n"); - fs.writeFileSync( - scriptPath, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `export PATH=${JSON.stringify(`${fakeBin}:${process.env.PATH || ""}`)}`, - `OPENCLAW=${JSON.stringify(path.join(fakeBin, "openclaw"))}`, - "export OPENCLAW_HOME=/sandbox", - "export OPENCLAW_STATE_DIR=/sandbox/.openclaw", - "export OPENCLAW_CONFIG_PATH=/sandbox/.openclaw/openclaw.json", - "export OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials", - '_DASHBOARD_PORT="19000"', - "start_persistent_gateway_log_mirror() { sleep 30 & GATEWAY_LOG_PERSIST_PID=$!; }", - "start_auto_pair() { sleep 30 & AUTO_PAIR_PID=$!; }", - "start_plugin_registry_refresh() { :; }", - "cleanup_on_signal() { :; }; record_gateway_pid() { :; }", // record_gateway_pid: #4952 - `mark_in_container_gateway() { : > ${JSON.stringify(markerPath)}; }`, - "STEP_DOWN_PREFIX_SANDBOX=(gosu sandbox)", - "STEP_DOWN_PREFIX_GATEWAY=(gosu gateway)", - launchBlock(kind, gatewayLog), - kind === "root" - ? `for _ in ${waitForLaunchLogIterations}; do [ -s ${JSON.stringify(gosuLog)} ] && [ -s ${JSON.stringify(openclawLog)} ] && break; sleep 0.1; done` - : `for _ in ${waitForLaunchLogIterations}; do [ -s ${JSON.stringify(openclawLog)} ] && break; sleep 0.1; done`, - 'printf "GATEWAY_PID=%s\\n" "$GATEWAY_PID"', - 'printf "AUTO_PAIR_PID=%s\\n" "${AUTO_PAIR_PID:-}"', - 'printf "TAIL_PID=%s\\n" "${GATEWAY_LOG_TAIL_PID:-}"', - 'printf "PERSIST_PID=%s\\n" "${GATEWAY_LOG_PERSIST_PID:-}"', - 'printf "WAIT_PID=%s\\n" "$SANDBOX_WAIT_PID"', - 'printf "CHILD_PIDS=%s\\n" "${SANDBOX_CHILD_PIDS[*]}"', - "trap -p SIGTERM", - 'for pid in "${SANDBOX_CHILD_PIDS[@]}"; do pkill -P "$pid" 2>/dev/null || true; kill "$pid" 2>/dev/null || true; done', - 'for pid in "${SANDBOX_CHILD_PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done', - ].join("\n"), - { mode: 0o700 }, - ); - - const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 15_000 }); - const openclaw = fs.existsSync(openclawLog) ? fs.readFileSync(openclawLog, "utf-8") : ""; - const gosu = fs.existsSync(gosuLog) ? fs.readFileSync(gosuLog, "utf-8") : ""; - const gateway = fs.existsSync(gatewayLog) ? fs.readFileSync(gatewayLog, "utf-8") : ""; - fs.rmSync(tmpDir, { recursive: true, force: true }); - return { result, openclaw, gosu, gateway }; - } - - it("registers child PIDs, redirects gateway output, and traps signals in non-root mode", () => { - const { result, openclaw, gateway } = runLaunchBlock("non-root"); - expect(result.status).toBe(0); - expect(openclaw).toContain("gateway run --port 19000"); - expect(openclaw).toContain("marker=present"); - expect(openclaw).not.toContain("marker=absent"); - expect(openclaw).toContain( - "state=/sandbox/.openclaw oauth=/sandbox/.openclaw/credentials home=/sandbox config=/sandbox/.openclaw/openclaw.json", - ); - expect(gateway).toContain("gateway stdout marker"); - expect(gateway).toContain("gateway stderr marker"); - expect(result.stdout).not.toContain("gateway stdout marker"); - const stdout = result.stdout; - const gatewayPid = stdout.match(/GATEWAY_PID=(\d+)/)?.[1]; - expect(gatewayPid).toBeTruthy(); - expect(stdout).toContain(`WAIT_PID=${gatewayPid}`); - expect(stdout).toContain(`CHILD_PIDS=${gatewayPid}`); - expect(stdout).toMatch(/AUTO_PAIR_PID=\d+/); - expect(stdout).toMatch(/TAIL_PID=\d+/); - expect(stdout).toMatch(/PERSIST_PID=\d+/); - expect(stdout).toContain("cleanup_on_signal"); - }); - - it("launches the root gateway through gosu with the configured port and tracks child PIDs", () => { - const { result, openclaw, gosu } = runLaunchBlock("root"); - expect(result.status).toBe(0); - expect(gosu).toContain("user=gateway"); - expect(gosu).toContain("gateway run --port 19000"); - expect(openclaw).toContain("marker=present"); - expect(openclaw).not.toContain("marker=absent"); - expect(openclaw).toContain( - "state=/sandbox/.openclaw oauth=/sandbox/.openclaw/credentials home=/sandbox config=/sandbox/.openclaw/openclaw.json", - ); - const gatewayPid = result.stdout.match(/GATEWAY_PID=(\d+)/)?.[1]; - expect(gatewayPid).toBeTruthy(); - expect(result.stdout).toContain(`WAIT_PID=${gatewayPid}`); - expect(result.stdout).toContain(`CHILD_PIDS=${gatewayPid}`); - expect(result.stdout).toMatch(/AUTO_PAIR_PID=\d+/); - expect(result.stdout).toMatch(/TAIL_PID=\d+/); - expect(result.stdout).toMatch(/PERSIST_PID=\d+/); - expect(result.stdout).toContain("cleanup_on_signal"); - }); -}); - // ------------------------------------------------------------------- // NC-2227-01: Legacy migration behavior // ------------------------------------------------------------------- diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index 3f0290d9cf..38824d5413 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -470,10 +470,32 @@ describe("sandbox provisioning: image health checks (#1430)", () => { // --ignore-ancestors prevents pgrep from self-matching the // healthcheck shell whose argv contains the gateway pattern. // The [ -] class matches both `openclaw gateway` (launcher) and - // `openclaw-gateway` (re-execed binary). + // `openclaw-gateway` (re-execed binary). Bare `openclaw` is verified + // through the recorded PID fallback below so unrelated CLI invocations + // cannot keep the container healthy. expect(probe.calls).toContain("pgrep --ignore-ancestors -f openclaw[ -]gateway"); }); + it("uses a pgrep liveness pattern that matches gateway argv and rewritten titles but not ordinary openclaw CLI use (#4710)", () => { + const dockerfile = fs.readFileSync(DOCKERFILE, "utf-8"); + const match = dockerfile.match(/pgrep --ignore-ancestors -f '([^']+)'/); + const pattern = match?.[1] ?? "$.^"; + const matches = (cmdline: string) => + spawnSync("grep", ["-qE", pattern], { input: cmdline, encoding: "utf-8" }).status === 0; + + // The launcher argv and legacy rewritten-title form must match. + expect(matches("node /usr/local/bin/openclaw gateway run --port 18789")).toBe(true); + expect(matches("openclaw-gateway")).toBe(true); + + // Bare `openclaw` and ordinary agent CLI invocations must not satisfy the + // pgrep liveness probe; the #4952 path below checks bare gateway argv + // through the recorded PID instead. + expect(matches("openclaw")).toBe(false); + expect(matches("openclaw plugins registry --refresh")).toBe(false); + expect(matches("node /usr/local/bin/openclaw devices list")).toBe(false); + expect(matches("vim openclaw-notes.txt")).toBe(false); + }); + it("reports unhealthy when curl times out (wedged HTTP server, not namespace mismatch)", () => { // A connect timeout means a listener exists but is not responding, // e.g. a wedged HTTP server. We deliberately do not fall back to the