From f4b7c0837db93f426a7920da08c1506c4105df9d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 11 Jun 2026 00:42:55 +0200 Subject: [PATCH 1/4] fix(recover): require sustained gateway serving after recovery (#4710) A wedged in-sandbox OpenClaw gateway serves for ~20 seconds after logging ready and then drops its HTTP listener while the process stays alive (a failed in-process restart triggered by a post-launch config write). The recovery wait declared success on a single health probe inside that window, so 'nemoclaw recover' reported a healthy gateway that was already on its way back to the wedge. After the first successful probe, wait out a settle window (NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS, default 25, 0 disables) and require a confirm probe to still succeed before declaring recovery. On confirm failure, surface the #4710 wedge signature from /tmp/gateway.log (config-reload restart, gateway startup failed, process-will-stay-alive) so the operator sees why the gateway is unreachable despite a live PID. Signed-off-by: Aaron Erickson --- .../actions/sandbox/process-recovery.test.ts | 119 +++++++++++++++++- src/lib/actions/sandbox/process-recovery.ts | 72 ++++++++++- 2 files changed, 184 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 49282e146b6..224e307dad6 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -1,10 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, it, expect } from "vitest"; +import { afterEach, describe, it, expect } from "vitest"; // Import from compiled dist for parity with the other CLI tests in this project. -import { probeSandboxInferenceGatewayHealth } from "../../../../dist/lib/actions/sandbox/process-recovery"; +import { + collectGatewayWedgeDiagnostics, + probeSandboxInferenceGatewayHealth, + waitForRecoveredSandboxGateway, +} from "../../../../dist/lib/actions/sandbox/process-recovery"; describe("probeSandboxInferenceGatewayHealth — #3265 gateway-chain subprobe", () => { const makeExec = @@ -54,3 +58,114 @@ describe("probeSandboxInferenceGatewayHealth — #3265 gateway-chain subprobe", expect(result).toBeNull(); }); }); + +describe("waitForRecoveredSandboxGateway — #4710 settle-window confirm", () => { + const ENV_KEYS = [ + "NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", + "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", + "NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", + ]; + const saved = new Map(ENV_KEYS.map((key) => [key, process.env[key]])); + + afterEach(() => { + for (const key of ENV_KEYS) { + const value = saved.get(key); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + + // A probe whose answers play out in order; the last answer repeats. + const makeProbe = (answers: Array) => { + const remaining = [...answers]; + return () => (remaining.length > 1 ? remaining.shift() : remaining[0]) ?? null; + }; + + it("confirms the gateway is still serving after the settle window", () => { + const sleeps: number[] = []; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + probeImpl: makeProbe([true, true]), + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(true); + // Default settle window of 25s between the two probes. + expect(sleeps).toEqual([25]); + }); + + it("fails recovery when the gateway serves once and then drops its listener (wedge)", () => { + const sleeps: number[] = []; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + probeImpl: makeProbe([true, false]), + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(false); + expect(sleeps).toEqual([25]); + }); + + it("skips the settle confirm when NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS=0", () => { + process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "0"; + const sleeps: number[] = []; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + // A second probe would report the wedge; with the settle disabled the + // first success must win and no second probe may run. + probeImpl: makeProbe([true, false]), + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(true); + expect(sleeps).toEqual([]); + }); + + it("still polls through initial failures before reaching the settle confirm", () => { + process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "5"; + const sleeps: number[] = []; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + probeImpl: makeProbe([false, false, true, true]), + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(true); + // Two poll intervals (default 3s) before the first success, then the + // settle window. + expect(sleeps).toEqual([3, 3, 5]); + }); + + it("returns false when the gateway never serves within the wait budget", () => { + process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS = "0"; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + probeImpl: makeProbe([false]), + sleepImpl: () => {}, + }); + expect(ok).toBe(false); + }); +}); + +describe("collectGatewayWedgeDiagnostics — #4710 wedge signature", () => { + it("returns the matching gateway.log lines, trimmed", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", { + execImpl: () => ({ + status: 0, + stdout: + " [reload] config change requires gateway restart (plugins.installs)\n" + + " gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.\n", + stderr: "", + }), + }); + expect(lines).toEqual([ + "[reload] config change requires gateway restart (plugins.installs)", + "gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.", + ]); + }); + + it("returns [] when nothing matches (grep exits non-zero)", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", { + execImpl: () => ({ status: 1, stdout: "", stderr: "" }), + }); + expect(lines).toEqual([]); + }); + + it("returns [] when the sandbox exec is unavailable", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", { + execImpl: () => null, + }); + expect(lines).toEqual([]); + }); +}); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 441a083369b..f81d11c95e0 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -310,7 +310,16 @@ function readNonNegativeNumberEnv(name: string, fallback: number): number { return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; } -function waitForRecoveredSandboxGateway(sandboxName: string): boolean { +export function waitForRecoveredSandboxGateway( + sandboxName: string, + options: { + probeImpl?: (sandboxName: string) => boolean | null; + sleepImpl?: (seconds: number) => void; + quiet?: boolean; + } = {}, +): boolean { + const probe = options.probeImpl ?? isSandboxGatewayRunning; + const sleep = options.sleepImpl ?? sleepSeconds; const timeoutSeconds = readNonNegativeNumberEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", 30); const intervalSeconds = readNonNegativeNumberEnv( "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", @@ -322,16 +331,60 @@ function waitForRecoveredSandboxGateway(sandboxName: string): boolean { : Math.max(1, Math.floor(timeoutSeconds) + 1); for (let attempt = 0; attempt < attempts; attempt += 1) { - if (isSandboxGatewayRunning(sandboxName) === true) { - return true; + if (probe(sandboxName) === true) { + // #4710: a freshly relaunched gateway can serve for ~20s and then drop + // its HTTP listener while the process stays alive (a failed in-process + // restart triggered by a post-launch config write parks it deaf). One + // successful probe inside that window is not proof of recovery — wait + // out a settle window and require the gateway to still be serving. + // 0 disables the settle confirm. + const settleSeconds = readNonNegativeNumberEnv( + "NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", + 25, + ); + if (settleSeconds <= 0) { + return true; + } + if (!options.quiet) { + console.log(` Confirming the gateway stays responsive (~${settleSeconds}s)...`); + } + sleep(settleSeconds); + return probe(sandboxName) === true; } if (attempt < attempts - 1) { - sleepSeconds(intervalSeconds); + sleep(intervalSeconds); } } return false; } +/** + * Collect the #4710 wedge signature from the sandbox gateway log: the + * sequence a self-initiated in-process gateway restart leaves behind when it + * closes the HTTP listener and then fails, parking the process alive. + * Returns up to the last five matching lines, or [] when none match or the + * log cannot be read. + */ +export function collectGatewayWedgeDiagnostics( + sandboxName: string, + options: { + execImpl?: (sandboxName: string, command: string) => SandboxCommandResult | null; + } = {}, +): string[] { + const exec = options.execImpl ?? executeSandboxExecCommand; + const signature = + "config change requires gateway restart|gateway startup failed|Process will stay alive"; + const command = `grep -E ${shellQuote(signature)} /tmp/gateway.log 2>/dev/null | tail -5`; + const result = exec(sandboxName, command); + if (!result || result.status !== 0) { + return []; + } + return result.stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); +} + /** * Re-establish the dashboard port forward to the sandbox. * Uses the recorded dashboard port for OpenClaw sandboxes, or the agent's @@ -489,9 +542,18 @@ export function checkAndRecoverSandboxProcesses( if (recovered) { // Wait for gateway to bind its HTTP port before declaring success. The // recovered process can be alive before the OpenAI-compatible API is ready. - if (!waitForRecoveredSandboxGateway(sandboxName)) { + if (!waitForRecoveredSandboxGateway(sandboxName, { quiet })) { if (!quiet) { console.error(" Gateway process started but is not responding."); + const wedgeLines = collectGatewayWedgeDiagnostics(sandboxName); + if (wedgeLines.length > 0) { + console.error( + " The gateway served briefly and then dropped its HTTP listener (#4710 wedge signature):", + ); + for (const line of wedgeLines) { + console.error(` ${line}`); + } + } console.error(" Check /tmp/gateway.log inside the sandbox for details."); console.error(" Connect to the sandbox and run manually:"); console.error( From 29503179ae5fcc88f8e2565a0bdb3b28c90818b0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 11 Jun 2026 07:53:34 +0200 Subject: [PATCH 2/4] fix(recover): surface wedge diagnostics on probe-only failure (#4710) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe-only connect path runs recovery with quiet=true and prints its own failure summary, so the #4710 wedge signature added to the non-quiet recovery path never reached the operator there. Extract a shared printGatewayWedgeDiagnostics helper and call it from the probe-only failure path too. CLI test fallout from the settle-confirm: runWithEnv now disables the 25s settle by default (NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS=0) so the existing connect-recovery tests stay fast — settle behavior keeps its dedicated unit coverage — and a new focused CLI suite drives the full wedge shape (serve once, then refuse) through 'connect --probe-only' with a short settle window, asserting the failure exit, the wedge-signature output, and that the confirm probe ran. Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/connect.ts | 6 +- src/lib/actions/sandbox/process-recovery.ts | 29 +++++-- test/cli/connect-recovery-settle.test.ts | 95 +++++++++++++++++++++ test/cli/helpers.ts | 5 ++ 4 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 test/cli/connect-recovery-settle.test.ts diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 874234b015a..811418e5c96 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -51,7 +51,7 @@ import { import { preflightVllmModelEnvOrExit } from "./connect-vllm-preflight"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; import { ensureLiveSandboxOrExit, printGatewayLifecycleHint } from "./gateway-state"; -import { checkAndRecoverSandboxProcesses } from "./process-recovery"; +import { checkAndRecoverSandboxProcesses, printGatewayWedgeDiagnostics } from "./process-recovery"; import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; const NEMOCLAW_GATEWAY_NAME = "nemoclaw"; @@ -218,6 +218,10 @@ function runSandboxConnectProbe(sandboxName: string): void { console.error( ` Probe failed: ${agentName} gateway is not running in '${sandboxName}' and automatic recovery failed.`, ); + // Surface the #4710 wedge signature: recovery ran with quiet=true, so this + // is the operator's only window into a gateway that served briefly and + // then dropped its listener. + printGatewayWedgeDiagnostics(sandboxName); console.error(" Check /tmp/gateway.log inside the sandbox for details."); process.exit(1); } diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index f81d11c95e0..6d23f6ed26c 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -385,6 +385,25 @@ export function collectGatewayWedgeDiagnostics( .filter(Boolean); } +/** + * Print the #4710 wedge signature (if present) to stderr so the operator + * sees why the gateway is unreachable despite a live process. Returns true + * when signature lines were found and printed. + */ +export function printGatewayWedgeDiagnostics(sandboxName: string): boolean { + const wedgeLines = collectGatewayWedgeDiagnostics(sandboxName); + if (wedgeLines.length === 0) { + return false; + } + console.error( + " The gateway served briefly and then dropped its HTTP listener (#4710 wedge signature):", + ); + for (const line of wedgeLines) { + console.error(` ${line}`); + } + return true; +} + /** * Re-establish the dashboard port forward to the sandbox. * Uses the recorded dashboard port for OpenClaw sandboxes, or the agent's @@ -545,15 +564,7 @@ export function checkAndRecoverSandboxProcesses( if (!waitForRecoveredSandboxGateway(sandboxName, { quiet })) { if (!quiet) { console.error(" Gateway process started but is not responding."); - const wedgeLines = collectGatewayWedgeDiagnostics(sandboxName); - if (wedgeLines.length > 0) { - console.error( - " The gateway served briefly and then dropped its HTTP listener (#4710 wedge signature):", - ); - for (const line of wedgeLines) { - console.error(` ${line}`); - } - } + printGatewayWedgeDiagnostics(sandboxName); console.error(" Check /tmp/gateway.log inside the sandbox for details."); console.error(" Connect to the sandbox and run manually:"); console.error( diff --git a/test/cli/connect-recovery-settle.test.ts b/test/cli/connect-recovery-settle.test.ts new file mode 100644 index 00000000000..f7188465ea6 --- /dev/null +++ b/test/cli/connect-recovery-settle.test.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// CLI coverage for the #4710 post-recovery settle-confirm: a wedged gateway +// serves the first probe after relaunch and then drops its HTTP listener, so +// `connect --probe-only` must fail and surface the wedge signature instead of +// declaring a recovery that is already dying. Split from +// connect-recovery.test.ts, which is at the default size budget. + +import { describe, it, expect } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { runWithEnv, writeSandboxRegistry } from "./helpers"; + +describe("CLI dispatch", () => { + it("fails probe-only when the gateway serves once and then drops its listener (#4710 wedge)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-wedge-")); + const localBin = path.join(home, "bin"); + const markerFile = path.join(home, "openshell-calls"); + const stateFile = path.join(home, "probe-state"); + const readyCountFile = path.join(home, "ready-count"); + fs.mkdirSync(localBin, { recursive: true }); + writeSandboxRegistry(home); + fs.writeFileSync(stateFile, "stopped"); + fs.writeFileSync( + path.join(localBin, "openshell"), + [ + "#!/usr/bin/env bash", + `marker_file=${JSON.stringify(markerFile)}`, + `state_file=${JSON.stringify(stateFile)}`, + `ready_count_file=${JSON.stringify(readyCountFile)}`, + 'printf \'%s\\n\' "$*" >> "$marker_file"', + 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', + " echo 'Sandbox:'", + " echo", + " echo ' Id: abc'", + " echo ' Name: alpha'", + " echo ' Namespace: openshell'", + " echo ' Phase: Ready'", + " exit 0", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', + ' cmd="$8"', + ' case "$cmd" in', + ' *"OPENCLAW="*)', + ' echo recovered > "$state_file"', + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + " echo 'GATEWAY_PID=123'", + " exit 0", + " ;;", + " *'curl -so'*)", + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + ' if [ "$(cat "$state_file")" != recovered ]; then echo STOPPED; exit 0; fi', + // The wedge shape: the relaunched gateway answers the first + // post-recovery probe, then drops its listener — every later probe + // refuses. + ' count=$(cat "$ready_count_file" 2>/dev/null || echo 0)', + " count=$((count + 1))", + ' echo "$count" > "$ready_count_file"', + ' if [ "$count" -le 1 ]; then echo RUNNING; else echo STOPPED; fi', + " exit 0", + " ;;", + " *'grep -E'*)", + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + " echo '[reload] config change requires gateway restart (plugins.installs)'", + " echo 'gateway startup failed: listen failure. Process will stay alive; fix the issue and restart.'", + " exit 0", + " ;;", + " esac", + "fi", + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + + const r = runWithEnv("alpha connect --probe-only", { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS: "3", + NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS: "0", + NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS: "1", + }); + + expect(r.code).toBe(1); + expect(r.out).toContain( + "Probe failed: OpenClaw gateway is not running in 'alpha' and automatic recovery failed.", + ); + expect(r.out).toContain("#4710 wedge signature"); + expect(r.out).toContain("config change requires gateway restart (plugins.installs)"); + // First probe succeeded, settle confirm observed the dropped listener. + expect(fs.readFileSync(readyCountFile, "utf8").trim()).toBe("2"); + }); +}); diff --git a/test/cli/helpers.ts b/test/cli/helpers.ts index 9d1eb34197a..b02a8c263f8 100644 --- a/test/cli/helpers.ts +++ b/test/cli/helpers.ts @@ -171,6 +171,11 @@ export function runWithEnv( HOME: fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-test-")), NEMOCLAW_HEALTH_POLL_COUNT: "1", NEMOCLAW_HEALTH_POLL_INTERVAL: "0", + // #4710: the post-recovery settle-confirm waits 25s by default; CLI + // tests disable it to stay fast. Settle behavior has dedicated + // coverage in process-recovery.test.ts and a targeted CLI test in + // connect-recovery.test.ts that overrides this with a short window. + NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS: "0", ...env, }, }); From c9c71cf4cb44552efc9243e98dcaf0ad27146abd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 11 Jun 2026 08:14:12 +0200 Subject: [PATCH 3/4] refactor(recover): extract wedge diagnostics module and sanitize log lines (#4710) PR Review Advisor follow-ups on #5182: - Move the wedge-diagnostics helpers out of the high-churn process-recovery.ts monolith into a focused gateway-wedge-diagnostics module with the sandbox exec passed explicitly (keeps the import graph acyclic) and document the source-of-truth contract there: the invalid OpenClaw park-alive state, the upstream source boundary, and the removal condition for this detection. - The printed lines come from a sandbox-writable log, so treat them as untrusted: strip terminal control characters and redact common credential shapes (bearer headers, key/token/secret/password assignments, nvapi- keys) before they reach the operator's terminal. Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/connect.ts | 5 +- .../sandbox/gateway-wedge-diagnostics.test.ts | 75 ++++++++++++++++++ .../sandbox/gateway-wedge-diagnostics.ts | 79 +++++++++++++++++++ .../actions/sandbox/process-recovery.test.ts | 33 -------- src/lib/actions/sandbox/process-recovery.ts | 51 +----------- 5 files changed, 161 insertions(+), 82 deletions(-) create mode 100644 src/lib/actions/sandbox/gateway-wedge-diagnostics.test.ts create mode 100644 src/lib/actions/sandbox/gateway-wedge-diagnostics.ts diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 811418e5c96..6ebac084677 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -51,7 +51,8 @@ import { import { preflightVllmModelEnvOrExit } from "./connect-vllm-preflight"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; import { ensureLiveSandboxOrExit, printGatewayLifecycleHint } from "./gateway-state"; -import { checkAndRecoverSandboxProcesses, printGatewayWedgeDiagnostics } from "./process-recovery"; +import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; +import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand } from "./process-recovery"; import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; const NEMOCLAW_GATEWAY_NAME = "nemoclaw"; @@ -221,7 +222,7 @@ function runSandboxConnectProbe(sandboxName: string): void { // Surface the #4710 wedge signature: recovery ran with quiet=true, so this // is the operator's only window into a gateway that served briefly and // then dropped its listener. - printGatewayWedgeDiagnostics(sandboxName); + printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); process.exit(1); } diff --git a/src/lib/actions/sandbox/gateway-wedge-diagnostics.test.ts b/src/lib/actions/sandbox/gateway-wedge-diagnostics.test.ts new file mode 100644 index 00000000000..b0e719a412f --- /dev/null +++ b/src/lib/actions/sandbox/gateway-wedge-diagnostics.test.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, it, expect } from "vitest"; + +// Import from compiled dist for parity with the other CLI tests in this project. +import { + collectGatewayWedgeDiagnostics, + sanitizeWedgeLogLine, +} from "../../../../dist/lib/actions/sandbox/gateway-wedge-diagnostics"; + +describe("collectGatewayWedgeDiagnostics — #4710 wedge signature", () => { + it("returns the matching gateway.log lines, trimmed", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({ + status: 0, + stdout: + " [reload] config change requires gateway restart (plugins.installs)\n" + + " gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.\n", + stderr: "", + })); + expect(lines).toEqual([ + "[reload] config change requires gateway restart (plugins.installs)", + "gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.", + ]); + }); + + it("returns [] when nothing matches (grep exits non-zero)", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({ + status: 1, + stdout: "", + stderr: "", + })); + expect(lines).toEqual([]); + }); + + it("returns [] when the sandbox exec is unavailable", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => null); + expect(lines).toEqual([]); + }); + + it("sanitizes sandbox-controlled log lines before returning them", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({ + status: 0, + stdout: + "gateway startup failed: Authorization: Bearer abc.def.ghi rejected\n" + + 'gateway startup failed: api_key="nv-secret-123" invalid\n' + + "gateway startup failed: \u001b[31mboom\u001b[0m Process will stay alive\n", + stderr: "", + })); + expect(lines[0]).toBe("gateway startup failed: Authorization: Bearer [REDACTED] rejected"); + expect(lines[1]).toBe('gateway startup failed: api_key="[REDACTED] invalid'); + // Terminal escape sequences are stripped so sandbox output cannot forge + // operator-terminal content. + expect(lines[2]).toBe("gateway startup failed: [31mboom[0m Process will stay alive"); + expect(lines[2]).not.toContain("\u001b"); + }); +}); + +describe("sanitizeWedgeLogLine", () => { + it("redacts nvapi keys and token assignments", () => { + expect(sanitizeWedgeLogLine("auth with nvapi-AbC123xyz failed")).toBe( + "auth with [REDACTED] failed", + ); + expect(sanitizeWedgeLogLine("retry token=sk-live-456 now")).toBe("retry token=[REDACTED] now"); + expect(sanitizeWedgeLogLine("PASSWORD: hunter2 rejected")).toBe( + "PASSWORD: [REDACTED] rejected", + ); + }); + + it("leaves ordinary wedge lines untouched", () => { + const line = + "gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart."; + expect(sanitizeWedgeLogLine(line)).toBe(line); + }); +}); diff --git a/src/lib/actions/sandbox/gateway-wedge-diagnostics.ts b/src/lib/actions/sandbox/gateway-wedge-diagnostics.ts new file mode 100644 index 00000000000..168827b1e3a --- /dev/null +++ b/src/lib/actions/sandbox/gateway-wedge-diagnostics.ts @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// #4710 wedge diagnostics — source-of-truth contract: +// +// Invalid state: the in-sandbox OpenClaw gateway performs a self-initiated +// in-process restart on restart-class config changes; in containers a failed +// restart parks the process alive with its HTTP listener closed, logging +// "gateway startup failed: ... Process will stay alive; fix the issue and +// restart." to /tmp/gateway.log. +// Source boundary: that park-alive behavior lives in OpenClaw's gateway run +// loop, outside NemoClaw; NemoClaw can only detect it and hand recovery back +// to its supervisor. The sandbox-side prevention (gateway.reload.mode=hot pin +// and the serving watchdog) ships separately in the #4710 sandbox PR. +// Removal condition: when sandbox images pin an OpenClaw release whose failed +// in-process restart exits non-zero (so the PID-wait supervisor respawns it), +// this detection can be narrowed and the recovery settle window shortened or +// defaulted off. + +import { shellQuote } from "../../runner"; +import type { SandboxCommandResult } from "./process-recovery"; + +export type SandboxExec = (sandboxName: string, command: string) => SandboxCommandResult | null; + +const WEDGE_LOG_SIGNATURE = + "config change requires gateway restart|gateway startup failed|Process will stay alive"; + +// The matched lines come from a sandbox-writable log, so they are untrusted: +// strip terminal control characters (no escape-sequence forgery in operator +// terminals) and redact common credential shapes before printing. +const CONTROL_CHARS_RE = new RegExp("[\\u0000-\\u0008\\u000b-\\u001f\\u007f-\\u009f]", "g"); +const SECRET_PATTERNS: RegExp[] = [ + /\b(authorization\s*:\s*bearer)\s+\S+/gi, + /\b(api[-_]?key|token|secret|password)(["']?\s*[=:]\s*["']?)\S+/gi, + /\bnvapi-\S+/gi, +]; + +export function sanitizeWedgeLogLine(line: string): string { + let sanitized = line.replace(CONTROL_CHARS_RE, ""); + sanitized = sanitized.replace(SECRET_PATTERNS[0], "$1 [REDACTED]"); + sanitized = sanitized.replace(SECRET_PATTERNS[1], "$1$2[REDACTED]"); + sanitized = sanitized.replace(SECRET_PATTERNS[2], "[REDACTED]"); + return sanitized.trim(); +} + +/** + * Collect the #4710 wedge signature from the sandbox gateway log: the + * sequence a self-initiated in-process gateway restart leaves behind when it + * closes the HTTP listener and then fails, parking the process alive. + * Returns up to the last five matching lines (sanitized), or [] when none + * match or the log cannot be read. + */ +export function collectGatewayWedgeDiagnostics(sandboxName: string, exec: SandboxExec): string[] { + const command = `grep -E ${shellQuote(WEDGE_LOG_SIGNATURE)} /tmp/gateway.log 2>/dev/null | tail -5`; + const result = exec(sandboxName, command); + if (!result || result.status !== 0) { + return []; + } + return result.stdout.split("\n").map(sanitizeWedgeLogLine).filter(Boolean); +} + +/** + * Print the #4710 wedge signature (if present) to stderr so the operator + * sees why the gateway is unreachable despite a live process. Returns true + * when signature lines were found and printed. + */ +export function printGatewayWedgeDiagnostics(sandboxName: string, exec: SandboxExec): boolean { + const wedgeLines = collectGatewayWedgeDiagnostics(sandboxName, exec); + if (wedgeLines.length === 0) { + return false; + } + console.error( + " The gateway served briefly and then dropped its HTTP listener (#4710 wedge signature):", + ); + for (const line of wedgeLines) { + console.error(` ${line}`); + } + return true; +} diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 224e307dad6..d0003b4cb12 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -5,7 +5,6 @@ import { afterEach, describe, it, expect } from "vitest"; // Import from compiled dist for parity with the other CLI tests in this project. import { - collectGatewayWedgeDiagnostics, probeSandboxInferenceGatewayHealth, waitForRecoveredSandboxGateway, } from "../../../../dist/lib/actions/sandbox/process-recovery"; @@ -137,35 +136,3 @@ describe("waitForRecoveredSandboxGateway — #4710 settle-window confirm", () => expect(ok).toBe(false); }); }); - -describe("collectGatewayWedgeDiagnostics — #4710 wedge signature", () => { - it("returns the matching gateway.log lines, trimmed", () => { - const lines = collectGatewayWedgeDiagnostics("my-sandbox", { - execImpl: () => ({ - status: 0, - stdout: - " [reload] config change requires gateway restart (plugins.installs)\n" + - " gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.\n", - stderr: "", - }), - }); - expect(lines).toEqual([ - "[reload] config change requires gateway restart (plugins.installs)", - "gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.", - ]); - }); - - it("returns [] when nothing matches (grep exits non-zero)", () => { - const lines = collectGatewayWedgeDiagnostics("my-sandbox", { - execImpl: () => ({ status: 1, stdout: "", stderr: "" }), - }); - expect(lines).toEqual([]); - }); - - it("returns [] when the sandbox exec is unavailable", () => { - const lines = collectGatewayWedgeDiagnostics("my-sandbox", { - execImpl: () => null, - }); - expect(lines).toEqual([]); - }); -}); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 6d23f6ed26c..1fdc557d202 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -22,6 +22,7 @@ import { ROOT, shellQuote } from "../../runner"; import * as registry from "../../state/registry"; import { parseForwardList } from "../../state/sandbox-session"; import { classifyForwardHealthWithReachability, isLocalForwardReachable } from "./forward-health"; +import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; import { ensureHermesDashboardPortForwardIfEnabled as ensureHermesDashboardPortForward, getHermesDashboardRecoveryConfig, @@ -338,6 +339,8 @@ export function waitForRecoveredSandboxGateway( // successful probe inside that window is not proof of recovery — wait // out a settle window and require the gateway to still be serving. // 0 disables the settle confirm. + // Source boundary and removal condition for this detection live in + // gateway-wedge-diagnostics.ts. const settleSeconds = readNonNegativeNumberEnv( "NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", 25, @@ -358,52 +361,6 @@ export function waitForRecoveredSandboxGateway( return false; } -/** - * Collect the #4710 wedge signature from the sandbox gateway log: the - * sequence a self-initiated in-process gateway restart leaves behind when it - * closes the HTTP listener and then fails, parking the process alive. - * Returns up to the last five matching lines, or [] when none match or the - * log cannot be read. - */ -export function collectGatewayWedgeDiagnostics( - sandboxName: string, - options: { - execImpl?: (sandboxName: string, command: string) => SandboxCommandResult | null; - } = {}, -): string[] { - const exec = options.execImpl ?? executeSandboxExecCommand; - const signature = - "config change requires gateway restart|gateway startup failed|Process will stay alive"; - const command = `grep -E ${shellQuote(signature)} /tmp/gateway.log 2>/dev/null | tail -5`; - const result = exec(sandboxName, command); - if (!result || result.status !== 0) { - return []; - } - return result.stdout - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); -} - -/** - * Print the #4710 wedge signature (if present) to stderr so the operator - * sees why the gateway is unreachable despite a live process. Returns true - * when signature lines were found and printed. - */ -export function printGatewayWedgeDiagnostics(sandboxName: string): boolean { - const wedgeLines = collectGatewayWedgeDiagnostics(sandboxName); - if (wedgeLines.length === 0) { - return false; - } - console.error( - " The gateway served briefly and then dropped its HTTP listener (#4710 wedge signature):", - ); - for (const line of wedgeLines) { - console.error(` ${line}`); - } - return true; -} - /** * Re-establish the dashboard port forward to the sandbox. * Uses the recorded dashboard port for OpenClaw sandboxes, or the agent's @@ -564,7 +521,7 @@ export function checkAndRecoverSandboxProcesses( if (!waitForRecoveredSandboxGateway(sandboxName, { quiet })) { if (!quiet) { console.error(" Gateway process started but is not responding."); - printGatewayWedgeDiagnostics(sandboxName); + printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); console.error(" Connect to the sandbox and run manually:"); console.error( From 14a88e652f8fc0b251fb99f66c34a13556a6e18b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 11 Jun 2026 08:23:53 +0200 Subject: [PATCH 4/4] docs(test): point settle-disable comment at connect-recovery-settle.test.ts Signed-off-by: Aaron Erickson --- test/cli/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/helpers.ts b/test/cli/helpers.ts index b02a8c263f8..929c5607584 100644 --- a/test/cli/helpers.ts +++ b/test/cli/helpers.ts @@ -174,7 +174,7 @@ export function runWithEnv( // #4710: the post-recovery settle-confirm waits 25s by default; CLI // tests disable it to stay fast. Settle behavior has dedicated // coverage in process-recovery.test.ts and a targeted CLI test in - // connect-recovery.test.ts that overrides this with a short window. + // connect-recovery-settle.test.ts that overrides this with a short window. NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS: "0", ...env, },