fix(recover): require sustained gateway serving after recovery (#4710)

src/lib/actions/sandbox/connect.ts

@@ -51,7 +51,8 @@ import {
 import { preflightVllmModelEnvOrExit } from "./connect-vllm-preflight";
 import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier";
 import { ensureLiveSandboxOrExit, printGatewayLifecycleHint } from "./gateway-state";
-import { checkAndRecoverSandboxProcesses } from "./process-recovery";
+import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics";
+import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand } from "./process-recovery";
 import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch";
 
 const NEMOCLAW_GATEWAY_NAME = "nemoclaw";
@@ -218,6 +219,10 @@ function runSandboxConnectProbe(sandboxName: string): void {
   console.error(
     `  Probe failed: ${agentName} gateway is not running in '${sandboxName}' and automatic recovery failed.`,
   );
+  // Surface the #4710 wedge signature: recovery ran with quiet=true, so this
+  // is the operator's only window into a gateway that served briefly and
+  // then dropped its listener.
+  printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand);
   console.error("  Check /tmp/gateway.log inside the sandbox for details.");
   process.exit(1);
 }

src/lib/actions/sandbox/gateway-wedge-diagnostics.test.ts

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, it, expect } from "vitest";
+
+// Import from compiled dist for parity with the other CLI tests in this project.
+import {
+  collectGatewayWedgeDiagnostics,
+  sanitizeWedgeLogLine,
+} from "../../../../dist/lib/actions/sandbox/gateway-wedge-diagnostics";
+
+describe("collectGatewayWedgeDiagnostics — #4710 wedge signature", () => {
+  it("returns the matching gateway.log lines, trimmed", () => {
+    const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({
+      status: 0,
+      stdout:
+        "  [reload] config change requires gateway restart (plugins.installs)\n" +
+        "  gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.\n",
+      stderr: "",
+    }));
+    expect(lines).toEqual([
+      "[reload] config change requires gateway restart (plugins.installs)",
+      "gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.",
+    ]);
+  });
+
+  it("returns [] when nothing matches (grep exits non-zero)", () => {
+    const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({
+      status: 1,
+      stdout: "",
+      stderr: "",
+    }));
+    expect(lines).toEqual([]);
+  });
+
+  it("returns [] when the sandbox exec is unavailable", () => {
+    const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => null);
+    expect(lines).toEqual([]);
+  });
+
+  it("sanitizes sandbox-controlled log lines before returning them", () => {
+    const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({
+      status: 0,
+      stdout:
+        "gateway startup failed: Authorization: Bearer abc.def.ghi rejected\n" +
+        'gateway startup failed: api_key="nv-secret-123" invalid\n' +
+        "gateway startup failed: \u001b[31mboom\u001b[0m Process will stay alive\n",
+      stderr: "",
+    }));
+    expect(lines[0]).toBe("gateway startup failed: Authorization: Bearer [REDACTED] rejected");
+    expect(lines[1]).toBe('gateway startup failed: api_key="[REDACTED] invalid');
+    // Terminal escape sequences are stripped so sandbox output cannot forge
+    // operator-terminal content.
+    expect(lines[2]).toBe("gateway startup failed: [31mboom[0m Process will stay alive");
+    expect(lines[2]).not.toContain("\u001b");
+  });
+});
+
+describe("sanitizeWedgeLogLine", () => {
+  it("redacts nvapi keys and token assignments", () => {
+    expect(sanitizeWedgeLogLine("auth with nvapi-AbC123xyz failed")).toBe(
+      "auth with [REDACTED] failed",
+    );
+    expect(sanitizeWedgeLogLine("retry token=sk-live-456 now")).toBe("retry token=[REDACTED] now");
+    expect(sanitizeWedgeLogLine("PASSWORD: hunter2 rejected")).toBe(
+      "PASSWORD: [REDACTED] rejected",
+    );
+  });
+
+  it("leaves ordinary wedge lines untouched", () => {
+    const line =
+      "gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.";
+    expect(sanitizeWedgeLogLine(line)).toBe(line);
+  });
+});

src/lib/actions/sandbox/gateway-wedge-diagnostics.ts

@@ -0,0 +1,79 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// #4710 wedge diagnostics — source-of-truth contract:
+//
+// Invalid state: the in-sandbox OpenClaw gateway performs a self-initiated
+// in-process restart on restart-class config changes; in containers a failed
+// restart parks the process alive with its HTTP listener closed, logging
+// "gateway startup failed: ... Process will stay alive; fix the issue and
+// restart." to /tmp/gateway.log.
+// Source boundary: that park-alive behavior lives in OpenClaw's gateway run
+// loop, outside NemoClaw; NemoClaw can only detect it and hand recovery back
+// to its supervisor. The sandbox-side prevention (gateway.reload.mode=hot pin
+// and the serving watchdog) ships separately in the #4710 sandbox PR.
+// Removal condition: when sandbox images pin an OpenClaw release whose failed
+// in-process restart exits non-zero (so the PID-wait supervisor respawns it),
+// this detection can be narrowed and the recovery settle window shortened or
+// defaulted off.
+
+import { shellQuote } from "../../runner";
+import type { SandboxCommandResult } from "./process-recovery";
+
+export type SandboxExec = (sandboxName: string, command: string) => SandboxCommandResult | null;
+
+const WEDGE_LOG_SIGNATURE =
+  "config change requires gateway restart|gateway startup failed|Process will stay alive";
+
+// The matched lines come from a sandbox-writable log, so they are untrusted:
+// strip terminal control characters (no escape-sequence forgery in operator
+// terminals) and redact common credential shapes before printing.
+const CONTROL_CHARS_RE = new RegExp("[\\u0000-\\u0008\\u000b-\\u001f\\u007f-\\u009f]", "g");
+const SECRET_PATTERNS: RegExp[] = [
+  /\b(authorization\s*:\s*bearer)\s+\S+/gi,
+  /\b(api[-_]?key|token|secret|password)(["']?\s*[=:]\s*["']?)\S+/gi,
+  /\bnvapi-\S+/gi,
+];
+
+export function sanitizeWedgeLogLine(line: string): string {
+  let sanitized = line.replace(CONTROL_CHARS_RE, "");
+  sanitized = sanitized.replace(SECRET_PATTERNS[0], "$1 [REDACTED]");
+  sanitized = sanitized.replace(SECRET_PATTERNS[1], "$1$2[REDACTED]");
+  sanitized = sanitized.replace(SECRET_PATTERNS[2], "[REDACTED]");
+  return sanitized.trim();
+}
+
+/**
+ * Collect the #4710 wedge signature from the sandbox gateway log: the
+ * sequence a self-initiated in-process gateway restart leaves behind when it
+ * closes the HTTP listener and then fails, parking the process alive.
+ * Returns up to the last five matching lines (sanitized), or [] when none
+ * match or the log cannot be read.
+ */
+export function collectGatewayWedgeDiagnostics(sandboxName: string, exec: SandboxExec): string[] {
+  const command = `grep -E ${shellQuote(WEDGE_LOG_SIGNATURE)} /tmp/gateway.log 2>/dev/null | tail -5`;
+  const result = exec(sandboxName, command);
+  if (!result || result.status !== 0) {
+    return [];
+  }
+  return result.stdout.split("\n").map(sanitizeWedgeLogLine).filter(Boolean);
+}
+
+/**
+ * Print the #4710 wedge signature (if present) to stderr so the operator
+ * sees why the gateway is unreachable despite a live process. Returns true
+ * when signature lines were found and printed.
+ */
+export function printGatewayWedgeDiagnostics(sandboxName: string, exec: SandboxExec): boolean {
+  const wedgeLines = collectGatewayWedgeDiagnostics(sandboxName, exec);
+  if (wedgeLines.length === 0) {
+    return false;
+  }
+  console.error(
+    "  The gateway served briefly and then dropped its HTTP listener (#4710 wedge signature):",
+  );
+  for (const line of wedgeLines) {
+    console.error(`    ${line}`);
+  }
+  return true;
+}

src/lib/actions/sandbox/process-recovery.test.ts

@@ -1,10 +1,13 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { describe, it, expect } from "vitest";
+import { afterEach, describe, it, expect } from "vitest";
 
 // Import from compiled dist for parity with the other CLI tests in this project.
-import { probeSandboxInferenceGatewayHealth } from "../../../../dist/lib/actions/sandbox/process-recovery";
+import {
+  probeSandboxInferenceGatewayHealth,
+  waitForRecoveredSandboxGateway,
+} from "../../../../dist/lib/actions/sandbox/process-recovery";
 
 describe("probeSandboxInferenceGatewayHealth — #3265 gateway-chain subprobe", () => {
   const makeExec =
@@ -54,3 +57,82 @@ describe("probeSandboxInferenceGatewayHealth — #3265 gateway-chain subprobe",
     expect(result).toBeNull();
   });
 });
+
+describe("waitForRecoveredSandboxGateway — #4710 settle-window confirm", () => {
+  const ENV_KEYS = [
+    "NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS",
+    "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS",
+    "NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS",
+  ];
+  const saved = new Map(ENV_KEYS.map((key) => [key, process.env[key]]));
+
+  afterEach(() => {
+    for (const key of ENV_KEYS) {
+      const value = saved.get(key);
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+  });
+
+  // A probe whose answers play out in order; the last answer repeats.
+  const makeProbe = (answers: Array<boolean | null>) => {
+    const remaining = [...answers];
+    return () => (remaining.length > 1 ? remaining.shift() : remaining[0]) ?? null;
+  };
+
+  it("confirms the gateway is still serving after the settle window", () => {
+    const sleeps: number[] = [];
+    const ok = waitForRecoveredSandboxGateway("my-sandbox", {
+      probeImpl: makeProbe([true, true]),
+      sleepImpl: (seconds: number) => sleeps.push(seconds),
+    });
+    expect(ok).toBe(true);
+    // Default settle window of 25s between the two probes.
+    expect(sleeps).toEqual([25]);
+  });
+
+  it("fails recovery when the gateway serves once and then drops its listener (wedge)", () => {
+    const sleeps: number[] = [];
+    const ok = waitForRecoveredSandboxGateway("my-sandbox", {
+      probeImpl: makeProbe([true, false]),
+      sleepImpl: (seconds: number) => sleeps.push(seconds),
+    });
+    expect(ok).toBe(false);
+    expect(sleeps).toEqual([25]);
+  });
+
+  it("skips the settle confirm when NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS=0", () => {
+    process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "0";
+    const sleeps: number[] = [];
+    const ok = waitForRecoveredSandboxGateway("my-sandbox", {
+      // A second probe would report the wedge; with the settle disabled the
+      // first success must win and no second probe may run.
+      probeImpl: makeProbe([true, false]),
+      sleepImpl: (seconds: number) => sleeps.push(seconds),
+    });
+    expect(ok).toBe(true);
+    expect(sleeps).toEqual([]);
+  });
+
+  it("still polls through initial failures before reaching the settle confirm", () => {
+    process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "5";
+    const sleeps: number[] = [];
+    const ok = waitForRecoveredSandboxGateway("my-sandbox", {
+      probeImpl: makeProbe([false, false, true, true]),
+      sleepImpl: (seconds: number) => sleeps.push(seconds),
+    });
+    expect(ok).toBe(true);
+    // Two poll intervals (default 3s) before the first success, then the
+    // settle window.
+    expect(sleeps).toEqual([3, 3, 5]);
+  });
+
+  it("returns false when the gateway never serves within the wait budget", () => {
+    process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS = "0";
+    const ok = waitForRecoveredSandboxGateway("my-sandbox", {
+      probeImpl: makeProbe([false]),
+      sleepImpl: () => {},
+    });
+    expect(ok).toBe(false);
+  });
+});

src/lib/actions/sandbox/process-recovery.ts

@@ -22,6 +22,7 @@ import { ROOT, shellQuote } from "../../runner";
 import * as registry from "../../state/registry";
 import { parseForwardList } from "../../state/sandbox-session";
 import { classifyForwardHealthWithReachability, isLocalForwardReachable } from "./forward-health";
+import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics";
 import {
   ensureHermesDashboardPortForwardIfEnabled as ensureHermesDashboardPortForward,
   getHermesDashboardRecoveryConfig,
@@ -310,7 +311,16 @@ function readNonNegativeNumberEnv(name: string, fallback: number): number {
   return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
 }
 
-function waitForRecoveredSandboxGateway(sandboxName: string): boolean {
+export function waitForRecoveredSandboxGateway(
+  sandboxName: string,
+  options: {
+    probeImpl?: (sandboxName: string) => boolean | null;
+    sleepImpl?: (seconds: number) => void;
+    quiet?: boolean;
+  } = {},
+): boolean {
+  const probe = options.probeImpl ?? isSandboxGatewayRunning;
+  const sleep = options.sleepImpl ?? sleepSeconds;
   const timeoutSeconds = readNonNegativeNumberEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", 30);
   const intervalSeconds = readNonNegativeNumberEnv(
     "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS",
@@ -321,13 +331,32 @@ function waitForRecoveredSandboxGateway(sandboxName: string): boolean {
       ? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1)
       : Math.max(1, Math.floor(timeoutSeconds) + 1);
 
-  return waitUntil(() => isSandboxGatewayRunning(sandboxName) === true, {
+  const recovered = waitUntil(() => probe(sandboxName) === true, {
     initialIntervalMs: intervalSeconds * 1000,
     maxIntervalMs: intervalSeconds * 1000,
     backoffFactor: 1,
     maxAttempts: attempts,
-    sleep: (ms) => sleepSeconds(ms / 1000),
+    sleep: (ms) => sleep(ms / 1000),
   });
+  if (!recovered) return false;
+
+  // #4710: a freshly relaunched gateway can serve for ~20s and then drop
+  // its HTTP listener while the process stays alive (a failed in-process
+  // restart triggered by a post-launch config write parks it deaf). One
+  // successful probe inside that window is not proof of recovery — wait
+  // out a settle window and require the gateway to still be serving.
+  // 0 disables the settle confirm.
+  // Source boundary and removal condition for this detection live in
+  // gateway-wedge-diagnostics.ts.
+  const settleSeconds = readNonNegativeNumberEnv("NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", 25);
+  if (settleSeconds <= 0) {
+    return true;
+  }
+  if (!options.quiet) {
+    console.log(`  Confirming the gateway stays responsive (~${settleSeconds}s)...`);
+  }
+  sleep(settleSeconds);
+  return probe(sandboxName) === true;
 }
 
 /**
@@ -487,9 +516,10 @@ export function checkAndRecoverSandboxProcesses(
   if (recovered) {
     // Wait for gateway to bind its HTTP port before declaring success. The
     // recovered process can be alive before the OpenAI-compatible API is ready.
-    if (!waitForRecoveredSandboxGateway(sandboxName)) {
+    if (!waitForRecoveredSandboxGateway(sandboxName, { quiet })) {
       if (!quiet) {
         console.error("  Gateway process started but is not responding.");
+        printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand);
         console.error("  Check /tmp/gateway.log inside the sandbox for details.");
         console.error("  Connect to the sandbox and run manually:");
         console.error(