diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 874234b015a..6ebac084677 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -51,7 +51,8 @@ import { import { preflightVllmModelEnvOrExit } from "./connect-vllm-preflight"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; import { ensureLiveSandboxOrExit, printGatewayLifecycleHint } from "./gateway-state"; -import { checkAndRecoverSandboxProcesses } from "./process-recovery"; +import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; +import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand } from "./process-recovery"; import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; const NEMOCLAW_GATEWAY_NAME = "nemoclaw"; @@ -218,6 +219,10 @@ function runSandboxConnectProbe(sandboxName: string): void { console.error( ` Probe failed: ${agentName} gateway is not running in '${sandboxName}' and automatic recovery failed.`, ); + // Surface the #4710 wedge signature: recovery ran with quiet=true, so this + // is the operator's only window into a gateway that served briefly and + // then dropped its listener. + printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); process.exit(1); } diff --git a/src/lib/actions/sandbox/gateway-wedge-diagnostics.test.ts b/src/lib/actions/sandbox/gateway-wedge-diagnostics.test.ts new file mode 100644 index 00000000000..b0e719a412f --- /dev/null +++ b/src/lib/actions/sandbox/gateway-wedge-diagnostics.test.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, it, expect } from "vitest"; + +// Import from compiled dist for parity with the other CLI tests in this project. +import { + collectGatewayWedgeDiagnostics, + sanitizeWedgeLogLine, +} from "../../../../dist/lib/actions/sandbox/gateway-wedge-diagnostics"; + +describe("collectGatewayWedgeDiagnostics — #4710 wedge signature", () => { + it("returns the matching gateway.log lines, trimmed", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({ + status: 0, + stdout: + " [reload] config change requires gateway restart (plugins.installs)\n" + + " gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.\n", + stderr: "", + })); + expect(lines).toEqual([ + "[reload] config change requires gateway restart (plugins.installs)", + "gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart.", + ]); + }); + + it("returns [] when nothing matches (grep exits non-zero)", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({ + status: 1, + stdout: "", + stderr: "", + })); + expect(lines).toEqual([]); + }); + + it("returns [] when the sandbox exec is unavailable", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => null); + expect(lines).toEqual([]); + }); + + it("sanitizes sandbox-controlled log lines before returning them", () => { + const lines = collectGatewayWedgeDiagnostics("my-sandbox", () => ({ + status: 0, + stdout: + "gateway startup failed: Authorization: Bearer abc.def.ghi rejected\n" + + 'gateway startup failed: api_key="nv-secret-123" invalid\n' + + "gateway startup failed: \u001b[31mboom\u001b[0m Process will stay alive\n", + stderr: "", + })); + expect(lines[0]).toBe("gateway startup failed: Authorization: Bearer [REDACTED] rejected"); + expect(lines[1]).toBe('gateway startup failed: api_key="[REDACTED] invalid'); + // Terminal escape sequences are stripped so sandbox output cannot forge + // operator-terminal content. + expect(lines[2]).toBe("gateway startup failed: [31mboom[0m Process will stay alive"); + expect(lines[2]).not.toContain("\u001b"); + }); +}); + +describe("sanitizeWedgeLogLine", () => { + it("redacts nvapi keys and token assignments", () => { + expect(sanitizeWedgeLogLine("auth with nvapi-AbC123xyz failed")).toBe( + "auth with [REDACTED] failed", + ); + expect(sanitizeWedgeLogLine("retry token=sk-live-456 now")).toBe("retry token=[REDACTED] now"); + expect(sanitizeWedgeLogLine("PASSWORD: hunter2 rejected")).toBe( + "PASSWORD: [REDACTED] rejected", + ); + }); + + it("leaves ordinary wedge lines untouched", () => { + const line = + "gateway startup failed: listen EADDRINUSE. Process will stay alive; fix the issue and restart."; + expect(sanitizeWedgeLogLine(line)).toBe(line); + }); +}); diff --git a/src/lib/actions/sandbox/gateway-wedge-diagnostics.ts b/src/lib/actions/sandbox/gateway-wedge-diagnostics.ts new file mode 100644 index 00000000000..168827b1e3a --- /dev/null +++ b/src/lib/actions/sandbox/gateway-wedge-diagnostics.ts @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// #4710 wedge diagnostics — source-of-truth contract: +// +// Invalid state: the in-sandbox OpenClaw gateway performs a self-initiated +// in-process restart on restart-class config changes; in containers a failed +// restart parks the process alive with its HTTP listener closed, logging +// "gateway startup failed: ... Process will stay alive; fix the issue and +// restart." to /tmp/gateway.log. +// Source boundary: that park-alive behavior lives in OpenClaw's gateway run +// loop, outside NemoClaw; NemoClaw can only detect it and hand recovery back +// to its supervisor. The sandbox-side prevention (gateway.reload.mode=hot pin +// and the serving watchdog) ships separately in the #4710 sandbox PR. +// Removal condition: when sandbox images pin an OpenClaw release whose failed +// in-process restart exits non-zero (so the PID-wait supervisor respawns it), +// this detection can be narrowed and the recovery settle window shortened or +// defaulted off. + +import { shellQuote } from "../../runner"; +import type { SandboxCommandResult } from "./process-recovery"; + +export type SandboxExec = (sandboxName: string, command: string) => SandboxCommandResult | null; + +const WEDGE_LOG_SIGNATURE = + "config change requires gateway restart|gateway startup failed|Process will stay alive"; + +// The matched lines come from a sandbox-writable log, so they are untrusted: +// strip terminal control characters (no escape-sequence forgery in operator +// terminals) and redact common credential shapes before printing. +const CONTROL_CHARS_RE = new RegExp("[\\u0000-\\u0008\\u000b-\\u001f\\u007f-\\u009f]", "g"); +const SECRET_PATTERNS: RegExp[] = [ + /\b(authorization\s*:\s*bearer)\s+\S+/gi, + /\b(api[-_]?key|token|secret|password)(["']?\s*[=:]\s*["']?)\S+/gi, + /\bnvapi-\S+/gi, +]; + +export function sanitizeWedgeLogLine(line: string): string { + let sanitized = line.replace(CONTROL_CHARS_RE, ""); + sanitized = sanitized.replace(SECRET_PATTERNS[0], "$1 [REDACTED]"); + sanitized = sanitized.replace(SECRET_PATTERNS[1], "$1$2[REDACTED]"); + sanitized = sanitized.replace(SECRET_PATTERNS[2], "[REDACTED]"); + return sanitized.trim(); +} + +/** + * Collect the #4710 wedge signature from the sandbox gateway log: the + * sequence a self-initiated in-process gateway restart leaves behind when it + * closes the HTTP listener and then fails, parking the process alive. + * Returns up to the last five matching lines (sanitized), or [] when none + * match or the log cannot be read. + */ +export function collectGatewayWedgeDiagnostics(sandboxName: string, exec: SandboxExec): string[] { + const command = `grep -E ${shellQuote(WEDGE_LOG_SIGNATURE)} /tmp/gateway.log 2>/dev/null | tail -5`; + const result = exec(sandboxName, command); + if (!result || result.status !== 0) { + return []; + } + return result.stdout.split("\n").map(sanitizeWedgeLogLine).filter(Boolean); +} + +/** + * Print the #4710 wedge signature (if present) to stderr so the operator + * sees why the gateway is unreachable despite a live process. Returns true + * when signature lines were found and printed. + */ +export function printGatewayWedgeDiagnostics(sandboxName: string, exec: SandboxExec): boolean { + const wedgeLines = collectGatewayWedgeDiagnostics(sandboxName, exec); + if (wedgeLines.length === 0) { + return false; + } + console.error( + " The gateway served briefly and then dropped its HTTP listener (#4710 wedge signature):", + ); + for (const line of wedgeLines) { + console.error(` ${line}`); + } + return true; +} diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 49282e146b6..d0003b4cb12 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -1,10 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, it, expect } from "vitest"; +import { afterEach, describe, it, expect } from "vitest"; // Import from compiled dist for parity with the other CLI tests in this project. -import { probeSandboxInferenceGatewayHealth } from "../../../../dist/lib/actions/sandbox/process-recovery"; +import { + probeSandboxInferenceGatewayHealth, + waitForRecoveredSandboxGateway, +} from "../../../../dist/lib/actions/sandbox/process-recovery"; describe("probeSandboxInferenceGatewayHealth — #3265 gateway-chain subprobe", () => { const makeExec = @@ -54,3 +57,82 @@ describe("probeSandboxInferenceGatewayHealth — #3265 gateway-chain subprobe", expect(result).toBeNull(); }); }); + +describe("waitForRecoveredSandboxGateway — #4710 settle-window confirm", () => { + const ENV_KEYS = [ + "NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", + "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", + "NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", + ]; + const saved = new Map(ENV_KEYS.map((key) => [key, process.env[key]])); + + afterEach(() => { + for (const key of ENV_KEYS) { + const value = saved.get(key); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + + // A probe whose answers play out in order; the last answer repeats. + const makeProbe = (answers: Array) => { + const remaining = [...answers]; + return () => (remaining.length > 1 ? remaining.shift() : remaining[0]) ?? null; + }; + + it("confirms the gateway is still serving after the settle window", () => { + const sleeps: number[] = []; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + probeImpl: makeProbe([true, true]), + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(true); + // Default settle window of 25s between the two probes. + expect(sleeps).toEqual([25]); + }); + + it("fails recovery when the gateway serves once and then drops its listener (wedge)", () => { + const sleeps: number[] = []; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + probeImpl: makeProbe([true, false]), + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(false); + expect(sleeps).toEqual([25]); + }); + + it("skips the settle confirm when NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS=0", () => { + process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "0"; + const sleeps: number[] = []; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + // A second probe would report the wedge; with the settle disabled the + // first success must win and no second probe may run. + probeImpl: makeProbe([true, false]), + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(true); + expect(sleeps).toEqual([]); + }); + + it("still polls through initial failures before reaching the settle confirm", () => { + process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "5"; + const sleeps: number[] = []; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + probeImpl: makeProbe([false, false, true, true]), + sleepImpl: (seconds: number) => sleeps.push(seconds), + }); + expect(ok).toBe(true); + // Two poll intervals (default 3s) before the first success, then the + // settle window. + expect(sleeps).toEqual([3, 3, 5]); + }); + + it("returns false when the gateway never serves within the wait budget", () => { + process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS = "0"; + const ok = waitForRecoveredSandboxGateway("my-sandbox", { + probeImpl: makeProbe([false]), + sleepImpl: () => {}, + }); + expect(ok).toBe(false); + }); +}); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 5cae60baafd..ffdfa4728fc 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -22,6 +22,7 @@ import { ROOT, shellQuote } from "../../runner"; import * as registry from "../../state/registry"; import { parseForwardList } from "../../state/sandbox-session"; import { classifyForwardHealthWithReachability, isLocalForwardReachable } from "./forward-health"; +import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; import { ensureHermesDashboardPortForwardIfEnabled as ensureHermesDashboardPortForward, getHermesDashboardRecoveryConfig, @@ -310,7 +311,16 @@ function readNonNegativeNumberEnv(name: string, fallback: number): number { return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; } -function waitForRecoveredSandboxGateway(sandboxName: string): boolean { +export function waitForRecoveredSandboxGateway( + sandboxName: string, + options: { + probeImpl?: (sandboxName: string) => boolean | null; + sleepImpl?: (seconds: number) => void; + quiet?: boolean; + } = {}, +): boolean { + const probe = options.probeImpl ?? isSandboxGatewayRunning; + const sleep = options.sleepImpl ?? sleepSeconds; const timeoutSeconds = readNonNegativeNumberEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", 30); const intervalSeconds = readNonNegativeNumberEnv( "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", @@ -321,13 +331,32 @@ function waitForRecoveredSandboxGateway(sandboxName: string): boolean { ? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1) : Math.max(1, Math.floor(timeoutSeconds) + 1); - return waitUntil(() => isSandboxGatewayRunning(sandboxName) === true, { + const recovered = waitUntil(() => probe(sandboxName) === true, { initialIntervalMs: intervalSeconds * 1000, maxIntervalMs: intervalSeconds * 1000, backoffFactor: 1, maxAttempts: attempts, - sleep: (ms) => sleepSeconds(ms / 1000), + sleep: (ms) => sleep(ms / 1000), }); + if (!recovered) return false; + + // #4710: a freshly relaunched gateway can serve for ~20s and then drop + // its HTTP listener while the process stays alive (a failed in-process + // restart triggered by a post-launch config write parks it deaf). One + // successful probe inside that window is not proof of recovery — wait + // out a settle window and require the gateway to still be serving. + // 0 disables the settle confirm. + // Source boundary and removal condition for this detection live in + // gateway-wedge-diagnostics.ts. + const settleSeconds = readNonNegativeNumberEnv("NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", 25); + if (settleSeconds <= 0) { + return true; + } + if (!options.quiet) { + console.log(` Confirming the gateway stays responsive (~${settleSeconds}s)...`); + } + sleep(settleSeconds); + return probe(sandboxName) === true; } /** @@ -487,9 +516,10 @@ export function checkAndRecoverSandboxProcesses( if (recovered) { // Wait for gateway to bind its HTTP port before declaring success. The // recovered process can be alive before the OpenAI-compatible API is ready. - if (!waitForRecoveredSandboxGateway(sandboxName)) { + if (!waitForRecoveredSandboxGateway(sandboxName, { quiet })) { if (!quiet) { console.error(" Gateway process started but is not responding."); + printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); console.error(" Connect to the sandbox and run manually:"); console.error( diff --git a/test/cli/connect-recovery-settle.test.ts b/test/cli/connect-recovery-settle.test.ts new file mode 100644 index 00000000000..f7188465ea6 --- /dev/null +++ b/test/cli/connect-recovery-settle.test.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// CLI coverage for the #4710 post-recovery settle-confirm: a wedged gateway +// serves the first probe after relaunch and then drops its HTTP listener, so +// `connect --probe-only` must fail and surface the wedge signature instead of +// declaring a recovery that is already dying. Split from +// connect-recovery.test.ts, which is at the default size budget. + +import { describe, it, expect } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { runWithEnv, writeSandboxRegistry } from "./helpers"; + +describe("CLI dispatch", () => { + it("fails probe-only when the gateway serves once and then drops its listener (#4710 wedge)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-wedge-")); + const localBin = path.join(home, "bin"); + const markerFile = path.join(home, "openshell-calls"); + const stateFile = path.join(home, "probe-state"); + const readyCountFile = path.join(home, "ready-count"); + fs.mkdirSync(localBin, { recursive: true }); + writeSandboxRegistry(home); + fs.writeFileSync(stateFile, "stopped"); + fs.writeFileSync( + path.join(localBin, "openshell"), + [ + "#!/usr/bin/env bash", + `marker_file=${JSON.stringify(markerFile)}`, + `state_file=${JSON.stringify(stateFile)}`, + `ready_count_file=${JSON.stringify(readyCountFile)}`, + 'printf \'%s\\n\' "$*" >> "$marker_file"', + 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', + " echo 'Sandbox:'", + " echo", + " echo ' Id: abc'", + " echo ' Name: alpha'", + " echo ' Namespace: openshell'", + " echo ' Phase: Ready'", + " exit 0", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', + ' cmd="$8"', + ' case "$cmd" in', + ' *"OPENCLAW="*)', + ' echo recovered > "$state_file"', + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + " echo 'GATEWAY_PID=123'", + " exit 0", + " ;;", + " *'curl -so'*)", + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + ' if [ "$(cat "$state_file")" != recovered ]; then echo STOPPED; exit 0; fi', + // The wedge shape: the relaunched gateway answers the first + // post-recovery probe, then drops its listener — every later probe + // refuses. + ' count=$(cat "$ready_count_file" 2>/dev/null || echo 0)', + " count=$((count + 1))", + ' echo "$count" > "$ready_count_file"', + ' if [ "$count" -le 1 ]; then echo RUNNING; else echo STOPPED; fi', + " exit 0", + " ;;", + " *'grep -E'*)", + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + " echo '[reload] config change requires gateway restart (plugins.installs)'", + " echo 'gateway startup failed: listen failure. Process will stay alive; fix the issue and restart.'", + " exit 0", + " ;;", + " esac", + "fi", + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + + const r = runWithEnv("alpha connect --probe-only", { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS: "3", + NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS: "0", + NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS: "1", + }); + + expect(r.code).toBe(1); + expect(r.out).toContain( + "Probe failed: OpenClaw gateway is not running in 'alpha' and automatic recovery failed.", + ); + expect(r.out).toContain("#4710 wedge signature"); + expect(r.out).toContain("config change requires gateway restart (plugins.installs)"); + // First probe succeeded, settle confirm observed the dropped listener. + expect(fs.readFileSync(readyCountFile, "utf8").trim()).toBe("2"); + }); +}); diff --git a/test/cli/helpers.ts b/test/cli/helpers.ts index 09ffa7676a5..f2d0126d26e 100644 --- a/test/cli/helpers.ts +++ b/test/cli/helpers.ts @@ -190,6 +190,11 @@ function runWithEnvInternal( HOME: fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-test-")), NEMOCLAW_HEALTH_POLL_COUNT: "1", NEMOCLAW_HEALTH_POLL_INTERVAL: "0", + // #4710: the post-recovery settle-confirm waits 25s by default; CLI + // tests disable it to stay fast. Settle behavior has dedicated + // coverage in process-recovery.test.ts and a targeted CLI test in + // connect-recovery-settle.test.ts that overrides this with a short window. + NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS: "0", ...env, }, }); diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index 971c1c24299..08349f6043f 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createRequire } from "node:module"; import fs from "node:fs"; +import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; @@ -240,10 +240,12 @@ beta 127.0.0.1 18789 12345 running`; beta 127.0.0.1 18789 12345 running`; const previousWaitSeconds = process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS; const previousPollInterval = process.env.NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS; + const previousSettleSeconds = process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS; let healthProbeCalls = 0; process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS = "2"; process.env.NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS = "0"; + process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = "0"; try { vi.spyOn(childProcess, "spawnSync").mockImplementation( @@ -295,6 +297,11 @@ beta 127.0.0.1 18789 12345 running`; } else { process.env.NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS = previousPollInterval; } + if (previousSettleSeconds === undefined) { + delete process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS; + } else { + process.env.NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS = previousSettleSeconds; + } } }); });