diff --git a/docs/manage-sandboxes/lifecycle.mdx b/docs/manage-sandboxes/lifecycle.mdx index e933ecc1d2a..c5de28ce253 100644 --- a/docs/manage-sandboxes/lifecycle.mdx +++ b/docs/manage-sandboxes/lifecycle.mdx @@ -232,6 +232,10 @@ If the gateway is already healthy, `recover` exits after the probe and does not For built-in OpenClaw sandboxes, recovery sends an authenticated request through registry-scoped privileged direct-container control. The request reaches either the root PID 1 supervisor in a direct root-entrypoint container or the installed root controller in an OpenShell-managed container. It does not fall back to ordinary `openshell sandbox exec` or a manual relaunch inside the sandbox. +For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command. +NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check, and attempts to restore it if that proof fails. +The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. +If the gateway still does not recover, for example when the sandbox needs credentials that only re-onboarding can restore, recreate the runtime with `$$nemoclaw rebuild --yes`, or destroy and re-onboard the sandbox. Refer to [`$$nemoclaw recover`](../reference/commands#$$nemoclaw-name-recover) for details. Use `gateway restart` when you intentionally need a supported OpenClaw gateway to reload runtime configuration or plugins. @@ -257,6 +261,10 @@ If the gateway is already healthy, `recover` exits after the probe and does not For built-in Hermes sandboxes, recovery sends an authenticated request through registry-scoped privileged direct-container control. The request reaches either the root PID 1 supervisor in a direct root-entrypoint container or the installed root controller in an OpenShell-managed container. It does not fall back to ordinary `openshell sandbox exec` or a manual relaunch inside the sandbox. +For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command. +NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check, and attempts to restore it if that proof fails. +The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. +If the gateway still does not recover, for example when the sandbox needs credentials that only re-onboarding can restore, recreate the runtime with `$$nemoclaw rebuild --yes`, or destroy and re-onboard the sandbox. Refer to [`$$nemoclaw recover`](../reference/commands#$$nemoclaw-name-recover) for details. Use `gateway restart` when you intentionally need a supported Hermes gateway to reload runtime configuration or plugins. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 76a1f2a2bc8..6a4fbe42dd7 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1033,6 +1033,9 @@ The host selects the controller from the live container topology. In a direct root-entrypoint container, the request reaches the root PID 1 supervisor. In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1. It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback. +When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation. +The recreation uses a credential-free managed startup command, pins the registered container identity, retains the previous container for rollback, and commits only after managed gateway health and the settle check pass. +The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. It is idempotent. When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it retries only when stdout is empty and stderr is exactly one `SUPERVISOR_BUSY` or `SUPERVISOR_UNAVAILABLE` line, with at most three controller attempts. Other controller failures stop immediately. @@ -1107,7 +1110,7 @@ Run `$$nemoclaw shields down` before a Hermes config or inference change; -The command can fail at these layers: unsupported agent, privileged control unavailable, secret-boundary refusal, unsafe config path, config hash mismatch when a strict hash is available, launch failure, health timeout, or forward recovery failure. +The command can fail at these layers: unsupported agent, privileged control unavailable, supervisor not running, secret-boundary refusal, unsafe config path, config hash mismatch when a strict hash is available, launch failure, health timeout, or forward recovery failure. An older direct-container image without the matching supervisor or managed controller helper reports `privileged control unavailable` and requires `$$nemoclaw rebuild --yes`. Ordinary OpenShell exec and manual in-sandbox relaunch are not fallback paths. Terminal agents do not have a gateway runtime and fail as unsupported. @@ -3135,7 +3138,9 @@ Keep the OpenShell gateway on loopback and use `NEMOCLAW_DASHBOARD_BIND` when yo `NEMOCLAW_DASHBOARD_BIND` controls the dashboard or API port forward bind address. Outside WSL, the forward stays on `127.0.0.1` (loopback only) by default. On WSL, NemoClaw binds the host-side forward on all interfaces so the Windows host can reach it, while the ready summary continues to print a loopback dashboard URL. -Set `NEMOCLAW_DASHBOARD_BIND=0.0.0.0` before `$$nemoclaw onboard` to prepare the sandbox for remote exposure and bind the forward on all interfaces. Use the same setting for later `$$nemoclaw connect` calls. A sandbox created without this opt-in must be recreated with `NEMOCLAW_DASHBOARD_BIND=0.0.0.0 $$nemoclaw onboard --recreate-sandbox` before a remote-bind connect is allowed. +On non-WSL SSH-deployed hosts, set `NEMOCLAW_DASHBOARD_BIND=0.0.0.0` before `$$nemoclaw onboard` to prepare the sandbox for remote exposure and bind the forward on all interfaces. +Use the same setting for later `$$nemoclaw connect` calls. +A sandbox created without this opt-in must be recreated with `NEMOCLAW_DASHBOARD_BIND=0.0.0.0 $$nemoclaw onboard --recreate-sandbox` before a remote-bind connect is allowed. Only `0.0.0.0` enables the remote bind; onboarding rejects any other non-empty value. @@ -3509,6 +3514,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_CLEANUP_GATEWAY` | `1`, `true`, or `yes` to enable; `0`, `false`, or `no` to disable | Overrides the platform default (macOS unattended: cleanup; Linux/Windows: preserve) for whether `$$nemoclaw destroy` removes the shared gateway when destroying the last sandbox. Command-line `--cleanup-gateway` and `--no-cleanup-gateway` still take precedence. | | `NEMOCLAW_CONFIRM_LEGACY_MANAGED_RECREATE` | Exact JSON array of sandbox names | Confirms to the installer that the exact listed set of pre-fingerprint OpenClaw or Hermes sandboxes used NemoClaw-managed images, allowing recovery onto the current managed image. The normalized names must exactly match the installer's printed array. Set it only after verifying every named sandbox. Recorded custom-image evidence remains blocked. | | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw connect` and `$$nemoclaw connect --probe-only`. Use only as a troubleshooting escape hatch. | +| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | | `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index bcc98013e5f..17953d27bb3 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -931,8 +931,11 @@ $$nemoclaw status When `recover` repairs a stopped built-in gateway, NemoClaw automatically retries only when stdout is empty and stderr is exactly one `SUPERVISOR_BUSY` or `SUPERVISOR_UNAVAILABLE` line, with at most three controller attempts. It does not retry other status or output combinations. +`SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop. +On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health and settle checks pass. +To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If that bounded retry is exhausted, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. -If the error mentions `SUPERVISOR_REBUILD_REQUIRED`, `SUPERVISOR_UNAVAILABLE`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. +If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, `SUPERVISOR_UNAVAILABLE`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. The managed controller also refuses an ambiguous or changing process tree instead of guessing which same-UID process is the gateway. Rebuild the image before retrying: diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index 72bdee38296..e402af6bc43 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -748,18 +748,20 @@ def _sandbox_uid() -> int: raise ControlError("SUPERVISOR_UNAVAILABLE") from exc -def _discover_supervisor(reader: ProcReader) -> ProcessIdentity: - pid1 = reader.capture(1) - if not _is_openshell(pid1): - raise ControlError("SUPERVISOR_UNAVAILABLE") - sandbox_uid = _sandbox_uid() +def _supervisor_candidates( + reader: ProcReader, pid1: ProcessIdentity, sandbox_uid: int +) -> tuple[list[ProcessIdentity], bool]: matches: list[ProcessIdentity] = [] + inconclusive = False for pid in reader.pids(): if pid == 1: continue try: identity = reader.capture(pid) - except (ControlError, FileNotFoundError, ProcessLookupError, PermissionError): + except (FileNotFoundError, ProcessLookupError): + continue + except (ControlError, PermissionError): + inconclusive = True continue if ( _is_nemoclaw_start(identity, sandbox_uid) @@ -771,6 +773,37 @@ def _discover_supervisor(reader: ProcReader) -> ProcessIdentity: matches.append(identity) if len(matches) > 1: break + return matches, inconclusive + + +def _discover_supervisor(reader: ProcReader) -> ProcessIdentity: + pid1 = reader.capture(1) + if not _is_openshell(pid1): + raise ControlError("SUPERVISOR_UNAVAILABLE") + sandbox_uid = _sandbox_uid() + matches, inconclusive = _supervisor_candidates(reader, pid1, sandbox_uid) + if inconclusive: + raise ControlError("SUPERVISOR_UNAVAILABLE") + if len(matches) == 0: + # A zero-match scan is the only absence signal that may authorize the + # host to recreate a legacy Docker container with its managed startup + # command. Re-scan the complete process table and pin PID 1 around both + # observations so ambiguity, process churn, and supervisor startup + # races remain generic unavailability rather than destructive-recovery + # authorization. + between_pid1 = reader.capture(1) + second_matches, second_inconclusive = _supervisor_candidates( + reader, pid1, sandbox_uid + ) + after_pid1 = reader.capture(1) + if ( + between_pid1.stable_key() == pid1.stable_key() + and after_pid1.stable_key() == pid1.stable_key() + and not second_inconclusive + and len(second_matches) == 0 + ): + raise ControlError("SUPERVISOR_NOT_RUNNING") + raise ControlError("SUPERVISOR_UNAVAILABLE") if len(matches) != 1: raise ControlError("SUPERVISOR_UNAVAILABLE") current_pid1 = reader.capture(1) diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index 6d296ab78d8..29df32bba6f 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -32,6 +32,11 @@ type SandboxPortDeps = { getSessionAgent?: (sandboxName?: string) => SandboxPortAgent; }; +type SandboxForwardRecoveryOptions = { + afterSuccess?: () => boolean; + beforeStart?: () => boolean; +}; + function isValidPort(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 65535; } @@ -62,7 +67,10 @@ export function resolveSandboxDashboardPort( * Returns true when `forward start` succeeded and a follow-up probe * confirms the new entry is running, false otherwise. */ -export function ensureSandboxPortForward(sandboxName: string): boolean { +export function ensureSandboxPortForward( + sandboxName: string, + options: SandboxForwardRecoveryOptions = {}, +): boolean { const port = resolveSandboxDashboardPort(sandboxName); const remoteBindRequested = isRemoteDashboardBindRequested(process.env.NEMOCLAW_DASHBOARD_BIND); const allInterfaceBindRequired = remoteBindRequested || isWsl(); @@ -79,9 +87,11 @@ export function ensureSandboxPortForward(sandboxName: string): boolean { forwardTarget: allInterfaceBindRequired ? `0.0.0.0:${port}` : String(port), forceRestart: remoteBindRequested, expectedBind: allInterfaceBindRequired ? "0.0.0.0" : "127.0.0.1", - beforeStart: remoteBindRequested - ? () => registry.getSandbox(sandboxName)?.dashboardRemoteBindPrepared === true - : undefined, + afterSuccess: options.afterSuccess, + beforeStart: () => + (!remoteBindRequested || + registry.getSandbox(sandboxName)?.dashboardRemoteBindPrepared === true) && + (options.beforeStart?.() ?? true), }); } @@ -133,6 +143,7 @@ export function ensureSandboxPortForwardForPort( sandboxName: string, port: number, options: { + afterSuccess?: () => boolean; forwardTarget?: string; forceRestart?: boolean; expectedBind?: string; @@ -140,13 +151,28 @@ export function ensureSandboxPortForwardForPort( } = {}, ): boolean { const { + afterSuccess = () => true, forwardTarget = String(port), forceRestart = false, expectedBind, beforeStart = () => true, } = options; + const acceptSuccessfulForward = () => { + let accepted = false; + try { + accepted = afterSuccess(); + } catch { + accepted = false; + } + if (accepted) return true; + runOpenshell(["forward", "stop", String(port), sandboxName], { + ignoreError: true, + stdio: "ignore", + }); + return false; + }; let forwardHealth = isSandboxPortForwardHealthy(sandboxName, port, expectedBind); - if (forwardHealth === true && !forceRestart) return true; + if (forwardHealth === true && !forceRestart) return acceptSuccessfulForward(); if (forwardHealth === "occupied") return false; const configuredWaitMs = Number(process.env.NEMOCLAW_FORWARD_RECOVERY_WAIT_MS ?? "3000"); const waitMs = Number.isFinite(configuredWaitMs) ? Math.max(0, configuredWaitMs) : 3000; @@ -195,7 +221,7 @@ export function ensureSandboxPortForwardForPort( backoffFactor: 1.5, }, ); - if (stopState.health === true && !forceRestart) return true; + if (stopState.health === true && !forceRestart) return acceptSuccessfulForward(); if (stopState.health === "occupied" || !stopSettled || !stopState.portReleased) return false; } @@ -204,6 +230,11 @@ export function ensureSandboxPortForwardForPort( ["forward", "start", "--background", forwardTarget, sandboxName], { ignoreError: true, + // OpenShell 0.0.72 leaves the background SSH forward attached to the + // caller's inherited descriptors. Detach them so a scripted `recover` + // can finish after the foreground OpenShell command exits. Keep this + // until every supported OpenShell release redirects those descriptors. + stdio: "ignore", }, ); if (startResult.status !== 0) return false; @@ -213,7 +244,7 @@ export function ensureSandboxPortForwardForPort( // of accepting an arbitrary reachable listener or failing on the first // metadata refresh. let health = isSandboxPortForwardHealthy(sandboxName, port, expectedBind); - if (health === true) return true; + if (health === true) return acceptSuccessfulForward(); if (health === "occupied") return false; if (waitMs === 0) return false; @@ -234,7 +265,7 @@ export function ensureSandboxPortForwardForPort( backoffFactor: 1.5, }, ); - return settled && !occupied; + return settled && !occupied && acceptSuccessfulForward(); } export function ensureHermesDashboardPortForwardIfEnabled(sandboxName: string): boolean | null { diff --git a/src/lib/actions/sandbox/gateway-restart.test.ts b/src/lib/actions/sandbox/gateway-restart.test.ts index e41540c18d7..db05d0400ff 100644 --- a/src/lib/actions/sandbox/gateway-restart.test.ts +++ b/src/lib/actions/sandbox/gateway-restart.test.ts @@ -18,6 +18,8 @@ describe("gateway restart failure markers", () => { [string, ReturnType["layer"]] > = [ ["PRIVILEGED_CONTROL_UNAVAILABLE", "privileged control unavailable"], + ["SUPERVISOR_UNAVAILABLE", "privileged control unavailable"], + ["SUPERVISOR_NOT_RUNNING", "supervisor not running"], ["SUPERVISOR_REBUILD_REQUIRED", "privileged control unavailable"], ["SUPERVISOR_BUSY", "privileged control unavailable"], [MARKERS.SECRET_BOUNDARY_REFUSED, "secret-boundary refusal"], diff --git a/src/lib/actions/sandbox/gateway-restart.ts b/src/lib/actions/sandbox/gateway-restart.ts index 529b5922978..54c9ef4ff03 100644 --- a/src/lib/actions/sandbox/gateway-restart.ts +++ b/src/lib/actions/sandbox/gateway-restart.ts @@ -17,6 +17,7 @@ export type GatewayRestartCommandResult = { export type GatewayRestartFailureLayer = | "unsupported agent" | "privileged control unavailable" + | "supervisor not running" | "secret-boundary refusal" | "unsafe config path" | "config hash mismatch" @@ -127,6 +128,12 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul const output = gatewayRestartOutput(result); const detail = sanitizeGatewayRestartFailureDetail(output.trim()); + if (output.includes("SUPERVISOR_NOT_RUNNING")) { + return { + layer: "supervisor not running", + detail: detail || "the in-sandbox gateway supervisor is not running", + }; + } if ( output.includes(MARKERS.ROOT_EXEC_UNAVAILABLE) || output.includes("PRIVILEGED_CONTROL_UNAVAILABLE") || diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 8140d3656d0..96bba814c93 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -7,8 +7,120 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { confirmRecoveredSandboxGatewayManaged, waitForRecoveredSandboxGateway, + waitForRecreatedSandboxOpenShellReady, } from "./process-recovery"; +const OPENSHELL_SANDBOX_NOT_READY_STDERR = `Error: × code: 'The system is not in a state required for the operation's + │ execution', message: "sandbox is not ready" +`; + +describe("recreated sandbox OpenShell readiness", () => { + it("retries only the structured not-ready state until OpenShell accepts the sandbox", () => { + const notReady = { + status: 1, + output: OPENSHELL_SANDBOX_NOT_READY_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_SANDBOX_NOT_READY_STDERR, + }; + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce(notReady) + .mockReturnValueOnce(notReady) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const beforeProbe = vi.fn(() => true); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 6, + }), + ).toBe(true); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(3); + expect(captureOpenshellImpl).toHaveBeenCalledWith( + ["sandbox", "exec", "--name", "recreated-box", "--", "true"], + expect.objectContaining({ + ignoreError: true, + includeStderr: true, + includeStreams: true, + }), + ); + expect(beforeProbe).toHaveBeenCalledTimes(3); + expect(sleeps).toEqual([3, 3]); + }); + + it("fails immediately on an unknown OpenShell error", () => { + const captureOpenshellImpl = vi.fn(() => ({ + status: 1, + output: "permission denied", + stdout: "", + stderr: "permission denied", + })); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(false); + expect(captureOpenshellImpl).toHaveBeenCalledOnce(); + expect(sleeps).toEqual([]); + }); + + it("does not retry an outcome-uncertain OpenShell timeout", () => { + const timeoutError = Object.assign(new Error("timed out"), { code: "ETIMEDOUT" }); + const captureOpenshellImpl = vi.fn(() => ({ + status: null, + output: "", + stdout: "", + stderr: "", + error: timeoutError, + })); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(false); + expect(captureOpenshellImpl).toHaveBeenCalledOnce(); + expect(sleeps).toEqual([]); + }); + + it("rechecks the pinned managed guard before every readiness retry", () => { + const captureOpenshellImpl = vi.fn(() => ({ + status: 1, + output: OPENSHELL_SANDBOX_NOT_READY_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_SANDBOX_NOT_READY_STDERR, + })); + const beforeProbe = vi.fn().mockReturnValueOnce(true).mockReturnValue(false); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 6, + }), + ).toBe(false); + expect(beforeProbe).toHaveBeenCalledTimes(2); + expect(captureOpenshellImpl).toHaveBeenCalledOnce(); + expect(sleeps).toEqual([3]); + }); +}); + describe("confirmRecoveredSandboxGatewayManaged scope", () => { const requestGatewaySupervisorAction = vi.fn(() => ({ status: 0, diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 7de6b5abe4a..caf6138a196 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -4,7 +4,9 @@ import { spawnSync } from "node:child_process"; import { randomBytes } from "node:crypto"; import { dockerSpawnSync } from "../../adapters/docker"; +import { stripAnsi } from "../../adapters/openshell/client"; import { + captureOpenshell, captureOpenshellForStatus, captureSandboxSshConfig, getOpenshellBinary, @@ -34,6 +36,7 @@ import { import { classifyGatewayRestartFailure, type GatewayRestartDeps, + type GatewayRestartFailureLayer, type GatewayRestartResult, printGatewayRestartFailure, type RestartSandboxGatewayOptions, @@ -50,6 +53,10 @@ import { buildSandboxExecMarkedCommand, extractSandboxExecCommandStdout, } from "./sandbox-exec-output"; +import { + type ManagedSupervisorRelaunch, + relaunchManagedSupervisorSession, +} from "./supervisor-relaunch"; export type { SandboxForwardHealth, SandboxForwardListEntry } from "./forward-health"; export { @@ -235,10 +242,11 @@ export function executeSandboxExecCommand( return executeLocalDockerSandboxCommand(sandboxName, markedCommand, effectiveTimeout); } -export function executeGatewaySupervisorAction( +function executeGatewaySupervisorActionPinned( sandboxName: string, action: "restart" | "recover" | "probe", - timeout = 210000, + timeout: number, + expectedContainerId?: string, ): SandboxCommandResult | null { const nonce = randomBytes(32).toString("hex"); let argv: string[]; @@ -248,6 +256,7 @@ export function executeGatewaySupervisorAction( ["/usr/local/bin/nemoclaw-gateway-control", action, nonce], false, true, + expectedContainerId, ); } catch (error) { const detail = error instanceof Error ? error.message : "privileged container unavailable"; @@ -278,6 +287,16 @@ export function executeGatewaySupervisorAction( return { status, stdout, stderr }; } +type RequestPinnedGatewaySupervisorAction = typeof executeGatewaySupervisorActionPinned; + +export function executeGatewaySupervisorAction( + sandboxName: string, + action: "restart" | "recover" | "probe", + timeout = 210000, +): SandboxCommandResult | null { + return executeGatewaySupervisorActionPinned(sandboxName, action, timeout); +} + async function executeSandboxExecCommandForStatus( sandboxName: string, command: string, @@ -353,6 +372,15 @@ function isExactlyRetryableManagedRecoveryFailure(result: SandboxCommandResult | return lines.length === 1 && ["SUPERVISOR_UNAVAILABLE", "SUPERVISOR_BUSY"].includes(lines[0]); } +function isExactlyMissingManagedSupervisor(result: SandboxCommandResult | null): boolean { + if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; + const lines = result.stderr + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + return lines.length === 1 && lines[0] === "SUPERVISOR_NOT_RUNNING"; +} + export function confirmRecoveredSandboxGatewayManaged( sandboxName: string, options: { @@ -392,20 +420,31 @@ export async function isSandboxGatewayRunningForStatus( } /** - * Restart the gateway process inside the sandbox after a pod restart. - * Cleans stale lock/temp files, sources proxy config, and launches the gateway - * in the background. Returns true on success. + * Recover a gateway through the registered agent's managed control boundary. + * Legacy custom agents retain their SSH-owned compatibility path. Built-in + * agents may return a transactional supervisor relaunch that the caller must + * commit or roll back after the managed health gate. */ +type SandboxProcessRecovery = + | { kind: "managed" | "custom" } + | { kind: "relaunched"; relaunch: ManagedSupervisorRelaunch }; + function recoverSandboxProcesses( sandboxName: string, { quiet = false, requestGatewaySupervisorAction = executeGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction = executeGatewaySupervisorActionPinned, + relaunchManagedSupervisorSessionImpl = relaunchManagedSupervisorSession, + onFailureLayer, }: { quiet?: boolean; requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; + requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; + relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; + onFailureLayer?: (layer: GatewayRestartFailureLayer) => void; } = {}, -): "managed" | "custom" | null { +): SandboxProcessRecovery | null { const agent = agentRuntime.getSessionAgent(sandboxName); const dashboardPort = resolveSandboxDashboardPort(sandboxName); let persistedAgent: string | null; @@ -419,9 +458,9 @@ function recoverSandboxProcesses( quiet || printGatewayRestartFailure(sandboxName, "unsupported agent", detail); return null; } - const recoveredSsh = (result: SandboxCommandResult | null) => - !!(result && result.status === 0 && hasGatewayRecoveryMarker(result)); - const recoverManagedGateway = (): boolean => { + const recoveredSsh = (result: SandboxCommandResult | null): SandboxProcessRecovery | null => + result && result.status === 0 && hasGatewayRecoveryMarker(result) ? { kind: "custom" } : null; + const recoverManagedGateway = (): SandboxProcessRecovery | null => { const maxAttempts = 3; const retryIntervalSeconds = readNonNegativeNumberEnv( "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", @@ -430,7 +469,7 @@ function recoverSandboxProcesses( let execResult: SandboxCommandResult | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { execResult = requestGatewaySupervisorAction(sandboxName, "recover"); - if (hasGatewayRecoveryMarker(execResult)) return true; + if (hasGatewayRecoveryMarker(execResult)) return { kind: "managed" }; // PID 1 may replace the gateway between the host's stopped observation // and the controller's process-tree capture. Retry only exact transient @@ -439,8 +478,24 @@ function recoverSandboxProcesses( sleepSeconds(retryIntervalSeconds); } const failure = classifyGatewayRestartFailure(execResult); + onFailureLayer?.(failure.layer); + if ( + failure.layer === "supervisor not running" && + isExactlyMissingManagedSupervisor(execResult) + ) { + const relaunch = relaunchManagedSupervisorSessionImpl(sandboxName, { + quiet, + deps: { + confirmMissingSupervisor: (containerId) => + isExactlyMissingManagedSupervisor( + requestPinnedGatewaySupervisorAction(sandboxName, "probe", 210000, containerId), + ), + }, + }); + if (relaunch) return { kind: "relaunched", relaunch }; + } if (!quiet) printGatewayRestartFailure(sandboxName, failure.layer, failure.detail); - return false; + return null; }; if (persistedAgent === "hermes") { if (!isHermesAgent(agent)) { @@ -448,7 +503,7 @@ function recoverSandboxProcesses( if (!quiet) printGatewayRestartFailure(sandboxName, "unsupported agent", detail); return null; } - return recoverManagedGateway() ? "managed" : null; + return recoverManagedGateway(); } // A persisted non-OpenClaw runtime whose manifest cannot be loaded is not @@ -463,7 +518,7 @@ function recoverSandboxProcesses( } if ((!persistedAgent || persistedAgent === "openclaw") && (!agent || agent.name === "openclaw")) { - return recoverManagedGateway() ? "managed" : null; + return recoverManagedGateway(); } const agentScript = agentRuntime.buildRecoveryScript(agent, dashboardPort); @@ -472,7 +527,7 @@ function recoverSandboxProcesses( // Non-Hermes custom manifests do not yet declare a supported host-side // runtime user. Recover them over SSH so the launch inherits the sandbox // login user instead of creating root-owned agent state under /sandbox. - return recoveredSsh(executeSandboxCommand(sandboxName, agentScript)) ? "custom" : null; + return recoveredSsh(executeSandboxCommand(sandboxName, agentScript)); } return null; @@ -521,6 +576,80 @@ function readNonNegativeNumberEnv(name: string, fallback: number): number { return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; } +const OPENSHELL_SANDBOX_NOT_READY = `Error: code: 'The system is not in a state required for the operation's execution', message: "sandbox is not ready"`; + +function normalizeOpenshellStructuredError(value: string): string { + return stripAnsi(value).replace(/[×│]/gu, " ").replace(/\s+/gu, " ").trim(); +} + +function isExactlyRetryableOpenshellSandboxNotReady( + result: ReturnType, +): boolean { + return ( + result.status === 1 && + !result.error && + String(result.stdout ?? "").trim() === "" && + normalizeOpenshellStructuredError(String(result.stderr ?? "")) === OPENSHELL_SANDBOX_NOT_READY + ); +} + +/** + * Wait until OpenShell has re-registered a directly recreated sandbox as + * ready. This probe deliberately has no direct-Docker or SSH fallback: it is + * proving control-plane readiness, not authorizing the already completed + * replacement-container recovery. + */ +export function waitForRecreatedSandboxOpenShellReady( + sandboxName: string, + options: { + captureOpenshellImpl?: typeof captureOpenshell; + beforeProbe?: (timeoutMs: number) => boolean; + intervalSeconds?: number; + nowImpl?: () => number; + sleepImpl?: (seconds: number) => void; + timeoutSeconds?: number; + } = {}, +): boolean { + const capture = options.captureOpenshellImpl ?? captureOpenshell; + const now = options.nowImpl ?? Date.now; + const sleep = options.sleepImpl ?? sleepSeconds; + const timeoutSeconds = readNonNegativeNumberEnv( + "NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", + options.timeoutSeconds ?? 30, + ); + const intervalSeconds = readNonNegativeNumberEnv( + "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", + options.intervalSeconds ?? 3, + ); + const deadlineMs = now() + timeoutSeconds * 1000; + const maxAttempts = + intervalSeconds > 0 + ? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1) + : Math.max(1, Math.floor(timeoutSeconds) + 1); + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const preGuardRemainingMs = deadlineMs - now(); + if (attempt > 1 && preGuardRemainingMs <= 0) return false; + const guardBudgetMs = Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, preGuardRemainingMs)); + if (options.beforeProbe?.(guardBudgetMs) === false) return false; + const remainingMs = deadlineMs - now(); + if (attempt > 1 && remainingMs <= 0) return false; + const result = capture(["sandbox", "exec", "--name", sandboxName, "--", "true"], { + ignoreError: true, + includeStderr: true, + includeStreams: true, + timeout: Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, remainingMs)), + }); + if (result.status === 0 && !result.error) return true; + if (!isExactlyRetryableOpenshellSandboxNotReady(result)) return false; + if (attempt === maxAttempts) return false; + const postProbeRemainingMs = deadlineMs - now(); + if (postProbeRemainingMs <= 0) return false; + sleep(Math.min(intervalSeconds * 1000, postProbeRemainingMs) / 1000); + } + return false; +} + function gatewayRecoveryTimeoutSeconds( agent: ReturnType, ): number { @@ -535,8 +664,17 @@ function gatewayRecoveryTimeoutSeconds( function printHostManagedGatewayRecoveryHints( sandboxName: string, agent: ReturnType, + failureLayer: GatewayRestartFailureLayer | null = null, ): void { const quotedSandboxName = shellQuote(sandboxName); + if (failureLayer === "supervisor not running") { + console.error(" The in-sandbox supervisor is not running, and trusted container recovery"); + console.error(" could not restore a managed supervisor and healthy gateway."); + console.error(" Recreate the sandbox runtime to restore it:"); + console.error(` nemoclaw ${quotedSandboxName} rebuild --yes`); + console.error(" If rebuild is blocked, destroy and re-onboard the sandbox to restore it."); + return; + } let agentName = agent?.name ?? null; if (!agentName) { try { @@ -581,6 +719,7 @@ export function waitForRecoveredSandboxGateway( sleepImpl?: (seconds: number) => void; quiet?: boolean; timeoutSeconds?: number; + requireManagedProbe?: boolean; } = {}, ): boolean { const probe = options.probeImpl ?? isSandboxGatewayRunning; @@ -609,6 +748,7 @@ export function waitForRecoveredSandboxGateway( const probeDuringRecoveryWait = () => { const managedResult = managedProbe?.(sandboxName) ?? null; if (managedResult !== null) return managedResult; + if (options.requireManagedProbe) return false; return probe(sandboxName); }; @@ -684,9 +824,17 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( { quiet = false, requestGatewaySupervisorAction = executeGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction = executeGatewaySupervisorActionPinned, + relaunchManagedSupervisorSessionImpl = relaunchManagedSupervisorSession, + isSandboxGatewayRunningImpl = isSandboxGatewayRunning, + waitForRecreatedSandboxOpenShellReadyImpl = waitForRecreatedSandboxOpenShellReady, }: { quiet?: boolean; requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; + requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; + relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; + isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; + waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; } = {}, ) { const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); @@ -700,7 +848,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( runtime: "terminal" as const, }; } - const running = isSandboxGatewayRunning(sandboxName); + const running = isSandboxGatewayRunningImpl(sandboxName); if (running === null) { return { checked: false, wasRunning: null, recovered: false, forwardRecovered: false }; } @@ -848,35 +996,138 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( console.log(" Recovering..."); } - const recoveryKind = recoverSandboxProcesses(sandboxName, { + let managedRecoveryFailureLayer: GatewayRestartFailureLayer | null = null; + const recovery = recoverSandboxProcesses(sandboxName, { quiet, requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + onFailureLayer: (layer) => { + managedRecoveryFailureLayer = layer; + }, }); - if (recoveryKind !== null) { + if (recovery !== null) { + const relaunch = recovery.kind === "relaunched" ? recovery.relaunch : null; + const requestManagedProbe = relaunch + ? (name: string, action: "restart" | "recover" | "probe", timeout = 210000) => + requestPinnedGatewaySupervisorAction(name, action, timeout, relaunch.containerId) + : requestGatewaySupervisorAction; + let relaunchedIdentityRejected = false; + const confirmRelaunchedManagedHealth = relaunch + ? (timeout = OPENSHELL_PROBE_TIMEOUT_MS) => { + let confirmed = false; + try { + confirmed = + confirmRecoveredSandboxGatewayManaged(sandboxName, { + requestGatewaySupervisorActionImpl: (name, action) => + requestManagedProbe(name, action, timeout), + }) === true; + } catch { + confirmed = false; + } + relaunchedIdentityRejected ||= !confirmed; + return confirmed; + } + : null; // Wait for gateway to bind its HTTP port before declaring success. The // recovered process can be alive before the OpenAI-compatible API is ready. - if ( - !waitForRecoveredSandboxGateway(sandboxName, { + let gatewayReady = false; + try { + gatewayReady = waitForRecoveredSandboxGateway(sandboxName, { quiet, - initialManagedHealthPassed: recoveryKind === "managed", + initialManagedHealthPassed: recovery.kind === "managed", + requireManagedProbe: recovery.kind === "relaunched", timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent), managedProbeImpl: (name) => confirmRecoveredSandboxGatewayManaged(name, { - requestGatewaySupervisorActionImpl: requestGatewaySupervisorAction, + requestGatewaySupervisorActionImpl: requestManagedProbe, }), - }) - ) { + }); + } catch (error) { + try { + relaunch?.finalize(false); + } catch { + // Preserve the original recovery error; the failure path below will + // direct the operator to inspect/rebuild the sandbox. + } + throw error; + } + if (!gatewayReady) { + let rolledBack = true; + if (relaunch) { + try { + rolledBack = relaunch.finalize(false).rolledBack; + } catch { + rolledBack = false; + } + } if (!quiet) { console.error(" Gateway process started but is not responding."); printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); - printHostManagedGatewayRecoveryHints(sandboxName, recoveryAgent); + if (!rolledBack) { + console.error( + " Automatic rollback of the previous sandbox container failed; inspect Docker state before retrying.", + ); + } + printHostManagedGatewayRecoveryHints( + sandboxName, + recoveryAgent, + managedRecoveryFailureLayer, + ); } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } + if (relaunch) { + try { + const completion = relaunch.finalize(true); + if (!completion.backupRemoved && !quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.", + ); + } + } catch { + if (!quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but container transaction cleanup could not be confirmed.", + ); + } + } + } + if ( + relaunch && + !waitForRecreatedSandboxOpenShellReadyImpl(sandboxName, { + beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) === true, + timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent), + }) + ) { + return { + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: + "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started", + }; + } const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, false); if (mcpRefusal) return mcpRefusal; - const forwardRecovered = ensureSandboxPortForward(sandboxName); + const forwardRecovered = ensureSandboxPortForward(sandboxName, { + afterSuccess: confirmRelaunchedManagedHealth ?? undefined, + beforeStart: confirmRelaunchedManagedHealth ?? undefined, + }); + if (!forwardRecovered && relaunchedIdentityRejected) { + return { + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: + "the primary dashboard/API host forward could not be re-established", + }; + } const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { quiet }); const declaredForwardsRecovered = recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { @@ -930,7 +1181,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } if (!quiet) { console.error(` Could not restart ${recoveryDisplayName} gateway automatically.`); - printHostManagedGatewayRecoveryHints(sandboxName, recoveryAgent); + printHostManagedGatewayRecoveryHints(sandboxName, recoveryAgent, managedRecoveryFailureLayer); } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; @@ -941,6 +1192,10 @@ export function checkAndRecoverSandboxProcesses( options: { quiet?: boolean; requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; + requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; + relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; + isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; + waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; } = {}, ) { return withTimerBoundShieldsMutationLock(sandboxName, "gateway process recovery", () => diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts new file mode 100644 index 00000000000..f66776e636d --- /dev/null +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DockerGpuPatchResult } from "../../onboard/docker-gpu-patch"; +import { + type ManagedSupervisorRelaunchDeps, + relaunchManagedSupervisorSession, +} from "./supervisor-relaunch"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +function patchResult(): DockerGpuPatchResult { + return { + applied: true, + oldContainerId: "old-container-id", + newContainerId: "new-container-id", + originalName: "openshell-alpha", + backupContainerName: "openshell-alpha-nemoclaw-backup", + mode: { + kind: "startup-command", + label: "persistent sandbox startup command", + device: "", + args: [], + }, + backupRemoved: false, + }; +} + +function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { + return { + getSandbox: vi.fn(() => ({ + name: "alpha", + agent: "openclaw", + dashboardPort: 18789, + openshellDriver: "docker", + })), + getSessionAgent: vi.fn( + () => + ({ + name: "openclaw", + displayName: "OpenClaw", + forwardPort: 18789, + }) as never, + ), + resolveDashboardPort: vi.fn(() => 18789), + resolveContainer: vi.fn(() => "old-container-id"), + inspectContainer: vi.fn(() => ({ + Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"] }, + })), + confirmMissingSupervisor: vi.fn(() => true), + recreate: vi.fn(() => patchResult()), + finalize: vi.fn(({ supervisorReady }) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ), + ...overrides, + } satisfies ManagedSupervisorRelaunchDeps; +} + +describe("relaunchManagedSupervisorSession", () => { + it("returns null without Docker discovery when the sandbox is not registered", () => { + const deps = baseDeps({ getSandbox: vi.fn(() => null) }); + + expect(relaunchManagedSupervisorSession("missing-box", { quiet: true, deps })).toBeNull(); + expect(deps.resolveContainer).not.toHaveBeenCalled(); + expect(deps.recreate).not.toHaveBeenCalled(); + }); + + it("honors the troubleshooting kill switch without mutating Docker", () => { + vi.stubEnv("NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH", "1"); + const deps = baseDeps(); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + expect(deps.resolveContainer).not.toHaveBeenCalled(); + expect(deps.recreate).not.toHaveBeenCalled(); + }); + + it("refuses a container that no longer has the legacy keepalive startup", () => { + const deps = baseDeps({ + inspectContainer: vi.fn(() => ({ + Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=env nemoclaw-start"] }, + })), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + expect(deps.recreate).not.toHaveBeenCalled(); + }); + + it("refuses recreation when the pinned container no longer proves supervisor absence", () => { + const deps = baseDeps({ confirmMissingSupervisor: vi.fn(() => false) }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("old-container-id"); + expect(deps.recreate).not.toHaveBeenCalled(); + }); + + it("pins the selected container and persists only a credential-free startup command", () => { + vi.stubEnv("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS", "CUSTOM_PROVIDER_CREDENTIAL"); + vi.stubEnv("CUSTOM_PROVIDER_CREDENTIAL", "s3cr3t-token"); + vi.stubEnv("HTTPS_PROXY", "http://proxyuser:proxypass@proxy.example:8080"); + const deps = baseDeps(); + + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch).not.toBeNull(); + expect(relaunch?.containerId).toBe("new-container-id"); + expect(deps.recreate).toHaveBeenCalledOnce(); + const options = vi.mocked(deps.recreate).mock.calls[0]?.[0]; + expect(options).toMatchObject({ + sandboxName: "alpha", + expectedOldContainerId: "old-container-id", + waitForSupervisor: false, + }); + const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; + expect(serialized).toContain("NEMOCLAW_DASHBOARD_PORT=18789"); + expect(serialized).toMatch(/nemoclaw-start$/); + expect(serialized).not.toContain("s3cr3t-token"); + expect(serialized).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); + expect(serialized).not.toContain("proxypass"); + + expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, rolledBack: false }); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ newContainerId: "new-container-id" }), + supervisorReady: true, + }); + }); + + it("rolls the container transaction back when managed readiness is not proven", () => { + const deps = baseDeps(); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(false)).toEqual({ backupRemoved: false, rolledBack: true }); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ backupContainerName: expect.any(String) }), + supervisorReady: false, + }); + }); + + it("returns null when the pinned recreation fails", () => { + const deps = baseDeps({ + recreate: vi.fn(() => { + throw new Error("container identity changed"); + }), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + }); + + it("redacts diagnostics when trusted recreation fails", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const deps = baseDeps({ + recreate: vi.fn(() => { + throw new Error( + "OPENAI_API_KEY=sk-recovery-secret HTTPS_PROXY=http://proxyuser:proxypass@proxy.example:8080", + ); + }), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: false, deps })).toBeNull(); + const output = errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain("OPENAI_API_KEY="); + expect(output).not.toContain("sk-recovery-secret"); + expect(output).not.toContain("proxyuser"); + expect(output).not.toContain("proxypass"); + }); +}); diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts new file mode 100644 index 00000000000..729827fa866 --- /dev/null +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerCapture } from "../../adapters/docker"; +import * as agentRuntime from "../../agent/runtime"; +import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; +import { + type DockerContainerInspect, + parseDockerInspectJson, +} from "../../onboard/docker-gpu-patch"; +import { + type DockerGpuPatchFinalizeOutcome, + finalizeDockerGpuPatchBackup, +} from "../../onboard/docker-gpu-patch-finalize"; +import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; +import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; +import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec"; +import { redact, redactFull } from "../../security/redact"; +import * as registry from "../../state/registry"; +import { resolveSandboxDashboardPort } from "./forward-recovery"; + +/** + * Compatibility boundary for OpenShell 0.0.71's Docker driver: legacy + * sandboxes persist `OPENSHELL_SANDBOX_COMMAND=sleep infinity` while + * `scripts/nemoclaw-start.sh` owns the managed workload as a sibling process. + * Only that inspected value authorizes this migration. Regression coverage is + * named in `supervisor-relaunch.test.ts` and `gateway-guard-recovery.test.ts`. + * Remove this path after supported upgrades rebuild every legacy keepalive + * container with `nemoclaw-start` as its persisted startup command. + */ +const LEGACY_OPENSHELL_KEEPALIVE = "sleep infinity"; +const DOCKER_INSPECT_TIMEOUT_MS = 15000; + +export type ManagedSupervisorRelaunch = { + containerId: string; + finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome; +}; + +export type ManagedSupervisorRelaunchDeps = { + getSandbox?: typeof registry.getSandbox; + getSessionAgent?: typeof agentRuntime.getSessionAgent; + resolveDashboardPort?: typeof resolveSandboxDashboardPort; + resolveContainer?: typeof resolveDirectSandboxContainer; + inspectContainer?: (containerId: string) => DockerContainerInspect; + confirmMissingSupervisor?: (containerId: string) => boolean; + recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand; + finalize?: typeof finalizeDockerGpuPatchBackup; +}; + +function inspectContainer(containerId: string): DockerContainerInspect { + return parseDockerInspectJson( + dockerCapture(["inspect", "--type", "container", containerId], { + ignoreError: true, + timeout: DOCKER_INSPECT_TIMEOUT_MS, + }), + ); +} + +function hasLegacyKeepaliveStartup(inspect: DockerContainerInspect): boolean { + const prefix = "OPENSHELL_SANDBOX_COMMAND="; + const values = (inspect.Config?.Env ?? []) + .filter((entry) => entry.startsWith(prefix)) + .map((entry) => entry.slice(prefix.length)); + return values.length === 1 && values[0] === LEGACY_OPENSHELL_KEEPALIVE; +} + +function reconstructSupervisorLaunchCommand( + sandboxName: string, + entry: NonNullable>, + deps: ManagedSupervisorRelaunchDeps, +): string[] | null { + const getSessionAgent = deps.getSessionAgent ?? agentRuntime.getSessionAgent; + const agent = getSessionAgent(sandboxName) ?? null; + const persistedAgent = entry.agent ?? "openclaw"; + if (!["openclaw", "hermes"].includes(persistedAgent)) return null; + if (persistedAgent === "hermes" && agent?.name !== "hermes") return null; + if (agent && agent.name !== "openclaw" && agent.name !== "hermes") return null; + + const manageDashboard = shouldManageDashboardForAgent(agent); + const resolveDashboardPort = deps.resolveDashboardPort ?? resolveSandboxDashboardPort; + const dashboardPort = String(resolveDashboardPort(sandboxName)); + const chatUiUrl = manageDashboard ? `http://127.0.0.1:${dashboardPort}` : ""; + const hermesDashboardEnabled = entry.hermesDashboardEnabled === true; + const { envArgs } = buildSandboxRuntimeEnvArgs({ + agent, + chatUiUrl, + manageDashboard, + getDashboardForwardPort: () => dashboardPort, + hermesDashboardState: { + enabled: hermesDashboardEnabled, + config: hermesDashboardEnabled + ? { + enabled: true, + port: entry.hermesDashboardPort ?? 0, + internalPort: entry.hermesDashboardInternalPort ?? 0, + tuiEnabled: entry.hermesDashboardTui === true, + } + : null, + }, + extraPlaceholderKeys: [], + observabilityEnabled: entry.observabilityEnabled === true, + sandboxName, + env: process.env, + omitCredentialEnv: true, + }); + return ["env", ...envArgs, "nemoclaw-start"]; +} + +export function relaunchManagedSupervisorSession( + sandboxName: string, + { + quiet, + deps = {}, + }: { + quiet: boolean; + deps?: ManagedSupervisorRelaunchDeps; + }, +): ManagedSupervisorRelaunch | null { + if (process.env.NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH === "1") return null; + const getSandbox = deps.getSandbox ?? registry.getSandbox; + const entry = getSandbox(sandboxName); + if (!entry) return null; + const driver = entry.openshellDriver?.trim().toLowerCase() ?? null; + if (driver !== null && driver !== "docker" && driver !== "vm") return null; + const startupCommand = reconstructSupervisorLaunchCommand(sandboxName, entry, deps); + if (startupCommand === null) return null; + + const resolveContainer = deps.resolveContainer ?? resolveDirectSandboxContainer; + const inspect = deps.inspectContainer ?? inspectContainer; + const confirmMissingSupervisor = deps.confirmMissingSupervisor; + const recreate = deps.recreate ?? recreateOpenShellDockerSandboxWithStartupCommand; + const finalize = deps.finalize ?? finalizeDockerGpuPatchBackup; + try { + const containerId = resolveContainer(sandboxName, driver); + if (!hasLegacyKeepaliveStartup(inspect(containerId))) return null; + if (!confirmMissingSupervisor?.(containerId)) return null; + if (!quiet) { + console.log(" Recreating the sandbox container with its managed startup command..."); + } + const result = recreate({ + sandboxName, + openshellSandboxCommand: startupCommand, + expectedOldContainerId: containerId, + waitForSupervisor: false, + }); + let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null = + null; + return { + containerId: result.newContainerId, + finalize(supervisorReady) { + if (completed) { + if (completed.supervisorReady !== supervisorReady) { + throw new Error( + "Supervisor relaunch transaction was finalized with conflicting state.", + ); + } + return completed.outcome; + } + const outcome = finalize({ result, supervisorReady }); + completed = { supervisorReady, outcome }; + return outcome; + }, + }; + } catch (error) { + if (!quiet) { + const detail = error instanceof Error ? error.message : String(error); + console.error(` Trusted container recovery could not start: ${redactFull(redact(detail))}`); + } + return null; + } +} diff --git a/src/lib/onboard/docker-gpu-diagnostic-redaction.test.ts b/src/lib/onboard/docker-gpu-diagnostic-redaction.test.ts index b838131d98f..f18823a0a53 100644 --- a/src/lib/onboard/docker-gpu-diagnostic-redaction.test.ts +++ b/src/lib/onboard/docker-gpu-diagnostic-redaction.test.ts @@ -73,7 +73,7 @@ describe("Docker GPU diagnostic redaction", () => { }; const dockerResponses = new Map([ [ - "ps -a --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", + "ps -a --no-trunc --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", "new-container-id\n", ], ["inspect new-container-id", JSON.stringify([inspect])], diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index f2404735609..9c87316f2ed 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -81,6 +81,20 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(dockerStart).not.toHaveBeenCalled(); }); + it("does not report rollback success when restarting the backup has no exit status", () => { + const outcome = finalizeDockerGpuPatchBackup( + { result: deferredCreateResult(), supervisorReady: false }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: null, error: new Error("spawn timed out") })), + }, + ); + + expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); + }); + it("is a no-op when the backup was already removed by the patch helper", () => { const dockerRm = vi.fn((_name: string) => ({ status: 0 })); const result = { ...deferredCreateResult(), backupRemoved: true }; @@ -104,4 +118,13 @@ describe("finalizeDockerGpuPatchBackup", () => { expect.objectContaining({ ignoreError: true }), ); }); + + it("does not report backup removal when Docker returns no exit status", () => { + const outcome = finalizeDockerGpuPatchBackup( + { result: deferredCreateResult(), supervisorReady: true }, + { dockerRm: vi.fn(() => ({ status: null, error: new Error("spawn timed out") })) }, + ); + + expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); + }); }); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index be56a74e8c0..542e27148cb 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -56,7 +56,7 @@ type ResolvedRollbackDeps = { }; function isZeroStatus(result: DockerRunResult | null | undefined): boolean { - return Number(result?.status ?? 0) === 0; + return result?.status === 0; } function resolveRollbackDeps(deps: DockerGpuPatchDeps): ResolvedRollbackDeps { diff --git a/src/lib/onboard/docker-gpu-patch-validation.test.ts b/src/lib/onboard/docker-gpu-patch-validation.test.ts index 0d949b84285..0d66d97e2e5 100644 --- a/src/lib/onboard/docker-gpu-patch-validation.test.ts +++ b/src/lib/onboard/docker-gpu-patch-validation.test.ts @@ -17,6 +17,7 @@ import { function inspectFixture(): DockerContainerInspect { return { Id: "old-container-id", + Image: `sha256:${"c".repeat(64)}`, Name: "/openshell-alpha", Config: { Image: "openshell/sandbox:abc", diff --git a/src/lib/onboard/docker-gpu-patch.test.ts b/src/lib/onboard/docker-gpu-patch.test.ts index 336de53805d..a51a707b5ad 100644 --- a/src/lib/onboard/docker-gpu-patch.test.ts +++ b/src/lib/onboard/docker-gpu-patch.test.ts @@ -31,6 +31,7 @@ import { function inspectFixture(): DockerContainerInspect { return { Id: "old-container-id", + Image: `sha256:${"c".repeat(64)}`, Name: "/openshell-alpha", Config: { Image: "openshell/sandbox:abc", @@ -638,11 +639,11 @@ describe("docker-gpu-patch", () => { expect.arrayContaining([ "--env", "OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", - "openshell/sandbox:abc", + `sha256:${"c".repeat(64)}`, ]), ); - expect(cloneArgs.slice(cloneArgs.indexOf("openshell/sandbox:abc"))).toEqual([ - "openshell/sandbox:abc", + expect(cloneArgs.slice(cloneArgs.indexOf(`sha256:${"c".repeat(64)}`))).toEqual([ + `sha256:${"c".repeat(64)}`, ]); expect(dockerRunDetached).toHaveBeenCalledWith( cloneArgs, diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index 09fab6dfb07..6dc3c7ac8c2 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -12,6 +12,7 @@ import { dockerRm, dockerRun, dockerRunDetached, + dockerStart, dockerStop, } from "../adapters/docker"; import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redaction"; @@ -58,6 +59,7 @@ type DockerRunResult = { status?: number | null; stdout?: string | Buffer | null; stderr?: string | Buffer | null; + error?: Error | null; }; type DockerRunOptions = Record; @@ -147,6 +149,7 @@ export type DockerGpuPatchResult = { }; export type DockerGpuCloneRunOptions = { + image?: string | null; networkMode?: string | null; openshellEndpoint?: string | null; sandboxFallbackDns?: string | null; @@ -216,6 +219,7 @@ export type DockerGpuPatchFailureClassification = { export type DockerContainerInspect = { Id?: string; + Image?: string; Name?: string; Config?: { Image?: string; @@ -279,6 +283,7 @@ function depsWithDefaults( | "dockerRunDetached" | "dockerRename" | "dockerRm" + | "dockerStart" | "dockerStop" | "dockerLogs" | "sleep" @@ -295,6 +300,7 @@ function depsWithDefaults( dockerRunDetached, dockerRename, dockerRm, + dockerStart, dockerStop, dockerLogs, sleep: (seconds: number) => { @@ -366,11 +372,13 @@ export function detectTegraDeviceGroupGids( function resultText(result: DockerRunResult | null | undefined): string { if (!result) return ""; - return `${String(result.stderr || "")} ${String(result.stdout || "")}`.trim(); + return `${String(result.stderr || "")} ${String(result.stdout || "")} ${String( + result.error?.message || "", + )}`.trim(); } function isZeroStatus(result: DockerRunResult | null | undefined): boolean { - return Number(result?.status ?? 0) === 0; + return result?.status === 0; } function sanitizePathPart(value: string): string { @@ -662,7 +670,7 @@ export function buildDockerGpuCloneRunArgs( ): string[] { const config = inspect.Config || {}; const host = inspect.HostConfig || {}; - const image = String(config.Image || "").trim(); + const image = String(options.image || config.Image || "").trim(); if (!image) throw new Error("Docker inspect output did not include Config.Image."); const args: string[] = ["--name", dockerContainerName(inspect), ...mode.args]; @@ -818,6 +826,7 @@ export function findOpenShellDockerSandboxContainerIds( [ "ps", "-a", + "--no-trunc", "--filter", `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, "--filter", @@ -1054,6 +1063,7 @@ export function recreateOpenShellDockerSandboxContainer( timeoutSecs?: number; waitForSupervisor?: boolean; openshellSandboxCommand?: readonly string[] | null; + expectedOldContainerId?: string | null; backend?: DockerGpuPatchBackend; dockerDesktopWsl?: boolean; modeOverride?: DockerGpuPatchMode; @@ -1073,11 +1083,37 @@ export function recreateOpenShellDockerSandboxContainer( `Could not find OpenShell Docker container for sandbox '${options.sandboxName}'.`, ); } + if ( + options.expectedOldContainerId != null && + (containerIds.length !== 1 || oldContainerId !== options.expectedOldContainerId) + ) { + throw new Error( + `OpenShell Docker container identity changed for sandbox '${options.sandboxName}'; ` + + "refusing startup-command recreation because the observed container differs from the pinned identity.", + ); + } + if (options.openshellSandboxCommand != null) { + // Validate the persisted command before image selection so malformed + // tokens remain the first fail-closed result and no container mutation + // can begin regardless of inspect metadata quality. + openshellSandboxCommandEnvValue(options.openshellSandboxCommand); + } context.oldContainerId = oldContainerId; const inspect = inspectDockerContainer(oldContainerId, deps); - const image = String(inspect.Config?.Image || "").trim(); - if (!image) throw new Error("OpenShell sandbox container inspect did not include an image."); + const configuredImage = String(inspect.Config?.Image || "").trim(); + if (!configuredImage) { + throw new Error("OpenShell sandbox container inspect did not include an image."); + } + const immutableImage = String(inspect.Image || "").trim(); + const requiresImmutableImage = options.openshellSandboxCommand != null; + if (requiresImmutableImage && !/^sha256:[0-9a-f]{64}$/i.test(immutableImage)) { + throw new Error( + "OpenShell sandbox container inspect did not include a valid immutable image ID; " + + "refusing startup-command recreation from a mutable image tag.", + ); + } + const image = requiresImmutableImage ? immutableImage : configuredImage; const selection = options.modeOverride ? { mode: options.modeOverride, attempts: [] } @@ -1105,6 +1141,7 @@ export function recreateOpenShellDockerSandboxContainer( context.backupContainerName = backupContainerName; const cloneOptions = buildDockerGpuCloneRunOptions(inspect); + cloneOptions.image = image; cloneOptions.openshellSandboxCommand = options.openshellSandboxCommand ?? null; const sandboxFallbackDns = d.detectSandboxFallbackDns(); if (sandboxFallbackDns) cloneOptions.sandboxFallbackDns = sandboxFallbackDns; @@ -1133,19 +1170,48 @@ export function recreateOpenShellDockerSandboxContainer( // renaming the user's working sandbox. const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); - d.dockerStop(oldContainerId, { + const containerMutationOptions = { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - const renameResult = d.dockerRename(oldContainerId, backupContainerName, { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); + }; + const stopResult = d.dockerStop(oldContainerId, containerMutationOptions); + if (!isZeroStatus(stopResult)) { + context.rolledBack = isZeroStatus(d.dockerStart(oldContainerId, containerMutationOptions)); + throw new Error( + `Could not stop original sandbox container: ${resultText(stopResult)}; ${ + context.rolledBack + ? "original sandbox container confirmed running" + : "restart failed; original sandbox container may be stopped" + }`, + ); + } + const renameResult = d.dockerRename( + oldContainerId, + backupContainerName, + containerMutationOptions, + ); if (!isZeroStatus(renameResult)) { + // A timed-out rename can still have reached the daemon. Normalize both + // possible outcomes toward the original name, then prove by container ID + // that the original is named correctly and running before calling the + // recovery successful. + d.dockerRename(backupContainerName, originalName, containerMutationOptions); + const restarted = isZeroStatus(d.dockerStart(oldContainerId, containerMutationOptions)); + let originalNameRestored = false; + try { + originalNameRestored = + dockerContainerName(inspectDockerContainer(oldContainerId, deps)) === originalName; + } catch { + originalNameRestored = false; + } + context.rolledBack = restarted && originalNameRestored; throw new Error( - `Could not move original sandbox container aside: ${resultText(renameResult)}`, + `Could not move original sandbox container aside: ${resultText(renameResult)}; ${ + context.rolledBack + ? "original sandbox container restored" + : "restore failed; original sandbox container state is uncertain" + }`, ); } @@ -1188,11 +1254,25 @@ export function recreateOpenShellDockerSandboxContainer( deps, ); if (!newContainerId) { + context.rolledBack = rollbackDockerGpuPatchOnRecreateFailure( + // Docker accepted `run --name originalName`, but neither stdout nor + // labeled discovery identified the replacement. Use the deterministic + // requested name to remove any partial replacement before restoring + // the pinned backup. + { newContainerId: originalName, backupContainerName, originalName }, + deps, + ); const containerDescription = selection.mode.kind === "startup-command" ? "Recreated sandbox container" : "GPU-enabled sandbox container"; - throw new Error(`${containerDescription} started, but Docker did not report its ID.`); + throw new Error( + `${containerDescription} started, but Docker did not report its ID; ${ + context.rolledBack + ? "pre-patch sandbox restored" + : "rollback failed; pre-patch sandbox was NOT restored" + }`, + ); } context.newContainerId = newContainerId; diff --git a/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.test.ts b/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.test.ts index 9fa9bd464b6..5f58c9c4f02 100644 --- a/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.test.ts +++ b/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.test.ts @@ -62,7 +62,7 @@ describe("Docker GPU pre-rollback diagnostics (#6110)", () => { ]); const dockerResponses = new Map([ [ - "ps -a --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", + "ps -a --no-trunc --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", "new-container-id\ndiscovered-container-id\n", ], [ @@ -186,7 +186,7 @@ describe("Docker GPU pre-rollback diagnostics (#6110)", () => { ]); const dockerResponses = new Map([ [ - "ps -a --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", + "ps -a --no-trunc --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", "new-container-id\n", ], ["inspect new-container-id", inspectOutput], diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 4d078f2ad21..ad93417c5fe 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -77,7 +77,7 @@ function defaultSleep(seconds: number): void { } function isZeroStatus(result: DockerRunResult | null | undefined): boolean { - return Number(result?.status ?? 0) === 0; + return result?.status === 0; } const ANSI_RE = /\x1b\[[0-9;]*m/g; diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index d6f13f91882..4ec319cd4eb 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -9,6 +9,7 @@ import { recreateOpenShellDockerSandboxWithStartupCommand } from "./docker-start function inspectFixture(): DockerContainerInspect { return { Id: "old-container-id", + Image: `sha256:${"c".repeat(64)}`, Name: "/openshell-alpha", Config: { Image: "openshell/sandbox:abc", @@ -76,6 +77,12 @@ describe("Docker startup-command patch", () => { expect(cloneArgs).not.toEqual( expect.arrayContaining(["--security-opt", "apparmor=unconfined"]), ); + expect(cloneArgs).toContain(`sha256:${"c".repeat(64)}`); + expect(cloneArgs).not.toContain("openshell/sandbox:abc"); + expect(dockerCapture).toHaveBeenCalledWith( + expect.arrayContaining(["ps", "-a", "--no-trunc"]), + expect.objectContaining({ ignoreError: true }), + ); }); it("rejects an empty restart-persistence command before Docker mutation", () => { @@ -117,6 +124,182 @@ describe("Docker startup-command patch", () => { expect(dockerRunDetached).not.toHaveBeenCalled(); }); + it("refuses startup-command recreation without an immutable image ID", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const inspect = inspectFixture(); + delete inspect.Image; + + expect(() => + recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspect]) + : "", + ), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop, + }, + ), + ).toThrow(/refusing startup-command recreation from a mutable image tag/); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }); + + it.each([ + "different-container-id", + "", + ])("refuses to mutate when the pinned container identity is changed or empty", (expectedOldContainerId) => { + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + + expect(() => + recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + openshellSandboxCommand: ["env", "nemoclaw-start"], + expectedOldContainerId, + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ), + dockerRunDetached, + dockerRename, + dockerStop, + }, + ), + ).toThrow("observed container differs from the pinned identity"); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRename).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }); + + it("does not rename or recreate when the original container cannot be stopped", () => { + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const dockerStart = vi.fn(() => ({ status: 0 })); + + expect(() => + recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ), + dockerRunDetached, + dockerRename, + dockerStart, + dockerStop: vi.fn(() => ({ status: null, error: new Error("stop timed out") })), + }, + ), + ).toThrow( + /Could not stop original sandbox container: stop timed out; original sandbox container confirmed running/, + ); + expect(dockerStart).toHaveBeenCalledWith( + "old-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRename).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }); + + it("normalizes and restarts the original container after an uncertain backup rename", () => { + const dockerStart = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const dockerRename = vi + .fn() + .mockReturnValueOnce({ status: null, error: new Error("rename timed out") }) + .mockReturnValueOnce({ status: 0 }); + const dockerCapture = vi + .fn() + .mockReturnValueOnce("old-container-id\n") + .mockReturnValueOnce(JSON.stringify([inspectFixture()])) + .mockReturnValueOnce(JSON.stringify([inspectFixture()])); + + expect(() => + recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture, + dockerRunDetached, + dockerRename, + dockerStart, + dockerStop: vi.fn(() => ({ status: 0 })), + }, + ), + ).toThrow(/rename timed out; original sandbox container restored/); + expect(dockerRename).toHaveBeenNthCalledWith( + 2, + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStart).toHaveBeenCalledWith( + "old-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerCapture).toHaveBeenNthCalledWith( + 3, + ["inspect", "--type", "container", "old-container-id"], + expect.any(Object), + ); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }); + + it("uses the full labeled replacement ID when detached run output is empty", () => { + const oldContainerId = "a".repeat(64); + const newContainerId = "b".repeat(64); + const dockerCapture = vi + .fn() + .mockReturnValueOnce(`${oldContainerId}\n`) + .mockReturnValueOnce(JSON.stringify([{ ...inspectFixture(), Id: oldContainerId }])) + .mockReturnValueOnce(`${oldContainerId}\n${newContainerId}\n`); + + const result = recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + timeoutSecs: 1, + expectedOldContainerId: oldContainerId, + waitForSupervisor: false, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture, + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: "" })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-07-10T00:00:00Z"), + }, + ); + + expect(result.newContainerId).toBe(newContainerId); + }); + it("restores the original sandbox when startup-command recreation fails", () => { const dockerRunDetached = vi.fn(() => ({ status: 1, stderr: "boom" })); @@ -144,4 +327,55 @@ describe("Docker startup-command patch", () => { ), ).toThrow(/Could not start recreated sandbox container: boom; pre-patch sandbox restored/); }); + + it("restores the original sandbox when Docker omits the replacement container ID", () => { + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + const now = vi.spyOn(Date, "now").mockReturnValueOnce(0).mockReturnValue(2_000); + + try { + expect(() => + recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + timeoutSecs: 1, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: "" })), + dockerRename, + dockerRm, + dockerStart, + dockerStop: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-07-10T00:00:00Z"), + }, + ), + ).toThrow(/Docker did not report its ID; pre-patch sandbox restored/); + } finally { + now.mockRestore(); + } + + expect(dockerRm).toHaveBeenCalledWith( + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRename).toHaveBeenLastCalledWith( + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStart).toHaveBeenCalledWith( + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + }); }); diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index df44ea981b0..f9f48ba41c0 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -16,6 +16,7 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( timeoutSecs?: number; waitForSupervisor?: boolean; openshellSandboxCommand: readonly string[]; + expectedOldContainerId?: string | null; }, deps: DockerGpuPatchDeps = {}, ): DockerGpuPatchResult { diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index 9da7968adfd..a84482ac09e 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -39,6 +39,7 @@ function makeDeps() { function inspectFixture(): DockerContainerInspect { return { Id: "old-container-id", + Image: `sha256:${"c".repeat(64)}`, Name: "/openshell-alpha", Config: { Image: "openshell/sandbox:abc", diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index e07a10f93ee..3db3e6047e6 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -11,6 +11,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; import { createOpenshellCliHelpers } from "./openshell-cli"; import { + buildSandboxRuntimeEnvArgs, prepareSandboxCreateLaunch, prepareSandboxCreateLaunchWithPrebuild, } from "./sandbox-create-launch"; @@ -31,6 +32,35 @@ afterEach(() => { } }); +describe("buildSandboxRuntimeEnvArgs", () => { + it("omits credential-bearing env when omitCredentialEnv is set", () => { + const base = { + agent: { name: "openclaw", configPaths: { dir: "/sandbox/.openclaw" } } as any, + chatUiUrl: "http://127.0.0.1:19000/", + manageDashboard: true, + getDashboardForwardPort: () => "19000", + hermesDashboardState: disabledHermesDashboardState, + extraPlaceholderKeys: ["TELEGRAM_BOT_TOKEN_AGENT_A"], + env: { + HTTPS_PROXY: "http://proxyuser:proxypass@proxy.example:8080", + NEMOCLAW_PROXY_HOST: "host.docker.internal", + NEMOCLAW_PROXY_PORT: "3129", + } as NodeJS.ProcessEnv, + }; + + const included = buildSandboxRuntimeEnvArgs(base).envArgs; + expect(included).toContain("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=TELEGRAM_BOT_TOKEN_AGENT_A"); + expect(included.some((arg) => arg.startsWith("HTTPS_PROXY="))).toBe(true); + + const omitted = buildSandboxRuntimeEnvArgs({ ...base, omitCredentialEnv: true }).envArgs; + expect(omitted.some((arg) => arg.startsWith("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS"))).toBe(false); + expect(omitted.some((arg) => arg.includes("proxypass"))).toBe(false); + expect(omitted.some((arg) => arg.startsWith("HTTPS_PROXY="))).toBe(false); + expect(omitted).toContain("NEMOCLAW_DASHBOARD_PORT=19000"); + expect(omitted).toContain("NEMOCLAW_PROXY_HOST=host.docker.internal"); + }); +}); + describe("prepareSandboxCreateLaunch", () => { it("builds the sandbox create command and runtime env envelope", () => { const openshellShellCommand = vi.fn((args: string[]) => `openshell ${args.join(" ")}`); diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index d74a932a764..9446ba5ec94 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -75,9 +75,24 @@ export interface SandboxCreateLaunchWithPrebuild extends SandboxCreateLaunch { prebuild: SandboxPrebuildResult; } -export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): SandboxCreateLaunch { - const env = input.env ?? process.env; - const manageDashboard = input.manageDashboard ?? true; +export interface SandboxRuntimeEnvArgsInput { + agent: AgentDefinition | null; + chatUiUrl: string; + manageDashboard: boolean; + getDashboardForwardPort(chatUiUrl: string): string; + hermesDashboardState: HermesDashboardOnboardState; + extraPlaceholderKeys: readonly string[]; + observabilityEnabled?: boolean; + sandboxName?: string; + env: NodeJS.ProcessEnv; + omitCredentialEnv?: boolean; +} + +export function buildSandboxRuntimeEnvArgs(input: SandboxRuntimeEnvArgsInput): { + envArgs: string[]; + effectiveDashboardPort: string; +} { + const { agent, env, manageDashboard } = input; const envArgs = manageDashboard ? [formatEnvAssignment("CHAT_UI_URL", input.chatUiUrl)] : []; // When manageDashboard is enabled, pass the effective dashboard port into @@ -96,11 +111,12 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San } } - appendOpenClawRuntimeEnvArgs(envArgs, input.agent ?? null); - appendOpenClawAutoPairRuntimeEnvArgs(envArgs, input.agent ?? null, env); + appendOpenClawRuntimeEnvArgs(envArgs, agent); + appendOpenClawAutoPairRuntimeEnvArgs(envArgs, agent, env); appendHermesDashboardEnvArgs(envArgs, input.hermesDashboardState, formatEnvAssignment); appendHostProxyEnvArgs(envArgs, env, { - dropCredentialBearingProxyUrls: input.agent?.name === "langchain-deepagents-code", + dropCredentialBearingProxyUrls: + agent?.name === "langchain-deepagents-code" || input.omitCredentialEnv === true, }); // Propagate NEMOCLAW_PROXY_HOST / NEMOCLAW_PROXY_PORT to runtime containers @@ -119,7 +135,7 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San envArgs.push(formatEnvAssignment("NEMOCLAW_PROXY_PORT", sandboxProxyPort)); } - if (input.agent?.name === "langchain-deepagents-code") { + if (agent?.name === "langchain-deepagents-code") { const sandboxName = input.sandboxName; if (sandboxName) { envArgs.push(formatEnvAssignment("NEMOCLAW_SANDBOX_NAME", sandboxName)); @@ -132,7 +148,27 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San ); } - appendExtraPlaceholderKeysEnvArg(envArgs, input.extraPlaceholderKeys, formatEnvAssignment); + if (!input.omitCredentialEnv) { + appendExtraPlaceholderKeysEnvArg(envArgs, input.extraPlaceholderKeys, formatEnvAssignment); + } + + return { envArgs, effectiveDashboardPort }; +} + +export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): SandboxCreateLaunch { + const env = input.env ?? process.env; + const manageDashboard = input.manageDashboard ?? true; + const { envArgs, effectiveDashboardPort } = buildSandboxRuntimeEnvArgs({ + agent: input.agent ?? null, + chatUiUrl: input.chatUiUrl, + manageDashboard, + getDashboardForwardPort: input.getDashboardForwardPort, + hermesDashboardState: input.hermesDashboardState, + extraPlaceholderKeys: input.extraPlaceholderKeys, + observabilityEnabled: input.observabilityEnabled, + sandboxName: input.sandboxName, + env, + }); const sandboxEnv = (input.buildEnv ?? buildSubprocessEnv)(); // Remove host-infrastructure credentials that the generic allowlist diff --git a/src/lib/sandbox/privileged-exec.test.ts b/src/lib/sandbox/privileged-exec.test.ts index eda0e5cecb2..2598fbd6b48 100644 --- a/src/lib/sandbox/privileged-exec.test.ts +++ b/src/lib/sandbox/privileged-exec.test.ts @@ -196,6 +196,42 @@ describe("privileged sandbox exec routing", () => { ); }); + it("refuses privileged execution when the pinned container identity changed", () => { + withPrivilegedExecMocks( + { + getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), + listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), + dockerCapture: () => "current-container-id\topenshell-alpha\n", + }, + ({ privilegedSandboxExecArgv }) => { + expect(() => + privilegedSandboxExecArgv( + "alpha", + ["/trusted/control"], + false, + true, + "previous-container-id", + ), + ).toThrow(/container identity changed.*refusing privileged execution/i); + }, + ); + }); + + it("refuses privileged execution when the pinned container identity is empty", () => { + withPrivilegedExecMocks( + { + getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), + listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), + dockerCapture: () => "current-container-id\topenshell-alpha\n", + }, + ({ privilegedSandboxExecArgv }) => { + expect(() => + privilegedSandboxExecArgv("alpha", ["/trusted/control"], false, true, ""), + ).toThrow(/container identity changed.*refusing privileged execution/i); + }, + ); + }); + it("fails before docker discovery when the sandbox registry entry is unavailable", () => { let dockerPsCalls = 0; withPrivilegedExecMocks( diff --git a/src/lib/sandbox/privileged-exec.ts b/src/lib/sandbox/privileged-exec.ts index 02a4bb9a6a2..c45f59a60a4 100644 --- a/src/lib/sandbox/privileged-exec.ts +++ b/src/lib/sandbox/privileged-exec.ts @@ -206,6 +206,7 @@ function privilegedSandboxExecArgv( cmd: string[], stdin = false, sanitizeEnvironment = false, + expectedContainerId?: string, ): string[] { const entry = readSandboxEntry(sandboxName); if (!entry) throw missingRegistryEntryError(sandboxName); @@ -219,6 +220,12 @@ function privilegedSandboxExecArgv( // clearly if no matching sandbox container is running. const container = findDirectSandboxContainer(sandboxName); if (container) { + if (expectedContainerId !== undefined && container !== expectedContainerId) { + throw new Error( + `OpenShell container identity changed for sandbox '${sandboxName}'; ` + + "refusing privileged execution against a different container.", + ); + } const sanitizedEnvArgs = sanitizeEnvironment ? SANITIZED_PRIVILEGED_ENV.flatMap((value) => ["--env", value]) : []; diff --git a/test/dashboard-remote-bind-lifecycle.test.ts b/test/dashboard-remote-bind-lifecycle.test.ts index b78348b4185..c10d8184172 100644 --- a/test/dashboard-remote-bind-lifecycle.test.ts +++ b/test/dashboard-remote-bind-lifecycle.test.ts @@ -593,7 +593,7 @@ describe("remote dashboard bind production lifecycle", () => { ); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "0.0.0.0:18789", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); @@ -620,7 +620,7 @@ describe("remote dashboard bind production lifecycle", () => { expect(ensureSandboxPortForward("beta")).toBe(false); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "0.0.0.0:18789", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); @@ -712,7 +712,7 @@ describe("remote dashboard bind production lifecycle", () => { ); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "18789", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); @@ -746,7 +746,7 @@ describe("remote dashboard bind production lifecycle", () => { expect(ensureSandboxPortForward("beta")).toBe(true); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "0.0.0.0:18789", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); @@ -783,7 +783,7 @@ describe("remote dashboard bind production lifecycle", () => { expect(ensureSandboxPortForward("beta")).toBe(true); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "18789", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); }); diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index 03ae0b1eba3..b7303e83de3 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -32,19 +32,21 @@ * that recovered runtime state. * - Deliberately out of scope for this merge gate: physical DGX Spark / * GB10 / aarch64 hardware, provider breadth beyond `cloud-openclaw`, and - * destructive host reboot / OOM / supervisor crash / manual - * `kubectl delete pod` triggers. The current live Vitest runner exposes a - * Docker-driver OpenShell sandbox and does not provide a stable per-test - * Kubernetes pod handle that can be deleted without destabilizing shared - * gateway state. Those trigger/hardware/provider clauses need a dedicated - * platform-runtime job; this test locks down the shared recovery contract - * they all depend on. + * destructive host reboot / OOM / manual `kubectl delete pod` triggers. + * The Docker-driver branch below does restart the registered sandbox + * container, then proves the legacy keepalive migration restores the + * managed supervisor topology without relying on ordinary sandbox exec. + * Kubernetes triggers still need a dedicated platform-runtime job. * * This Vitest coverage owns both the #2478 WARNING assertion lineage and the * #2701 guard-chain assertion. */ +import { Buffer } from "node:buffer"; +import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { ubuntuRepoDocker } from "../registry/matrix.ts"; @@ -60,6 +62,88 @@ const ENVIRONMENT = ubuntuRepoDocker("cloud-openclaw"); const SANDBOX_NAME = "e2e-2701"; +const STARTUP_COMMAND_INSPECT_SCRIPT = String.raw` +const { spawnSync } = require("node:child_process"); +const id = process.argv[1]; +const result = spawnSync("docker", ["inspect", "--type", "container", id], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], +}); +if (result.status !== 0) { + process.stderr.write(result.stderr || "docker inspect failed\n"); + process.exit(result.status || 1); +} +const rows = JSON.parse(result.stdout); +const prefix = "OPENSHELL_SANDBOX_COMMAND="; +const matches = (rows[0]?.Config?.Env || []).filter((entry) => entry.startsWith(prefix)); +if (matches.length !== 1) { + process.stderr.write("expected one OpenShell sandbox startup command\n"); + process.exit(1); +} +process.stdout.write(matches[0].slice(prefix.length) + "\n"); +`; + +const SUPERVISOR_TOPOLOGY_SCRIPT = String.raw`from pathlib import Path +import pwd +expected_uid=str(pwd.getpwnam("sandbox").pw_uid) +assert expected_uid != "0", expected_uid +rows=[] +for entry in Path("/proc").iterdir(): + if not entry.name.isdigit() or entry.name == "1": + continue + try: + stat=(entry / "stat").read_text().rsplit(")", 1)[1].split() + cmd=(entry / "cmdline").read_bytes().rstrip(b"\0").split(b"\0") + status=(entry / "status").read_text() + except (FileNotFoundError, PermissionError, ProcessLookupError): + continue + if int(stat[1]) != 1 or not cmd: + continue + if cmd[0].rsplit(b"/", 1)[-1] == b"nemoclaw-start" or (len(cmd) > 1 and cmd[0].rsplit(b"/", 1)[-1] == b"bash" and cmd[1].rsplit(b"/", 1)[-1] == b"nemoclaw-start"): + rows.append((entry.name, status)) +assert len(rows) == 1, rows +uid_line=next(line for line in rows[0][1].splitlines() if line.startswith("Uid:")) +assert uid_line.split()[1:] == [expected_uid] * 4, uid_line +print("MANAGED_SUPERVISOR=" + rows[0][0] + ":PPID1")`; + +const SUPERVISOR_TOPOLOGY_COMMAND = `import base64;exec(base64.b64decode("${Buffer.from( + SUPERVISOR_TOPOLOGY_SCRIPT, +).toString("base64")}"))`; + +async function findSandboxContainer(host: HostCliClient, artifactName: string): Promise { + const result = await host.command( + "docker", + [ + "ps", + "--no-trunc", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`, + "--format", + "{{.ID}}", + ], + { artifactName, env: buildAvailabilityProbeEnv() }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + const ids = result.stdout.trim().split(/\s+/).filter(Boolean); + expect(ids, resultText(result)).toHaveLength(1); + return ids[0] ?? ""; +} + +async function inspectStartupCommand( + host: HostCliClient, + containerId: string, + artifactName: string, +): Promise { + const result = await host.command("node", ["-e", STARTUP_COMMAND_INSPECT_SCRIPT, containerId], { + artifactName, + env: buildAvailabilityProbeEnv(), + }); + expect(result.exitCode, resultText(result)).toBe(0); + return result.stdout.trim(); +} + test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)", async ({ artifacts, environment, @@ -75,18 +159,20 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) await artifacts.target.declare({ id: "gateway-guard-recovery", boundary: "sandbox-lifecycle", - issues: ["#2701", "#2478"], + issues: ["#2701", "#2478", "#6635"], acceptanceCoverage: { covered: [ "production connect --probe-only recovery route", "authenticated PID 1 OpenClaw recovery supervisor", "pod-recreate-equivalent empty /tmp guard chain plus missing gateway process", + "Docker container restart with a legacy keepalive startup", + "container-identity-pinned supervisor recreation with managed health proof", "no rebuild required for the recovered runtime state", ], intentionallyOutOfScope: [ "DGX Spark / GB10 / aarch64 hardware matrix", "provider breadth beyond cloud-openclaw", - "host reboot / OOM / supervisor crash / manual kubectl delete pod triggers", + "host reboot / OOM / manual kubectl delete pod triggers", ], }, }); @@ -183,4 +269,96 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) }); expect(stablePid).toBeGreaterThan(0); + + // ── Assert #6635 legacy Docker restart recovery ──────────────── + // Fresh non-GPU OpenClaw containers on this OpenShell floor still carry the + // legacy keepalive. Restarting the container therefore kills the initial + // OpenShell workload session and deterministically leaves no managed + // supervisor. Recovery must upgrade that container through the host-side + // transaction and commit only after managed control accepts the new tree. + const originalContainerId = await findSandboxContainer(host, "legacy-restart-container-before"); + expect( + await inspectStartupCommand(host, originalContainerId, "legacy-restart-command-before"), + ).toBe("sleep infinity"); + await host.cleanupForward(18789, { + artifactName: "legacy-restart-stop-dashboard-forward", + env: buildAvailabilityProbeEnv(), + }); + const restart = await host.command("docker", ["restart", originalContainerId], { + artifactName: "legacy-restart-docker-restart", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }); + expect(restart.exitCode, resultText(restart)).toBe(0); + + const credentialCanary = "nemoclaw-e2e-recovery-secret-6635"; + const trustedRecovery = await host.nemoclaw([instance.sandboxName, "recover"], { + artifactName: "legacy-restart-trusted-recover", + env: { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: "CUSTOM_PROVIDER_CREDENTIAL", + CUSTOM_PROVIDER_CREDENTIAL: credentialCanary, + }, + redactionValues: [credentialCanary], + timeoutMs: 240_000, + }); + expect(trustedRecovery.timedOut, resultText(trustedRecovery)).toBe(false); + expect(trustedRecovery.exitCode, resultText(trustedRecovery)).toBe(0); + expect(resultText(trustedRecovery)).toContain("Probe complete: recovered OpenClaw gateway"); + + const recoveredContainerId = await findSandboxContainer(host, "legacy-restart-container-after"); + expect(recoveredContainerId).not.toBe(originalContainerId); + const recoveredStartupCommand = await inspectStartupCommand( + host, + recoveredContainerId, + "legacy-restart-command-after", + ); + expect(recoveredStartupCommand).toMatch(/(?:^| )nemoclaw-start$/); + expect(recoveredStartupCommand).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); + expect(recoveredStartupCommand).not.toContain(credentialCanary); + + expect(SUPERVISOR_TOPOLOGY_COMMAND).not.toMatch(/[\r\n]/); + const topology = await sandbox.exec( + instance.sandboxName, + ["python3", "-c", SUPERVISOR_TOPOLOGY_COMMAND], + { + artifactName: "legacy-restart-managed-supervisor-topology", + env: buildAvailabilityProbeEnv(), + }, + ); + expect(topology.exitCode, resultText(topology)).toBe(0); + expect(topology.stdout).toMatch(/MANAGED_SUPERVISOR=[0-9]+:PPID1/); + + const forwardedHealth = await host.command( + "curl", + ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "http://127.0.0.1:18789/health"], + { + artifactName: "legacy-restart-forwarded-health", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(forwardedHealth.exitCode, resultText(forwardedHealth)).toBe(0); + expect(forwardedHealth.stdout.trim()).toMatch(/^(200|401)$/); + + const inference = await host.nemoclaw( + [ + instance.sandboxName, + "agent", + "--agent", + "main", + "--json", + "--session-id", + `e2e-6635-${Date.now()}-${process.pid}`, + "-m", + "What is 6 multiplied by 7? Reply with only the integer, no extra words.", + ], + { + artifactName: "legacy-restart-agent-inference", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }, + ); + expect(inference.exitCode, resultText(inference)).toBe(0); + expect(containsInteger42Answer(inference.stdout), resultText(inference)).toBe(true); }); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 5b0b06cd764..0307f351e3d 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -18,6 +18,17 @@ "live": "test/e2e/live/full-e2e.test.ts", "fast": ["test/e2e/support/onboard-performance.test.ts"] }, + { + "live": "test/e2e/live/gateway-guard-recovery.test.ts", + "fast": [ + "src/lib/actions/sandbox/supervisor-relaunch.test.ts", + "src/lib/onboard/docker-startup-command-patch.test.ts", + "src/lib/sandbox/privileged-exec.test.ts", + "test/managed-gateway-control.test.ts", + "test/nemoclaw-start-guard-recovery.test.ts", + "test/process-recovery-supervisor-relaunch.test.ts" + ] + }, { "live": "test/e2e/live/network-policy.test.ts", "fast": ["test/validate-blueprint.test.ts"] diff --git a/test/managed-gateway-control.test.ts b/test/managed-gateway-control.test.ts index dc690c6e265..e5709adc82c 100644 --- a/test/managed-gateway-control.test.ts +++ b/test/managed-gateway-control.test.ts @@ -169,6 +169,98 @@ with tempfile.TemporaryDirectory() as root: finally: control._namespace_inode = real_namespace_inode + remove_process(proc_root, 41) + remove_process(proc_root, 40) + try: + control._discover_supervisor(reader) + missing_supervisor = "accepted" + except control.ControlError as error: + missing_supervisor = error.code + real_supervisor_candidates = control._supervisor_candidates + supervisor_candidate_calls = [] + def supervisor_appears_between_scans(reader, pid1, sandbox_uid): + matches, inconclusive = real_supervisor_candidates(reader, pid1, sandbox_uid) + supervisor_candidate_calls.append(len(matches)) + if len(supervisor_candidate_calls) == 1: + write_process( + proc_root, + namespace_path, + 40, + 222, + 1, + 1000, + b"bash\0/usr/local/bin/nemoclaw-start\0", + b"PATH=/usr/bin\0NEMOCLAW_DASHBOARD_PORT=18789\0", + ) + return matches, inconclusive + control._supervisor_candidates = supervisor_appears_between_scans + try: + control._discover_supervisor(reader) + appearing_supervisor = "accepted" + except control.ControlError as error: + appearing_supervisor = error.code + finally: + control._supervisor_candidates = real_supervisor_candidates + remove_process(proc_root, 40) + write_process( + proc_root, + namespace_path, + 46, + 666, + 1, + 1000, + b"unreadable-process\0", + ) + real_capture = reader.capture + def capture_with_permission_denial(pid): + if pid == 46: + raise PermissionError("denied") + return real_capture(pid) + reader.capture = capture_with_permission_denial + try: + control._discover_supervisor(reader) + unreadable_process = "accepted" + except control.ControlError as error: + unreadable_process = error.code + finally: + reader.capture = real_capture + remove_process(proc_root, 46) + write_process( + proc_root, + namespace_path, + 40, + 222, + 1, + 1000, + b"bash\0/usr/local/bin/nemoclaw-start\0", + b"PATH=/usr/bin\0NEMOCLAW_DASHBOARD_PORT=18789\0", + ) + write_process( + proc_root, + namespace_path, + 41, + 333, + 40, + 1000, + b"/usr/local/bin/hermes.real\0gateway\0run\0", + listener_inode="77777", + ) + write_process( + proc_root, + namespace_path, + 45, + 555, + 1, + 1000, + b"/usr/local/bin/nemoclaw-start\0", + ) + try: + control._discover_supervisor(reader) + duplicate_supervisor = "accepted" + except control.ControlError as error: + duplicate_supervisor = error.code + remove_process(proc_root, 45) + preflight_steps = [] real_validator = control._run_fixed_validator real_runtime_validator = control._validate_runtime_environment @@ -598,6 +690,10 @@ with tempfile.TemporaryDirectory() as root: "namespace_denied": namespace_denied, "preflight": preflight_steps, "runtime_validation": runtime_validation, + "missing_supervisor": missing_supervisor, + "appearing_supervisor": appearing_supervisor, + "unreadable_process": unreadable_process, + "duplicate_supervisor": duplicate_supervisor, "duplicate": duplicate, "signals": sent, "reused": reused, @@ -665,6 +761,10 @@ describe("managed gateway root control", () => { { hash: "checked" }, ], runtime_validation: "in-process", + missing_supervisor: "SUPERVISOR_NOT_RUNNING", + appearing_supervisor: "SUPERVISOR_UNAVAILABLE", + unreadable_process: "SUPERVISOR_UNAVAILABLE", + duplicate_supervisor: "SUPERVISOR_UNAVAILABLE", duplicate: "SUPERVISOR_UNAVAILABLE", signals: [15, 9], reused: "SUPERVISOR_UNAVAILABLE", diff --git a/test/process-recovery-forward-failure.test.ts b/test/process-recovery-forward-failure.test.ts index 8b4c31f95e5..977d4ab4ec5 100644 --- a/test/process-recovery-forward-failure.test.ts +++ b/test/process-recovery-forward-failure.test.ts @@ -179,7 +179,7 @@ beta 127.0.0.1 18789 12345 dead`, expect(teamsForwardStarted).toBe(true); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "3978", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); }); diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts new file mode 100644 index 00000000000..945cfbdc629 --- /dev/null +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -0,0 +1,337 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as forwardHealth from "../src/lib/actions/sandbox/forward-health.ts"; +import { checkAndRecoverSandboxProcesses } from "../src/lib/actions/sandbox/process-recovery.ts"; +import { relaunchManagedSupervisorSession } from "../src/lib/actions/sandbox/supervisor-relaunch.ts"; +import * as openshellRuntime from "../src/lib/adapters/openshell/runtime.ts"; +import * as agentRuntime from "../src/lib/agent/runtime.ts"; +import * as registry from "../src/lib/state/registry.ts"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +function mockOpenClawSandbox(sandboxName: string) { + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "openclaw", + displayName: "OpenClaw", + forwardPort: 18789, + healthProbe: { url: "http://127.0.0.1:18789/health", port: 18789, timeout_seconds: 30 }, + } as never); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: sandboxName, + agent: "openclaw", + dashboardPort: 18789, + openshellDriver: "docker", + }); +} + +function setImmediateRecoveryPolling() { + vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); + vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "0"); + vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", "0"); + vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); +} + +describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { + it("does not turn ambiguous supervisor unavailability into a container mutation", () => { + mockOpenClawSandbox("ambiguous-box"); + setImmediateRecoveryPolling(); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_UNAVAILABLE", + })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => null); + + const result = checkAndRecoverSandboxProcesses("ambiguous-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ checked: true, wasRunning: false, recovered: false }); + expect(requestGatewaySupervisorAction).toHaveBeenCalledTimes(3); + expect(relaunchManagedSupervisorSessionImpl).not.toHaveBeenCalled(); + }); + + it("does not mutate on an embellished no-supervisor marker", () => { + mockOpenClawSandbox("embellished-box"); + setImmediateRecoveryPolling(); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "prefix SUPERVISOR_NOT_RUNNING suffix", + })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => null); + + const result = checkAndRecoverSandboxProcesses("embellished-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ checked: true, wasRunning: false, recovered: false }); + expect(requestGatewaySupervisorAction).toHaveBeenCalledOnce(); + expect(relaunchManagedSupervisorSessionImpl).not.toHaveBeenCalled(); + }); + + it("honors the relaunch kill switch through stable no-supervisor recovery", () => { + vi.stubEnv("NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH", "1"); + mockOpenClawSandbox("legacy-box"); + setImmediateRecoveryPolling(); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_NOT_RUNNING", + })); + const resolveContainer = vi.fn(() => "old-container-id"); + const recreate = vi.fn(() => { + throw new Error("kill switch allowed container mutation"); + }); + const requestPinnedGatewaySupervisorAction = vi.fn(() => null); + const relaunchManagedSupervisorSessionImpl = vi.fn( + (sandboxName: string, options: Parameters[1]) => + relaunchManagedSupervisorSession(sandboxName, { + quiet: options.quiet, + deps: { ...options.deps, resolveContainer, recreate }, + }), + ); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "log").mockImplementation(() => undefined); + + const result = checkAndRecoverSandboxProcesses("legacy-box", { + quiet: false, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ checked: true, wasRunning: false, recovered: false }); + expect(requestGatewaySupervisorAction).toHaveBeenCalledOnce(); + expect(relaunchManagedSupervisorSessionImpl).toHaveBeenCalledWith( + "legacy-box", + expect.objectContaining({ quiet: false }), + ); + expect(resolveContainer).not.toHaveBeenCalled(); + expect(requestPinnedGatewaySupervisorAction).not.toHaveBeenCalled(); + expect(recreate).not.toHaveBeenCalled(); + const errorLines = errorSpy.mock.calls.map((call) => String(call[0])); + expect(errorLines).toContainEqual( + expect.stringContaining("Failure layer: supervisor not running"), + ); + expect(errorLines).toContainEqual(expect.stringContaining("trusted container recovery")); + expect(errorLines).toContainEqual(expect.stringContaining("rebuild --yes")); + expect(errorLines).not.toContainEqual( + expect.stringContaining("Retry the managed restart from the host"), + ); + }); + + it("rolls back when recreation starts but managed control never accepts it", () => { + mockOpenClawSandbox("rejected-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => + action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, + ); + const requestPinnedGatewaySupervisorAction = vi.fn(() => null); + + const result = checkAndRecoverSandboxProcesses("rejected-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ checked: true, wasRunning: false, recovered: false }); + expect(requestPinnedGatewaySupervisorAction).toHaveBeenCalledWith( + "rejected-box", + "probe", + 210000, + "replacement-container-id", + ); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(false); + }); + + it("commits only after managed health accepts the recreated supervisor", () => { + mockOpenClawSandbox("recovered-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn((supervisorReady: boolean) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => + action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, + ); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + })); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: "SANDBOX BIND PORT PID STATUS\nrecovered-box 127.0.0.1 18789 12345 running", + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = checkAndRecoverSandboxProcesses("recovered-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ checked: true, wasRunning: false, recovered: true }); + expect(requestGatewaySupervisorAction).toHaveBeenCalledWith("recovered-box", "recover"); + expect(requestPinnedGatewaySupervisorAction).toHaveBeenCalledWith( + "recovered-box", + "probe", + 210000, + "replacement-container-id", + ); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(true); + }); + + it("retains a healthy replacement but does not start a forward when OpenShell stays unready", () => { + mockOpenClawSandbox("unready-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_NOT_RUNNING", + })); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + })); + const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => false); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); + + const result = checkAndRecoverSandboxProcesses("unready-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + waitForRecreatedSandboxOpenShellReadyImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: expect.stringContaining("did not become ready in OpenShell"), + }); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(true); + expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledWith( + "unready-box", + expect.objectContaining({ beforeProbe: expect.any(Function), timeoutSeconds: 30 }), + ); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + + it("rejects a healthy forward when the replacement identity changes after readiness", () => { + mockOpenClawSandbox("drifted-box"); + vi.mocked(agentRuntime.getSessionAgent).mockReturnValue({ + name: "openclaw", + displayName: "OpenClaw", + forwardPort: 18789, + forward_ports: [19000], + healthProbe: { url: "http://127.0.0.1:18789/health", port: 18789, timeout_seconds: 30 }, + } as never); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_NOT_RUNNING", + })); + const acceptedProbe = { + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + }; + const requestPinnedGatewaySupervisorAction = vi + .fn() + .mockReturnValueOnce(acceptedProbe) + .mockReturnValueOnce(acceptedProbe) + .mockImplementationOnce(() => { + throw new Error("replacement identity changed"); + }) + .mockReturnValue(acceptedProbe); + const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn( + (_name, options) => options.beforeProbe?.(1000) === true, + ); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: "SANDBOX BIND PORT PID STATUS\ndrifted-box 127.0.0.1 18789 12345 running", + }); + const runOpenshell = vi + .spyOn(openshellRuntime, "runOpenshell") + .mockReturnValue({ status: 0 } as never); + + const result = checkAndRecoverSandboxProcesses("drifted-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + waitForRecreatedSandboxOpenShellReadyImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + }); + expect(requestPinnedGatewaySupervisorAction).toHaveBeenCalledTimes(3); + expect(requestPinnedGatewaySupervisorAction).toHaveBeenLastCalledWith( + "drifted-box", + "probe", + 15000, + "replacement-container-id", + ); + expect(finalize).toHaveBeenCalledWith(true); + expect(runOpenshell).toHaveBeenCalledOnce(); + expect(runOpenshell).toHaveBeenCalledWith(["forward", "stop", "18789", "drifted-box"], { + ignoreError: true, + stdio: "ignore", + }); + }); +}); diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index 4c18045f1d4..ce0348f80b8 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -225,7 +225,7 @@ beta 127.0.0.1 18789 12345 running`; }); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "0.0.0.0:18789", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); @@ -263,7 +263,7 @@ beta 127.0.0.1 18789 12345 running`; expect(events).toEqual(["stale-listener", "stale-listener", "released", "start"]); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "8642", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); @@ -346,7 +346,7 @@ beta 127.0.0.1 3978 12346 running`; }); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "3978", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); @@ -400,7 +400,7 @@ beta 127.0.0.1 18789 12345 running`; }); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "3978", "beta"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); }); @@ -1211,7 +1211,7 @@ hermes-box 127.0.0.1 8642 12346 running`; expect(requestGatewaySupervisorAction).toHaveBeenCalledWith("hermes-box", "recover"); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "18789", "hermes-box"], - { ignoreError: true }, + { ignoreError: true, stdio: "ignore" }, ); });