diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 3f97cd96baf..5ba8479d1ee 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -129,8 +129,11 @@ During rebuild or restore, NemoClaw merges those settings with the freshly gener If the restored config cannot be parsed or applied safely, NemoClaw stops the restore instead of replacing the generated config with an unsafe fallback. OpenClaw's device identity keys and paired-device tokens are intentionally excluded from snapshots because backup sanitization scrubs them beyond use. -Restore never touches the sandbox's current gateway pairing state, even when an older snapshot still contains those files. -OpenClaw regenerates its device identity on demand, and NemoClaw auto-pair re-pairs CLI clients on connect. +Snapshot state replacement does not overwrite the destination sandbox's gateway pairing files, even when an older snapshot still contains them. +After a cross-sandbox restore creates the destination, NemoClaw establishes gateway pairing and verifies it with an authenticated agent run. +If verification fails, the restored state remains in the destination and the command exits nonzero. +Run `$$nemoclaw connect` to retry pairing before you run an agent. +OpenClaw regenerates its device identity on demand. Credential-bearing Hermes files such as `auth.json` are intentionally excluded from snapshots. diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.test.ts b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts new file mode 100644 index 00000000000..81cbd251979 --- /dev/null +++ b/src/lib/actions/sandbox/restore-gateway-pairing.test.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { establishRestoredSandboxGatewayPairing } from "./restore-gateway-pairing"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("establishRestoredSandboxGatewayPairing", () => { + it("provokes the scope upgrade before approving it (#7431)", () => { + const order: string[] = []; + const warmupScopeUpgrade = vi.fn(() => order.push("warmup")); + const autoPairScopeApproval = vi.fn(() => order.push("approve")); + const verifyGatewayPairing = vi.fn(() => { + order.push("verify"); + return true; + }); + + establishRestoredSandboxGatewayPairing("beta", { + warmupScopeUpgrade, + autoPairScopeApproval, + verifyGatewayPairing, + }); + + expect(warmupScopeUpgrade).toHaveBeenCalledWith("beta"); + expect(autoPairScopeApproval).toHaveBeenCalledWith("beta"); + expect(verifyGatewayPairing).toHaveBeenCalledWith("beta"); + expect(order).toEqual(["warmup", "approve", "verify"]); + }); + + it("fails when the pairing warm-up does not complete (#7431)", () => { + const warmupScopeUpgrade = vi.fn(() => { + throw new Error("gateway not up"); + }); + const autoPairScopeApproval = vi.fn(); + const verifyGatewayPairing = vi.fn(() => true); + + expect(() => + establishRestoredSandboxGatewayPairing("beta", { + warmupScopeUpgrade, + autoPairScopeApproval, + verifyGatewayPairing, + }), + ).toThrow("gateway not up"); + expect(autoPairScopeApproval).not.toHaveBeenCalled(); + expect(verifyGatewayPairing).not.toHaveBeenCalled(); + }); + + it("fails when the authenticated verification run cannot use the restored gateway (#7431)", () => { + expect(() => + establishRestoredSandboxGatewayPairing("beta", { + warmupScopeUpgrade: vi.fn(), + autoPairScopeApproval: vi.fn(), + verifyGatewayPairing: vi.fn(() => false), + }), + ).toThrow("authenticated gateway verification run did not succeed"); + }); +}); diff --git a/src/lib/actions/sandbox/restore-gateway-pairing.ts b/src/lib/actions/sandbox/restore-gateway-pairing.ts new file mode 100644 index 00000000000..a9e16361fc5 --- /dev/null +++ b/src/lib/actions/sandbox/restore-gateway-pairing.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { verifyRestoredSandboxGatewayPairing } from "../../adapters/openshell/restore-gateway-pairing"; +import { WARMUP_SESSION_ID_PREFIX } from "./warmup-session"; + +export type RestoreGatewayPairingDeps = { + warmupScopeUpgrade: (sandboxName: string) => void; + autoPairScopeApproval: (sandboxName: string) => void; + verifyGatewayPairing: (sandboxName: string) => boolean; +}; + +function defaultRestoreGatewayPairingDeps(): RestoreGatewayPairingDeps { + const warmup: typeof import("./auto-pair-warmup") = require("./auto-pair-warmup"); + const connect: typeof import("./connect") = require("./connect"); + return { + warmupScopeUpgrade: warmup.runSandboxScopeWarmupRun, + autoPairScopeApproval: connect.runConnectAutoPairApprovalPass, + verifyGatewayPairing: (sandboxName) => + verifyRestoredSandboxGatewayPairing(sandboxName, WARMUP_SESSION_ID_PREFIX), + }; +} + +export function establishRestoredSandboxGatewayPairing( + targetSandbox: string, + deps: RestoreGatewayPairingDeps = defaultRestoreGatewayPairingDeps(), +): void { + try { + deps.warmupScopeUpgrade(targetSandbox); + deps.autoPairScopeApproval(targetSandbox); + if (!deps.verifyGatewayPairing(targetSandbox)) { + throw new Error("the authenticated gateway verification run did not succeed"); + } + } catch (err) { + throw new Error( + `could not establish gateway pairing for '${targetSandbox}': ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } +} diff --git a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts index ff244e01b83..57b1de4e173 100644 --- a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts @@ -5,7 +5,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as f from "./snapshot-restore-test-fixture"; -beforeEach(f.resetSnapshotRestoreMocks); +beforeEach(() => { + f.resetSnapshotRestoreMocks(); +}); afterEach(f.cleanupSnapshotRestoreMocks); describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { it("restores the latest snapshot into the source sandbox", async () => { @@ -188,3 +190,146 @@ describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { expect(f.registerSandboxMock).not.toHaveBeenCalled(); }); }); + +describe("runSandboxSnapshot restore: gateway pairing on a freshly created destination", () => { + it("provokes and approves device pairing after a cross-sandbox restore", async () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + f.getSandboxMock.mockImplementation((name) => + name === "alpha" + ? { + name: "alpha", + agent: "openclaw", + imageTag: "nemoclaw-alpha:test", + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + } + : null, + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + f.restoreSandboxStateMock.mockReturnValue({ + success: true, + restoredDirs: ["workspace"], + restoredFiles: ["user.md"], + failedDirs: [], + failedFiles: [], + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true }); + + expect(f.restoreSandboxStateMock).toHaveBeenCalledWith("beta", "/tmp/backup-alpha"); + expect(f.establishRestoredSandboxGatewayPairingMock).toHaveBeenCalledWith("beta"); + }); + + it("fails with repair guidance when restored gateway pairing cannot be verified (#7431)", async () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + f.getSandboxMock.mockImplementation((name) => + name === "alpha" + ? { + name: "alpha", + agent: "openclaw", + imageTag: "nemoclaw-alpha:test", + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + } + : null, + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + f.restoreSandboxStateMock.mockReturnValue({ + success: true, + restoredDirs: ["workspace"], + restoredFiles: ["user.md"], + failedDirs: [], + failedFiles: [], + }); + f.establishRestoredSandboxGatewayPairingMock.mockImplementationOnce(() => { + throw new Error("authenticated gateway verification failed"); + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true }), + ).rejects.toMatchObject({ + exitCode: 1, + lines: [ + "State restored into 'beta', but gateway pairing could not be verified.", + "Run `nemoclaw beta connect` to retry pairing before running an agent.", + expect.stringContaining("authenticated gateway verification failed"), + ], + }); + }); + + it.each([ + "hermes", + "langchain-deepagents-code", + ])("does not run OpenClaw pairing for a cross-sandbox %s restore (#7431)", async (agent) => { + vi.spyOn(console, "log").mockImplementation(() => {}); + f.getSandboxMock.mockImplementation((name) => + name === "alpha" + ? { + name: "alpha", + agent, + imageTag: "nemoclaw-alpha:test", + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + } + : null, + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + f.restoreSandboxStateMock.mockReturnValue({ + success: true, + restoredDirs: ["workspace"], + restoredFiles: [], + failedDirs: [], + failedFiles: [], + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true }); + + expect(f.restoreSandboxStateMock).toHaveBeenCalledWith("beta", "/tmp/backup-alpha"); + expect(f.establishRestoredSandboxGatewayPairingMock).not.toHaveBeenCalled(); + }); + + it("leaves the working gateway credentials untouched on a self-restore", async () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + f.restoreSandboxStateMock.mockReturnValue({ + success: true, + restoredDirs: ["workspace"], + restoredFiles: ["user.md"], + failedDirs: [], + failedFiles: [], + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(f.restoreSandboxStateMock).toHaveBeenCalledWith("alpha", "/tmp/backup-alpha"); + expect(f.establishRestoredSandboxGatewayPairingMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 684ff3fa05c..1579d5e477e 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -108,6 +108,7 @@ export const captureOpenshellMock = vi.fn< (args: string[], opts?: Record) => OpenshellCaptureResult >((args) => defaultOpenshellResponses(args)); export const dockerInspectMock = vi.fn(() => ({ status: 0, stdout: "true\n" })); +export const establishRestoredSandboxGatewayPairingMock = vi.fn(); export const findBackupMock = vi.fn(); export const getAppliedPresetsMock = vi.fn(() => [] as string[]); export const getCustomPoliciesMock = vi.fn( @@ -247,6 +248,10 @@ vi.mock("./destroy", () => ({ removeSandboxRegistryEntry: vi.fn(), })); +vi.mock("./restore-gateway-pairing", () => ({ + establishRestoredSandboxGatewayPairing: establishRestoredSandboxGatewayPairingMock, +})); + export function resetSnapshotRestoreMocks(): void { vi.clearAllMocks(); shieldsMock.setIsShieldsDownExport(shieldsMock.isShieldsDownMock); @@ -256,6 +261,7 @@ export function resetSnapshotRestoreMocks(): void { lifecycleMock.readTimerMarkerMock.mockReturnValue(null); captureOpenshellMock.mockImplementation((args) => defaultOpenshellResponses(args)); dockerInspectMock.mockReturnValue({ status: 0, stdout: "true\n" }); + establishRestoredSandboxGatewayPairingMock.mockReset(); findBackupMock.mockReturnValue({ match: null }); getAppliedPresetsMock.mockReturnValue([]); getCustomPoliciesMock.mockReturnValue([]); diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 903eadee680..80eebe7e7b6 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -236,6 +236,10 @@ vi.mock("../../state/sandbox", () => ({ restoreSandboxState: restoreSandboxStateMock, })); +vi.mock("./restore-gateway-pairing", () => ({ + establishRestoredSandboxGatewayPairing: vi.fn(), +})); + vi.mock("./destroy", () => ({ cleanupShieldsDestroyArtifacts: lifecycleMock.cleanupShieldsDestroyArtifactsMock, removeSandboxRegistryEntry: vi.fn(), diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 13f4d003979..69e06668470 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -61,6 +61,7 @@ import { parseDcodeProbeState, } from "./dcode-activity-probe"; import { cleanupShieldsDestroyArtifacts, removeSandboxRegistryEntry } from "./destroy"; +import { establishRestoredSandboxGatewayPairing } from "./restore-gateway-pairing"; import { buildSandboxExecMarkedCommand, createSandboxExecMarker, @@ -874,6 +875,7 @@ async function runSnapshotRestoreUnlocked( " Failed to query live sandbox state from OpenShell.", ); const isCrossSandboxRestore = targetSandbox !== sandboxName; + let crossSandboxRestoreAgent: string | null = null; const targetEntry = isCrossSandboxRestore ? registry.getSandbox(targetSandbox) : null; const targetExists = sourceLiveNames.has(targetSandbox) || Boolean(targetEntry); @@ -992,6 +994,7 @@ async function runSnapshotRestoreUnlocked( ); snapshotExit(1); } + crossSandboxRestoreAgent = lockedSourceEntry.agent || "openclaw"; if (getSandboxEntryInference(lockedSourceEntry).kind !== "configured") { console.error( ` Cannot auto-create '${targetSandbox}': source '${sandboxName}' has no complete durable inference route.`, @@ -1099,6 +1102,18 @@ async function runSnapshotRestoreUnlocked( // managed observability binding from current target state. reconcileSnapshotPolicyPresets(targetSandbox, resolvedSnapshot); }); + if (isCrossSandboxRestore && crossSandboxRestoreAgent === "openclaw") { + try { + establishRestoredSandboxGatewayPairing(targetSandbox); + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + throw new SnapshotCommandError([ + `State restored into '${targetSandbox}', but gateway pairing could not be verified.`, + `Run \`${CLI_NAME} ${targetSandbox} connect\` to retry pairing before running an agent.`, + `Details: ${detail}`, + ]); + } + } } export async function runSandboxSnapshot( diff --git a/src/lib/adapters/openshell/restore-gateway-pairing.test.ts b/src/lib/adapters/openshell/restore-gateway-pairing.test.ts new file mode 100644 index 00000000000..3e7dc43fcfb --- /dev/null +++ b/src/lib/adapters/openshell/restore-gateway-pairing.test.ts @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + type RestoreGatewayPairingVerifierDeps, + verifyRestoredSandboxGatewayPairing, +} from "./restore-gateway-pairing"; + +const SESSION_ID_PREFIX = "nemoclaw-onboard-warmup-"; + +function verifierDeps( + result: ReturnType, +): RestoreGatewayPairingVerifierDeps { + return { + resolveOpenshell: vi.fn(() => "/usr/bin/openshell"), + spawnSync: vi.fn(() => result), + }; +} + +describe("verifyRestoredSandboxGatewayPairing", () => { + it("accepts an authenticated gateway verification run that exits successfully (#7431)", () => { + const deps = verifierDeps({ status: 0, stdout: '{"result":"pong"}', stderr: "" }); + + expect(verifyRestoredSandboxGatewayPairing("beta", SESSION_ID_PREFIX, deps)).toBe(true); + expect(deps.spawnSync).toHaveBeenCalledWith( + "/usr/bin/openshell", + expect.arrayContaining(["sandbox", "exec", "--name", "beta"]), + expect.objectContaining({ timeout: 30_000 }), + ); + }); + + it("rejects an authenticated gateway verification run that exits unsuccessfully (#7431)", () => { + expect( + verifyRestoredSandboxGatewayPairing("beta", SESSION_ID_PREFIX, verifierDeps({ status: 1 })), + ).toBe(false); + }); + + it.each([ + "EMBEDDED FALLBACK: gateway unavailable", + '{"fallbackFrom":"gateway"}', + '{"transport":"embedded"}', + "gateway connect failed: device pairing required", + "scope upgrade pending approval", + ])("rejects a zero-exit verification run with fallback or pairing output (#7431)", (output) => { + expect( + verifyRestoredSandboxGatewayPairing( + "beta", + SESSION_ID_PREFIX, + verifierDeps({ status: 0, stdout: output }), + ), + ).toBe(false); + }); + + it("accepts changed output when the gateway run exits successfully without a failure signal (#7431)", () => { + expect( + verifyRestoredSandboxGatewayPairing( + "beta", + SESSION_ID_PREFIX, + verifierDeps({ status: 0, stdout: '{"futureResult":"ok"}' }), + ), + ).toBe(true); + }); + + it("rejects an authenticated gateway verification run that times out (#7431)", () => { + const error = new Error("timed out") as NodeJS.ErrnoException; + error.code = "ETIMEDOUT"; + + expect( + verifyRestoredSandboxGatewayPairing( + "beta", + SESSION_ID_PREFIX, + verifierDeps({ status: null, error }), + ), + ).toBe(false); + }); + + it("rejects verification when the OpenShell executable cannot be resolved (#7431)", () => { + const spawn = vi.fn(() => ({ status: 0 })); + + expect( + verifyRestoredSandboxGatewayPairing("beta", SESSION_ID_PREFIX, { + resolveOpenshell: () => null, + spawnSync: spawn, + }), + ).toBe(false); + expect(spawn).not.toHaveBeenCalled(); + }); + + it("rejects verification when OpenShell cannot be started (#7431)", () => { + expect( + verifyRestoredSandboxGatewayPairing("beta", SESSION_ID_PREFIX, { + resolveOpenshell: () => "/usr/bin/openshell", + spawnSync: () => { + throw new Error("spawn failed"); + }, + }), + ).toBe(false); + }); +}); diff --git a/src/lib/adapters/openshell/restore-gateway-pairing.ts b/src/lib/adapters/openshell/restore-gateway-pairing.ts new file mode 100644 index 00000000000..540fc01ace8 --- /dev/null +++ b/src/lib/adapters/openshell/restore-gateway-pairing.ts @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncOptionsWithStringEncoding, spawnSync } from "node:child_process"; + +import { ROOT } from "../../state/paths"; +import { resolveOpenshell } from "./resolve"; + +const RESTORE_GATEWAY_PAIRING_VERIFY_TIMEOUT_MS = 30_000; + +const RESTORE_GATEWAY_PAIRING_VERIFY_SCRIPT = ` +PROXY_ENV=/tmp/nemoclaw-proxy-env.sh +[ -r "$PROXY_ENV" ] && . "$PROXY_ENV" +command -v openclaw >/dev/null 2>&1 || exit 1 +openclaw agent --agent main --json -m "ping" \ + --session-id "$1restore-verify-$$-$(date +%s)" +`; + +// OpenClaw can currently exit zero after using its embedded fallback, and its +// JSON output does not expose a supported, stable transport discriminator. +// These compatibility signals match the gateway-auth live tests. Remove this +// classifier once OpenClaw provides a machine-readable gateway-only result. +const RESTORE_GATEWAY_PAIRING_REJECTION = + /EMBEDDED FALLBACK|gateway connect failed|scope upgrade pending approval|device pairing required|pairing required|fallbackFrom[": ]+gateway|transport[": ]+embedded/i; + +type RestoreGatewayPairingSpawnResult = { + status: number | null; + error?: Error; + stdout?: string | null; + stderr?: string | null; +}; + +export type RestoreGatewayPairingVerifierDeps = { + resolveOpenshell: () => string | null; + spawnSync: ( + command: string, + args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, + ) => RestoreGatewayPairingSpawnResult; +}; + +const defaultDeps: RestoreGatewayPairingVerifierDeps = { + resolveOpenshell, + spawnSync, +}; + +export function verifyRestoredSandboxGatewayPairing( + targetSandbox: string, + sessionIdPrefix: string, + deps: RestoreGatewayPairingVerifierDeps = defaultDeps, +): boolean { + try { + const openshellBinary = deps.resolveOpenshell(); + if (!openshellBinary) return false; + + const result = deps.spawnSync( + openshellBinary, + [ + "sandbox", + "exec", + "--name", + targetSandbox, + "--", + "sh", + "-c", + RESTORE_GATEWAY_PAIRING_VERIFY_SCRIPT, + "restore-gateway-pairing", + sessionIdPrefix, + ], + { + cwd: ROOT, + encoding: "utf8", + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + timeout: RESTORE_GATEWAY_PAIRING_VERIFY_TIMEOUT_MS, + }, + ); + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + return ( + result.status === 0 && + result.error === undefined && + !RESTORE_GATEWAY_PAIRING_REJECTION.test(output) + ); + } catch { + return false; + } +}