Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions docs/manage-sandboxes/backup-restore.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,11 @@ During rebuild or restore, NemoClaw merges those settings with the freshly gener
If the restored config cannot be parsed or applied safely, NemoClaw stops the restore instead of replacing the generated config with an unsafe fallback.

OpenClaw's device identity keys and paired-device tokens are intentionally excluded from snapshots because backup sanitization scrubs them beyond use.
Restore never touches the sandbox's current gateway pairing state, even when an older snapshot still contains those files.
OpenClaw regenerates its device identity on demand, and NemoClaw auto-pair re-pairs CLI clients on connect.
Snapshot state replacement does not overwrite the destination sandbox's gateway pairing files, even when an older snapshot still contains them.
After a cross-sandbox restore creates the destination, NemoClaw establishes gateway pairing and verifies it with an authenticated agent run.
If verification fails, the restored state remains in the destination and the command exits nonzero.
Run `$$nemoclaw <destination> connect` to retry pairing before you run an agent.
OpenClaw regenerates its device identity on demand.
</AgentOnly>
<AgentOnly variant="hermes">
Credential-bearing Hermes files such as `auth.json` are intentionally excluded from snapshots.
Expand Down
61 changes: 61 additions & 0 deletions src/lib/actions/sandbox/restore-gateway-pairing.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { afterEach, describe, expect, it, vi } from "vitest";

import { establishRestoredSandboxGatewayPairing } from "./restore-gateway-pairing";

afterEach(() => {
vi.restoreAllMocks();
});

describe("establishRestoredSandboxGatewayPairing", () => {
it("provokes the scope upgrade before approving it (#7431)", () => {
const order: string[] = [];
const warmupScopeUpgrade = vi.fn(() => order.push("warmup"));
const autoPairScopeApproval = vi.fn(() => order.push("approve"));
const verifyGatewayPairing = vi.fn(() => {
order.push("verify");
return true;
});

establishRestoredSandboxGatewayPairing("beta", {
warmupScopeUpgrade,
autoPairScopeApproval,
verifyGatewayPairing,
});

expect(warmupScopeUpgrade).toHaveBeenCalledWith("beta");
expect(autoPairScopeApproval).toHaveBeenCalledWith("beta");
expect(verifyGatewayPairing).toHaveBeenCalledWith("beta");
expect(order).toEqual(["warmup", "approve", "verify"]);
});

it("fails when the pairing warm-up does not complete (#7431)", () => {
const warmupScopeUpgrade = vi.fn(() => {
throw new Error("gateway not up");
});
const autoPairScopeApproval = vi.fn();
const verifyGatewayPairing = vi.fn(() => true);

expect(() =>
establishRestoredSandboxGatewayPairing("beta", {
warmupScopeUpgrade,
autoPairScopeApproval,
verifyGatewayPairing,
}),
).toThrow("gateway not up");
expect(autoPairScopeApproval).not.toHaveBeenCalled();
expect(verifyGatewayPairing).not.toHaveBeenCalled();
});

it("fails when the authenticated verification run cannot use the restored gateway (#7431)", () => {
expect(() =>
establishRestoredSandboxGatewayPairing("beta", {
warmupScopeUpgrade: vi.fn(),
autoPairScopeApproval: vi.fn(),
verifyGatewayPairing: vi.fn(() => false),
}),
).toThrow("authenticated gateway verification run did not succeed");
});
});
41 changes: 41 additions & 0 deletions src/lib/actions/sandbox/restore-gateway-pairing.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { verifyRestoredSandboxGatewayPairing } from "../../adapters/openshell/restore-gateway-pairing";
import { WARMUP_SESSION_ID_PREFIX } from "./warmup-session";

export type RestoreGatewayPairingDeps = {
warmupScopeUpgrade: (sandboxName: string) => void;
autoPairScopeApproval: (sandboxName: string) => void;
verifyGatewayPairing: (sandboxName: string) => boolean;
};

function defaultRestoreGatewayPairingDeps(): RestoreGatewayPairingDeps {
const warmup: typeof import("./auto-pair-warmup") = require("./auto-pair-warmup");
const connect: typeof import("./connect") = require("./connect");
return {
warmupScopeUpgrade: warmup.runSandboxScopeWarmupRun,
autoPairScopeApproval: connect.runConnectAutoPairApprovalPass,
verifyGatewayPairing: (sandboxName) =>
verifyRestoredSandboxGatewayPairing(sandboxName, WARMUP_SESSION_ID_PREFIX),
};
}

export function establishRestoredSandboxGatewayPairing(
targetSandbox: string,
deps: RestoreGatewayPairingDeps = defaultRestoreGatewayPairingDeps(),
): void {
try {
deps.warmupScopeUpgrade(targetSandbox);
deps.autoPairScopeApproval(targetSandbox);
if (!deps.verifyGatewayPairing(targetSandbox)) {
throw new Error("the authenticated gateway verification run did not succeed");
}
} catch (err) {
throw new Error(
`could not establish gateway pairing for '${targetSandbox}': ${
err instanceof Error ? err.message : String(err)
}`,
);
}
}
147 changes: 146 additions & 1 deletion src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";

import * as f from "./snapshot-restore-test-fixture";

beforeEach(f.resetSnapshotRestoreMocks);
beforeEach(() => {
f.resetSnapshotRestoreMocks();
});
afterEach(f.cleanupSnapshotRestoreMocks);
describe("runSandboxSnapshot restore: lifecycle and destination safety", () => {
it("restores the latest snapshot into the source sandbox", async () => {
Expand Down Expand Up @@ -188,3 +190,146 @@ describe("runSandboxSnapshot restore: lifecycle and destination safety", () => {
expect(f.registerSandboxMock).not.toHaveBeenCalled();
});
});

describe("runSandboxSnapshot restore: gateway pairing on a freshly created destination", () => {
it("provokes and approves device pairing after a cross-sandbox restore", async () => {
vi.spyOn(console, "log").mockImplementation(() => {});
f.getSandboxMock.mockImplementation((name) =>
name === "alpha"
? {
name: "alpha",
agent: "openclaw",
imageTag: "nemoclaw-alpha:test",
openshellDriver: "docker",
provider: "nvidia-nim",
model: "nvidia/model-a",
}
: null,
);
f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"]));
f.captureOpenshellMock.mockImplementation((args) =>
f.openshellResponses(args, {
"sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") },
"sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" },
}),
);
f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture });
f.restoreSandboxStateMock.mockReturnValue({
success: true,
restoredDirs: ["workspace"],
restoredFiles: ["user.md"],
failedDirs: [],
failedFiles: [],
});
const { runSandboxSnapshot } = await import("./snapshot");

await runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true });

expect(f.restoreSandboxStateMock).toHaveBeenCalledWith("beta", "/tmp/backup-alpha");
expect(f.establishRestoredSandboxGatewayPairingMock).toHaveBeenCalledWith("beta");
});

it("fails with repair guidance when restored gateway pairing cannot be verified (#7431)", async () => {
vi.spyOn(console, "log").mockImplementation(() => {});
f.getSandboxMock.mockImplementation((name) =>
name === "alpha"
? {
name: "alpha",
agent: "openclaw",
imageTag: "nemoclaw-alpha:test",
openshellDriver: "docker",
provider: "nvidia-nim",
model: "nvidia/model-a",
}
: null,
);
f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"]));
f.captureOpenshellMock.mockImplementation((args) =>
f.openshellResponses(args, {
"sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") },
"sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" },
}),
);
f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture });
f.restoreSandboxStateMock.mockReturnValue({
success: true,
restoredDirs: ["workspace"],
restoredFiles: ["user.md"],
failedDirs: [],
failedFiles: [],
});
f.establishRestoredSandboxGatewayPairingMock.mockImplementationOnce(() => {
throw new Error("authenticated gateway verification failed");
});
const { runSandboxSnapshot } = await import("./snapshot");

await expect(
runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true }),
).rejects.toMatchObject({
exitCode: 1,
lines: [
"State restored into 'beta', but gateway pairing could not be verified.",
"Run `nemoclaw beta connect` to retry pairing before running an agent.",
expect.stringContaining("authenticated gateway verification failed"),
],
});
});

it.each([
"hermes",
"langchain-deepagents-code",
])("does not run OpenClaw pairing for a cross-sandbox %s restore (#7431)", async (agent) => {
vi.spyOn(console, "log").mockImplementation(() => {});
f.getSandboxMock.mockImplementation((name) =>
name === "alpha"
? {
name: "alpha",
agent,
imageTag: "nemoclaw-alpha:test",
openshellDriver: "docker",
provider: "nvidia-nim",
model: "nvidia/model-a",
}
: null,
);
f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"]));
f.captureOpenshellMock.mockImplementation((args) =>
f.openshellResponses(args, {
"sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") },
"sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" },
}),
);
f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture });
f.restoreSandboxStateMock.mockReturnValue({
success: true,
restoredDirs: ["workspace"],
restoredFiles: [],
failedDirs: [],
failedFiles: [],
});
const { runSandboxSnapshot } = await import("./snapshot");

await runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true });

expect(f.restoreSandboxStateMock).toHaveBeenCalledWith("beta", "/tmp/backup-alpha");
expect(f.establishRestoredSandboxGatewayPairingMock).not.toHaveBeenCalled();
});

it("leaves the working gateway credentials untouched on a self-restore", async () => {
vi.spyOn(console, "log").mockImplementation(() => {});
f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture });
f.restoreSandboxStateMock.mockReturnValue({
success: true,
restoredDirs: ["workspace"],
restoredFiles: ["user.md"],
failedDirs: [],
failedFiles: [],
});
const { runSandboxSnapshot } = await import("./snapshot");

await runSandboxSnapshot("alpha", { kind: "restore" });

expect(f.restoreSandboxStateMock).toHaveBeenCalledWith("alpha", "/tmp/backup-alpha");
expect(f.establishRestoredSandboxGatewayPairingMock).not.toHaveBeenCalled();
});
});
6 changes: 6 additions & 0 deletions src/lib/actions/sandbox/snapshot-restore-test-fixture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ export const captureOpenshellMock = vi.fn<
(args: string[], opts?: Record<string, unknown>) => OpenshellCaptureResult
>((args) => defaultOpenshellResponses(args));
export const dockerInspectMock = vi.fn(() => ({ status: 0, stdout: "true\n" }));
export const establishRestoredSandboxGatewayPairingMock = vi.fn();
export const findBackupMock = vi.fn();
export const getAppliedPresetsMock = vi.fn(() => [] as string[]);
export const getCustomPoliciesMock = vi.fn(
Expand Down Expand Up @@ -247,6 +248,10 @@ vi.mock("./destroy", () => ({
removeSandboxRegistryEntry: vi.fn(),
}));

vi.mock("./restore-gateway-pairing", () => ({
establishRestoredSandboxGatewayPairing: establishRestoredSandboxGatewayPairingMock,
}));

export function resetSnapshotRestoreMocks(): void {
vi.clearAllMocks();
shieldsMock.setIsShieldsDownExport(shieldsMock.isShieldsDownMock);
Expand All @@ -256,6 +261,7 @@ export function resetSnapshotRestoreMocks(): void {
lifecycleMock.readTimerMarkerMock.mockReturnValue(null);
captureOpenshellMock.mockImplementation((args) => defaultOpenshellResponses(args));
dockerInspectMock.mockReturnValue({ status: 0, stdout: "true\n" });
establishRestoredSandboxGatewayPairingMock.mockReset();
findBackupMock.mockReturnValue({ match: null });
getAppliedPresetsMock.mockReturnValue([]);
getCustomPoliciesMock.mockReturnValue([]);
Expand Down
4 changes: 4 additions & 0 deletions src/lib/actions/sandbox/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,10 @@ vi.mock("../../state/sandbox", () => ({
restoreSandboxState: restoreSandboxStateMock,
}));

vi.mock("./restore-gateway-pairing", () => ({
establishRestoredSandboxGatewayPairing: vi.fn(),
}));

vi.mock("./destroy", () => ({
cleanupShieldsDestroyArtifacts: lifecycleMock.cleanupShieldsDestroyArtifactsMock,
removeSandboxRegistryEntry: vi.fn(),
Expand Down
15 changes: 15 additions & 0 deletions src/lib/actions/sandbox/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ import {
parseDcodeProbeState,
} from "./dcode-activity-probe";
import { cleanupShieldsDestroyArtifacts, removeSandboxRegistryEntry } from "./destroy";
import { establishRestoredSandboxGatewayPairing } from "./restore-gateway-pairing";
import {
buildSandboxExecMarkedCommand,
createSandboxExecMarker,
Expand Down Expand Up @@ -874,6 +875,7 @@ async function runSnapshotRestoreUnlocked(
" Failed to query live sandbox state from OpenShell.",
);
const isCrossSandboxRestore = targetSandbox !== sandboxName;
let crossSandboxRestoreAgent: string | null = null;
const targetEntry = isCrossSandboxRestore ? registry.getSandbox(targetSandbox) : null;
const targetExists = sourceLiveNames.has(targetSandbox) || Boolean(targetEntry);

Expand Down Expand Up @@ -992,6 +994,7 @@ async function runSnapshotRestoreUnlocked(
);
snapshotExit(1);
}
crossSandboxRestoreAgent = lockedSourceEntry.agent || "openclaw";
if (getSandboxEntryInference(lockedSourceEntry).kind !== "configured") {
console.error(
` Cannot auto-create '${targetSandbox}': source '${sandboxName}' has no complete durable inference route.`,
Expand Down Expand Up @@ -1099,6 +1102,18 @@ async function runSnapshotRestoreUnlocked(
// managed observability binding from current target state.
reconcileSnapshotPolicyPresets(targetSandbox, resolvedSnapshot);
});
if (isCrossSandboxRestore && crossSandboxRestoreAgent === "openclaw") {
try {
establishRestoredSandboxGatewayPairing(targetSandbox);
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
throw new SnapshotCommandError([
`State restored into '${targetSandbox}', but gateway pairing could not be verified.`,
`Run \`${CLI_NAME} ${targetSandbox} connect\` to retry pairing before running an agent.`,
`Details: ${detail}`,
]);
}
}
}

export async function runSandboxSnapshot(
Expand Down
Loading
Loading