diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 555c1939fbd..8d1def477b4 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -8,7 +8,7 @@ "src/lib/adapters/docker/index.ts": 43, "src/lib/adapters/openshell/client.ts": 23, "src/lib/adapters/openshell/resolve.ts": 27, - "src/lib/adapters/openshell/runtime.ts": 51, + "src/lib/adapters/openshell/runtime.ts": 52, "src/lib/adapters/openshell/timeouts.ts": 37, "src/lib/agent/defs.ts": 32, "src/lib/cli/branding.ts": 84, @@ -23,7 +23,7 @@ "src/lib/inference/config.ts": 29, "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 26, - "src/lib/onboard/gateway-binding.ts": 47, + "src/lib/onboard/gateway-binding.ts": 48, "src/lib/runner.ts": 89, "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 36, @@ -43,7 +43,7 @@ "src/lib/actions/sandbox/policy-channel.ts": 29, "src/lib/actions/sandbox/process-recovery.ts": 22, "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, - "src/lib/actions/sandbox/snapshot.ts": 38, + "src/lib/actions/sandbox/snapshot.ts": 39, "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 23, diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index 3e4e8288b03..515ef28c6d1 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ listSandboxes: vi.fn(), + getSandbox: vi.fn(), backupSandboxState: vi.fn(), captureSandboxListWithGatewayPreflightOrExit: vi.fn(), parseReadySandboxNames: vi.fn(), @@ -18,17 +19,25 @@ const mocks = vi.hoisted(() => ({ isSandboxContainerDefinitivelyAbsent: vi.fn(), openBackupShieldsWindow: vi.fn(), relockBackupShieldsWindow: vi.fn(), + withSandboxMutationLock: vi.fn(), })); vi.mock("../state/registry", () => ({ isRouteOnlySandboxReservation: (entry: { pendingRouteReservation?: true; createdAt?: string }) => entry.pendingRouteReservation === true && entry.createdAt === undefined, listSandboxes: mocks.listSandboxes, + getSandbox: mocks.getSandbox, })); vi.mock("../state/sandbox", () => ({ backupSandboxState: mocks.backupSandboxState, BackupResult: {}, })); +vi.mock("../state/mcp-lifecycle-lock", () => ({ + withSandboxMutationLock: mocks.withSandboxMutationLock, +})); +vi.mock("./sandbox/snapshot/backup-authority", () => ({ + backupSandboxStateWithManagedAuthority: (name: string) => mocks.backupSandboxState(name), +})); vi.mock("../openshell-sandbox-list", () => ({ captureSandboxListWithGatewayPreflightOrExit: mocks.captureSandboxListWithGatewayPreflightOrExit, })); @@ -100,6 +109,7 @@ describe("backupAll", () => { wasLocked: false, })); mocks.relockBackupShieldsWindow.mockReturnValue(true); + mocks.withSandboxMutationLock.mockImplementation((_name, callback) => callback()); }); afterEach(() => { @@ -217,6 +227,43 @@ describe("backupAll", () => { logSpy.mockRestore(); }); + it("counts a mutation-lock acquisition failure and continues with later sandboxes", async () => { + mocks.listSandboxes.mockReturnValue({ + sandboxes: [{ name: "alpha" }, { name: "beta" }], + defaultSandbox: "alpha", + }); + mocks.parseReadySandboxNames.mockReturnValue(new Set(["alpha", "beta"])); + mocks.withSandboxMutationLock + .mockRejectedValueOnce(new Error("Timed out waiting for the sandbox mutation lock")) + .mockImplementationOnce((_name, callback) => callback()); + mocks.backupSandboxState.mockReturnValue({ + success: true, + backedUpDirs: ["workspace"], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + manifest: { backupPath: "/backups/beta/timestamp" }, + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`exit:${code}`); + }) as never); + + await expect(backupAll()).rejects.toThrow("exit:1"); + + expect(mocks.withSandboxMutationLock.mock.calls.map(([name]) => name)).toEqual([ + "alpha", + "beta", + ]); + expect(mocks.backupSandboxState).toHaveBeenCalledOnce(); + expect(mocks.backupSandboxState).toHaveBeenCalledWith("beta"); + expect(logSpy.mock.calls.flat().join("\n")).toContain("1 backed up, 1 failed, 0 skipped"); + expect(errorSpy.mock.calls.flat().join("\n")).toContain( + "alpha: backup failed (mutation lock: Timed out waiting for the sandbox mutation lock)", + ); + }); + it("does not back up when gateway preflight exits", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "sb-good" }], diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 9792f2be1c7..1025b327fd7 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -22,6 +22,7 @@ import { SANDBOX_IMAGE_REPOS } from "../domain/sandbox/image-tag"; import { resolveGatewayName, resolveSandboxGatewayName } from "../onboard/gateway-binding"; import { captureSandboxListWithGatewayPreflightOrExit } from "../openshell-sandbox-list"; import { parseLiveSandboxNames, parseReadySandboxNames } from "../runtime-recovery"; +import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock"; import * as registry from "../state/registry"; import * as sandboxState from "../state/sandbox"; import { nemoclawStateRoot, resolveHome } from "../state/state-root"; @@ -30,6 +31,7 @@ import { openBackupShieldsWindow, relockBackupShieldsWindow, } from "./sandbox/backup-shields-window"; +import * as snapshotBackup from "./sandbox/snapshot/backup-authority"; import { backupStartedSandboxState, isSandboxContainerDefinitivelyAbsent, @@ -198,7 +200,7 @@ export async function backupAll(): Promise { let unreachableRunning = 0; let notRunningSkipped = 0; const strandedOrphans: string[] = []; - for (const sb of sandboxes) { + const backupRegisteredSandbox = async (sb: (typeof sandboxes)[number]): Promise => { // A registered docker-driver sandbox whose container is merely stopped is // backupable: start it for the duration of the backup and return it to // its stopped state after (#6500). Anything else that is not Ready keeps @@ -211,12 +213,12 @@ export async function backupAll(): Promise { // Tracked separately from `skipped` so the strict gate stays // untripped: there is nothing to back up and nothing to start. strandedOrphans.push(sb.name); - continue; + return; } console.log(` ${D}${notRunningBackupSkipMessage(sb.name)}${R}`); skipped++; notRunningSkipped++; - continue; + return; } console.log(` Starting stopped sandbox '${sb.name}' to back it up...`); } @@ -229,7 +231,13 @@ export async function backupAll(): Promise { const attempt = await backupSandboxWithinShieldsWindow(sb.name, () => startedForBackup ? backupStartedSandboxState(sb.name) - : sandboxState.backupSandboxState(sb.name), + : snapshotBackup.backupSandboxStateWithManagedAuthority( + sb.name, + {}, + { + getSandbox: registry.getSandbox, + }, + ), ); result = attempt.result; orphanManifestMessage = attempt.orphanManifestMessage; @@ -248,17 +256,17 @@ export async function backupAll(): Promise { } if (!returnedToStopped) { failed++; - continue; + return; } if (!shieldsWindowOpened) { console.error(` ${RD}✗${R} ${sb.name}: backup failed (could not safely unlock shields)`); failed++; - continue; + return; } if (orphanManifestMessage) { console.log(` ${YW}⚠${R} Skipped '${sb.name}' (orphan manifest): ${orphanManifestMessage}`); skipped++; - continue; + return; } if (!result) throw new Error(`Backup for '${sb.name}' completed without a result`); if (result.success) { @@ -273,7 +281,7 @@ export async function backupAll(): Promise { ` ${YW}⚠${R} Skipped '${sb.name}' (running but SSH-unreachable; NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1 set). Any uncommitted state since the last successful backup will be lost.`, ); skipped++; - continue; + return; } unreachableRunning++; } @@ -284,6 +292,24 @@ export async function backupAll(): Promise { console.error(` ${RD}✗${R} ${sb.name}: backup failed (${failedItems})`); failed++; } + }; + for (const sb of sandboxes) { + let enteredMutationLock = false; + try { + await withSandboxMutationLock(sb.name, () => { + enteredMutationLock = true; + return backupRegisteredSandbox(sb); + }); + } catch (error) { + // Callback failures retain the existing fail-fast behavior. A lock that + // could not be acquired is instead one failed sandbox attempt so the + // remaining backups, orphan confirmation, summary, and strict gate all + // still run. + if (enteredMutationLock) throw error; + const detail = error instanceof Error ? error.message : String(error); + console.error(` ${RD}✗${R} ${sb.name}: backup failed (mutation lock: ${detail})`); + failed++; + } } // The classification above is only as fresh as the pre-loop listing, and // the backup loop can run for minutes. Confirm with a second pinned listing diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts index 1740d1bbfa8..b480b74facd 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts @@ -9,6 +9,7 @@ import * as gatewayRuntime from "../../gateway-runtime-action"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import * as sandboxState from "../../state/sandbox"; import * as userManagedFilesProbe from "../../state/user-managed-files-probe"; +import * as snapshotBackup from "./snapshot/backup-authority"; import { backupSandboxStateForRebuild, disposeRebuildAgentBaseImagePreflight, @@ -549,7 +550,7 @@ describe("backupSandboxStateForRebuild with --force", () => { errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(console, "log").mockImplementation(() => undefined); - backupSpy = vi.spyOn(sandboxState, "backupSandboxState"); + backupSpy = vi.spyOn(snapshotBackup, "backupSandboxStateWithManagedAuthority"); }); afterEach(() => { @@ -843,7 +844,9 @@ describe("warnUnpreservedUserManagedFiles", () => { logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - backupSpy = vi.spyOn(sandboxState, "backupSandboxState").mockReturnValue(makeBackupResult()); + backupSpy = vi + .spyOn(snapshotBackup, "backupSandboxStateWithManagedAuthority") + .mockReturnValue(makeBackupResult()); probeSpy = vi.spyOn(userManagedFilesProbe, "probeUserManagedFiles").mockReturnValue({ declared: [], existing: [], diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.ts index b69f2d59054..f30e0ca5247 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.ts @@ -49,6 +49,7 @@ import { printWrongGatewayActiveGuidance, } from "./gateway-state"; import { openRebuildShieldsWindow, type RebuildShieldsWindow } from "./rebuild-shields"; +import * as snapshotBackup from "./snapshot/backup-authority"; export type RebuildSandboxEntry = SandboxEntry & { agents?: unknown[] }; @@ -446,7 +447,13 @@ export function backupSandboxStateForRebuild( console.log(" Backing up sandbox state..."); log(`Agent type: ${sb.agent || "openclaw"}, stateDirs from manifest`); - const backup = sandboxState.backupSandboxState(sandboxName); + const backup = snapshotBackup.backupSandboxStateWithManagedAuthority( + sandboxName, + {}, + { + getSandbox: (name) => loadRegistry().sandboxes[name] ?? null, + }, + ); log( `Backup result: success=${backup.success}, backed=${backup.backedUpDirs.join(",")}; files=${backup.backedUpFiles.join(",")}, failed=${backup.failedDirs.join(",")}; failedFiles=${backup.failedFiles.join(",")}`, ); diff --git a/src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts b/src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts index baeef9340ca..84b3e9a3369 100644 --- a/src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts +++ b/src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts @@ -3,8 +3,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import * as sandboxState from "../../state/sandbox"; import { runRebuildRestorePhase } from "./rebuild-restore-phase"; +import * as snapshotRestore from "./snapshot/restore-authority"; afterEach(() => { vi.restoreAllMocks(); @@ -14,7 +14,7 @@ describe("rebuild restore target forwarding", () => { it("forwards the recreated target identity and explicit custom-image capability", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const restoreRecreatedSandboxState = vi - .spyOn(sandboxState, "restoreRecreatedSandboxState") + .spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority") .mockReturnValue({ success: true, restoredDirs: [], @@ -34,9 +34,14 @@ describe("rebuild restore target forwarding", () => { log: vi.fn(), }); - expect(restoreRecreatedSandboxState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backup", { - targetAgentType: "langchain-deepagents-code", - allowCustomImageWholeStateFileRestore: true, - }); + expect(restoreRecreatedSandboxState).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ backupPath: "/tmp/rebuild-backup" }), + { + targetAgentType: "langchain-deepagents-code", + allowCustomImageWholeStateFileRestore: true, + }, + { getSandbox: expect.any(Function) }, + ); }); }); diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts index dbde96ca34f..831101486d8 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts @@ -11,6 +11,7 @@ import { resolveRestoredPolicyRegistryState, } from "./rebuild-post-restore-phase"; import { runRebuildRestorePhase } from "./rebuild-restore-phase"; +import * as snapshotRestore from "./snapshot/restore-authority"; const BUILTIN_OBSERVABILITY_CONTENT = "network_policies:\n observability-otlp-local:\n name: observability-otlp-local\n"; @@ -44,7 +45,7 @@ describe("rebuild policy restore fidelity", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); const log = vi.fn(); - vi.spyOn(sandboxState, "restoreRecreatedSandboxState").mockReturnValue({ + vi.spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority").mockReturnValue({ success: false, restoredDirs: [], restoredFiles: [], @@ -76,7 +77,7 @@ describe("rebuild policy restore fidelity", () => { vi.spyOn(console, "error").mockImplementation(() => undefined); const parsePresetPolicyKeys = vi.spyOn(policies, "parsePresetPolicyKeys"); const restoreRecreatedSandboxState = vi - .spyOn(sandboxState, "restoreRecreatedSandboxState") + .spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority") .mockReturnValue({ success: true, restoredDirs: [], @@ -104,9 +105,14 @@ describe("rebuild policy restore fidelity", () => { log: vi.fn(), }); - expect(restoreRecreatedSandboxState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backup", { - targetAgentType: "openclaw", - }); + expect(restoreRecreatedSandboxState).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ backupPath: "/tmp/rebuild-backup" }), + { + targetAgentType: "openclaw", + }, + { getSandbox: expect.any(Function) }, + ); expect(applyPreset).toHaveBeenCalledOnce(); expect(applyPreset).toHaveBeenCalledWith("alpha", "npm"); for (const entry of customPolicies) { diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.ts b/src/lib/actions/sandbox/rebuild-restore-phase.ts index 2ce0640077c..057d7bd9ba9 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.ts @@ -8,11 +8,13 @@ import { OBSERVABILITY_POLICY_BINDING, } from "../../onboard/observability-policy-presets"; import * as policies from "../../policy"; +import { load as loadRegistry } from "../../state/registry/persistence"; import * as sandboxState from "../../state/sandbox"; import { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import type { RebuildLog } from "./rebuild-credential-preflight"; import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import * as snapshotRestore from "./snapshot/restore-authority"; export interface RebuildRestorePhaseInput { sandboxName: string; @@ -191,13 +193,16 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild console.log(""); console.log(" Restoring workspace state..."); log(`Restoring from: ${backupManifest.backupPath} into sandbox: ${sandboxName}`); - const restore = sandboxState.restoreRecreatedSandboxState( + const restore = snapshotRestore.restoreRecreatedSandboxStateWithManagedAuthority( sandboxName, - backupManifest.backupPath, + backupManifest, { targetAgentType, ...(targetImageIsCustom ? { allowCustomImageWholeStateFileRestore: true } : {}), }, + { + getSandbox: (name) => loadRegistry().sandboxes[name] ?? null, + }, ); log( `Restore result: success=${restore.success}, restored=${restore.restoredDirs.join(",")}; files=${restore.restoredFiles.join(",")}, failed=${restore.failedDirs.join(",")}; failedFiles=${restore.failedFiles.join(",")}${restore.error ? `; error=${restore.error}` : ""}`, diff --git a/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts b/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts new file mode 100644 index 00000000000..f5082a04cbb --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../onboard/managed-image/contract"; +import { encodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; + +import * as fixture from "./snapshot-restore-test-fixture"; + +const providerRestore = vi.hoisted(() => { + const events: string[] = []; + const provider = { identity: { id: "docker" } }; + const managedProfile = { + agent: "openclaw", + profileFingerprint: "a".repeat(64), + }; + const source = { + schemaVersion: 1, + providerId: "docker", + providerHandle: "snapshot-provider-handle", + lifecycleState: "running", + lifecycleGeneration: "snapshot-generation", + runtime: { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "container-id" }, + acceleration: { kind: "none" }, + }, + }; + const readManagedSnapshotProfileAuthority = vi.fn( + (_source?: unknown): { agent: string } | null => ({ + agent: "openclaw", + }), + ); + const prepareManagedSnapshotProfileRestore = vi.fn(() => ({ + providerRestoreAuthority: managedProfile, + })); + const requireCurrentSnapshotRuntimeProvider = vi.fn(() => provider); + const prepareSandboxRuntimeRestore = vi.fn(() => { + events.push("provider-preflight"); + return { + phase: "preflighted", + targetProviderId: "docker", + targetSandboxName: "alpha", + source, + preflight: {}, + managedProfile, + }; + }); + const confirmSandboxRuntimeRestore = vi.fn(() => { + events.push("provider-restore-proof"); + return { phase: "validated" }; + }); + return { + events, + source, + readManagedSnapshotProfileAuthority, + prepareManagedSnapshotProfileRestore, + requireCurrentSnapshotRuntimeProvider, + prepareSandboxRuntimeRestore, + confirmSandboxRuntimeRestore, + }; +}); + +vi.mock("./snapshot/dependencies", () => ({ + backupSandboxStateWithManagedAuthority: vi.fn(), + captureSandboxRuntimeSnapshot: vi.fn(), + confirmSandboxRuntimeRestore: providerRestore.confirmSandboxRuntimeRestore, + prepareManagedSnapshotProfileRestore: providerRestore.prepareManagedSnapshotProfileRestore, + prepareSandboxRuntimeRestore: providerRestore.prepareSandboxRuntimeRestore, + readManagedSnapshotProfileAuthority: providerRestore.readManagedSnapshotProfileAuthority, + rejectManagedSnapshotCloneUntilRebind: vi.fn(), + requireCurrentSnapshotRuntimeProvider: providerRestore.requireCurrentSnapshotRuntimeProvider, +})); + +function managedWorkload(agent: ShippedManagedImageAgent = "openclaw") { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile(agent)); + return { + schemaVersion: 1 as const, + kind: "managed-image" as const, + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64" as const, + release: "v0.0.100", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1 as const, + startupProfileContractVersion: 1 as const, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true as const, + }; +} + +function managedSnapshot(agent: ShippedManagedImageAgent = "openclaw") { + return { + snapshotVersion: 4, + timestamp: "2026-07-30T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + agentType: agent, + workload: managedWorkload(agent), + runtimeSnapshot: providerRestore.source, + }; +} + +beforeEach(() => { + fixture.resetSnapshotRestoreMocks(); + providerRestore.events.length = 0; + providerRestore.readManagedSnapshotProfileAuthority.mockClear(); + providerRestore.prepareManagedSnapshotProfileRestore.mockClear(); + providerRestore.requireCurrentSnapshotRuntimeProvider.mockClear(); + providerRestore.prepareSandboxRuntimeRestore.mockClear(); + providerRestore.confirmSandboxRuntimeRestore.mockClear(); + fixture.getLatestBackupMock.mockReturnValue(managedSnapshot()); + fixture.getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "openclaw", + openshellDriver: "docker", + }); + fixture.restoreSandboxStateMock.mockImplementation((_name, _path, options) => { + try { + options?.validateBeforeMutation?.(); + } catch (error) { + return { + success: false, + restoredDirs: [], + restoredFiles: [], + failedDirs: ["workspace"], + failedFiles: [], + error: error instanceof Error ? error.message : String(error), + }; + } + providerRestore.events.push("filesystem-restore"); + return { + success: true, + restoredDirs: ["workspace"], + restoredFiles: [], + failedDirs: [], + failedFiles: [], + }; + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); +}); +afterEach(() => { + fixture.cleanupSnapshotRestoreMocks(); +}); + +describe("managed snapshot provider restore ordering", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("refreshes %s provider authority at the mutation edge and proves the profile", async (agent) => { + fixture.getLatestBackupMock.mockReturnValue(managedSnapshot(agent)); + fixture.getSandboxMock.mockReturnValue({ + name: "alpha", + agent, + openshellDriver: "docker", + }); + providerRestore.readManagedSnapshotProfileAuthority.mockReturnValue({ agent }); + providerRestore.prepareManagedSnapshotProfileRestore.mockReturnValue({ + providerRestoreAuthority: { + agent, + profileFingerprint: "a".repeat(64), + }, + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(providerRestore.events).toEqual([ + "provider-preflight", + "provider-preflight", + "filesystem-restore", + "provider-restore-proof", + ]); + expect(providerRestore.prepareSandboxRuntimeRestore).toHaveBeenCalledTimes(2); + expect(providerRestore.confirmSandboxRuntimeRestore).toHaveBeenCalledOnce(); + }); + + it("aborts before filesystem mutation when mutation-edge validation fails", async () => { + providerRestore.prepareSandboxRuntimeRestore + .mockImplementationOnce(() => { + providerRestore.events.push("provider-preflight"); + return { + phase: "preflighted", + targetProviderId: "docker", + targetSandboxName: "alpha", + source: providerRestore.source, + preflight: {}, + managedProfile: { + agent: "openclaw", + profileFingerprint: "a".repeat(64), + }, + }; + }) + .mockImplementationOnce(() => { + providerRestore.events.push("provider-preflight-rejected"); + throw new Error("runtime changed after snapshot preflight"); + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect(runSandboxSnapshot("alpha", { kind: "restore" })).rejects.toMatchObject({ + exitCode: 1, + }); + + expect(providerRestore.events).toEqual(["provider-preflight", "provider-preflight-rejected"]); + expect(fixture.restoreSandboxStateMock).toHaveBeenCalledWith( + "alpha", + "/tmp/backup-alpha", + expect.objectContaining({ validateBeforeMutation: expect.any(Function) }), + ); + expect(providerRestore.confirmSandboxRuntimeRestore).not.toHaveBeenCalled(); + }); +}); + +describe("legacy snapshot compatibility gate", () => { + beforeEach(() => { + fixture.getLatestBackupMock.mockReturnValue({ + snapshotVersion: 3, + timestamp: "2026-07-29T00:00:00.000Z", + backupPath: "/tmp/legacy-backup-alpha", + agentType: "openclaw", + }); + providerRestore.readManagedSnapshotProfileAuthority.mockImplementation((source: unknown) => + (source as { workload?: unknown }).workload ? { agent: "openclaw" } : null, + ); + }); + + it("rejects self-restore when the current target is managed", async () => { + fixture.getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "openclaw", + openshellDriver: "docker", + workload: managedWorkload(), + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect(runSandboxSnapshot("alpha", { kind: "restore" })).rejects.toMatchObject({ + exitCode: 1, + }); + + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("legacy snapshot lacks managed workload"), + ); + expect(fixture.restoreSandboxStateMock).not.toHaveBeenCalled(); + expect(providerRestore.prepareSandboxRuntimeRestore).not.toHaveBeenCalled(); + }); + + it.each([ + "source", + "destination", + ] as const)("rejects cross-clone when the current %s is managed", async (managedSide) => { + const source = { + name: "alpha", + agent: "openclaw" as const, + openshellDriver: "docker", + imageTag: "legacy-source:test", + ...(managedSide === "source" ? { workload: managedWorkload() } : {}), + }; + const destination = + managedSide === "destination" + ? { + name: "beta", + agent: "openclaw" as const, + openshellDriver: "docker", + imageTag: "managed-target@test", + workload: managedWorkload(), + } + : null; + fixture.getSandboxMock.mockImplementation((name) => + name === "alpha" ? source : name === "beta" ? destination : null, + ); + fixture.parseLiveSandboxNamesMock.mockReturnValue( + new Set(managedSide === "destination" ? ["alpha", "beta"] : ["alpha"]), + ); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("legacy snapshot lacks managed workload"), + ); + expect( + fixture.runOpenshellMock.mock.calls.some( + ([args]) => args[0] === "sandbox" && args[1] === "delete", + ), + ).toBe(false); + expect(fixture.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(fixture.restoreSandboxStateMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts index 2cfbd8af86e..db2c42cf0ae 100644 --- a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts @@ -20,6 +20,54 @@ afterEach(() => { } }); describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { + it("holds the per-sandbox mutation lock across snapshot creation", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-snapshot-create-lock-")); + tempHomes.push(tempHome); + vi.stubEnv("HOME", tempHome); + let releaseLock: (() => void) | undefined; + let signalLocked: (() => void) | undefined; + const locked = new Promise((resolve) => { + signalLocked = resolve; + }); + const release = new Promise((resolve) => { + releaseLock = resolve; + }); + const externalMutation = withSandboxMutationLock("alpha", async () => { + signalLocked?.(); + await release; + }); + await locked; + f.backupSandboxStateMock.mockReturnValue({ + success: true, + manifest: { + timestamp: "2026-07-31T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + }, + backedUpDirs: [], + restoredDirs: [], + backedUpFiles: [], + failedDirs: [], + failedFiles: [], + }); + f.findBackupMock.mockReturnValue({ + match: { + snapshotVersion: 4, + timestamp: "2026-07-31T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + }, + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + const create = runSandboxSnapshot("alpha", { kind: "create" }); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(f.backupSandboxStateMock).not.toHaveBeenCalled(); + + releaseLock?.(); + await externalMutation; + await create; + expect(f.backupSandboxStateMock).toHaveBeenCalledWith("alpha", { name: null }); + }); + it("restores the latest snapshot into the source sandbox", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); f.getLatestBackupMock.mockReturnValue({ diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index ee45ff695b4..66819eb0951 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { vi } from "vitest"; +import type { SandboxWorkloadReceipt } from "../../state/registry/types"; import { SANDBOX_EXEC_STARTED_MARKER } from "./sandbox-exec-output"; import type { SnapshotStreamSandboxCreateMock } from "./snapshot-create-stream-test-types"; @@ -41,12 +42,7 @@ export type SandboxRecord = { fromDockerfile?: string | null; gatewayName?: string | null; imageTag?: string | null; - workload?: { - schemaVersion: 1; - kind: "legacy-dockerfile"; - reference: string | null; - shared: false; - }; + workload?: SandboxWorkloadReceipt; openshellDriver?: string | null; observabilityEnabled?: boolean; provider?: string | null; @@ -132,6 +128,11 @@ const lifecycleMock = vi.hoisted(() => { }); export const backupSandboxStateMock = vi.fn(); +export const captureSnapshotRestoreAuthorityMock = vi.fn(() => ({ + schemaVersion: 1 as const, + backupPath: "/tmp/backup-alpha", + contentSha256: "a".repeat(64), +})); export const loadAgentMock = vi.fn((name: string) => ({ name, policyAdditionsPath: name === "openclaw" ? null : `/repo/agents/${name}/policy-additions.yaml`, @@ -291,6 +292,7 @@ vi.mock("../../state/gateway", () => ({ })); vi.mock("../../state/registry", () => ({ + getBaselineExclusions: vi.fn(() => []), getConfiguredMessagingChannelsFromEntry: vi.fn(() => []), getCustomPolicies: getCustomPoliciesMock, getDisabledMessagingChannelsFromEntry: vi.fn(() => []), @@ -306,6 +308,7 @@ vi.mock("../../state/registry", () => ({ vi.mock("../../state/sandbox", () => ({ backupSandboxState: backupSandboxStateMock, + captureSnapshotRestoreAuthority: captureSnapshotRestoreAuthorityMock, findBackup: findBackupMock, getLatestBackup: getLatestBackupMock, listBackups: listBackupsMock, @@ -336,6 +339,11 @@ vi.mock("./restore-gateway-pairing", () => ({ export function resetSnapshotRestoreMocks(): void { vi.clearAllMocks(); + captureSnapshotRestoreAuthorityMock.mockReturnValue({ + schemaVersion: 1, + backupPath: "/tmp/backup-alpha", + contentSha256: "a".repeat(64), + }); shieldsMock.setIsShieldsDownExport(shieldsMock.isShieldsDownMock); shieldsMock.isShieldsDownMock.mockReturnValue(true); shieldsMock.shieldsUpMock.mockImplementation(() => lifecycleMock.events.push("harden")); diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index eb9023636a3..2a1f1562a80 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -79,6 +79,16 @@ import { selectSandboxGatewayIfRegistered, usesGatewayMetadataProbe, } from "./sandbox-gateway-routing"; +import { + backupSandboxStateWithManagedAuthority, + confirmSandboxRuntimeRestore, + type PreparedSandboxRuntimeRestore, + prepareManagedSnapshotProfileRestore, + prepareSandboxRuntimeRestore, + readManagedSnapshotProfileAuthority, + rejectManagedSnapshotCloneUntilRebind, + requireCurrentSnapshotRuntimeProvider, +} from "./snapshot/dependencies"; import { formatSnapshotBaselineExclusionSummary } from "./snapshot-baseline-exclusion-summary"; import { printHermesGatewayRestoreHint } from "./snapshot-hermes-gateway-hint"; @@ -678,9 +688,13 @@ function runSnapshotCreate( } const label = request.name ? ` (--name ${request.name})` : ""; console.log(` Creating snapshot of '${sandboxName}'${label}...`); - const result = sandboxState.backupSandboxState(sandboxName, { - name: request.name ?? null, - }); + const result = backupSandboxStateWithManagedAuthority( + sandboxName, + { + name: request.name ?? null, + }, + { getSandbox: registry.getSandbox }, + ); if (result.success) { const manifest = result.manifest!; const entry = sandboxState.findBackup(sandboxName, manifest.timestamp).match ?? manifest; @@ -949,6 +963,18 @@ function reconcileSnapshotCustomPolicies( } } +function readCurrentManagedSnapshotProfileAuthority(entry: SandboxEntry | null) { + return entry + ? readManagedSnapshotProfileAuthority({ + sandboxName: entry.name, + agentType: entry.agent ?? "", + imageTag: entry.imageTag, + fromDockerfile: entry.fromDockerfile, + workload: entry.workload, + }) + : null; +} + async function runSnapshotRestore( sandboxName: string, request: Extract, @@ -1026,6 +1052,53 @@ async function runSnapshotRestoreUnlocked( console.log(` Using latest snapshot ${v}${nameSuffix} (${latest.timestamp})`); } + const snapshotProfileSource = { + sandboxName, + agentType: resolvedSnapshot.agentType, + workload: resolvedSnapshot.workload, + }; + const currentSourceEntry = registry.getSandbox(sandboxName); + let hasManagedProfileAuthority = false; + let snapshotRestoreAuthority: sandboxState.SnapshotRestoreAuthority | null = null; + try { + const snapshotAuthority = readManagedSnapshotProfileAuthority(snapshotProfileSource); + hasManagedProfileAuthority = snapshotAuthority !== null; + if (hasManagedProfileAuthority && !resolvedSnapshot.runtimeSnapshot) { + throw new Error("managed snapshot is missing provider runtime authority"); + } + const currentSourceAuthority = readCurrentManagedSnapshotProfileAuthority(currentSourceEntry); + const currentTargetAuthority = + targetEntry && targetEntry !== currentSourceEntry + ? readCurrentManagedSnapshotProfileAuthority(targetEntry) + : currentSourceAuthority; + if (!hasManagedProfileAuthority && (currentSourceAuthority || currentTargetAuthority)) { + throw new Error( + "legacy snapshot lacks managed workload and provider runtime authority required by the current source or destination", + ); + } + if (isCrossSandboxRestore && hasManagedProfileAuthority) { + rejectManagedSnapshotCloneUntilRebind(snapshotProfileSource, targetSandbox); + } + if (hasManagedProfileAuthority) { + snapshotRestoreAuthority = sandboxState.captureSnapshotRestoreAuthority( + backupPath, + resolvedSnapshot, + ); + if (!snapshotRestoreAuthority) { + throw new Error("selected snapshot content changed during restore preflight"); + } + } + } catch (error) { + console.error( + ` Cannot restore managed snapshot authority: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + console.error(` Destination '${targetSandbox}' was not changed.`); + snapshotExit(1); + } + + let preparedRuntimeRestore: PreparedSandboxRuntimeRestore | null = null; if (!isCrossSandboxRestore) { // Self-restore: target is `sandboxName`. Cannot auto-create; the // source pod is the target, so it must already be live. @@ -1033,6 +1106,39 @@ async function runSnapshotRestoreUnlocked( console.error(` Sandbox '${targetSandbox}' is not running. Cannot restore snapshot.`); snapshotExit(1); } + if (hasManagedProfileAuthority) { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget || !resolvedSnapshot.runtimeSnapshot) { + console.error( + ` Cannot restore managed snapshot '${sandboxName}': target or provider runtime authority is missing.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + const profileRestore = prepareManagedSnapshotProfileRestore( + snapshotProfileSource, + currentTarget, + provider, + ); + if (!profileRestore) { + throw new Error("managed profile restore authority is missing"); + } + preparedRuntimeRestore = prepareSandboxRuntimeRestore( + provider, + currentTarget, + resolvedSnapshot.runtimeSnapshot, + profileRestore.providerRestoreAuthority, + ); + } catch (error) { + console.error( + ` Cannot preflight managed snapshot restore: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + snapshotExit(1); + } + } } else { // #3756: cross-sandbox restore into a destination that already exists // used to overlay onto the live filesystem silently. Refuse by default @@ -1179,13 +1285,74 @@ async function runSnapshotRestoreUnlocked( // reconciliation under the active timer generation. Normal auto-restore // waits; the absolute deadline may preempt this process and reclaim the // token, preventing policy/config mutation after lockdown resumes. + const validateManagedRestoreBeforeMutation = preparedRuntimeRestore + ? () => { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + throw new Error(`target '${targetSandbox}' is no longer registered`); + } + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + const profileRestore = prepareManagedSnapshotProfileRestore( + snapshotProfileSource, + currentTarget, + provider, + ); + if (!profileRestore) { + throw new Error("managed profile restore authority is missing"); + } + const prepared = preparedRuntimeRestore; + if (!prepared) throw new Error("managed runtime restore authority is missing"); + // The state layer invokes this after local tar staging and + // immediately before its first remote filesystem mutation. + preparedRuntimeRestore = prepareSandboxRuntimeRestore( + provider, + currentTarget, + prepared.source, + profileRestore.providerRestoreAuthority, + ); + } + : null; if (targetSandbox !== sandboxName) { console.log(` Restoring snapshot from '${sandboxName}' into '${targetSandbox}'...`); } else { console.log(` Restoring snapshot into '${sandboxName}'...`); } - const result = sandboxState.restoreSandboxState(targetSandbox, backupPath); + if (Boolean(snapshotRestoreAuthority) !== Boolean(validateManagedRestoreBeforeMutation)) { + console.error( + ` Cannot restore managed snapshot '${sandboxName}': content authority and the runtime mutation fence must both be present.`, + ); + console.error(` Destination '${targetSandbox}' was not changed.`); + snapshotExit(1); + } + const result = + snapshotRestoreAuthority && validateManagedRestoreBeforeMutation + ? sandboxState.restoreSandboxState(targetSandbox, backupPath, { + authority: snapshotRestoreAuthority, + validateBeforeMutation: validateManagedRestoreBeforeMutation, + }) + : sandboxState.restoreSandboxState(targetSandbox, backupPath); if (result.success) { + if (preparedRuntimeRestore) { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + console.error( + ` Managed snapshot state was restored, but target '${targetSandbox}' is no longer registered.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + confirmSandboxRuntimeRestore(provider, currentTarget, preparedRuntimeRestore); + } catch (error) { + console.error( + ` Managed snapshot state was restored, but provider restore proof failed: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + console.error(" Retry this exact snapshot after the runtime provider stabilizes."); + snapshotExit(1); + } + } console.log( ` ${G}\u2713${R} Restored ${result.restoredDirs.length} directories, ${result.restoredFiles.length} files`, ); @@ -1206,6 +1373,9 @@ async function runSnapshotRestoreUnlocked( if (result.failedFiles.length > 0) { console.error(` Failed files: ${result.failedFiles.join(", ")}`); } + if (result.error) { + console.error(` Reason: ${result.error}`); + } snapshotExit(1); } // Post-restore security-state reconciliation is best-effort by design: the @@ -1245,7 +1415,7 @@ export async function runSandboxSnapshot( ) { switch (request.kind) { case "create": { - runSnapshotCreate(sandboxName, request); + await withSandboxMutationLock(sandboxName, () => runSnapshotCreate(sandboxName, request)); break; } case "list": { diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts new file mode 100644 index 00000000000..8221b1535bc --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -0,0 +1,255 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../../onboard/managed-image/contract"; +import { encodeManagedStartupProfile } from "../../../onboard/managed-startup/profile"; +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; +import type { BackupOptions, BackupResult } from "../../../state/sandbox"; +import { backupSandboxStateWithManagedAuthority } from "./backup-authority"; + +function workload( + agent: ShippedManagedImageAgent, + changedProfile = false, +): Extract { + const encodedProfile = encodeManagedStartupProfile( + managedStartupE2eProfile(agent, changedProfile), + ); + return { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.88", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function sandbox( + agent: ShippedManagedImageAgent, + receipt: SandboxWorkloadReceipt = workload(agent), +): SandboxEntry { + return { + name: "alpha", + agent, + openshellDriver: "mxc", + imageTag: receipt.kind === "managed-image" ? receipt.reference : null, + fromDockerfile: null, + workload: receipt, + }; +} + +function runtime(handle = "session-1") { + return { + schemaVersion: 1, + providerId: "mxc", + providerHandle: `opaque-${handle}`, + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "session", handle }, + acceleration: { kind: "none" }, + }, + } as const; +} + +function provider(acceptsReceipt = true): RuntimeProviderBundle { + return { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + workload: { + providerId: "mxc", + supported: true, + profile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: false, + }, + acceptsReceipt: () => acceptsReceipt, + }, + } as unknown as RuntimeProviderBundle; +} + +function successfulBackup(options: BackupOptions): BackupResult { + try { + options.validateBeforePublish?.(); + } catch (error) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: error instanceof Error ? error.message : String(error), + }; + } + return { + success: true, + manifest: { + version: 1, + sandboxName: "alpha", + timestamp: "2026-07-31T00-00-00-000Z", + agentType: "openclaw", + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox", + backupPath: "/tmp/alpha", + blueprintDigest: null, + }, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + }; +} + +describe("managed snapshot backup authority", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("captures and republishes exact %s provider authority", (agent) => { + const entry = sandbox(agent); + const getSandbox = vi.fn(() => entry); + const requireProvider = vi.fn(() => provider()); + const captureRuntime = vi.fn(() => runtime()); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "stable" }, + { getSandbox, requireProvider, captureRuntime, backup }, + ); + + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + name: "stable", + workload: entry.workload, + runtimeSnapshot: runtime(), + validateBeforePublish: expect.any(Function), + }), + ); + expect(getSandbox).toHaveBeenCalledTimes(2); + expect(requireProvider).toHaveBeenCalledTimes(2); + expect(captureRuntime).toHaveBeenCalledTimes(2); + }); + + it("keeps explicit Dockerfile backups on the legacy state-only path", () => { + const entry = { + name: "alpha", + agent: "openclaw", + openshellDriver: "mxc", + fromDockerfile: "/tmp/Dockerfile", + } satisfies SandboxEntry; + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + const requireProvider = vi.fn(); + const captureRuntime = vi.fn(); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "legacy" }, + { + getSandbox: () => entry, + requireProvider, + captureRuntime: captureRuntime as never, + backup, + }, + ); + + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith("alpha", { name: "legacy" }); + expect(requireProvider).not.toHaveBeenCalled(); + expect(captureRuntime).not.toHaveBeenCalled(); + }); + + it("fails before filesystem capture when the provider rejects managed authority", () => { + const entry = sandbox("openclaw"); + const backup = vi.fn(); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox: () => entry, + requireProvider: () => provider(false), + captureRuntime: vi.fn() as never, + backup, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("does not accept the managed workload receipt"), + }); + expect(backup).not.toHaveBeenCalled(); + }); + + it.each([ + { + label: "workload", + secondEntry: sandbox("openclaw", workload("openclaw", true)), + secondRuntime: runtime(), + error: "managed workload changed during backup", + }, + { + label: "runtime", + secondEntry: sandbox("openclaw"), + secondRuntime: runtime("session-2"), + error: "runtime changed during backup", + }, + ])("rejects $label drift before manifest publication", ({ + secondEntry, + secondRuntime, + error, + }) => { + const initialEntry = sandbox("openclaw"); + const getSandbox = vi + .fn<() => SandboxEntry | null>() + .mockReturnValueOnce(initialEntry) + .mockReturnValueOnce(secondEntry); + const captureRuntime = vi + .fn<() => ReturnType>() + .mockReturnValueOnce(runtime()) + .mockReturnValueOnce(secondRuntime); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox, + requireProvider: () => provider(), + captureRuntime: captureRuntime as ( + bundle: RuntimeProviderBundle, + entry: SandboxEntry, + ) => ReturnType, + backup, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining(error), + }); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts new file mode 100644 index 00000000000..bc8ccb371b3 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; +import type { SandboxEntry } from "../../../state/registry/types"; +import * as sandboxState from "../../../state/sandbox"; +import { readManagedSnapshotProfileAuthority } from "./managed-profile"; +import { captureSandboxRuntimeSnapshot } from "./provider-lifecycle"; + +type SnapshotBackupAuthority = Pick< + sandboxState.BackupOptions, + "runtimeSnapshot" | "workload" | "validateBeforePublish" +>; + +interface SnapshotBackupAuthorityDependencies { + readonly getSandbox: (sandboxName: string) => SandboxEntry | null; + readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; + readonly captureRuntime: typeof captureSandboxRuntimeSnapshot; + readonly backup: typeof sandboxState.backupSandboxState; +} + +const defaultDependencies: Omit = { + requireProvider: (sandbox) => + requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), + captureRuntime: captureSandboxRuntimeSnapshot, + // Keep the call late-bound so tests and alternative state stores can replace + // the module export without this adapter retaining an import-time reference. + backup: (...args) => sandboxState.backupSandboxState(...args), +}; + +function failure(error: unknown): sandboxState.BackupResult { + const detail = error instanceof Error ? error.message : String(error); + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: `Cannot capture managed snapshot authority: ${detail}.`, + }; +} + +function backupStateOnly( + dependencies: SnapshotBackupAuthorityDependencies, + sandboxName: string, + options: Pick, +): sandboxState.BackupResult { + return options.name === undefined + ? dependencies.backup(sandboxName) + : dependencies.backup(sandboxName, options); +} + +function readAuthority(entry: SandboxEntry) { + return readManagedSnapshotProfileAuthority({ + sandboxName: entry.name, + agentType: entry.agent ?? "", + imageTag: entry.imageTag, + fromDockerfile: entry.fromDockerfile, + workload: entry.workload, + }); +} + +function captureManagedAuthority( + entry: SandboxEntry, + dependencies: SnapshotBackupAuthorityDependencies, +): SnapshotBackupAuthority | null { + const authority = readAuthority(entry); + if (!authority) return null; + const provider = dependencies.requireProvider(entry); + if (!provider.workload.acceptsReceipt(authority.receipt)) { + throw new Error( + `runtime provider '${provider.identity.id}' does not accept the managed workload receipt`, + ); + } + const runtimeSnapshot = dependencies.captureRuntime(provider, entry); + const workload = authority.receipt; + + return { + runtimeSnapshot, + workload, + validateBeforePublish: () => { + const current = dependencies.getSandbox(entry.name); + if (!current) { + throw new Error(`sandbox '${entry.name}' is no longer registered`); + } + const currentAuthority = readAuthority(current); + if (!currentAuthority || !isDeepStrictEqual(currentAuthority.receipt, workload)) { + throw new Error(`sandbox '${entry.name}' managed workload changed during backup`); + } + const currentProvider = dependencies.requireProvider(current); + if ( + currentProvider.identity.id !== provider.identity.id || + !currentProvider.workload.acceptsReceipt(currentAuthority.receipt) + ) { + throw new Error(`sandbox '${entry.name}' runtime provider changed during backup`); + } + const currentRuntime = dependencies.captureRuntime(currentProvider, current); + if (!isDeepStrictEqual(currentRuntime, runtimeSnapshot)) { + throw new Error(`sandbox '${entry.name}' runtime changed during backup`); + } + }, + }; +} + +/** + * Capture one managed workload and runtime authority pair around the complete + * filesystem copy. The state layer publishes the manifest only after the + * final callback confirms that the same provider authority remains live. + */ +export function backupSandboxStateWithManagedAuthority( + sandboxName: string, + options: Pick = {}, + overrides: Pick & + Partial>, +): sandboxState.BackupResult { + const dependencies = { ...defaultDependencies, ...overrides }; + const entry = dependencies.getSandbox(sandboxName); + if (!entry) return backupStateOnly(dependencies, sandboxName, options); + + let authority: SnapshotBackupAuthority | null; + try { + authority = captureManagedAuthority(entry, dependencies); + } catch (error) { + return failure(error); + } + return authority + ? dependencies.backup(sandboxName, { ...options, ...authority }) + : backupStateOnly(dependencies, sandboxName, options); +} diff --git a/src/lib/actions/sandbox/snapshot/dependencies.ts b/src/lib/actions/sandbox/snapshot/dependencies.ts new file mode 100644 index 00000000000..e428143b209 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/dependencies.ts @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; +import type { SandboxEntry } from "../../../state/registry/types"; + +export { + ManagedSnapshotProfileRestoreError, + prepareManagedSnapshotProfileRestore, + readManagedSnapshotProfileAuthority, + rejectManagedSnapshotCloneUntilRebind, +} from "./managed-profile"; +export type { + PreparedSandboxRuntimeRestore, + ValidatedSandboxRuntimeRestore, +} from "./provider-lifecycle"; +export { + captureSandboxRuntimeSnapshot, + confirmSandboxRuntimeRestore, + prepareSandboxRuntimeRestore, + SandboxSnapshotProviderError, +} from "./provider-lifecycle"; +export { backupSandboxStateWithManagedAuthority } from "./backup-authority"; + +/** + * Resolve the one already-registered provider bundle for a durable sandbox. + * Snapshot actions never maintain a second provider map or infer a container + * engine from host state. + */ +export function requireCurrentSnapshotRuntimeProvider( + sandbox: SandboxEntry, +): RuntimeProviderBundle { + return requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES); +} diff --git a/src/lib/actions/sandbox/snapshot/managed-profile.test.ts b/src/lib/actions/sandbox/snapshot/managed-profile.test.ts new file mode 100644 index 00000000000..29596cb5275 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/managed-profile.test.ts @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../../onboard/managed-image/contract"; +import { + encodeManagedStartupProfile, + fingerprintManagedStartupProfile, +} from "../../../onboard/managed-startup/profile"; +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; +import { + prepareManagedSnapshotProfileRestore, + readManagedSnapshotProfileAuthority, + rejectManagedSnapshotCloneUntilRebind, +} from "./managed-profile"; + +function workload( + agent: ShippedManagedImageAgent, + changedProfile = false, +): Extract { + const encodedProfile = encodeManagedStartupProfile( + managedStartupE2eProfile(agent, changedProfile), + ); + return { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.88", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function sandbox(agent: ShippedManagedImageAgent, receipt = workload(agent)): SandboxEntry { + return { + name: "alpha", + agent, + openshellDriver: "mxc", + imageTag: receipt.reference, + fromDockerfile: null, + workload: receipt, + }; +} + +function provider(accepted = true, managedProfileRestore = true): RuntimeProviderBundle { + return { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + workload: { + providerId: "mxc", + supported: true, + profile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: false, + }, + acceptsReceipt: () => accepted, + }, + snapshot: { + providerId: "mxc", + supported: true, + contractVersion: 1, + capabilities: { + backup: true, + restore: true, + managedProfileRestore, + }, + preflight: () => { + throw new Error("profile preflight must not perform runtime effects"); + }, + capture: () => { + throw new Error("profile preflight must not perform runtime effects"); + }, + validateRestore: () => { + throw new Error("profile preflight must not perform runtime effects"); + }, + restore: () => { + throw new Error("profile preflight must not perform runtime effects"); + }, + }, + } as unknown as RuntimeProviderBundle; +} + +describe("managed snapshot profile restore", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("validates exact secret-free %s profile authority", (agent) => { + const receipt = workload(agent); + const source = { sandboxName: "alpha", agentType: agent, workload: receipt }; + + const plan = prepareManagedSnapshotProfileRestore(source, sandbox(agent, receipt), provider()); + + expect(plan).toMatchObject({ + schemaVersion: 1, + providerId: "mxc", + sourceSandboxName: "alpha", + targetSandboxName: "alpha", + authority: { + agent, + receipt, + profile: { agent }, + }, + providerRestoreAuthority: { + agent, + profileFingerprint: fingerprintManagedStartupProfile(managedStartupE2eProfile(agent)), + }, + }); + }); + + it("returns null for legacy snapshots without managed workload authority", () => { + expect( + readManagedSnapshotProfileAuthority({ + sandboxName: "legacy", + agentType: "openclaw", + }), + ).toBeNull(); + }); + + it("rejects malformed snapshot authority before consulting the target", () => { + const receipt = { + ...workload("hermes"), + startupProfileSha256: "0".repeat(64), + }; + expect(() => + prepareManagedSnapshotProfileRestore( + { sandboxName: "alpha", agentType: "hermes", workload: receipt }, + sandbox("hermes"), + provider(), + ), + ).toThrow(/invalid managed workload authority/u); + }); + + it("rejects target profile drift and provider refusal", () => { + const receipt = workload("openclaw"); + const source = { sandboxName: "alpha", agentType: "openclaw", workload: receipt }; + expect(() => + prepareManagedSnapshotProfileRestore( + source, + sandbox("openclaw", workload("openclaw", true)), + provider(), + ), + ).toThrow(/requires a managed image or startup-profile rebind/u); + expect(() => + prepareManagedSnapshotProfileRestore(source, sandbox("openclaw", receipt), provider(false)), + ).toThrow(/does not accept the snapshot workload receipt/u); + expect(() => + prepareManagedSnapshotProfileRestore( + source, + sandbox("openclaw", receipt), + provider(true, false), + ), + ).toThrow(/does not support managed-profile restore/u); + }); + + it("fails before a managed cross-sandbox clone can reach image-only creation", () => { + expect(() => + rejectManagedSnapshotCloneUntilRebind( + { + sandboxName: "alpha", + agentType: "langchain-deepagents-code", + workload: workload("langchain-deepagents-code"), + }, + "beta", + ), + ).toThrow(/requires managed-profile clone rebind/u); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/managed-profile.ts b/src/lib/actions/sandbox/snapshot/managed-profile.ts new file mode 100644 index 00000000000..a35ceaa9062 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/managed-profile.ts @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; +import { fingerprintManagedStartupProfile } from "../../../onboard/managed-startup/profile"; +import type { + RuntimeProviderBundle, + RuntimeProviderManagedProfileRestoreAuthority, +} from "../../../onboard/runtime-provider/contract"; +import { normalizeRuntimeProviderIdentity } from "../../../onboard/runtime-provider/registry"; +import { + type ManagedWorkloadAuthority, + readManagedWorkloadAuthority, +} from "../../../onboard/workload/authority"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; + +export interface ManagedSnapshotProfileSource { + readonly sandboxName: string; + readonly agentType: string; + readonly imageTag?: string | null; + readonly fromDockerfile?: string | null; + readonly workload?: SandboxWorkloadReceipt; +} + +export interface ManagedSnapshotProfileRestorePlan { + readonly schemaVersion: 1; + readonly providerId: string; + readonly sourceSandboxName: string; + readonly targetSandboxName: string; + readonly authority: ManagedWorkloadAuthority; + readonly providerRestoreAuthority: RuntimeProviderManagedProfileRestoreAuthority; +} + +export class ManagedSnapshotProfileRestoreError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Managed snapshot profile restore failed: ${message}`, options); + this.name = "ManagedSnapshotProfileRestoreError"; + } +} + +/** + * Reconstruct and validate the secret-free managed startup authority stored in + * a snapshot. No mutable release catalog or current image alias is consulted. + */ +export function readManagedSnapshotProfileAuthority( + source: ManagedSnapshotProfileSource, +): ManagedWorkloadAuthority | null { + try { + return readManagedWorkloadAuthority({ + agent: source.agentType, + fromDockerfile: source.fromDockerfile ?? null, + imageTag: + source.imageTag ?? + (source.workload?.kind === "managed-image" ? source.workload.reference : null), + workload: source.workload, + }); + } catch (error) { + throw new ManagedSnapshotProfileRestoreError( + `snapshot '${source.sandboxName}' has invalid managed workload authority`, + { cause: error }, + ); + } +} + +/** + * Validate an in-place managed-profile restore against the selected provider + * and the current exact workload. PR3.8 restores profile-backed state only + * when no image/profile rebind is required; cross-sandbox rebind and activation + * are intentionally owned by the later clone transaction. + */ +export function prepareManagedSnapshotProfileRestore( + source: ManagedSnapshotProfileSource, + target: SandboxEntry, + provider: RuntimeProviderBundle, +): ManagedSnapshotProfileRestorePlan | null { + const sourceAuthority = readManagedSnapshotProfileAuthority(source); + if (!sourceAuthority) return null; + + const targetProviderId = normalizeRuntimeProviderIdentity(target.openshellDriver); + if ( + targetProviderId !== provider.identity.id || + provider.snapshot.providerId !== provider.identity.id + ) { + throw new ManagedSnapshotProfileRestoreError( + `target '${target.name}' does not belong to provider '${provider.identity.id}'`, + ); + } + if ( + provider.snapshot.supported !== true || + provider.snapshot.capabilities.managedProfileRestore !== true + ) { + throw new ManagedSnapshotProfileRestoreError( + `provider '${provider.identity.id}' does not support managed-profile restore`, + ); + } + if (!provider.workload.acceptsReceipt(sourceAuthority.receipt)) { + throw new ManagedSnapshotProfileRestoreError( + `provider '${provider.identity.id}' does not accept the snapshot workload receipt`, + ); + } + + let targetAuthority: ManagedWorkloadAuthority | null; + try { + targetAuthority = readManagedWorkloadAuthority(target); + } catch (error) { + throw new ManagedSnapshotProfileRestoreError( + `target '${target.name}' has invalid managed workload authority`, + { cause: error }, + ); + } + if (!targetAuthority) { + throw new ManagedSnapshotProfileRestoreError( + `target '${target.name}' is not the snapshot's managed workload`, + ); + } + if ( + targetAuthority.agent !== sourceAuthority.agent || + !isDeepStrictEqual(targetAuthority.receipt, sourceAuthority.receipt) || + !isDeepStrictEqual(targetAuthority.contract, sourceAuthority.contract) || + !isDeepStrictEqual(targetAuthority.profile, sourceAuthority.profile) + ) { + throw new ManagedSnapshotProfileRestoreError( + `target '${target.name}' requires a managed image or startup-profile rebind`, + ); + } + + return Object.freeze({ + schemaVersion: 1 as const, + providerId: provider.identity.id, + sourceSandboxName: source.sandboxName, + targetSandboxName: target.name, + authority: sourceAuthority, + providerRestoreAuthority: { + agent: sourceAuthority.agent, + profileFingerprint: fingerprintManagedStartupProfile(sourceAuthority.profile), + }, + }); +} + +/** + * Managed cross-sandbox restore cannot reuse the legacy image-only create path: + * its startup profile contains sandbox-scoped authority. Validate the source + * first, then stop before deletion or creation until clone/rebind is available. + */ +export function rejectManagedSnapshotCloneUntilRebind( + source: ManagedSnapshotProfileSource, + targetSandboxName: string, +): void { + const authority = readManagedSnapshotProfileAuthority(source); + if (!authority) return; + throw new ManagedSnapshotProfileRestoreError( + `restoring '${source.sandboxName}' as '${targetSandboxName}' requires managed-profile clone rebind`, + ); +} diff --git a/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts new file mode 100644 index 00000000000..0770debc921 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts @@ -0,0 +1,411 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { + RuntimeProviderBundle, + RuntimeProviderManagedProfileRestoreAuthority, + RuntimeProviderRuntimeReceipt, +} from "../../../onboard/runtime-provider/contract"; +import type { SandboxEntry } from "../../../state/registry/types"; +import { + captureSandboxRuntimeSnapshot, + confirmSandboxRuntimeRestore, + prepareSandboxRuntimeRestore, +} from "./provider-lifecycle"; + +function sandbox(name = "alpha"): SandboxEntry { + return { name, agent: "openclaw", openshellDriver: "mxc" }; +} + +function runtime(providerId = "mxc"): RuntimeProviderRuntimeReceipt { + return { + schemaVersion: 1, + providerId, + runtime: { kind: "session", handle: `opaque-${providerId}-session` }, + acceleration: { kind: "none" }, + }; +} + +const managedProfile = { + agent: "openclaw", + profileFingerprint: "a".repeat(64), +} as const satisfies RuntimeProviderManagedProfileRestoreAuthority; + +function provider( + options: { + providerId?: string; + preflightProviderId?: string; + runtimeProviderId?: string; + restoreProviderId?: string; + } = {}, +): { + readonly bundle: RuntimeProviderBundle; + readonly preflight: ReturnType; + readonly capture: ReturnType; + readonly validateRestore: ReturnType; + readonly restore: ReturnType; +} { + const providerId = options.providerId ?? "mxc"; + const preflight = vi.fn((operation: "backup" | "restore", entry: SandboxEntry) => ({ + schemaVersion: 1 as const, + providerId: options.preflightProviderId ?? providerId, + operation, + sandboxName: entry.name, + providerHandle: `opaque-${providerId}-preflight`, + lifecycleState: "running" as const, + lifecycleGeneration: "generation-1", + })); + const capture = vi.fn(() => runtime(options.runtimeProviderId ?? providerId)); + const validateRestore = vi.fn(); + const restore = vi.fn( + ( + entry: SandboxEntry, + _preflight: unknown, + _runtime: unknown, + authority: RuntimeProviderManagedProfileRestoreAuthority, + ) => ({ + schemaVersion: 1 as const, + providerId: options.restoreProviderId ?? providerId, + sandboxName: entry.name, + providerHandle: `opaque-${providerId}-restore`, + lifecycleState: "running" as const, + lifecycleGeneration: "generation-1", + runtime: runtime(options.runtimeProviderId ?? providerId), + managedProfile: authority, + }), + ); + return { + bundle: { + identity: { contractVersion: 1, id: providerId, displayName: providerId }, + snapshot: { + providerId, + supported: true, + contractVersion: 1, + capabilities: { backup: true, restore: true, managedProfileRestore: true }, + preflight, + capture, + validateRestore, + restore, + }, + } as unknown as RuntimeProviderBundle, + preflight, + capture, + validateRestore, + restore, + }; +} + +describe("snapshot provider lifecycle", () => { + it("captures provider-neutral runtime and lifecycle state behind opaque handles", () => { + const { bundle, preflight, capture } = provider(); + + expect(captureSandboxRuntimeSnapshot(bundle, sandbox())).toEqual({ + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-mxc-preflight", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: runtime(), + }); + expect(preflight).toHaveBeenCalledWith("backup", expect.objectContaining({ name: "alpha" })); + expect(capture).toHaveBeenCalledOnce(); + }); + + it("preflights before restore and revalidates through the same injected facet", () => { + const { bundle, restore, validateRestore } = provider(); + const target = sandbox("target"); + const source = { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }; + + const prepared = prepareSandboxRuntimeRestore(bundle, target, source, managedProfile); + const validated = confirmSandboxRuntimeRestore(bundle, target, prepared); + + expect(prepared.phase).toBe("preflighted"); + expect(validateRestore).toHaveBeenCalledWith( + target, + prepared.preflight, + expect.objectContaining({ providerId: "mxc" }), + managedProfile, + ); + expect(validated.phase).toBe("validated"); + expect(restore).toHaveBeenCalledWith( + target, + prepared.preflight, + expect.objectContaining({ providerId: "mxc" }), + managedProfile, + ); + expect(validated.restoreReceipt).toMatchObject({ + providerId: "mxc", + managedProfile, + }); + }); + + it("leaves opaque provider and runtime handles under provider ownership", () => { + const { bundle, restore } = provider(); + const target = sandbox("target"); + const prepared = prepareSandboxRuntimeRestore( + bundle, + target, + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source-provider", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: { + ...runtime(), + runtime: { kind: "session", handle: "opaque-source-runtime" }, + }, + }, + managedProfile, + ); + restore.mockReturnValueOnce({ + schemaVersion: 1, + providerId: "mxc", + sandboxName: "target", + providerHandle: "opaque-provider-owned-restore", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + ...runtime(), + runtime: { kind: "replacement-session", handle: "opaque-provider-owned-runtime" }, + }, + managedProfile, + }); + + expect(confirmSandboxRuntimeRestore(bundle, target, prepared).restoreReceipt).toMatchObject({ + providerHandle: "opaque-provider-owned-restore", + runtime: { + runtime: { kind: "replacement-session", handle: "opaque-provider-owned-runtime" }, + }, + }); + }); + + it("rejects provider identity drift before returning snapshot authority", () => { + expect(() => + captureSandboxRuntimeSnapshot(provider({ preflightProviderId: "other" }).bundle, sandbox()), + ).toThrow(/invalid backup preflight authority/u); + expect(() => + captureSandboxRuntimeSnapshot(provider({ runtimeProviderId: "other" }).bundle, sandbox()), + ).toThrow(/unrepresentable runtime state/u); + }); + + it("fails preflight when the target cannot represent snapshot lifecycle state", () => { + const { bundle, restore } = provider(); + expect(() => + prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "paused", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }, + managedProfile, + ), + ).toThrow(/cannot represent the snapshot lifecycle state/u); + expect(restore).not.toHaveBeenCalled(); + }); + + it("propagates provider restore refusal from the read-only preflight edge", () => { + const { bundle, validateRestore, restore } = provider(); + validateRestore.mockImplementationOnce(() => { + throw new Error("source provider handle is invalid"); + }); + + expect(() => + prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "tampered-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }, + managedProfile, + ), + ).toThrow(/source provider handle is invalid/u); + expect(restore).not.toHaveBeenCalled(); + }); + + it("rejects stale target authority without calling provider restore", () => { + const { bundle, restore } = provider(); + const prepared = prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: runtime(), + }, + managedProfile, + ); + + expect(() => confirmSandboxRuntimeRestore(bundle, sandbox("replacement"), prepared)).toThrow( + /restore preflight authority is stale/u, + ); + expect(restore).not.toHaveBeenCalled(); + }); + + it("rejects cross-provider runtime authority before target preflight", () => { + const { bundle, preflight } = provider(); + expect(() => + prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "other", + providerHandle: "opaque-other", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: runtime("other"), + }, + managedProfile, + ), + ).toThrow(/does not match target provider/u); + expect(preflight).not.toHaveBeenCalled(); + }); + + it("rejects an invalid managed profile authority and provider restore proof", () => { + const source = { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: runtime(), + }; + expect(() => + prepareSandboxRuntimeRestore(provider().bundle, sandbox("target"), source, { + ...managedProfile, + profileFingerprint: "not-a-digest", + }), + ).toThrow(/managed profile restore authority is invalid/u); + + const { bundle } = provider({ restoreProviderId: "other" }); + const prepared = prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + source, + managedProfile, + ); + expect(() => confirmSandboxRuntimeRestore(bundle, sandbox("target"), prepared)).toThrow( + /invalid managed restore proof/u, + ); + }); + + it.each([ + { field: "lifecycle state", lifecycleState: "stopped", lifecycleGeneration: "generation-1" }, + { field: "lifecycle generation", lifecycleState: "running", lifecycleGeneration: "changed" }, + ] as const)("rejects restore proof with changed $field", ({ + lifecycleState, + lifecycleGeneration, + }) => { + const { bundle, restore } = provider(); + const target = sandbox("target"); + const prepared = prepareSandboxRuntimeRestore( + bundle, + target, + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }, + managedProfile, + ); + restore.mockReturnValueOnce({ + schemaVersion: 1, + providerId: "mxc", + sandboxName: "target", + providerHandle: "provider-owned-restore-handle", + lifecycleState, + lifecycleGeneration, + runtime: runtime(), + managedProfile, + }); + + expect(() => confirmSandboxRuntimeRestore(bundle, target, prepared)).toThrow( + /invalid managed restore proof/u, + ); + }); + + it("rejects restore proof that changes acceleration authority", () => { + const { bundle } = provider(); + const target = sandbox("target"); + const prepared = prepareSandboxRuntimeRestore( + bundle, + target, + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: { + ...runtime(), + acceleration: { kind: "gpu", vendor: "nvidia", devices: ["GPU-0"] }, + }, + }, + managedProfile, + ); + + expect(() => confirmSandboxRuntimeRestore(bundle, target, prepared)).toThrow( + /invalid managed restore proof/u, + ); + }); + + it("isolates central authority from a hostile provider that mutates backup inputs", () => { + const { bundle, capture } = provider(); + capture.mockImplementationOnce((_entry, preflight) => { + (preflight as { providerHandle: string }).providerHandle = "mutated"; + return runtime(); + }); + + expect(() => captureSandboxRuntimeSnapshot(bundle, sandbox())).toThrow(TypeError); + }); + + it("isolates central authority from a hostile MXC-style restore facet", () => { + const { bundle, validateRestore } = provider(); + validateRestore.mockImplementationOnce((_entry, _preflight, source, authority) => { + (source as { providerHandle: string }).providerHandle = "mutated"; + (authority as { agent: string }).agent = "other"; + }); + + expect(() => + prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }, + managedProfile, + ), + ).toThrow(TypeError); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts b/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts new file mode 100644 index 00000000000..623de52a1ed --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +import { cloneAndDeepFreeze } from "../../../core/immutable"; +import type { + RuntimeProviderBundle, + RuntimeProviderManagedProfileRestoreAuthority, + RuntimeProviderRuntimeReceipt, + RuntimeProviderSnapshotPreflightReceipt, + RuntimeProviderSnapshotRestoreReceipt, + RuntimeProviderSnapshotSurface, +} from "../../../onboard/runtime-provider/contract"; +import { + normalizeRuntimeProviderManagedProfileRestoreAuthority, + normalizeRuntimeProviderRuntimeReceipt, + normalizeRuntimeProviderSnapshotPreflightReceipt, + normalizeRuntimeProviderSnapshotRestoreReceipt, +} from "../../../onboard/runtime-provider/registry"; +import { + cloneSandboxRuntimeSnapshot, + SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, + type SandboxRuntimeSnapshot, +} from "../../../state/registry/runtime-snapshot"; +import type { SandboxEntry } from "../../../state/registry/types"; + +type SupportedSnapshotSurface = Extract; + +export class SandboxSnapshotProviderError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Sandbox snapshot provider failed: ${message}`, options); + this.name = "SandboxSnapshotProviderError"; + } +} + +/** + * Provider facets are extension points, so readonly TypeScript annotations are + * not a runtime trust boundary. Give every provider a detached, deeply frozen + * copy and retain only separately normalized values in central orchestration. + */ + +function requireSnapshotSurface( + bundle: RuntimeProviderBundle, + capability: keyof SupportedSnapshotSurface["capabilities"], +): SupportedSnapshotSurface { + const surface = bundle.snapshot; + if ( + surface.supported !== true || + surface.providerId !== bundle.identity.id || + surface.capabilities[capability] !== true + ) { + throw new SandboxSnapshotProviderError( + `runtime provider '${bundle.identity.id}' does not support ${capability}`, + ); + } + return surface; +} + +function requirePreflight( + bundle: RuntimeProviderBundle, + sandbox: SandboxEntry, + operation: "backup" | "restore", + value: unknown, +): RuntimeProviderSnapshotPreflightReceipt { + const preflight = normalizeRuntimeProviderSnapshotPreflightReceipt(value); + if ( + !preflight || + preflight.providerId !== bundle.identity.id || + preflight.operation !== operation || + preflight.sandboxName !== sandbox.name + ) { + throw new SandboxSnapshotProviderError( + `runtime provider '${bundle.identity.id}' returned invalid ${operation} preflight authority`, + ); + } + return preflight; +} + +function requireRuntimeReceipt( + bundle: RuntimeProviderBundle, + value: unknown, +): RuntimeProviderRuntimeReceipt { + const receipt = normalizeRuntimeProviderRuntimeReceipt(value); + if (!receipt || receipt.providerId !== bundle.identity.id) { + throw new SandboxSnapshotProviderError( + `runtime provider '${bundle.identity.id}' returned unrepresentable runtime state`, + ); + } + return receipt; +} + +function requireRestoreReceipt( + bundle: RuntimeProviderBundle, + sandbox: SandboxEntry, + authority: RuntimeProviderManagedProfileRestoreAuthority, + preflight: RuntimeProviderSnapshotPreflightReceipt, + source: SandboxRuntimeSnapshot, + value: unknown, +): RuntimeProviderSnapshotRestoreReceipt { + const receipt = normalizeRuntimeProviderSnapshotRestoreReceipt(value); + if ( + !receipt || + receipt.providerId !== bundle.identity.id || + receipt.sandboxName !== sandbox.name || + receipt.managedProfile.agent !== authority.agent || + receipt.managedProfile.profileFingerprint !== authority.profileFingerprint || + receipt.lifecycleState !== preflight.lifecycleState || + receipt.lifecycleGeneration !== preflight.lifecycleGeneration || + !isDeepStrictEqual(receipt.runtime.acceleration, source.runtime.acceleration) + ) { + throw new SandboxSnapshotProviderError( + `runtime provider '${bundle.identity.id}' returned invalid managed restore proof`, + ); + } + return receipt; +} + +/** + * Capture the complete provider-neutral snapshot state. Both provider calls + * occur inside the caller's quiescence lock; the provider re-observes runtime + * identity at capture so a stale preflight can never be persisted. + */ +export function captureSandboxRuntimeSnapshot( + bundle: RuntimeProviderBundle, + sandbox: SandboxEntry, +): SandboxRuntimeSnapshot { + const surface = requireSnapshotSurface(bundle, "backup"); + const providerSandbox = cloneAndDeepFreeze(sandbox); + const preflight = requirePreflight( + bundle, + sandbox, + "backup", + surface.preflight("backup", providerSandbox), + ); + const immutablePreflight = cloneAndDeepFreeze(preflight); + const runtime = requireRuntimeReceipt( + bundle, + surface.capture(providerSandbox, immutablePreflight), + ); + return cloneAndDeepFreeze({ + schemaVersion: SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, + providerId: bundle.identity.id, + providerHandle: immutablePreflight.providerHandle, + lifecycleState: immutablePreflight.lifecycleState, + lifecycleGeneration: immutablePreflight.lifecycleGeneration, + runtime: cloneAndDeepFreeze(runtime), + }); +} + +export interface PreparedSandboxRuntimeRestore { + readonly phase: "preflighted"; + readonly targetProviderId: string; + readonly targetSandboxName: string; + readonly source: SandboxRuntimeSnapshot; + readonly preflight: RuntimeProviderSnapshotPreflightReceipt; + readonly managedProfile: RuntimeProviderManagedProfileRestoreAuthority; +} + +export interface ValidatedSandboxRuntimeRestore { + readonly phase: "validated"; + readonly targetProviderId: string; + readonly targetSandboxName: string; + readonly source: SandboxRuntimeSnapshot; + readonly restoreReceipt: RuntimeProviderSnapshotRestoreReceipt; +} + +/** + * Perform the read-only restore preflight before a force-delete or filesystem + * mutation. Source provider handles remain opaque; PR3.8 self-restore requires + * the exact owning provider, while cross-provider rebinding remains deferred. + */ +export function prepareSandboxRuntimeRestore( + bundle: RuntimeProviderBundle, + target: SandboxEntry, + sourceValue: unknown, + managedProfileValue: unknown, +): PreparedSandboxRuntimeRestore { + const source = cloneSandboxRuntimeSnapshot(sourceValue); + if (!source) { + throw new SandboxSnapshotProviderError("snapshot runtime state is invalid"); + } + if (source.providerId !== bundle.identity.id) { + throw new SandboxSnapshotProviderError( + `snapshot runtime provider '${source.providerId}' does not match target provider '${bundle.identity.id}'`, + ); + } + const surface = requireSnapshotSurface(bundle, "restore"); + const providerTarget = cloneAndDeepFreeze(target); + const immutableSource = cloneAndDeepFreeze(source); + const managedProfile = + normalizeRuntimeProviderManagedProfileRestoreAuthority(managedProfileValue); + if (!managedProfile) { + throw new SandboxSnapshotProviderError("managed profile restore authority is invalid"); + } + const preflight = requirePreflight( + bundle, + target, + "restore", + surface.preflight("restore", providerTarget), + ); + if (preflight.lifecycleState !== source.lifecycleState) { + throw new SandboxSnapshotProviderError( + `target '${target.name}' cannot represent the snapshot lifecycle state`, + ); + } + const immutablePreflight = cloneAndDeepFreeze(preflight); + const immutableManagedProfile = cloneAndDeepFreeze(managedProfile); + surface.validateRestore( + providerTarget, + immutablePreflight, + immutableSource, + immutableManagedProfile, + ); + return cloneAndDeepFreeze({ + phase: "preflighted" as const, + targetProviderId: bundle.identity.id, + targetSandboxName: target.name, + source: immutableSource, + preflight: immutablePreflight, + managedProfile: immutableManagedProfile, + }); +} + +function normalizePreparedRestore( + bundle: RuntimeProviderBundle, + target: SandboxEntry, + prepared: PreparedSandboxRuntimeRestore, +): PreparedSandboxRuntimeRestore { + const source = cloneSandboxRuntimeSnapshot(prepared.source); + const preflight = normalizeRuntimeProviderSnapshotPreflightReceipt(prepared.preflight); + const managedProfile = normalizeRuntimeProviderManagedProfileRestoreAuthority( + prepared.managedProfile, + ); + if ( + prepared.phase !== "preflighted" || + prepared.targetProviderId !== bundle.identity.id || + prepared.targetSandboxName !== target.name || + !source || + source.providerId !== bundle.identity.id || + !preflight || + preflight.providerId !== bundle.identity.id || + preflight.operation !== "restore" || + preflight.sandboxName !== target.name || + !managedProfile + ) { + throw new SandboxSnapshotProviderError("restore preflight authority is stale"); + } + return cloneAndDeepFreeze({ + phase: "preflighted" as const, + targetProviderId: bundle.identity.id, + targetSandboxName: target.name, + source: cloneAndDeepFreeze(source), + preflight: cloneAndDeepFreeze(preflight), + managedProfile: cloneAndDeepFreeze(managedProfile), + }); +} + +/** + * Invoke the owning provider after filesystem restoration. The provider + * consumes its exact preflight authority, proves the managed profile is live, + * and returns a normalized runtime/restore receipt; central orchestration + * never interprets either opaque handle. + */ +export function confirmSandboxRuntimeRestore( + bundle: RuntimeProviderBundle, + target: SandboxEntry, + prepared: PreparedSandboxRuntimeRestore, +): ValidatedSandboxRuntimeRestore { + const authority = normalizePreparedRestore(bundle, target, prepared); + const surface = requireSnapshotSurface(bundle, "restore"); + const providerTarget = cloneAndDeepFreeze(target); + const restoreReceipt = requireRestoreReceipt( + bundle, + target, + authority.managedProfile, + authority.preflight, + authority.source, + surface.restore( + providerTarget, + authority.preflight, + authority.source, + authority.managedProfile, + ), + ); + return cloneAndDeepFreeze({ + phase: "validated" as const, + targetProviderId: bundle.identity.id, + targetSandboxName: target.name, + source: authority.source, + restoreReceipt: cloneAndDeepFreeze(restoreReceipt), + }); +} diff --git a/src/lib/actions/sandbox/snapshot/restore-authority.test.ts b/src/lib/actions/sandbox/snapshot/restore-authority.test.ts new file mode 100644 index 00000000000..719b42a3099 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/restore-authority.test.ts @@ -0,0 +1,247 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../../onboard/managed-image/contract"; +import { encodeManagedStartupProfile } from "../../../onboard/managed-startup/profile"; +import type { + RuntimeProviderBundle, + RuntimeProviderManagedProfileRestoreAuthority, +} from "../../../onboard/runtime-provider/contract"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; +import type { + RebuildManifest, + RecreatedSandboxRestoreOptions, + RestoreResult, +} from "../../../state/sandbox"; +import { restoreRecreatedSandboxStateWithManagedAuthority } from "./restore-authority"; + +function workload( + agent: ShippedManagedImageAgent, +): Extract { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile(agent)); + return { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.88", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function runtimeSnapshot() { + return { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-preflight", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "session", handle: "session-1" }, + acceleration: { kind: "none" }, + }, + } as const; +} + +function manifest(agent: ShippedManagedImageAgent): RebuildManifest { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-07-31T00-00-00-000Z", + agentType: agent, + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox", + backupPath: "/tmp/alpha", + blueprintDigest: null, + workload: workload(agent), + runtimeSnapshot: runtimeSnapshot(), + }; +} + +function sandbox(agent: ShippedManagedImageAgent): SandboxEntry { + const receipt = workload(agent); + return { + name: "alpha", + agent, + openshellDriver: "mxc", + imageTag: receipt.reference, + fromDockerfile: null, + workload: receipt, + }; +} + +function provider(agent: ShippedManagedImageAgent) { + const preflight = vi.fn((operation: "backup" | "restore", entry: SandboxEntry) => ({ + schemaVersion: 1 as const, + providerId: "mxc", + operation, + sandboxName: entry.name, + providerHandle: "opaque-preflight", + lifecycleState: "running" as const, + lifecycleGeneration: "generation-1", + })); + const validateRestore = vi.fn(); + const restore = vi.fn( + ( + entry: SandboxEntry, + _preflight: unknown, + _source: unknown, + authority: RuntimeProviderManagedProfileRestoreAuthority, + ) => ({ + schemaVersion: 1 as const, + providerId: "mxc", + sandboxName: entry.name, + providerHandle: "opaque-restore", + lifecycleState: "running" as const, + lifecycleGeneration: "generation-1", + runtime: runtimeSnapshot().runtime, + managedProfile: authority, + }), + ); + const bundle = { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + workload: { + providerId: "mxc", + supported: true, + profile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: false, + }, + acceptsReceipt: (receipt: SandboxWorkloadReceipt | undefined) => + receipt?.kind === "managed-image" && receipt.reference === workload(agent).reference, + }, + snapshot: { + providerId: "mxc", + supported: true, + contractVersion: 1, + capabilities: { backup: true, restore: true, managedProfileRestore: true }, + preflight, + capture: () => runtimeSnapshot().runtime, + validateRestore, + restore, + }, + } as unknown as RuntimeProviderBundle; + return { bundle, preflight, validateRestore, restore }; +} + +describe("managed rebuild restore authority", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("revalidates %s content and provider authority at the mutation edge", (agent) => { + const target = sandbox(agent); + const runtimeProvider = provider(agent); + const restore = vi.fn( + (_name: string, _path: string, options: RecreatedSandboxRestoreOptions): RestoreResult => { + options.validateBeforeMutation?.(); + return { + success: true, + restoredDirs: ["workspace"], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + }; + }, + ); + + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + manifest(agent), + { targetAgentType: agent }, + { + getSandbox: () => target, + requireProvider: () => runtimeProvider.bundle, + captureContentAuthority: () => ({ + schemaVersion: 1, + backupPath: "/tmp/alpha", + contentSha256: "c".repeat(64), + }), + restore, + }, + ); + + expect(result.success).toBe(true); + expect(restore).toHaveBeenCalledWith( + "alpha", + "/tmp/alpha", + expect.objectContaining({ + authority: expect.objectContaining({ contentSha256: "c".repeat(64) }), + validateBeforeMutation: expect.any(Function), + }), + ); + expect(runtimeProvider.preflight).toHaveBeenCalledTimes(2); + expect(runtimeProvider.validateRestore).toHaveBeenCalledTimes(2); + expect(runtimeProvider.restore).toHaveBeenCalledOnce(); + }); + + it("keeps legacy rebuild manifests on the state-only restore path", () => { + const legacy = { ...manifest("openclaw"), workload: undefined, runtimeSnapshot: undefined }; + const restore = vi.fn(() => ({ + success: true, + restoredDirs: [], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + })); + + expect( + restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + legacy, + { targetAgentType: "openclaw" }, + { + getSandbox: vi.fn(), + requireProvider: vi.fn() as never, + captureContentAuthority: vi.fn(), + restore, + }, + ).success, + ).toBe(true); + expect(restore).toHaveBeenCalledWith("alpha", "/tmp/alpha", { + targetAgentType: "openclaw", + }); + }); + + it("rejects a managed manifest without provider runtime authority", () => { + const restore = vi.fn(); + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + { ...manifest("hermes"), runtimeSnapshot: undefined }, + { targetAgentType: "hermes" }, + { + getSandbox: vi.fn(), + requireProvider: vi.fn() as never, + captureContentAuthority: vi.fn(), + restore, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("missing provider runtime authority"), + }); + expect(restore).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/restore-authority.ts b/src/lib/actions/sandbox/snapshot/restore-authority.ts new file mode 100644 index 00000000000..8d5b2198e58 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/restore-authority.ts @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; +import type { SandboxEntry } from "../../../state/registry/types"; +import * as sandboxState from "../../../state/sandbox"; +import { + prepareManagedSnapshotProfileRestore, + readManagedSnapshotProfileAuthority, +} from "./managed-profile"; +import { + confirmSandboxRuntimeRestore, + type PreparedSandboxRuntimeRestore, + prepareSandboxRuntimeRestore, +} from "./provider-lifecycle"; + +interface ManagedRestoreAuthorityDependencies { + readonly getSandbox: (sandboxName: string) => SandboxEntry | null; + readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; + readonly captureContentAuthority: typeof sandboxState.captureSnapshotRestoreAuthority; + readonly restore: typeof sandboxState.restoreRecreatedSandboxState; +} + +const defaultDependencies: Omit = { + requireProvider: (sandbox) => + requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), + captureContentAuthority: (...args) => sandboxState.captureSnapshotRestoreAuthority(...args), + restore: (...args) => sandboxState.restoreRecreatedSandboxState(...args), +}; + +function failure(error: unknown): sandboxState.RestoreResult { + const detail = error instanceof Error ? error.message : String(error); + return { + success: false, + restoredDirs: [], + failedDirs: ["manifest"], + restoredFiles: [], + failedFiles: [], + error: `Cannot restore managed snapshot authority: ${detail}.`, + }; +} + +/** + * Restore a rebuild backup through the same provider and content authority + * boundary as an explicit snapshot restore. Legacy/custom-image manifests + * retain their existing state-only path. + */ +export function restoreRecreatedSandboxStateWithManagedAuthority( + sandboxName: string, + manifest: sandboxState.RebuildManifest, + options: sandboxState.RecreatedSandboxRestoreOptions, + overrides: Pick & + Partial>, +): sandboxState.RestoreResult { + const dependencies = { ...defaultDependencies, ...overrides }; + let snapshotProfile; + try { + snapshotProfile = readManagedSnapshotProfileAuthority({ + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }); + } catch (error) { + return failure(error); + } + if (!snapshotProfile) { + return dependencies.restore(sandboxName, manifest.backupPath, options); + } + if (!manifest.runtimeSnapshot) { + return failure("managed snapshot is missing provider runtime authority"); + } + + let prepared: PreparedSandboxRuntimeRestore; + let providerId: string; + let contentAuthority: sandboxState.SnapshotRestoreAuthority; + try { + const target = dependencies.getSandbox(sandboxName); + if (!target) throw new Error(`target '${sandboxName}' is not registered`); + const provider = dependencies.requireProvider(target); + providerId = provider.identity.id; + const profileRestore = prepareManagedSnapshotProfileRestore( + { + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }, + target, + provider, + ); + if (!profileRestore) throw new Error("managed profile restore authority is missing"); + const captured = dependencies.captureContentAuthority(manifest.backupPath, manifest); + if (!captured) throw new Error("selected snapshot content changed during restore preflight"); + contentAuthority = captured; + prepared = prepareSandboxRuntimeRestore( + provider, + target, + manifest.runtimeSnapshot, + profileRestore.providerRestoreAuthority, + ); + } catch (error) { + return failure(error); + } + + const restore = dependencies.restore(sandboxName, manifest.backupPath, { + ...options, + authority: contentAuthority, + validateBeforeMutation: () => { + const current = dependencies.getSandbox(sandboxName); + if (!current) throw new Error(`target '${sandboxName}' is no longer registered`); + const provider = dependencies.requireProvider(current); + if (provider.identity.id !== providerId) { + throw new Error(`target '${sandboxName}' runtime provider changed before restore`); + } + const profileRestore = prepareManagedSnapshotProfileRestore( + { + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }, + current, + provider, + ); + if (!profileRestore) throw new Error("managed profile restore authority is missing"); + prepared = prepareSandboxRuntimeRestore( + provider, + current, + prepared.source, + profileRestore.providerRestoreAuthority, + ); + }, + }); + if (!restore.success) return restore; + + try { + const current = dependencies.getSandbox(sandboxName); + if (!current) throw new Error(`target '${sandboxName}' is no longer registered`); + const provider = dependencies.requireProvider(current); + if (provider.identity.id !== providerId) { + throw new Error(`target '${sandboxName}' runtime provider changed during restore`); + } + confirmSandboxRuntimeRestore(provider, current, prepared); + return restore; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { + ...restore, + success: false, + error: + `State was restored, but managed runtime proof failed: ${detail}. ` + + `Retry this exact snapshot after the runtime stabilizes.`, + }; + } +} diff --git a/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts b/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts index e13fc7264ff..ff73c8101a4 100644 --- a/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts +++ b/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it, vi } from "vitest"; const adapterMocks = vi.hoisted(() => ({ dockerRun: vi.fn(), dockerCapture: vi.fn(), + backupWithAuthority: vi.fn(), })); vi.mock("../../adapters/docker/run", () => ({ @@ -19,6 +20,9 @@ vi.mock("../../state/registry", () => ({ vi.mock("../../state/sandbox", () => ({ backupSandboxState: vi.fn(), })); +vi.mock("./snapshot/backup-authority", () => ({ + backupSandboxStateWithManagedAuthority: (name: string) => adapterMocks.backupWithAuthority(name), +})); import * as registry from "../../state/registry"; import { @@ -225,6 +229,14 @@ describe("backupStartedSandboxState", () => { const unreachable = { ...ok, success: false, unreachable: true }; const denied = { ...ok, success: false }; + it("uses managed provider authority through the default stopped-backup path", async () => { + adapterMocks.backupWithAuthority.mockReturnValueOnce(ok); + + await expect(backupStartedSandboxState("my-sb")).resolves.toEqual(ok); + + expect(adapterMocks.backupWithAuthority).toHaveBeenCalledWith("my-sb"); + }); + it("retries while the just-started container's SSH endpoint is unreachable (#6500)", async () => { const backup = vi .fn() diff --git a/src/lib/actions/sandbox/stopped-sandbox-backup.ts b/src/lib/actions/sandbox/stopped-sandbox-backup.ts index b1a9c9d6e7f..667584104ad 100644 --- a/src/lib/actions/sandbox/stopped-sandbox-backup.ts +++ b/src/lib/actions/sandbox/stopped-sandbox-backup.ts @@ -12,6 +12,7 @@ import { import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; import { resolveSandboxContainerOwner } from "./sandbox-container-owner"; +import * as snapshotBackup from "./snapshot/backup-authority"; /** Read a registered sandbox's OpenShell driver, treating registry read * failure as unknown so callers fail closed on driver-gated decisions. */ @@ -182,7 +183,14 @@ interface BackupRetryDeps { } const defaultBackupRetryDeps: BackupRetryDeps = { - backup: (name) => sandboxState.backupSandboxState(name), + backup: (name) => + snapshotBackup.backupSandboxStateWithManagedAuthority( + name, + {}, + { + getSandbox: registry.getSandbox, + }, + ), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), attempts: 5, delayMs: 2000, diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index f0224759b04..7061252f12c 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -97,6 +97,10 @@ export function setupGpuFlowMocks(mocks: Record imageId: GPU_IMAGE_ID, bookkeepingImageRef: "openshell/sandbox-from:test", stateError: "", + deviceRequests: null, + devices: null, + runtime: "nvidia", + nvidiaVisibleDevices: "all", nativeGpuAttachmentState: "present", containerId: "container-a", }); diff --git a/src/lib/onboard/created-sandbox-finalization.test.ts b/src/lib/onboard/created-sandbox-finalization.test.ts index f470719fa59..bb068a9b454 100644 --- a/src/lib/onboard/created-sandbox-finalization.test.ts +++ b/src/lib/onboard/created-sandbox-finalization.test.ts @@ -504,6 +504,52 @@ describe("created OpenClaw sandbox finalization", () => { expect(register).toHaveBeenCalledWith(pluginInstalls); }); + it("defers managed restore before unregistered target authority can be bound", () => { + const register = vi.fn(); + const error = vi.fn(); + + expect(() => + finalizeCreatedSandbox( + { + sandboxName: "openclaw", + restoreBackupPath: "/tmp/managed-openclaw-backup", + preUpgradeBackup: false, + targetAgentType: "openclaw", + validateManagedDcode: false, + provider: "compatible-endpoint", + model: "demo", + preferredInferenceApi: "openai-completions", + }, + { + discoverFreshOpenClawImagePluginInstalls: vi.fn(), + restoreRecreatedSandboxState: () => ({ + success: false, + restoredDirs: [], + failedDirs: ["manifest"], + restoredFiles: [], + failedFiles: [], + error: sandboxState.MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR, + }), + getDcodeSelectionDrift: vi.fn(), + register, + note: vi.fn(), + error, + exitProcess: (code): never => { + throw new Error(`exit ${code}`); + }, + }, + ), + ).toThrow("exit 1"); + + expect(register).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringContaining("restore is deferred")); + expect(error).toHaveBeenCalledWith( + " State was not restored and registry metadata was not updated.", + ); + expect(error).toHaveBeenCalledWith(' openshell sandbox delete "openclaw"'); + expect(error).toHaveBeenCalledWith(" Manual recovery: /tmp/managed-openclaw-backup"); + }); + it("fails closed before restore and registration when provenance discovery fails", () => { const restoreRecreatedSandboxState = vi.fn(); const register = vi.fn(); diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index f95dc3c22ee..c0efd29d416 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -6,6 +6,7 @@ import type { OpenClawManagedExtensionDiscoveryResult, } from "../state/openclaw-plugin-restore"; import { + MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR, OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR, type RecreatedSandboxRestoreOptions, type RestoreResult, @@ -90,6 +91,16 @@ export function finalizeCreatedSandbox( ` ✓ State restored (${restore.restoredDirs.length} directories, ${restore.restoredFiles.length} files)`, ); } else { + if (restore.error === MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR) { + deps.error( + ` Managed snapshot restore is deferred for newly created sandbox '${options.sandboxName}' until its runtime authority can be bound before registry publication.`, + ); + deps.error(" State was not restored and registry metadata was not updated."); + deps.error(" Remove the unregistered sandbox before retrying:"); + deps.error(` openshell sandbox delete ${JSON.stringify(options.sandboxName)}`); + deps.error(` Manual recovery: ${options.restoreBackupPath}`); + return deps.exitProcess(1); + } if (restore.error === OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR) { deps.error( ` OpenClaw image plugin provenance validation failed for sandbox '${options.sandboxName}': ${restore.error}`, diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 544b5170984..36ce0c7c201 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -140,6 +140,41 @@ The handler clears the journal after it records both create and registration rec This slice covers resumed onboard replacement, including not-ready repair and non-default gateways. Rebuild and non-resumed re-onboard remain under #6492. +## Managed snapshot and rebuild restore authority + +Managed-image backups use one provider-neutral authority path for explicit snapshot creation, `backup-all`, stopped-sandbox backup retries, and rebuild backups. +The contract applies to OpenClaw, Hermes, and Deep Agents Code. +The path records the exact managed workload receipt and a versioned runtime receipt from the sandbox's registered provider. +The state layer copies and sanitizes the backup before it invokes the provider fence. +The fence re-reads the registry row and re-observes the provider runtime. +The state layer publishes the manifest atomically only when the workload, provider, lifecycle generation, runtime identity, and acceleration receipt still match. +It removes the unpublished backup directory when the fence rejects publication. + +A managed restore binds the selected manifest and every backup payload to one content digest. +The state layer recomputes that digest after local restore staging and before its first remote filesystem mutation. +At the same mutation edge, central orchestration asks the registered provider to revalidate the target runtime and managed startup profile. +After state restoration, the provider must prove the managed profile and runtime state again. +Explicit snapshot restore and rebuild restore use this same boundary. + +The snapshot provider facet has its own contract version. +Provider inputs are detached and deeply frozen at the extension boundary, and central orchestration retains only normalized receipts. +Docker lifecycle inspection and GPU inspection remain inside the Docker provider adapter. +The provider-neutral receipt can represent another provider, including an MXC-style implementation, without adding provider switches to snapshot or rebuild orchestration. + +Legacy and custom-image snapshots retain their state-only backup and restore path. The managed +authority path may become the default for managed images only after the +[incremental runtime epic](https://github.com/NVIDIA/NemoClaw/issues/7744) completes create +finalization, clone/rebind, recovery, and activation for every supported agent with authority proven +before mutation. Any later consolidation must preserve legacy and custom-image restore parity. +This contract does not activate another runtime provider or managed-image onboarding path. +Ordinary onboard recreation and create finalization remain deferred under +[#7744](https://github.com/NVIDIA/NemoClaw/issues/7744) because the replacement target is not +registered when that restore currently runs; the raw state layer rejects a managed manifest unless +both exact content authority and a runtime-validation fence are present. Cross-provider clone and +rebind, durable interrupted-restore recovery, ordinary recreate integration, and user-visible +runtime activation are separately reviewable units tracked by that epic. +If provider proof fails after filesystem restoration, NemoClaw reports that state changed and requires the operator to retry the exact snapshot after the runtime stabilizes. + ## Agent-specific differences | Agent | Lifecycle difference | @@ -208,5 +243,6 @@ PR #5955 moved the rebuild messaging conflict check before destruction. | Session sanitation, sandbox prompt checkpoints, and no-secret persistence | `src/lib/state/onboard-session-sandbox-prompts.test.ts`, `src/lib/state/onboard-checkpoint.test.ts`, `machine/handlers/sandbox-create-intent-boundary.test.ts` | Tri-state decisions remain scoped to checkpointed sandbox choices. | | Versioned checkpoint schema, tri-state decisions, migration, and unknown-future fail-safe | `src/lib/state/onboard-checkpoint.test.ts`, `src/lib/state/onboard-checkpoint-migrate.test.ts` | Live decision reads still use legacy fields | | Resumable create replay, durable identity, and stale-binding fail-closed | `src/lib/onboard/checkpoint-replay.test.ts`, `src/lib/onboard/checkpoint-resume-guard.test.ts`, `machine/handlers/sandbox-checkpoint-crash-recovery.test.ts` | None at the sandbox-handler boundary. | +| Managed snapshot workload, content, and provider authority across explicit and rebuild flows | `src/lib/actions/sandbox/snapshot/backup-authority.test.ts`, `restore-authority.test.ts`, `managed-profile.test.ts`, `provider-lifecycle.test.ts`, and `snapshot-managed-provider-restore-order.test.ts` | Cross-provider clone and rebind, durable interrupted-restore recovery, and user-visible runtime activation remain separate review units. | When lifecycle behavior changes one of these contracts, update the map and the narrow owning test in that same PR. Do not add source-text scans or production scaffolding solely to preserve current orchestration order. diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts index e26eb9b8ec5..60bc55fef2c 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts @@ -8,11 +8,19 @@ const IMAGE_ID = `sha256:${"a".repeat(64)}`; const BOOKKEEPING_IMAGE_REF = "openshell/sandbox-from:alpha"; const EMPTY_RUNTIME_FIELDS = [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "runc"]; -function querySnapshot(fields: unknown) { +function querySnapshot(fields: unknown, nvidiaVisibleDevices?: string) { const dockerRun = vi .fn() .mockReturnValueOnce({ status: 0, stdout: "container-a\n", stderr: "" }) - .mockReturnValueOnce({ status: 0, stdout: JSON.stringify(fields), stderr: "" }); + .mockReturnValueOnce({ status: 0, stdout: JSON.stringify(fields), stderr: "" }) + .mockReturnValueOnce({ + status: 0, + stdout: + nvidiaVisibleDevices === undefined + ? "" + : `NVIDIA_VISIBLE_DEVICES=${nvidiaVisibleDevices}\n`, + stderr: "", + }); return { dockerRun, result: queryOpenShellDockerSandboxRuntimeSnapshot("alpha", { dockerRun }), @@ -31,6 +39,7 @@ describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { deviceRequests: null, devices: [], runtime: "runc", + nvidiaVisibleDevices: null, nativeGpuAttachmentState: "absent", containerId: "container-a", }); @@ -114,14 +123,10 @@ describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { ], ["NVIDIA runtime", null, [], "nvidia"], ])("detects a host-configured GPU attachment from %s", (_label, requests, devices, runtime) => { - const { result } = querySnapshot([ - IMAGE_ID, - BOOKKEEPING_IMAGE_REF, - "", - requests, - devices, - runtime, - ]); + const { result } = querySnapshot( + [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", requests, devices, runtime], + runtime === "nvidia" ? "all" : undefined, + ); expect(result).toMatchObject({ ok: true, @@ -129,6 +134,51 @@ describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { }); }); + it.each([ + ["all devices", "all", "present"], + ["an exact device list", "0,GPU-live-1", "present"], + ["no devices", "none", "absent"], + ["runtime bypass", "void", "absent"], + ] as const)("reads only NVIDIA_VISIBLE_DEVICES for %s", (_label, value, expectedState) => { + const { dockerRun, result } = querySnapshot( + [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "nvidia"], + value, + ); + + expect(result).toMatchObject({ + ok: true, + nvidiaVisibleDevices: value, + nativeGpuAttachmentState: expectedState, + }); + expect(dockerRun).toHaveBeenLastCalledWith( + [ + "inspect", + "--type", + "container", + "--format", + '{{range .Config.Env}}{{if eq (index (split . "=") 0) "NVIDIA_VISIBLE_DEVICES"}}{{println .}}{{end}}{{end}}', + "container-a", + ], + expect.objectContaining({ suppressOutput: true }), + ); + }); + + it.each([ + "all,0", + "0,0", + "GPU-0 with-space", + ])("rejects ambiguous NVIDIA_VISIBLE_DEVICES value %s", (value) => { + const { result } = querySnapshot( + [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "nvidia"], + value, + ); + + expect(result).toEqual({ + ok: false, + error: "docker inspect returned invalid NVIDIA_VISIBLE_DEVICES", + }); + }); + it.each([ ["unknown runtime", null, [], "nvidia-container-runtime"], [ diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index 2c045cdb086..a8934027fd6 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -107,6 +107,11 @@ export type OpenShellDockerSandboxRuntimeSnapshotQuery = deviceRequests: OpenShellDockerDeviceRequest[] | null; devices: OpenShellDockerDeviceMapping[] | null; runtime: string; + /** + * Exact allowlisted NVIDIA Container Runtime selector. Other container + * environment entries are deliberately never returned by this query. + */ + nvidiaVisibleDevices: string | null; /** Closed-world classification of host-owned Docker GPU configuration. */ nativeGpuAttachmentState: OpenShellDockerGpuAttachmentState; containerId: string; @@ -196,9 +201,9 @@ function classifyGpuAttachment( deviceRequests: OpenShellDockerDeviceRequest[] | null, devices: OpenShellDockerDeviceMapping[] | null, runtime: string, + nvidiaVisibleDevices: string | null, ): OpenShellDockerGpuAttachmentState { const normalizedRuntime = runtime.trim().toLowerCase(); - if (normalizedRuntime === "nvidia") return "present"; if ( deviceRequests?.some( (request) => @@ -211,6 +216,10 @@ function classifyGpuAttachment( ) { return "present"; } + if (normalizedRuntime === "nvidia") { + if (nvidiaVisibleDevices === null) return "unknown"; + return ["", "none", "void"].includes(nvidiaVisibleDevices) ? "absent" : "present"; + } if ( devices?.some( (mapping) => @@ -228,6 +237,45 @@ function classifyGpuAttachment( return noDeviceRequests && noDeviceMappings && knownNonGpuRuntime ? "absent" : "unknown"; } +function parseNvidiaVisibleDevices(result: { + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; + status?: number | null; +}): { ok: true; value: string | null } | { ok: false; error: string } { + if (Number(result.status ?? 1) !== 0) { + return { + ok: false, + error: + commandResultText(result) || "docker inspect could not read NVIDIA_VISIBLE_DEVICES safely", + }; + } + const lines = String(result.stdout ?? "") + .split(/\r?\n/u) + .filter((line) => line.length > 0); + if (lines.length === 0) return { ok: true, value: null }; + if (lines.length !== 1 || !lines[0]?.startsWith("NVIDIA_VISIBLE_DEVICES=")) { + return { ok: false, error: "docker inspect returned ambiguous NVIDIA_VISIBLE_DEVICES" }; + } + const value = lines[0].slice("NVIDIA_VISIBLE_DEVICES=".length); + const identifiers = value.split(","); + const validKeyword = ["", "all", "none", "void"].includes(value); + const validIdentifiers = + !validKeyword && + identifiers.length > 0 && + identifiers.length <= 64 && + identifiers.every( + (identifier) => + !["all", "none", "void"].includes(identifier) && + /^[A-Za-z0-9._:/=-]{1,256}$/u.test(identifier) && + Buffer.byteLength(identifier, "utf8") <= 256, + ) && + new Set(identifiers).size === identifiers.length; + if (!validKeyword && !validIdentifiers) { + return { ok: false, error: "docker inspect returned invalid NVIDIA_VISIBLE_DEVICES" }; + } + return { ok: true, value }; +} + /** * Inspect the one exactly labeled native container before deletion. * @@ -295,6 +343,28 @@ export function queryOpenShellDockerSandboxRuntimeSnapshot( const deviceRequests = fields[3]; const devices = fields[4]; const runtime = fields[5]; + let nvidiaVisibleDevices: string | null = null; + if (runtime.trim().toLowerCase() === "nvidia") { + const visibleDevices = parseNvidiaVisibleDevices( + run( + [ + "inspect", + "--type", + "container", + "--format", + '{{range .Config.Env}}{{if eq (index (split . "=") 0) "NVIDIA_VISIBLE_DEVICES"}}{{println .}}{{end}}{{end}}', + containerId, + ], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_SANDBOX_QUERY_TIMEOUT_MS, + }, + ), + ); + if (!visibleDevices.ok) return visibleDevices; + nvidiaVisibleDevices = visibleDevices.value; + } return { ok: true, imageId: fields[0].toLowerCase(), @@ -303,7 +373,13 @@ export function queryOpenShellDockerSandboxRuntimeSnapshot( deviceRequests, devices, runtime, - nativeGpuAttachmentState: classifyGpuAttachment(deviceRequests, devices, runtime), + nvidiaVisibleDevices, + nativeGpuAttachmentState: classifyGpuAttachment( + deviceRequests, + devices, + runtime, + nvidiaVisibleDevices, + ), containerId, }; } diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index d629eb51eed..644239b3a7a 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -5,6 +5,8 @@ import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/ import type { ManagedImageSelectionPolicy } from "../workload/source"; export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; +export const RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION = 1 as const; +export const RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION = 1 as const; export type RuntimeProviderGatewayLauncher = "nemoclaw" | "openshell"; export type RuntimeProviderLifecycleAction = "start" | "stop"; @@ -152,7 +154,7 @@ export interface RuntimeProviderCleanupOperations { } /** - * Provider-neutral, bounded state that later snapshot work may persist. + * Provider-neutral, bounded state persisted by provider-backed snapshots. * Provider handles remain opaque strings; acceleration is normalized so no * action module needs a Docker-, CDI-, or device-specific DTO. */ @@ -174,6 +176,49 @@ export interface RuntimeProviderRuntimeReceipt { }; } +export type RuntimeProviderSnapshotOperation = "backup" | "restore"; +export type RuntimeProviderSnapshotLifecycleState = "running" | "paused" | "stopped"; + +export interface RuntimeProviderSnapshotPreflightReceipt { + readonly schemaVersion: typeof RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION; + readonly providerId: string; + readonly operation: RuntimeProviderSnapshotOperation; + readonly sandboxName: string; + readonly providerHandle: string; + readonly lifecycleState: RuntimeProviderSnapshotLifecycleState; + readonly lifecycleGeneration: string; +} + +export interface RuntimeProviderManagedProfileRestoreAuthority { + readonly agent: string; + readonly profileFingerprint: string; +} + +/** + * Complete normalized source state supplied to the owning restore facet. + * `providerHandle` binds the lifecycle generation and full runtime receipt. + */ +export interface RuntimeProviderSnapshotRestoreSource { + readonly schemaVersion: 1; + readonly providerId: string; + readonly providerHandle: string; + readonly lifecycleState: RuntimeProviderSnapshotLifecycleState; + readonly lifecycleGeneration: string; + readonly runtime: RuntimeProviderRuntimeReceipt; +} + +export interface RuntimeProviderSnapshotRestoreReceipt { + readonly schemaVersion: 1; + readonly providerId: string; + readonly sandboxName: string; + /** Provider-authored proof over preflight, source state, profile, and live runtime. */ + readonly providerHandle: string; + readonly lifecycleState: RuntimeProviderSnapshotLifecycleState; + readonly lifecycleGeneration: string; + readonly runtime: RuntimeProviderRuntimeReceipt; + readonly managedProfile: RuntimeProviderManagedProfileRestoreAuthority; +} + export type RuntimeProviderPreflightDoctorSurface = RuntimeProviderSupportedSurface<{ inspectHost(): RuntimeProviderDoctorCheck; preflightLifecycle( @@ -221,8 +266,37 @@ export type RuntimeProviderBootstrapSurface = export type RuntimeProviderSnapshotSurface = | RuntimeProviderSupportedSurface<{ - capture(sandbox: SandboxEntry): RuntimeProviderRuntimeReceipt; - restore(sandbox: SandboxEntry, receipt: RuntimeProviderRuntimeReceipt): void; + /** + * Version the snapshot facet independently so providers can reject a + * central contract they do not implement without forcing unrelated + * bundle surfaces to rev in lockstep. + */ + readonly contractVersion: typeof RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION; + readonly capabilities: { + readonly backup: boolean; + readonly restore: boolean; + readonly managedProfileRestore: boolean; + }; + preflight( + operation: RuntimeProviderSnapshotOperation, + sandbox: SandboxEntry, + ): RuntimeProviderSnapshotPreflightReceipt; + capture( + sandbox: SandboxEntry, + preflight: RuntimeProviderSnapshotPreflightReceipt, + ): RuntimeProviderRuntimeReceipt; + validateRestore( + sandbox: SandboxEntry, + preflight: RuntimeProviderSnapshotPreflightReceipt, + source: RuntimeProviderSnapshotRestoreSource, + managedProfile: RuntimeProviderManagedProfileRestoreAuthority, + ): void; + restore( + sandbox: SandboxEntry, + preflight: RuntimeProviderSnapshotPreflightReceipt, + source: RuntimeProviderSnapshotRestoreSource, + managedProfile: RuntimeProviderManagedProfileRestoreAuthority, + ): RuntimeProviderSnapshotRestoreReceipt; }> | RuntimeProviderUnsupportedSurface; diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index e0bb280ee21..f8481c37c3c 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -17,6 +17,7 @@ import { MANAGED_IMAGE_REPOSITORIES, MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, } from "../managed-image/contract"; +import { queryOpenShellDockerSandboxRuntimeSnapshot } from "../openshell-docker-sandbox-containers"; import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, type RuntimeProviderBundle, @@ -31,6 +32,7 @@ import { type RuntimeProviderWorkloadCleanupResult, type RuntimeProviderWorkloadProfile, } from "./contract"; +import { createDockerRuntimeProviderSnapshotSurface } from "./snapshot"; type DockerOpResult = { status?: number | null }; type DockerStop = (name: string, options?: Record) => DockerOpResult; @@ -50,6 +52,7 @@ export interface DockerRuntimeProviderDependencies { readonly isRuntimeDown: typeof isDockerRuntimeDown; readonly printRuntimeDownGuidance: typeof printDockerRuntimeDownGuidance; readonly recoverSandbox: typeof recoverDockerDriverSandbox; + readonly queryRuntimeSnapshot: typeof queryOpenShellDockerSandboxRuntimeSnapshot; readonly removeImage: DockerRemoveImage; readonly stopContainer: DockerStop; readonly unpauseContainer: DockerUnpause; @@ -82,6 +85,8 @@ function resolveDependencies( isRuntimeDown: overrides.isRuntimeDown ?? isDockerRuntimeDown, printRuntimeDownGuidance: overrides.printRuntimeDownGuidance ?? printDockerRuntimeDownGuidance, recoverSandbox: overrides.recoverSandbox ?? recoverDockerDriverSandbox, + queryRuntimeSnapshot: + overrides.queryRuntimeSnapshot ?? queryOpenShellDockerSandboxRuntimeSnapshot, removeImage: overrides.removeImage ?? ((reference, options) => loadDockerRemoveImage()(reference, options)), @@ -348,7 +353,10 @@ export function createDockerRuntimeProviderBundle( ], }, bootstrap: unsupported(providerId, futureReason), - snapshot: unsupported(providerId, futureReason), + snapshot: createDockerRuntimeProviderSnapshotSurface(providerId, { + captureHostCommand: deps.captureHostCommand, + queryRuntimeSnapshot: deps.queryRuntimeSnapshot, + }), recovery: unsupported(providerId, futureReason), cleanup: { providerId, diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 81ee3ba7ae4..765f1feed9f 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -4,12 +4,19 @@ import type { SandboxEntry } from "../../state/registry/types"; import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, + RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, type RuntimeProviderBundle, type RuntimeProviderBundleRegistry, type RuntimeProviderChannelStopTransport, type RuntimeProviderContainerEngineOperation, + type RuntimeProviderManagedProfileRestoreAuthority, type RuntimeProviderMutationOperation, type RuntimeProviderRuntimeReceipt, + type RuntimeProviderSnapshotLifecycleState, + type RuntimeProviderSnapshotPreflightReceipt, + type RuntimeProviderSnapshotRestoreReceipt, + type RuntimeProviderSnapshotRestoreSource, } from "./contract"; const PROVIDER_ID_PATTERN = /^[a-z][a-z0-9-]{0,62}$/u; @@ -30,6 +37,15 @@ const BUNDLE_SURFACES = [ ] as const; const MAX_RECEIPT_HANDLE_BYTES = 4096; const MAX_RECEIPT_DEVICES = 64; +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const MANAGED_PROFILE_AGENT_PATTERN = /^[a-z][a-z0-9-]{0,127}$/u; +const SNAPSHOT_OPERATIONS = new Set(["backup", "restore"]); +const SNAPSHOT_LIFECYCLE_STATES = new Set([ + "running", + "paused", + "stopped", +]); const GATEWAY_LAUNCHERS = new Set(["nemoclaw", "openshell"]); const CHANNEL_STOP_TRANSPORTS: ReadonlySet = new Set([ "docker-kubectl-first", @@ -324,10 +340,26 @@ function validateBootstrapSurface(surface: Record): void { } } -function validateSnapshotSurface(surface: Record): void { +function validateSnapshotSurface(providerId: string, surface: Record): void { if (surface.supported === true) { + if (surface.contractVersion !== RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION) { + throw new RuntimeProviderRegistrationError( + `snapshot for '${providerId}' has an unsupported contract version`, + ); + } + const capabilities = requireOwnRecord(surface, "capabilities"); + for (const capability of ["backup", "restore", "managedProfileRestore"] as const) { + requireBoolean(capabilities, capability, "snapshot capabilities"); + } + requireFunction(surface, "preflight", "snapshot"); requireFunction(surface, "capture", "snapshot"); + requireFunction(surface, "validateRestore", "snapshot"); requireFunction(surface, "restore", "snapshot"); + if (capabilities.managedProfileRestore === true && capabilities.restore !== true) { + throw new RuntimeProviderRegistrationError( + `snapshot for '${providerId}' cannot restore managed profiles without restore support`, + ); + } } } @@ -396,7 +428,7 @@ function validateSupportedSurfaceSchemas( validateLifecycleSurface(providerId, surfaces.lifecycle); validateMutationAuthoritySurface(providerId, surfaces.mutationAuthority); validateBootstrapSurface(surfaces.bootstrap); - validateSnapshotSurface(surfaces.snapshot); + validateSnapshotSurface(providerId, surfaces.snapshot); validateRecoverySurface(surfaces.recovery); validateCleanupSurface(surfaces.cleanup); validateContainerEngineSurface(providerId, surfaces.containerEngine); @@ -583,7 +615,10 @@ export function runtimeProviderContainerEngineIdentity( function boundedString(value: unknown, maxBytes: number): value is string { return ( - typeof value === "string" && value.trim() !== "" && Buffer.byteLength(value, "utf8") <= maxBytes + typeof value === "string" && + value.trim() !== "" && + Buffer.byteLength(value, "utf8") <= maxBytes && + !CONTROL_CHARACTERS.test(value) ); } @@ -632,3 +667,104 @@ export function normalizeRuntimeProviderRuntimeReceipt( }, }; } + +export function normalizeRuntimeProviderManagedProfileRestoreAuthority( + value: unknown, +): RuntimeProviderManagedProfileRestoreAuthority | null { + if ( + !isPlainRecord(value) || + typeof value.agent !== "string" || + !MANAGED_PROFILE_AGENT_PATTERN.test(value.agent) || + typeof value.profileFingerprint !== "string" || + !SHA256_PATTERN.test(value.profileFingerprint) + ) { + return null; + } + return { + agent: value.agent, + profileFingerprint: value.profileFingerprint, + }; +} + +export function normalizeRuntimeProviderSnapshotPreflightReceipt( + value: unknown, +): RuntimeProviderSnapshotPreflightReceipt | null { + if ( + !isPlainRecord(value) || + value.schemaVersion !== RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION || + !validProviderId(value.providerId) || + typeof value.operation !== "string" || + !SNAPSHOT_OPERATIONS.has(value.operation) || + !boundedString(value.sandboxName, 512) || + !boundedString(value.providerHandle, MAX_RECEIPT_HANDLE_BYTES) || + !SNAPSHOT_LIFECYCLE_STATES.has(value.lifecycleState as RuntimeProviderSnapshotLifecycleState) || + !boundedString(value.lifecycleGeneration, MAX_RECEIPT_HANDLE_BYTES) + ) { + return null; + } + return { + schemaVersion: RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, + providerId: value.providerId, + operation: value.operation as RuntimeProviderSnapshotPreflightReceipt["operation"], + sandboxName: value.sandboxName, + providerHandle: value.providerHandle, + lifecycleState: value.lifecycleState as RuntimeProviderSnapshotLifecycleState, + lifecycleGeneration: value.lifecycleGeneration, + }; +} + +export function normalizeRuntimeProviderSnapshotRestoreSource( + value: unknown, +): RuntimeProviderSnapshotRestoreSource | null { + if ( + !isPlainRecord(value) || + value.schemaVersion !== 1 || + !validProviderId(value.providerId) || + !boundedString(value.providerHandle, MAX_RECEIPT_HANDLE_BYTES) || + !SNAPSHOT_LIFECYCLE_STATES.has(value.lifecycleState as RuntimeProviderSnapshotLifecycleState) || + !boundedString(value.lifecycleGeneration, MAX_RECEIPT_HANDLE_BYTES) + ) { + return null; + } + const runtime = normalizeRuntimeProviderRuntimeReceipt(value.runtime); + if (!runtime || runtime.providerId !== value.providerId) return null; + return { + schemaVersion: 1, + providerId: value.providerId, + providerHandle: value.providerHandle, + lifecycleState: value.lifecycleState as RuntimeProviderSnapshotLifecycleState, + lifecycleGeneration: value.lifecycleGeneration, + runtime, + }; +} + +export function normalizeRuntimeProviderSnapshotRestoreReceipt( + value: unknown, +): RuntimeProviderSnapshotRestoreReceipt | null { + if ( + !isPlainRecord(value) || + value.schemaVersion !== 1 || + !validProviderId(value.providerId) || + !boundedString(value.sandboxName, 512) || + !boundedString(value.providerHandle, MAX_RECEIPT_HANDLE_BYTES) || + !SNAPSHOT_LIFECYCLE_STATES.has(value.lifecycleState as RuntimeProviderSnapshotLifecycleState) || + !boundedString(value.lifecycleGeneration, MAX_RECEIPT_HANDLE_BYTES) + ) { + return null; + } + const runtime = normalizeRuntimeProviderRuntimeReceipt(value.runtime); + const managedProfile = normalizeRuntimeProviderManagedProfileRestoreAuthority( + value.managedProfile, + ); + if (!runtime || runtime.providerId !== value.providerId || !managedProfile) return null; + return { + schemaVersion: 1, + providerId: value.providerId, + sandboxName: value.sandboxName, + providerHandle: value.providerHandle, + lifecycleState: value.lifecycleState as RuntimeProviderSnapshotLifecycleState, + lifecycleGeneration: value.lifecycleGeneration, + runtime, + managedProfile, + }; +} diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index 4a790f2947b..bf2a47fe4b4 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -31,7 +31,11 @@ import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./current"; import { createDockerRuntimeProviderBundle } from "./docker"; import { createRuntimeProviderBundleRegistry, + normalizeRuntimeProviderManagedProfileRestoreAuthority, normalizeRuntimeProviderRuntimeReceipt, + normalizeRuntimeProviderSnapshotPreflightReceipt, + normalizeRuntimeProviderSnapshotRestoreReceipt, + normalizeRuntimeProviderSnapshotRestoreSource, RuntimeProviderRegistrationError, resolveRuntimeProviderBundle, } from "./registry"; @@ -125,7 +129,18 @@ describe("RuntimeProviderBundle registry contract", () => { expect(bundle[surface].providerId, `${providerId}.${surface}`).toBe(providerId); } expect(bundle.bootstrap).toMatchObject({ supported: false }); - expect(bundle.snapshot).toMatchObject({ supported: false }); + expect(bundle.snapshot).toMatchObject( + providerId === "docker" + ? { + supported: true, + capabilities: { + backup: true, + restore: true, + managedProfileRestore: true, + }, + } + : { supported: false }, + ); expect(bundle.recovery).toMatchObject({ supported: false }); } }); @@ -410,6 +425,40 @@ describe("RuntimeProviderBundle registry contract", () => { ).toThrow(/duplicate operation identities/u); }); + it("versions supported snapshot facets and enforces managed-profile capability dependencies", () => { + const docker = CURRENT_RUNTIME_PROVIDER_BUNDLES.docker!; + const snapshot = docker.snapshot; + expectSupportedSurface(snapshot); + expect(snapshot.contractVersion).toBe(1); + + expect(() => + createRuntimeProviderBundleRegistry([ + [ + "docker", + replaceSurface(docker, "snapshot", { + ...snapshot, + contractVersion: 2, + }), + ], + ]), + ).toThrow(/unsupported contract version/u); + expect(() => + createRuntimeProviderBundleRegistry([ + [ + "docker", + replaceSurface(docker, "snapshot", { + ...snapshot, + capabilities: { + ...snapshot.capabilities, + restore: false, + managedProfileRestore: true, + }, + }), + ], + ]), + ).toThrow(/cannot restore managed profiles/u); + }); + it("normalizes bounded opaque runtime receipts and rejects duplicate GPU devices", () => { const receipt = { schemaVersion: 1, @@ -430,6 +479,77 @@ describe("RuntimeProviderBundle registry contract", () => { runtime: { ...receipt.runtime, handle: "x".repeat(4097) }, }), ).toBeNull(); + expect( + normalizeRuntimeProviderRuntimeReceipt({ + ...receipt, + runtime: { ...receipt.runtime, handle: "opaque\ninjection" }, + }), + ).toBeNull(); + }); + + it("normalizes snapshot preflight and managed restore proof as one bounded contract", () => { + const managedProfile = { + agent: "openclaw", + profileFingerprint: "f".repeat(64), + }; + const preflight = { + schemaVersion: 1, + providerId: "docker", + operation: "restore", + sandboxName: "alpha", + providerHandle: "opaque-preflight", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + }; + const runtime = { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "c".repeat(64) }, + acceleration: { kind: "none" }, + }; + const source = { + schemaVersion: 1, + providerId: "docker", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation-1", + runtime, + }; + const restore = { + schemaVersion: 1, + providerId: "docker", + sandboxName: "alpha", + providerHandle: "opaque-restore-proof", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime, + managedProfile, + }; + + expect(normalizeRuntimeProviderManagedProfileRestoreAuthority(managedProfile)).toEqual( + managedProfile, + ); + expect(normalizeRuntimeProviderSnapshotPreflightReceipt(preflight)).toEqual(preflight); + expect(normalizeRuntimeProviderSnapshotRestoreSource(source)).toEqual(source); + expect(normalizeRuntimeProviderSnapshotRestoreReceipt(restore)).toEqual(restore); + expect( + normalizeRuntimeProviderSnapshotPreflightReceipt({ + ...preflight, + lifecycleGeneration: "generation\ninjection", + }), + ).toBeNull(); + expect( + normalizeRuntimeProviderSnapshotPreflightReceipt({ + ...preflight, + operation: { toString: () => "restore" }, + }), + ).toBeNull(); + expect( + normalizeRuntimeProviderSnapshotRestoreReceipt({ + ...restore, + runtime: { ...runtime, providerId: "other" }, + }), + ).toBeNull(); }); }); diff --git a/src/lib/onboard/runtime-provider/snapshot.test.ts b/src/lib/onboard/runtime-provider/snapshot.test.ts new file mode 100644 index 00000000000..d473a4da700 --- /dev/null +++ b/src/lib/onboard/runtime-provider/snapshot.test.ts @@ -0,0 +1,795 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { SandboxEntry } from "../../state/registry/types"; +import type { OpenShellDockerSandboxRuntimeSnapshotQuery } from "../openshell-docker-sandbox-containers"; +import type { + RuntimeProviderManagedProfileRestoreAuthority, + RuntimeProviderRuntimeReceipt, + RuntimeProviderSnapshotPreflightReceipt, +} from "./contract"; +import { + createDockerRuntimeProviderSnapshotSurface, + createRuntimeProviderSnapshotSurface, + observeDockerRuntimeSnapshot, + observeOpenShellRuntimeSnapshot, + type RuntimeProviderSnapshotObservation, +} from "./snapshot"; + +const managedProfile = { + agent: "openclaw", + profileFingerprint: "f".repeat(64), +} as const satisfies RuntimeProviderManagedProfileRestoreAuthority; + +function sandbox(overrides: Partial = {}): SandboxEntry { + return { + name: "alpha", + agent: "openclaw", + openshellDriver: "mxc", + gatewayName: "nemoclaw-18080", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + sandboxGpuEnabled: false, + sandboxGpuMode: "0", + sandboxGpuDevice: null, + ...overrides, + }; +} + +function observation( + providerId = "mxc", + overrides: Partial = {}, +): RuntimeProviderSnapshotObservation { + return { + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId, + runtime: { kind: "session", handle: "opaque-mxc-session" }, + acceleration: { kind: "none" }, + }, + ...overrides, + }; +} + +function surfaceDriver( + observe: () => RuntimeProviderSnapshotObservation, + restoreManagedProfile = vi.fn(() => "provider-restore-proof"), +) { + return { observe, restoreManagedProfile }; +} + +function snapshotSource( + preflight: RuntimeProviderSnapshotPreflightReceipt, + runtime: RuntimeProviderRuntimeReceipt, +) { + return { + schemaVersion: 1 as const, + providerId: preflight.providerId, + providerHandle: preflight.providerHandle, + lifecycleState: preflight.lifecycleState, + lifecycleGeneration: preflight.lifecycleGeneration, + runtime, + }; +} + +function requireSupportedSurface( + surface: T, +): Extract { + expect(surface.supported).toBe(true); + return surface as Extract; +} + +describe("runtime provider snapshot surface", () => { + it("binds the full runtime and lifecycle generation into opaque backup authority", () => { + const observe = vi.fn(() => observation()); + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe)), + ); + + const preflight = surface.preflight("backup", sandbox()); + const receipt = surface.capture(sandbox(), preflight); + + expect(preflight).toMatchObject({ + lifecycleGeneration: "generation-1", + lifecycleState: "running", + }); + expect(preflight.providerHandle).toMatch(/^[a-f0-9]{64}$/u); + expect(preflight.providerHandle).not.toContain("opaque-mxc-session"); + expect(receipt.runtime.handle).toBe("opaque-mxc-session"); + expect(observe).toHaveBeenCalledTimes(2); + }); + + it("invokes the owning provider restore facet and returns managed profile/runtime proof", () => { + const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ), + ); + const target = sandbox(); + const preflight = surface.preflight("restore", target); + + const receipt = surface.restore( + target, + preflight, + snapshotSource(preflight, observation().runtime), + managedProfile, + ); + + expect(restoreManagedProfile).toHaveBeenCalledWith(target, managedProfile); + expect(receipt).toMatchObject({ + providerId: "mxc", + sandboxName: "alpha", + lifecycleGeneration: "generation-1", + runtime: { providerId: "mxc", runtime: { handle: "opaque-mxc-session" } }, + managedProfile, + }); + expect(receipt.providerHandle).toMatch(/^[a-f0-9]{64}$/u); + }); + + it("restores after a runtime restart and binds proof to current runtime plus source provenance", () => { + const target = sandbox(); + const current = observation("mxc", { + lifecycleGeneration: "current-generation", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "current-session" }, + }, + }); + const restoreFrom = ( + targetObservation: RuntimeProviderSnapshotObservation, + sourceGeneration: string, + sourceHandle: string, + ) => { + const sourceObservation = observation("mxc", { + lifecycleGeneration: sourceGeneration, + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: sourceHandle }, + }, + }); + const sourceSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => sourceObservation), + ), + ); + const sourcePreflight = sourceSurface.preflight("backup", target); + const source = snapshotSource( + sourcePreflight, + sourceSurface.capture(target, sourcePreflight), + ); + const targetSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => targetObservation), + ), + ); + const targetPreflight = targetSurface.preflight("restore", target); + return targetSurface.restore(target, targetPreflight, source, managedProfile); + }; + + const first = restoreFrom(current, "source-generation-1", "source-session-1"); + const changedSource = restoreFrom(current, "source-generation-2", "source-session-2"); + const changedCurrent = restoreFrom( + observation("mxc", { + lifecycleGeneration: "next-current-generation", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "next-current-session" }, + }, + }), + "source-generation-1", + "source-session-1", + ); + + expect(first).toMatchObject({ + lifecycleGeneration: "current-generation", + runtime: { runtime: { handle: "current-session" } }, + }); + expect(changedSource.providerHandle).not.toBe(first.providerHandle); + expect(changedCurrent.providerHandle).not.toBe(first.providerHandle); + }); + + it.each([ + { + label: "before managed-profile proof", + observations: [ + observation(), + observation("mxc", { + lifecycleGeneration: "changed-generation", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "changed-session" }, + }, + }), + ], + expectedRestoreCalls: 0, + }, + { + label: "after managed-profile proof", + observations: [ + observation(), + observation(), + observation("mxc", { + lifecycleGeneration: "changed-generation", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "changed-session" }, + }, + }), + ], + expectedRestoreCalls: 1, + }, + ])("rejects current-runtime changes $label", ({ observations, expectedRestoreCalls }) => { + const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); + const observe = vi.fn<() => RuntimeProviderSnapshotObservation>(); + for (const value of observations) observe.mockReturnValueOnce(value); + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe, restoreManagedProfile)), + ); + const target = sandbox(); + const preflight = surface.preflight("restore", target); + const source = snapshotSource(preflight, observation().runtime); + + expect(() => surface.restore(target, preflight, source, managedProfile)).toThrow( + /runtime changed after snapshot preflight/u, + ); + expect(restoreManagedProfile).toHaveBeenCalledTimes(expectedRestoreCalls); + }); + + it("fails before provider restore when the target cannot represent source acceleration", () => { + const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); + const targetSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ), + ); + const sourceObservation = observation("mxc", { + runtime: { + ...observation().runtime, + acceleration: { kind: "gpu", vendor: "nvidia", devices: ["live-device-0"] }, + }, + }); + const sourceSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => sourceObservation), + ), + ); + const target = sandbox(); + const sourcePreflight = sourceSurface.preflight("backup", target); + const source = snapshotSource(sourcePreflight, sourceSurface.capture(target, sourcePreflight)); + const preflight = targetSurface.preflight("restore", target); + + expect(() => targetSurface.restore(target, preflight, source, managedProfile)).toThrow( + /cannot represent the snapshot acceleration state/u, + ); + expect(restoreManagedProfile).not.toHaveBeenCalled(); + }); + + it("fails before provider restore when the target cannot represent source lifecycle", () => { + const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); + const targetSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ), + ); + const stopped = observation("mxc", { lifecycleState: "stopped" }); + const sourceSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => stopped), + ), + ); + const target = sandbox(); + const sourcePreflight = sourceSurface.preflight("backup", target); + const source = snapshotSource(sourcePreflight, sourceSurface.capture(target, sourcePreflight)); + const targetPreflight = targetSurface.preflight("restore", target); + + expect(() => targetSurface.restore(target, targetPreflight, source, managedProfile)).toThrow( + /cannot represent the snapshot lifecycle state/u, + ); + expect(restoreManagedProfile).not.toHaveBeenCalled(); + }); + + it.each([ + { + label: "runtime identity/lifecycle", + changed: observation("mxc", { + lifecycleState: "paused", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "replacement-session" }, + }, + }), + }, + { + label: "acceleration", + changed: observation("mxc", { + runtime: { + ...observation().runtime, + acceleration: { + kind: "gpu", + vendor: "nvidia", + devices: ["nvidia.com/gpu=0"], + }, + }, + }), + }, + { + label: "lifecycle generation", + changed: observation("mxc", { lifecycleGeneration: "generation-2" }), + }, + ])("rejects a $label race after preflight", ({ changed }) => { + const observe = vi + .fn<() => RuntimeProviderSnapshotObservation>() + .mockReturnValueOnce(observation()) + .mockReturnValueOnce(changed); + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe)), + ); + const target = sandbox(); + const preflight = surface.preflight("backup", target); + + expect(() => surface.capture(target, preflight)).toThrow(/runtime changed after/u); + }); + + it("rejects a preflight receipt from another operation or sandbox", () => { + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation()), + ), + ); + const target = sandbox(); + const restorePreflight = surface.preflight("restore", target); + const otherTarget = sandbox({ name: "other" }); + + expect(() => surface.capture(target, restorePreflight)).toThrow( + /stale snapshot preflight authority/u, + ); + expect(() => + surface.restore( + otherTarget, + restorePreflight, + snapshotSource(restorePreflight, observation().runtime), + managedProfile, + ), + ).toThrow(/stale snapshot preflight authority/u); + }); + + it("rejects invalid runtime receipts, restore authority, and provider proof", () => { + const invalidRuntime = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation("other-provider")), + ), + ); + expect(() => invalidRuntime.preflight("backup", sandbox())).toThrow(/invalid runtime receipt/u); + + const invalidProof = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver( + () => observation(), + vi.fn(() => "proof\ninjection"), + ), + ), + ); + const preflight = invalidProof.preflight("restore", sandbox()); + const source = snapshotSource(preflight, observation().runtime); + expect(() => invalidProof.restore(sandbox(), preflight, source, managedProfile)).toThrow( + /invalid managed profile restore proof/u, + ); + + expect(() => + invalidProof.restore( + sandbox(), + preflight, + { + ...source, + runtime: { + ...source.runtime, + acceleration: { kind: "gpu", vendor: "nvidia", devices: ["tampered-device"] }, + }, + }, + managedProfile, + ), + ).toThrow(/does not match its provider handle/u); + }); +}); + +describe("OpenShell snapshot observation", () => { + const liveAcceleration = { + kind: "gpu", + vendor: "nvidia", + devices: ["provider-live-device-0"], + } as const satisfies RuntimeProviderRuntimeReceipt["acceleration"]; + + it("requires exact live identity, lifecycle generation, and provider acceleration", () => { + const capture = vi.fn(() => ({ + status: 0, + output: "Name: alpha\nId: openshell-alpha-id\nState: Ready\nGeneration: live-generation-7\n", + stdout: "", + stderr: "", + })); + const observeAcceleration = vi.fn(() => liveAcceleration); + + expect( + observeOpenShellRuntimeSnapshot( + sandbox({ + // Contradictory durable fields must not influence the live receipt. + sandboxGpuEnabled: false, + sandboxGpuMode: "0", + sandboxGpuDevice: null, + }), + "mxc", + { capture: capture as never, observeAcceleration }, + ), + ).toEqual({ + lifecycleState: "running", + lifecycleGeneration: "live-generation-7", + runtime: { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "openshell-sandbox", handle: "openshell-alpha-id" }, + acceleration: liveAcceleration, + }, + }); + expect(observeAcceleration).toHaveBeenCalledWith( + expect.objectContaining({ name: "alpha" }), + "openshell-alpha-id", + ); + }); + + it("rejects durable identity and acceleration fallbacks", () => { + const result = { + status: 0, + output: "alpha Ready\nGeneration: live-generation-7\n", + stdout: "", + stderr: "", + }; + expect(() => + observeOpenShellRuntimeSnapshot( + sandbox({ + lifecycleLiveIdentityFingerprint: "a".repeat(64), + sandboxGpuEnabled: true, + sandboxGpuMode: "1", + sandboxGpuDevice: "all", + }), + "mxc", + { + capture: (() => result) as never, + observeAcceleration: () => ({ kind: "none" }), + }, + ), + ).toThrow(/exact live runtime identity/u); + + expect(() => + observeOpenShellRuntimeSnapshot(sandbox(), "mxc", { + capture: (() => ({ + ...result, + output: "Id: sandbox-id\nState: Ready\nGeneration: live-generation-7\n", + })) as never, + }), + ).toThrow(/did not supply live acceleration evidence/u); + }); + + it.each([ + ["Paused", "paused"], + ["Stopped", "stopped"], + ["Exited", "stopped"], + ["Created", "stopped"], + ] as const)("normalizes the OpenShell %s lifecycle as %s", (state, expected) => { + expect( + observeOpenShellRuntimeSnapshot(sandbox(), "mxc", { + capture: (() => ({ + status: 0, + output: `Id: sandbox-id\nState: ${state}\nGeneration: generation-1\n`, + stdout: "", + stderr: "", + })) as never, + observeAcceleration: () => ({ kind: "none" }), + }).lifecycleState, + ).toBe(expected); + }); + + it("rejects mismatched provider, failed inspection, or missing generation", () => { + const capture = vi.fn(); + expect(() => + observeOpenShellRuntimeSnapshot(sandbox({ openshellDriver: "docker" }), "mxc", { + capture: capture as never, + }), + ).toThrow(/belongs to another runtime provider/u); + expect(capture).not.toHaveBeenCalled(); + + for (const result of [ + { status: 1, output: "not found", stdout: "", stderr: "" }, + { + status: 0, + signal: "SIGTERM", + output: "Id: sandbox-id\nState: Ready\nGeneration: generation-1\n", + stdout: "", + stderr: "", + }, + ]) { + expect(() => + observeOpenShellRuntimeSnapshot(sandbox(), "mxc", { + capture: (() => result) as never, + observeAcceleration: () => ({ kind: "none" }), + }), + ).toThrow(/runtime identity could not be inspected/u); + } + expect(() => + observeOpenShellRuntimeSnapshot(sandbox(), "mxc", { + capture: (() => ({ + status: 0, + output: "Id: sandbox-id\nState: Ready\n", + stdout: "", + stderr: "", + })) as never, + observeAcceleration: () => ({ kind: "none" }), + }), + ).toThrow(/lifecycle generation cannot be represented/u); + }); +}); + +function dockerSnapshot( + overrides: Partial> = {}, +): Extract { + return { + ok: true, + imageId: `sha256:${"b".repeat(64)}`, + bookkeepingImageRef: "managed@example", + stateError: "", + deviceRequests: null, + devices: null, + runtime: "runc", + nvidiaVisibleDevices: null, + nativeGpuAttachmentState: "absent", + containerId: "c".repeat(64), + ...overrides, + }; +} + +function dockerLifecycleCapture( + containerId = "c".repeat(64), + overrides: { status?: string; paused?: boolean; restartCount?: number } = {}, +) { + return vi.fn(() => ({ + status: 0, + stdout: JSON.stringify([ + containerId, + overrides.status ?? "running", + overrides.paused ?? false, + "2026-07-30T12:00:00Z", + "0001-01-01T00:00:00Z", + overrides.restartCount ?? 0, + ]), + stderr: "", + })); +} + +describe("Docker provider snapshot evidence", () => { + it("normalizes Docker's explicit paused status", () => { + const observed = observeDockerRuntimeSnapshot( + sandbox({ openshellDriver: "docker" }), + "docker", + { + captureHostCommand: dockerLifecycleCapture(undefined, { status: "paused", paused: true }), + queryRuntimeSnapshot: () => dockerSnapshot(), + }, + ); + + expect(observed.lifecycleState).toBe("paused"); + }); + + it("captures exact live container, lifecycle, and device selectors", () => { + const queryRuntimeSnapshot = vi.fn(() => + dockerSnapshot({ + deviceRequests: [ + { + Driver: "nvidia", + Count: 0, + DeviceIDs: ["GPU-live-0"], + Capabilities: [["gpu"]], + Options: null, + }, + ], + nativeGpuAttachmentState: "present", + runtime: "nvidia", + nvidiaVisibleDevices: "GPU-live-0", + }), + ); + const observed = observeDockerRuntimeSnapshot( + sandbox({ + openshellDriver: "docker", + sandboxGpuEnabled: false, + sandboxGpuMode: "0", + sandboxGpuDevice: null, + }), + "docker", + { captureHostCommand: dockerLifecycleCapture(), queryRuntimeSnapshot }, + ); + + expect(observed).toMatchObject({ + lifecycleState: "running", + lifecycleGeneration: expect.stringMatching(/^[a-f0-9]{64}$/u), + runtime: { + providerId: "docker", + runtime: { kind: "docker-container", handle: "c".repeat(64) }, + acceleration: { + kind: "gpu", + vendor: "nvidia", + devices: ["docker-device-id:GPU-live-0", "docker-nvidia-visible-device:GPU-live-0"], + }, + }, + }); + }); + + it("accepts Docker's explicit count=-1 selector but rejects inferred or ambiguous GPU state", () => { + const allDevices = observeDockerRuntimeSnapshot( + sandbox({ openshellDriver: "docker" }), + "docker", + { + captureHostCommand: dockerLifecycleCapture(), + queryRuntimeSnapshot: () => + dockerSnapshot({ + deviceRequests: [ + { + Driver: "nvidia", + Count: -1, + DeviceIDs: null, + Capabilities: [["gpu"]], + Options: null, + }, + ], + nativeGpuAttachmentState: "present", + runtime: "nvidia", + nvidiaVisibleDevices: "all", + }), + }, + ); + expect(allDevices.runtime.acceleration).toMatchObject({ + devices: ["docker-device-request:nvidia:count=-1", "docker-nvidia-visible-devices:all"], + }); + + for (const snapshot of [ + dockerSnapshot({ + deviceRequests: null, + nativeGpuAttachmentState: "present", + runtime: "nvidia", + nvidiaVisibleDevices: null, + }), + dockerSnapshot({ + deviceRequests: [ + { + Driver: "nvidia", + Count: 1, + DeviceIDs: null, + Capabilities: [["gpu"]], + Options: null, + }, + ], + nativeGpuAttachmentState: "present", + runtime: "nvidia", + }), + dockerSnapshot({ nativeGpuAttachmentState: "unknown", runtime: "custom-runtime" }), + ]) { + expect(() => + observeDockerRuntimeSnapshot(sandbox({ openshellDriver: "docker" }), "docker", { + captureHostCommand: dockerLifecycleCapture(), + queryRuntimeSnapshot: () => snapshot, + }), + ).toThrow(/acceleration|exact live device selectors/u); + } + }); + + it("captures the NVIDIA Container Runtime selector used by Jetson", () => { + const observed = observeDockerRuntimeSnapshot( + sandbox({ openshellDriver: "docker" }), + "docker", + { + captureHostCommand: dockerLifecycleCapture(), + queryRuntimeSnapshot: () => + dockerSnapshot({ + deviceRequests: null, + devices: null, + nativeGpuAttachmentState: "present", + runtime: "nvidia", + nvidiaVisibleDevices: "0,GPU-live-1", + }), + }, + ); + + expect(observed.runtime.acceleration).toEqual({ + kind: "gpu", + vendor: "nvidia", + devices: ["docker-nvidia-visible-device:0", "docker-nvidia-visible-device:GPU-live-1"], + }); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("runs the in-sandbox %s profile verifier and fails closed on refusal", (agent) => { + const authority = { + agent, + profileFingerprint: managedProfile.profileFingerprint, + }; + const captureOpenShell = vi.fn(() => ({ + status: 0, + output: `[managed-startup] verified ${agent} profile completion\n`, + stdout: "", + stderr: "", + })); + const dependencies = { + captureHostCommand: dockerLifecycleCapture(), + captureOpenShell: captureOpenShell as never, + queryRuntimeSnapshot: () => dockerSnapshot(), + }; + const surface = requireSupportedSurface( + createDockerRuntimeProviderSnapshotSurface("docker", dependencies), + ); + const target = sandbox({ agent, openshellDriver: "docker" }); + const preflight = surface.preflight("restore", target); + const source = snapshotSource(preflight, { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "c".repeat(64) }, + acceleration: { kind: "none" }, + }); + const receipt = surface.restore(target, preflight, source, authority); + expect(receipt.managedProfile).toEqual(authority); + expect(captureOpenShell).toHaveBeenCalledWith( + [ + "sandbox", + "exec", + "--name", + "alpha", + "-g", + "nemoclaw-18080", + "--no-tty", + "--timeout", + "10", + "--", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--verify-completion", + "--agent", + agent, + "--profile-fingerprint", + authority.profileFingerprint, + ], + expect.objectContaining({ + ignoreError: true, + includeStderr: true, + timeout: 15_000, + }), + ); + + const denied = requireSupportedSurface( + createDockerRuntimeProviderSnapshotSurface("docker", { + ...dependencies, + captureOpenShell: (() => ({ + status: 1, + output: "profile mismatch", + stdout: "", + stderr: "", + })) as never, + }), + ); + const deniedPreflight = denied.preflight("restore", target); + const deniedSource = snapshotSource(deniedPreflight, source.runtime); + expect(() => denied.restore(target, deniedPreflight, deniedSource, authority)).toThrow( + /managed profile restoration could not be proven/u, + ); + }); +}); diff --git a/src/lib/onboard/runtime-provider/snapshot.ts b/src/lib/onboard/runtime-provider/snapshot.ts new file mode 100644 index 00000000000..3f088c1ef0c --- /dev/null +++ b/src/lib/onboard/runtime-provider/snapshot.ts @@ -0,0 +1,679 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; + +import { captureOpenshell } from "../../adapters/openshell/runtime"; +import type { SandboxEntry } from "../../state/registry/types"; +import { resolveSandboxGatewayName } from "../gateway-binding"; +import { + type OpenShellDockerSandboxRuntimeSnapshotQuery, + queryOpenShellDockerSandboxRuntimeSnapshot, +} from "../openshell-docker-sandbox-containers"; +import { + RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, + RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, + type RuntimeProviderCommandCapture, + type RuntimeProviderManagedProfileRestoreAuthority, + type RuntimeProviderRuntimeReceipt, + type RuntimeProviderSnapshotLifecycleState, + type RuntimeProviderSnapshotOperation, + type RuntimeProviderSnapshotPreflightReceipt, + type RuntimeProviderSnapshotRestoreReceipt, + type RuntimeProviderSnapshotRestoreSource, + type RuntimeProviderSnapshotSurface, +} from "./contract"; +import { + normalizeRuntimeProviderIdentity, + normalizeRuntimeProviderManagedProfileRestoreAuthority, + normalizeRuntimeProviderRuntimeReceipt, + normalizeRuntimeProviderSnapshotPreflightReceipt, + normalizeRuntimeProviderSnapshotRestoreSource, +} from "./registry"; + +const SANDBOX_ID_PATTERN = /^[A-Za-z0-9._-]{1,512}$/u; +const DOCKER_CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/u; +const MANAGED_STARTUP_RUNTIME_EXECUTABLE = + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs"; +const LIFECYCLE_GENERATION_PATTERN = /^[A-Za-z0-9._:/=-]{1,512}$/u; +const ANSI_PATTERN = /\u001b\[[0-9;]*m/gu; +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; + +export interface RuntimeProviderSnapshotObservation { + readonly lifecycleState: RuntimeProviderSnapshotLifecycleState; + readonly lifecycleGeneration: string; + readonly runtime: RuntimeProviderRuntimeReceipt; +} + +export type RuntimeProviderSnapshotObserver = ( + sandbox: SandboxEntry, + providerId: string, +) => RuntimeProviderSnapshotObservation; + +export type RuntimeProviderManagedProfileRestorer = ( + sandbox: SandboxEntry, + authority: RuntimeProviderManagedProfileRestoreAuthority, +) => string; + +export interface RuntimeProviderSnapshotDriver { + readonly observe: RuntimeProviderSnapshotObserver; + readonly restoreManagedProfile: RuntimeProviderManagedProfileRestorer; +} + +export interface OpenShellRuntimeSnapshotDependencies { + readonly capture: typeof captureOpenshell; + /** + * The owning provider must supply acceleration observed from its live + * runtime. Durable registry intent is deliberately not accepted here. + */ + readonly observeAcceleration: ( + sandbox: SandboxEntry, + runtimeId: string, + ) => RuntimeProviderRuntimeReceipt["acceleration"]; +} + +export interface DockerRuntimeSnapshotDependencies { + readonly captureHostCommand: ( + command: string, + args: string[], + timeout?: number, + ) => RuntimeProviderCommandCapture; + readonly captureOpenShell: typeof captureOpenshell; + readonly queryRuntimeSnapshot: ( + sandboxName: string, + ) => OpenShellDockerSandboxRuntimeSnapshotQuery; +} + +export class RuntimeProviderSnapshotError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Runtime snapshot provider failed: ${message}`, options); + this.name = "RuntimeProviderSnapshotError"; + } +} + +function gatewayScopedSandboxGetArgs(sandbox: SandboxEntry): string[] { + const gatewayName = resolveSandboxGatewayName(sandbox); + return gatewayName + ? ["sandbox", "get", "-g", gatewayName, sandbox.name] + : ["sandbox", "get", sandbox.name]; +} + +function gatewayScopedManagedProfileVerifyArgs( + sandbox: SandboxEntry, + authority: RuntimeProviderManagedProfileRestoreAuthority, +): string[] { + const args = ["sandbox", "exec", "--name", sandbox.name]; + const gatewayName = resolveSandboxGatewayName(sandbox); + if (gatewayName) args.push("-g", gatewayName); + args.push( + "--no-tty", + "--timeout", + "10", + "--", + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--verify-completion", + "--agent", + authority.agent, + "--profile-fingerprint", + authority.profileFingerprint, + ); + return args; +} + +function cleanOutput(value: string): string { + return value.replace(ANSI_PATTERN, ""); +} + +function parseSandboxId(output: string): string | null { + const match = cleanOutput(output).match(/^\s*(?:Id|ID):\s*([A-Za-z0-9._-]+)\s*$/mu); + return match && SANDBOX_ID_PATTERN.test(match[1] ?? "") ? (match[1] ?? null) : null; +} + +function parseLifecycleState( + output: string, + sandboxName: string, +): RuntimeProviderSnapshotLifecycleState | null { + const clean = cleanOutput(output); + const field = clean.match(/^\s*(?:State|Phase|Status):\s*([A-Za-z][A-Za-z0-9_-]*)\s*$/imu)?.[1]; + const row = clean + .split(/\r?\n/u) + .map((line) => line.trim().split(/\s+/u)) + .find((columns) => columns[0] === sandboxName); + const phase = field ?? row?.slice(1).find((value) => /^[A-Za-z][A-Za-z0-9_-]*$/u.test(value)); + if (phase === "Ready" || phase === "Running") return "running"; + if (phase === "Paused") return "paused"; + if (phase === "Stopped" || phase === "Exited" || phase === "Created") return "stopped"; + return null; +} + +function parseLifecycleGeneration(output: string): string | null { + const match = cleanOutput(output).match( + /^\s*(?:Generation|ResourceVersion|Resource Version):\s*([A-Za-z0-9._:/=-]+)\s*$/imu, + ); + const generation = match?.[1] ?? ""; + return LIFECYCLE_GENERATION_PATTERN.test(generation) ? generation : null; +} + +/** + * Observe an OpenShell-owned runtime without exposing its CLI shape to the + * snapshot action. Exact live identity, lifecycle generation, and provider + * acceleration evidence are all mandatory; durable fallbacks fail closed. + */ +export function observeOpenShellRuntimeSnapshot( + sandbox: SandboxEntry, + providerId: string, + dependencies: Partial = {}, +): RuntimeProviderSnapshotObservation { + if (normalizeRuntimeProviderIdentity(sandbox.openshellDriver) !== providerId) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' belongs to another runtime provider`, + ); + } + const capture = dependencies.capture ?? captureOpenshell; + const result = capture(gatewayScopedSandboxGetArgs(sandbox), { + ignoreError: true, + includeStderr: true, + timeout: 10_000, + }); + if (result.status !== 0 || result.error || result.signal) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' runtime identity could not be inspected`, + ); + } + const output = result.output || ""; + const sandboxId = parseSandboxId(output); + if (!sandboxId) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' exact live runtime identity cannot be represented`, + ); + } + const lifecycleState = parseLifecycleState(output, sandbox.name); + const lifecycleGeneration = parseLifecycleGeneration(output); + if (!lifecycleState || !lifecycleGeneration) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' lifecycle generation cannot be represented`, + ); + } + if (!dependencies.observeAcceleration) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' did not supply live acceleration evidence`, + ); + } + return { + lifecycleState, + lifecycleGeneration, + runtime: { + schemaVersion: 1, + providerId, + runtime: { + kind: "openshell-sandbox", + handle: sandboxId, + }, + acceleration: dependencies.observeAcceleration(sandbox, sandboxId), + }, + }; +} + +function dockerRequestUsesGpu( + request: NonNullable< + Extract["deviceRequests"] + >[number], +): boolean { + return ( + request.Driver.trim().toLowerCase() === "nvidia" || + request.DeviceIDs?.some((device) => /^nvidia[.]com\/gpu(?:=|$)/iu.test(device.trim())) === + true || + request.Capabilities?.some((group) => + group.some((capability) => capability.trim().toLowerCase() === "gpu"), + ) === true + ); +} + +function dockerGpuSelectors( + snapshot: Extract, +): RuntimeProviderRuntimeReceipt["acceleration"] { + if (snapshot.nativeGpuAttachmentState === "absent") return { kind: "none" }; + if (snapshot.nativeGpuAttachmentState !== "present") { + throw new RuntimeProviderSnapshotError("Docker returned ambiguous live acceleration evidence"); + } + + const selectors: string[] = []; + if (snapshot.runtime.trim().toLowerCase() === "nvidia") { + const visibleDevices = snapshot.nvidiaVisibleDevices; + if (visibleDevices === "all") { + selectors.push("docker-nvidia-visible-devices:all"); + } else if (visibleDevices && !["none", "void"].includes(visibleDevices)) { + for (const device of visibleDevices.split(",")) { + selectors.push(`docker-nvidia-visible-device:${device}`); + } + } + } + for (const request of snapshot.deviceRequests ?? []) { + if (!dockerRequestUsesGpu(request)) continue; + if (request.DeviceIDs && request.DeviceIDs.length > 0) { + for (const device of request.DeviceIDs) { + selectors.push(`docker-device-id:${device}`); + } + continue; + } + if (request.Count === -1) { + // Count=-1 is Docker's explicit live all-device selector. Never infer + // this value from a durable "GPU enabled" flag. + selectors.push(`docker-device-request:${request.Driver || "default"}:count=-1`); + continue; + } + throw new RuntimeProviderSnapshotError( + "Docker GPU attachment does not expose exact live device selectors", + ); + } + for (const mapping of snapshot.devices ?? []) { + const rendered = + `docker-device-path:${mapping.PathOnHost}=>${mapping.PathInContainer}` + + `:${mapping.CgroupPermissions}`; + if ( + /^\/dev\/(?:nvidia|dri|nvhost|nvmap|tegra)/iu.test(mapping.PathOnHost.trim()) || + /^\/dev\/(?:nvidia|dri|nvhost|nvmap|tegra)/iu.test(mapping.PathInContainer.trim()) + ) { + selectors.push(rendered); + } + } + const devices = [...new Set(selectors)].sort(); + if ( + devices.length === 0 || + devices.some( + (device) => + device.trim() === "" || + Buffer.byteLength(device, "utf8") > 512 || + CONTROL_CHARACTERS.test(device), + ) + ) { + throw new RuntimeProviderSnapshotError( + "Docker GPU attachment does not expose exact live device selectors", + ); + } + return { kind: "gpu", vendor: "nvidia", devices }; +} + +function parseDockerLifecycle( + result: RuntimeProviderCommandCapture, + expectedContainerId: string, +): { + readonly state: RuntimeProviderSnapshotLifecycleState; + readonly generation: string; +} { + if (result.status !== 0 || result.error) { + throw new RuntimeProviderSnapshotError("Docker lifecycle state could not be inspected"); + } + let fields: unknown; + try { + fields = JSON.parse(result.stdout.trim()); + } catch { + throw new RuntimeProviderSnapshotError("Docker returned malformed lifecycle state"); + } + if ( + !Array.isArray(fields) || + fields.length !== 6 || + fields[0] !== expectedContainerId || + typeof fields[1] !== "string" || + typeof fields[2] !== "boolean" || + typeof fields[3] !== "string" || + typeof fields[4] !== "string" || + !Number.isSafeInteger(fields[5]) || + fields[5] < 0 + ) { + throw new RuntimeProviderSnapshotError("Docker returned malformed lifecycle state"); + } + const status = fields[1].trim().toLowerCase(); + let state: RuntimeProviderSnapshotLifecycleState; + if (status === "running") state = fields[2] ? "paused" : "running"; + else if (status === "paused" && fields[2] === true) state = "paused"; + else if (["created", "exited", "dead"].includes(status) && fields[2] === false) state = "stopped"; + else { + throw new RuntimeProviderSnapshotError( + `Docker lifecycle '${status || "unknown"}' cannot be represented`, + ); + } + const generation = createHash("sha256") + .update( + JSON.stringify({ + containerId: fields[0], + status, + paused: fields[2], + startedAt: fields[3], + finishedAt: fields[4], + restartCount: fields[5], + }), + "utf8", + ) + .digest("hex"); + return { state, generation }; +} + +export function observeDockerRuntimeSnapshot( + sandbox: SandboxEntry, + providerId: string, + dependencies: Pick< + DockerRuntimeSnapshotDependencies, + "captureHostCommand" | "queryRuntimeSnapshot" + >, +): RuntimeProviderSnapshotObservation { + if (normalizeRuntimeProviderIdentity(sandbox.openshellDriver) !== providerId) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' belongs to another runtime provider`, + ); + } + const snapshot = dependencies.queryRuntimeSnapshot(sandbox.name); + if (!snapshot.ok || !DOCKER_CONTAINER_ID_PATTERN.test(snapshot.containerId)) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' exact Docker runtime identity could not be inspected`, + ); + } + const lifecycle = parseDockerLifecycle( + dependencies.captureHostCommand( + "docker", + [ + "inspect", + "--type", + "container", + "--format", + "[{{json .Id}},{{json .State.Status}},{{json .State.Paused}},{{json .State.StartedAt}},{{json .State.FinishedAt}},{{json .RestartCount}}]", + snapshot.containerId, + ], + 10_000, + ), + snapshot.containerId, + ); + return { + lifecycleState: lifecycle.state, + lifecycleGeneration: lifecycle.generation, + runtime: { + schemaVersion: 1, + providerId, + runtime: { kind: "docker-container", handle: snapshot.containerId }, + acceleration: dockerGpuSelectors(snapshot), + }, + }; +} + +export function verifyOpenShellManagedProfileRestore( + sandbox: SandboxEntry, + authorityValue: RuntimeProviderManagedProfileRestoreAuthority, + dependencies: Pick, +): string { + const authority = normalizeRuntimeProviderManagedProfileRestoreAuthority(authorityValue); + if (!authority) { + throw new RuntimeProviderSnapshotError("managed profile restore authority is invalid"); + } + const result = dependencies.captureOpenShell( + gatewayScopedManagedProfileVerifyArgs(sandbox, authority), + { + ignoreError: true, + includeStderr: true, + timeout: 15_000, + }, + ); + if (result.status !== 0 || result.error || result.signal) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' managed profile restoration could not be proven`, + ); + } + return createHash("sha256") + .update(sandbox.name, "utf8") + .update("\0", "utf8") + .update(authority.agent, "utf8") + .update("\0", "utf8") + .update(authority.profileFingerprint, "utf8") + .update("\0", "utf8") + .update(cleanOutput(result.output || ""), "utf8") + .digest("hex"); +} + +function opaqueProviderHandle( + providerId: string, + observation: RuntimeProviderSnapshotObservation, +): string { + return createHash("sha256") + .update( + JSON.stringify({ + providerId, + lifecycleState: observation.lifecycleState, + lifecycleGeneration: observation.lifecycleGeneration, + runtime: observation.runtime, + }), + "utf8", + ) + .digest("hex"); +} + +function observeAndNormalize( + observer: RuntimeProviderSnapshotObserver, + sandbox: SandboxEntry, + providerId: string, +): RuntimeProviderSnapshotObservation { + const observed = observer(sandbox, providerId); + const runtime = normalizeRuntimeProviderRuntimeReceipt(observed.runtime); + if (!runtime || runtime.providerId !== providerId) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' returned an invalid runtime receipt`, + ); + } + if ( + !["running", "paused", "stopped"].includes(observed.lifecycleState) || + !LIFECYCLE_GENERATION_PATTERN.test(observed.lifecycleGeneration) + ) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' returned invalid lifecycle authority`, + ); + } + return { + lifecycleState: observed.lifecycleState, + lifecycleGeneration: observed.lifecycleGeneration, + runtime, + }; +} + +function requireStablePreflight( + value: RuntimeProviderSnapshotPreflightReceipt, + providerId: string, + operation: RuntimeProviderSnapshotOperation, + sandbox: SandboxEntry, +): RuntimeProviderSnapshotPreflightReceipt { + const normalized = normalizeRuntimeProviderSnapshotPreflightReceipt(value); + if ( + !normalized || + normalized.providerId !== providerId || + normalized.operation !== operation || + normalized.sandboxName !== sandbox.name + ) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' received stale snapshot preflight authority`, + ); + } + return normalized; +} + +function assertUnchanged( + providerId: string, + expected: RuntimeProviderSnapshotPreflightReceipt, + observed: RuntimeProviderSnapshotObservation, +): void { + if ( + opaqueProviderHandle(providerId, observed) !== expected.providerHandle || + observed.lifecycleState !== expected.lifecycleState || + observed.lifecycleGeneration !== expected.lifecycleGeneration + ) { + throw new RuntimeProviderSnapshotError( + `sandbox '${expected.sandboxName}' runtime changed after snapshot preflight`, + ); + } +} + +function restoreProviderHandle( + preflight: RuntimeProviderSnapshotPreflightReceipt, + source: RuntimeProviderSnapshotRestoreSource, + authority: RuntimeProviderManagedProfileRestoreAuthority, + providerProof: string, + observed: RuntimeProviderSnapshotObservation, +): string { + return createHash("sha256") + .update( + JSON.stringify({ + preflight, + source, + authority, + providerProof, + observed, + }), + "utf8", + ) + .digest("hex"); +} + +function validateRestoreRequest( + providerId: string, + driver: RuntimeProviderSnapshotDriver, + sandbox: SandboxEntry, + preflightValue: RuntimeProviderSnapshotPreflightReceipt, + sourceValue: RuntimeProviderSnapshotRestoreSource, + managedProfileValue: RuntimeProviderManagedProfileRestoreAuthority, +): { + readonly expected: RuntimeProviderSnapshotPreflightReceipt; + readonly source: RuntimeProviderSnapshotRestoreSource; + readonly managedProfile: RuntimeProviderManagedProfileRestoreAuthority; +} { + const expected = requireStablePreflight(preflightValue, providerId, "restore", sandbox); + const source = normalizeRuntimeProviderSnapshotRestoreSource(sourceValue); + if (!source || source.providerId !== providerId) { + throw new RuntimeProviderSnapshotError( + "source runtime authority is invalid or belongs to another provider", + ); + } + const sourceObservation = { + lifecycleState: source.lifecycleState, + lifecycleGeneration: source.lifecycleGeneration, + runtime: source.runtime, + }; + if (opaqueProviderHandle(providerId, sourceObservation) !== source.providerHandle) { + throw new RuntimeProviderSnapshotError( + "source runtime receipt does not match its provider handle", + ); + } + // A recovery may legitimately follow a runtime restart. Preserve the exact + // current handle/generation and bind them into the restore receipt rather + // than requiring them to equal the historical source identity. + if (source.lifecycleState !== expected.lifecycleState) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' cannot represent the snapshot lifecycle state`, + ); + } + const managedProfile = + normalizeRuntimeProviderManagedProfileRestoreAuthority(managedProfileValue); + if (!managedProfile) { + throw new RuntimeProviderSnapshotError("managed profile restore authority is invalid"); + } + const observed = observeAndNormalize(driver.observe, sandbox, providerId); + assertUnchanged(providerId, expected, observed); + if (!isDeepStrictEqual(source.runtime.acceleration, observed.runtime.acceleration)) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' cannot represent the snapshot acceleration state`, + ); + } + return { expected, source, managedProfile }; +} + +export function createRuntimeProviderSnapshotSurface( + providerId: string, + driver: RuntimeProviderSnapshotDriver, +): RuntimeProviderSnapshotSurface { + const capabilities = { + backup: true, + restore: true, + managedProfileRestore: true, + } as const; + return { + providerId, + supported: true, + contractVersion: RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, + capabilities, + preflight(operation, sandbox) { + const observed = observeAndNormalize(driver.observe, sandbox, providerId); + return { + schemaVersion: RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, + providerId, + operation, + sandboxName: sandbox.name, + providerHandle: opaqueProviderHandle(providerId, observed), + lifecycleState: observed.lifecycleState, + lifecycleGeneration: observed.lifecycleGeneration, + }; + }, + capture(sandbox, preflight) { + const expected = requireStablePreflight(preflight, providerId, "backup", sandbox); + const observed = observeAndNormalize(driver.observe, sandbox, providerId); + assertUnchanged(providerId, expected, observed); + return observed.runtime; + }, + validateRestore(sandbox, preflight, source, managedProfile) { + validateRestoreRequest(providerId, driver, sandbox, preflight, source, managedProfile); + }, + restore(sandbox, preflight, sourceValue, managedProfileValue) { + const { expected, source, managedProfile } = validateRestoreRequest( + providerId, + driver, + sandbox, + preflight, + sourceValue, + managedProfileValue, + ); + const providerProof = driver.restoreManagedProfile(sandbox, managedProfile); + if ( + typeof providerProof !== "string" || + providerProof.trim() === "" || + Buffer.byteLength(providerProof, "utf8") > 4096 || + CONTROL_CHARACTERS.test(providerProof) + ) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' returned invalid managed profile restore proof`, + ); + } + const after = observeAndNormalize(driver.observe, sandbox, providerId); + assertUnchanged(providerId, expected, after); + const receipt = { + schemaVersion: 1 as const, + providerId, + sandboxName: sandbox.name, + providerHandle: restoreProviderHandle( + expected, + source, + managedProfile, + providerProof, + after, + ), + lifecycleState: after.lifecycleState, + lifecycleGeneration: after.lifecycleGeneration, + runtime: after.runtime, + managedProfile, + } satisfies RuntimeProviderSnapshotRestoreReceipt; + return receipt; + }, + }; +} + +export function createDockerRuntimeProviderSnapshotSurface( + providerId: string, + dependencies: Partial & + Pick, +): RuntimeProviderSnapshotSurface { + const resolved = { + captureHostCommand: dependencies.captureHostCommand, + captureOpenShell: dependencies.captureOpenShell ?? captureOpenshell, + queryRuntimeSnapshot: + dependencies.queryRuntimeSnapshot ?? queryOpenShellDockerSandboxRuntimeSnapshot, + }; + return createRuntimeProviderSnapshotSurface(providerId, { + observe: (sandbox, id) => observeDockerRuntimeSnapshot(sandbox, id, resolved), + restoreManagedProfile: (sandbox, authority) => + verifyOpenShellManagedProfileRestore(sandbox, authority, resolved), + }); +} diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index f11e1e21142..c7671ff4e60 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -84,6 +84,10 @@ const DEFAULT_RUNTIME_SNAPSHOT = { imageId: IMAGE_ID, bookkeepingImageRef: "openshell/sandbox-from:test", stateError: "", + deviceRequests: null, + devices: null, + runtime: "runc", + nvidiaVisibleDevices: null, nativeGpuAttachmentState: "absent" as const, containerId: "container-a", }; diff --git a/src/lib/state/registry/runtime-snapshot.test.ts b/src/lib/state/registry/runtime-snapshot.test.ts new file mode 100644 index 00000000000..597decd9fdc --- /dev/null +++ b/src/lib/state/registry/runtime-snapshot.test.ts @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + cloneSandboxRuntimeSnapshot, + SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, +} from "./runtime-snapshot"; + +function gpuSnapshot() { + return { + schemaVersion: SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, + providerId: "mxc", + providerHandle: "mxc-snapshot:opaque-123", + lifecycleState: "running", + lifecycleGeneration: "generation-42", + runtime: { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "session", handle: "opaque-session-42" }, + acceleration: { + kind: "gpu", + vendor: "nvidia", + devices: ["nvidia.com/gpu=0"], + }, + }, + } as const; +} + +describe("sandbox runtime snapshot normalization", () => { + it("clones opaque provider and runtime handles without interpreting them", () => { + const input = gpuSnapshot(); + const normalized = cloneSandboxRuntimeSnapshot(input); + + expect(normalized).toEqual(input); + expect(normalized).not.toBe(input); + expect(normalized?.runtime).not.toBe(input.runtime); + expect(normalized?.runtime.acceleration).not.toBe(input.runtime.acceleration); + }); + + it("rejects provider identity drift between the wrapper and runtime receipt", () => { + expect( + cloneSandboxRuntimeSnapshot({ + ...gpuSnapshot(), + providerId: "docker", + }), + ).toBeUndefined(); + }); + + it.each([ + { lifecycleState: "restarting" }, + { lifecycleGeneration: "" }, + { providerHandle: "" }, + { providerHandle: "opaque\nhandle" }, + { schemaVersion: 2 }, + ])("rejects an unrepresentable persisted wrapper: %j", (change) => { + expect(cloneSandboxRuntimeSnapshot({ ...gpuSnapshot(), ...change })).toBeUndefined(); + }); + + it("rejects malformed or duplicate normalized acceleration devices", () => { + expect( + cloneSandboxRuntimeSnapshot({ + ...gpuSnapshot(), + runtime: { + ...gpuSnapshot().runtime, + acceleration: { + kind: "gpu", + vendor: "nvidia", + devices: ["nvidia.com/gpu=0", "nvidia.com/gpu=0"], + }, + }, + }), + ).toBeUndefined(); + }); + + it("drops unknown persisted keys instead of widening snapshot authority", () => { + expect( + cloneSandboxRuntimeSnapshot({ + ...gpuSnapshot(), + engine: "podman", + containerName: "must-not-become-authority", + runtime: { + ...gpuSnapshot().runtime, + command: ["delete", "by-name"], + }, + }), + ).toEqual(gpuSnapshot()); + }); + + it.each([ + { + label: "runtime control characters", + runtime: { + ...gpuSnapshot().runtime, + runtime: { kind: "session", handle: "opaque\nsession" }, + }, + }, + { + label: "empty runtime kind", + runtime: { + ...gpuSnapshot().runtime, + runtime: { kind: "", handle: "opaque" }, + }, + }, + { + label: "empty GPU device inventory", + runtime: { + ...gpuSnapshot().runtime, + acceleration: { kind: "gpu", vendor: "nvidia", devices: [] }, + }, + }, + { + label: "unknown acceleration kind", + runtime: { + ...gpuSnapshot().runtime, + acceleration: { kind: "tpu", devices: ["all"] }, + }, + }, + ])("rejects $label in the nested provider receipt", ({ runtime }) => { + expect(cloneSandboxRuntimeSnapshot({ ...gpuSnapshot(), runtime })).toBeUndefined(); + }); + + it("accepts a bounded provider-neutral no-acceleration receipt", () => { + expect( + cloneSandboxRuntimeSnapshot({ + schemaVersion: 1, + providerId: "kubernetes", + providerHandle: "opaque-provider-handle", + lifecycleState: "stopped", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "kubernetes", + runtime: { kind: "sandbox", handle: "opaque-runtime-handle" }, + acceleration: { kind: "none" }, + }, + }), + ).toEqual({ + schemaVersion: 1, + providerId: "kubernetes", + providerHandle: "opaque-provider-handle", + lifecycleState: "stopped", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "kubernetes", + runtime: { kind: "sandbox", handle: "opaque-runtime-handle" }, + acceleration: { kind: "none" }, + }, + }); + }); +}); diff --git a/src/lib/state/registry/runtime-snapshot.ts b/src/lib/state/registry/runtime-snapshot.ts new file mode 100644 index 00000000000..d4fc90886cb --- /dev/null +++ b/src/lib/state/registry/runtime-snapshot.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderRuntimeReceipt } from "../../onboard/runtime-provider/contract"; +import { normalizeRuntimeProviderRuntimeReceipt } from "../../onboard/runtime-provider/registry"; + +export const SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION = 1 as const; + +export type SandboxRuntimeLifecycleState = "running" | "paused" | "stopped"; + +/** + * Provider-neutral runtime state persisted beside a filesystem snapshot. + * + * `providerHandle` and `runtime.handle` remain opaque to the state and action + * layers. Only the owning provider may interpret either value. + */ +export interface SandboxRuntimeSnapshot { + readonly schemaVersion: typeof SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION; + readonly providerId: string; + readonly providerHandle: string; + readonly lifecycleState: SandboxRuntimeLifecycleState; + readonly lifecycleGeneration: string; + readonly runtime: RuntimeProviderRuntimeReceipt; +} + +const LIFECYCLE_STATES = new Set(["running", "paused", "stopped"]); +const MAX_PROVIDER_HANDLE_BYTES = 4096; +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; + +function isPlainRecord(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function validProviderHandle(value: unknown): value is string { + return ( + typeof value === "string" && + value.trim() !== "" && + Buffer.byteLength(value, "utf8") <= MAX_PROVIDER_HANDLE_BYTES && + !CONTROL_CHARACTERS.test(value) + ); +} + +/** + * Validate and deeply clone an untrusted persisted runtime snapshot. + * Unknown keys are deliberately dropped, while the nested runtime receipt is + * normalized by the sole runtime-provider receipt boundary. + */ +export function cloneSandboxRuntimeSnapshot(value: unknown): SandboxRuntimeSnapshot | undefined { + if ( + !isPlainRecord(value) || + value.schemaVersion !== SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION || + typeof value.providerId !== "string" || + !validProviderHandle(value.providerHandle) || + typeof value.lifecycleState !== "string" || + !LIFECYCLE_STATES.has(value.lifecycleState as SandboxRuntimeLifecycleState) || + !validProviderHandle(value.lifecycleGeneration) + ) { + return undefined; + } + const runtime = normalizeRuntimeProviderRuntimeReceipt(value.runtime); + if (!runtime || runtime.providerId !== value.providerId) return undefined; + return { + schemaVersion: SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, + providerId: value.providerId, + providerHandle: value.providerHandle, + lifecycleState: value.lifecycleState as SandboxRuntimeLifecycleState, + lifecycleGeneration: value.lifecycleGeneration, + runtime, + }; +} diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 72e3ef9028b..6425384573b 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -13,7 +13,9 @@ import { createHash } from "node:crypto"; import { chmodSync, closeSync, + constants, existsSync, + fstatSync, lstatSync, mkdirSync, mkdtempSync, @@ -21,6 +23,7 @@ import { readdirSync, readFileSync, readlinkSync, + readSync, renameSync, rmSync, statSync, @@ -28,6 +31,7 @@ import { } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; import { spawnSync } from "child_process"; import { captureSandboxSshConfigCommand } from "../adapters/openshell/client.js"; @@ -56,6 +60,12 @@ import { parseOpenClawImagePluginInstalls, planOpenClawPluginRestore, } from "./openclaw-plugin-restore.js"; +import { + cloneSandboxRuntimeSnapshot, + type SandboxRuntimeSnapshot, +} from "./registry/runtime-snapshot.js"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "./registry/types.js"; +import { cloneSandboxWorkloadReceipt } from "./registry/workload.js"; import type { CustomPolicyEntry } from "./registry.js"; import * as registry from "./registry.js"; import { isSshTransportFailure } from "./ssh-transport.js"; @@ -69,6 +79,8 @@ const REBUILD_BACKUPS_DIR = path.join(nemoclawStateRoot(HOME_DIR, GATEWAY_PORT), const MANIFEST_VERSION = 1; export const OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR = "custom-image OpenClaw plugin provenance is missing or invalid"; +export const MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR = + "managed snapshot restore requires exact content and runtime authority"; function parseJson(text: string): T { return JSON.parse(text); @@ -109,6 +121,16 @@ export interface RebuildManifest { * zero-custom snapshot); absent only on legacy manifests. */ customPolicies?: CustomPolicyEntry[]; + /** + * Provider-neutral runtime and acceleration state captured before the + * filesystem copy. Required when `workload` is a managed-image receipt. + */ + runtimeSnapshot?: SandboxRuntimeSnapshot; + /** + * Exact immutable managed workload/profile authority associated with this + * snapshot. Older and explicit Dockerfile snapshots omit this field. + */ + workload?: SandboxWorkloadReceipt; instances?: InstanceBackup[]; // Optional user-provided label for `snapshot restore `. name?: string; @@ -121,6 +143,14 @@ export type SnapshotEntry = RebuildManifest & { snapshotVersion: number }; export interface BackupOptions { name?: string | null; + runtimeSnapshot?: SandboxRuntimeSnapshot; + workload?: SandboxWorkloadReceipt; + /** + * Internal publication fence for provider-backed backups. The callback + * runs after data capture and sanitization but before the manifest becomes + * visible to restore and rebuild flows. + */ + validateBeforePublish?: () => void; } export interface InstanceBackup { @@ -172,7 +202,24 @@ export interface RestoreResult { error?: string; } -export interface RecreatedSandboxRestoreOptions { +export interface SnapshotRestoreAuthority { + readonly schemaVersion: 1; + readonly backupPath: string; + readonly contentSha256: string; +} + +export interface SnapshotRestoreOptions { + /** + * Content identity captured from the selected manifest and every backup + * payload. The state layer revalidates it after local staging and before + * the first remote filesystem mutation. + */ + readonly authority?: SnapshotRestoreAuthority; + /** Internal provider fence invoked at the same last-safe mutation edge. */ + readonly validateBeforeMutation?: () => void; +} + +export interface RecreatedSandboxRestoreOptions extends SnapshotRestoreOptions { /** Agent in the newly created target image, not the backup manifest agent. */ targetAgentType: string; /** Explicit capability for custom images whose config must be restored wholesale. */ @@ -186,6 +233,8 @@ interface InternalRestoreOptions { allowCustomImageWholeStateFileRestore?: true; discoverFreshOpenClawImagePluginInstalls?: true; freshOpenClawImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; + authority?: SnapshotRestoreAuthority; + validateBeforeMutation?: () => void; } export interface TarValidationResult { @@ -265,6 +314,12 @@ export function hasAuthoritativeOpenClawImagePluginProvenance(value: { function isRebuildManifest(value: unknown): value is RebuildManifest { if (!isObjectRecord(value) || !isStateDirArray(value.stateDirs)) return false; const dir = typeof value.dir === "string" ? value.dir : value.writableDir; + const runtimeSnapshot = + value.runtimeSnapshot === undefined + ? undefined + : cloneSandboxRuntimeSnapshot(value.runtimeSnapshot); + const workload = + value.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(value.workload as never); return ( typeof value.version === "number" && typeof value.sandboxName === "string" && @@ -290,6 +345,9 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { typeof value.blueprintDigest === "string") && (value.policyPresets === undefined || isStringArray(value.policyPresets)) && (value.customPolicies === undefined || isCustomPolicyEntryArray(value.customPolicies)) && + (value.runtimeSnapshot === undefined || runtimeSnapshot !== undefined) && + (value.workload === undefined || workload !== undefined) && + (workload?.kind !== "managed-image" || runtimeSnapshot !== undefined) && (value.instances === undefined || (Array.isArray(value.instances) && value.instances.every((entry) => isInstanceBackup(entry)))) && @@ -875,6 +933,89 @@ export { buildStateFileRestoreCommand } from "./state-file-restore.js"; // module. Prefer importing directly from ./ssh-transport in new code. export { isSshTransportFailure }; +function normalizeSnapshotBackupAuthority(options: BackupOptions): { + readonly runtimeSnapshot?: SandboxRuntimeSnapshot; + readonly workload?: SandboxWorkloadReceipt; + readonly error?: string; +} { + const runtimeSnapshot = + options.runtimeSnapshot === undefined + ? undefined + : cloneSandboxRuntimeSnapshot(options.runtimeSnapshot); + const workload = + options.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(options.workload); + if (options.runtimeSnapshot !== undefined && runtimeSnapshot === undefined) { + return { error: "snapshot runtime state is invalid or cannot be represented" }; + } + if (options.workload !== undefined && workload === undefined) { + return { error: "snapshot workload authority is invalid" }; + } + if (workload?.kind === "managed-image" && runtimeSnapshot === undefined) { + return { error: "managed snapshot is missing provider runtime state" }; + } + return { + ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), + ...(workload === undefined ? {} : { workload }), + }; +} + +function validateSnapshotPublication( + backupPath: string, + validateBeforePublish: BackupOptions["validateBeforePublish"], +): string | null { + if (!validateBeforePublish) return null; + try { + validateBeforePublish(); + return null; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + try { + rmSync(backupPath, { recursive: true, force: true }); + return `Snapshot authority changed during backup: ${detail}`; + } catch (cleanupError) { + const cleanupDetail = + cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + return ( + `Snapshot authority changed during backup: ${detail}. ` + + `The unpublished backup at '${backupPath}' could not be removed: ${cleanupDetail}` + ); + } + } +} + +function resolveOpenClawBackupMetadata( + agentName: string, + sandbox: SandboxEntry | null, + configDir: string, +): { + readonly reconcileImagePluginProvenance: boolean; + readonly pluginInstalls?: OpenClawImagePluginInstall[]; + readonly error?: string; +} { + const reconcileImagePluginProvenance = + agentName === "openclaw" && Boolean(sandbox?.fromDockerfile); + if ( + agentName !== "openclaw" || + (!reconcileImagePluginProvenance && sandbox?.openclawImagePluginInstalls === undefined) + ) { + return { reconcileImagePluginProvenance }; + } + const provenance = parseOpenClawImagePluginInstalls( + sandbox?.openclawImagePluginInstalls, + configDir, + ); + if (!provenance.ok) { + return { + reconcileImagePluginProvenance, + error: "registered OpenClaw image plugin provenance is missing or invalid", + }; + } + return { + reconcileImagePluginProvenance, + pluginInstalls: cloneOpenClawImagePluginInstalls(provenance.pluginInstalls), + }; +} + export function backupSandboxState(sandboxName: string, options: BackupOptions = {}): BackupResult { const sb = registry.getSandbox(sandboxName); const agentName = sb?.agent || "openclaw"; @@ -890,25 +1031,28 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = `backupSandboxState: agent=${agentName}, dir=${dir}, stateDirs=[${stateDirs.join(",")}], stateFiles=[${stateFiles.map((f) => f.path).join(",")}]`, ); - const reconcileOpenClawImagePluginProvenance = - agentName === "openclaw" && Boolean(sb?.fromDockerfile); - let openclawImagePluginInstalls: OpenClawImagePluginInstall[] | undefined; - if ( - agentName === "openclaw" && - (reconcileOpenClawImagePluginProvenance || sb?.openclawImagePluginInstalls !== undefined) - ) { - const provenance = parseOpenClawImagePluginInstalls(sb?.openclawImagePluginInstalls, dir); - if (!provenance.ok) { - return { - success: false, - backedUpDirs: [], - failedDirs: [], - backedUpFiles: [], - failedFiles: [], - error: "registered OpenClaw image plugin provenance is missing or invalid", - }; - } - openclawImagePluginInstalls = cloneOpenClawImagePluginInstalls(provenance.pluginInstalls); + const snapshotAuthority = normalizeSnapshotBackupAuthority(options); + if (snapshotAuthority.error) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: snapshotAuthority.error, + }; + } + + const openClawMetadata = resolveOpenClawBackupMetadata(agentName, sb, dir); + if (openClawMetadata.error) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: openClawMetadata.error, + }; } // Validate user-supplied name and check for conflicts BEFORE creating any @@ -945,6 +1089,16 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const backupPath = path.join(REBUILD_BACKUPS_DIR, sandboxName, timestamp); + if (existsSync(backupPath)) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: `Snapshot path '${backupPath}' already exists; retry the backup.`, + }; + } // SECURITY: Verify backup destination ancestors are not symlinks. // Without this check, an attacker who plants ~/.nemoclaw/rebuild-backups @@ -975,8 +1129,10 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = agentType: agentName, agentVersion: sb?.agentVersion || null, expectedVersion: agent.expectedVersion, - ...(openclawImagePluginInstalls !== undefined ? { openclawImagePluginInstalls } : {}), - ...(reconcileOpenClawImagePluginProvenance + ...(openClawMetadata.pluginInstalls !== undefined + ? { openclawImagePluginInstalls: openClawMetadata.pluginInstalls } + : {}), + ...(openClawMetadata.reconcileImagePluginProvenance ? { reconcileOpenClawImagePluginProvenance: true } : {}), stateDirs, @@ -987,6 +1143,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = blueprintDigest: computeBlueprintDigest(), policyPresets, customPolicies, + ...snapshotAuthority, ...(providedName !== null ? { name: providedName } : {}), }; @@ -999,6 +1156,17 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = if (stateDirs.length === 0 && stateFiles.length === 0) { _log("WARNING: Agent manifest declares no state_dirs or state_files — nothing to back up"); + const publicationError = validateSnapshotPublication(backupPath, options.validateBeforePublish); + if (publicationError) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: publicationError, + }; + } writeManifest(backupPath, manifest); return { success: true, manifest, backedUpDirs, failedDirs, backedUpFiles, failedFiles }; } @@ -1324,6 +1492,17 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = manifest.stateDirs.includes(failedDir), ); + const publicationError = validateSnapshotPublication(backupPath, options.validateBeforePublish); + if (publicationError) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: publicationError, + }; + } writeManifest(backupPath, manifest); manifest.backupPath = backupPath; @@ -1341,10 +1520,152 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = // ── Restore ──────────────────────────────────────────────────────── +function snapshotManifestAuthority(manifest: RebuildManifest): RebuildManifest { + const normalized = { + ...manifest, + backupPath: path.resolve(manifest.backupPath), + } as RebuildManifest & { snapshotVersion?: unknown }; + // snapshotVersion is a list-time cursor, not persisted restore authority. + // Every other normalized manifest field can affect restore behavior and + // therefore remains bound to the operator's selected snapshot. + delete normalized.snapshotVersion; + return normalized; +} + +function hashSnapshotTree(backupPath: string): string { + if (typeof constants.O_NOFOLLOW !== "number") { + throw new Error("snapshot hashing requires O_NOFOLLOW support"); + } + const openFlags = + constants.O_RDONLY | + constants.O_NOFOLLOW | + (typeof constants.O_NONBLOCK === "number" ? constants.O_NONBLOCK : 0); + const hash = createHash("sha256"); + const visit = (directory: string, relativeDirectory: string): void => { + const entries = readdirSync(directory, { withFileTypes: true }).sort((left, right) => + left.name === right.name ? 0 : left.name < right.name ? -1 : 1, + ); + for (const entry of entries) { + const fullPath = path.join(directory, entry.name); + const relativePath = path.posix.join( + relativeDirectory.split(path.sep).join(path.posix.sep), + entry.name, + ); + if (entry.isDirectory()) { + hash.update(JSON.stringify(["directory", relativePath]), "utf8"); + visit(fullPath, relativePath); + continue; + } + if (entry.isSymbolicLink()) { + hash.update(JSON.stringify(["symlink", relativePath, readlinkSync(fullPath)]), "utf8"); + continue; + } + if (!entry.isFile()) { + throw new Error(`snapshot contains unsupported entry '${relativePath}'`); + } + const descriptor = openSync(fullPath, openFlags); + try { + const opened = fstatSync(descriptor); + if (!opened.isFile()) { + throw new Error(`snapshot entry '${relativePath}' changed while it was opened`); + } + hash.update(JSON.stringify(["file", relativePath, opened.size]), "utf8"); + const buffer = Buffer.allocUnsafe(64 * 1024); + for (;;) { + const bytesRead = readSync(descriptor, buffer, 0, buffer.byteLength, null); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + } + const after = fstatSync(descriptor); + const pathAfter = lstatSync(fullPath); + if ( + after.size !== opened.size || + after.mtimeMs !== opened.mtimeMs || + pathAfter.isSymbolicLink() || + !pathAfter.isFile() || + pathAfter.dev !== opened.dev || + pathAfter.ino !== opened.ino || + pathAfter.size !== opened.size || + pathAfter.mtimeMs !== opened.mtimeMs + ) { + throw new Error(`snapshot entry '${relativePath}' changed while it was read`); + } + } finally { + closeSync(descriptor); + } + } + }; + visit(backupPath, ""); + return hash.digest("hex"); +} + +/** + * Bind a selected, validated manifest to all bytes that restore can consume. + * Returns null for an unsafe path, malformed manifest, selection drift, or a + * payload that changes while it is being hashed. + */ +export function captureSnapshotRestoreAuthority( + backupPath: string, + expectedManifest?: RebuildManifest, +): SnapshotRestoreAuthority | null { + try { + const root = path.resolve(REBUILD_BACKUPS_DIR); + const candidate = path.resolve(backupPath); + if (candidate === root || !isWithinRoot(candidate, root)) return null; + rejectSymlinksOnPath(candidate); + if (!lstatSync(path.join(candidate, "rebuild-manifest.json")).isFile()) return null; + const manifest = readManifest(candidate); + if (!manifest || path.resolve(manifest.backupPath) !== candidate) return null; + if ( + expectedManifest && + !isDeepStrictEqual( + snapshotManifestAuthority(manifest), + snapshotManifestAuthority(expectedManifest), + ) + ) { + return null; + } + return { + schemaVersion: 1, + backupPath: candidate, + contentSha256: hashSnapshotTree(candidate), + }; + } catch { + return null; + } +} + +function validateSnapshotRestoreMutation( + backupPath: string, + options: Pick, +): string | null { + if (options.authority) { + const current = captureSnapshotRestoreAuthority(backupPath); + if ( + !current || + current.backupPath !== options.authority.backupPath || + current.contentSha256 !== options.authority.contentSha256 + ) { + return "Selected snapshot content changed before filesystem mutation"; + } + } + try { + options.validateBeforeMutation?.(); + return null; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return `Runtime authority changed before filesystem mutation: ${detail}`; + } +} + /** * Restore state directories into a sandbox from a prior backup. */ -export function restoreSandboxState(sandboxName: string, backupPath: string): RestoreResult { +export function restoreSandboxState( + sandboxName: string, + backupPath: string, + options: SnapshotRestoreOptions = {}, +): RestoreResult { const target = registry.getSandbox(sandboxName); if (!target) { return { @@ -1359,6 +1680,10 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re return restoreSandboxStateInternal(sandboxName, backupPath, { targetAgentType: String(target.agent || "openclaw"), ...(target.fromDockerfile ? { allowCustomImageWholeStateFileRestore: true } : {}), + ...(options.authority ? { authority: options.authority } : {}), + ...(options.validateBeforeMutation + ? { validateBeforeMutation: options.validateBeforeMutation } + : {}), }); } @@ -1377,6 +1702,10 @@ export function restoreRecreatedSandboxState( ? { discoverFreshOpenClawImagePluginInstalls: true } : {}), freshOpenClawImagePluginInstalls: options.freshOpenClawImagePluginInstalls, + ...(options.authority ? { authority: options.authority } : {}), + ...(options.validateBeforeMutation + ? { validateBeforeMutation: options.validateBeforeMutation } + : {}), }); } @@ -1443,6 +1772,12 @@ function restoreSandboxStateInternal( error, }; }; + if ( + manifest.workload?.kind === "managed-image" && + (!options.authority || !options.validateBeforeMutation) + ) { + return failRestoreContract(MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR); + } if (options.targetAgentType !== manifest.agentType) { return failRestoreContract( `Backup agent '${manifest.agentType}' does not match target agent '${options.targetAgentType}'`, @@ -1551,6 +1886,10 @@ function restoreSandboxStateInternal( } if (cleanupStateDirs.length === 0 && localFiles.length === 0) { + const mutationAuthorityError = validateSnapshotRestoreMutation(backupPath, options); + if (mutationAuthorityError) { + return failRestoreContract(mutationAuthorityError); + } _log("No dirs or files to restore"); return { success: true, restoredDirs, failedDirs, restoredFiles, failedFiles }; } @@ -1642,6 +1981,11 @@ function restoreSandboxStateInternal( restoreTar = tarResult.stdout; } + const mutationAuthorityError = validateSnapshotRestoreMutation(backupPath, options); + if (mutationAuthorityError) { + return failRestoreContract(mutationAuthorityError); + } + // Remove existing state dirs before extracting so stale files from later // snapshots don't persist after restoring an earlier one. OpenClaw's // image-managed extensions are preserved from the freshly built image and @@ -1845,6 +2189,12 @@ function readManifest(backupPath: string): RebuildManifest | null { const manifest = parsed as RebuildManifest & { dir?: string; writableDir?: string }; const dir = manifest.dir ?? manifest.writableDir; if (!dir) return null; + const runtimeSnapshot = + manifest.runtimeSnapshot === undefined + ? undefined + : cloneSandboxRuntimeSnapshot(manifest.runtimeSnapshot); + const workload = + manifest.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(manifest.workload); return { ...manifest, dir, @@ -1852,6 +2202,8 @@ function readManifest(backupPath: string): RebuildManifest | null { // restore contract can reject them instead of silently de-duplicating. stateFiles: normalizeStateFileSpecsPreservingDuplicates(manifest.stateFiles ?? []), blueprintDigest: manifest.blueprintDigest ?? null, + ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), + ...(workload === undefined ? {} : { workload }), }; } catch { return null; diff --git a/test/snapshot-managed-restore-authority.test.ts b/test/snapshot-managed-restore-authority.test.ts new file mode 100644 index 00000000000..c195699751d --- /dev/null +++ b/test/snapshot-managed-restore-authority.test.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { encodeManagedStartupProfile } from "../src/lib/onboard/managed-startup/profile"; + +const ORIGINAL_HOME = process.env.HOME; +const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-snapshot-authority-")); +process.env.HOME = TMP_HOME; +const sandboxState = await import("../src/lib/state/sandbox.js"); +const BACKUPS_ROOT = path.join(TMP_HOME, ".nemoclaw", "rebuild-backups"); + +afterAll(() => { + void (ORIGINAL_HOME === undefined + ? Reflect.deleteProperty(process.env, "HOME") + : Reflect.set(process.env, "HOME", ORIGINAL_HOME)); + fs.rmSync(TMP_HOME, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(BACKUPS_ROOT, { recursive: true, force: true }); +}); + +function managedAuthority() { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); + return { + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.97", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }, + runtimeSnapshot: { + schemaVersion: 1, + providerId: "docker", + providerHandle: "opaque-provider-handle", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "opaque-container-id" }, + acceleration: { kind: "none" }, + }, + }, + } as const; +} + +function writeBackup(overrides: Record = {}) { + const timestamp = "2026-04-21T14-00-00-000Z"; + const backupPath = path.join(BACKUPS_ROOT, "alpha", timestamp); + fs.mkdirSync(backupPath, { recursive: true }); + const manifest = { + version: 1, + sandboxName: "alpha", + timestamp, + agentType: "openclaw", + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox/.openclaw", + backupPath, + blueprintDigest: null, + ...overrides, + }; + fs.writeFileSync( + path.join(backupPath, "rebuild-manifest.json"), + JSON.stringify(manifest, null, 2), + ); + return manifest; +} + +function writeOpenClawRegistry(): void { + fs.mkdirSync(path.join(TMP_HOME, ".nemoclaw"), { recursive: true }); + fs.writeFileSync( + path.join(TMP_HOME, ".nemoclaw", "sandboxes.json"), + JSON.stringify({ + defaultSandbox: "alpha", + sandboxes: { + alpha: { + name: "alpha", + model: "demo", + provider: "compatible-endpoint", + gpuEnabled: false, + policies: [], + agent: "openclaw", + }, + }, + }), + ); +} + +describe("managed snapshot restore authority", () => { + it("binds every normalized restore-relevant manifest field selected by the operator", () => { + const manifest = writeBackup({ backedUpDirs: ["workspace"], stateDirs: ["workspace"] }); + const selected = sandboxState.getLatestBackup("alpha"); + expect(selected).not.toBeNull(); + + fs.writeFileSync( + path.join(manifest.backupPath, "rebuild-manifest.json"), + JSON.stringify({ ...manifest, stateDirs: ["workspace", "agents"] }, null, 2), + ); + + expect(sandboxState.captureSnapshotRestoreAuthority(manifest.backupPath, selected!)).toBeNull(); + }); + + it("requires both content and runtime fences at each raw state entry point", () => { + const manifest = writeBackup(managedAuthority()); + const contentAuthority = sandboxState.captureSnapshotRestoreAuthority(manifest.backupPath); + expect(contentAuthority).not.toBeNull(); + + for (const partialAuthority of [ + {}, + { authority: contentAuthority! }, + { validateBeforeMutation: vi.fn() }, + ]) { + expect( + sandboxState.restoreRecreatedSandboxState("alpha", manifest.backupPath, { + targetAgentType: "openclaw", + ...partialAuthority, + }), + ).toMatchObject({ + success: false, + error: sandboxState.MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR, + }); + } + + writeOpenClawRegistry(); + expect(sandboxState.restoreSandboxState("alpha", manifest.backupPath)).toMatchObject({ + success: false, + error: sandboxState.MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR, + }); + + const validateBeforeMutation = vi.fn(); + expect( + sandboxState.restoreRecreatedSandboxState("alpha", manifest.backupPath, { + targetAgentType: "openclaw", + freshOpenClawImagePluginInstalls: [], + authority: contentAuthority!, + validateBeforeMutation, + }), + ).toMatchObject({ success: true }); + expect(validateBeforeMutation).toHaveBeenCalledOnce(); + }); +}); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 89be74b5589..26f7f05ee33 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -5,12 +5,16 @@ // - validateSnapshotName accepts/rejects names // - listBackups computes virtual v versions by timestamp-ascending position // - findBackup resolves selectors (v, name, exact timestamp) + +import { createHash } from "node:crypto"; import fs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { afterAll, beforeEach, describe, expect, it } from "vitest"; +import { managedStartupE2eProfile } from "../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { encodeManagedStartupProfile } from "../src/lib/onboard/managed-startup/profile"; // Override HOME BEFORE importing sandbox-state — it reads process.env.HOME // at module-load time to compute REBUILD_BACKUPS_DIR. Captured original is @@ -66,6 +70,39 @@ function writeBackup( fs.writeFileSync(path.join(dir, "rebuild-manifest.json"), JSON.stringify(manifest, null, 2)); return manifest; } +function managedSnapshotAuthority() { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); + return { + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.97", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }, + runtimeSnapshot: { + schemaVersion: 1, + providerId: "docker", + providerHandle: "opaque-provider-handle", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "opaque-container-id" }, + acceleration: { kind: "none" }, + }, + }, + } as const; +} afterAll(() => { if (ORIGINAL_HOME === undefined) { delete process.env.HOME; @@ -189,6 +226,38 @@ describe("listBackups computes virtual versions", () => { expect(entry.customPolicies).toEqual(custom); }); + it("round-trips normalized managed workload and provider runtime authority", () => { + const authority = managedSnapshotAuthority(); + writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z", { + workload: { ...authority.workload, ignored: "not-authority" }, + runtimeSnapshot: { + ...authority.runtimeSnapshot, + containerName: "not-authority", + }, + }); + + const [entry] = sandboxState.listBackups("test-sandbox"); + + expect(entry?.workload).toEqual(authority.workload); + expect(entry?.runtimeSnapshot).toEqual(authority.runtimeSnapshot); + }); + + it("rejects a managed snapshot manifest without valid provider runtime authority", () => { + const authority = managedSnapshotAuthority(); + writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z", { + workload: authority.workload, + }); + writeBackup("test-sandbox", "2026-04-21T14-01-00-000Z", { + ...authority, + runtimeSnapshot: { + ...authority.runtimeSnapshot, + lifecycleGeneration: "", + }, + }); + + expect(sandboxState.listBackups("test-sandbox")).toEqual([]); + }); + it("preserves an empty customPolicies array so restore can distinguish zero-custom from legacy snapshots", () => { writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z", { customPolicies: [] }); const [entry] = sandboxState.listBackups("test-sandbox"); @@ -294,6 +363,32 @@ describe("listBackups computes virtual versions", () => { }); }); +describe("snapshot restore content authority", () => { + it("binds the selected manifest and payload bytes to one digest", () => { + const manifest = writeBackup("alpha", "2026-04-21T14-00-00-000Z", { + backedUpDirs: ["workspace"], + stateDirs: ["workspace"], + }); + const backupPath = String(manifest.backupPath); + fs.mkdirSync(path.join(backupPath, "workspace")); + fs.writeFileSync(path.join(backupPath, "workspace", "state.txt"), "before\n"); + const selected = sandboxState.getLatestBackup("alpha"); + expect(selected).not.toBeNull(); + + const authority = sandboxState.captureSnapshotRestoreAuthority(backupPath, selected!); + expect(authority).toMatchObject({ + schemaVersion: 1, + backupPath, + contentSha256: expect.stringMatching(/^[a-f0-9]{64}$/u), + }); + + fs.writeFileSync(path.join(backupPath, "workspace", "state.txt"), "after\n"); + expect(sandboxState.captureSnapshotRestoreAuthority(backupPath)?.contentSha256).not.toBe( + authority?.contentSha256, + ); + }); +}); + describe("findBackup", () => { it("matches v against the computed version", () => { writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z"); // v1 (oldest) @@ -537,6 +632,25 @@ process.exit(0); expect(backup.manifest?.reconcileOpenClawImagePluginProvenance).toBe(true); expect(backup.manifest?.openclawImagePluginInstalls).toEqual([]); expect(fs.readdirSync(stagingRoot)).toEqual([]); + + const rejected = sandboxState.backupSandboxState("alpha", { + validateBeforePublish: () => { + throw new Error("runtime generation changed"); + }, + }); + expect(rejected).toMatchObject({ + success: false, + error: expect.stringContaining( + "Snapshot authority changed during backup: runtime generation changed", + ), + }); + const published = fs + .readdirSync(path.join(BACKUPS_ROOT, "alpha")) + .filter((entry) => + fs.existsSync(path.join(BACKUPS_ROOT, "alpha", entry, "rebuild-manifest.json")), + ); + expect(published).toHaveLength(1); + expect(fs.readdirSync(stagingRoot)).toEqual([]); } finally { restoreEnv("NEMOCLAW_OPENSHELL_BIN", oldOpenshell); restoreEnv("TMPDIR", oldTmpdir);