feat(snapshot): preserve managed runtime authority

ci/source-architecture-budget.json

@@ -8,7 +8,7 @@
       "src/lib/adapters/docker/index.ts": 43,
       "src/lib/adapters/openshell/client.ts": 23,
       "src/lib/adapters/openshell/resolve.ts": 27,
-      "src/lib/adapters/openshell/runtime.ts": 51,
+      "src/lib/adapters/openshell/runtime.ts": 52,
       "src/lib/adapters/openshell/timeouts.ts": 37,
       "src/lib/agent/defs.ts": 32,
       "src/lib/cli/branding.ts": 84,
@@ -23,7 +23,7 @@
       "src/lib/inference/config.ts": 29,
       "src/lib/inference/web-search.ts": 21,
       "src/lib/messaging/channels/index.ts": 26,
-      "src/lib/onboard/gateway-binding.ts": 47,
+      "src/lib/onboard/gateway-binding.ts": 48,
       "src/lib/runner.ts": 89,
       "src/lib/security/redact.ts": 51,
       "src/lib/state/onboard-session.ts": 36,
@@ -43,7 +43,7 @@
       "src/lib/actions/sandbox/policy-channel.ts": 29,
       "src/lib/actions/sandbox/process-recovery.ts": 22,
       "src/lib/actions/sandbox/rebuild-pipeline.ts": 28,
-      "src/lib/actions/sandbox/snapshot.ts": 38,
+      "src/lib/actions/sandbox/snapshot.ts": 39,
       "src/lib/actions/uninstall/run-plan.ts": 25,
       "src/lib/inference/onboard-probes.ts": 21,
       "src/lib/inference/vllm.ts": 23,

src/lib/actions/maintenance.test.ts

@@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 const mocks = vi.hoisted(() => ({
   listSandboxes: vi.fn(),
+  getSandbox: vi.fn(),
   backupSandboxState: vi.fn(),
   captureSandboxListWithGatewayPreflightOrExit: vi.fn(),
   parseReadySandboxNames: vi.fn(),
@@ -18,17 +19,25 @@ const mocks = vi.hoisted(() => ({
   isSandboxContainerDefinitivelyAbsent: vi.fn(),
   openBackupShieldsWindow: vi.fn(),
   relockBackupShieldsWindow: vi.fn(),
+  withSandboxMutationLock: vi.fn(),
 }));
 
 vi.mock("../state/registry", () => ({
   isRouteOnlySandboxReservation: (entry: { pendingRouteReservation?: true; createdAt?: string }) =>
     entry.pendingRouteReservation === true && entry.createdAt === undefined,
   listSandboxes: mocks.listSandboxes,
+  getSandbox: mocks.getSandbox,
 }));
 vi.mock("../state/sandbox", () => ({
   backupSandboxState: mocks.backupSandboxState,
   BackupResult: {},
 }));
+vi.mock("../state/mcp-lifecycle-lock", () => ({
+  withSandboxMutationLock: mocks.withSandboxMutationLock,
+}));
+vi.mock("./sandbox/snapshot/backup-authority", () => ({
+  backupSandboxStateWithManagedAuthority: (name: string) => mocks.backupSandboxState(name),
+}));
 vi.mock("../openshell-sandbox-list", () => ({
   captureSandboxListWithGatewayPreflightOrExit: mocks.captureSandboxListWithGatewayPreflightOrExit,
 }));
@@ -100,6 +109,7 @@ describe("backupAll", () => {
       wasLocked: false,
     }));
     mocks.relockBackupShieldsWindow.mockReturnValue(true);
+    mocks.withSandboxMutationLock.mockImplementation((_name, callback) => callback());
   });
 
   afterEach(() => {
@@ -217,6 +227,43 @@ describe("backupAll", () => {
     logSpy.mockRestore();
   });
 
+  it("counts a mutation-lock acquisition failure and continues with later sandboxes", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "alpha" }, { name: "beta" }],
+      defaultSandbox: "alpha",
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set(["alpha", "beta"]));
+    mocks.withSandboxMutationLock
+      .mockRejectedValueOnce(new Error("Timed out waiting for the sandbox mutation lock"))
+      .mockImplementationOnce((_name, callback) => callback());
+    mocks.backupSandboxState.mockReturnValue({
+      success: true,
+      backedUpDirs: ["workspace"],
+      failedDirs: [],
+      backedUpFiles: [],
+      failedFiles: [],
+      manifest: { backupPath: "/backups/beta/timestamp" },
+    });
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
+    vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    expect(mocks.withSandboxMutationLock.mock.calls.map(([name]) => name)).toEqual([
+      "alpha",
+      "beta",
+    ]);
+    expect(mocks.backupSandboxState).toHaveBeenCalledOnce();
+    expect(mocks.backupSandboxState).toHaveBeenCalledWith("beta");
+    expect(logSpy.mock.calls.flat().join("\n")).toContain("1 backed up, 1 failed, 0 skipped");
+    expect(errorSpy.mock.calls.flat().join("\n")).toContain(
+      "alpha: backup failed (mutation lock: Timed out waiting for the sandbox mutation lock)",
+    );
+  });
+
   it("does not back up when gateway preflight exits", async () => {
     mocks.listSandboxes.mockReturnValue({
       sandboxes: [{ name: "sb-good" }],

src/lib/actions/maintenance.ts

@@ -22,6 +22,7 @@ import { SANDBOX_IMAGE_REPOS } from "../domain/sandbox/image-tag";
 import { resolveGatewayName, resolveSandboxGatewayName } from "../onboard/gateway-binding";
 import { captureSandboxListWithGatewayPreflightOrExit } from "../openshell-sandbox-list";
 import { parseLiveSandboxNames, parseReadySandboxNames } from "../runtime-recovery";
+import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock";
 import * as registry from "../state/registry";
 import * as sandboxState from "../state/sandbox";
 import { nemoclawStateRoot, resolveHome } from "../state/state-root";
@@ -30,6 +31,7 @@ import {
   openBackupShieldsWindow,
   relockBackupShieldsWindow,
 } from "./sandbox/backup-shields-window";
+import * as snapshotBackup from "./sandbox/snapshot/backup-authority";
 import {
   backupStartedSandboxState,
   isSandboxContainerDefinitivelyAbsent,
@@ -198,7 +200,7 @@ export async function backupAll(): Promise<void> {
   let unreachableRunning = 0;
   let notRunningSkipped = 0;
   const strandedOrphans: string[] = [];
-  for (const sb of sandboxes) {
+  const backupRegisteredSandbox = async (sb: (typeof sandboxes)[number]): Promise<void> => {
     // A registered docker-driver sandbox whose container is merely stopped is
     // backupable: start it for the duration of the backup and return it to
     // its stopped state after (#6500). Anything else that is not Ready keeps
@@ -211,12 +213,12 @@ export async function backupAll(): Promise<void> {
           // Tracked separately from `skipped` so the strict gate stays
           // untripped: there is nothing to back up and nothing to start.
           strandedOrphans.push(sb.name);
-          continue;
+          return;
         }
         console.log(`  ${D}${notRunningBackupSkipMessage(sb.name)}${R}`);
         skipped++;
         notRunningSkipped++;
-        continue;
+        return;
       }
       console.log(`  Starting stopped sandbox '${sb.name}' to back it up...`);
     }
@@ -229,7 +231,13 @@ export async function backupAll(): Promise<void> {
       const attempt = await backupSandboxWithinShieldsWindow(sb.name, () =>
         startedForBackup
           ? backupStartedSandboxState(sb.name)
-          : sandboxState.backupSandboxState(sb.name),
+          : snapshotBackup.backupSandboxStateWithManagedAuthority(
+              sb.name,
+              {},
+              {
+                getSandbox: registry.getSandbox,
+              },
+            ),
       );
       result = attempt.result;
       orphanManifestMessage = attempt.orphanManifestMessage;
@@ -248,17 +256,17 @@ export async function backupAll(): Promise<void> {
     }
     if (!returnedToStopped) {
       failed++;
-      continue;
+      return;
     }
     if (!shieldsWindowOpened) {
       console.error(`  ${RD}✗${R} ${sb.name}: backup failed (could not safely unlock shields)`);
       failed++;
-      continue;
+      return;
     }
     if (orphanManifestMessage) {
       console.log(`  ${YW}⚠${R} Skipped '${sb.name}' (orphan manifest): ${orphanManifestMessage}`);
       skipped++;
-      continue;
+      return;
     }
     if (!result) throw new Error(`Backup for '${sb.name}' completed without a result`);
     if (result.success) {
@@ -273,7 +281,7 @@ export async function backupAll(): Promise<void> {
             `  ${YW}⚠${R} Skipped '${sb.name}' (running but SSH-unreachable; NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1 set). Any uncommitted state since the last successful backup will be lost.`,
           );
           skipped++;
-          continue;
+          return;
         }
         unreachableRunning++;
       }
@@ -284,6 +292,24 @@ export async function backupAll(): Promise<void> {
       console.error(`  ${RD}✗${R} ${sb.name}: backup failed (${failedItems})`);
       failed++;
     }
+  };
+  for (const sb of sandboxes) {
+    let enteredMutationLock = false;
+    try {
+      await withSandboxMutationLock(sb.name, () => {
+        enteredMutationLock = true;
+        return backupRegisteredSandbox(sb);
+      });
+    } catch (error) {
+      // Callback failures retain the existing fail-fast behavior. A lock that
+      // could not be acquired is instead one failed sandbox attempt so the
+      // remaining backups, orphan confirmation, summary, and strict gate all
+      // still run.
+      if (enteredMutationLock) throw error;
+      const detail = error instanceof Error ? error.message : String(error);
+      console.error(`  ${RD}✗${R} ${sb.name}: backup failed (mutation lock: ${detail})`);
+      failed++;
+    }
   }
   // The classification above is only as fresh as the pre-loop listing, and
   // the backup loop can run for minutes. Confirm with a second pinned listing

src/lib/actions/sandbox/rebuild-flow-helpers.test.ts

@@ -9,6 +9,7 @@ import * as gatewayRuntime from "../../gateway-runtime-action";
 import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image";
 import * as sandboxState from "../../state/sandbox";
 import * as userManagedFilesProbe from "../../state/user-managed-files-probe";
+import * as snapshotBackup from "./snapshot/backup-authority";
 import {
   backupSandboxStateForRebuild,
   disposeRebuildAgentBaseImagePreflight,
@@ -549,7 +550,7 @@ describe("backupSandboxStateForRebuild with --force", () => {
     errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
     vi.spyOn(console, "log").mockImplementation(() => undefined);
 
-    backupSpy = vi.spyOn(sandboxState, "backupSandboxState");
+    backupSpy = vi.spyOn(snapshotBackup, "backupSandboxStateWithManagedAuthority");
   });
 
   afterEach(() => {
@@ -843,7 +844,9 @@ describe("warnUnpreservedUserManagedFiles", () => {
     logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
     errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
 
-    backupSpy = vi.spyOn(sandboxState, "backupSandboxState").mockReturnValue(makeBackupResult());
+    backupSpy = vi
+      .spyOn(snapshotBackup, "backupSandboxStateWithManagedAuthority")
+      .mockReturnValue(makeBackupResult());
     probeSpy = vi.spyOn(userManagedFilesProbe, "probeUserManagedFiles").mockReturnValue({
       declared: [],
       existing: [],

src/lib/actions/sandbox/rebuild-flow-helpers.ts

@@ -49,6 +49,7 @@ import {
   printWrongGatewayActiveGuidance,
 } from "./gateway-state";
 import { openRebuildShieldsWindow, type RebuildShieldsWindow } from "./rebuild-shields";
+import * as snapshotBackup from "./snapshot/backup-authority";
 
 export type RebuildSandboxEntry = SandboxEntry & { agents?: unknown[] };
 
@@ -446,7 +447,13 @@ export function backupSandboxStateForRebuild(
 
   console.log("  Backing up sandbox state...");
   log(`Agent type: ${sb.agent || "openclaw"}, stateDirs from manifest`);
-  const backup = sandboxState.backupSandboxState(sandboxName);
+  const backup = snapshotBackup.backupSandboxStateWithManagedAuthority(
+    sandboxName,
+    {},
+    {
+      getSandbox: (name) => loadRegistry().sandboxes[name] ?? null,
+    },
+  );
   log(
     `Backup result: success=${backup.success}, backed=${backup.backedUpDirs.join(",")}; files=${backup.backedUpFiles.join(",")}, failed=${backup.failedDirs.join(",")}; failedFiles=${backup.failedFiles.join(",")}`,
   );

src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts

@@ -3,8 +3,8 @@
 
 import { afterEach, describe, expect, it, vi } from "vitest";
 
-import * as sandboxState from "../../state/sandbox";
 import { runRebuildRestorePhase } from "./rebuild-restore-phase";
+import * as snapshotRestore from "./snapshot/restore-authority";
 
 afterEach(() => {
   vi.restoreAllMocks();
@@ -14,7 +14,7 @@ describe("rebuild restore target forwarding", () => {
   it("forwards the recreated target identity and explicit custom-image capability", () => {
     vi.spyOn(console, "log").mockImplementation(() => undefined);
     const restoreRecreatedSandboxState = vi
-      .spyOn(sandboxState, "restoreRecreatedSandboxState")
+      .spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority")
       .mockReturnValue({
         success: true,
         restoredDirs: [],
@@ -34,9 +34,14 @@ describe("rebuild restore target forwarding", () => {
       log: vi.fn(),
     });
 
-    expect(restoreRecreatedSandboxState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backup", {
-      targetAgentType: "langchain-deepagents-code",
-      allowCustomImageWholeStateFileRestore: true,
-    });
+    expect(restoreRecreatedSandboxState).toHaveBeenCalledWith(
+      "alpha",
+      expect.objectContaining({ backupPath: "/tmp/rebuild-backup" }),
+      {
+        targetAgentType: "langchain-deepagents-code",
+        allowCustomImageWholeStateFileRestore: true,
+      },
+      { getSandbox: expect.any(Function) },
+    );
   });
 });

src/lib/actions/sandbox/rebuild-restore-phase.test.ts

@@ -11,6 +11,7 @@ import {
   resolveRestoredPolicyRegistryState,
 } from "./rebuild-post-restore-phase";
 import { runRebuildRestorePhase } from "./rebuild-restore-phase";
+import * as snapshotRestore from "./snapshot/restore-authority";
 
 const BUILTIN_OBSERVABILITY_CONTENT =
   "network_policies:\n  observability-otlp-local:\n    name: observability-otlp-local\n";
@@ -44,7 +45,7 @@ describe("rebuild policy restore fidelity", () => {
     vi.spyOn(console, "log").mockImplementation(() => undefined);
     const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined);
     const log = vi.fn();
-    vi.spyOn(sandboxState, "restoreRecreatedSandboxState").mockReturnValue({
+    vi.spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority").mockReturnValue({
       success: false,
       restoredDirs: [],
       restoredFiles: [],
@@ -76,7 +77,7 @@ describe("rebuild policy restore fidelity", () => {
     vi.spyOn(console, "error").mockImplementation(() => undefined);
     const parsePresetPolicyKeys = vi.spyOn(policies, "parsePresetPolicyKeys");
     const restoreRecreatedSandboxState = vi
-      .spyOn(sandboxState, "restoreRecreatedSandboxState")
+      .spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority")
       .mockReturnValue({
         success: true,
         restoredDirs: [],
@@ -104,9 +105,14 @@ describe("rebuild policy restore fidelity", () => {
       log: vi.fn(),
     });
 
-    expect(restoreRecreatedSandboxState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backup", {
-      targetAgentType: "openclaw",
-    });
+    expect(restoreRecreatedSandboxState).toHaveBeenCalledWith(
+      "alpha",
+      expect.objectContaining({ backupPath: "/tmp/rebuild-backup" }),
+      {
+        targetAgentType: "openclaw",
+      },
+      { getSandbox: expect.any(Function) },
+    );
     expect(applyPreset).toHaveBeenCalledOnce();
     expect(applyPreset).toHaveBeenCalledWith("alpha", "npm");
     for (const entry of customPolicies) {

src/lib/actions/sandbox/rebuild-restore-phase.ts

@@ -8,11 +8,13 @@ import {
   OBSERVABILITY_POLICY_BINDING,
 } from "../../onboard/observability-policy-presets";
 import * as policies from "../../policy";
+import { load as loadRegistry } from "../../state/registry/persistence";
 import * as sandboxState from "../../state/sandbox";
 import { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts";
 import type { RebuildBackupManifest } from "./rebuild-backup-phase";
 import type { RebuildLog } from "./rebuild-credential-preflight";
 import type { RebuildSandboxEntry } from "./rebuild-flow-helpers";
+import * as snapshotRestore from "./snapshot/restore-authority";
 
 export interface RebuildRestorePhaseInput {
   sandboxName: string;
@@ -191,13 +193,16 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild
     console.log("");
     console.log("  Restoring workspace state...");
     log(`Restoring from: ${backupManifest.backupPath} into sandbox: ${sandboxName}`);
-    const restore = sandboxState.restoreRecreatedSandboxState(
+    const restore = snapshotRestore.restoreRecreatedSandboxStateWithManagedAuthority(
       sandboxName,
-      backupManifest.backupPath,
+      backupManifest,
       {
         targetAgentType,
         ...(targetImageIsCustom ? { allowCustomImageWholeStateFileRestore: true } : {}),
       },
+      {
+        getSandbox: (name) => loadRegistry().sandboxes[name] ?? null,
+      },
     );
     log(
       `Restore result: success=${restore.success}, restored=${restore.restoredDirs.join(",")}; files=${restore.restoredFiles.join(",")}, failed=${restore.failedDirs.join(",")}; failedFiles=${restore.failedFiles.join(",")}${restore.error ? `; error=${restore.error}` : ""}`,