fix(state): restore preserved backups into replacement sandboxes

docs/manage-sandboxes/backup-restore.mdx

@@ -43,6 +43,7 @@ Hermes snapshots include `SOUL.md`, the Web Dashboard profile under `.hermes/das
 The default-profile snapshot also includes cron execution history in `.hermes/runtime/cron-executions.db` and Discord replay state in `.hermes/gateway/discord_message_recovery.db`.
 NemoClaw captures cron job definitions from `.hermes/cron` as directory state.
 NemoClaw uses SQLite's online backup API and restores these databases through SQLite instead of copying live raw database files.
+After it replaces a database, NemoClaw opens a write transaction against the result and fails the restore when the database cannot be written.
 Named-profile cron and Discord databases under `.hermes/profiles/<name>/` use raw directory capture and can be inconsistent if a write overlaps the snapshot.
 
 Kanban backup is limited to the backward-compatible default board in `kanban.db`.
@@ -121,6 +122,11 @@ $$nemoclaw my-assistant snapshot restore before-upgrade --to my-assistant-clone
 $$nemoclaw my-assistant snapshot restore before-upgrade --to my-assistant-clone --force --yes
 ```
 
+Cross-sandbox restore from a stopped source is available for Docker- and VM-driver sandboxes.
+For a stopped source, its registry entry must record both the sandbox image and a complete inference route; NemoClaw creates the destination from the recorded image.
+NemoClaw stops before creating or replacing the destination when either record is missing, and directs you to run `$$nemoclaw onboard` when no image is recorded.
+For a Kubernetes-driver source, the pod image must remain resolvable through its gateway.
+
 For dashboard-enabled agents, NemoClaw allocates the destination sandbox its own dashboard port instead of reusing the source port.
 If no port is available, restore stops before deleting an existing `--force` destination.
 

docs/reference/commands.mdx

@@ -2906,7 +2906,7 @@ $$nemoclaw my-assistant snapshot list
 ### `$$nemoclaw <name> snapshot restore [selector] [--to <dst>] [--force] [--yes|-y]`
 
 Restore sandbox state from a snapshot.
-The sandbox must be running before you restore.
+For an in-place restore, the sandbox must be running.
 If no selector is provided, the latest snapshot is used.
 Restore removes files added after the snapshot only from state directories selected for cleanup.
 It preserves directories that exist only in the target manifest or whose backup failed.
@@ -2920,16 +2920,18 @@ The selector accepts any of:
 - An exact timestamp.
 
 Pass `--to <dst>` to restore the snapshot into a different sandbox instead of the source.
-When `dst` does not exist, it is auto-created by reusing the source sandbox's container image.
-No re-onboarding is needed.
+When `dst` does not exist, it is auto-created from the source image.
+For a Docker- or VM-driver source, the source can be stopped when its registry entry records both the sandbox image and a complete inference route.
+For a Kubernetes-driver source, the pod image must remain resolvable through its gateway.
+No re-onboarding is needed when those prerequisites are present.
 A cross-sandbox restore refuses to clone a source or replace an existing destination whose baseline exclusion transaction needs repair, before it creates or deletes anything.
 When `dst` already exists, `snapshot restore --to <dst>` refuses by default to avoid silently mutating the destination's filesystem.
 To overwrite an existing destination, pass `--force`: the command deletes `dst`, then recreates it from the source's image and restores the snapshot into the fresh copy.
 If the existing destination has an active shields timer, the force path restores and verifies lockdown, revokes the timer, and then deletes the destination.
 It clears the remaining local shields state only after deletion succeeds.
 The `--force` path prompts interactively to confirm the destination name before deleting.
 Pass `--yes` (or set `NEMOCLAW_NON_INTERACTIVE=1`) to skip the prompt.
-The snapshot selector and source pod image are both validated before any deletion, so a bad selector or unresolvable image cannot destroy `dst` and only fail afterwards.
+The snapshot selector, source image, and durable inference route are validated before any deletion. If any prerequisite is invalid, restore stops before it deletes `dst`.
 
 ```bash
 # restore latest snapshot in-place

src/lib/actions/sandbox/snapshot-restore-offline-source.test.ts

@@ -0,0 +1,101 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import * as f from "./snapshot-restore-test-fixture";
+
+const offlineSourceEntry: {
+  name: string;
+  agent: string;
+  imageTag: string | null;
+  openshellDriver: string;
+  provider: string | null;
+  model: string | null;
+} = {
+  name: "alpha",
+  agent: "openclaw",
+  imageTag: "nemoclaw-alpha:test",
+  openshellDriver: "docker",
+  provider: "nvidia-nim",
+  model: "nvidia/model-a",
+};
+
+function stubOfflineSource(entry: typeof offlineSourceEntry): void {
+  f.getSandboxMock.mockImplementation((name) => (name === "alpha" ? entry : null));
+  f.parseLiveSandboxNamesMock.mockReturnValue(new Set<string>());
+  f.captureOpenshellMock.mockImplementation((args) =>
+    f.openshellResponses(args, {
+      "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") },
+      "sandbox list": { status: 0, output: "beta Ready\n" },
+    }),
+  );
+  f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture });
+}
+
+beforeEach(() => {
+  f.resetSnapshotRestoreMocks();
+});
+afterEach(() => {
+  f.cleanupSnapshotRestoreMocks();
+});
+
+describe("runSandboxSnapshot restore: source sandbox no longer running", () => {
+  it("restores into a replacement sandbox built from the registered source image", async () => {
+    vi.spyOn(console, "log").mockImplementation(() => {});
+    stubOfflineSource(offlineSourceEntry);
+    f.restoreSandboxStateMock.mockReturnValue({
+      success: true,
+      restoredDirs: ["workspace"],
+      restoredFiles: ["user.md"],
+      failedDirs: [],
+      failedFiles: [],
+    });
+    const { runSandboxSnapshot } = await import("./snapshot");
+
+    await runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true });
+
+    expect(f.streamSandboxCreateMock).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.arrayContaining(["--name", "beta", "--from", "nemoclaw-alpha:test"]),
+      expect.any(Object),
+      expect.any(Object),
+    );
+    expect(f.restoreSandboxStateMock).toHaveBeenCalledWith("beta", "/tmp/backup-alpha");
+  });
+
+  it("stops before creating a replacement when the source records no image", async () => {
+    vi.spyOn(console, "log").mockImplementation(() => {});
+    const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+    stubOfflineSource({ ...offlineSourceEntry, imageTag: null });
+    const { runSandboxSnapshot } = await import("./snapshot");
+
+    await expect(
+      runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true }),
+    ).rejects.toMatchObject({ exitCode: 1 });
+
+    const errors = consoleError.mock.calls.flat().join("\n");
+    expect(errors).toContain(
+      "source 'alpha' is not running and its registry entry records no image",
+    );
+    expect(f.streamSandboxCreateMock).not.toHaveBeenCalled();
+    expect(f.restoreSandboxStateMock).not.toHaveBeenCalled();
+  });
+
+  it("stops before creating a replacement when the source inference route is incomplete", async () => {
+    vi.spyOn(console, "log").mockImplementation(() => {});
+    const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+    stubOfflineSource({ ...offlineSourceEntry, model: null });
+    const { runSandboxSnapshot } = await import("./snapshot");
+
+    await expect(
+      runSandboxSnapshot("alpha", { kind: "restore", to: "beta", yes: true }),
+    ).rejects.toMatchObject({ exitCode: 1 });
+
+    expect(consoleError.mock.calls.flat().join("\n")).toContain(
+      "source 'alpha' has no complete durable inference route",
+    );
+    expect(f.streamSandboxCreateMock).not.toHaveBeenCalled();
+    expect(f.restoreSandboxStateMock).not.toHaveBeenCalled();
+  });
+});

src/lib/actions/sandbox/snapshot.ts

@@ -986,32 +986,28 @@ async function runSnapshotRestoreUnlocked(
       );
       snapshotExit(1);
     }
-    // Cross-sandbox restore — whether dst exists (with --force) or not,
-    // we must be able to clone the source's running pod image. Resolve it
-    // upfront so a missing source / unresolvable image cannot delete the
-    // destination first (#3756 P1).
-    if (!sourceLiveNames.has(sandboxName)) {
-      if (targetExists) {
+    // Cross-sandbox restore — whether dst exists (with --force) or not, we
+    // must be able to clone the source's image. Resolve it upfront so a
+    // missing source / unresolvable image cannot delete the destination first
+    // (#3756 P1). A source that is no longer running stays restorable while
+    // its registry entry still records the image and inference route, because
+    // that is the case a replacement sandbox exists to recover from.
+    const srcEntry = registry.getSandbox(sandboxName) || { name: sandboxName };
+    const fromImage = resolveSrcPodImage(sandboxName, srcEntry);
+    if (!fromImage) {
+      if (!sourceLiveNames.has(sandboxName)) {
         console.error(
-          `  Cannot recreate '${targetSandbox}' from snapshot: source '${sandboxName}' not found.`,
+          `  Cannot ${targetExists ? "recreate" : "auto-create"} '${targetSandbox}': source '${sandboxName}' is not running and its registry entry records no image.`,
         );
+        console.error(`  Create '${targetSandbox}' manually with '${CLI_NAME} onboard'.`);
       } else {
         console.error(
-          `  Cannot auto-create '${targetSandbox}': source '${sandboxName}' not found.`,
+          `  Cannot resolve image for source sandbox '${sandboxName}' — aborting before ` +
+            (targetExists ? `deleting '${targetSandbox}'.` : `creating '${targetSandbox}'.`),
         );
-        console.error(`  Create '${targetSandbox}' manually with '${CLI_NAME} onboard'.`);
       }
       snapshotExit(1);
     }
-    const srcEntry = registry.getSandbox(sandboxName) || { name: sandboxName };
-    const fromImage = resolveSrcPodImage(sandboxName, srcEntry);
-    if (!fromImage) {
-      console.error(
-        `  Cannot resolve image for source sandbox '${sandboxName}' — aborting before ` +
-          (targetExists ? `deleting '${targetSandbox}'.` : `creating '${targetSandbox}'.`),
-      );
-      snapshotExit(1);
-    }
     if (targetExists) {
       // --force confirmed above. Prompt for the destination name (unless
       // --yes or NEMOCLAW_NON_INTERACTIVE=1), then delete and recreate.

src/lib/state/state-file-restore.ts

@@ -32,6 +32,18 @@ const SQLITE_RESTORE_PY = [
   "    src_conn.close()",
 ].join("\n");
 
+const SQLITE_WRITE_CHECK_PY = [
+  "import sqlite3, sys",
+  "dst = sys.argv[1]",
+  "conn = sqlite3.connect(dst, timeout=30)",
+  "try:",
+  "    conn.execute('PRAGMA busy_timeout=30000')",
+  "    conn.execute('BEGIN IMMEDIATE')",
+  "    conn.execute('ROLLBACK')",
+  "finally:",
+  "    conn.close()",
+].join("\n");
+
 function stateFileRemotePath(dir: string, filePath: string): string {
   return `${dir.replace(/\/+$/, "")}/${filePath}`;
 }
@@ -44,13 +56,18 @@ export function buildStateFileRestoreCommand(
   const remotePath = stateFileRemotePath(dir, spec.path);
   const quotedRemotePath = shellQuote(remotePath);
   if (spec.strategy === "sqlite_backup") {
-    // The agent gateway process can own the live database under a distinct
-    // uid. Even when its parent and file are intentionally group-writable,
-    // restoring in place would expose a partially replaced SQLite file to the
-    // gateway. Validate the backup into a staged database the restoring user
-    // owns, then replace the target atomically through the group-writable
-    // parent. The stale WAL/SHM sidecars belong to the replaced database, so
-    // drop them after the swap (#7312).
+    // The agent gateway can own the live database under a distinct uid, so
+    // restoring in place can fail for the sandbox user and expose a partially
+    // replaced SQLite file to the gateway (#7312). Validate the backup into a
+    // staged database this user owns, then replace the target atomically;
+    // replacement only needs write permission on the parent directory. The
+    // stale WAL/SHM sidecars belong to the replaced database, so drop them.
+    //
+    // A successful swap does not prove the agent can persist to the result, so
+    // open a write transaction against the replaced database before reporting
+    // success. The check runs under the same umask as the restore so its own
+    // sidecars stay group-writable, and both sidecar pairs are dropped: the
+    // stale ones before the check reads them, the check's own after it ends.
     return [
       `dst=${quotedRemotePath}`,
       'parent="$(dirname "$dst")"',
@@ -59,13 +76,15 @@ export function buildStateFileRestoreCommand(
       'mkdir -p "$parent"',
       'tmp="$(mktemp /tmp/nemoclaw-sqlite-restore.XXXXXX)"',
       'staged="$(mktemp "${parent}/.nemoclaw-sqlite-staged.XXXXXX")"',
-      'trap \'rm -f "$tmp" "$staged"\' EXIT',
+      'trap \'rm -f "$tmp" "$staged" "${staged}-wal" "${staged}-shm"\' EXIT',
       'cat > "$tmp"',
       'chmod 600 "$tmp"',
       `(umask 0007; /usr/bin/python3 -I -S -c ${shellQuote(SQLITE_RESTORE_PY)} "$tmp" "$staged")`,
       'chmod 660 "$staged"',
       'mv -f "$staged" "$dst"',
       'rm -f -- "${dst}-wal" "${dst}-shm"',
+      `(umask 0007; /usr/bin/python3 -I -S -c ${shellQuote(SQLITE_WRITE_CHECK_PY)} "$dst") || { echo "restored database is not writable: $dst" >&2; exit 12; }`,
+      'rm -f -- "${dst}-wal" "${dst}-shm"',
     ].join(" && ");
   }
 

src/lib/state/state-file-sqlite-restore-behavior.test.ts

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { execFileSync, spawnSync } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import { buildStateFileRestoreCommand } from "./state-file-restore";
+
+const CREATE_SOURCE_DB_PY = [
+  "import sqlite3, sys",
+  "conn = sqlite3.connect(sys.argv[1])",
+  "conn.execute('CREATE TABLE sessions (id TEXT)')",
+  "conn.execute(\"INSERT INTO sessions VALUES ('restored')\")",
+  "conn.commit()",
+  "conn.close()",
+].join("\n");
+
+const WRITE_TO_DB_PY = [
+  "import sqlite3, sys",
+  "conn = sqlite3.connect(sys.argv[1])",
+  "conn.execute(\"INSERT INTO sessions VALUES ('after-restore')\")",
+  "conn.commit()",
+  "print(conn.execute('SELECT count(*) FROM sessions').fetchone()[0])",
+  "conn.close()",
+].join("\n");
+
+const hasSystemPython = fs.existsSync("/usr/bin/python3");
+const fixtures: string[] = [];
+
+function makeFixture(): { dir: string; backup: Buffer } {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sqlite-restore-"));
+  fixtures.push(dir);
+  const sourceDb = path.join(dir, "source.db");
+  execFileSync("/usr/bin/python3", ["-c", CREATE_SOURCE_DB_PY, sourceDb]);
+  return { dir, backup: fs.readFileSync(sourceDb) };
+}
+
+function occupyRollbackJournalPath(restored: string): void {
+  fs.mkdirSync(`${restored}-journal`, { recursive: true });
+}
+
+function runRestore(stateDir: string, backup: Buffer): { status: number | null; stderr: string } {
+  const command = buildStateFileRestoreCommand(stateDir, {
+    path: "runtime/state.db",
+    strategy: "sqlite_backup",
+  });
+  const result = spawnSync("bash", ["-c", command], { input: backup });
+  return { status: result.status, stderr: result.stderr.toString() };
+}
+
+afterEach(() => {
+  for (const fixture of fixtures.splice(0)) {
+    fs.rmSync(fixture, { recursive: true, force: true });
+  }
+});
+
+describe.skipIf(!hasSystemPython)("sqlite state-file restore", () => {
+  it("leaves the restored database writable and free of stale sidecars", () => {
+    const { dir, backup } = makeFixture();
+    const stateDir = path.join(dir, "state");
+    const restored = path.join(stateDir, "runtime", "state.db");
+    fs.mkdirSync(path.join(stateDir, "runtime"), { recursive: true });
+    fs.writeFileSync(`${restored}-wal`, "stale");
+    fs.writeFileSync(`${restored}-shm`, "stale");
+
+    const result = runRestore(stateDir, backup);
+
+    expect(result.stderr).toBe("");
+    expect(result.status).toBe(0);
+    expect(fs.existsSync(`${restored}-wal`)).toBe(false);
+    expect(fs.existsSync(`${restored}-shm`)).toBe(false);
+    const rows = execFileSync("/usr/bin/python3", ["-c", WRITE_TO_DB_PY, restored]).toString();
+    expect(rows.trim()).toBe("2");
+  });
+
+  it("reports failure when the swapped database cannot open a write transaction", () => {
+    const { dir, backup } = makeFixture();
+    const stateDir = path.join(dir, "state");
+    const restored = path.join(stateDir, "runtime", "state.db");
+    fs.mkdirSync(path.join(stateDir, "runtime"), { recursive: true });
+    occupyRollbackJournalPath(restored);
+
+    const result = runRestore(stateDir, backup);
+
+    expect(fs.existsSync(restored)).toBe(true);
+    expect(result.stderr).toContain(`restored database is not writable: ${restored}`);
+    expect(result.status).toBe(12);
+  });
+});