fix(sandbox): clear stale restore dirs and repair hermes venv perms

agents/hermes/Dockerfile

@@ -38,6 +38,10 @@ RUN set -eu; \
 RUN /opt/hermes/.venv/bin/python -c \
     'import mcp; from tools import mcp_tool; assert getattr(mcp_tool, "_MCP_AVAILABLE", False), "Hermes MCP client runtime is unavailable"; assert getattr(mcp_tool, "_MCP_HTTP_AVAILABLE", False), "Hermes MCP Streamable HTTP runtime is unavailable"'
 
+RUN chmod -R a+rX /opt/hermes/.venv \
+    && test -r /opt/hermes/.venv/pyvenv.cfg \
+    && su -s /bin/sh sandbox -c '/opt/hermes/.venv/bin/python3 -c "import urllib.request"'
+
 # Published base images can lag Dockerfile.base while local feature branches
 # still layer this final image on top. Invalid state: the selected base has
 # Hermes source under /opt/hermes but lacks hermes_cli/web_dist. Prebuild the

agents/hermes/Dockerfile.base

@@ -304,6 +304,10 @@ RUN /usr/local/bin/hermes --version \
     && /opt/hermes/.venv/bin/python -c \
         'import mcp; from tools import mcp_tool; assert getattr(mcp_tool, "_MCP_AVAILABLE", False), "Hermes MCP client runtime is unavailable"; assert getattr(mcp_tool, "_MCP_HTTP_AVAILABLE", False), "Hermes MCP Streamable HTTP runtime is unavailable"'
 
+RUN chmod -R a+rX /opt/hermes/.venv \
+    && test -r /opt/hermes/.venv/pyvenv.cfg \
+    && su -s /bin/sh sandbox -c '/opt/hermes/.venv/bin/python3 -c "import urllib.request"'
+
 # Gate the exact completed base filesystem before it can be published.
 COPY scripts/checks/node-tar-image-scan.mts /scripts/checks/node-tar-image-scan.mts
 RUN install -d -m 0755 /usr/local/share/nemoclaw \

docs/manage-sandboxes/backup-restore.mdx

@@ -60,6 +60,8 @@ When you restore a snapshot, NemoClaw replays those recorded custom presets with
 
 The target sandbox's current agent manifest remains authoritative for state-file restore behavior.
 NemoClaw rejects the restore when the snapshot's agent, config directory, state-file path, or state-file strategy conflicts with that manifest.
+Restore limits directory cleanup to state directories declared by the snapshot manifest.
+It preserves directories that exist only in the target manifest or whose backup failed.
 
 For managed images, NemoClaw applies the current manifest's managed config merge rules by default and does not fall back to whole-file replacement.
 For Deep Agents targets, whole-file config replacement is limited to sandboxes created from a custom Dockerfile.

docs/reference/commands.mdx

@@ -2703,7 +2703,8 @@ $$nemoclaw my-assistant snapshot list
 Restore sandbox state from a snapshot.
 The sandbox must be running before you restore.
 If no selector is provided, the latest snapshot is used.
-Restore performs a clean replacement of each state directory, removing files that were added after the snapshot was taken.
+Restore removes files added after the snapshot only from state directories selected for cleanup.
+It preserves directories that exist only in the target manifest or whose backup failed.
 The state replacement, mutable-config permission repair, and policy reconciliation run under the same per-sandbox transition.
 An expired auto-restore timer can interrupt that work and restore lockdown.
 

src/lib/onboard/machine/handlers/finalization.test.ts

@@ -180,6 +180,28 @@ describe("handleFinalizationState", () => {
     );
   });
 
+  it("rechecks gateway and forwarding after finalization work and before verification", async () => {
+    const { deps, calls } = createDeps();
+    const agent = { name: "openclaw" };
+
+    await handleFinalizationState({
+      ...baseOptions(deps),
+      agent,
+      webSearchEnabled: true,
+    });
+
+    const recoveryOrders = calls.recoverProcesses.mock.invocationCallOrder;
+    const refreshOrder = calls.ensureAgentDashboard.mock.invocationCallOrder[0];
+    expect(recoveryOrders).toHaveLength(2);
+    expect(refreshOrder).toBeLessThan(recoveryOrders[0]);
+    expect(recoveryOrders[1]).toBeGreaterThan(calls.warmupScopeUpgrade.mock.invocationCallOrder[0]);
+    expect(recoveryOrders[1]).toBeGreaterThan(
+      calls.autoPairScopeApproval.mock.invocationCallOrder[0],
+    );
+    expect(recoveryOrders[1]).toBeGreaterThan(calls.verifyWebSearch.mock.invocationCallOrder[0]);
+    expect(recoveryOrders[1]).toBeLessThan(calls.verify.mock.invocationCallOrder[0]);
+  });
+
   it("skips dashboard and gateway verification for terminal agents without forwards", async () => {
     const { deps, calls } = createDeps();
     const agent = {

src/lib/onboard/machine/handlers/finalization.ts

@@ -184,6 +184,12 @@ export async function handleFinalizationState<Agent, VerifyChain, VerificationRe
     deps.verifyWebSearchInsideSandbox(sandboxName, agent);
   }
 
+  if (manageDashboard) {
+    // Scope warm-up can outlive a forward that was healthy after policy recovery.
+    // Recheck the gateway and forward before verification, restarting only when needed.
+    deps.checkAndRecoverSandboxProcesses(sandboxName, { quiet: true });
+  }
+
   await deps.recordPostVerifyStarted();
 
   let verificationDiagnostics: string[] = [];

src/lib/state/openclaw-managed-extensions.test.ts

@@ -258,3 +258,72 @@ describe("OpenClaw managed extension cleanup", () => {
     expect(buildRestoreCleanupCommand("/sandbox/.openclaw", [], [], new Set())).toBe(":");
   });
 });
+
+describe("restore stale content cleanup", () => {
+  it("clears stale contents of declared dirs missing from the backup while preserving the directory", () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-stale-content-"));
+    try {
+      const workspace = path.join(root, "workspace");
+      const nested = path.join(workspace, "sub");
+      fs.mkdirSync(nested, { recursive: true });
+      fs.writeFileSync(path.join(workspace, "stale.txt"), "post-snapshot");
+      fs.writeFileSync(path.join(nested, "child"), "post-snapshot");
+      const sessions = path.join(root, "sessions");
+      fs.mkdirSync(sessions);
+      fs.writeFileSync(path.join(sessions, "s"), "1");
+
+      const command = buildRestoreCleanupCommand(root, ["sessions"], [], new Set(), [
+        "workspace",
+        "sessions",
+        "memories",
+      ]);
+      expect(command).toContain("rm -rf -- '" + sessions + "'");
+      expect(command).not.toContain("d='" + sessions + "'");
+      execFileSync("bash", ["-c", command], { stdio: "pipe" });
+
+      expect(fs.existsSync(workspace)).toBe(true);
+      expect(fs.existsSync(path.join(workspace, "stale.txt"))).toBe(false);
+      expect(fs.existsSync(nested)).toBe(false);
+      expect(fs.existsSync(sessions)).toBe(false);
+    } finally {
+      fs.rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  it("preserves the directory mode when clearing stale contents", () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-stale-mode-"));
+    try {
+      const workspace = path.join(root, "workspace");
+      fs.mkdirSync(workspace);
+      fs.chmodSync(workspace, 0o2770);
+      fs.writeFileSync(path.join(workspace, "stale"), "x");
+
+      const command = buildRestoreCleanupCommand(root, [], [], new Set(), ["workspace"]);
+      execFileSync("bash", ["-c", command], { stdio: "pipe" });
+
+      expect(fs.existsSync(path.join(workspace, "stale"))).toBe(false);
+      expect(fs.statSync(workspace).mode & 0o777).toBe(0o770);
+    } finally {
+      fs.rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  it("does not clear a declared dir that is a symlink", () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-stale-symlink-"));
+    try {
+      const realDir = path.join(root, "real");
+      fs.mkdirSync(realDir);
+      fs.writeFileSync(path.join(realDir, "keep"), "1");
+      const workspace = path.join(root, "workspace");
+      fs.symlinkSync(realDir, workspace);
+
+      const command = buildRestoreCleanupCommand(root, [], [], new Set(), ["workspace"]);
+      execFileSync("bash", ["-c", command], { stdio: "pipe" });
+
+      expect(fs.lstatSync(workspace).isSymbolicLink()).toBe(true);
+      expect(fs.existsSync(path.join(realDir, "keep"))).toBe(true);
+    } finally {
+      fs.rmSync(root, { recursive: true, force: true });
+    }
+  });
+});

src/lib/state/openclaw-managed-extensions.ts

@@ -136,11 +136,21 @@ function buildOpenClawExtensionsCleanupCommand(
   ].join(" && ");
 }
 
+function buildStaleStateDirContentsCleanupCommand(dir: string, dirName: string): string {
+  const target = shellQuote(`${dir}/${dirName}`);
+  return (
+    `d=${target}; ` +
+    'if [ -d "$d" ] && [ ! -L "$d" ]; then ' +
+    'find "$d" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; fi'
+  );
+}
+
 export function buildRestoreCleanupCommand(
   dir: string,
   localDirs: readonly string[],
   managedExtensionDirs: readonly string[],
   requiredExtensionDirs: ReadonlySet<string>,
+  staleContentDirs: readonly string[] = [],
 ): string {
   const preserveManagedExtensions = managedExtensionDirs.length > 0;
   const commands: string[] = [];
@@ -153,5 +163,11 @@ export function buildRestoreCleanupCommand(
       buildOpenClawExtensionsCleanupCommand(dir, managedExtensionDirs, requiredExtensionDirs),
     );
   }
+  const localDirSet = new Set(localDirs);
+  for (const dirName of staleContentDirs) {
+    if (localDirSet.has(dirName)) continue;
+    if (preserveManagedExtensions && dirName === "extensions") continue;
+    commands.push(buildStaleStateDirContentsCleanupCommand(dir, dirName));
+  }
   return commands.length > 0 ? commands.join(" && ") : ":";
 }

src/lib/state/sandbox.ts

@@ -89,6 +89,8 @@ export interface RebuildManifest {
   stateDirs: string[];
   /** Directories verified as safe to restore. Absent on older manifests. */
   backedUpDirs?: string[];
+  /** Declared directories that could not be backed up. Absent on older manifests. */
+  failedBackupDirs?: string[];
   stateFiles?: StateFileSpec[];
   /** Single config/state directory */
   dir: string;
@@ -270,6 +272,8 @@ function isRebuildManifest(value: unknown): value is RebuildManifest {
     (value.agentVersion === null || typeof value.agentVersion === "string") &&
     (value.expectedVersion === null || typeof value.expectedVersion === "string") &&
     (value.backedUpDirs === undefined || isBackedUpDirArray(value.backedUpDirs, value.stateDirs)) &&
+    (value.failedBackupDirs === undefined ||
+      isBackedUpDirArray(value.failedBackupDirs, value.stateDirs)) &&
     typeof dir === "string" &&
     (value.openclawImagePluginInstalls === undefined ||
       parseOpenClawImagePluginInstalls(value.openclawImagePluginInstalls, dir).ok) &&
@@ -983,6 +987,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions =
       ? { reconcileOpenClawImagePluginProvenance: true }
       : {}),
     stateDirs,
+    failedBackupDirs: [],
     stateFiles,
     dir,
     backupPath,
@@ -1322,6 +1327,9 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions =
     );
   }
   manifest.backedUpDirs = backedUpDirs;
+  manifest.failedBackupDirs = failedDirs.filter((failedDir) =>
+    manifest.stateDirs.includes(failedDir),
+  );
 
   writeManifest(backupPath, manifest);
   manifest.backupPath = backupPath;
@@ -1475,6 +1483,21 @@ function restoreSandboxStateInternal(
       localDirs.splice(localDirs.indexOf(d), 1);
     }
   }
+  // Only manifests that distinguish failed backups from absent directories can
+  // authorize cleanup without deleting data that a failed backup did not capture.
+  // Older manifests leave this field absent, so preserve their historical restore behavior.
+  const failedBackupDirs = new Set(manifest.failedBackupDirs ?? []);
+  const localDirSet = new Set(localDirs);
+  const staleContentDirs =
+    manifest.failedBackupDirs === undefined
+      ? []
+      : manifest.stateDirs.filter(
+          (stateDir) =>
+            !targetRuntimeAuthDirs.has(stateDir) &&
+            !localDirSet.has(stateDir) &&
+            !failedBackupDirs.has(stateDir),
+        );
+  const cleanupStateDirs = [...new Set([...localDirs, ...staleContentDirs])];
   const targetStateFiles = new Map<string, AgentStateFile>();
   for (const targetFile of targetAgent.stateFiles) {
     const normalized = normalizeStateFilePath(targetFile.path);
@@ -1534,7 +1557,7 @@ function restoreSandboxStateInternal(
     freshOpenClawImagePluginInstalls = discovery.pluginInstalls;
   }
 
-  if (localDirs.length === 0 && localFiles.length === 0) {
+  if (cleanupStateDirs.length === 0 && localFiles.length === 0) {
     _log("No dirs or files to restore");
     return { success: true, restoredDirs, failedDirs, restoredFiles, failedFiles };
   }
@@ -1546,7 +1569,7 @@ function restoreSandboxStateInternal(
     return {
       success: false,
       restoredDirs,
-      failedDirs: [...localDirs],
+      failedDirs: [...cleanupStateDirs],
       restoredFiles,
       failedFiles: localFiles.map((f) => f.path),
     };
@@ -1571,15 +1594,15 @@ function restoreSandboxStateInternal(
     const pluginRestorePlan = planOpenClawPluginRestore({
       agentType: manifest.agentType,
       dir,
-      localDirs,
+      localDirs: cleanupStateDirs,
       freshImagePluginInstalls: freshOpenClawImagePluginInstalls,
       previousImagePluginInstalls: previousOpenClawImagePluginInstalls,
     });
     if (!pluginRestorePlan.ok) {
       return {
         success: false,
         restoredDirs,
-        failedDirs: [...localDirs],
+        failedDirs: [...cleanupStateDirs],
         restoredFiles,
         failedFiles: localFiles.map((f) => f.path),
         error:
@@ -1600,6 +1623,7 @@ function restoreSandboxStateInternal(
       );
     }
 
+    let restoreTar: Buffer | undefined;
     if (localDirs.length > 0) {
       // Upload via tar pipe
       // NC-2227-04: Removed -h flag from restore as well — no symlink following.
@@ -1617,22 +1641,26 @@ function restoreSandboxStateInternal(
         return {
           success: false,
           restoredDirs,
-          failedDirs: [...localDirs],
+          failedDirs: [...cleanupStateDirs],
           restoredFiles,
           failedFiles: localFiles.map((f) => f.path),
         };
       }
+      restoreTar = tarResult.stdout;
+    }
 
-      // Remove existing state dirs before extracting so stale files from later
-      // snapshots don't persist after restoring an earlier one. OpenClaw's
-      // image-managed extensions are preserved from the freshly built image and
-      // excluded from the restore tar; only user/non-managed extension entries
-      // are cleared and restored from the backup.
+    // Remove existing state dirs before extracting so stale files from later
+    // snapshots don't persist after restoring an earlier one. OpenClaw's
+    // image-managed extensions are preserved from the freshly built image and
+    // excluded from the restore tar; only user/non-managed extension entries
+    // are cleared and restored from the backup.
+    if (cleanupStateDirs.length > 0) {
       const rmCmd = buildRestoreCleanupCommand(
         dir,
         localDirs,
         pluginRestorePlan.preservedExtensionDirs,
         new Set(pluginRestorePlan.requiredFreshExtensionDirs),
+        staleContentDirs,
       );
       _log(`Cleaning target dirs before restore: ${rmCmd}`);
       const rmResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), rmCmd], {
@@ -1649,15 +1677,17 @@ function restoreSandboxStateInternal(
         return {
           success: false,
           restoredDirs,
-          failedDirs: [...localDirs],
+          failedDirs: [...cleanupStateDirs],
           restoredFiles,
           failedFiles: localFiles.map((f) => f.path),
         };
       }
+    }
 
+    if (restoreTar !== undefined) {
       const extractCmd = `tar --no-same-owner -xf - -C ${shellQuote(dir)}`;
       const sshResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), extractCmd], {
-        input: tarResult.stdout,
+        input: restoreTar,
         stdio: ["pipe", "pipe", "pipe"],
         timeout: 120000,
       });