diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 552c1dda916..726004d2d2d 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -38,6 +38,10 @@ RUN set -eu; \ RUN /opt/hermes/.venv/bin/python -c \ 'import mcp; from tools import mcp_tool; assert getattr(mcp_tool, "_MCP_AVAILABLE", False), "Hermes MCP client runtime is unavailable"; assert getattr(mcp_tool, "_MCP_HTTP_AVAILABLE", False), "Hermes MCP Streamable HTTP runtime is unavailable"' +RUN chmod -R a+rX /opt/hermes/.venv \ + && test -r /opt/hermes/.venv/pyvenv.cfg \ + && su -s /bin/sh sandbox -c '/opt/hermes/.venv/bin/python3 -c "import urllib.request"' + # Published base images can lag Dockerfile.base while local feature branches # still layer this final image on top. Invalid state: the selected base has # Hermes source under /opt/hermes but lacks hermes_cli/web_dist. Prebuild the diff --git a/agents/hermes/Dockerfile.base b/agents/hermes/Dockerfile.base index c4a3a84b869..dedccea37c0 100644 --- a/agents/hermes/Dockerfile.base +++ b/agents/hermes/Dockerfile.base @@ -304,6 +304,10 @@ RUN /usr/local/bin/hermes --version \ && /opt/hermes/.venv/bin/python -c \ 'import mcp; from tools import mcp_tool; assert getattr(mcp_tool, "_MCP_AVAILABLE", False), "Hermes MCP client runtime is unavailable"; assert getattr(mcp_tool, "_MCP_HTTP_AVAILABLE", False), "Hermes MCP Streamable HTTP runtime is unavailable"' +RUN chmod -R a+rX /opt/hermes/.venv \ + && test -r /opt/hermes/.venv/pyvenv.cfg \ + && su -s /bin/sh sandbox -c '/opt/hermes/.venv/bin/python3 -c "import urllib.request"' + # Gate the exact completed base filesystem before it can be published. COPY scripts/checks/node-tar-image-scan.mts /scripts/checks/node-tar-image-scan.mts RUN install -d -m 0755 /usr/local/share/nemoclaw \ diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 5ba8479d1ee..463d48f9acb 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -60,6 +60,8 @@ When you restore a snapshot, NemoClaw replays those recorded custom presets with The target sandbox's current agent manifest remains authoritative for state-file restore behavior. NemoClaw rejects the restore when the snapshot's agent, config directory, state-file path, or state-file strategy conflicts with that manifest. +Restore limits directory cleanup to state directories declared by the snapshot manifest. +It preserves directories that exist only in the target manifest or whose backup failed. For managed images, NemoClaw applies the current manifest's managed config merge rules by default and does not fall back to whole-file replacement. For Deep Agents targets, whole-file config replacement is limited to sandboxes created from a custom Dockerfile. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index e0c8800b3ce..95703aa0e84 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2703,7 +2703,8 @@ $$nemoclaw my-assistant snapshot list Restore sandbox state from a snapshot. The sandbox must be running before you restore. If no selector is provided, the latest snapshot is used. -Restore performs a clean replacement of each state directory, removing files that were added after the snapshot was taken. +Restore removes files added after the snapshot only from state directories selected for cleanup. +It preserves directories that exist only in the target manifest or whose backup failed. The state replacement, mutable-config permission repair, and policy reconciliation run under the same per-sandbox transition. An expired auto-restore timer can interrupt that work and restore lockdown. diff --git a/src/lib/onboard/machine/handlers/finalization.test.ts b/src/lib/onboard/machine/handlers/finalization.test.ts index dea8667b8b7..1130ccfc0ef 100644 --- a/src/lib/onboard/machine/handlers/finalization.test.ts +++ b/src/lib/onboard/machine/handlers/finalization.test.ts @@ -180,6 +180,28 @@ describe("handleFinalizationState", () => { ); }); + it("rechecks gateway and forwarding after finalization work and before verification", async () => { + const { deps, calls } = createDeps(); + const agent = { name: "openclaw" }; + + await handleFinalizationState({ + ...baseOptions(deps), + agent, + webSearchEnabled: true, + }); + + const recoveryOrders = calls.recoverProcesses.mock.invocationCallOrder; + const refreshOrder = calls.ensureAgentDashboard.mock.invocationCallOrder[0]; + expect(recoveryOrders).toHaveLength(2); + expect(refreshOrder).toBeLessThan(recoveryOrders[0]); + expect(recoveryOrders[1]).toBeGreaterThan(calls.warmupScopeUpgrade.mock.invocationCallOrder[0]); + expect(recoveryOrders[1]).toBeGreaterThan( + calls.autoPairScopeApproval.mock.invocationCallOrder[0], + ); + expect(recoveryOrders[1]).toBeGreaterThan(calls.verifyWebSearch.mock.invocationCallOrder[0]); + expect(recoveryOrders[1]).toBeLessThan(calls.verify.mock.invocationCallOrder[0]); + }); + it("skips dashboard and gateway verification for terminal agents without forwards", async () => { const { deps, calls } = createDeps(); const agent = { diff --git a/src/lib/onboard/machine/handlers/finalization.ts b/src/lib/onboard/machine/handlers/finalization.ts index 1350e28758d..747c7143369 100644 --- a/src/lib/onboard/machine/handlers/finalization.ts +++ b/src/lib/onboard/machine/handlers/finalization.ts @@ -184,6 +184,12 @@ export async function handleFinalizationState { expect(buildRestoreCleanupCommand("/sandbox/.openclaw", [], [], new Set())).toBe(":"); }); }); + +describe("restore stale content cleanup", () => { + it("clears stale contents of declared dirs missing from the backup while preserving the directory", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-stale-content-")); + try { + const workspace = path.join(root, "workspace"); + const nested = path.join(workspace, "sub"); + fs.mkdirSync(nested, { recursive: true }); + fs.writeFileSync(path.join(workspace, "stale.txt"), "post-snapshot"); + fs.writeFileSync(path.join(nested, "child"), "post-snapshot"); + const sessions = path.join(root, "sessions"); + fs.mkdirSync(sessions); + fs.writeFileSync(path.join(sessions, "s"), "1"); + + const command = buildRestoreCleanupCommand(root, ["sessions"], [], new Set(), [ + "workspace", + "sessions", + "memories", + ]); + expect(command).toContain("rm -rf -- '" + sessions + "'"); + expect(command).not.toContain("d='" + sessions + "'"); + execFileSync("bash", ["-c", command], { stdio: "pipe" }); + + expect(fs.existsSync(workspace)).toBe(true); + expect(fs.existsSync(path.join(workspace, "stale.txt"))).toBe(false); + expect(fs.existsSync(nested)).toBe(false); + expect(fs.existsSync(sessions)).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("preserves the directory mode when clearing stale contents", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-stale-mode-")); + try { + const workspace = path.join(root, "workspace"); + fs.mkdirSync(workspace); + fs.chmodSync(workspace, 0o2770); + fs.writeFileSync(path.join(workspace, "stale"), "x"); + + const command = buildRestoreCleanupCommand(root, [], [], new Set(), ["workspace"]); + execFileSync("bash", ["-c", command], { stdio: "pipe" }); + + expect(fs.existsSync(path.join(workspace, "stale"))).toBe(false); + expect(fs.statSync(workspace).mode & 0o777).toBe(0o770); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("does not clear a declared dir that is a symlink", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-stale-symlink-")); + try { + const realDir = path.join(root, "real"); + fs.mkdirSync(realDir); + fs.writeFileSync(path.join(realDir, "keep"), "1"); + const workspace = path.join(root, "workspace"); + fs.symlinkSync(realDir, workspace); + + const command = buildRestoreCleanupCommand(root, [], [], new Set(), ["workspace"]); + execFileSync("bash", ["-c", command], { stdio: "pipe" }); + + expect(fs.lstatSync(workspace).isSymbolicLink()).toBe(true); + expect(fs.existsSync(path.join(realDir, "keep"))).toBe(true); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/state/openclaw-managed-extensions.ts b/src/lib/state/openclaw-managed-extensions.ts index 2a07e2c75d5..55de115a9d4 100644 --- a/src/lib/state/openclaw-managed-extensions.ts +++ b/src/lib/state/openclaw-managed-extensions.ts @@ -136,11 +136,21 @@ function buildOpenClawExtensionsCleanupCommand( ].join(" && "); } +function buildStaleStateDirContentsCleanupCommand(dir: string, dirName: string): string { + const target = shellQuote(`${dir}/${dirName}`); + return ( + `d=${target}; ` + + 'if [ -d "$d" ] && [ ! -L "$d" ]; then ' + + 'find "$d" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; fi' + ); +} + export function buildRestoreCleanupCommand( dir: string, localDirs: readonly string[], managedExtensionDirs: readonly string[], requiredExtensionDirs: ReadonlySet, + staleContentDirs: readonly string[] = [], ): string { const preserveManagedExtensions = managedExtensionDirs.length > 0; const commands: string[] = []; @@ -153,5 +163,11 @@ export function buildRestoreCleanupCommand( buildOpenClawExtensionsCleanupCommand(dir, managedExtensionDirs, requiredExtensionDirs), ); } + const localDirSet = new Set(localDirs); + for (const dirName of staleContentDirs) { + if (localDirSet.has(dirName)) continue; + if (preserveManagedExtensions && dirName === "extensions") continue; + commands.push(buildStaleStateDirContentsCleanupCommand(dir, dirName)); + } return commands.length > 0 ? commands.join(" && ") : ":"; } diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 0a98ddb7b79..af6040273a7 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -89,6 +89,8 @@ export interface RebuildManifest { stateDirs: string[]; /** Directories verified as safe to restore. Absent on older manifests. */ backedUpDirs?: string[]; + /** Declared directories that could not be backed up. Absent on older manifests. */ + failedBackupDirs?: string[]; stateFiles?: StateFileSpec[]; /** Single config/state directory */ dir: string; @@ -270,6 +272,8 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { (value.agentVersion === null || typeof value.agentVersion === "string") && (value.expectedVersion === null || typeof value.expectedVersion === "string") && (value.backedUpDirs === undefined || isBackedUpDirArray(value.backedUpDirs, value.stateDirs)) && + (value.failedBackupDirs === undefined || + isBackedUpDirArray(value.failedBackupDirs, value.stateDirs)) && typeof dir === "string" && (value.openclawImagePluginInstalls === undefined || parseOpenClawImagePluginInstalls(value.openclawImagePluginInstalls, dir).ok) && @@ -983,6 +987,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = ? { reconcileOpenClawImagePluginProvenance: true } : {}), stateDirs, + failedBackupDirs: [], stateFiles, dir, backupPath, @@ -1322,6 +1327,9 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = ); } manifest.backedUpDirs = backedUpDirs; + manifest.failedBackupDirs = failedDirs.filter((failedDir) => + manifest.stateDirs.includes(failedDir), + ); writeManifest(backupPath, manifest); manifest.backupPath = backupPath; @@ -1475,6 +1483,21 @@ function restoreSandboxStateInternal( localDirs.splice(localDirs.indexOf(d), 1); } } + // Only manifests that distinguish failed backups from absent directories can + // authorize cleanup without deleting data that a failed backup did not capture. + // Older manifests leave this field absent, so preserve their historical restore behavior. + const failedBackupDirs = new Set(manifest.failedBackupDirs ?? []); + const localDirSet = new Set(localDirs); + const staleContentDirs = + manifest.failedBackupDirs === undefined + ? [] + : manifest.stateDirs.filter( + (stateDir) => + !targetRuntimeAuthDirs.has(stateDir) && + !localDirSet.has(stateDir) && + !failedBackupDirs.has(stateDir), + ); + const cleanupStateDirs = [...new Set([...localDirs, ...staleContentDirs])]; const targetStateFiles = new Map(); for (const targetFile of targetAgent.stateFiles) { const normalized = normalizeStateFilePath(targetFile.path); @@ -1534,7 +1557,7 @@ function restoreSandboxStateInternal( freshOpenClawImagePluginInstalls = discovery.pluginInstalls; } - if (localDirs.length === 0 && localFiles.length === 0) { + if (cleanupStateDirs.length === 0 && localFiles.length === 0) { _log("No dirs or files to restore"); return { success: true, restoredDirs, failedDirs, restoredFiles, failedFiles }; } @@ -1546,7 +1569,7 @@ function restoreSandboxStateInternal( return { success: false, restoredDirs, - failedDirs: [...localDirs], + failedDirs: [...cleanupStateDirs], restoredFiles, failedFiles: localFiles.map((f) => f.path), }; @@ -1571,7 +1594,7 @@ function restoreSandboxStateInternal( const pluginRestorePlan = planOpenClawPluginRestore({ agentType: manifest.agentType, dir, - localDirs, + localDirs: cleanupStateDirs, freshImagePluginInstalls: freshOpenClawImagePluginInstalls, previousImagePluginInstalls: previousOpenClawImagePluginInstalls, }); @@ -1579,7 +1602,7 @@ function restoreSandboxStateInternal( return { success: false, restoredDirs, - failedDirs: [...localDirs], + failedDirs: [...cleanupStateDirs], restoredFiles, failedFiles: localFiles.map((f) => f.path), error: @@ -1600,6 +1623,7 @@ function restoreSandboxStateInternal( ); } + let restoreTar: Buffer | undefined; if (localDirs.length > 0) { // Upload via tar pipe // NC-2227-04: Removed -h flag from restore as well — no symlink following. @@ -1617,22 +1641,26 @@ function restoreSandboxStateInternal( return { success: false, restoredDirs, - failedDirs: [...localDirs], + failedDirs: [...cleanupStateDirs], restoredFiles, failedFiles: localFiles.map((f) => f.path), }; } + restoreTar = tarResult.stdout; + } - // Remove existing state dirs before extracting so stale files from later - // snapshots don't persist after restoring an earlier one. OpenClaw's - // image-managed extensions are preserved from the freshly built image and - // excluded from the restore tar; only user/non-managed extension entries - // are cleared and restored from the backup. + // Remove existing state dirs before extracting so stale files from later + // snapshots don't persist after restoring an earlier one. OpenClaw's + // image-managed extensions are preserved from the freshly built image and + // excluded from the restore tar; only user/non-managed extension entries + // are cleared and restored from the backup. + if (cleanupStateDirs.length > 0) { const rmCmd = buildRestoreCleanupCommand( dir, localDirs, pluginRestorePlan.preservedExtensionDirs, new Set(pluginRestorePlan.requiredFreshExtensionDirs), + staleContentDirs, ); _log(`Cleaning target dirs before restore: ${rmCmd}`); const rmResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), rmCmd], { @@ -1649,15 +1677,17 @@ function restoreSandboxStateInternal( return { success: false, restoredDirs, - failedDirs: [...localDirs], + failedDirs: [...cleanupStateDirs], restoredFiles, failedFiles: localFiles.map((f) => f.path), }; } + } + if (restoreTar !== undefined) { const extractCmd = `tar --no-same-owner -xf - -C ${shellQuote(dir)}`; const sshResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), extractCmd], { - input: tarResult.stdout, + input: restoreTar, stdio: ["pipe", "pipe", "pipe"], timeout: 120000, }); diff --git a/test/snapshot-stale-directory-restore.test.ts b/test/snapshot-stale-directory-restore.test.ts new file mode 100644 index 00000000000..9ca6d8c428e --- /dev/null +++ b/test/snapshot-stale-directory-restore.test.ts @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeEach, expect, it } from "vitest"; + +const ORIGINAL_HOME = process.env.HOME; +const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-stale-dir-restore-")); +process.env.HOME = TMP_HOME; +const sandboxState = await import("../src/lib/state/sandbox.js"); +const BACKUPS_ROOT = path.join(TMP_HOME, ".nemoclaw", "rebuild-backups"); + +afterAll(() => { + restoreEnv("HOME", ORIGINAL_HOME); + fs.rmSync(TMP_HOME, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(BACKUPS_ROOT, { recursive: true, force: true }); +}); + +function writeExecutable(filePath: string, source: string): void { + fs.writeFileSync(filePath, source, { mode: 0o755 }); +} + +function restoreEnv(name: string, value: string | undefined): void { + value === undefined + ? Reflect.deleteProperty(process.env, name) + : Reflect.set(process.env, name, value); +} + +function writeSandboxRegistry(sandboxName: string, agent: string | null = null): void { + const stateRoot = path.join(TMP_HOME, ".nemoclaw"); + fs.mkdirSync(stateRoot, { recursive: true }); + fs.writeFileSync( + path.join(stateRoot, "sandboxes.json"), + JSON.stringify({ + defaultSandbox: sandboxName, + sandboxes: { + [sandboxName]: { + name: sandboxName, + model: "m", + provider: "p", + gpuEnabled: false, + policies: [], + agent, + }, + }, + }), + ); +} + +function writeFakeOpenshell(binDir: string): string { + const openshell = path.join(binDir, "openshell"); + writeExecutable( + openshell, + `#!/usr/bin/env node +const args = process.argv.slice(2); +if (args[0] === "sandbox" && args[1] === "ssh-config") { + process.stdout.write("Host openshell-alpha\\n HostName 127.0.0.1\\n User sandbox\\n"); +} +process.exit(0); +`, + ); + return openshell; +} + +it("clears snapshot-declared absent directories while preserving target-only state (#7428)", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-absent-dirs-")); + const oldPath = process.env.PATH; + const oldOpenshell = process.env.NEMOCLAW_OPENSHELL_BIN; + try { + const binDir = path.join(fixture, "bin"); + const sshLog = path.join(fixture, "ssh-log.jsonl"); + fs.mkdirSync(binDir, { recursive: true }); + + const openshell = writeFakeOpenshell(binDir); + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const fs = require("node:fs"); +const cmd = process.argv[process.argv.length - 1] || ""; +fs.appendFileSync(${JSON.stringify(sshLog)}, JSON.stringify({ cmd }) + "\\n"); +if (cmd.includes("[ -d ") && cmd.includes("printf")) { + process.exit(0); +} +if (cmd.includes("openclaw.json") && cmd.includes("cat --")) { + process.exit(2); +} +if (cmd.includes("rm -rf")) { + process.exit(0); +} +process.exit(0); +`, + ); + + writeSandboxRegistry("alpha"); + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}${path.delimiter}${oldPath || ""}`; + + const backup = sandboxState.backupSandboxState("alpha"); + expect(backup.success).toBe(true); + expect(backup.manifest?.backedUpDirs).toEqual([]); + expect(backup.manifest?.failedBackupDirs).toEqual([]); + const manifestPath = path.join(backup.manifest!.backupPath, "rebuild-manifest.json"); + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8")); + expect(manifest.stateDirs).toContain("agents"); + manifest.stateDirs = manifest.stateDirs.filter((stateDir: string) => stateDir !== "agents"); + fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + + const restore = sandboxState.restoreSandboxState("alpha", backup.manifest!.backupPath); + expect(restore.success).toBe(true); + expect(restore.restoredDirs).toEqual([]); + + const loggedCommands = fs + .readFileSync(sshLog, "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line).cmd as string); + const cleanupCommand = loggedCommands.find((cmd) => + cmd.includes("d='/sandbox/.openclaw/workspace'"), + ); + expect(cleanupCommand).toBeDefined(); + expect(cleanupCommand).toContain("! -name 'nemoclaw'"); + expect(cleanupCommand).toContain("! -name 'openclaw-weixin'"); + expect(cleanupCommand).not.toContain("rm -rf -- '/sandbox/.openclaw/extensions'"); + expect(cleanupCommand).not.toContain("d='/sandbox/.openclaw/extensions'"); + expect(loggedCommands).not.toEqual( + expect.arrayContaining([expect.stringContaining("d='/sandbox/.openclaw/agents'")]), + ); + } finally { + restoreEnv("NEMOCLAW_OPENSHELL_BIN", oldOpenshell); + restoreEnv("PATH", oldPath); + fs.rmSync(fixture, { recursive: true, force: true }); + } +}); + +it("clears a Hermes directory declared absent by the snapshot (#7428)", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-absent-dir-")); + const oldPath = process.env.PATH; + const oldOpenshell = process.env.NEMOCLAW_OPENSHELL_BIN; + try { + const binDir = path.join(fixture, "bin"); + const workspaceMarker = path.join(fixture, "workspace-content"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync(workspaceMarker, "stale"); + + const openshell = writeFakeOpenshell(binDir); + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const fs = require("node:fs"); +const cmd = process.argv[process.argv.length - 1] || ""; +if (cmd.includes("[ -d ") && cmd.includes("printf")) { + process.exit(0); +} +if ( + cmd.includes("/sandbox/.hermes/SOUL.md") || + cmd.includes("/sandbox/.hermes/.hermes_history") || + cmd.includes("/sandbox/.hermes/runtime/state.db") || + cmd.includes("/sandbox/.hermes/kanban.db") +) { + process.exit(2); +} +if (cmd.includes("d='/sandbox/.hermes/workspace'")) { + fs.rmSync(${JSON.stringify(workspaceMarker)}, { force: true }); +} +process.exit(0); +`, + ); + + writeSandboxRegistry("alpha", "hermes"); + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}${path.delimiter}${oldPath || ""}`; + + const backup = sandboxState.backupSandboxState("alpha"); + expect(backup.success).toBe(true); + expect(backup.manifest?.stateDirs).toContain("workspace"); + expect(backup.manifest?.backedUpDirs).not.toContain("workspace"); + expect(backup.manifest?.failedBackupDirs).not.toContain("workspace"); + + const restore = sandboxState.restoreSandboxState("alpha", backup.manifest!.backupPath); + + expect(restore.success).toBe(true); + expect(fs.existsSync(workspaceMarker)).toBe(false); + } finally { + restoreEnv("NEMOCLAW_OPENSHELL_BIN", oldOpenshell); + restoreEnv("PATH", oldPath); + fs.rmSync(fixture, { recursive: true, force: true }); + } +}); + +it("preserves stale content for directories whose backup failed (#7428)", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-failed-dir-")); + const oldPath = process.env.PATH; + const oldOpenshell = process.env.NEMOCLAW_OPENSHELL_BIN; + try { + const binDir = path.join(fixture, "bin"); + const sshLog = path.join(fixture, "ssh-log.jsonl"); + const workspaceMarker = path.join(fixture, "workspace-content"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync(workspaceMarker, "preserve"); + + const openshell = writeFakeOpenshell(binDir); + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const fs = require("node:fs"); +const cmd = process.argv[process.argv.length - 1] || ""; +fs.appendFileSync(${JSON.stringify(sshLog)}, JSON.stringify({ cmd }) + "\\n"); +if (cmd.includes("[ -d ") && cmd.includes("printf")) { + process.exit(0); +} +if (cmd.includes("openclaw.json") && cmd.includes("cat --")) { + process.exit(2); +} +if (cmd.includes("d='/sandbox/.openclaw/workspace'")) { + fs.rmSync(${JSON.stringify(workspaceMarker)}, { force: true }); +} +process.exit(0); +`, + ); + + writeSandboxRegistry("alpha"); + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}${path.delimiter}${oldPath || ""}`; + + const backup = sandboxState.backupSandboxState("alpha"); + expect(backup.success).toBe(true); + const manifestPath = path.join(backup.manifest!.backupPath, "rebuild-manifest.json"); + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8")); + manifest.failedBackupDirs = ["workspace"]; + fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + + const restore = sandboxState.restoreSandboxState("alpha", backup.manifest!.backupPath); + + expect(restore.success).toBe(true); + const cleanupCommands = fs + .readFileSync(sshLog, "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line).cmd as string) + .filter((cmd) => cmd.includes("rm -rf")); + expect(cleanupCommands).not.toEqual( + expect.arrayContaining([expect.stringContaining("d='/sandbox/.openclaw/workspace'")]), + ); + expect(fs.existsSync(workspaceMarker)).toBe(true); + + Reflect.deleteProperty(manifest, "failedBackupDirs"); + fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + fs.writeFileSync(sshLog, ""); + const legacyRestore = sandboxState.restoreSandboxState("alpha", backup.manifest!.backupPath); + + expect(legacyRestore.success).toBe(true); + expect(fs.readFileSync(sshLog, "utf-8")).not.toContain("d='/sandbox/.openclaw/workspace'"); + expect(fs.existsSync(workspaceMarker)).toBe(true); + } finally { + restoreEnv("NEMOCLAW_OPENSHELL_BIN", oldOpenshell); + restoreEnv("PATH", oldPath); + fs.rmSync(fixture, { recursive: true, force: true }); + } +}); + +it("reports stale directories when restore cannot obtain SSH configuration (#7428)", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-stale-dir-ssh-failure-")); + const oldPath = process.env.PATH; + const oldOpenshell = process.env.NEMOCLAW_OPENSHELL_BIN; + try { + const binDir = path.join(fixture, "bin"); + fs.mkdirSync(binDir, { recursive: true }); + const openshell = writeFakeOpenshell(binDir); + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const cmd = process.argv[process.argv.length - 1] || ""; +if (cmd.includes("[ -d ") && cmd.includes("printf")) { + process.exit(0); +} +if (cmd.includes("openclaw.json") && cmd.includes("cat --")) { + process.exit(2); +} +process.exit(0); +`, + ); + + writeSandboxRegistry("alpha"); + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}${path.delimiter}${oldPath || ""}`; + const backup = sandboxState.backupSandboxState("alpha"); + expect(backup.success).toBe(true); + expect(backup.manifest?.failedBackupDirs).toEqual([]); + + const failingOpenshell = path.join(binDir, "openshell-fail"); + writeExecutable(failingOpenshell, "#!/usr/bin/env node\nprocess.exit(1);\n"); + process.env.NEMOCLAW_OPENSHELL_BIN = failingOpenshell; + const restore = sandboxState.restoreSandboxState("alpha", backup.manifest!.backupPath); + + expect(restore.success).toBe(false); + expect(restore.failedDirs).toEqual( + expect.arrayContaining(["agents", "extensions", "workspace"]), + ); + } finally { + restoreEnv("NEMOCLAW_OPENSHELL_BIN", oldOpenshell); + restoreEnv("PATH", oldPath); + fs.rmSync(fixture, { recursive: true, force: true }); + } +});