diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 675b147faed..dfc009e32af 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -109,7 +109,10 @@ After the installer launches `nemoclaw onboard`, the wizard runs preflight check It prints a review summary before it registers the provider with OpenShell. After you confirm, NemoClaw registers inference, prompts for optional web search and messaging channels, builds and starts the sandbox, sets up OpenClaw, then applies the selected network policy tier and presets. At any prompt, press Enter to accept the default shown in `[brackets]`, type `back` to return to the previous prompt, or type `exit` to quit. -If existing sandbox sessions are running, the installer warns before onboarding because the setup can rebuild or upgrade sandboxes after the new sandbox launches. +If registered sandboxes already exist, the installer runs `nemoclaw backup-all` when the installed CLI supports it, then runs `nemoclaw upgrade-sandboxes --auto` before generic onboarding. +For a registered sandbox that is non-Ready after the host upgrade, the installer restores its validated latest backup only when the backup identity matches and its registry entry has a NemoClaw-managed image fingerprint. +Pre-fingerprint and custom-image sandboxes are not recreated automatically because matching agent versions do not prove image provenance. +If an automatic rebuild fails, or a non-Ready recovery is blocked or fails, the installer exits with a nonzero status and does not start generic onboarding. The inference provider prompt presents a numbered list. diff --git a/docs/manage-sandboxes/lifecycle.mdx b/docs/manage-sandboxes/lifecycle.mdx index 3da9707351d..b29a819bc5f 100644 --- a/docs/manage-sandboxes/lifecycle.mdx +++ b/docs/manage-sandboxes/lifecycle.mdx @@ -262,6 +262,9 @@ $$nemoclaw upgrade-sandboxes --check ``` Before upgrade work, the installer runs `$$nemoclaw backup-all` when the installed CLI supports it. +After the host CLI and OpenShell update, the installer runs `$$nemoclaw upgrade-sandboxes --auto` before generic onboarding. +If an existing sandbox is non-Ready, the automatic path requires a validated latest backup whose sandbox and agent identity match the registry and positive evidence that NemoClaw managed the image. +The installer attempts every eligible recovery, exits with a nonzero status if any recovery fails, and does not continue to generic onboarding after that failure. For manual upgrade flows, create a snapshot first and then run the update or rebuild command you need: ```bash diff --git a/scripts/install.sh b/scripts/install.sh index eea0c416a06..920c74b30a9 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -532,7 +532,7 @@ print_done() { local _needs_cli_refresh=false needs_shell_reload && _needs_cli_refresh=true - # #5735: do not claim a clean install when the post-onboard auto-upgrade of a + # #5735: do not claim a clean install when the automatic upgrade of a # pre-existing sandbox failed (it may have been destroyed before its recreate # failed). Surface an explicit incomplete/recovery status instead. if [[ "${_UPGRADE_SANDBOXES_FAILED:-false}" == true ]]; then @@ -909,7 +909,7 @@ ONBOARD_RAN=false # auto-onboarding (#3276). _CLI_PATH="" _PREEXISTING_SANDBOX_COUNT=0 -# #5735: set when the post-onboard auto-upgrade of pre-existing sandboxes +# #5735: set when automatic recovery/upgrade of pre-existing sandboxes # reported a failure. A failed/destructive rebuild must not be reported as a # clean install, so print_done downgrades the final banner when this is true. _UPGRADE_SANDBOXES_FAILED=false @@ -1867,12 +1867,12 @@ EOF preinstall_backup_and_retire_legacy_gateway() { local reg_file="${HOME}/.nemoclaw/sandboxes.json" [ -f "$reg_file" ] || return 0 - command_exists openshell || return 0 local sandbox_count sandbox_count="$(registered_sandbox_count)" _PREEXISTING_SANDBOX_COUNT="$sandbox_count" [ "$sandbox_count" -gt 0 ] 2>/dev/null || return 0 + command_exists openshell || return 0 if [[ "${NEMOCLAW_SINGLE_SESSION:-}" == "1" ]]; then error "Aborting — NEMOCLAW_SINGLE_SESSION is set. Destroy existing sessions with '${_CLI_BIN} destroy' before reinstalling." @@ -2147,6 +2147,27 @@ run_installer_host_preflight() { [[ "$status" -ne 10 ]] } +recover_preexisting_sandboxes_before_onboard() { + local cli_runner="$1" + if [ "${_PREEXISTING_SANDBOX_COUNT:-0}" -le 0 ] 2>/dev/null; then + return 0 + fi + + info "Recovering and upgrading pre-existing sandboxes before onboarding…" + # `--auto` is the existing non-interactive maintenance path. When the + # pre-upgrade backup signal is present, the CLI also recovers registered + # non-Ready sandboxes from their validated latest backup. It attempts every + # eligible sandbox before returning non-zero for any failure. + if "$cli_runner" upgrade-sandboxes --auto 2>&1; then + return 0 + fi + + _UPGRADE_SANDBOXES_FAILED=true + warn "One or more existing sandboxes could not be recovered automatically." + warn "Generic onboarding will not run; review the affected sandbox and preserved backup diagnostics above." + return 1 +} + run_onboard() { show_usage_notice info "Running ${_CLI_BIN} onboard…" @@ -2726,23 +2747,12 @@ main() { warn "Set NEMOCLAW_SINGLE_SESSION=1 to abort the installer when sessions are active." fi if run_installer_host_preflight; then + if ! recover_preexisting_sandboxes_before_onboard "$_cli_runner"; then + finalize_install + return 1 + fi run_onboard || error "Onboarding did not complete successfully." ONBOARD_RAN=true - # After onboard, check for stale sandboxes that need rebuilding (#1904). - # Uses --auto so it runs non-interactively in piped/CI contexts. - if [ "${_PREEXISTING_SANDBOX_COUNT:-0}" -gt 0 ] 2>/dev/null && [ -n "$_cli_runner" ]; then - info "Checking for sandboxes that need upgrading…" - # #5735: a non-zero exit here can mean an existing sandbox was rebuilt - # destructively and its recreate failed. Record it so print_done reports - # the install as incomplete with recovery guidance instead of a clean - # banner. The CLI already prints the affected sandbox name and the - # preserved backup path on failure. - if ! "$_cli_runner" upgrade-sandboxes --auto 2>&1; then - _UPGRADE_SANDBOXES_FAILED=true - warn "One or more existing sandboxes could not be upgraded automatically." - warn "Review the messages above — affected sandboxes may need '${_CLI_BIN} onboard --resume' or '${_CLI_BIN} rebuild', and any backup path shown above can restore workspace state." - fi - fi restore_onboard_forward_after_post_checks || error "Hermes host forward restore failed." elif [ "${NON_INTERACTIVE:-}" = "1" ]; then error "Skipping onboarding until the host prerequisites above are fixed." @@ -2757,9 +2767,8 @@ main() { } # Print the completion summary, then propagate a fatal/non-zero result when the -# post-onboard auto-upgrade of a pre-existing sandbox failed (#5735, PRA-5). The -# new sandbox may have onboarded fine, but a failed auto-upgrade can have left an -# *existing* sandbox destroyed or backup-only, so the install must not be +# automatic recovery of a pre-existing sandbox failed (#5735, PRA-5). A failed +# recovery can have left an existing sandbox destroyed or backup-only, so the install must not be # reported as success. print_done() has already shown the affected sandbox and # recovery guidance (and the "completed with warnings" banner); exiting non-zero # here is what keeps automation and operators from treating it as a clean diff --git a/src/lib/actions/sandbox/rebuild-flow.test.ts b/src/lib/actions/sandbox/rebuild-flow.test.ts index d843ecfa6f9..e8b0aaedc84 100644 --- a/src/lib/actions/sandbox/rebuild-flow.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow.test.ts @@ -52,7 +52,14 @@ type RebuildFlowOverrides = { buildMessagingRebuildPlan?: () => Promise | unknown; sandboxEntry?: Record; sessionSandboxName?: string; + sandboxListOutput?: string; backupPolicyPresets?: string[]; + preDeleteSandboxEntry?: Record; + preDeleteDefaultSandbox?: string | null; + preDeleteLatestManifest?: Record | null; + recoveryManifestValidation?: ( + manifest: Record, + ) => { ok: true; manifest: Record } | { ok: false; reason: string }; }; type RebuildFlowHarness = { @@ -68,6 +75,7 @@ type RebuildFlowHarness = { registryUpdateSpy: MockInstance; releaseOnboardLockSpy: MockInstance; relockSpy: MockInstance; + restoreSandboxEntrySpy: MockInstance; restoreSandboxStateSpy: MockInstance; runOpenshellSpy: MockInstance; messagingRebuildPlanSpy: MockInstance; @@ -193,7 +201,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ - result: { status: 0, output: "alpha Ready" }, + result: { status: 0, output: overrides.sandboxListOutput ?? "alpha Ready" }, }); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); vi.spyOn(agentDefs, "loadAgent").mockReturnValue(agentDef); @@ -212,17 +220,36 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild .mockImplementation(() => undefined); const markStepFailedSpy = installTerminalStepFailureMock(onboardSession, session); session.sandboxName = overrides.sessionSandboxName ?? session.sandboxName; - vi.spyOn(registry, "getSandbox").mockReturnValue({ + const sandboxEntry = { name: "alpha", provider: "ollama-local", model: "nvidia/nemotron", policies: ["npm"], agent: null, + agentVersion: "0.1.0", nimContainer: null, ...(overrides.sandboxEntry ?? {}), + }; + vi.spyOn(registry, "getSandbox").mockReturnValue(sandboxEntry); + let registryLoadCount = 0; + vi.spyOn(registry, "load").mockImplementation(() => { + const isPreDeleteRead = registryLoadCount > 0; + registryLoadCount++; + return { + defaultSandbox: isPreDeleteRead ? (overrides.preDeleteDefaultSandbox ?? "alpha") : "alpha", + sandboxes: { + alpha: + isPreDeleteRead && overrides.preDeleteSandboxEntry + ? overrides.preDeleteSandboxEntry + : sandboxEntry, + }, + }; }); vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [] }); const registryUpdateSpy = vi.spyOn(registry, "updateSandbox").mockImplementation(() => undefined); + const restoreSandboxEntrySpy = vi + .spyOn(registry, "restoreSandboxEntry") + .mockImplementation(() => undefined); vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ detected: false, sessions: [], @@ -251,6 +278,19 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild policyPresets: overrides.backupPolicyPresets ?? ["npm", "bad", "throw"], }, }); + vi.spyOn(sandboxState, "validateRebuildRecoveryManifest").mockImplementation( + (...args: unknown[]) => { + const manifest = args[2] as Record; + return overrides.recoveryManifestValidation?.(manifest) ?? { ok: true as const, manifest }; + }, + ); + vi.spyOn(sandboxState, "getLatestBackup").mockImplementation( + () => + (overrides.preDeleteLatestManifest === undefined + ? makePreparedRecoveryManifest() + : overrides.preDeleteLatestManifest) as ReturnType, + ); + vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence").mockReturnValue(true); const restoreSandboxStateSpy = vi.spyOn(sandboxState, "restoreSandboxState").mockImplementation( overrides.restoreSandboxState ?? (() => ({ @@ -286,6 +326,8 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild vi.spyOn(shields, "repairMutableConfigPerms").mockImplementation( overrides.repairMutableConfigPerms ?? (() => ({ applied: true, verified: true, errors: [] })), ); + vi.spyOn(shields, "isShieldsDown").mockReturnValue(true); + vi.spyOn(shields, "clearShieldsState").mockImplementation(() => undefined); const messagingRebuildPlanSpy = vi .spyOn(messaging.MessagingWorkflowPlanner.prototype, "buildRebuildPlanFromSandboxEntry") .mockImplementation(overrides.buildMessagingRebuildPlan ?? (() => null)); @@ -310,6 +352,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild registryUpdateSpy, releaseOnboardLockSpy, relockSpy, + restoreSandboxEntrySpy, restoreSandboxStateSpy, runOpenshellSpy, messagingRebuildPlanSpy, @@ -387,6 +430,25 @@ function makeActiveTeamsMessagingPlan() { }; } +function makePreparedRecoveryManifest() { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-07-01T06-50-42-044Z", + agentType: "openclaw", + agentVersion: "0.1.0", + expectedVersion: "0.2.0", + stateDirs: ["workspace"], + backedUpDirs: ["workspace"], + stateFiles: [], + dir: "/sandbox/.openclaw", + backupPath: "/tmp/rebuild-backups/alpha/2026-07-01T06-50-42-044Z", + blueprintDigest: null, + policyPresets: ["npm"], + customPolicies: [], + }; +} + describe("rebuildSandbox flow", () => { beforeEach(() => { delete process.env.NEMOCLAW_SANDBOX_NAME; @@ -443,6 +505,175 @@ describe("rebuildSandbox flow", () => { ); }); + it("restores the validated pre-upgrade manifest without taking a second backup (#6114)", async () => { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxListOutput: "alpha Error", + }); + const recoveryManifest = makePreparedRecoveryManifest(); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest, + }), + ).resolves.toBeUndefined(); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith( + "alpha", + recoveryManifest.backupPath, + ); + }); + + it("rejects a mismatched prepared manifest before deleting the sandbox (#6114)", async () => { + const harness = createRebuildFlowHarness({ + recoveryManifestValidation: () => ({ + ok: false, + reason: "manifest sandbox 'beta' does not match 'alpha'", + }), + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Invalid recovery manifest"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("revalidates the prepared manifest immediately before deleting the sandbox (#6114)", async () => { + let validationCount = 0; + const harness = createRebuildFlowHarness({ + recoveryManifestValidation: (manifest) => { + validationCount++; + return validationCount === 1 + ? { ok: true as const, manifest } + : { ok: false as const, reason: "persisted backup identity changed during validation" }; + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Invalid recovery manifest"); + + expect(validationCount).toBe(2); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("rejects same-agent registry configuration drift before deleting the sandbox (#6114)", async () => { + const harness = createRebuildFlowHarness({ + preDeleteSandboxEntry: { + name: "alpha", + provider: "compatible-endpoint", + model: "new-model", + policies: ["npm", "github"], + agent: null, + agentVersion: "0.1.0", + nemoclawVersion: "0.0.71", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recovery registry configuration changed during preflight"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("uses the single refreshed registry snapshot for recreate rollback (#6114)", async () => { + const harness = createRebuildFlowHarness({ + preDeleteDefaultSandbox: "beta", + onboard: () => { + throw new Error("recreate failed"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( + expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), + { reclaimDefault: null }, + ); + }); + + it("rejects a latest-backup change immediately before deleting the sandbox (#6114)", async () => { + const harness = createRebuildFlowHarness({ + preDeleteLatestManifest: { + ...makePreparedRecoveryManifest(), + timestamp: "2026-07-01T07-00-00-000Z", + backupPath: "/tmp/rebuild-backups/alpha/2026-07-01T07-00-00-000Z", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recovery backup identity changed during preflight"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("restores the registry entry when prepared-backup recreation fails (#6114)", async () => { + const harness = createRebuildFlowHarness({ + onboard: () => { + throw new Error("recreate failed"); + }, + }); + const recoveryManifest = makePreparedRecoveryManifest(); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest, + }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( + expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), + { reclaimDefault: "alpha" }, + ); + expect(harness.restoreSandboxStateSpy).not.toHaveBeenCalled(); + }); + it("restores enabled messaging presets while pruning disabled ones from final policies", async () => { const disabledSlackPlan = { schemaVersion: 1, diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 309843fd9a2..0d323decc82 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isDeepStrictEqual } from "node:util"; + import { CLI_NAME } from "../../cli/branding"; import { prompt as askPrompt } from "../../credentials/store"; import { @@ -577,10 +579,99 @@ async function reapplyMessagingManifestAfterOpenClawDoctor( * `Dockerfile.base` changes fail before destructive work and are applied to the * recreated sandbox image. */ +interface RebuildSandboxExecutionOptions { + throwOnError?: boolean; + /** Internal installer recovery input; never exposed as a CLI option. */ + recoveryManifest?: sandboxState.RebuildManifest; +} + +type RebuildBail = (message: string, code?: number) => never; + +function failPreparedRecoveryPreDelete( + detail: string, + errorMessage: string, + bail: RebuildBail, +): never { + console.error(""); + console.error(` ${_RD}Recovery pre-delete check failed:${R} ${detail}.`); + console.error(" Sandbox is untouched — no data was lost."); + return bail(errorMessage); +} + +function revalidatePreparedRecoveryBeforeDelete( + sandboxName: string, + initialEntry: RebuildSandboxEntry, + candidate: sandboxState.RebuildManifest | null, + registrySnapshot: registry.SandboxRegistry | null, + bail: RebuildBail, +): { + manifest: sandboxState.RebuildManifest | null; + registrySnapshot: registry.SandboxRegistry | null; +} { + if (!candidate) return { manifest: null, registrySnapshot }; + + const refreshedRegistrySnapshot = JSON.parse( + JSON.stringify(registry.load()), + ) as registry.SandboxRegistry; + const currentEntry = refreshedRegistrySnapshot.sandboxes[sandboxName]; + if (!currentEntry) { + return failPreparedRecoveryPreDelete( + "registry entry no longer exists", + "Recovery registry identity changed during preflight.", + bail, + ); + } + if (!isDeepStrictEqual(currentEntry, initialEntry)) { + return failPreparedRecoveryPreDelete( + "registered sandbox configuration changed during preflight", + "Recovery registry configuration changed during preflight.", + bail, + ); + } + + const latestManifest = sandboxState.getLatestBackup(sandboxName); + if ( + !latestManifest || + latestManifest.timestamp !== candidate.timestamp || + latestManifest.backupPath !== candidate.backupPath + ) { + return failPreparedRecoveryPreDelete( + "latest prepared backup changed during preflight", + "Recovery backup identity changed during preflight.", + bail, + ); + } + + const validation = sandboxState.validateRebuildRecoveryManifest( + sandboxName, + currentEntry.agent, + latestManifest, + ); + if (!validation.ok) { + return failPreparedRecoveryPreDelete( + validation.reason, + `Invalid recovery manifest: ${validation.reason}`, + bail, + ); + } + if (!sandboxState.hasPositiveManagedImageEvidence(currentEntry)) { + return failPreparedRecoveryPreDelete( + "registry no longer has a NemoClaw-managed image fingerprint", + "Recovery registry entry has no NemoClaw-managed image fingerprint.", + bail, + ); + } + + return { + manifest: validation.manifest, + registrySnapshot: refreshedRegistrySnapshot, + }; +} + export async function rebuildSandbox( sandboxName: string, options: string[] | RebuildSandboxOptions = {}, - opts: { throwOnError?: boolean } = {}, + opts: RebuildSandboxExecutionOptions = {}, ): Promise { const normalized = normalizeRebuildSandboxOptions(options); const verbose = normalized.verbose === true || process.env.NEMOCLAW_REBUILD_VERBOSE === "1"; @@ -588,7 +679,7 @@ export async function rebuildSandbox( const skipConfirm = normalized.yes === true || normalized.force === true; // When called from upgradeSandboxes in a loop, throwOnError prevents // process.exit from aborting the entire batch on the first failure. - const bail = opts.throwOnError + const bail: RebuildBail = opts.throwOnError ? (msg: string, _code = 1) => { throw new Error(msg); } @@ -600,6 +691,35 @@ export async function rebuildSandbox( const sb = getRebuildSandboxEntryOrBail(sandboxName, bail); if (!sb) return; + let recoveryManifest: sandboxState.RebuildManifest | null = null; + if (opts.recoveryManifest) { + const validation = sandboxState.validateRebuildRecoveryManifest( + sandboxName, + sb.agent, + opts.recoveryManifest, + ); + if (!validation.ok) { + console.error(""); + console.error(` ${_RD}Recovery preflight failed:${R} ${validation.reason}.`); + console.error(" Sandbox is untouched — no data was lost."); + bail(`Invalid recovery manifest: ${validation.reason}`); + return; + } + if (!sandboxState.hasPositiveManagedImageEvidence(sb)) { + console.error(""); + console.error( + ` ${_RD}Recovery preflight failed:${R} registry has no NemoClaw-managed image fingerprint.`, + ); + console.error( + " Pre-fingerprint and custom-image sandboxes are not recreated automatically.", + ); + console.error(" Sandbox is untouched — no data was lost."); + bail("Recovery registry entry has no NemoClaw-managed image fingerprint."); + return; + } + recoveryManifest = validation.manifest; + } + // Multi-agent guard (temporary — until swarm lands) if (!isSingleAgentRebuildSupported(sb, bail)) return; @@ -650,7 +770,15 @@ export async function rebuildSandbox( // Step 1: Ensure sandbox is live for backup, or identify stale-sandbox recovery. const liveState = await resolveRebuildLiveState(sandboxName, sb, log, bail); if (!liveState) return; - const { staleRecovery, staleRegistrySnapshot } = liveState; + const { staleRecovery } = liveState; + const preparedBackupRecovery = recoveryManifest !== null; + const recoveryRecreate = staleRecovery || preparedBackupRecovery; + // A prepared pre-upgrade backup can recover a sandbox that still appears in + // OpenShell but is stuck in Provisioning/Error. Capture the same registry + // rollback state used by missing-live-sandbox recovery before deletion. + let recoveryRegistrySnapshot = preparedBackupRecovery + ? JSON.parse(JSON.stringify(registry.load())) + : liveState.staleRegistrySnapshot; // Build agent base layers before backup/delete so Dockerfile.base errors leave // the existing sandbox intact. This is what applies local Hermes version edits. @@ -661,7 +789,7 @@ export async function rebuildSandbox( // clearing old shields state until recreate succeeds (#4497). const { rebuildShieldsWindow, staleSandboxWasLocked } = openRebuildShieldsWindowForState( sandboxName, - staleRecovery, + recoveryRecreate, ); if (!rebuildShieldsWindow) return bail("Failed to auto-unlock shields."); @@ -671,15 +799,32 @@ export async function rebuildSandbox( let sandboxStillExists = true; try { - // Step 2: Backup (skipped on stale-sandbox recovery -- no live state exists) - const backupManifest = backupSandboxStateForRebuild( + // Re-read the prepared manifest immediately before the destructive phase. + // Base-image builds and other preflight work can take long enough that the + // on-disk backup may have been replaced since the initial validation. + const preDeleteRecovery = revalidatePreparedRecoveryBeforeDelete( sandboxName, sb, - staleRecovery, - log, - relockShieldsIfNeeded, + recoveryManifest, + recoveryRegistrySnapshot, bail, ); + recoveryManifest = preDeleteRecovery.manifest; + recoveryRegistrySnapshot = preDeleteRecovery.registrySnapshot; + + // Step 2: Backup (skipped on stale-sandbox recovery -- no live state exists) + // Installer recovery already has a validated pre-upgrade backup. Reuse it + // instead of trying to reach a non-Ready sandbox to create a second backup. + const backupManifest = + recoveryManifest ?? + backupSandboxStateForRebuild( + sandboxName, + sb, + staleRecovery, + log, + relockShieldsIfNeeded, + bail, + ); if (backupManifest === undefined) return; // Step 3: Delete sandbox without tearing down gateway or session. @@ -896,32 +1041,30 @@ export async function rebuildSandbox( /* best effort */ } - // Stale-sandbox recovery had no backup to fall back on and already removed - // the registry entry before the recreate. If the recreate failed, restore - // the captured entry so the recommended `rebuild --yes` (and `connect`) + // Recovery already removed the registry entry before the recreate. If the + // recreate failed, restore the captured entry so the recommended + // `rebuild --yes` (and `connect`) // remain retryable instead of failing at dispatch with "not found in // registry" (#4497). Restore unconditionally — overwriting any partial entry // a failed `onboard` may have registered — so the original metadata // (defaultSandbox, customPolicies, every field) wins, not a half-written // recreate entry. The restore targets only this sandbox under the registry // lock, leaving other sandboxes' concurrent changes intact. - const snapshotEntry = staleRegistrySnapshot?.sandboxes?.[sandboxName]; - if (staleRecovery && snapshotEntry) { + const snapshotEntry = recoveryRegistrySnapshot?.sandboxes?.[sandboxName]; + if (recoveryRecreate && snapshotEntry) { try { registry.restoreSandboxEntry(snapshotEntry, { reclaimDefault: - staleRegistrySnapshot?.defaultSandbox === sandboxName ? sandboxName : null, + recoveryRegistrySnapshot?.defaultSandbox === sandboxName ? sandboxName : null, }); - log("Stale-recovery recreate failed: restored preserved registry entry for retry"); + log("Recovery recreate failed: restored preserved registry entry for retry"); } catch (err) { - log( - `Failed to restore registry entry after stale-recovery recreate failure: ${String(err)}`, - ); + log(`Failed to restore registry entry after recovery recreate failure: ${String(err)}`); } } console.error(""); - if (staleRecovery) { + if (recoveryRecreate) { console.error(` ${_RD}Recovery recreate failed.${R}`); console.error( " Your local registry entry has been preserved — you can retry once the issue above is fixed.", @@ -955,11 +1098,10 @@ export async function rebuildSandbox( return; } - // Recreate succeeded. For stale recovery, reset the now-stale shields state so - // the freshly recreated (mutable) sandbox reports its true posture instead of - // the gone sandbox's old lock seal. Deferred until here so a failed recreate - // above leaves the lockdown record intact for a retry (#4497). - if (staleRecovery) { + // Recreate succeeded. Reset the prior shields state so the freshly recreated + // (mutable) sandbox reports its true posture. Deferred until here so a failed + // recreate above leaves the lockdown record intact for a retry (#4497). + if (recoveryRecreate) { shields.clearShieldsState(sandboxName); } @@ -1182,15 +1324,15 @@ export async function rebuildSandbox( } console.log(""); - if ( + const postRestoreComplete = restoreSucceeded && !mutablePermsRepairUnverified && !mutableConfigHashRefreshUnverified && !messagingHostForwardUnverified && - !policyPresetRestoreIncomplete - ) { + !policyPresetRestoreIncomplete; + if (postRestoreComplete) { console.log(` ${G}\u2713${R} Sandbox '${sandboxName}' rebuilt successfully`); - if (staleRecovery) { + if (staleRecovery && !backupManifest) { console.log( ` ${D}Recovered from a stale registry entry \u2014 no prior workspace state was available to restore.${R}`, ); @@ -1235,11 +1377,16 @@ export async function rebuildSandbox( // Stale recovery reset the shields state to mutable (the gone sandbox's lock // seal could not carry over to the fresh image). If lockdown had been enabled, // tell the operator to re-apply it on the recreated sandbox (#4497). - if (staleRecovery && staleSandboxWasLocked) { + if (recoveryRecreate && staleSandboxWasLocked) { console.log( ` ${YW}\u26a0${R} Shields were previously enabled but the recreated sandbox starts unlocked \u2014 run \`${CLI_NAME} ${sandboxName} shields up\` to restore lockdown.`, ); } + if (preparedBackupRecovery && !postRestoreComplete) { + bail( + `Prepared backup recovery for '${sandboxName}' completed with unverified post-restore state.`, + ); + } } finally { if (!rebuildShieldsWindow.relocked) { relockShieldsIfNeeded(sandboxStillExists); diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts new file mode 100644 index 00000000000..939d037323a --- /dev/null +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +type UpgradeSandboxes = typeof import("./upgrade-sandboxes")["upgradeSandboxes"]; + +const requireDist = createRequire(import.meta.url); +const upgradeModulePath = "./upgrade-sandboxes.js"; + +// Warm the CommonJS source graph outside the first test's timeout. Each harness +// still reloads the entry module after installing its dependency spies. +requireDist(upgradeModulePath); +delete require.cache[requireDist.resolve(upgradeModulePath)]; + +function makeManifest(sandboxName: string) { + const timestamp = `2026-07-01T06-50-4${sandboxName.length}-044Z`; + return { + version: 1, + sandboxName, + timestamp, + agentType: "openclaw", + agentVersion: "2026.5.27", + expectedVersion: "2026.5.27", + stateDirs: ["workspace"], + backedUpDirs: ["workspace"], + stateFiles: [], + dir: "/sandbox/.openclaw", + backupPath: `/tmp/rebuild-backups/${sandboxName}/${timestamp}`, + blueprintDigest: null, + policyPresets: [], + customPolicies: [], + snapshotVersion: 1, + }; +} + +function createRecoveryHarness( + names: string[], + options: { + gatewayNames?: Record; + liveOutput?: string; + latestBackup?: ReturnType | null; + registryOverrides?: Record< + string, + Partial<{ + agent: "openclaw" | "hermes" | null; + agentVersion: string | null; + nemoclawVersion: string | null; + }> + >; + staleNames?: string[]; + useRealManagedEvidence?: boolean; + } = {}, +): { + upgradeSandboxes: UpgradeSandboxes; + rebuildSpy: ReturnType; + latestBackupSpy: ReturnType; + managedEvidenceSpy: ReturnType; +} { + delete require.cache[requireDist.resolve(upgradeModulePath)]; + vi.stubEnv("NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE", "1"); + + const gatewayDrift = requireDist("../adapters/openshell/gateway-drift.js"); + const coreVersion = requireDist("../core/version.js"); + const sandboxList = requireDist("../openshell-sandbox-list.js"); + const sandboxVersion = requireDist("../sandbox/version.js"); + const registry = requireDist("../state/registry.js"); + const sandboxState = requireDist("../state/sandbox.js"); + const rebuild = requireDist("./sandbox/rebuild.js"); + + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); + vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); + vi.spyOn(coreVersion, "getVersion").mockReturnValue("0.0.71"); + vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ + result: { + status: 0, + output: options.liveOutput ?? names.map((name) => `${name} Error`).join("\n"), + }, + recoveryAttempted: false, + recoverySucceeded: false, + }); + vi.spyOn(registry, "listSandboxes").mockReturnValue({ + sandboxes: names.map((name) => ({ + name, + agent: null, + agentVersion: "2026.5.27", + gatewayName: options.gatewayNames?.[name], + nemoclawVersion: "0.0.71", + ...options.registryOverrides?.[name], + })), + }); + vi.spyOn(sandboxVersion, "checkAgentVersion").mockImplementation((...args: unknown[]) => { + const name = String(args[0]); + return { + sandboxVersion: options.staleNames?.includes(name) === true ? "2026.5.26" : "2026.5.27", + expectedVersion: "2026.5.27", + isStale: options.staleNames?.includes(name) === true, + detectionMethod: "registry", + }; + }); + const latestBackupSpy = vi + .spyOn(sandboxState, "getLatestBackup") + .mockImplementation((...args: unknown[]) => + options.latestBackup === undefined ? makeManifest(String(args[0])) : options.latestBackup, + ); + vi.spyOn(sandboxState, "validateRebuildRecoveryManifest").mockImplementation( + (...args: unknown[]) => ({ + ok: true as const, + manifest: args[2] as ReturnType, + }), + ); + const managedEvidenceSpy = options.useRealManagedEvidence + ? vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence") + : vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence").mockReturnValue(true); + const rebuildSpy = vi.spyOn(rebuild, "rebuildSandbox").mockResolvedValue(undefined); + + return { + upgradeSandboxes: requireDist(upgradeModulePath).upgradeSandboxes, + rebuildSpy, + latestBackupSpy, + managedEvidenceSpy, + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + delete require.cache[requireDist.resolve(upgradeModulePath)]; +}); + +describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { + it("passes every non-Ready sandbox's validated manifest into rebuild", async () => { + const harness = createRecoveryHarness(["alpha", "beta"]); + + await expect(harness.upgradeSandboxes({ auto: true })).resolves.toBeUndefined(); + + expect(harness.rebuildSpy).toHaveBeenCalledTimes(2); + for (const name of ["alpha", "beta"]) { + expect(harness.rebuildSpy).toHaveBeenCalledWith(name, ["--yes"], { + throwOnError: true, + recoveryManifest: expect.objectContaining({ sandboxName: name }), + }); + } + }); + + it("continues through all eligible sandboxes before reporting a recovery failure", async () => { + const harness = createRecoveryHarness(["alpha", "beta"]); + harness.rebuildSpy.mockRejectedValueOnce(new Error("alpha failed")); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + + expect(harness.rebuildSpy).toHaveBeenCalledTimes(2); + expect(harness.rebuildSpy.mock.calls.map((call) => call[0])).toEqual(["alpha", "beta"]); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("Failed to recover 'alpha': alpha failed"), + ); + }); + + it("fails closed for a probed v0.0.55 custom image with matching backup agent version", async () => { + const probedAgentVersion = "2026.5.27"; + const harness = createRecoveryHarness(["custom-box"], { + latestBackup: { + ...makeManifest("custom-box"), + agentVersion: probedAgentVersion, + }, + registryOverrides: { + "custom-box": { + agentVersion: probedAgentVersion, + nemoclawVersion: null, + }, + }, + useRealManagedEvidence: true, + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + + expect(harness.managedEvidenceSpy).toHaveBeenCalledWith( + expect.objectContaining({ + agentVersion: probedAgentVersion, + nemoclawVersion: null, + }), + ); + expect(harness.rebuildSpy).not.toHaveBeenCalled(); + expect(exitSpy).toHaveBeenCalledWith(1); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("registry has no NemoClaw-managed image fingerprint"), + ); + }); + + it("warns and does not recover a stale registered sandbox absent from the selected gateway", async () => { + const harness = createRecoveryHarness(["registered-elsewhere"], { + gatewayNames: { "registered-elsewhere": "gateway-b" }, + liveOutput: "selected-gateway-box Ready", + latestBackup: null, + staleNames: ["registered-elsewhere"], + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).resolves.toBeUndefined(); + + expect(harness.rebuildSpy).not.toHaveBeenCalled(); + expect(harness.latestBackupSpy).not.toHaveBeenCalled(); + expect(console.log).toHaveBeenCalledWith( + expect.stringContaining("Skipping 1 sandbox(es) not observed on the selected gateway"), + ); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it("attempts both a live stale rebuild and a prepared non-Ready recovery", async () => { + const harness = createRecoveryHarness(["stale-box", "recovery-box"], { + liveOutput: "stale-box Ready\nrecovery-box Error", + staleNames: ["stale-box"], + }); + + await expect(harness.upgradeSandboxes({ auto: true })).resolves.toBeUndefined(); + + expect(harness.rebuildSpy).toHaveBeenCalledTimes(2); + expect(harness.rebuildSpy).toHaveBeenNthCalledWith(1, "stale-box", ["--yes"], { + throwOnError: true, + recoveryManifest: undefined, + }); + expect(harness.rebuildSpy).toHaveBeenNthCalledWith(2, "recovery-box", ["--yes"], { + throwOnError: true, + recoveryManifest: expect.objectContaining({ sandboxName: "recovery-box" }), + }); + }); + + it("fails closed for a live Error sandbox with no latest backup", async () => { + const harness = createRecoveryHarness(["broken-box"], { + latestBackup: null, + staleNames: ["broken-box"], + }); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + + expect(harness.rebuildSpy).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("broken-box")); + expect(console.log).not.toHaveBeenCalledWith( + expect.stringContaining("verify their recorded gateway or start them first"), + ); + }); + + it("continues after one live sandbox's backup assessment throws", async () => { + const harness = createRecoveryHarness(["alpha", "beta"]); + harness.latestBackupSpy + .mockImplementationOnce(() => { + throw new Error("ENOTDIR: unreadable backup root"); + }) + .mockImplementationOnce((name: string) => makeManifest(name)); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + + expect(harness.rebuildSpy).toHaveBeenCalledOnce(); + expect(harness.rebuildSpy).toHaveBeenCalledWith("beta", ["--yes"], { + throwOnError: true, + recoveryManifest: expect.objectContaining({ sandboxName: "beta" }), + }); + }); +}); diff --git a/src/lib/actions/upgrade-sandboxes.ts b/src/lib/actions/upgrade-sandboxes.ts index f0b066dfd1b..fb49da4d13a 100644 --- a/src/lib/actions/upgrade-sandboxes.ts +++ b/src/lib/actions/upgrade-sandboxes.ts @@ -24,9 +24,10 @@ import { captureSandboxListWithGatewayRecovery, printSandboxListFailureWithRecoveryContext, } from "../openshell-sandbox-list"; -import { parseReadySandboxNames } from "../runtime-recovery"; +import { parseLiveSandboxEntries, parseReadySandboxNames } from "../runtime-recovery"; import * as sandboxVersion from "../sandbox/version"; import * as registry from "../state/registry"; +import * as sandboxState from "../state/sandbox"; import { rebuildSandbox } from "./sandbox/rebuild"; // ── Upgrade sandboxes (#1904) ──────────────────────────────────── @@ -78,6 +79,53 @@ function describeStaleUpgrade(s: UpgradeSandboxCandidate): string { return parts.join("; "); } +type PreparedBackupRecovery = { + sandbox: registry.SandboxEntry; + manifest: sandboxState.RebuildManifest; +}; + +type RejectedBackupRecovery = { + sandbox: registry.SandboxEntry; + reason: string; +}; + +function prepareBackupRecovery( + sandbox: registry.SandboxEntry, +): PreparedBackupRecovery | RejectedBackupRecovery { + try { + const latest = sandboxState.getLatestBackup(sandbox.name); + if (!latest) { + return { sandbox, reason: "no validated pre-upgrade backup was found" }; + } + + const validation = sandboxState.validateRebuildRecoveryManifest( + sandbox.name, + sandbox.agent, + latest, + ); + if (!validation.ok) { + return { sandbox, reason: validation.reason }; + } + if (!sandboxState.hasPositiveManagedImageEvidence(sandbox)) { + return { + sandbox, + reason: + "registry has no NemoClaw-managed image fingerprint (pre-fingerprint and custom images are not auto-recreated)", + }; + } + return { sandbox, manifest: validation.manifest }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { sandbox, reason: `backup recovery assessment failed: ${detail}` }; + } +} + +function isPreparedBackupRecovery( + candidate: PreparedBackupRecovery | RejectedBackupRecovery, +): candidate is PreparedBackupRecovery { + return "manifest" in candidate; +} + export async function upgradeSandboxes( options: string[] | UpgradeSandboxesOptions = {}, ): Promise { @@ -116,6 +164,16 @@ export async function upgradeSandboxes( process.exit(liveResult.status || 1); } const liveNames = parseReadySandboxNames(liveResult.output || ""); + // Absence from the selected gateway is not evidence of failure: a registered + // sandbox may be Ready on another recorded gateway. Only an explicitly + // observed, known non-Ready phase is eligible for prepared-backup recovery. + const nonReadyLiveNames = new Set( + parseLiveSandboxEntries(liveResult.output || "") + .filter( + (entry) => entry.phase !== null && entry.phase !== "Ready" && entry.phase !== "Running", + ) + .map((entry) => entry.name), + ); // Classify sandboxes as stale, unknown, or current. Pass the running NemoClaw // build so a NemoClaw image/build change is detected even when the agent @@ -127,7 +185,35 @@ export async function upgradeSandboxes( { currentNemoclawVersion: resolveCurrentNemoclawVersion() }, ); - if (stale.length === 0 && unknown.length === 0) { + // Source boundary (#6114): a v0.0.55/legacy-OpenShell install can leave its + // already-registered sandboxes in Provisioning/Error after the host upgrade. + // That state comes from the already-installed legacy CLI/gateway and cannot be + // prevented at its source by this candidate. install.sh exports this signal only + // after that CLI completes backup-all, or after an operator asserts prepared + // upgrade state. Recovery remains limited to registry entries with a managed-image + // fingerprint; pre-fingerprint entries cannot prove provenance and fail closed. + // upgrade-sandboxes-recovery.test.ts and + // install-preexisting-sandbox-recovery.test.ts guard the handoff. Remove this + // bridge with onboard's matching consumer once prepared-backup installer recovery + // is no longer supported. + const recoverPreparedBackups = process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE === "1"; + const backupRecoveryAssessments = recoverPreparedBackups + ? sandboxes.filter((sandbox) => nonReadyLiveNames.has(sandbox.name)).map(prepareBackupRecovery) + : []; + const preparedRecoveries = backupRecoveryAssessments.filter(isPreparedBackupRecovery); + const rejectedRecoveries = backupRecoveryAssessments.filter( + (candidate): candidate is RejectedBackupRecovery => !isPreparedBackupRecovery(candidate), + ); + const assessedRecoveryNames = new Set( + backupRecoveryAssessments.map((candidate) => candidate.sandbox.name), + ); + + if ( + stale.length === 0 && + unknown.length === 0 && + preparedRecoveries.length === 0 && + rejectedRecoveries.length === 0 + ) { console.log(" All sandboxes are up to date."); return; } @@ -146,6 +232,20 @@ export async function upgradeSandboxes( console.log(` ${s.name} v? → v${s.expected} (${status})`); } } + if (preparedRecoveries.length > 0) { + console.log(`\n ${B}Prepared backup recovery:${R}`); + for (const recovery of preparedRecoveries) { + console.log( + ` ${recovery.sandbox.name} ${D}${recovery.manifest.timestamp}${R} (non-Ready)`, + ); + } + } + if (rejectedRecoveries.length > 0) { + console.log(`\n ${YW}Backup recovery blocked:${R}`); + for (const recovery of rejectedRecoveries) { + console.error(` ${recovery.sandbox.name} ${recovery.reason}`); + } + } console.log(""); if (checkOnly) { @@ -155,35 +255,67 @@ export async function upgradeSandboxes( ` ${unknown.length} sandbox(es) could not be version-checked; start them and rerun, or rebuild manually.`, ); } + if (preparedRecoveries.length > 0) { + console.log( + ` ${preparedRecoveries.length} non-Ready sandbox(es) have a validated pre-upgrade backup.`, + ); + } + if (rejectedRecoveries.length > 0) { + console.log( + ` ${rejectedRecoveries.length} non-Ready sandbox(es) cannot be recovered automatically.`, + ); + } console.log(` Run \`${CLI_NAME} upgrade-sandboxes\` to rebuild them.`); return; } const { rebuildable, stopped } = splitRebuildableSandboxes(stale); - if (stopped.length > 0) { - console.log(` ${D}Skipping ${stopped.length} stopped sandbox(es) — start them first.${R}`); + const notObservedReadyOrNonReady = stopped.filter( + (sandbox) => !assessedRecoveryNames.has(sandbox.name), + ); + if (notObservedReadyOrNonReady.length > 0) { + console.log( + ` ${D}Skipping ${notObservedReadyOrNonReady.length} sandbox(es) not observed on the selected gateway — verify their recorded gateway or start them first.${R}`, + ); } - if (rebuildable.length === 0) { + if ( + rebuildable.length === 0 && + preparedRecoveries.length === 0 && + rejectedRecoveries.length === 0 + ) { console.log(" No running stale sandboxes to rebuild."); return; } let rebuilt = 0; - let failed = 0; - for (const s of rebuildable) { + let failed = rejectedRecoveries.length; + const work = [ + ...rebuildable.map((sandbox) => ({ sandbox, manifest: null })), + ...preparedRecoveries.map((recovery) => ({ + sandbox: { name: recovery.sandbox.name }, + manifest: recovery.manifest, + })), + ]; + for (const item of work) { + const { sandbox, manifest } = item; if (!skipConfirm) { - const answer = await askPrompt(` Rebuild '${s.name}'? [y/N]: `); + const verb = manifest ? "Recover" : "Rebuild"; + const answer = await askPrompt(` ${verb} '${sandbox.name}'? [y/N]: `); if (answer.trim().toLowerCase() !== "y" && answer.trim().toLowerCase() !== "yes") { - console.log(` Skipped '${s.name}'.`); + console.log(` Skipped '${sandbox.name}'.`); continue; } } try { - await rebuildSandbox(s.name, ["--yes"], { throwOnError: true }); + await rebuildSandbox(sandbox.name, ["--yes"], { + throwOnError: true, + recoveryManifest: manifest ?? undefined, + }); rebuilt++; } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); - console.error(` ${YW}⚠${R} Failed to rebuild '${s.name}': ${errorMessage}`); + const verb = manifest ? "recover" : "rebuild"; + console.error(` ${YW}⚠${R} Failed to ${verb} '${sandbox.name}': ${errorMessage}`); failed++; } } diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index ea86923781d..0cedf413923 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -75,6 +75,8 @@ describe("buildCreatedSandboxRegistryEntry", () => { openshellVersion: "0.1.2", }); expect(entry.agent).toBeNull(); + expect(entry.agentVersion).toBeTruthy(); + expect(entry.nemoclawVersion).toBeTruthy(); expect(entry.messaging).toBe(plannedMessagingState); const rawEntry = entry as unknown as Record; expect(rawEntry.messagingChannels).toBeUndefined(); @@ -115,6 +117,8 @@ describe("buildCreatedSandboxRegistryEntry", () => { expect(entry.credentialEnv).toBeNull(); expect(entry.preferredInferenceApi).toBeNull(); expect(entry.nimContainer).toBeNull(); + expect(entry.agentVersion).toBeNull(); + expect(entry.nemoclawVersion).toBeNull(); const rawEntry = entry as unknown as Record; expect(rawEntry.messagingChannels).toBeUndefined(); expect(rawEntry.messagingChannelConfig).toBeUndefined(); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index cffe18df1f1..f96b4611e9d 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1643,6 +1643,82 @@ function readManifest(backupPath: string): RebuildManifest | null { // ── Listing ──────────────────────────────────────────────────────── +export type RebuildRecoveryManifestValidation = + | { ok: true; manifest: RebuildManifest } + | { ok: false; reason: string }; + +/** + * Re-read and validate a prepared rebuild backup before a destructive recovery. + * + * `getLatestBackup()` validates the manifest schema. Recovery additionally pins + * the backup to the target sandbox's own timestamped directory and requires the + * persisted sandbox/agent identity to match the registry entry. This keeps an + * installer recovery from deleting a sandbox based on a renamed, copied, or + * otherwise mismatched manifest. + */ +export function validateRebuildRecoveryManifest( + sandboxName: string, + agentName: string | null | undefined, + candidate: RebuildManifest, +): RebuildRecoveryManifestValidation { + const expectedAgent = String(agentName || "openclaw").trim() || "openclaw"; + const sandboxBackupRoot = path.resolve(REBUILD_BACKUPS_DIR, sandboxName); + const expectedBackupPath = path.resolve(sandboxBackupRoot, candidate.timestamp); + const candidateBackupPath = path.resolve(candidate.backupPath); + + if ( + candidateBackupPath !== expectedBackupPath || + path.dirname(candidateBackupPath) !== sandboxBackupRoot || + path.basename(candidateBackupPath) !== candidate.timestamp + ) { + return { + ok: false, + reason: `backup path does not match '${sandboxName}' and timestamp '${candidate.timestamp}'`, + }; + } + + const persisted = readManifest(candidateBackupPath); + if (!persisted || persisted.version !== MANIFEST_VERSION) { + return { ok: false, reason: "latest backup manifest is missing, malformed, or unsupported" }; + } + if (persisted.sandboxName !== sandboxName) { + return { + ok: false, + reason: `manifest sandbox '${persisted.sandboxName}' does not match '${sandboxName}'`, + }; + } + if (persisted.agentType !== expectedAgent) { + return { + ok: false, + reason: `manifest agent '${persisted.agentType}' does not match registry agent '${expectedAgent}'`, + }; + } + if ( + persisted.timestamp !== candidate.timestamp || + path.resolve(persisted.backupPath) !== candidateBackupPath + ) { + return { ok: false, reason: "persisted backup identity changed during validation" }; + } + + return { ok: true, manifest: persisted }; +} + +/** + * Confirm that a registry entry carries positive NemoClaw-managed image + * provenance. Managed images built by current releases receive a non-empty + * `nemoclawVersion` fingerprint, while custom images do not. + * + * `agentVersion` is not provenance: a live version probe can populate it for a + * legacy custom image, and backup then copies that value into the manifest. + * Pre-fingerprint entries therefore fail closed instead of inferring image + * ownership from matching agent versions. + */ +export function hasPositiveManagedImageEvidence( + sandbox: Pick, +): boolean { + return Boolean(String(sandbox.nemoclawVersion || "").trim()); +} + /** * List available backups for a sandbox, newest first, each enriched with a * virtual `snapshotVersion` number. diff --git a/test/install-preexisting-sandbox-recovery.test.ts b/test/install-preexisting-sandbox-recovery.test.ts new file mode 100644 index 00000000000..8056dd5520a --- /dev/null +++ b/test/install-preexisting-sandbox-recovery.test.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const INSTALLER_PAYLOAD = path.join(import.meta.dirname, "..", "scripts", "install.sh"); + +function runRecoveryBeforeOnboard( + preexistingCount: number, + recoveryExitCode: number, +): { status: number | null; calls: string[]; output: string } { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-install-recovery-order-")); + const cli = path.join(tmp, "nemoclaw"); + const callLog = path.join(tmp, "calls.log"); + const payloadDir = path.join(tmp, "payload"); + fs.mkdirSync(payloadDir); + fs.writeFileSync(path.join(payloadDir, "setup-jetson.sh"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + fs.writeFileSync( + cli, + `#!/usr/bin/env bash +printf 'restore=%s argv=%s\n' "\${NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE:-}" "$*" >> "${callLog}" +if [ "\${1:-}" = "upgrade-sandboxes" ]; then + if [ ${recoveryExitCode} -ne 0 ]; then + printf "Failed to recover 'broken-box': prepared backup restore failed\n" >&2 + fi + exit ${recoveryExitCode} +fi +exit 0 +`, + { mode: 0o755 }, + ); + + const snippet = ` + set -e + source "${INSTALLER_PAYLOAD}" >/dev/null 2>&1 + _CLI_BIN=nemoclaw + _UPGRADE_SANDBOXES_FAILED=false + SCRIPT_DIR="${payloadDir}" + info() { printf 'INFO:%s\n' "$*"; } + warn() { printf 'WARN:%s\n' "$*"; } + error() { printf 'ERROR:%s\n' "$*" >&2; exit 1; } + print_banner() { :; } + preflight_usage_notice_prompt() { :; } + ensure_docker() { :; } + ensure_openshell_build_deps() { :; } + maybe_offer_express_install() { :; } + step() { :; } + install_nodejs() { :; } + ensure_supported_runtime() { :; } + fix_npm_permissions() { :; } + preinstall_backup_and_retire_legacy_gateway() { + _PREEXISTING_SANDBOX_COUNT=${preexistingCount} + } + install_nemoclaw() { :; } + verify_nemoclaw() { _CLI_PATH="${cli}"; } + run_installer_host_preflight() { return 0; } + run_onboard() { "${cli}" onboard; } + restore_onboard_forward_after_post_checks() { return 0; } + print_done() { printf 'PRINT_DONE\n'; } + main --non-interactive --yes-i-accept-third-party-software + `; + const result = spawnSync("bash", ["-c", snippet], { + encoding: "utf-8", + env: { + ...process.env, + BASH_ENV: "", + ENV: "", + HOME: tmp, + NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: "1", + }, + }); + const calls = fs.existsSync(callLog) + ? fs.readFileSync(callLog, "utf-8").trim().split(/\r?\n/).filter(Boolean) + : []; + return { status: result.status, calls, output: `${result.stdout}${result.stderr}` }; +} + +describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { + it("runs automatic recovery before generic onboarding", () => { + const result = runRecoveryBeforeOnboard(2, 0); + + expect(result.status, result.output).toBe(0); + expect(result.calls).toEqual([ + "restore=1 argv=upgrade-sandboxes --auto", + "restore=1 argv=onboard", + ]); + }); + + it("stops before onboarding when any automatic recovery fails", () => { + const result = runRecoveryBeforeOnboard(2, 7); + + expect(result.status).toBe(1); + expect(result.calls).toEqual(["restore=1 argv=upgrade-sandboxes --auto"]); + expect(result.output).toContain("Failed to recover 'broken-box'"); + expect(result.output).toContain("Generic onboarding will not run"); + expect(result.output).toContain( + "Installation incomplete: one or more existing sandboxes failed to upgrade", + ); + }); + + it("leaves fresh installs unchanged", () => { + const result = runRecoveryBeforeOnboard(0, 7); + + expect(result.status, result.output).toBe(0); + expect(result.calls).toEqual(["restore=1 argv=onboard"]); + }); +}); diff --git a/test/snapshot-recovery-validation.test.ts b/test/snapshot-recovery-validation.test.ts new file mode 100644 index 00000000000..1abaefeb722 --- /dev/null +++ b/test/snapshot-recovery-validation.test.ts @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; + +const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-recovery-validation-")); +vi.stubEnv("HOME", TMP_HOME); +const REPO_ROOT = path.join(import.meta.dirname, ".."); +const sandboxState = (await import( + pathToFileURL(path.join(REPO_ROOT, "src", "lib", "state", "sandbox.ts")).href +)) as typeof import("../src/lib/state/sandbox.js"); +const BACKUPS_ROOT = path.join(TMP_HOME, ".nemoclaw", "rebuild-backups"); + +function writeBackup( + sandboxName: string, + timestamp: string, + overrides: Record = {}, +): Record { + const backupPath = path.join(BACKUPS_ROOT, sandboxName, timestamp); + fs.mkdirSync(backupPath, { recursive: true }); + const manifest = { + version: 1, + sandboxName, + timestamp, + agentType: "openclaw", + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox/.openclaw", + backupPath, + blueprintDigest: null, + ...overrides, + }; + fs.writeFileSync( + path.join(backupPath, "rebuild-manifest.json"), + JSON.stringify(manifest, null, 2), + ); + return manifest; +} + +afterAll(() => { + vi.unstubAllEnvs(); + fs.rmSync(TMP_HOME, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(BACKUPS_ROOT, { recursive: true, force: true }); +}); + +describe("prepared rebuild backup recovery validation (#6114)", () => { + it("does not expose a latest backup with a missing or malformed manifest", () => { + const backupPath = path.join(BACKUPS_ROOT, "alpha", "2026-07-01T06-50-41-044Z"); + fs.mkdirSync(backupPath, { recursive: true }); + + expect(sandboxState.getLatestBackup("alpha")).toBeNull(); + + fs.writeFileSync(path.join(backupPath, "rebuild-manifest.json"), "{malformed"); + expect(sandboxState.getLatestBackup("alpha")).toBeNull(); + }); + + it("accepts an exact sandbox and agent identity from its timestamped backup path", () => { + writeBackup("alpha", "2026-07-01T06-50-42-044Z", { + agentVersion: "2026.5.27", + expectedVersion: "2026.5.27", + }); + const latest = sandboxState.getLatestBackup("alpha"); + + expect(latest).not.toBeNull(); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", null, latest!)).toEqual({ + ok: true, + manifest: expect.objectContaining({ + sandboxName: "alpha", + agentType: "openclaw", + timestamp: "2026-07-01T06-50-42-044Z", + }), + }); + }); + + it("rejects a persisted manifest that disappears or becomes malformed after discovery", () => { + const candidate = writeBackup("alpha", "2026-07-01T06-50-42-044Z", { + agentVersion: "2026.5.27", + expectedVersion: "2026.5.27", + }); + const manifestPath = path.join( + BACKUPS_ROOT, + "alpha", + "2026-07-01T06-50-42-044Z", + "rebuild-manifest.json", + ); + + fs.unlinkSync(manifestPath); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", null, candidate as never)).toEqual( + { + ok: false, + reason: "latest backup manifest is missing, malformed, or unsupported", + }, + ); + + fs.writeFileSync(manifestPath, "{malformed"); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", null, candidate as never)).toEqual( + { + ok: false, + reason: "latest backup manifest is missing, malformed, or unsupported", + }, + ); + }); + + it("rejects sandbox, agent, and backup-path identity mismatches", () => { + writeBackup("alpha", "2026-07-01T06-50-42-044Z", { + sandboxName: "beta", + agentType: "hermes", + }); + const mismatched = sandboxState.getLatestBackup("alpha"); + + expect(mismatched).not.toBeNull(); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", "hermes", mismatched!)).toEqual({ + ok: false, + reason: "manifest sandbox 'beta' does not match 'alpha'", + }); + + writeBackup("alpha", "2026-07-01T06-50-43-044Z", { agentType: "hermes" }); + const agentMismatch = sandboxState.getLatestBackup("alpha"); + expect(agentMismatch).not.toBeNull(); + expect( + sandboxState.validateRebuildRecoveryManifest("alpha", "openclaw", agentMismatch!), + ).toEqual({ + ok: false, + reason: "manifest agent 'hermes' does not match registry agent 'openclaw'", + }); + + const exact = writeBackup("alpha", "2026-07-01T06-51-42-044Z", { + backupPath: path.join(BACKUPS_ROOT, "alpha", "some-other-backup"), + }); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", null, exact as never)).toEqual({ + ok: false, + reason: "backup path does not match 'alpha' and timestamp '2026-07-01T06-51-42-044Z'", + }); + }); + + it("requires a non-empty managed-image fingerprint", () => { + expect(sandboxState.hasPositiveManagedImageEvidence({ nemoclawVersion: "0.0.71" })).toBe(true); + expect(sandboxState.hasPositiveManagedImageEvidence({ nemoclawVersion: null })).toBe(false); + expect(sandboxState.hasPositiveManagedImageEvidence({ nemoclawVersion: " " })).toBe(false); + }); +});