From 20f42f64ab818fa7d2dec323f4515297bd45ac29 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 1 Jul 2026 07:50:06 -0700 Subject: [PATCH 1/7] fix(installer): recover sandboxes before onboarding Signed-off-by: Aaron Erickson --- docs/get-started/quickstart.mdx | 4 +- docs/manage-sandboxes/lifecycle.mdx | 3 + scripts/install.sh | 50 ++-- src/lib/actions/sandbox/rebuild-flow.test.ts | 235 +++++++++++++++++- src/lib/actions/sandbox/rebuild.ts | 205 ++++++++++++--- .../upgrade-sandboxes-recovery.test.ts | 153 ++++++++++++ src/lib/actions/upgrade-sandboxes.ts | 125 +++++++++- src/lib/state/sandbox.ts | 79 ++++++ ...stall-preexisting-sandbox-recovery.test.ts | 107 ++++++++ test/snapshot-recovery-validation.test.ts | 130 ++++++++++ 10 files changed, 1027 insertions(+), 64 deletions(-) create mode 100644 src/lib/actions/upgrade-sandboxes-recovery.test.ts create mode 100644 test/install-preexisting-sandbox-recovery.test.ts create mode 100644 test/snapshot-recovery-validation.test.ts diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 675b147faed..b0da348f734 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -109,7 +109,9 @@ After the installer launches `nemoclaw onboard`, the wizard runs preflight check It prints a review summary before it registers the provider with OpenShell. After you confirm, NemoClaw registers inference, prompts for optional web search and messaging channels, builds and starts the sandbox, sets up OpenClaw, then applies the selected network policy tier and presets. At any prompt, press Enter to accept the default shown in `[brackets]`, type `back` to return to the previous prompt, or type `exit` to quit. -If existing sandbox sessions are running, the installer warns before onboarding because the setup can rebuild or upgrade sandboxes after the new sandbox launches. +If registered sandboxes already exist, the installer runs `nemoclaw backup-all` when the installed CLI supports it, then runs `nemoclaw upgrade-sandboxes --auto` before generic onboarding. +For a registered sandbox that is non-Ready after the host upgrade, the installer restores its validated latest backup only when the backup identity matches and there is positive managed-image evidence: a NemoClaw build fingerprint or, for legacy managed images, matching registry and backup agent versions. +If any existing sandbox cannot recover, the installer exits with a nonzero status and does not start generic onboarding. The inference provider prompt presents a numbered list. diff --git a/docs/manage-sandboxes/lifecycle.mdx b/docs/manage-sandboxes/lifecycle.mdx index 3da9707351d..b29a819bc5f 100644 --- a/docs/manage-sandboxes/lifecycle.mdx +++ b/docs/manage-sandboxes/lifecycle.mdx @@ -262,6 +262,9 @@ $$nemoclaw upgrade-sandboxes --check ``` Before upgrade work, the installer runs `$$nemoclaw backup-all` when the installed CLI supports it. +After the host CLI and OpenShell update, the installer runs `$$nemoclaw upgrade-sandboxes --auto` before generic onboarding. +If an existing sandbox is non-Ready, the automatic path requires a validated latest backup whose sandbox and agent identity match the registry and positive evidence that NemoClaw managed the image. +The installer attempts every eligible recovery, exits with a nonzero status if any recovery fails, and does not continue to generic onboarding after that failure. For manual upgrade flows, create a snapshot first and then run the update or rebuild command you need: ```bash diff --git a/scripts/install.sh b/scripts/install.sh index eea0c416a06..f6f912b6aaf 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -532,7 +532,7 @@ print_done() { local _needs_cli_refresh=false needs_shell_reload && _needs_cli_refresh=true - # #5735: do not claim a clean install when the post-onboard auto-upgrade of a + # #5735: do not claim a clean install when the automatic upgrade of a # pre-existing sandbox failed (it may have been destroyed before its recreate # failed). Surface an explicit incomplete/recovery status instead. if [[ "${_UPGRADE_SANDBOXES_FAILED:-false}" == true ]]; then @@ -909,7 +909,7 @@ ONBOARD_RAN=false # auto-onboarding (#3276). _CLI_PATH="" _PREEXISTING_SANDBOX_COUNT=0 -# #5735: set when the post-onboard auto-upgrade of pre-existing sandboxes +# #5735: set when automatic recovery/upgrade of pre-existing sandboxes # reported a failure. A failed/destructive rebuild must not be reported as a # clean install, so print_done downgrades the final banner when this is true. _UPGRADE_SANDBOXES_FAILED=false @@ -1867,12 +1867,12 @@ EOF preinstall_backup_and_retire_legacy_gateway() { local reg_file="${HOME}/.nemoclaw/sandboxes.json" [ -f "$reg_file" ] || return 0 - command_exists openshell || return 0 local sandbox_count sandbox_count="$(registered_sandbox_count)" _PREEXISTING_SANDBOX_COUNT="$sandbox_count" [ "$sandbox_count" -gt 0 ] 2>/dev/null || return 0 + command_exists openshell || return 0 if [[ "${NEMOCLAW_SINGLE_SESSION:-}" == "1" ]]; then error "Aborting — NEMOCLAW_SINGLE_SESSION is set. Destroy existing sessions with '${_CLI_BIN} destroy' before reinstalling." @@ -2147,6 +2147,27 @@ run_installer_host_preflight() { [[ "$status" -ne 10 ]] } +recover_preexisting_sandboxes_before_onboard() { + local cli_runner="$1" + if [ "${_PREEXISTING_SANDBOX_COUNT:-0}" -le 0 ] 2>/dev/null; then + return 0 + fi + + info "Recovering and upgrading pre-existing sandboxes before onboarding…" + # `--auto` is the existing non-interactive maintenance path. When the + # pre-upgrade backup signal is present, the CLI also recovers registered + # non-Ready sandboxes from their validated latest backup. It attempts every + # eligible sandbox before returning non-zero for any failure. + if "$cli_runner" upgrade-sandboxes --auto 2>&1; then + return 0 + fi + + _UPGRADE_SANDBOXES_FAILED=true + warn "One or more existing sandboxes could not be recovered automatically." + warn "Generic onboarding will not run; review the affected sandbox and preserved backup diagnostics above." + return 1 +} + run_onboard() { show_usage_notice info "Running ${_CLI_BIN} onboard…" @@ -2726,23 +2747,11 @@ main() { warn "Set NEMOCLAW_SINGLE_SESSION=1 to abort the installer when sessions are active." fi if run_installer_host_preflight; then + if ! recover_preexisting_sandboxes_before_onboard "$_cli_runner"; then + error "Installation incomplete: one or more existing sandboxes failed to recover before onboarding." + fi run_onboard || error "Onboarding did not complete successfully." ONBOARD_RAN=true - # After onboard, check for stale sandboxes that need rebuilding (#1904). - # Uses --auto so it runs non-interactively in piped/CI contexts. - if [ "${_PREEXISTING_SANDBOX_COUNT:-0}" -gt 0 ] 2>/dev/null && [ -n "$_cli_runner" ]; then - info "Checking for sandboxes that need upgrading…" - # #5735: a non-zero exit here can mean an existing sandbox was rebuilt - # destructively and its recreate failed. Record it so print_done reports - # the install as incomplete with recovery guidance instead of a clean - # banner. The CLI already prints the affected sandbox name and the - # preserved backup path on failure. - if ! "$_cli_runner" upgrade-sandboxes --auto 2>&1; then - _UPGRADE_SANDBOXES_FAILED=true - warn "One or more existing sandboxes could not be upgraded automatically." - warn "Review the messages above — affected sandboxes may need '${_CLI_BIN} onboard --resume' or '${_CLI_BIN} rebuild', and any backup path shown above can restore workspace state." - fi - fi restore_onboard_forward_after_post_checks || error "Hermes host forward restore failed." elif [ "${NON_INTERACTIVE:-}" = "1" ]; then error "Skipping onboarding until the host prerequisites above are fixed." @@ -2757,9 +2766,8 @@ main() { } # Print the completion summary, then propagate a fatal/non-zero result when the -# post-onboard auto-upgrade of a pre-existing sandbox failed (#5735, PRA-5). The -# new sandbox may have onboarded fine, but a failed auto-upgrade can have left an -# *existing* sandbox destroyed or backup-only, so the install must not be +# automatic recovery of a pre-existing sandbox failed (#5735, PRA-5). A failed +# recovery can have left an existing sandbox destroyed or backup-only, so the install must not be # reported as success. print_done() has already shown the affected sandbox and # recovery guidance (and the "completed with warnings" banner); exiting non-zero # here is what keeps automation and operators from treating it as a clean diff --git a/src/lib/actions/sandbox/rebuild-flow.test.ts b/src/lib/actions/sandbox/rebuild-flow.test.ts index d843ecfa6f9..e8b0aaedc84 100644 --- a/src/lib/actions/sandbox/rebuild-flow.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow.test.ts @@ -52,7 +52,14 @@ type RebuildFlowOverrides = { buildMessagingRebuildPlan?: () => Promise | unknown; sandboxEntry?: Record; sessionSandboxName?: string; + sandboxListOutput?: string; backupPolicyPresets?: string[]; + preDeleteSandboxEntry?: Record; + preDeleteDefaultSandbox?: string | null; + preDeleteLatestManifest?: Record | null; + recoveryManifestValidation?: ( + manifest: Record, + ) => { ok: true; manifest: Record } | { ok: false; reason: string }; }; type RebuildFlowHarness = { @@ -68,6 +75,7 @@ type RebuildFlowHarness = { registryUpdateSpy: MockInstance; releaseOnboardLockSpy: MockInstance; relockSpy: MockInstance; + restoreSandboxEntrySpy: MockInstance; restoreSandboxStateSpy: MockInstance; runOpenshellSpy: MockInstance; messagingRebuildPlanSpy: MockInstance; @@ -193,7 +201,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ - result: { status: 0, output: "alpha Ready" }, + result: { status: 0, output: overrides.sandboxListOutput ?? "alpha Ready" }, }); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); vi.spyOn(agentDefs, "loadAgent").mockReturnValue(agentDef); @@ -212,17 +220,36 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild .mockImplementation(() => undefined); const markStepFailedSpy = installTerminalStepFailureMock(onboardSession, session); session.sandboxName = overrides.sessionSandboxName ?? session.sandboxName; - vi.spyOn(registry, "getSandbox").mockReturnValue({ + const sandboxEntry = { name: "alpha", provider: "ollama-local", model: "nvidia/nemotron", policies: ["npm"], agent: null, + agentVersion: "0.1.0", nimContainer: null, ...(overrides.sandboxEntry ?? {}), + }; + vi.spyOn(registry, "getSandbox").mockReturnValue(sandboxEntry); + let registryLoadCount = 0; + vi.spyOn(registry, "load").mockImplementation(() => { + const isPreDeleteRead = registryLoadCount > 0; + registryLoadCount++; + return { + defaultSandbox: isPreDeleteRead ? (overrides.preDeleteDefaultSandbox ?? "alpha") : "alpha", + sandboxes: { + alpha: + isPreDeleteRead && overrides.preDeleteSandboxEntry + ? overrides.preDeleteSandboxEntry + : sandboxEntry, + }, + }; }); vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [] }); const registryUpdateSpy = vi.spyOn(registry, "updateSandbox").mockImplementation(() => undefined); + const restoreSandboxEntrySpy = vi + .spyOn(registry, "restoreSandboxEntry") + .mockImplementation(() => undefined); vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ detected: false, sessions: [], @@ -251,6 +278,19 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild policyPresets: overrides.backupPolicyPresets ?? ["npm", "bad", "throw"], }, }); + vi.spyOn(sandboxState, "validateRebuildRecoveryManifest").mockImplementation( + (...args: unknown[]) => { + const manifest = args[2] as Record; + return overrides.recoveryManifestValidation?.(manifest) ?? { ok: true as const, manifest }; + }, + ); + vi.spyOn(sandboxState, "getLatestBackup").mockImplementation( + () => + (overrides.preDeleteLatestManifest === undefined + ? makePreparedRecoveryManifest() + : overrides.preDeleteLatestManifest) as ReturnType, + ); + vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence").mockReturnValue(true); const restoreSandboxStateSpy = vi.spyOn(sandboxState, "restoreSandboxState").mockImplementation( overrides.restoreSandboxState ?? (() => ({ @@ -286,6 +326,8 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild vi.spyOn(shields, "repairMutableConfigPerms").mockImplementation( overrides.repairMutableConfigPerms ?? (() => ({ applied: true, verified: true, errors: [] })), ); + vi.spyOn(shields, "isShieldsDown").mockReturnValue(true); + vi.spyOn(shields, "clearShieldsState").mockImplementation(() => undefined); const messagingRebuildPlanSpy = vi .spyOn(messaging.MessagingWorkflowPlanner.prototype, "buildRebuildPlanFromSandboxEntry") .mockImplementation(overrides.buildMessagingRebuildPlan ?? (() => null)); @@ -310,6 +352,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild registryUpdateSpy, releaseOnboardLockSpy, relockSpy, + restoreSandboxEntrySpy, restoreSandboxStateSpy, runOpenshellSpy, messagingRebuildPlanSpy, @@ -387,6 +430,25 @@ function makeActiveTeamsMessagingPlan() { }; } +function makePreparedRecoveryManifest() { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-07-01T06-50-42-044Z", + agentType: "openclaw", + agentVersion: "0.1.0", + expectedVersion: "0.2.0", + stateDirs: ["workspace"], + backedUpDirs: ["workspace"], + stateFiles: [], + dir: "/sandbox/.openclaw", + backupPath: "/tmp/rebuild-backups/alpha/2026-07-01T06-50-42-044Z", + blueprintDigest: null, + policyPresets: ["npm"], + customPolicies: [], + }; +} + describe("rebuildSandbox flow", () => { beforeEach(() => { delete process.env.NEMOCLAW_SANDBOX_NAME; @@ -443,6 +505,175 @@ describe("rebuildSandbox flow", () => { ); }); + it("restores the validated pre-upgrade manifest without taking a second backup (#6114)", async () => { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxListOutput: "alpha Error", + }); + const recoveryManifest = makePreparedRecoveryManifest(); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest, + }), + ).resolves.toBeUndefined(); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith( + "alpha", + recoveryManifest.backupPath, + ); + }); + + it("rejects a mismatched prepared manifest before deleting the sandbox (#6114)", async () => { + const harness = createRebuildFlowHarness({ + recoveryManifestValidation: () => ({ + ok: false, + reason: "manifest sandbox 'beta' does not match 'alpha'", + }), + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Invalid recovery manifest"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("revalidates the prepared manifest immediately before deleting the sandbox (#6114)", async () => { + let validationCount = 0; + const harness = createRebuildFlowHarness({ + recoveryManifestValidation: (manifest) => { + validationCount++; + return validationCount === 1 + ? { ok: true as const, manifest } + : { ok: false as const, reason: "persisted backup identity changed during validation" }; + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Invalid recovery manifest"); + + expect(validationCount).toBe(2); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("rejects same-agent registry configuration drift before deleting the sandbox (#6114)", async () => { + const harness = createRebuildFlowHarness({ + preDeleteSandboxEntry: { + name: "alpha", + provider: "compatible-endpoint", + model: "new-model", + policies: ["npm", "github"], + agent: null, + agentVersion: "0.1.0", + nemoclawVersion: "0.0.71", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recovery registry configuration changed during preflight"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("uses the single refreshed registry snapshot for recreate rollback (#6114)", async () => { + const harness = createRebuildFlowHarness({ + preDeleteDefaultSandbox: "beta", + onboard: () => { + throw new Error("recreate failed"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( + expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), + { reclaimDefault: null }, + ); + }); + + it("rejects a latest-backup change immediately before deleting the sandbox (#6114)", async () => { + const harness = createRebuildFlowHarness({ + preDeleteLatestManifest: { + ...makePreparedRecoveryManifest(), + timestamp: "2026-07-01T07-00-00-000Z", + backupPath: "/tmp/rebuild-backups/alpha/2026-07-01T07-00-00-000Z", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recovery backup identity changed during preflight"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("restores the registry entry when prepared-backup recreation fails (#6114)", async () => { + const harness = createRebuildFlowHarness({ + onboard: () => { + throw new Error("recreate failed"); + }, + }); + const recoveryManifest = makePreparedRecoveryManifest(); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest, + }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( + expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), + { reclaimDefault: "alpha" }, + ); + expect(harness.restoreSandboxStateSpy).not.toHaveBeenCalled(); + }); + it("restores enabled messaging presets while pruning disabled ones from final policies", async () => { const disabledSlackPlan = { schemaVersion: 1, diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 309843fd9a2..6fa672c1d2f 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isDeepStrictEqual } from "node:util"; + import { CLI_NAME } from "../../cli/branding"; import { prompt as askPrompt } from "../../credentials/store"; import { @@ -577,10 +579,99 @@ async function reapplyMessagingManifestAfterOpenClawDoctor( * `Dockerfile.base` changes fail before destructive work and are applied to the * recreated sandbox image. */ +export interface RebuildSandboxExecutionOptions { + throwOnError?: boolean; + /** Internal installer recovery input; never exposed as a CLI option. */ + recoveryManifest?: sandboxState.RebuildManifest; +} + +type RebuildBail = (message: string, code?: number) => never; + +function failPreparedRecoveryPreDelete( + detail: string, + errorMessage: string, + bail: RebuildBail, +): never { + console.error(""); + console.error(` ${_RD}Recovery pre-delete check failed:${R} ${detail}.`); + console.error(" Sandbox is untouched — no data was lost."); + return bail(errorMessage); +} + +function revalidatePreparedRecoveryBeforeDelete( + sandboxName: string, + initialEntry: RebuildSandboxEntry, + candidate: sandboxState.RebuildManifest | null, + registrySnapshot: registry.SandboxRegistry | null, + bail: RebuildBail, +): { + manifest: sandboxState.RebuildManifest | null; + registrySnapshot: registry.SandboxRegistry | null; +} { + if (!candidate) return { manifest: null, registrySnapshot }; + + const refreshedRegistrySnapshot = JSON.parse( + JSON.stringify(registry.load()), + ) as registry.SandboxRegistry; + const currentEntry = refreshedRegistrySnapshot.sandboxes[sandboxName]; + if (!currentEntry) { + return failPreparedRecoveryPreDelete( + "registry entry no longer exists", + "Recovery registry identity changed during preflight.", + bail, + ); + } + if (!isDeepStrictEqual(currentEntry, initialEntry)) { + return failPreparedRecoveryPreDelete( + "registered sandbox configuration changed during preflight", + "Recovery registry configuration changed during preflight.", + bail, + ); + } + + const latestManifest = sandboxState.getLatestBackup(sandboxName); + if ( + !latestManifest || + latestManifest.timestamp !== candidate.timestamp || + latestManifest.backupPath !== candidate.backupPath + ) { + return failPreparedRecoveryPreDelete( + "latest prepared backup changed during preflight", + "Recovery backup identity changed during preflight.", + bail, + ); + } + + const validation = sandboxState.validateRebuildRecoveryManifest( + sandboxName, + currentEntry.agent, + latestManifest, + ); + if (!validation.ok) { + return failPreparedRecoveryPreDelete( + validation.reason, + `Invalid recovery manifest: ${validation.reason}`, + bail, + ); + } + if (!sandboxState.hasPositiveManagedImageEvidence(currentEntry, validation.manifest)) { + return failPreparedRecoveryPreDelete( + "backup no longer has positive NemoClaw-managed image evidence", + "Recovery backup is not proven to come from a NemoClaw-managed image.", + bail, + ); + } + + return { + manifest: validation.manifest, + registrySnapshot: refreshedRegistrySnapshot, + }; +} + export async function rebuildSandbox( sandboxName: string, options: string[] | RebuildSandboxOptions = {}, - opts: { throwOnError?: boolean } = {}, + opts: RebuildSandboxExecutionOptions = {}, ): Promise { const normalized = normalizeRebuildSandboxOptions(options); const verbose = normalized.verbose === true || process.env.NEMOCLAW_REBUILD_VERBOSE === "1"; @@ -588,7 +679,7 @@ export async function rebuildSandbox( const skipConfirm = normalized.yes === true || normalized.force === true; // When called from upgradeSandboxes in a loop, throwOnError prevents // process.exit from aborting the entire batch on the first failure. - const bail = opts.throwOnError + const bail: RebuildBail = opts.throwOnError ? (msg: string, _code = 1) => { throw new Error(msg); } @@ -600,6 +691,33 @@ export async function rebuildSandbox( const sb = getRebuildSandboxEntryOrBail(sandboxName, bail); if (!sb) return; + let recoveryManifest: sandboxState.RebuildManifest | null = null; + if (opts.recoveryManifest) { + const validation = sandboxState.validateRebuildRecoveryManifest( + sandboxName, + sb.agent, + opts.recoveryManifest, + ); + if (!validation.ok) { + console.error(""); + console.error(` ${_RD}Recovery preflight failed:${R} ${validation.reason}.`); + console.error(" Sandbox is untouched — no data was lost."); + bail(`Invalid recovery manifest: ${validation.reason}`); + return; + } + if (!sandboxState.hasPositiveManagedImageEvidence(sb, validation.manifest)) { + console.error(""); + console.error( + ` ${_RD}Recovery preflight failed:${R} backup has no positive NemoClaw-managed image evidence.`, + ); + console.error(" Legacy custom-image sandboxes are not recreated automatically."); + console.error(" Sandbox is untouched — no data was lost."); + bail("Recovery backup is not proven to come from a NemoClaw-managed image."); + return; + } + recoveryManifest = validation.manifest; + } + // Multi-agent guard (temporary — until swarm lands) if (!isSingleAgentRebuildSupported(sb, bail)) return; @@ -650,7 +768,15 @@ export async function rebuildSandbox( // Step 1: Ensure sandbox is live for backup, or identify stale-sandbox recovery. const liveState = await resolveRebuildLiveState(sandboxName, sb, log, bail); if (!liveState) return; - const { staleRecovery, staleRegistrySnapshot } = liveState; + const { staleRecovery } = liveState; + const preparedBackupRecovery = recoveryManifest !== null; + const recoveryRecreate = staleRecovery || preparedBackupRecovery; + // A prepared pre-upgrade backup can recover a sandbox that still appears in + // OpenShell but is stuck in Provisioning/Error. Capture the same registry + // rollback state used by missing-live-sandbox recovery before deletion. + let recoveryRegistrySnapshot = preparedBackupRecovery + ? JSON.parse(JSON.stringify(registry.load())) + : liveState.staleRegistrySnapshot; // Build agent base layers before backup/delete so Dockerfile.base errors leave // the existing sandbox intact. This is what applies local Hermes version edits. @@ -661,7 +787,7 @@ export async function rebuildSandbox( // clearing old shields state until recreate succeeds (#4497). const { rebuildShieldsWindow, staleSandboxWasLocked } = openRebuildShieldsWindowForState( sandboxName, - staleRecovery, + recoveryRecreate, ); if (!rebuildShieldsWindow) return bail("Failed to auto-unlock shields."); @@ -671,15 +797,32 @@ export async function rebuildSandbox( let sandboxStillExists = true; try { - // Step 2: Backup (skipped on stale-sandbox recovery -- no live state exists) - const backupManifest = backupSandboxStateForRebuild( + // Re-read the prepared manifest immediately before the destructive phase. + // Base-image builds and other preflight work can take long enough that the + // on-disk backup may have been replaced since the initial validation. + const preDeleteRecovery = revalidatePreparedRecoveryBeforeDelete( sandboxName, sb, - staleRecovery, - log, - relockShieldsIfNeeded, + recoveryManifest, + recoveryRegistrySnapshot, bail, ); + recoveryManifest = preDeleteRecovery.manifest; + recoveryRegistrySnapshot = preDeleteRecovery.registrySnapshot; + + // Step 2: Backup (skipped on stale-sandbox recovery -- no live state exists) + // Installer recovery already has a validated pre-upgrade backup. Reuse it + // instead of trying to reach a non-Ready sandbox to create a second backup. + const backupManifest = + recoveryManifest ?? + backupSandboxStateForRebuild( + sandboxName, + sb, + staleRecovery, + log, + relockShieldsIfNeeded, + bail, + ); if (backupManifest === undefined) return; // Step 3: Delete sandbox without tearing down gateway or session. @@ -896,32 +1039,30 @@ export async function rebuildSandbox( /* best effort */ } - // Stale-sandbox recovery had no backup to fall back on and already removed - // the registry entry before the recreate. If the recreate failed, restore - // the captured entry so the recommended `rebuild --yes` (and `connect`) + // Recovery already removed the registry entry before the recreate. If the + // recreate failed, restore the captured entry so the recommended + // `rebuild --yes` (and `connect`) // remain retryable instead of failing at dispatch with "not found in // registry" (#4497). Restore unconditionally — overwriting any partial entry // a failed `onboard` may have registered — so the original metadata // (defaultSandbox, customPolicies, every field) wins, not a half-written // recreate entry. The restore targets only this sandbox under the registry // lock, leaving other sandboxes' concurrent changes intact. - const snapshotEntry = staleRegistrySnapshot?.sandboxes?.[sandboxName]; - if (staleRecovery && snapshotEntry) { + const snapshotEntry = recoveryRegistrySnapshot?.sandboxes?.[sandboxName]; + if (recoveryRecreate && snapshotEntry) { try { registry.restoreSandboxEntry(snapshotEntry, { reclaimDefault: - staleRegistrySnapshot?.defaultSandbox === sandboxName ? sandboxName : null, + recoveryRegistrySnapshot?.defaultSandbox === sandboxName ? sandboxName : null, }); - log("Stale-recovery recreate failed: restored preserved registry entry for retry"); + log("Recovery recreate failed: restored preserved registry entry for retry"); } catch (err) { - log( - `Failed to restore registry entry after stale-recovery recreate failure: ${String(err)}`, - ); + log(`Failed to restore registry entry after recovery recreate failure: ${String(err)}`); } } console.error(""); - if (staleRecovery) { + if (recoveryRecreate) { console.error(` ${_RD}Recovery recreate failed.${R}`); console.error( " Your local registry entry has been preserved — you can retry once the issue above is fixed.", @@ -955,11 +1096,10 @@ export async function rebuildSandbox( return; } - // Recreate succeeded. For stale recovery, reset the now-stale shields state so - // the freshly recreated (mutable) sandbox reports its true posture instead of - // the gone sandbox's old lock seal. Deferred until here so a failed recreate - // above leaves the lockdown record intact for a retry (#4497). - if (staleRecovery) { + // Recreate succeeded. Reset the prior shields state so the freshly recreated + // (mutable) sandbox reports its true posture. Deferred until here so a failed + // recreate above leaves the lockdown record intact for a retry (#4497). + if (recoveryRecreate) { shields.clearShieldsState(sandboxName); } @@ -1182,15 +1322,15 @@ export async function rebuildSandbox( } console.log(""); - if ( + const postRestoreComplete = restoreSucceeded && !mutablePermsRepairUnverified && !mutableConfigHashRefreshUnverified && !messagingHostForwardUnverified && - !policyPresetRestoreIncomplete - ) { + !policyPresetRestoreIncomplete; + if (postRestoreComplete) { console.log(` ${G}\u2713${R} Sandbox '${sandboxName}' rebuilt successfully`); - if (staleRecovery) { + if (staleRecovery && !backupManifest) { console.log( ` ${D}Recovered from a stale registry entry \u2014 no prior workspace state was available to restore.${R}`, ); @@ -1235,11 +1375,16 @@ export async function rebuildSandbox( // Stale recovery reset the shields state to mutable (the gone sandbox's lock // seal could not carry over to the fresh image). If lockdown had been enabled, // tell the operator to re-apply it on the recreated sandbox (#4497). - if (staleRecovery && staleSandboxWasLocked) { + if (recoveryRecreate && staleSandboxWasLocked) { console.log( ` ${YW}\u26a0${R} Shields were previously enabled but the recreated sandbox starts unlocked \u2014 run \`${CLI_NAME} ${sandboxName} shields up\` to restore lockdown.`, ); } + if (preparedBackupRecovery && !postRestoreComplete) { + bail( + `Prepared backup recovery for '${sandboxName}' completed with unverified post-restore state.`, + ); + } } finally { if (!rebuildShieldsWindow.relocked) { relockShieldsIfNeeded(sandboxStillExists); diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts new file mode 100644 index 00000000000..945a9cc390b --- /dev/null +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +type UpgradeSandboxes = typeof import("./upgrade-sandboxes")["upgradeSandboxes"]; + +const requireDist = createRequire(import.meta.url); +const upgradeModulePath = "./upgrade-sandboxes.js"; +const originalRecoverySignal = process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; + +// Warm the CommonJS source graph outside the first test's timeout. Each harness +// still reloads the entry module after installing its dependency spies. +requireDist(upgradeModulePath); +delete require.cache[requireDist.resolve(upgradeModulePath)]; + +function makeManifest(sandboxName: string) { + const timestamp = `2026-07-01T06-50-4${sandboxName.length}-044Z`; + return { + version: 1, + sandboxName, + timestamp, + agentType: "openclaw", + agentVersion: "2026.5.27", + expectedVersion: "2026.5.27", + stateDirs: ["workspace"], + backedUpDirs: ["workspace"], + stateFiles: [], + dir: "/sandbox/.openclaw", + backupPath: `/tmp/rebuild-backups/${sandboxName}/${timestamp}`, + blueprintDigest: null, + policyPresets: [], + customPolicies: [], + snapshotVersion: 1, + }; +} + +function createRecoveryHarness(names: string[]): { + upgradeSandboxes: UpgradeSandboxes; + rebuildSpy: ReturnType; + managedEvidenceSpy: ReturnType; +} { + delete require.cache[requireDist.resolve(upgradeModulePath)]; + process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = "1"; + + const gatewayDrift = requireDist("../adapters/openshell/gateway-drift.js"); + const coreVersion = requireDist("../core/version.js"); + const sandboxList = requireDist("../openshell-sandbox-list.js"); + const sandboxVersion = requireDist("../sandbox/version.js"); + const registry = requireDist("../state/registry.js"); + const sandboxState = requireDist("../state/sandbox.js"); + const rebuild = requireDist("./sandbox/rebuild.js"); + + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); + vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); + vi.spyOn(coreVersion, "getVersion").mockReturnValue("0.0.71"); + vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ + result: { + status: 0, + output: names.map((name) => `${name} Error`).join("\n"), + }, + recoveryAttempted: false, + recoverySucceeded: false, + }); + vi.spyOn(registry, "listSandboxes").mockReturnValue({ + sandboxes: names.map((name) => ({ + name, + agent: null, + agentVersion: "2026.5.27", + nemoclawVersion: "0.0.71", + })), + }); + vi.spyOn(sandboxVersion, "checkAgentVersion").mockReturnValue({ + sandboxVersion: "2026.5.27", + expectedVersion: "2026.5.27", + isStale: false, + detectionMethod: "registry", + }); + vi.spyOn(sandboxState, "getLatestBackup").mockImplementation((...args: unknown[]) => + makeManifest(String(args[0])), + ); + vi.spyOn(sandboxState, "validateRebuildRecoveryManifest").mockImplementation( + (...args: unknown[]) => ({ + ok: true as const, + manifest: args[2] as ReturnType, + }), + ); + const managedEvidenceSpy = vi + .spyOn(sandboxState, "hasPositiveManagedImageEvidence") + .mockReturnValue(true); + const rebuildSpy = vi.spyOn(rebuild, "rebuildSandbox").mockResolvedValue(undefined); + + return { + upgradeSandboxes: requireDist(upgradeModulePath).upgradeSandboxes, + rebuildSpy, + managedEvidenceSpy, + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + delete require.cache[requireDist.resolve(upgradeModulePath)]; + if (originalRecoverySignal === undefined) { + delete process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; + } else { + process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = originalRecoverySignal; + } +}); + +describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { + it("passes every non-Ready sandbox's validated manifest into rebuild", async () => { + const harness = createRecoveryHarness(["alpha", "beta"]); + + await expect(harness.upgradeSandboxes({ auto: true })).resolves.toBeUndefined(); + + expect(harness.rebuildSpy).toHaveBeenCalledTimes(2); + for (const name of ["alpha", "beta"]) { + expect(harness.rebuildSpy).toHaveBeenCalledWith(name, ["--yes"], { + throwOnError: true, + recoveryManifest: expect.objectContaining({ sandboxName: name }), + }); + } + }); + + it("continues through all eligible sandboxes before reporting a recovery failure", async () => { + const harness = createRecoveryHarness(["alpha", "beta"]); + harness.rebuildSpy.mockRejectedValueOnce(new Error("alpha failed")); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + + expect(harness.rebuildSpy).toHaveBeenCalledTimes(2); + expect(harness.rebuildSpy.mock.calls.map((call) => call[0])).toEqual(["alpha", "beta"]); + }); + + it("fails closed without rebuilding a legacy custom-image backup", async () => { + const harness = createRecoveryHarness(["custom-box"]); + harness.managedEvidenceSpy.mockReturnValue(false); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + + expect(harness.rebuildSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/upgrade-sandboxes.ts b/src/lib/actions/upgrade-sandboxes.ts index f0b066dfd1b..96e974c7256 100644 --- a/src/lib/actions/upgrade-sandboxes.ts +++ b/src/lib/actions/upgrade-sandboxes.ts @@ -27,6 +27,7 @@ import { import { parseReadySandboxNames } from "../runtime-recovery"; import * as sandboxVersion from "../sandbox/version"; import * as registry from "../state/registry"; +import * as sandboxState from "../state/sandbox"; import { rebuildSandbox } from "./sandbox/rebuild"; // ── Upgrade sandboxes (#1904) ──────────────────────────────────── @@ -78,6 +79,48 @@ function describeStaleUpgrade(s: UpgradeSandboxCandidate): string { return parts.join("; "); } +type PreparedBackupRecovery = { + sandbox: registry.SandboxEntry; + manifest: sandboxState.RebuildManifest; +}; + +type RejectedBackupRecovery = { + sandbox: registry.SandboxEntry; + reason: string; +}; + +function prepareBackupRecovery( + sandbox: registry.SandboxEntry, +): PreparedBackupRecovery | RejectedBackupRecovery { + const latest = sandboxState.getLatestBackup(sandbox.name); + if (!latest) { + return { sandbox, reason: "no validated pre-upgrade backup was found" }; + } + + const validation = sandboxState.validateRebuildRecoveryManifest( + sandbox.name, + sandbox.agent, + latest, + ); + if (!validation.ok) { + return { sandbox, reason: validation.reason }; + } + if (!sandboxState.hasPositiveManagedImageEvidence(sandbox, validation.manifest)) { + return { + sandbox, + reason: + "backup has no positive NemoClaw-managed image evidence (legacy custom images are not auto-recreated)", + }; + } + return { sandbox, manifest: validation.manifest }; +} + +function isPreparedBackupRecovery( + candidate: PreparedBackupRecovery | RejectedBackupRecovery, +): candidate is PreparedBackupRecovery { + return "manifest" in candidate; +} + export async function upgradeSandboxes( options: string[] | UpgradeSandboxesOptions = {}, ): Promise { @@ -127,7 +170,24 @@ export async function upgradeSandboxes( { currentNemoclawVersion: resolveCurrentNemoclawVersion() }, ); - if (stale.length === 0 && unknown.length === 0) { + const recoverPreparedBackups = process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE === "1"; + const backupRecoveryAssessments = recoverPreparedBackups + ? sandboxes.filter((sandbox) => !liveNames.has(sandbox.name)).map(prepareBackupRecovery) + : []; + const preparedRecoveries = backupRecoveryAssessments.filter(isPreparedBackupRecovery); + const rejectedRecoveries = backupRecoveryAssessments.filter( + (candidate): candidate is RejectedBackupRecovery => !isPreparedBackupRecovery(candidate), + ); + const preparedRecoveryNames = new Set( + preparedRecoveries.map((candidate) => candidate.sandbox.name), + ); + + if ( + stale.length === 0 && + unknown.length === 0 && + preparedRecoveries.length === 0 && + rejectedRecoveries.length === 0 + ) { console.log(" All sandboxes are up to date."); return; } @@ -146,6 +206,20 @@ export async function upgradeSandboxes( console.log(` ${s.name} v? → v${s.expected} (${status})`); } } + if (preparedRecoveries.length > 0) { + console.log(`\n ${B}Prepared backup recovery:${R}`); + for (const recovery of preparedRecoveries) { + console.log( + ` ${recovery.sandbox.name} ${D}${recovery.manifest.timestamp}${R} (non-Ready)`, + ); + } + } + if (rejectedRecoveries.length > 0) { + console.log(`\n ${YW}Backup recovery blocked:${R}`); + for (const recovery of rejectedRecoveries) { + console.error(` ${recovery.sandbox.name} ${recovery.reason}`); + } + } console.log(""); if (checkOnly) { @@ -155,35 +229,66 @@ export async function upgradeSandboxes( ` ${unknown.length} sandbox(es) could not be version-checked; start them and rerun, or rebuild manually.`, ); } + if (preparedRecoveries.length > 0) { + console.log( + ` ${preparedRecoveries.length} non-Ready sandbox(es) have a validated pre-upgrade backup.`, + ); + } + if (rejectedRecoveries.length > 0) { + console.log( + ` ${rejectedRecoveries.length} non-Ready sandbox(es) cannot be recovered automatically.`, + ); + } console.log(` Run \`${CLI_NAME} upgrade-sandboxes\` to rebuild them.`); return; } const { rebuildable, stopped } = splitRebuildableSandboxes(stale); - if (stopped.length > 0) { - console.log(` ${D}Skipping ${stopped.length} stopped sandbox(es) — start them first.${R}`); + const stoppedWithoutPreparedBackup = stopped.filter( + (sandbox) => !preparedRecoveryNames.has(sandbox.name), + ); + if (stoppedWithoutPreparedBackup.length > 0 && !recoverPreparedBackups) { + console.log( + ` ${D}Skipping ${stoppedWithoutPreparedBackup.length} stopped sandbox(es) — start them first.${R}`, + ); } - if (rebuildable.length === 0) { + if ( + rebuildable.length === 0 && + preparedRecoveries.length === 0 && + rejectedRecoveries.length === 0 + ) { console.log(" No running stale sandboxes to rebuild."); return; } let rebuilt = 0; - let failed = 0; - for (const s of rebuildable) { + let failed = rejectedRecoveries.length; + const work = [ + ...rebuildable.map((sandbox) => ({ sandbox, manifest: null })), + ...preparedRecoveries.map((recovery) => ({ + sandbox: { name: recovery.sandbox.name }, + manifest: recovery.manifest, + })), + ]; + for (const item of work) { + const { sandbox, manifest } = item; if (!skipConfirm) { - const answer = await askPrompt(` Rebuild '${s.name}'? [y/N]: `); + const verb = manifest ? "Recover" : "Rebuild"; + const answer = await askPrompt(` ${verb} '${sandbox.name}'? [y/N]: `); if (answer.trim().toLowerCase() !== "y" && answer.trim().toLowerCase() !== "yes") { - console.log(` Skipped '${s.name}'.`); + console.log(` Skipped '${sandbox.name}'.`); continue; } } try { - await rebuildSandbox(s.name, ["--yes"], { throwOnError: true }); + await rebuildSandbox(sandbox.name, ["--yes"], { + throwOnError: true, + recoveryManifest: manifest ?? undefined, + }); rebuilt++; } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); - console.error(` ${YW}⚠${R} Failed to rebuild '${s.name}': ${errorMessage}`); + console.error(` ${YW}⚠${R} Failed to rebuild '${sandbox.name}': ${errorMessage}`); failed++; } } diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index cffe18df1f1..6634a79019d 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1643,6 +1643,85 @@ function readManifest(backupPath: string): RebuildManifest | null { // ── Listing ──────────────────────────────────────────────────────── +export type RebuildRecoveryManifestValidation = + | { ok: true; manifest: RebuildManifest } + | { ok: false; reason: string }; + +/** + * Re-read and validate a prepared rebuild backup before a destructive recovery. + * + * `getLatestBackup()` validates the manifest schema. Recovery additionally pins + * the backup to the target sandbox's own timestamped directory and requires the + * persisted sandbox/agent identity to match the registry entry. This keeps an + * installer recovery from deleting a sandbox based on a renamed, copied, or + * otherwise mismatched manifest. + */ +export function validateRebuildRecoveryManifest( + sandboxName: string, + agentName: string | null | undefined, + candidate: RebuildManifest, +): RebuildRecoveryManifestValidation { + const expectedAgent = String(agentName || "openclaw").trim() || "openclaw"; + const sandboxBackupRoot = path.resolve(REBUILD_BACKUPS_DIR, sandboxName); + const expectedBackupPath = path.resolve(sandboxBackupRoot, candidate.timestamp); + const candidateBackupPath = path.resolve(candidate.backupPath); + + if ( + candidateBackupPath !== expectedBackupPath || + path.dirname(candidateBackupPath) !== sandboxBackupRoot || + path.basename(candidateBackupPath) !== candidate.timestamp + ) { + return { + ok: false, + reason: `backup path does not match '${sandboxName}' and timestamp '${candidate.timestamp}'`, + }; + } + + const persisted = readManifest(candidateBackupPath); + if (!persisted || persisted.version !== MANIFEST_VERSION) { + return { ok: false, reason: "latest backup manifest is missing, malformed, or unsupported" }; + } + if (persisted.sandboxName !== sandboxName) { + return { + ok: false, + reason: `manifest sandbox '${persisted.sandboxName}' does not match '${sandboxName}'`, + }; + } + if (persisted.agentType !== expectedAgent) { + return { + ok: false, + reason: `manifest agent '${persisted.agentType}' does not match registry agent '${expectedAgent}'`, + }; + } + if ( + persisted.timestamp !== candidate.timestamp || + path.resolve(persisted.backupPath) !== candidateBackupPath + ) { + return { ok: false, reason: "persisted backup identity changed during validation" }; + } + + return { ok: true, manifest: persisted }; +} + +/** + * Confirm that a backup came from a NemoClaw-managed image. Current registry + * entries carry a build fingerprint; v0.0.55 managed images predate that field + * but carry the same non-empty agent version in the registry and backup. Legacy + * custom images intentionally carry neither signal. + */ +export function hasPositiveManagedImageEvidence( + sandbox: Pick, + manifest: Pick, +): boolean { + const fingerprint = String(sandbox.nemoclawVersion || "").trim(); + if (fingerprint) return true; + const registryAgentVersion = String(sandbox.agentVersion || "").trim(); + const manifestAgentVersion = String(manifest.agentVersion || "").trim(); + return Boolean( + registryAgentVersion && manifestAgentVersion && registryAgentVersion === manifestAgentVersion, + ); +} + /** * List available backups for a sandbox, newest first, each enriched with a * virtual `snapshotVersion` number. diff --git a/test/install-preexisting-sandbox-recovery.test.ts b/test/install-preexisting-sandbox-recovery.test.ts new file mode 100644 index 00000000000..6bbce8d8204 --- /dev/null +++ b/test/install-preexisting-sandbox-recovery.test.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const INSTALLER_PAYLOAD = path.join(import.meta.dirname, "..", "scripts", "install.sh"); + +function runRecoveryBeforeOnboard( + preexistingCount: number, + recoveryExitCode: number, +): { status: number | null; calls: string[]; output: string } { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-install-recovery-order-")); + const cli = path.join(tmp, "nemoclaw"); + const callLog = path.join(tmp, "calls.log"); + const payloadDir = path.join(tmp, "payload"); + fs.mkdirSync(payloadDir); + fs.writeFileSync(path.join(payloadDir, "setup-jetson.sh"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + fs.writeFileSync( + cli, + `#!/usr/bin/env bash +printf 'restore=%s argv=%s\n' "\${NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE:-}" "$*" >> "${callLog}" +if [ "\${1:-}" = "upgrade-sandboxes" ]; then + exit ${recoveryExitCode} +fi +exit 0 +`, + { mode: 0o755 }, + ); + + const snippet = ` + set -e + source "${INSTALLER_PAYLOAD}" >/dev/null 2>&1 + _CLI_BIN=nemoclaw + _UPGRADE_SANDBOXES_FAILED=false + SCRIPT_DIR="${payloadDir}" + info() { printf 'INFO:%s\n' "$*"; } + warn() { printf 'WARN:%s\n' "$*"; } + error() { printf 'ERROR:%s\n' "$*" >&2; exit 1; } + print_banner() { :; } + preflight_usage_notice_prompt() { :; } + ensure_docker() { :; } + ensure_openshell_build_deps() { :; } + maybe_offer_express_install() { :; } + step() { :; } + install_nodejs() { :; } + ensure_supported_runtime() { :; } + fix_npm_permissions() { :; } + preinstall_backup_and_retire_legacy_gateway() { + _PREEXISTING_SANDBOX_COUNT=${preexistingCount} + } + install_nemoclaw() { :; } + verify_nemoclaw() { _CLI_PATH="${cli}"; } + run_installer_host_preflight() { return 0; } + run_onboard() { "${cli}" onboard; } + restore_onboard_forward_after_post_checks() { return 0; } + finalize_install() { :; } + main --non-interactive --yes-i-accept-third-party-software + `; + const result = spawnSync("bash", ["-c", snippet], { + encoding: "utf-8", + env: { + ...process.env, + BASH_ENV: "", + ENV: "", + HOME: tmp, + NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: "1", + }, + }); + const calls = fs.existsSync(callLog) + ? fs.readFileSync(callLog, "utf-8").trim().split(/\r?\n/).filter(Boolean) + : []; + return { status: result.status, calls, output: `${result.stdout}${result.stderr}` }; +} + +describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { + it("runs automatic recovery before generic onboarding", () => { + const result = runRecoveryBeforeOnboard(2, 0); + + expect(result.status, result.output).toBe(0); + expect(result.calls).toEqual([ + "restore=1 argv=upgrade-sandboxes --auto", + "restore=1 argv=onboard", + ]); + }); + + it("stops before onboarding when any automatic recovery fails", () => { + const result = runRecoveryBeforeOnboard(2, 7); + + expect(result.status).toBe(1); + expect(result.calls).toEqual(["restore=1 argv=upgrade-sandboxes --auto"]); + expect(result.output).toContain("Generic onboarding will not run"); + }); + + it("leaves fresh installs unchanged", () => { + const result = runRecoveryBeforeOnboard(0, 7); + + expect(result.status, result.output).toBe(0); + expect(result.calls).toEqual(["restore=1 argv=onboard"]); + }); +}); diff --git a/test/snapshot-recovery-validation.test.ts b/test/snapshot-recovery-validation.test.ts new file mode 100644 index 00000000000..3dec63cc2c2 --- /dev/null +++ b/test/snapshot-recovery-validation.test.ts @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { afterAll, beforeEach, describe, expect, it } from "vitest"; + +const ORIGINAL_HOME = process.env.HOME; +const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-recovery-validation-")); +process.env.HOME = TMP_HOME; +const REPO_ROOT = path.join(import.meta.dirname, ".."); +const sandboxState = (await import( + pathToFileURL(path.join(REPO_ROOT, "src", "lib", "state", "sandbox.ts")).href +)) as typeof import("../src/lib/state/sandbox.js"); +const BACKUPS_ROOT = path.join(TMP_HOME, ".nemoclaw", "rebuild-backups"); + +function writeBackup( + sandboxName: string, + timestamp: string, + overrides: Record = {}, +): Record { + const backupPath = path.join(BACKUPS_ROOT, sandboxName, timestamp); + fs.mkdirSync(backupPath, { recursive: true }); + const manifest = { + version: 1, + sandboxName, + timestamp, + agentType: "openclaw", + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox/.openclaw", + backupPath, + blueprintDigest: null, + ...overrides, + }; + fs.writeFileSync( + path.join(backupPath, "rebuild-manifest.json"), + JSON.stringify(manifest, null, 2), + ); + return manifest; +} + +afterAll(() => { + if (ORIGINAL_HOME === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = ORIGINAL_HOME; + } + fs.rmSync(TMP_HOME, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(BACKUPS_ROOT, { recursive: true, force: true }); +}); + +describe("prepared rebuild backup recovery validation (#6114)", () => { + it("accepts an exact sandbox and agent identity from its timestamped backup path", () => { + writeBackup("alpha", "2026-07-01T06-50-42-044Z", { + agentVersion: "2026.5.27", + expectedVersion: "2026.5.27", + }); + const latest = sandboxState.getLatestBackup("alpha"); + + expect(latest).not.toBeNull(); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", null, latest!)).toEqual({ + ok: true, + manifest: expect.objectContaining({ + sandboxName: "alpha", + agentType: "openclaw", + timestamp: "2026-07-01T06-50-42-044Z", + }), + }); + }); + + it("rejects sandbox, agent, and backup-path identity mismatches", () => { + writeBackup("alpha", "2026-07-01T06-50-42-044Z", { + sandboxName: "beta", + agentType: "hermes", + }); + const mismatched = sandboxState.getLatestBackup("alpha"); + + expect(mismatched).not.toBeNull(); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", "hermes", mismatched!)).toEqual({ + ok: false, + reason: "manifest sandbox 'beta' does not match 'alpha'", + }); + + writeBackup("alpha", "2026-07-01T06-50-43-044Z", { agentType: "hermes" }); + const agentMismatch = sandboxState.getLatestBackup("alpha"); + expect(agentMismatch).not.toBeNull(); + expect( + sandboxState.validateRebuildRecoveryManifest("alpha", "openclaw", agentMismatch!), + ).toEqual({ + ok: false, + reason: "manifest agent 'hermes' does not match registry agent 'openclaw'", + }); + + const exact = writeBackup("alpha", "2026-07-01T06-51-42-044Z", { + backupPath: path.join(BACKUPS_ROOT, "alpha", "some-other-backup"), + }); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", null, exact as never)).toEqual({ + ok: false, + reason: "backup path does not match 'alpha' and timestamp '2026-07-01T06-51-42-044Z'", + }); + }); + + it("requires a build fingerprint or matching legacy managed-agent versions", () => { + const legacyManaged = { agentVersion: "2026.5.27", nemoclawVersion: null }; + const legacyCustom = { agentVersion: null, nemoclawVersion: null }; + const manifest = { agentVersion: "2026.5.27" }; + + expect(sandboxState.hasPositiveManagedImageEvidence(legacyManaged, manifest)).toBe(true); + expect( + sandboxState.hasPositiveManagedImageEvidence( + { agentVersion: null, nemoclawVersion: "0.0.71" }, + { agentVersion: null }, + ), + ).toBe(true); + expect(sandboxState.hasPositiveManagedImageEvidence(legacyCustom, manifest)).toBe(false); + expect( + sandboxState.hasPositiveManagedImageEvidence(legacyManaged, { + agentVersion: "different-version", + }), + ).toBe(false); + }); +}); From 1309bf4bc6164c359f6c2ad0057cf0f74af4b586 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 1 Jul 2026 10:25:58 -0700 Subject: [PATCH 2/7] test(upgrade): avoid conditional teardown branches Signed-off-by: Aaron Erickson --- src/lib/actions/upgrade-sandboxes-recovery.test.ts | 12 +++++++----- test/snapshot-recovery-validation.test.ts | 7 ++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index 945a9cc390b..239ff5a3870 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -104,11 +104,13 @@ function createRecoveryHarness(names: string[]): { afterEach(() => { vi.restoreAllMocks(); delete require.cache[requireDist.resolve(upgradeModulePath)]; - if (originalRecoverySignal === undefined) { - delete process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; - } else { - process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = originalRecoverySignal; - } + delete process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; + Object.assign( + process.env, + originalRecoverySignal === undefined + ? {} + : { NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: originalRecoverySignal }, + ); }); describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { diff --git a/test/snapshot-recovery-validation.test.ts b/test/snapshot-recovery-validation.test.ts index 3dec63cc2c2..d0aa91eb562 100644 --- a/test/snapshot-recovery-validation.test.ts +++ b/test/snapshot-recovery-validation.test.ts @@ -45,11 +45,8 @@ function writeBackup( } afterAll(() => { - if (ORIGINAL_HOME === undefined) { - delete process.env.HOME; - } else { - process.env.HOME = ORIGINAL_HOME; - } + delete process.env.HOME; + Object.assign(process.env, ORIGINAL_HOME === undefined ? {} : { HOME: ORIGINAL_HOME }); fs.rmSync(TMP_HOME, { recursive: true, force: true }); }); From 5688e5f1e38a6b7872f02f939b00825b04aee95b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 1 Jul 2026 10:37:38 -0700 Subject: [PATCH 3/7] fix(upgrade): narrow prepared recovery eligibility Signed-off-by: Aaron Erickson --- .../upgrade-sandboxes-recovery.test.ts | 70 +++++++++++++++++-- src/lib/actions/upgrade-sandboxes.ts | 57 +++++++++------ 2 files changed, 101 insertions(+), 26 deletions(-) diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index 239ff5a3870..138e9f3bb3d 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -37,9 +37,17 @@ function makeManifest(sandboxName: string) { }; } -function createRecoveryHarness(names: string[]): { +function createRecoveryHarness( + names: string[], + options: { + gatewayNames?: Record; + liveOutput?: string; + latestBackup?: ReturnType | null; + } = {}, +): { upgradeSandboxes: UpgradeSandboxes; rebuildSpy: ReturnType; + latestBackupSpy: ReturnType; managedEvidenceSpy: ReturnType; } { delete require.cache[requireDist.resolve(upgradeModulePath)]; @@ -61,7 +69,7 @@ function createRecoveryHarness(names: string[]): { vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ result: { status: 0, - output: names.map((name) => `${name} Error`).join("\n"), + output: options.liveOutput ?? names.map((name) => `${name} Error`).join("\n"), }, recoveryAttempted: false, recoverySucceeded: false, @@ -71,6 +79,7 @@ function createRecoveryHarness(names: string[]): { name, agent: null, agentVersion: "2026.5.27", + gatewayName: options.gatewayNames?.[name], nemoclawVersion: "0.0.71", })), }); @@ -80,9 +89,11 @@ function createRecoveryHarness(names: string[]): { isStale: false, detectionMethod: "registry", }); - vi.spyOn(sandboxState, "getLatestBackup").mockImplementation((...args: unknown[]) => - makeManifest(String(args[0])), - ); + const latestBackupSpy = vi + .spyOn(sandboxState, "getLatestBackup") + .mockImplementation((...args: unknown[]) => + options.latestBackup === undefined ? makeManifest(String(args[0])) : options.latestBackup, + ); vi.spyOn(sandboxState, "validateRebuildRecoveryManifest").mockImplementation( (...args: unknown[]) => ({ ok: true as const, @@ -97,6 +108,7 @@ function createRecoveryHarness(names: string[]): { return { upgradeSandboxes: requireDist(upgradeModulePath).upgradeSandboxes, rebuildSpy, + latestBackupSpy, managedEvidenceSpy, }; } @@ -152,4 +164,52 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { expect(harness.rebuildSpy).not.toHaveBeenCalled(); }); + + it("does not recover a Ready sandbox registered on a different gateway", async () => { + const harness = createRecoveryHarness(["ready-on-gateway-b"], { + gatewayNames: { "ready-on-gateway-b": "gateway-b" }, + liveOutput: "No sandboxes found.", + latestBackup: null, + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).resolves.toBeUndefined(); + + expect(harness.rebuildSpy).not.toHaveBeenCalled(); + expect(harness.latestBackupSpy).not.toHaveBeenCalled(); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it("fails closed for a live Error sandbox with no latest backup", async () => { + const harness = createRecoveryHarness(["broken-box"], { latestBackup: null }); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + + expect(harness.rebuildSpy).not.toHaveBeenCalled(); + }); + + it("continues after one live sandbox's backup assessment throws", async () => { + const harness = createRecoveryHarness(["alpha", "beta"]); + harness.latestBackupSpy + .mockImplementationOnce(() => { + throw new Error("ENOTDIR: unreadable backup root"); + }) + .mockImplementationOnce((name: string) => makeManifest(name)); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + + await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + + expect(harness.rebuildSpy).toHaveBeenCalledOnce(); + expect(harness.rebuildSpy).toHaveBeenCalledWith("beta", ["--yes"], { + throwOnError: true, + recoveryManifest: expect.objectContaining({ sandboxName: "beta" }), + }); + }); }); diff --git a/src/lib/actions/upgrade-sandboxes.ts b/src/lib/actions/upgrade-sandboxes.ts index 96e974c7256..2c65586327e 100644 --- a/src/lib/actions/upgrade-sandboxes.ts +++ b/src/lib/actions/upgrade-sandboxes.ts @@ -24,7 +24,7 @@ import { captureSandboxListWithGatewayRecovery, printSandboxListFailureWithRecoveryContext, } from "../openshell-sandbox-list"; -import { parseReadySandboxNames } from "../runtime-recovery"; +import { parseLiveSandboxEntries, parseReadySandboxNames } from "../runtime-recovery"; import * as sandboxVersion from "../sandbox/version"; import * as registry from "../state/registry"; import * as sandboxState from "../state/sandbox"; @@ -92,27 +92,32 @@ type RejectedBackupRecovery = { function prepareBackupRecovery( sandbox: registry.SandboxEntry, ): PreparedBackupRecovery | RejectedBackupRecovery { - const latest = sandboxState.getLatestBackup(sandbox.name); - if (!latest) { - return { sandbox, reason: "no validated pre-upgrade backup was found" }; - } + try { + const latest = sandboxState.getLatestBackup(sandbox.name); + if (!latest) { + return { sandbox, reason: "no validated pre-upgrade backup was found" }; + } - const validation = sandboxState.validateRebuildRecoveryManifest( - sandbox.name, - sandbox.agent, - latest, - ); - if (!validation.ok) { - return { sandbox, reason: validation.reason }; - } - if (!sandboxState.hasPositiveManagedImageEvidence(sandbox, validation.manifest)) { - return { - sandbox, - reason: - "backup has no positive NemoClaw-managed image evidence (legacy custom images are not auto-recreated)", - }; + const validation = sandboxState.validateRebuildRecoveryManifest( + sandbox.name, + sandbox.agent, + latest, + ); + if (!validation.ok) { + return { sandbox, reason: validation.reason }; + } + if (!sandboxState.hasPositiveManagedImageEvidence(sandbox, validation.manifest)) { + return { + sandbox, + reason: + "backup has no positive NemoClaw-managed image evidence (legacy custom images are not auto-recreated)", + }; + } + return { sandbox, manifest: validation.manifest }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { sandbox, reason: `backup recovery assessment failed: ${detail}` }; } - return { sandbox, manifest: validation.manifest }; } function isPreparedBackupRecovery( @@ -159,6 +164,16 @@ export async function upgradeSandboxes( process.exit(liveResult.status || 1); } const liveNames = parseReadySandboxNames(liveResult.output || ""); + // Absence from the selected gateway is not evidence of failure: a registered + // sandbox may be Ready on another recorded gateway. Only an explicitly + // observed, known non-Ready phase is eligible for prepared-backup recovery. + const nonReadyLiveNames = new Set( + parseLiveSandboxEntries(liveResult.output || "") + .filter( + (entry) => entry.phase !== null && entry.phase !== "Ready" && entry.phase !== "Running", + ) + .map((entry) => entry.name), + ); // Classify sandboxes as stale, unknown, or current. Pass the running NemoClaw // build so a NemoClaw image/build change is detected even when the agent @@ -172,7 +187,7 @@ export async function upgradeSandboxes( const recoverPreparedBackups = process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE === "1"; const backupRecoveryAssessments = recoverPreparedBackups - ? sandboxes.filter((sandbox) => !liveNames.has(sandbox.name)).map(prepareBackupRecovery) + ? sandboxes.filter((sandbox) => nonReadyLiveNames.has(sandbox.name)).map(prepareBackupRecovery) : []; const preparedRecoveries = backupRecoveryAssessments.filter(isPreparedBackupRecovery); const rejectedRecoveries = backupRecoveryAssessments.filter( From 1645a3cfa45a4aa136ac40dbaace028945b1fd4d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 1 Jul 2026 11:03:30 -0700 Subject: [PATCH 4/7] fix(installer): finalize failed sandbox recovery Signed-off-by: Aaron Erickson --- scripts/install.sh | 3 +- .../upgrade-sandboxes-recovery.test.ts | 41 +++++++++++++++---- ...stall-preexisting-sandbox-recovery.test.ts | 5 ++- test/snapshot-recovery-validation.test.ts | 10 +++++ 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index f6f912b6aaf..920c74b30a9 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2748,7 +2748,8 @@ main() { fi if run_installer_host_preflight; then if ! recover_preexisting_sandboxes_before_onboard "$_cli_runner"; then - error "Installation incomplete: one or more existing sandboxes failed to recover before onboarding." + finalize_install + return 1 fi run_onboard || error "Onboarding did not complete successfully." ONBOARD_RAN=true diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index 138e9f3bb3d..546db082d9d 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -43,6 +43,7 @@ function createRecoveryHarness( gatewayNames?: Record; liveOutput?: string; latestBackup?: ReturnType | null; + staleNames?: string[]; } = {}, ): { upgradeSandboxes: UpgradeSandboxes; @@ -83,11 +84,14 @@ function createRecoveryHarness( nemoclawVersion: "0.0.71", })), }); - vi.spyOn(sandboxVersion, "checkAgentVersion").mockReturnValue({ - sandboxVersion: "2026.5.27", - expectedVersion: "2026.5.27", - isStale: false, - detectionMethod: "registry", + vi.spyOn(sandboxVersion, "checkAgentVersion").mockImplementation((...args: unknown[]) => { + const name = String(args[0]); + return { + sandboxVersion: options.staleNames?.includes(name) === true ? "2026.5.26" : "2026.5.27", + expectedVersion: "2026.5.27", + isStale: options.staleNames?.includes(name) === true, + detectionMethod: "registry", + }; }); const latestBackupSpy = vi .spyOn(sandboxState, "getLatestBackup") @@ -165,10 +169,10 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { expect(harness.rebuildSpy).not.toHaveBeenCalled(); }); - it("does not recover a Ready sandbox registered on a different gateway", async () => { - const harness = createRecoveryHarness(["ready-on-gateway-b"], { - gatewayNames: { "ready-on-gateway-b": "gateway-b" }, - liveOutput: "No sandboxes found.", + it("does not recover a registered sandbox absent from the selected gateway", async () => { + const harness = createRecoveryHarness(["registered-elsewhere"], { + gatewayNames: { "registered-elsewhere": "gateway-b" }, + liveOutput: "selected-gateway-box Ready", latestBackup: null, }); const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { @@ -182,6 +186,25 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { expect(exitSpy).not.toHaveBeenCalled(); }); + it("attempts both a live stale rebuild and a prepared non-Ready recovery", async () => { + const harness = createRecoveryHarness(["stale-box", "recovery-box"], { + liveOutput: "stale-box Ready\nrecovery-box Error", + staleNames: ["stale-box"], + }); + + await expect(harness.upgradeSandboxes({ auto: true })).resolves.toBeUndefined(); + + expect(harness.rebuildSpy).toHaveBeenCalledTimes(2); + expect(harness.rebuildSpy).toHaveBeenNthCalledWith(1, "stale-box", ["--yes"], { + throwOnError: true, + recoveryManifest: undefined, + }); + expect(harness.rebuildSpy).toHaveBeenNthCalledWith(2, "recovery-box", ["--yes"], { + throwOnError: true, + recoveryManifest: expect.objectContaining({ sandboxName: "recovery-box" }), + }); + }); + it("fails closed for a live Error sandbox with no latest backup", async () => { const harness = createRecoveryHarness(["broken-box"], { latestBackup: null }); vi.spyOn(process, "exit").mockImplementation(((code?: number) => { diff --git a/test/install-preexisting-sandbox-recovery.test.ts b/test/install-preexisting-sandbox-recovery.test.ts index 6bbce8d8204..f1ec38e3a6a 100644 --- a/test/install-preexisting-sandbox-recovery.test.ts +++ b/test/install-preexisting-sandbox-recovery.test.ts @@ -60,7 +60,7 @@ exit 0 run_installer_host_preflight() { return 0; } run_onboard() { "${cli}" onboard; } restore_onboard_forward_after_post_checks() { return 0; } - finalize_install() { :; } + print_done() { printf 'PRINT_DONE\n'; } main --non-interactive --yes-i-accept-third-party-software `; const result = spawnSync("bash", ["-c", snippet], { @@ -96,6 +96,9 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.status).toBe(1); expect(result.calls).toEqual(["restore=1 argv=upgrade-sandboxes --auto"]); expect(result.output).toContain("Generic onboarding will not run"); + expect(result.output).toContain( + "Installation incomplete: one or more existing sandboxes failed to upgrade", + ); }); it("leaves fresh installs unchanged", () => { diff --git a/test/snapshot-recovery-validation.test.ts b/test/snapshot-recovery-validation.test.ts index d0aa91eb562..73a168291e4 100644 --- a/test/snapshot-recovery-validation.test.ts +++ b/test/snapshot-recovery-validation.test.ts @@ -55,6 +55,16 @@ beforeEach(() => { }); describe("prepared rebuild backup recovery validation (#6114)", () => { + it("does not expose a latest backup with a missing or malformed manifest", () => { + const backupPath = path.join(BACKUPS_ROOT, "alpha", "2026-07-01T06-50-41-044Z"); + fs.mkdirSync(backupPath, { recursive: true }); + + expect(sandboxState.getLatestBackup("alpha")).toBeNull(); + + fs.writeFileSync(path.join(backupPath, "rebuild-manifest.json"), "{malformed"); + expect(sandboxState.getLatestBackup("alpha")).toBeNull(); + }); + it("accepts an exact sandbox and agent identity from its timestamped backup path", () => { writeBackup("alpha", "2026-07-01T06-50-42-044Z", { agentVersion: "2026.5.27", From 227afb805608bf4f1b978d531e721854efe3e4a7 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 1 Jul 2026 14:47:34 -0700 Subject: [PATCH 5/7] fix(upgrade): harden prepared recovery evidence Signed-off-by: Aaron Erickson --- docs/get-started/quickstart.mdx | 2 +- src/lib/actions/sandbox/rebuild.ts | 2 +- .../upgrade-sandboxes-recovery.test.ts | 29 +++++++++------ src/lib/actions/upgrade-sandboxes.ts | 24 ++++++++---- src/lib/onboard/sandbox-registration.test.ts | 4 ++ src/lib/state/sandbox.ts | 11 ++++-- ...stall-preexisting-sandbox-recovery.test.ts | 4 ++ test/snapshot-recovery-validation.test.ts | 37 ++++++++++++++++--- 8 files changed, 85 insertions(+), 28 deletions(-) diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index b0da348f734..9cb1424efe3 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -111,7 +111,7 @@ After you confirm, NemoClaw registers inference, prompts for optional web search At any prompt, press Enter to accept the default shown in `[brackets]`, type `back` to return to the previous prompt, or type `exit` to quit. If registered sandboxes already exist, the installer runs `nemoclaw backup-all` when the installed CLI supports it, then runs `nemoclaw upgrade-sandboxes --auto` before generic onboarding. For a registered sandbox that is non-Ready after the host upgrade, the installer restores its validated latest backup only when the backup identity matches and there is positive managed-image evidence: a NemoClaw build fingerprint or, for legacy managed images, matching registry and backup agent versions. -If any existing sandbox cannot recover, the installer exits with a nonzero status and does not start generic onboarding. +If an automatic rebuild fails, or a non-Ready recovery is blocked or fails, the installer exits with a nonzero status and does not start generic onboarding. The inference provider prompt presents a numbered list. diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 6fa672c1d2f..0ec0474eac0 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -579,7 +579,7 @@ async function reapplyMessagingManifestAfterOpenClawDoctor( * `Dockerfile.base` changes fail before destructive work and are applied to the * recreated sandbox image. */ -export interface RebuildSandboxExecutionOptions { +interface RebuildSandboxExecutionOptions { throwOnError?: boolean; /** Internal installer recovery input; never exposed as a CLI option. */ recoveryManifest?: sandboxState.RebuildManifest; diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index 546db082d9d..2e84a1eff16 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -9,7 +9,6 @@ type UpgradeSandboxes = typeof import("./upgrade-sandboxes")["upgradeSandboxes"] const requireDist = createRequire(import.meta.url); const upgradeModulePath = "./upgrade-sandboxes.js"; -const originalRecoverySignal = process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; // Warm the CommonJS source graph outside the first test's timeout. Each harness // still reloads the entry module after installing its dependency spies. @@ -52,7 +51,7 @@ function createRecoveryHarness( managedEvidenceSpy: ReturnType; } { delete require.cache[requireDist.resolve(upgradeModulePath)]; - process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = "1"; + vi.stubEnv("NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE", "1"); const gatewayDrift = requireDist("../adapters/openshell/gateway-drift.js"); const coreVersion = requireDist("../core/version.js"); @@ -119,14 +118,8 @@ function createRecoveryHarness( afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); delete require.cache[requireDist.resolve(upgradeModulePath)]; - delete process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; - Object.assign( - process.env, - originalRecoverySignal === undefined - ? {} - : { NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: originalRecoverySignal }, - ); }); describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { @@ -155,6 +148,9 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { expect(harness.rebuildSpy).toHaveBeenCalledTimes(2); expect(harness.rebuildSpy.mock.calls.map((call) => call[0])).toEqual(["alpha", "beta"]); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("Failed to recover 'alpha': alpha failed"), + ); }); it("fails closed without rebuilding a legacy custom-image backup", async () => { @@ -169,11 +165,12 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { expect(harness.rebuildSpy).not.toHaveBeenCalled(); }); - it("does not recover a registered sandbox absent from the selected gateway", async () => { + it("warns and does not recover a stale registered sandbox absent from the selected gateway", async () => { const harness = createRecoveryHarness(["registered-elsewhere"], { gatewayNames: { "registered-elsewhere": "gateway-b" }, liveOutput: "selected-gateway-box Ready", latestBackup: null, + staleNames: ["registered-elsewhere"], }); const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { throw new Error(`process.exit(${code})`); @@ -183,6 +180,9 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { expect(harness.rebuildSpy).not.toHaveBeenCalled(); expect(harness.latestBackupSpy).not.toHaveBeenCalled(); + expect(console.log).toHaveBeenCalledWith( + expect.stringContaining("Skipping 1 sandbox(es) not observed on the selected gateway"), + ); expect(exitSpy).not.toHaveBeenCalled(); }); @@ -206,7 +206,10 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { }); it("fails closed for a live Error sandbox with no latest backup", async () => { - const harness = createRecoveryHarness(["broken-box"], { latestBackup: null }); + const harness = createRecoveryHarness(["broken-box"], { + latestBackup: null, + staleNames: ["broken-box"], + }); vi.spyOn(process, "exit").mockImplementation(((code?: number) => { throw new Error(`process.exit(${code})`); }) as never); @@ -214,6 +217,10 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); expect(harness.rebuildSpy).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("broken-box")); + expect(console.log).not.toHaveBeenCalledWith( + expect.stringContaining("verify their recorded gateway or start them first"), + ); }); it("continues after one live sandbox's backup assessment throws", async () => { diff --git a/src/lib/actions/upgrade-sandboxes.ts b/src/lib/actions/upgrade-sandboxes.ts index 2c65586327e..a867d28f0e8 100644 --- a/src/lib/actions/upgrade-sandboxes.ts +++ b/src/lib/actions/upgrade-sandboxes.ts @@ -185,6 +185,15 @@ export async function upgradeSandboxes( { currentNemoclawVersion: resolveCurrentNemoclawVersion() }, ); + // Source boundary (#6114): a v0.0.55/legacy-OpenShell install can leave its + // already-registered sandboxes in Provisioning/Error after the host upgrade. + // That state comes from the already-installed legacy CLI/gateway and cannot be + // prevented at its source by this candidate. install.sh exports this signal only + // after that CLI completes backup-all, or after an operator asserts prepared + // upgrade state. upgrade-sandboxes-recovery.test.ts and + // install-preexisting-sandbox-recovery.test.ts guard the handoff. Remove this + // bridge with onboard's matching consumer once pre-fingerprint upgrades are no + // longer supported. const recoverPreparedBackups = process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE === "1"; const backupRecoveryAssessments = recoverPreparedBackups ? sandboxes.filter((sandbox) => nonReadyLiveNames.has(sandbox.name)).map(prepareBackupRecovery) @@ -193,8 +202,8 @@ export async function upgradeSandboxes( const rejectedRecoveries = backupRecoveryAssessments.filter( (candidate): candidate is RejectedBackupRecovery => !isPreparedBackupRecovery(candidate), ); - const preparedRecoveryNames = new Set( - preparedRecoveries.map((candidate) => candidate.sandbox.name), + const assessedRecoveryNames = new Set( + backupRecoveryAssessments.map((candidate) => candidate.sandbox.name), ); if ( @@ -259,12 +268,12 @@ export async function upgradeSandboxes( } const { rebuildable, stopped } = splitRebuildableSandboxes(stale); - const stoppedWithoutPreparedBackup = stopped.filter( - (sandbox) => !preparedRecoveryNames.has(sandbox.name), + const notObservedReadyOrNonReady = stopped.filter( + (sandbox) => !assessedRecoveryNames.has(sandbox.name), ); - if (stoppedWithoutPreparedBackup.length > 0 && !recoverPreparedBackups) { + if (notObservedReadyOrNonReady.length > 0) { console.log( - ` ${D}Skipping ${stoppedWithoutPreparedBackup.length} stopped sandbox(es) — start them first.${R}`, + ` ${D}Skipping ${notObservedReadyOrNonReady.length} sandbox(es) not observed on the selected gateway — verify their recorded gateway or start them first.${R}`, ); } if ( @@ -303,7 +312,8 @@ export async function upgradeSandboxes( rebuilt++; } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); - console.error(` ${YW}⚠${R} Failed to rebuild '${sandbox.name}': ${errorMessage}`); + const verb = manifest ? "recover" : "rebuild"; + console.error(` ${YW}⚠${R} Failed to ${verb} '${sandbox.name}': ${errorMessage}`); failed++; } } diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index ea86923781d..0cedf413923 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -75,6 +75,8 @@ describe("buildCreatedSandboxRegistryEntry", () => { openshellVersion: "0.1.2", }); expect(entry.agent).toBeNull(); + expect(entry.agentVersion).toBeTruthy(); + expect(entry.nemoclawVersion).toBeTruthy(); expect(entry.messaging).toBe(plannedMessagingState); const rawEntry = entry as unknown as Record; expect(rawEntry.messagingChannels).toBeUndefined(); @@ -115,6 +117,8 @@ describe("buildCreatedSandboxRegistryEntry", () => { expect(entry.credentialEnv).toBeNull(); expect(entry.preferredInferenceApi).toBeNull(); expect(entry.nimContainer).toBeNull(); + expect(entry.agentVersion).toBeNull(); + expect(entry.nemoclawVersion).toBeNull(); const rawEntry = entry as unknown as Record; expect(rawEntry.messagingChannels).toBeUndefined(); expect(rawEntry.messagingChannelConfig).toBeUndefined(); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 6634a79019d..84587ce2527 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1705,9 +1705,14 @@ export function validateRebuildRecoveryManifest( /** * Confirm that a backup came from a NemoClaw-managed image. Current registry - * entries carry a build fingerprint; v0.0.55 managed images predate that field - * but carry the same non-empty agent version in the registry and backup. Legacy - * custom images intentionally carry neither signal. + * entries carry a build fingerprint. The v0.0.55 source boundary predates that + * field: onboard registered `agentVersion` only when `fromDockerfile` was false, + * and backup copied that registry value into the manifest. Therefore an exact, + * non-empty legacy version match is positive managed-image evidence, while a + * `--from` custom image has a null registry version and fails closed. + * + * Remove the legacy version fallback once upgrades from releases without the + * `nemoclawVersion` fingerprint are no longer supported. */ export function hasPositiveManagedImageEvidence( sandbox: Pick, diff --git a/test/install-preexisting-sandbox-recovery.test.ts b/test/install-preexisting-sandbox-recovery.test.ts index f1ec38e3a6a..8056dd5520a 100644 --- a/test/install-preexisting-sandbox-recovery.test.ts +++ b/test/install-preexisting-sandbox-recovery.test.ts @@ -27,6 +27,9 @@ function runRecoveryBeforeOnboard( `#!/usr/bin/env bash printf 'restore=%s argv=%s\n' "\${NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE:-}" "$*" >> "${callLog}" if [ "\${1:-}" = "upgrade-sandboxes" ]; then + if [ ${recoveryExitCode} -ne 0 ]; then + printf "Failed to recover 'broken-box': prepared backup restore failed\n" >&2 + fi exit ${recoveryExitCode} fi exit 0 @@ -95,6 +98,7 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { expect(result.status).toBe(1); expect(result.calls).toEqual(["restore=1 argv=upgrade-sandboxes --auto"]); + expect(result.output).toContain("Failed to recover 'broken-box'"); expect(result.output).toContain("Generic onboarding will not run"); expect(result.output).toContain( "Installation incomplete: one or more existing sandboxes failed to upgrade", diff --git a/test/snapshot-recovery-validation.test.ts b/test/snapshot-recovery-validation.test.ts index 73a168291e4..271377acdb5 100644 --- a/test/snapshot-recovery-validation.test.ts +++ b/test/snapshot-recovery-validation.test.ts @@ -6,11 +6,10 @@ import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { afterAll, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; -const ORIGINAL_HOME = process.env.HOME; const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-recovery-validation-")); -process.env.HOME = TMP_HOME; +vi.stubEnv("HOME", TMP_HOME); const REPO_ROOT = path.join(import.meta.dirname, ".."); const sandboxState = (await import( pathToFileURL(path.join(REPO_ROOT, "src", "lib", "state", "sandbox.ts")).href @@ -45,8 +44,7 @@ function writeBackup( } afterAll(() => { - delete process.env.HOME; - Object.assign(process.env, ORIGINAL_HOME === undefined ? {} : { HOME: ORIGINAL_HOME }); + vi.unstubAllEnvs(); fs.rmSync(TMP_HOME, { recursive: true, force: true }); }); @@ -83,6 +81,35 @@ describe("prepared rebuild backup recovery validation (#6114)", () => { }); }); + it("rejects a persisted manifest that disappears or becomes malformed after discovery", () => { + const candidate = writeBackup("alpha", "2026-07-01T06-50-42-044Z", { + agentVersion: "2026.5.27", + expectedVersion: "2026.5.27", + }); + const manifestPath = path.join( + BACKUPS_ROOT, + "alpha", + "2026-07-01T06-50-42-044Z", + "rebuild-manifest.json", + ); + + fs.unlinkSync(manifestPath); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", null, candidate as never)).toEqual( + { + ok: false, + reason: "latest backup manifest is missing, malformed, or unsupported", + }, + ); + + fs.writeFileSync(manifestPath, "{malformed"); + expect(sandboxState.validateRebuildRecoveryManifest("alpha", null, candidate as never)).toEqual( + { + ok: false, + reason: "latest backup manifest is missing, malformed, or unsupported", + }, + ); + }); + it("rejects sandbox, agent, and backup-path identity mismatches", () => { writeBackup("alpha", "2026-07-01T06-50-42-044Z", { sandboxName: "beta", From 35ab003bbba4159cb630b9e04566261ce5401c8b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 1 Jul 2026 15:13:36 -0700 Subject: [PATCH 6/7] fix(upgrade): reject ambiguous legacy image provenance Signed-off-by: Aaron Erickson --- docs/get-started/quickstart.mdx | 3 +- src/lib/actions/sandbox/rebuild.ts | 16 ++++--- .../upgrade-sandboxes-recovery.test.ts | 45 ++++++++++++++++--- src/lib/actions/upgrade-sandboxes.ts | 12 ++--- src/lib/state/sandbox.ts | 26 ++++------- test/snapshot-recovery-validation.test.ts | 22 ++------- 6 files changed, 69 insertions(+), 55 deletions(-) diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 9cb1424efe3..dfc009e32af 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -110,7 +110,8 @@ It prints a review summary before it registers the provider with OpenShell. After you confirm, NemoClaw registers inference, prompts for optional web search and messaging channels, builds and starts the sandbox, sets up OpenClaw, then applies the selected network policy tier and presets. At any prompt, press Enter to accept the default shown in `[brackets]`, type `back` to return to the previous prompt, or type `exit` to quit. If registered sandboxes already exist, the installer runs `nemoclaw backup-all` when the installed CLI supports it, then runs `nemoclaw upgrade-sandboxes --auto` before generic onboarding. -For a registered sandbox that is non-Ready after the host upgrade, the installer restores its validated latest backup only when the backup identity matches and there is positive managed-image evidence: a NemoClaw build fingerprint or, for legacy managed images, matching registry and backup agent versions. +For a registered sandbox that is non-Ready after the host upgrade, the installer restores its validated latest backup only when the backup identity matches and its registry entry has a NemoClaw-managed image fingerprint. +Pre-fingerprint and custom-image sandboxes are not recreated automatically because matching agent versions do not prove image provenance. If an automatic rebuild fails, or a non-Ready recovery is blocked or fails, the installer exits with a nonzero status and does not start generic onboarding. The inference provider prompt presents a numbered list. diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 0ec0474eac0..0d323decc82 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -654,10 +654,10 @@ function revalidatePreparedRecoveryBeforeDelete( bail, ); } - if (!sandboxState.hasPositiveManagedImageEvidence(currentEntry, validation.manifest)) { + if (!sandboxState.hasPositiveManagedImageEvidence(currentEntry)) { return failPreparedRecoveryPreDelete( - "backup no longer has positive NemoClaw-managed image evidence", - "Recovery backup is not proven to come from a NemoClaw-managed image.", + "registry no longer has a NemoClaw-managed image fingerprint", + "Recovery registry entry has no NemoClaw-managed image fingerprint.", bail, ); } @@ -705,14 +705,16 @@ export async function rebuildSandbox( bail(`Invalid recovery manifest: ${validation.reason}`); return; } - if (!sandboxState.hasPositiveManagedImageEvidence(sb, validation.manifest)) { + if (!sandboxState.hasPositiveManagedImageEvidence(sb)) { console.error(""); console.error( - ` ${_RD}Recovery preflight failed:${R} backup has no positive NemoClaw-managed image evidence.`, + ` ${_RD}Recovery preflight failed:${R} registry has no NemoClaw-managed image fingerprint.`, + ); + console.error( + " Pre-fingerprint and custom-image sandboxes are not recreated automatically.", ); - console.error(" Legacy custom-image sandboxes are not recreated automatically."); console.error(" Sandbox is untouched — no data was lost."); - bail("Recovery backup is not proven to come from a NemoClaw-managed image."); + bail("Recovery registry entry has no NemoClaw-managed image fingerprint."); return; } recoveryManifest = validation.manifest; diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index 2e84a1eff16..e47686ac449 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -42,7 +42,16 @@ function createRecoveryHarness( gatewayNames?: Record; liveOutput?: string; latestBackup?: ReturnType | null; + registryOverrides?: Record< + string, + Partial<{ + agent: "openclaw" | "hermes" | null; + agentVersion: string | null; + nemoclawVersion: string | null; + }> + >; staleNames?: string[]; + useRealManagedEvidence?: boolean; } = {}, ): { upgradeSandboxes: UpgradeSandboxes; @@ -81,6 +90,7 @@ function createRecoveryHarness( agentVersion: "2026.5.27", gatewayName: options.gatewayNames?.[name], nemoclawVersion: "0.0.71", + ...options.registryOverrides?.[name], })), }); vi.spyOn(sandboxVersion, "checkAgentVersion").mockImplementation((...args: unknown[]) => { @@ -103,9 +113,8 @@ function createRecoveryHarness( manifest: args[2] as ReturnType, }), ); - const managedEvidenceSpy = vi - .spyOn(sandboxState, "hasPositiveManagedImageEvidence") - .mockReturnValue(true); + const managedEvidenceSpy = vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence"); + if (!options.useRealManagedEvidence) managedEvidenceSpy.mockReturnValue(true); const rebuildSpy = vi.spyOn(rebuild, "rebuildSandbox").mockResolvedValue(undefined); return { @@ -153,16 +162,38 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { ); }); - it("fails closed without rebuilding a legacy custom-image backup", async () => { - const harness = createRecoveryHarness(["custom-box"]); - harness.managedEvidenceSpy.mockReturnValue(false); - vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + it("fails closed for a probed v0.0.55 custom image with matching backup agent version", async () => { + const probedAgentVersion = "2026.5.27"; + const harness = createRecoveryHarness(["custom-box"], { + latestBackup: { + ...makeManifest("custom-box"), + agentVersion: probedAgentVersion, + }, + registryOverrides: { + "custom-box": { + agentVersion: probedAgentVersion, + nemoclawVersion: null, + }, + }, + useRealManagedEvidence: true, + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { throw new Error(`process.exit(${code})`); }) as never); await expect(harness.upgradeSandboxes({ auto: true })).rejects.toThrow("process.exit(1)"); + expect(harness.managedEvidenceSpy).toHaveBeenCalledWith( + expect.objectContaining({ + agentVersion: probedAgentVersion, + nemoclawVersion: null, + }), + ); expect(harness.rebuildSpy).not.toHaveBeenCalled(); + expect(exitSpy).toHaveBeenCalledWith(1); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("registry has no NemoClaw-managed image fingerprint"), + ); }); it("warns and does not recover a stale registered sandbox absent from the selected gateway", async () => { diff --git a/src/lib/actions/upgrade-sandboxes.ts b/src/lib/actions/upgrade-sandboxes.ts index a867d28f0e8..fb49da4d13a 100644 --- a/src/lib/actions/upgrade-sandboxes.ts +++ b/src/lib/actions/upgrade-sandboxes.ts @@ -106,11 +106,11 @@ function prepareBackupRecovery( if (!validation.ok) { return { sandbox, reason: validation.reason }; } - if (!sandboxState.hasPositiveManagedImageEvidence(sandbox, validation.manifest)) { + if (!sandboxState.hasPositiveManagedImageEvidence(sandbox)) { return { sandbox, reason: - "backup has no positive NemoClaw-managed image evidence (legacy custom images are not auto-recreated)", + "registry has no NemoClaw-managed image fingerprint (pre-fingerprint and custom images are not auto-recreated)", }; } return { sandbox, manifest: validation.manifest }; @@ -190,10 +190,12 @@ export async function upgradeSandboxes( // That state comes from the already-installed legacy CLI/gateway and cannot be // prevented at its source by this candidate. install.sh exports this signal only // after that CLI completes backup-all, or after an operator asserts prepared - // upgrade state. upgrade-sandboxes-recovery.test.ts and + // upgrade state. Recovery remains limited to registry entries with a managed-image + // fingerprint; pre-fingerprint entries cannot prove provenance and fail closed. + // upgrade-sandboxes-recovery.test.ts and // install-preexisting-sandbox-recovery.test.ts guard the handoff. Remove this - // bridge with onboard's matching consumer once pre-fingerprint upgrades are no - // longer supported. + // bridge with onboard's matching consumer once prepared-backup installer recovery + // is no longer supported. const recoverPreparedBackups = process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE === "1"; const backupRecoveryAssessments = recoverPreparedBackups ? sandboxes.filter((sandbox) => nonReadyLiveNames.has(sandbox.name)).map(prepareBackupRecovery) diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 84587ce2527..f96b4611e9d 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1704,27 +1704,19 @@ export function validateRebuildRecoveryManifest( } /** - * Confirm that a backup came from a NemoClaw-managed image. Current registry - * entries carry a build fingerprint. The v0.0.55 source boundary predates that - * field: onboard registered `agentVersion` only when `fromDockerfile` was false, - * and backup copied that registry value into the manifest. Therefore an exact, - * non-empty legacy version match is positive managed-image evidence, while a - * `--from` custom image has a null registry version and fails closed. + * Confirm that a registry entry carries positive NemoClaw-managed image + * provenance. Managed images built by current releases receive a non-empty + * `nemoclawVersion` fingerprint, while custom images do not. * - * Remove the legacy version fallback once upgrades from releases without the - * `nemoclawVersion` fingerprint are no longer supported. + * `agentVersion` is not provenance: a live version probe can populate it for a + * legacy custom image, and backup then copies that value into the manifest. + * Pre-fingerprint entries therefore fail closed instead of inferring image + * ownership from matching agent versions. */ export function hasPositiveManagedImageEvidence( - sandbox: Pick, - manifest: Pick, + sandbox: Pick, ): boolean { - const fingerprint = String(sandbox.nemoclawVersion || "").trim(); - if (fingerprint) return true; - const registryAgentVersion = String(sandbox.agentVersion || "").trim(); - const manifestAgentVersion = String(manifest.agentVersion || "").trim(); - return Boolean( - registryAgentVersion && manifestAgentVersion && registryAgentVersion === manifestAgentVersion, - ); + return Boolean(String(sandbox.nemoclawVersion || "").trim()); } /** diff --git a/test/snapshot-recovery-validation.test.ts b/test/snapshot-recovery-validation.test.ts index 271377acdb5..1abaefeb722 100644 --- a/test/snapshot-recovery-validation.test.ts +++ b/test/snapshot-recovery-validation.test.ts @@ -142,23 +142,9 @@ describe("prepared rebuild backup recovery validation (#6114)", () => { }); }); - it("requires a build fingerprint or matching legacy managed-agent versions", () => { - const legacyManaged = { agentVersion: "2026.5.27", nemoclawVersion: null }; - const legacyCustom = { agentVersion: null, nemoclawVersion: null }; - const manifest = { agentVersion: "2026.5.27" }; - - expect(sandboxState.hasPositiveManagedImageEvidence(legacyManaged, manifest)).toBe(true); - expect( - sandboxState.hasPositiveManagedImageEvidence( - { agentVersion: null, nemoclawVersion: "0.0.71" }, - { agentVersion: null }, - ), - ).toBe(true); - expect(sandboxState.hasPositiveManagedImageEvidence(legacyCustom, manifest)).toBe(false); - expect( - sandboxState.hasPositiveManagedImageEvidence(legacyManaged, { - agentVersion: "different-version", - }), - ).toBe(false); + it("requires a non-empty managed-image fingerprint", () => { + expect(sandboxState.hasPositiveManagedImageEvidence({ nemoclawVersion: "0.0.71" })).toBe(true); + expect(sandboxState.hasPositiveManagedImageEvidence({ nemoclawVersion: null })).toBe(false); + expect(sandboxState.hasPositiveManagedImageEvidence({ nemoclawVersion: " " })).toBe(false); }); }); From f6d60800ad6b0b4128fd450ffc7173726f32f3c7 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 1 Jul 2026 15:24:47 -0700 Subject: [PATCH 7/7] test(upgrade): keep recovery harness linear Signed-off-by: Aaron Erickson --- src/lib/actions/upgrade-sandboxes-recovery.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index e47686ac449..939d037323a 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -113,8 +113,9 @@ function createRecoveryHarness( manifest: args[2] as ReturnType, }), ); - const managedEvidenceSpy = vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence"); - if (!options.useRealManagedEvidence) managedEvidenceSpy.mockReturnValue(true); + const managedEvidenceSpy = options.useRealManagedEvidence + ? vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence") + : vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence").mockReturnValue(true); const rebuildSpy = vi.spyOn(rebuild, "rebuildSandbox").mockResolvedValue(undefined); return {