fix(installer): skip unreachable running sandboxes in pre-upgrade backup

docs/reference/commands-nemohermes.mdx

@@ -1399,6 +1399,8 @@ nemohermes backup-all
 
 The installer calls `backup-all` automatically before onboarding to protect against data loss during OpenShell upgrades.
 
+A running sandbox whose in-sandbox SSH endpoint does not answer fails its backup and aborts the run. Set `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1` to skip such sandboxes and continue the upgrade; they are recovered from their latest validated backup during onboarding.
+
 ### `nemohermes <name> snapshot create`
 
 Create a timestamped snapshot of sandbox state.

docs/reference/commands.mdx

@@ -1768,6 +1768,8 @@ $$nemoclaw backup-all
 
 The installer calls `backup-all` automatically before onboarding to protect against data loss during OpenShell upgrades.
 
+A running sandbox whose in-sandbox SSH endpoint does not answer fails its backup and aborts the run. Set `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1` to skip such sandboxes and continue the upgrade; they are recovered from their latest validated backup during onboarding.
+
 ### `$$nemoclaw <name> snapshot create`
 
 Create a timestamped snapshot of sandbox state.

scripts/install.sh

@@ -1722,18 +1722,14 @@ resolve_prepared_cli_runner() {
 }
 
 run_preupgrade_backup() {
-  local old_cli_runner="$1" old_openshell_version="$2"
+  local old_cli_runner="$1"
 
   if "$old_cli_runner" backup-all 2>&1; then
     return 0
   fi
 
-  if ! legacy_openshell_gateway_upgrade_needed "$old_openshell_version"; then
-    return 1
-  fi
-
   warn "Pre-upgrade backup with the existing ${_CLI_BIN} CLI failed."
-  warn "Retrying with the current ${_CLI_DISPLAY} CLI before retiring the legacy OpenShell gateway."
+  warn "Retrying with the current ${_CLI_DISPLAY} CLI, which supports NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP."
   if ! prepare_current_cli_for_preupgrade_backup; then
     warn "Could not prepare the current ${_CLI_DISPLAY} CLI for backup retry."
     return 1
@@ -1907,11 +1903,11 @@ preinstall_backup_and_retire_legacy_gateway() {
   fi
 
   info "Backing up ${sandbox_count} sandbox(es) before upgrading OpenShell…"
-  if ! run_preupgrade_backup "$old_cli_runner" "$old_openshell_version"; then
+  if ! run_preupgrade_backup "$old_cli_runner"; then
     if legacy_openshell_gateway_upgrade_needed "$old_openshell_version"; then
       error "Pre-upgrade backup failed. Aborting before retiring the legacy OpenShell gateway."
     fi
-    error "Pre-upgrade backup failed. Fix the OpenShell gateway state, rerun '${_CLI_BIN} backup-all', then rerun the installer."
+    error "Pre-upgrade backup failed. If the failures are running sandboxes whose in-sandbox SSH endpoint is unreachable, rerun the installer with NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1 to continue and recover them after the upgrade; otherwise restore the affected sandbox or stop its container, then rerun '${_CLI_BIN} backup-all'."
   fi
   export NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE=1
 

src/lib/actions/maintenance.test.ts

@@ -54,7 +54,7 @@ vi.mock("../domain/maintenance/images", () => ({
   parseSandboxImageRows: vi.fn().mockReturnValue([]),
 }));
 
-import { backupAll } from "./maintenance";
+import { backupAll, shouldSkipUnreachableSandboxBackup } from "./maintenance";
 
 describe("backupAll", () => {
   beforeEach(() => {
@@ -186,4 +186,95 @@ describe("backupAll", () => {
 
     await expect(backupAll()).rejects.toThrow(/binary/);
   });
+
+  it("skips a running but SSH-unreachable sandbox when NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-bad" }, { name: "sb-good" }],
+      defaultSandbox: null,
+    });
+    mocks.backupSandboxState.mockImplementation((name: string) =>
+      name === "sb-bad"
+        ? {
+            success: false,
+            unreachable: true,
+            backedUpDirs: [],
+            failedDirs: ["memories"],
+            backedUpFiles: [],
+            failedFiles: [],
+          }
+        : {
+            success: true,
+            backedUpDirs: ["dir1"],
+            failedDirs: [],
+            backedUpFiles: [],
+            failedFiles: [],
+            manifest: { backupPath: "/backups/sb-good/timestamp" },
+          },
+    );
+
+    process.env.NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await backupAll();
+
+    const output = logSpy.mock.calls.map((c) => c[0]).join("\n");
+    expect(output).toContain("Skipped 'sb-bad'");
+    expect(output).toContain("1 backed up, 0 failed, 1 skipped");
+    expect(exitSpy).not.toHaveBeenCalled();
+
+    delete process.env.NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP;
+    logSpy.mockRestore();
+    exitSpy.mockRestore();
+  });
+
+  it("fails with actionable guidance when a running sandbox is unreachable and the skip flag is unset", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-bad" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set(["sb-bad"]));
+    mocks.captureSandboxListWithGatewayRecovery.mockResolvedValue({
+      result: { status: 0, output: "sb-bad\n" },
+    });
+    mocks.backupSandboxState.mockImplementation(() => ({
+      success: false,
+      unreachable: true,
+      backedUpDirs: [],
+      failedDirs: ["memories"],
+      backedUpFiles: [],
+      failedFiles: [],
+    }));
+
+    delete process.env.NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP;
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    const errorOutput = errorSpy.mock.calls.map((c) => c[0]).join("\n");
+    expect(errorOutput).toContain("NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1");
+
+    errorSpy.mockRestore();
+    exitSpy.mockRestore();
+  });
+});
+
+describe("shouldSkipUnreachableSandboxBackup", () => {
+  it("is true only for exactly '1'", () => {
+    expect(
+      shouldSkipUnreachableSandboxBackup({ NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP: "1" }),
+    ).toBe(true);
+    expect(
+      shouldSkipUnreachableSandboxBackup({ NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP: "0" }),
+    ).toBe(false);
+    expect(
+      shouldSkipUnreachableSandboxBackup({ NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP: "true" }),
+    ).toBe(false);
+    expect(shouldSkipUnreachableSandboxBackup({})).toBe(false);
+  });
 });

src/lib/actions/maintenance.ts

@@ -31,6 +31,10 @@ const R = useColor ? "\x1b[0m" : "";
 const RD = useColor ? "\x1b[1;31m" : "";
 const YW = useColor ? "\x1b[1;33m" : "";
 
+export function shouldSkipUnreachableSandboxBackup(env: NodeJS.ProcessEnv): boolean {
+  return env.NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP === "1";
+}
+
 export async function backupAll(): Promise<void> {
   const { sandboxes } = registry.listSandboxes();
   if (sandboxes.length === 0) {
@@ -63,9 +67,11 @@ export async function backupAll(): Promise<void> {
   }
   const readyNames = parseReadySandboxNames(liveList.output || "");
 
+  const skipUnreachable = shouldSkipUnreachableSandboxBackup(process.env);
   let backed = 0;
   let failed = 0;
   let skipped = 0;
+  let unreachableRunning = 0;
   for (const sb of sandboxes) {
     if (!readyNames.has(sb.name)) {
       console.log(`  ${D}Skipping '${sb.name}' (not running)${R}`);
@@ -122,6 +128,16 @@ export async function backupAll(): Promise<void> {
       );
       backed++;
     } else {
+      if (result.unreachable) {
+        if (skipUnreachable) {
+          console.log(
+            `  ${YW}⚠${R} Skipped '${sb.name}' (running but SSH-unreachable; NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1 set)`,
+          );
+          skipped++;
+          continue;
+        }
+        unreachableRunning++;
+      }
       const failedItems = [...result.failedDirs, ...result.failedFiles];
       console.error(`  ${RD}✗${R} ${sb.name}: backup failed (${failedItems.join(", ")})`);
       failed++;
@@ -133,6 +149,18 @@ export async function backupAll(): Promise<void> {
     console.log(`  Backups stored in: ~/.nemoclaw/rebuild-backups/`);
   }
   if (failed > 0) {
+    if (unreachableRunning > 0) {
+      console.error("");
+      console.error(
+        `  ${unreachableRunning} running sandbox(es) could not be backed up because their in-sandbox SSH endpoint did not answer.`,
+      );
+      console.error(
+        `  To upgrade now and recover them afterwards from their latest validated backup, re-run with NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1.`,
+      );
+      console.error(
+        `  To preserve their current state first, stop the affected container (so it is skipped as not running) or restore its gateway health, then run '${CLI_NAME} backup-all' again.`,
+      );
+    }
     process.exit(1);
   }
 }

src/lib/state/sandbox.test.ts

@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import { isSshTransportFailure } from "./sandbox";
+
+describe("isSshTransportFailure", () => {
+  it("treats an ssh transport failure exit code (255) as unreachable", () => {
+    expect(isSshTransportFailure({ status: 255 })).toBe(true);
+  });
+
+  it("treats a timed-out or signal-killed probe (null status) as unreachable", () => {
+    expect(isSshTransportFailure({ status: null })).toBe(true);
+  });
+
+  it("treats a spawn error as unreachable", () => {
+    expect(isSshTransportFailure({ status: null, error: new Error("spawn ETIMEDOUT") })).toBe(true);
+  });
+
+  it("does not treat a reachable non-zero remote exit as unreachable", () => {
+    expect(isSshTransportFailure({ status: 1 })).toBe(false);
+    expect(isSshTransportFailure({ status: 2 })).toBe(false);
+  });
+
+  it("does not treat a successful probe as unreachable", () => {
+    expect(isSshTransportFailure({ status: 0 })).toBe(false);
+  });
+});

src/lib/state/sandbox.ts

@@ -121,6 +121,7 @@ export interface BackupResult {
   error?: string;
   backedUpFiles: string[];
   failedFiles: string[];
+  unreachable?: boolean;
 }
 
 export interface RestoreResult {
@@ -1018,6 +1019,12 @@ function restoreStateFile(
  * Back up all state directories from a running sandbox.
  * Uses the agent manifest to determine which directories contain state.
  */
+export function isSshTransportFailure(result: { status: number | null; error?: Error }): boolean {
+  if (result.error) return true;
+  if (result.status === null) return true;
+  return result.status === 255;
+}
+
 export function backupSandboxState(sandboxName: string, options: BackupOptions = {}): BackupResult {
   const sb = registry.getSandbox(sandboxName);
   const agentName = sb?.agent || "openclaw";
@@ -1168,6 +1175,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions =
         );
         return {
           success: false,
+          unreachable: isSshTransportFailure(existResult),
           manifest,
           backedUpDirs,
           failedDirs: [...stateDirs],

test/install-openshell-upgrade-prompt.test.ts

@@ -210,11 +210,28 @@ describe("install.sh OpenShell 0.0.37 gateway upgrade prompt", () => {
 
     expect(result.status).not.toBe(0);
     expect(result.stdout + result.stderr).toContain(
-      "Fix the OpenShell gateway state, rerun 'nemoclaw backup-all', then rerun the installer.",
+      "If the failures are running sandboxes whose in-sandbox SSH endpoint is unreachable, rerun the installer with NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1 to continue and recover them after the upgrade; otherwise restore the affected sandbox or stop its container, then rerun 'nemoclaw backup-all'.",
     );
     expect(cliLog.split(/\r?\n/)).toContain("old:backup-all");
     expect(cliLog).not.toContain("--help");
-    expect(cliLog).not.toContain("prepare-current");
+    expect(cliLog.split(/\r?\n/)).toContain("prepare-current");
+    expect(openshellLog).toBe("");
+  });
+
+  it("retries current-gateway backup with the current CLI when the old CLI fails", () => {
+    const { result, cliLog, openshellLog } = runPreinstallUpgradeGuard(
+      {
+        NON_INTERACTIVE: "1",
+      },
+      { backupSucceeds: false, fallbackAvailable: true, openshellVersion: "0.0.37" },
+    );
+
+    expect(result.status).toBe(0);
+    expect(result.stdout + result.stderr).toContain("Retrying with the current NemoClaw CLI");
+    expect(result.stdout).toContain("RESTORE=1");
+    expect(cliLog.split(/\r?\n/)).toContain("old:backup-all");
+    expect(cliLog.split(/\r?\n/)).toContain("prepare-current");
+    expect(cliLog.split(/\r?\n/)).toContain("current:backup-all");
     expect(openshellLog).toBe("");
   });