fix(installer): skip unreachable running sandboxes in pre-upgrade backup

docs/reference/commands-nemohermes.mdx

@@ -1399,6 +1399,12 @@ nemohermes backup-all
 
 The installer calls `backup-all` automatically before onboarding to protect against data loss during OpenShell upgrades.
 
+A running sandbox whose in-sandbox SSH endpoint does not answer fails its backup and aborts the run.
+Set `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1` (exactly — other values like `true`, `yes`, or `0` are not accepted) to skip such sandboxes and continue the upgrade.
+When the installer invokes `backup-all` before an OpenShell upgrade, skipped sandboxes are automatically restored from their latest validated backup during post-upgrade onboarding.
+Standalone `nemohermes backup-all` invocations only skip the failure — they do not schedule a subsequent restore.
+Any uncommitted state since the last successful backup will be lost.
+
 ### `nemohermes <name> snapshot create`
 
 Create a timestamped snapshot of sandbox state.
@@ -2133,6 +2139,7 @@ The following flags change defaults for commands that manage existing sandboxes.
 | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `nemohermes <name> connect` and `nemohermes <name> connect --probe-only`. Use only as a troubleshooting escape hatch. |
 | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. |
 | `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `nemohermes <name> shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. |
+| `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to the installer's automatic pre-upgrade `nemohermes backup-all` and to manual `nemohermes backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer so the upgrade proceeds instead of aborting. Skipped sandboxes are restored from their latest validated backup during the installer's post-upgrade onboarding; any uncommitted state since that backup is lost. Standalone `nemohermes backup-all` invocations only skip the failure — they do not schedule a subsequent restore. |
 | `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA` | `1` to opt in | Acknowledges data loss during `nemohermes uninstall` and removes the otherwise-preserved entries (`rebuild-backups/`, `backups/`, `sandboxes.json`) under `~/.nemoclaw/`. Equivalent to passing the `--destroy-user-data` flag; the global `Proceed?` confirmation still applies unless `--yes` is also passed. |
 
 ### Legacy `nemohermes setup`

docs/reference/commands.mdx

@@ -1768,6 +1768,12 @@ $$nemoclaw backup-all
 
 The installer calls `backup-all` automatically before onboarding to protect against data loss during OpenShell upgrades.
 
+A running sandbox whose in-sandbox SSH endpoint does not answer fails its backup and aborts the run.
+Set `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1` (exactly — other values like `true`, `yes`, or `0` are not accepted) to skip such sandboxes and continue the upgrade.
+When the installer invokes `backup-all` before an OpenShell upgrade, skipped sandboxes are automatically restored from their latest validated backup during post-upgrade onboarding.
+Standalone `$$nemoclaw backup-all` invocations only skip the failure — they do not schedule a subsequent restore.
+Any uncommitted state since the last successful backup will be lost.
+
 ### `$$nemoclaw <name> snapshot create`
 
 Create a timestamped snapshot of sandbox state.
@@ -2618,6 +2624,7 @@ The following flags change defaults for commands that manage existing sandboxes.
 | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw <name> connect` and `$$nemoclaw <name> connect --probe-only`. Use only as a troubleshooting escape hatch. |
 | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. |
 | `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw <name> shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. |
+| `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to the installer's automatic pre-upgrade `$$nemoclaw backup-all` and to manual `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer so the upgrade proceeds instead of aborting. Skipped sandboxes are restored from their latest validated backup during the installer's post-upgrade onboarding; any uncommitted state since that backup is lost. Standalone `$$nemoclaw backup-all` invocations only skip the failure — they do not schedule a subsequent restore. |
 | `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA` | `1` to opt in | Acknowledges data loss during `$$nemoclaw uninstall` and removes the otherwise-preserved entries (`rebuild-backups/`, `backups/`, `sandboxes.json`) under `~/.nemoclaw/`. Equivalent to passing the `--destroy-user-data` flag; the global `Proceed?` confirmation still applies unless `--yes` is also passed. |
 
 <AgentOnly variant="openclaw">

scripts/install.sh

@@ -1722,18 +1722,14 @@ resolve_prepared_cli_runner() {
 }
 
 run_preupgrade_backup() {
-  local old_cli_runner="$1" old_openshell_version="$2"
+  local old_cli_runner="$1"
 
   if "$old_cli_runner" backup-all 2>&1; then
     return 0
   fi
 
-  if ! legacy_openshell_gateway_upgrade_needed "$old_openshell_version"; then
-    return 1
-  fi
-
   warn "Pre-upgrade backup with the existing ${_CLI_BIN} CLI failed."
-  warn "Retrying with the current ${_CLI_DISPLAY} CLI before retiring the legacy OpenShell gateway."
+  warn "Retrying with the current ${_CLI_DISPLAY} CLI, which supports NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP."
   if ! prepare_current_cli_for_preupgrade_backup; then
     warn "Could not prepare the current ${_CLI_DISPLAY} CLI for backup retry."
     return 1
@@ -1907,11 +1903,11 @@ preinstall_backup_and_retire_legacy_gateway() {
   fi
 
   info "Backing up ${sandbox_count} sandbox(es) before upgrading OpenShell…"
-  if ! run_preupgrade_backup "$old_cli_runner" "$old_openshell_version"; then
+  if ! run_preupgrade_backup "$old_cli_runner"; then
     if legacy_openshell_gateway_upgrade_needed "$old_openshell_version"; then
       error "Pre-upgrade backup failed. Aborting before retiring the legacy OpenShell gateway."
     fi
-    error "Pre-upgrade backup failed. Fix the OpenShell gateway state, rerun '${_CLI_BIN} backup-all', then rerun the installer."
+    error "Pre-upgrade backup failed. If the failures are running sandboxes whose in-sandbox SSH endpoint is unreachable, rerun the installer with NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1 to continue and recover them after the upgrade (any uncommitted state since the last successful backup will be lost); otherwise restore the affected sandbox or stop its container, then rerun '${_CLI_BIN} backup-all'."
   fi
   export NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE=1
 

src/lib/actions/maintenance.test.ts

@@ -54,7 +54,7 @@ vi.mock("../domain/maintenance/images", () => ({
   parseSandboxImageRows: vi.fn().mockReturnValue([]),
 }));
 
-import { backupAll } from "./maintenance";
+import { backupAll, shouldSkipUnreachableSandboxBackup } from "./maintenance";
 
 describe("backupAll", () => {
   beforeEach(() => {
@@ -186,4 +186,95 @@ describe("backupAll", () => {
 
     await expect(backupAll()).rejects.toThrow(/binary/);
   });
+
+  it("skips a running but SSH-unreachable sandbox when NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-bad" }, { name: "sb-good" }],
+      defaultSandbox: null,
+    });
+    mocks.backupSandboxState.mockImplementation((name: string) =>
+      name === "sb-bad"
+        ? {
+            success: false,
+            unreachable: true,
+            backedUpDirs: [],
+            failedDirs: ["memories"],
+            backedUpFiles: [],
+            failedFiles: [],
+          }
+        : {
+            success: true,
+            backedUpDirs: ["dir1"],
+            failedDirs: [],
+            backedUpFiles: [],
+            failedFiles: [],
+            manifest: { backupPath: "/backups/sb-good/timestamp" },
+          },
+    );
+
+    process.env.NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await backupAll();
+
+    const output = logSpy.mock.calls.map((c) => c[0]).join("\n");
+    expect(output).toContain("Skipped 'sb-bad'");
+    expect(output).toContain("1 backed up, 0 failed, 1 skipped");
+    expect(exitSpy).not.toHaveBeenCalled();
+
+    delete process.env.NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP;
+    logSpy.mockRestore();
+    exitSpy.mockRestore();
+  });
+
+  it("fails with actionable guidance when a running sandbox is unreachable and the skip flag is unset", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-bad" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set(["sb-bad"]));
+    mocks.captureSandboxListWithGatewayRecovery.mockResolvedValue({
+      result: { status: 0, output: "sb-bad\n" },
+    });
+    mocks.backupSandboxState.mockImplementation(() => ({
+      success: false,
+      unreachable: true,
+      backedUpDirs: [],
+      failedDirs: ["memories"],
+      backedUpFiles: [],
+      failedFiles: [],
+    }));
+
+    delete process.env.NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP;
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    const errorOutput = errorSpy.mock.calls.map((c) => c[0]).join("\n");
+    expect(errorOutput).toContain("NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1");
+
+    errorSpy.mockRestore();
+    exitSpy.mockRestore();
+  });
+});
+
+describe("shouldSkipUnreachableSandboxBackup", () => {
+  it("is true only for exactly '1'", () => {
+    expect(
+      shouldSkipUnreachableSandboxBackup({ NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP: "1" }),
+    ).toBe(true);
+    expect(
+      shouldSkipUnreachableSandboxBackup({ NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP: "0" }),
+    ).toBe(false);
+    expect(
+      shouldSkipUnreachableSandboxBackup({ NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP: "true" }),
+    ).toBe(false);
+    expect(shouldSkipUnreachableSandboxBackup({})).toBe(false);
+  });
 });

src/lib/actions/maintenance.ts

@@ -31,6 +31,10 @@ const R = useColor ? "\x1b[0m" : "";
 const RD = useColor ? "\x1b[1;31m" : "";
 const YW = useColor ? "\x1b[1;33m" : "";
 
+export function shouldSkipUnreachableSandboxBackup(env: NodeJS.ProcessEnv): boolean {
+  return env.NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP === "1";
+}
+
 export async function backupAll(): Promise<void> {
   const { sandboxes } = registry.listSandboxes();
   if (sandboxes.length === 0) {
@@ -63,9 +67,11 @@ export async function backupAll(): Promise<void> {
   }
   const readyNames = parseReadySandboxNames(liveList.output || "");
 
+  const skipUnreachable = shouldSkipUnreachableSandboxBackup(process.env);
   let backed = 0;
   let failed = 0;
   let skipped = 0;
+  let unreachableRunning = 0;
   for (const sb of sandboxes) {
     if (!readyNames.has(sb.name)) {
       console.log(`  ${D}Skipping '${sb.name}' (not running)${R}`);
@@ -122,6 +128,16 @@ export async function backupAll(): Promise<void> {
       );
       backed++;
     } else {
+      if (result.unreachable) {
+        if (skipUnreachable) {
+          console.log(
+            `  ${YW}⚠${R} Skipped '${sb.name}' (running but SSH-unreachable; NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1 set). Any uncommitted state since the last successful backup will be lost.`,
+          );
+          skipped++;
+          continue;
+        }
+        unreachableRunning++;
+      }
       const failedItems = [...result.failedDirs, ...result.failedFiles];
       console.error(`  ${RD}✗${R} ${sb.name}: backup failed (${failedItems.join(", ")})`);
       failed++;
@@ -133,6 +149,18 @@ export async function backupAll(): Promise<void> {
     console.log(`  Backups stored in: ~/.nemoclaw/rebuild-backups/`);
   }
   if (failed > 0) {
+    if (unreachableRunning > 0) {
+      console.error("");
+      console.error(
+        `  ${unreachableRunning} running sandbox(es) could not be backed up because their in-sandbox SSH endpoint did not answer.`,
+      );
+      console.error(
+        `  To upgrade now and recover them afterwards from their latest validated backup, re-run with NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1. Any uncommitted state since the last successful backup will be lost.`,
+      );
+      console.error(
+        `  To preserve their current state first, stop the affected container (so it is skipped as not running) or restore its gateway health, then run '${CLI_NAME} backup-all' again.`,
+      );
+    }
     process.exit(1);
   }
 }