fix(backup): back up stopped docker sandboxes by starting them for the backup

docs/manage-sandboxes/backup-restore.mdx

@@ -268,7 +268,8 @@ $$nemoclaw backup-all
 ```
 
 `backup-all` walks the sandboxes registered on the host, creates a snapshot for each running sandbox, and stores the snapshot bundles under `~/.nemoclaw/rebuild-backups/<name>/`.
-If a registered sandbox is stopped, start the sandbox or its container and rerun `$$nemoclaw backup-all` so NemoClaw can capture a fresh snapshot before maintenance.
+If a registered docker-driver sandbox's container is stopped, `backup-all` starts the container for the duration of the backup and returns it to its stopped state afterwards.
+If a sandbox is not running and its container cannot be started this way, start the sandbox or its container and rerun `$$nemoclaw backup-all` so NemoClaw can capture a fresh snapshot before maintenance.
 When a backup fails, NemoClaw identifies the affected state item and reports one of these causes when available: `permission denied`, `tar read error`, or `absent after extraction`.
 Use `$$nemoclaw <name> snapshot list` and `$$nemoclaw <name> snapshot restore` to inspect or restore one sandbox's bundles later.
 

docs/reference/commands.mdx

@@ -2497,15 +2497,16 @@ A custom OpenClaw sandbox is recoverable only when the selected backup independe
 ### `$$nemoclaw backup-all`
 
 Back up all registered running sandboxes to `~/.nemoclaw/rebuild-backups/`.
-Sandboxes that are not running are skipped with remediation guidance.
+A registered docker-driver sandbox whose container is stopped is started for the duration of the backup and returned to its stopped state afterwards.
+Sandboxes that are not running and cannot be started this way are skipped with remediation guidance.
 
 ```bash
 $$nemoclaw backup-all
 ```
 
 Before an OpenShell upgrade, the installer prepares the current release CLI and uses it to run `backup-all` in strict mode.
 Strict mode requires every registered sandbox to produce a fresh backup and aborts before gateway changes if any sandbox is skipped or fails.
-When strict mode reports a skipped stopped sandbox, start that sandbox or its container and rerun the installer or `$$nemoclaw backup-all`.
+When strict mode reports a skipped sandbox, start that sandbox or its container and rerun the installer or `$$nemoclaw backup-all`.
 
 A running sandbox whose in-sandbox SSH endpoint does not answer fails its backup and aborts the run.
 For a standalone `$$nemoclaw backup-all` run, set `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1` exactly to skip such sandboxes instead of failing.

src/lib/actions/maintenance.test.ts

@@ -11,6 +11,9 @@ const mocks = vi.hoisted(() => ({
   dockerListImagesFormat: vi.fn().mockReturnValue(""),
   dockerRmi: vi.fn(),
   prompt: vi.fn(),
+  startStoppedSandboxContainerForBackup: vi.fn(),
+  backupStartedSandboxState: vi.fn(),
+  returnSandboxContainerToStopped: vi.fn(),
 }));
 
 vi.mock("../state/registry", () => ({
@@ -36,6 +39,11 @@ vi.mock("../cli/branding", () => ({
 vi.mock("../credentials/store", () => ({
   prompt: mocks.prompt,
 }));
+vi.mock("./sandbox/stopped-sandbox-backup", () => ({
+  startStoppedSandboxContainerForBackup: mocks.startStoppedSandboxContainerForBackup,
+  backupStartedSandboxState: mocks.backupStartedSandboxState,
+  returnSandboxContainerToStopped: mocks.returnSandboxContainerToStopped,
+}));
 vi.mock("../domain/lifecycle/options", () => ({
   normalizeGarbageCollectImagesOptions: (o: unknown) => o || {},
 }));
@@ -54,6 +62,8 @@ describe("backupAll", () => {
       output: "sb-good\nsb-bad\n",
     });
     mocks.parseReadySandboxNames.mockReturnValue(new Set(["sb-good", "sb-bad"]));
+    mocks.startStoppedSandboxContainerForBackup.mockReturnValue(null);
+    mocks.returnSandboxContainerToStopped.mockReturnValue(true);
   });
 
   afterEach(() => {
@@ -191,6 +201,131 @@ describe("backupAll", () => {
     expect(errorOutput).not.toContain("prepare the upgrade manually");
   });
 
+  it("starts a stopped container, backs it up, and returns it to stopped so strict mode passes (#6500)", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-good" }, { name: "sb-stopped" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set(["sb-good"]));
+    mocks.backupSandboxState.mockReturnValue({
+      success: true,
+      backedUpDirs: ["workspace"],
+      failedDirs: [],
+      backedUpFiles: [],
+      failedFiles: [],
+      manifest: { backupPath: "/backups/sb-good/timestamp" },
+    });
+    mocks.startStoppedSandboxContainerForBackup.mockImplementation((name: string) =>
+      name === "sb-stopped" ? { containerName: "openshell-sb-stopped-abc" } : null,
+    );
+    mocks.backupStartedSandboxState.mockResolvedValue({
+      success: true,
+      backedUpDirs: ["workspace"],
+      failedDirs: [],
+      backedUpFiles: [],
+      failedFiles: [],
+      manifest: { backupPath: "/backups/sb-stopped/timestamp" },
+    });
+    process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await backupAll();
+
+    expect(exitSpy).not.toHaveBeenCalled();
+    expect(mocks.backupStartedSandboxState).toHaveBeenCalledWith("sb-stopped");
+    expect(mocks.backupSandboxState).toHaveBeenCalledWith("sb-good");
+    expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith("openshell-sb-stopped-abc");
+    const logOutput = logSpy.mock.calls.flat().join("\n");
+    expect(logOutput).toContain("Starting stopped sandbox 'sb-stopped' to back it up");
+    expect(logOutput).toContain("Returned 'sb-stopped' to its stopped state");
+    expect(logOutput).toContain("2 backed up, 0 failed, 0 skipped");
+    expect(logOutput).not.toContain("Skipping 'sb-stopped'");
+  });
+
+  it("returns the container to stopped and counts a failure when the started backup fails (#6500)", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-stopped" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set());
+    mocks.startStoppedSandboxContainerForBackup.mockReturnValue({
+      containerName: "openshell-sb-stopped-abc",
+    });
+    mocks.backupStartedSandboxState.mockResolvedValue({
+      success: false,
+      backedUpDirs: [],
+      failedDirs: ["identity"],
+      failedDirReasons: { identity: "permission denied" },
+      backedUpFiles: [],
+      failedFiles: [],
+    });
+    process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
+    vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith("openshell-sb-stopped-abc");
+    expect(logSpy.mock.calls.flat().join("\n")).toContain("0 backed up, 1 failed, 0 skipped");
+    expect(errorSpy.mock.calls.flat().join("\n")).toContain(
+      "backup failed (identity (permission denied))",
+    );
+  });
+
+  it("warns when the started container cannot be returned to its stopped state (#6500)", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-stopped" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set());
+    mocks.startStoppedSandboxContainerForBackup.mockReturnValue({
+      containerName: "openshell-sb-stopped-abc",
+    });
+    mocks.backupStartedSandboxState.mockResolvedValue({
+      success: true,
+      backedUpDirs: ["workspace"],
+      failedDirs: [],
+      backedUpFiles: [],
+      failedFiles: [],
+      manifest: { backupPath: "/backups/sb-stopped/timestamp" },
+    });
+    mocks.returnSandboxContainerToStopped.mockReturnValue(false);
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+
+    await backupAll();
+
+    expect(logSpy.mock.calls.flat().join("\n")).toContain(
+      "Could not return 'sb-stopped' to its stopped state after backup",
+    );
+  });
+
+  it("keeps the not-running skip when no stopped container can be started (#6114)", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-stopped" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set());
+    mocks.startStoppedSandboxContainerForBackup.mockReturnValue(null);
+    process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    vi.spyOn(console, "error").mockImplementation(() => undefined);
+    vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    expect(mocks.backupStartedSandboxState).not.toHaveBeenCalled();
+    expect(mocks.returnSandboxContainerToStopped).not.toHaveBeenCalled();
+    expect(logSpy.mock.calls.flat().join("\n")).toContain("Skipping 'sb-stopped' (not running");
+  });
+
   it("continues backup loop when backupSandboxState throws for one sandbox", async () => {
     mocks.listSandboxes.mockReturnValue({
       sandboxes: [{ name: "sb-bad" }, { name: "sb-good" }],

src/lib/actions/maintenance.ts

@@ -15,6 +15,12 @@ import { captureSandboxListWithGatewayPreflightOrExit } from "../openshell-sandb
 import { parseReadySandboxNames } from "../runtime-recovery";
 import * as registry from "../state/registry";
 import * as sandboxState from "../state/sandbox";
+import {
+  backupStartedSandboxState,
+  returnSandboxContainerToStopped,
+  type StartedForBackup,
+  startStoppedSandboxContainerForBackup,
+} from "./sandbox/stopped-sandbox-backup";
 
 const useColor = !process.env.NO_COLOR && !!process.stdout.isTTY;
 const trueColor =
@@ -54,16 +60,27 @@ export async function backupAll(): Promise<void> {
   let unreachableRunning = 0;
   let notRunningSkipped = 0;
   for (const sb of sandboxes) {
+    // A registered docker-driver sandbox whose container is merely stopped is
+    // backupable: start it for the duration of the backup and return it to
+    // its stopped state after (#6500). Anything else that is not Ready keeps
+    // the existing skip (and, under installer-strict mode, the #6114 gate).
+    let startedForBackup: StartedForBackup | null = null;
     if (!readyNames.has(sb.name)) {
-      console.log(`  ${D}${notRunningBackupSkipMessage(sb.name)}${R}`);
-      skipped++;
-      notRunningSkipped++;
-      continue;
+      startedForBackup = startStoppedSandboxContainerForBackup(sb.name);
+      if (!startedForBackup) {
+        console.log(`  ${D}${notRunningBackupSkipMessage(sb.name)}${R}`);
+        skipped++;
+        notRunningSkipped++;
+        continue;
+      }
+      console.log(`  Starting stopped sandbox '${sb.name}' to back it up...`);
     }
     console.log(`  Backing up '${sb.name}'...`);
     let result: sandboxState.BackupResult;
     try {
-      result = sandboxState.backupSandboxState(sb.name);
+      result = startedForBackup
+        ? await backupStartedSandboxState(sb.name)
+        : sandboxState.backupSandboxState(sb.name);
     } catch (err: unknown) {
       // Source-of-truth review (#5734 / #5819):
       //
@@ -103,6 +120,16 @@ export async function backupAll(): Promise<void> {
       console.log(`  ${YW}⚠${R} Skipped '${sb.name}' (orphan manifest): ${msg}`);
       skipped++;
       continue;
+    } finally {
+      if (startedForBackup) {
+        if (returnSandboxContainerToStopped(startedForBackup.containerName)) {
+          console.log(`  ${D}Returned '${sb.name}' to its stopped state.${R}`);
+        } else {
+          console.log(
+            `  ${YW}⚠${R} Could not return '${sb.name}' to its stopped state after backup; its container was left running.`,
+          );
+        }
+      }
     }
     if (result.success) {
       console.log(

src/lib/actions/sandbox/stopped-sandbox-backup.test.ts

@@ -0,0 +1,141 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("../../state/registry", () => ({
+  getSandbox: vi.fn(),
+  listSandboxes: vi.fn(),
+}));
+vi.mock("../../state/sandbox", () => ({
+  backupSandboxState: vi.fn(),
+}));
+
+import {
+  backupStartedSandboxState,
+  returnSandboxContainerToStopped,
+  startStoppedSandboxContainerForBackup,
+} from "./stopped-sandbox-backup";
+
+describe("startStoppedSandboxContainerForBackup", () => {
+  const deps = (over: Record<string, unknown> = {}) => ({
+    getSandboxDriver: vi.fn().mockReturnValue("docker"),
+    listSandboxNames: vi.fn().mockReturnValue(["my-sb"]),
+    dockerPsAllNames: vi.fn().mockReturnValue("openshell-my-sb-abc123\n"),
+    dockerInspectStatus: vi.fn().mockReturnValue("exited"),
+    dockerStart: vi.fn().mockReturnValue("openshell-my-sb-abc123"),
+    ...over,
+  });
+
+  it("starts an exited docker-driver container and reports its name", () => {
+    const d = deps();
+    expect(startStoppedSandboxContainerForBackup("my-sb", d)).toEqual({
+      containerName: "openshell-my-sb-abc123",
+    });
+    expect(d.dockerStart).toHaveBeenCalledWith("openshell-my-sb-abc123");
+  });
+
+  it("starts a created container (onboarded but never run)", () => {
+    const d = deps({ dockerInspectStatus: vi.fn().mockReturnValue("created") });
+    expect(startStoppedSandboxContainerForBackup("my-sb", d)).not.toBeNull();
+  });
+
+  it("leaves non-docker-driver sandboxes alone", () => {
+    const d = deps({ getSandboxDriver: vi.fn().mockReturnValue("kubernetes") });
+    expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull();
+    expect(d.dockerPsAllNames).not.toHaveBeenCalled();
+  });
+
+  it("returns null when no container owns the sandbox name", () => {
+    const d = deps({ dockerPsAllNames: vi.fn().mockReturnValue("openshell-other-x\n") });
+    expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull();
+    expect(d.dockerStart).not.toHaveBeenCalled();
+  });
+
+  it("leaves a running-but-not-Ready container alone (crash loop, gateway drift)", () => {
+    const d = deps({ dockerInspectStatus: vi.fn().mockReturnValue("running") });
+    expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull();
+    expect(d.dockerStart).not.toHaveBeenCalled();
+  });
+
+  it("leaves a paused container alone (#4495)", () => {
+    const d = deps({ dockerInspectStatus: vi.fn().mockReturnValue("paused") });
+    expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull();
+    expect(d.dockerStart).not.toHaveBeenCalled();
+  });
+
+  it("returns null when docker start fails", () => {
+    const d = deps({ dockerStart: vi.fn().mockReturnValue("") });
+    expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull();
+  });
+});
+
+describe("returnSandboxContainerToStopped", () => {
+  it("reports success when docker stop echoes the container name", () => {
+    const dockerStop = vi.fn().mockReturnValue("openshell-my-sb-abc123");
+    expect(returnSandboxContainerToStopped("openshell-my-sb-abc123", { dockerStop })).toBe(true);
+    expect(dockerStop).toHaveBeenCalledWith("openshell-my-sb-abc123");
+  });
+
+  it("reports failure when docker stop produces no output", () => {
+    const dockerStop = vi.fn().mockReturnValue("");
+    expect(returnSandboxContainerToStopped("openshell-my-sb-abc123", { dockerStop })).toBe(false);
+  });
+});
+
+describe("backupStartedSandboxState", () => {
+  const ok = {
+    success: true,
+    backedUpDirs: [],
+    failedDirs: [],
+    backedUpFiles: [],
+    failedFiles: [],
+  };
+  const unreachable = { ...ok, success: false, unreachable: true };
+  const denied = { ...ok, success: false };
+
+  it("retries while the just-started container's SSH endpoint is unreachable (#6500)", async () => {
+    const backup = vi
+      .fn()
+      .mockReturnValueOnce(unreachable)
+      .mockReturnValueOnce(unreachable)
+      .mockReturnValueOnce(ok);
+    const sleep = vi.fn().mockResolvedValue(undefined);
+    const result = await backupStartedSandboxState("my-sb", {
+      backup,
+      sleep,
+      attempts: 5,
+      delayMs: 1,
+    });
+    expect(result.success).toBe(true);
+    expect(backup).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it("returns a non-transport failure without retrying", async () => {
+    const backup = vi.fn().mockReturnValue(denied);
+    const sleep = vi.fn().mockResolvedValue(undefined);
+    const result = await backupStartedSandboxState("my-sb", {
+      backup,
+      sleep,
+      attempts: 5,
+      delayMs: 1,
+    });
+    expect(result.success).toBe(false);
+    expect(backup).toHaveBeenCalledTimes(1);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it("gives up after the attempt budget while still unreachable", async () => {
+    const backup = vi.fn().mockReturnValue(unreachable);
+    const sleep = vi.fn().mockResolvedValue(undefined);
+    const result = await backupStartedSandboxState("my-sb", {
+      backup,
+      sleep,
+      attempts: 3,
+      delayMs: 1,
+    });
+    expect(result.unreachable).toBe(true);
+    expect(backup).toHaveBeenCalledTimes(3);
+  });
+});