fix(sandbox): recover gateway before maintenance sandbox-list retry

src/commands/backup-all.ts

@@ -15,6 +15,6 @@ export default class BackupAllCommand extends NemoClawCommand {
 
   public async run(): Promise<void> {
     await this.parse(BackupAllCommand);
-    runBackupAllAction();
+    await runBackupAllAction();
   }
 }

src/lib/actions/gateway-drift-preflight.test.ts

@@ -3,7 +3,7 @@
 
 import { createRequire } from "node:module";
 
-import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest";
 
 import type { OpenShellStateRpcIssue } from "../adapters/openshell/gateway-drift";
 
@@ -41,6 +41,7 @@ describe("gateway drift preflight for maintenance actions", () => {
   let detectPreflightIssueSpy: MockInstance;
   let detectResultIssueSpy: MockInstance;
   let printIssueSpy: MockInstance;
+  let recoverNamedGatewayRuntimeSpy: MockInstance;
 
   beforeEach(async () => {
     spies = [];
@@ -54,6 +55,7 @@ describe("gateway drift preflight for maintenance actions", () => {
     const sandboxVersion = requireDist("../../../dist/lib/sandbox/version.js");
     const upgradeDomain = requireDist("../../../dist/lib/domain/maintenance/upgrade.js");
     const rebuild = requireDist("../../../dist/lib/actions/sandbox/rebuild.js");
+    const gatewayRuntime = requireDist("../../../dist/lib/gateway-runtime-action.js");
 
     detectPreflightIssueSpy = vi
       .spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue")
@@ -78,6 +80,9 @@ describe("gateway drift preflight for maintenance actions", () => {
     classifyUpgradeableSandboxesSpy = vi
       .spyOn(upgradeDomain, "classifyUpgradeableSandboxes")
       .mockReturnValue({ stale: [], unknown: [] });
+    recoverNamedGatewayRuntimeSpy = vi
+      .spyOn(gatewayRuntime, "recoverNamedGatewayRuntime")
+      .mockResolvedValue({ recovered: true });
 
     spies.push(
       detectPreflightIssueSpy,
@@ -86,6 +91,7 @@ describe("gateway drift preflight for maintenance actions", () => {
       captureOpenshellSpy,
       backupSandboxStateSpy,
       classifyUpgradeableSandboxesSpy,
+      recoverNamedGatewayRuntimeSpy,
       vi.spyOn(registry, "listSandboxes").mockReturnValue({
         sandboxes: [{ name: "alpha", provider: "nvidia-prod", model: "nemotron" }],
       } as never),
@@ -108,35 +114,63 @@ describe("gateway drift preflight for maintenance actions", () => {
     errorSpy.mockRestore();
   });
 
-  it("backup-all fails before sandbox list when gateway image drift is detected", () => {
+  it("backup-all fails before sandbox list when gateway image drift is detected", async () => {
     detectPreflightIssueSpy.mockReturnValue(driftIssue);
 
-    expect(() => backupAll()).toThrow("process.exit(1)");
+    await expect(backupAll()).rejects.toThrow("process.exit(1)");
 
     expect(printIssueSpy).toHaveBeenCalledWith(
       driftIssue,
       expect.objectContaining({ command: "nemoclaw backup-all" }),
     );
     expect(captureOpenshellSpy).not.toHaveBeenCalled();
     expect(backupSandboxStateSpy).not.toHaveBeenCalled();
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
   });
 
-  it("backup-all fails closed on protobuf mismatch instead of treating sandboxes as stopped", () => {
+  it("backup-all recovers the named gateway and retries the sandbox list before backing up", async () => {
+    captureOpenshellSpy
+      .mockReturnValueOnce({ status: 1, output: "client error (Connect): Connection refused" })
+      .mockReturnValueOnce({ status: 0, output: "alpha Ready" });
+
+    await backupAll();
+
+    expect(recoverNamedGatewayRuntimeSpy).toHaveBeenCalledWith({
+      recoverableStates: ["missing_named", "named_unhealthy", "named_unreachable"],
+    });
+    expect(captureOpenshellSpy).toHaveBeenCalledTimes(2);
+    expect(captureOpenshellSpy).toHaveBeenNthCalledWith(1, ["sandbox", "list"]);
+    expect(captureOpenshellSpy).toHaveBeenNthCalledWith(2, ["sandbox", "list"]);
+    expect(backupSandboxStateSpy).toHaveBeenCalledWith("alpha");
+  });
+
+  it("backup-all does not recover generic sandbox list failures", async () => {
+    captureOpenshellSpy.mockReturnValue({ status: 1, output: "usage: openshell sandbox list" });
+
+    await expect(backupAll()).rejects.toThrow("process.exit(1)");
+
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
+    expect(captureOpenshellSpy).toHaveBeenCalledTimes(1);
+    expect(backupSandboxStateSpy).not.toHaveBeenCalled();
+  });
+
+  it("backup-all fails closed on protobuf mismatch instead of treating sandboxes as stopped", async () => {
     const protobufIssue: OpenShellStateRpcIssue = {
       kind: "protobuf_mismatch",
       output: "Sandbox.metadata: SandboxResponse.sandbox: invalid wire type value: 6",
     };
     captureOpenshellSpy.mockReturnValue({ status: 1, output: protobufIssue.output });
     detectResultIssueSpy.mockReturnValue(protobufIssue);
 
-    expect(() => backupAll()).toThrow("process.exit(1)");
+    await expect(backupAll()).rejects.toThrow("process.exit(1)");
 
     expect(printIssueSpy).toHaveBeenCalledWith(
       protobufIssue,
       expect.objectContaining({ command: "nemoclaw backup-all" }),
     );
     expect(captureOpenshellSpy).toHaveBeenCalledWith(["sandbox", "list"]);
     expect(backupSandboxStateSpy).not.toHaveBeenCalled();
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
   });
 
   it("upgrade-sandboxes fails before sandbox list when gateway image drift is detected", async () => {
@@ -150,6 +184,35 @@ describe("gateway drift preflight for maintenance actions", () => {
     );
     expect(captureOpenshellSpy).not.toHaveBeenCalled();
     expect(classifyUpgradeableSandboxesSpy).not.toHaveBeenCalled();
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
+  });
+
+  it("upgrade-sandboxes recovers the named gateway and retries before classifying sandboxes", async () => {
+    captureOpenshellSpy
+      .mockReturnValueOnce({ status: 1, output: "client error (Connect): Connection refused" })
+      .mockReturnValueOnce({ status: 0, output: "alpha Ready" });
+
+    await upgradeSandboxes({ check: true });
+
+    expect(recoverNamedGatewayRuntimeSpy).toHaveBeenCalledWith({
+      recoverableStates: ["missing_named", "named_unhealthy", "named_unreachable"],
+    });
+    expect(captureOpenshellSpy).toHaveBeenCalledTimes(2);
+    expect(classifyUpgradeableSandboxesSpy).toHaveBeenCalledWith(
+      [{ name: "alpha", provider: "nvidia-prod", model: "nemotron" }],
+      new Set(["alpha"]),
+      expect.any(Function),
+    );
+  });
+
+  it("upgrade-sandboxes does not recover generic sandbox list failures", async () => {
+    captureOpenshellSpy.mockReturnValue({ status: 1, output: "unknown option: --json" });
+
+    await expect(upgradeSandboxes({ check: true })).rejects.toThrow("process.exit(1)");
+
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
+    expect(captureOpenshellSpy).toHaveBeenCalledTimes(1);
+    expect(classifyUpgradeableSandboxesSpy).not.toHaveBeenCalled();
   });
 
   it("upgrade-sandboxes fails closed on protobuf mismatch before classifying stopped sandboxes", async () => {
@@ -168,5 +231,6 @@ describe("gateway drift preflight for maintenance actions", () => {
     );
     expect(captureOpenshellSpy).toHaveBeenCalledWith(["sandbox", "list"]);
     expect(classifyUpgradeableSandboxesSpy).not.toHaveBeenCalled();
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
   });
 });

src/lib/actions/global.test.ts

@@ -58,7 +58,7 @@ describe("global cli action facade", () => {
     await runSetupAction(["--fresh"]);
     await runSetupSparkAction(["--name", "alpha"]);
     await runDeployAction("gpu-alpha");
-    runBackupAllAction();
+    await runBackupAllAction();
     await runGarbageCollectImagesAction({ dryRun: true });
     showRootHelp();
     showVersion();

src/lib/actions/global.ts

@@ -1,11 +1,13 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { runDeployAction as executeDeployAction } from "./deploy";
+import { runOpenshell } from "../adapters/openshell/runtime";
 import {
   type GarbageCollectImagesOptions,
   type UpgradeSandboxesOptions,
 } from "../domain/lifecycle/options";
+import { recoverNamedGatewayRuntime as recoverNamedGatewayRuntimeAction } from "../gateway-runtime-action";
+import { runDeployAction as executeDeployAction } from "./deploy";
 import {
   backupAll as executeBackupAllAction,
   garbageCollectImages as executeGarbageCollectImagesAction,
@@ -15,8 +17,6 @@ import {
   runSetupAction as executeSetupAction,
   runSetupSparkAction as executeSetupSparkAction,
 } from "./onboard";
-import { recoverNamedGatewayRuntime as recoverNamedGatewayRuntimeAction } from "../gateway-runtime-action";
-import { runOpenshell } from "../adapters/openshell/runtime";
 import { help, version } from "./root-help";
 
 type GatewayRecovery = { recovered: boolean };
@@ -51,8 +51,8 @@ export async function runDeployAction(instanceName?: string): Promise<void> {
   await executeDeployAction(instanceName);
 }
 
-export function runBackupAllAction(): void {
-  executeBackupAllAction();
+export async function runBackupAllAction(): Promise<void> {
+  await executeBackupAllAction();
 }
 
 export async function runUpgradeSandboxesAction(

src/lib/actions/maintenance.ts

@@ -2,22 +2,25 @@
 // SPDX-License-Identifier: Apache-2.0
 
 
+import { dockerListImagesFormat, dockerRmi } from "../adapters/docker";
+import {
+  detectOpenShellStateRpcPreflightIssue,
+  detectOpenShellStateRpcResultIssue,
+  printOpenShellStateRpcIssue,
+} from "../adapters/openshell/gateway-drift";
+import { CLI_NAME } from "../cli/branding";
 import { prompt as askPrompt } from "../credentials/store";
 import {
   type GarbageCollectImagesOptions,
   normalizeGarbageCollectImagesOptions,
 } from "../domain/lifecycle/options";
-import { CLI_NAME } from "../cli/branding";
-import { dockerListImagesFormat, dockerRmi } from "../adapters/docker";
 import { findOrphanedSandboxImages, parseSandboxImageRows } from "../domain/maintenance/images";
-import { captureOpenshell } from "../adapters/openshell/runtime";
 import {
-  detectOpenShellStateRpcPreflightIssue,
-  detectOpenShellStateRpcResultIssue,
-  printOpenShellStateRpcIssue,
-} from "../adapters/openshell/gateway-drift";
-import * as registry from "../state/registry";
+  captureSandboxListWithGatewayRecovery,
+  printSandboxListFailureWithRecoveryContext,
+} from "../openshell-sandbox-list";
 import { parseLiveSandboxNames } from "../runtime-recovery";
+import * as registry from "../state/registry";
 import * as sandboxState from "../state/sandbox";
 
 const useColor = !process.env.NO_COLOR && !!process.stdout.isTTY;
@@ -29,7 +32,7 @@ const R = useColor ? "\x1b[0m" : "";
 const RD = useColor ? "\x1b[1;31m" : "";
 const YW = useColor ? "\x1b[1;33m" : "";
 
-export function backupAll(): void {
+export async function backupAll(): Promise<void> {
   const { sandboxes } = registry.listSandboxes();
   if (sandboxes.length === 0) {
     console.log("  No sandboxes registered. Nothing to back up.");
@@ -45,7 +48,8 @@ export function backupAll(): void {
     process.exit(1);
   }
 
-  const liveList = captureOpenshell(["sandbox", "list"]);
+  const liveListRecovery = await captureSandboxListWithGatewayRecovery();
+  const liveList = liveListRecovery.result;
   const resultIssue = detectOpenShellStateRpcResultIssue(liveList);
   if (resultIssue) {
     printOpenShellStateRpcIssue(resultIssue, {
@@ -55,8 +59,7 @@ export function backupAll(): void {
     process.exit(1);
   }
   if (liveList.status !== 0) {
-    console.error("  Failed to query running sandboxes from OpenShell.");
-    console.error("  Ensure OpenShell is running: openshell status");
+    printSandboxListFailureWithRecoveryContext(liveListRecovery);
     process.exit(liveList.status || 1);
   }
   const liveNames = parseLiveSandboxNames(liveList.output || "");

src/lib/actions/sandbox/rebuild-gateway-drift.test.ts

@@ -3,7 +3,7 @@
 
 import { createRequire } from "node:module";
 
-import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest";
 
 import type { OpenShellStateRpcIssue } from "../../adapters/openshell/gateway-drift";
 
@@ -33,33 +33,51 @@ describe("rebuild gateway drift preflight", () => {
   let errorSpy: MockInstance;
   let spies: MockInstance[];
   let checkAgentVersionSpy: MockInstance;
+  let detectPreflightIssueSpy: MockInstance;
+  let captureOpenshellSpy: MockInstance;
   let printIssueSpy: MockInstance;
+  let recoverNamedGatewayRuntimeSpy: MockInstance;
 
   beforeEach(async () => {
     spies = [];
     exitSpy = mockExit();
     errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
 
     const gatewayDrift = requireDist("../../../../dist/lib/adapters/openshell/gateway-drift.js");
+    const openshellRuntime = requireDist("../../../../dist/lib/adapters/openshell/runtime.js");
+    const gatewayRuntime = requireDist("../../../../dist/lib/gateway-runtime-action.js");
     const registry = requireDist("../../../../dist/lib/state/registry.js");
     const resolve = requireDist("../../../../dist/lib/adapters/openshell/resolve.js");
     const sandboxSession = requireDist("../../../../dist/lib/state/sandbox-session.js");
+    const onboardSession = requireDist("../../../../dist/lib/state/onboard-session.js");
     const sandboxVersion = requireDist("../../../../dist/lib/sandbox/version.js");
+    const agentRuntime = requireDist("../../../../dist/lib/agent/runtime.js");
 
     printIssueSpy = vi
       .spyOn(gatewayDrift, "printOpenShellStateRpcIssue")
       .mockImplementation(() => undefined);
+    detectPreflightIssueSpy = vi
+      .spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue")
+      .mockReturnValue(driftIssue);
     checkAgentVersionSpy = vi
       .spyOn(sandboxVersion, "checkAgentVersion")
       .mockReturnValue({ expectedVersion: "0.1.0", sandboxVersion: "0.0.1" } as never);
+    captureOpenshellSpy = vi
+      .spyOn(openshellRuntime, "captureOpenshell")
+      .mockReturnValue({ status: 0, output: "alpha Ready" });
+    recoverNamedGatewayRuntimeSpy = vi
+      .spyOn(gatewayRuntime, "recoverNamedGatewayRuntime")
+      .mockResolvedValue({ recovered: true });
 
     spies.push(
-      vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(driftIssue),
+      detectPreflightIssueSpy,
       vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null),
+      captureOpenshellSpy,
+      recoverNamedGatewayRuntimeSpy,
       printIssueSpy,
       vi.spyOn(registry, "getSandbox").mockReturnValue({
         name: "alpha",
-        provider: "nvidia-prod",
+        provider: "ollama-local",
         model: "nvidia/nemotron",
         policies: [],
         nimContainer: null,
@@ -70,6 +88,9 @@ describe("rebuild gateway drift preflight", () => {
         detected: false,
         sessions: [],
       }),
+      vi.spyOn(onboardSession, "loadSession").mockReturnValue(null),
+      vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null),
+      vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"),
       checkAgentVersionSpy,
     );
 
@@ -90,5 +111,37 @@ describe("rebuild gateway drift preflight", () => {
       expect.objectContaining({ command: "nemoclaw alpha rebuild" }),
     );
     expect(checkAgentVersionSpy).not.toHaveBeenCalled();
+    expect(captureOpenshellSpy).not.toHaveBeenCalled();
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
+  });
+
+  it("recovers the named gateway and retries the liveness query before deciding running state", async () => {
+    detectPreflightIssueSpy.mockReturnValue(null);
+    captureOpenshellSpy
+      .mockReturnValueOnce({ status: 1, output: "client error (Connect): Connection refused" })
+      .mockReturnValueOnce({ status: 0, output: "beta Ready" });
+
+    await expect(rebuildSandbox("alpha", ["--yes"], { throwOnError: true })).rejects.toThrow(
+      "Sandbox 'alpha' is not running.",
+    );
+
+    expect(recoverNamedGatewayRuntimeSpy).toHaveBeenCalledWith({
+      recoverableStates: ["missing_named", "named_unhealthy", "named_unreachable"],
+    });
+    expect(captureOpenshellSpy).toHaveBeenCalledTimes(2);
+    expect(captureOpenshellSpy).toHaveBeenNthCalledWith(1, ["sandbox", "list"]);
+    expect(captureOpenshellSpy).toHaveBeenNthCalledWith(2, ["sandbox", "list"]);
+  });
+
+  it("does not recover generic sandbox list failures", async () => {
+    detectPreflightIssueSpy.mockReturnValue(null);
+    captureOpenshellSpy.mockReturnValue({ status: 1, output: "unknown option: sandbox list" });
+
+    await expect(rebuildSandbox("alpha", ["--yes"], { throwOnError: true })).rejects.toThrow(
+      "Failed to query running sandboxes from OpenShell.",
+    );
+
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
+    expect(captureOpenshellSpy).toHaveBeenCalledTimes(1);
   });
 });