feat(status): classify failing layer when gateway probe fails (#3271)

src/lib/actions/sandbox/gateway-failure-classifier.ts

@@ -0,0 +1,121 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import net from "node:net";
+import { execSync } from "node:child_process";
+
+import { GATEWAY_PORT } from "../../core/ports";
+
+const DEFAULT_CONTAINER = "openshell-cluster-nemoclaw";
+const DOCKER_TIMEOUT_MS = 3000;
+const PORT_PROBE_TIMEOUT_MS = 2000;
+
+export type GatewayFailureLayer =
+  | "docker_unreachable"
+  | "container_exited_port_conflict"
+  | "container_exited"
+  | "gateway_unreachable";
+
+export type GatewayFailureResult = {
+  layer: GatewayFailureLayer;
+  detail: string;
+};
+
+export type GatewayFailureRunners = {
+  dockerInfo: () => boolean;
+  dockerIsRunning: (container: string) => boolean;
+  portProbe: (port: number) => Promise<boolean>;
+};
+
+function defaultDockerInfo(): boolean {
+  try {
+    execSync("docker info", { timeout: DOCKER_TIMEOUT_MS, stdio: "pipe" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function defaultDockerIsRunning(container: string): boolean {
+  try {
+    const out = execSync(`docker ps --filter name=${container} --format "{{.Names}}"`, {
+      timeout: DOCKER_TIMEOUT_MS,
+      stdio: "pipe",
+      encoding: "utf-8",
+    });
+    return out.trim().split("\n").some((line) => line.trim() === container);
+  } catch {
+    return false;
+  }
+}
+
+function defaultPortProbe(port: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const sock = net.connect({ host: "127.0.0.1", port }, () => {
+      sock.destroy();
+      resolve(true);
+    });
+    sock.setTimeout(PORT_PROBE_TIMEOUT_MS);
+    sock.on("timeout", () => {
+      sock.destroy();
+      resolve(false);
+    });
+    sock.on("error", () => {
+      resolve(false);
+    });
+  });
+}
+
+const defaultRunners: GatewayFailureRunners = {
+  dockerInfo: defaultDockerInfo,
+  dockerIsRunning: defaultDockerIsRunning,
+  portProbe: defaultPortProbe,
+};
+
+export async function classifyGatewayFailure(
+  _sandboxName: string,
+  opts?: { runners?: GatewayFailureRunners },
+): Promise<GatewayFailureResult> {
+  const runners = opts?.runners ?? defaultRunners;
+
+  if (!runners.dockerInfo()) {
+    return {
+      layer: "docker_unreachable",
+      detail: "Docker daemon is not reachable (docker info failed or timed out).",
+    };
+  }
+
+  const containerRunning = runners.dockerIsRunning(DEFAULT_CONTAINER);
+
+  if (!containerRunning) {
+    const portInUse = await runners.portProbe(GATEWAY_PORT);
+    if (portInUse) {
+      return {
+        layer: "container_exited_port_conflict",
+        detail: `Container '${DEFAULT_CONTAINER}' is not running, but port ${GATEWAY_PORT} is held by another process.`,
+      };
+    }
+    return {
+      layer: "container_exited",
+      detail: `Container '${DEFAULT_CONTAINER}' is not running.`,
+    };
+  }
+
+  return {
+    layer: "gateway_unreachable",
+    detail: `Container '${DEFAULT_CONTAINER}' is running but the gateway API is not responding.`,
+  };
+}
+
+const LAYER_HEADERS: Record<GatewayFailureLayer, string> = {
+  docker_unreachable: "Failure layer: docker_unreachable — Docker daemon is not reachable.",
+  container_exited_port_conflict:
+    "Failure layer: container_exited_port_conflict — container stopped, gateway port held by foreign process.",
+  container_exited: "Failure layer: container_exited — container is not running.",
+  gateway_unreachable:
+    "Failure layer: gateway_unreachable — container running but gateway API unresponsive.",
+};
+
+export function getLayerHeader(layer: GatewayFailureLayer): string {
+  return LAYER_HEADERS[layer];
+}

src/lib/actions/sandbox/status.ts

@@ -27,6 +27,7 @@ import {
   printGatewayLifecycleHint,
   printWrongGatewayActiveGuidance,
 } from "./gateway-state";
+import { classifyGatewayFailure, getLayerHeader } from "./gateway-failure-classifier";
 import {
   isSandboxGatewayRunningForStatus,
   probeSandboxInferenceGatewayHealth,
@@ -317,6 +318,8 @@ export async function showSandboxStatus(sandboxName: string): Promise<void> {
     if (lookup.output) {
       console.log(lookup.output);
     }
+    const failure = await classifyGatewayFailure(sandboxName);
+    console.log(`  ${YW}${getLayerHeader(failure.layer)}${R}`);
     printGatewayLifecycleHint(lookup.output, sandboxName, console.log);
     process.exit(1);
   }

test/gateway-failure-classifier.test.ts

@@ -0,0 +1,89 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import {
+  classifyGatewayFailure,
+  getLayerHeader,
+  type GatewayFailureRunners,
+} from "../dist/lib/actions/sandbox/gateway-failure-classifier.js";
+
+function makeRunners(overrides: Partial<GatewayFailureRunners> = {}): GatewayFailureRunners {
+  return {
+    dockerInfo: () => true,
+    dockerIsRunning: () => true,
+    portProbe: async () => false,
+    ...overrides,
+  };
+}
+
+describe("classifyGatewayFailure", () => {
+  it("returns docker_unreachable when docker info fails", async () => {
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({ dockerInfo: () => false }),
+    });
+    expect(result.layer).toBe("docker_unreachable");
+    expect(result.detail).toContain("Docker daemon");
+  });
+
+  it("returns container_exited_port_conflict when container is stopped and port is in use", async () => {
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({
+        dockerIsRunning: () => false,
+        portProbe: async () => true,
+      }),
+    });
+    expect(result.layer).toBe("container_exited_port_conflict");
+    expect(result.detail).toContain("port");
+    expect(result.detail).toContain("another process");
+  });
+
+  it("returns container_exited when container is stopped and port is free", async () => {
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({
+        dockerIsRunning: () => false,
+        portProbe: async () => false,
+      }),
+    });
+    expect(result.layer).toBe("container_exited");
+    expect(result.detail).toContain("not running");
+  });
+
+  it("returns gateway_unreachable when container is running but gateway unresponsive", async () => {
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners(),
+    });
+    expect(result.layer).toBe("gateway_unreachable");
+    expect(result.detail).toContain("not responding");
+  });
+
+  it("does not call dockerIsRunning or portProbe when docker info fails", async () => {
+    let dockerIsRunningCalled = false;
+    let portProbeCalled = false;
+    await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({
+        dockerInfo: () => false,
+        dockerIsRunning: () => {
+          dockerIsRunningCalled = true;
+          return false;
+        },
+        portProbe: async () => {
+          portProbeCalled = true;
+          return false;
+        },
+      }),
+    });
+    expect(dockerIsRunningCalled).toBe(false);
+    expect(portProbeCalled).toBe(false);
+  });
+});
+
+describe("getLayerHeader", () => {
+  it("returns a header containing the layer name", () => {
+    expect(getLayerHeader("docker_unreachable")).toContain("docker_unreachable");
+    expect(getLayerHeader("container_exited_port_conflict")).toContain("container_exited_port_conflict");
+    expect(getLayerHeader("container_exited")).toContain("container_exited");
+    expect(getLayerHeader("gateway_unreachable")).toContain("gateway_unreachable");
+  });
+});

test/repro-2666-silent-list-status.test.ts

@@ -200,6 +200,28 @@ describe("#2666 — subprocess regression: simulated (container-stopped + foreig
       { mode: 0o755 },
     );
 
+    // Fake docker that simulates: daemon reachable, container stopped
+    fs.writeFileSync(
+      path.join(binDir, "docker"),
+      [
+        "#!/usr/bin/env bash",
+        "case \"$1\" in",
+        "  info)",
+        "    echo 'Server Version: 24.0.0'",
+        "    exit 0",
+        "    ;;",
+        "  ps)",
+        "    echo ''",
+        "    exit 0",
+        "    ;;",
+        "  *)",
+        "    exit 0",
+        "    ;;",
+        "esac",
+      ].join("\n"),
+      { mode: 0o755 },
+    );
+
     const registryDir = path.join(home, ".nemoclaw");
     fs.mkdirSync(registryDir, { recursive: true });
     fs.writeFileSync(
@@ -268,4 +290,37 @@ describe("#2666 — subprocess regression: simulated (container-stopped + foreig
     // — that's the contract a watchdog wrapping the command relies on.
     expect(code).not.toBe(0);
   });
+
+  it("nemoclaw <name> status prints the failure layer when the gateway probe fails (#3271)", () => {
+    // Override the fake openshell with one that produces a generic gateway_error
+    // that does NOT match the recovery heuristics (no "Gateway: nemoclaw" etc.),
+    // so the status command falls through to the final else branch where the
+    // classifier runs.
+    fs.writeFileSync(
+      path.join(binDir, "openshell"),
+      [
+        "#!/usr/bin/env bash",
+        "case \"$*\" in",
+        '  "sandbox get my-assist")',
+        "    echo 'transport error: unexpected EOF' >&2",
+        "    exit 1",
+        "    ;;",
+        "  status)",
+        "    echo 'Status: Unknown'",
+        "    exit 1",
+        "    ;;",
+        "  *)",
+        "    exit 1",
+        "    ;;",
+        "esac",
+      ].join("\n"),
+      { mode: 0o755 },
+    );
+    const { stdout, stderr } = runCli(["my-assist", "status"]);
+    const combined = `${stdout}\n${stderr}`;
+    expect(combined).toContain("Failure layer:");
+    expect(combined).toMatch(
+      /\b(container_exited_port_conflict|container_exited|docker_unreachable|gateway_unreachable)\b/,
+    );
+  });
 });