feat(status): classify failing layer when gateway probe fails (#3271)

src/lib/actions/sandbox/gateway-failure-classifier.ts

@@ -0,0 +1,134 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import net from "node:net";
+
+import { dockerInfo } from "../../adapters/docker/info";
+import { dockerCapture } from "../../adapters/docker/run";
+import { GATEWAY_PORT } from "../../core/ports";
+
+const DEFAULT_CONTAINER = "openshell-cluster-nemoclaw";
+const DOCKER_TIMEOUT_MS = 3000;
+const PORT_PROBE_TIMEOUT_MS = 2000;
+
+export type GatewayFailureLayer =
+  | "docker_unreachable"
+  | "container_missing"
+  | "container_exited_port_conflict"
+  | "container_exited"
+  | "gateway_unreachable";
+
+export type GatewayFailureResult = {
+  layer: GatewayFailureLayer;
+  detail: string;
+};
+
+export type GatewayFailureRunners = {
+  dockerInfo: () => boolean;
+  dockerIsRunning: (container: string) => boolean;
+  dockerExists: (container: string) => boolean;
+  portProbe: (port: number) => Promise<boolean>;
+};
+
+function defaultDockerInfo(): boolean {
+  return dockerInfo({ ignoreError: true, timeout: DOCKER_TIMEOUT_MS }).length > 0;
+}
+
+function dockerContainerListed(container: string, allFlag: boolean): boolean {
+  const args = ["ps"];
+  if (allFlag) args.push("-a");
+  args.push("--filter", `name=${container}`, "--format", "{{.Names}}");
+  const out = dockerCapture(args, { ignoreError: true, timeout: DOCKER_TIMEOUT_MS });
+  return out.split("\n").some((line) => line.trim() === container);
+}
+
+function defaultDockerIsRunning(container: string): boolean {
+  return dockerContainerListed(container, false);
+}
+
+function defaultDockerExists(container: string): boolean {
+  return dockerContainerListed(container, true);
+}
+
+function defaultPortProbe(port: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const sock = net.connect({ host: "127.0.0.1", port }, () => {
+      sock.destroy();
+      resolve(true);
+    });
+    sock.setTimeout(PORT_PROBE_TIMEOUT_MS);
+    sock.on("timeout", () => {
+      sock.destroy();
+      resolve(false);
+    });
+    sock.on("error", () => {
+      resolve(false);
+    });
+  });
+}
+
+const defaultRunners: GatewayFailureRunners = {
+  dockerInfo: defaultDockerInfo,
+  dockerIsRunning: defaultDockerIsRunning,
+  dockerExists: defaultDockerExists,
+  portProbe: defaultPortProbe,
+};
+
+export async function classifyGatewayFailure(
+  _sandboxName: string,
+  opts?: { runners?: GatewayFailureRunners },
+): Promise<GatewayFailureResult> {
+  const runners = opts?.runners ?? defaultRunners;
+
+  if (!runners.dockerInfo()) {
+    return {
+      layer: "docker_unreachable",
+      detail: "Docker daemon is not reachable (docker info failed or timed out).",
+    };
+  }
+
+  if (runners.dockerIsRunning(DEFAULT_CONTAINER)) {
+    return {
+      layer: "gateway_unreachable",
+      detail: `Container '${DEFAULT_CONTAINER}' is running but the gateway API is not responding.`,
+    };
+  }
+
+  // Container is not running. Distinguish "exited and still present" from
+  // "removed/never created" — only the former can hit container_exited*. Per
+  // issue #3271 AC: container_exited_port_conflict requires `docker ps -a` to
+  // confirm the container exited rather than being absent.
+  if (!runners.dockerExists(DEFAULT_CONTAINER)) {
+    return {
+      layer: "container_missing",
+      detail: `Container '${DEFAULT_CONTAINER}' is not present (never created or removed).`,
+    };
+  }
+
+  const portInUse = await runners.portProbe(GATEWAY_PORT);
+  if (portInUse) {
+    return {
+      layer: "container_exited_port_conflict",
+      detail: `Container '${DEFAULT_CONTAINER}' exited, and port ${GATEWAY_PORT} is held by another process.`,
+    };
+  }
+  return {
+    layer: "container_exited",
+    detail: `Container '${DEFAULT_CONTAINER}' exited.`,
+  };
+}
+
+const LAYER_HEADERS: Record<GatewayFailureLayer, string> = {
+  docker_unreachable: "Failure layer: docker_unreachable — Docker daemon is not reachable.",
+  container_missing:
+    "Failure layer: container_missing — gateway container is not present; recreate the sandbox.",
+  container_exited_port_conflict:
+    "Failure layer: container_exited_port_conflict — container exited, gateway port held by foreign process.",
+  container_exited: "Failure layer: container_exited — container exited.",
+  gateway_unreachable:
+    "Failure layer: gateway_unreachable — container running but gateway API unresponsive.",
+};
+
+export function getLayerHeader(layer: GatewayFailureLayer): string {
+  return LAYER_HEADERS[layer];
+}

src/lib/actions/sandbox/status.ts

@@ -32,6 +32,7 @@ import {
   printGatewayLifecycleHint,
   printWrongGatewayActiveGuidance,
 } from "./gateway-state";
+import { classifyGatewayFailure, getLayerHeader } from "./gateway-failure-classifier";
 import {
   isSandboxGatewayRunningForStatus,
   probeSandboxInferenceGatewayHealth,
@@ -104,6 +105,11 @@ function maybeEnsureHermesToolGatewayBroker(sb: registry.SandboxEntry | null): v
   }
 }
 
+async function printGatewayFailureLayerHeader(sandboxName: string): Promise<void> {
+  const failure = await classifyGatewayFailure(sandboxName);
+  console.log(`  ${getLayerHeader(failure.layer)}`);
+}
+
 // eslint-disable-next-line complexity
 export async function showSandboxStatus(sandboxName: string): Promise<void> {
   const sb = registry.getSandbox(sandboxName);
@@ -282,6 +288,7 @@ export async function showSandboxStatus(sandboxName: string): Promise<void> {
       if (guard.state === "connected_other") {
         printWrongGatewayActiveGuidance(sandboxName, guard.activeGateway, console.log);
       } else {
+        await printGatewayFailureLayerHeader(sandboxName);
         printGatewayLifecycleHint(guard.status || "", sandboxName, console.log);
       }
     } else {
@@ -315,6 +322,7 @@ export async function showSandboxStatus(sandboxName: string): Promise<void> {
     process.exit(1);
   } else if (lookup.state === "gateway_unreachable_after_restart") {
     console.log("");
+    await printGatewayFailureLayerHeader(sandboxName);
     console.log(
       `  Sandbox '${sandboxName}' may still exist, but the selected ${CLI_DISPLAY_NAME} gateway is still refusing connections after restart.`,
     );
@@ -330,6 +338,7 @@ export async function showSandboxStatus(sandboxName: string): Promise<void> {
     process.exit(1);
   } else if (lookup.state === "gateway_missing_after_restart") {
     console.log("");
+    await printGatewayFailureLayerHeader(sandboxName);
     console.log(
       `  Sandbox '${sandboxName}' may still exist locally, but the ${CLI_DISPLAY_NAME} gateway is no longer configured after restart/rebuild.`,
     );
@@ -349,6 +358,7 @@ export async function showSandboxStatus(sandboxName: string): Promise<void> {
     if (lookup.output) {
       console.log(lookup.output);
     }
+    await printGatewayFailureLayerHeader(sandboxName);
     printGatewayLifecycleHint(lookup.output, sandboxName, console.log);
     process.exit(1);
   }

test/gateway-failure-classifier.test.ts

@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import {
+  classifyGatewayFailure,
+  getLayerHeader,
+  type GatewayFailureRunners,
+} from "../dist/lib/actions/sandbox/gateway-failure-classifier.js";
+
+function makeRunners(overrides: Partial<GatewayFailureRunners> = {}): GatewayFailureRunners {
+  return {
+    dockerInfo: () => true,
+    dockerIsRunning: () => true,
+    dockerExists: () => true,
+    portProbe: async () => false,
+    ...overrides,
+  };
+}
+
+describe("classifyGatewayFailure", () => {
+  it("returns docker_unreachable when docker info fails", async () => {
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({ dockerInfo: () => false }),
+    });
+    expect(result.layer).toBe("docker_unreachable");
+    expect(result.detail).toContain("Docker daemon");
+  });
+
+  it("returns gateway_unreachable when container is running but API is unresponsive", async () => {
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners(),
+    });
+    expect(result.layer).toBe("gateway_unreachable");
+    expect(result.detail).toContain("not responding");
+  });
+
+  it("returns container_missing when container is not running AND `docker ps -a` does not list it", async () => {
+    // This is the gap CodeRabbit flagged on #3309: a removed/never-created
+    // container must not be mislabeled as exited.
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({
+        dockerIsRunning: () => false,
+        dockerExists: () => false,
+      }),
+    });
+    expect(result.layer).toBe("container_missing");
+    expect(result.detail).toContain("not present");
+  });
+
+  it("returns container_exited_port_conflict when container exited AND port is held", async () => {
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({
+        dockerIsRunning: () => false,
+        dockerExists: () => true,
+        portProbe: async () => true,
+      }),
+    });
+    expect(result.layer).toBe("container_exited_port_conflict");
+    expect(result.detail).toContain("port");
+    expect(result.detail).toContain("another process");
+  });
+
+  it("returns container_exited when container exited AND port is free", async () => {
+    const result = await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({
+        dockerIsRunning: () => false,
+        dockerExists: () => true,
+        portProbe: async () => false,
+      }),
+    });
+    expect(result.layer).toBe("container_exited");
+    expect(result.detail).toContain("exited");
+  });
+
+  it("does not call dockerIsRunning / dockerExists / portProbe when docker info fails", async () => {
+    let dockerIsRunningCalled = false;
+    let dockerExistsCalled = false;
+    let portProbeCalled = false;
+    await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({
+        dockerInfo: () => false,
+        dockerIsRunning: () => {
+          dockerIsRunningCalled = true;
+          return false;
+        },
+        dockerExists: () => {
+          dockerExistsCalled = true;
+          return false;
+        },
+        portProbe: async () => {
+          portProbeCalled = true;
+          return false;
+        },
+      }),
+    });
+    expect(dockerIsRunningCalled).toBe(false);
+    expect(dockerExistsCalled).toBe(false);
+    expect(portProbeCalled).toBe(false);
+  });
+
+  it("does not call portProbe when the container is missing", async () => {
+    // Existence check fails fast — we should not probe the port for a
+    // non-existent container, since port_conflict isn't a meaningful
+    // classification without a container to recover.
+    let portProbeCalled = false;
+    await classifyGatewayFailure("my-sandbox", {
+      runners: makeRunners({
+        dockerIsRunning: () => false,
+        dockerExists: () => false,
+        portProbe: async () => {
+          portProbeCalled = true;
+          return false;
+        },
+      }),
+    });
+    expect(portProbeCalled).toBe(false);
+  });
+});
+
+describe("getLayerHeader", () => {
+  it("returns a header naming each layer", () => {
+    expect(getLayerHeader("docker_unreachable")).toContain("docker_unreachable");
+    expect(getLayerHeader("container_missing")).toContain("container_missing");
+    expect(getLayerHeader("container_exited_port_conflict")).toContain(
+      "container_exited_port_conflict",
+    );
+    expect(getLayerHeader("container_exited")).toContain("container_exited");
+    expect(getLayerHeader("gateway_unreachable")).toContain("gateway_unreachable");
+  });
+});