diff --git a/src/lib/actions/sandbox/gateway-failure-classifier.ts b/src/lib/actions/sandbox/gateway-failure-classifier.ts new file mode 100644 index 00000000000..dff2e3971f0 --- /dev/null +++ b/src/lib/actions/sandbox/gateway-failure-classifier.ts @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import net from "node:net"; + +import { dockerInfo } from "../../adapters/docker/info"; +import { dockerCapture } from "../../adapters/docker/run"; +import { GATEWAY_PORT } from "../../core/ports"; + +const DEFAULT_CONTAINER = "openshell-cluster-nemoclaw"; +const DOCKER_TIMEOUT_MS = 3000; +const PORT_PROBE_TIMEOUT_MS = 2000; + +export type GatewayFailureLayer = + | "docker_unreachable" + | "container_missing" + | "container_exited_port_conflict" + | "container_exited" + | "gateway_unreachable"; + +export type GatewayFailureResult = { + layer: GatewayFailureLayer; + detail: string; +}; + +export type GatewayFailureRunners = { + dockerInfo: () => boolean; + dockerIsRunning: (container: string) => boolean; + dockerExists: (container: string) => boolean; + portProbe: (port: number) => Promise; +}; + +function defaultDockerInfo(): boolean { + return dockerInfo({ ignoreError: true, timeout: DOCKER_TIMEOUT_MS }).length > 0; +} + +function dockerContainerListed(container: string, allFlag: boolean): boolean { + const args = ["ps"]; + if (allFlag) args.push("-a"); + args.push("--filter", `name=${container}`, "--format", "{{.Names}}"); + const out = dockerCapture(args, { ignoreError: true, timeout: DOCKER_TIMEOUT_MS }); + return out.split("\n").some((line) => line.trim() === container); +} + +function defaultDockerIsRunning(container: string): boolean { + return dockerContainerListed(container, false); +} + +function defaultDockerExists(container: string): boolean { + return dockerContainerListed(container, true); +} + +function defaultPortProbe(port: number): Promise { + return new Promise((resolve) => { + const sock = net.connect({ host: "127.0.0.1", port }, () => { + sock.destroy(); + resolve(true); + }); + sock.setTimeout(PORT_PROBE_TIMEOUT_MS); + sock.on("timeout", () => { + sock.destroy(); + resolve(false); + }); + sock.on("error", () => { + resolve(false); + }); + }); +} + +const defaultRunners: GatewayFailureRunners = { + dockerInfo: defaultDockerInfo, + dockerIsRunning: defaultDockerIsRunning, + dockerExists: defaultDockerExists, + portProbe: defaultPortProbe, +}; + +export async function classifyGatewayFailure( + _sandboxName: string, + opts?: { runners?: GatewayFailureRunners }, +): Promise { + const runners = opts?.runners ?? defaultRunners; + + if (!runners.dockerInfo()) { + return { + layer: "docker_unreachable", + detail: "Docker daemon is not reachable (docker info failed or timed out).", + }; + } + + if (runners.dockerIsRunning(DEFAULT_CONTAINER)) { + return { + layer: "gateway_unreachable", + detail: `Container '${DEFAULT_CONTAINER}' is running but the gateway API is not responding.`, + }; + } + + // Container is not running. Distinguish "exited and still present" from + // "removed/never created" — only the former can hit container_exited*. Per + // issue #3271 AC: container_exited_port_conflict requires `docker ps -a` to + // confirm the container exited rather than being absent. + if (!runners.dockerExists(DEFAULT_CONTAINER)) { + return { + layer: "container_missing", + detail: `Container '${DEFAULT_CONTAINER}' is not present (never created or removed).`, + }; + } + + const portInUse = await runners.portProbe(GATEWAY_PORT); + if (portInUse) { + return { + layer: "container_exited_port_conflict", + detail: `Container '${DEFAULT_CONTAINER}' exited, and port ${GATEWAY_PORT} is held by another process.`, + }; + } + return { + layer: "container_exited", + detail: `Container '${DEFAULT_CONTAINER}' exited.`, + }; +} + +const LAYER_HEADERS: Record = { + docker_unreachable: "Failure layer: docker_unreachable — Docker daemon is not reachable.", + container_missing: + "Failure layer: container_missing — gateway container is not present; recreate the sandbox.", + container_exited_port_conflict: + "Failure layer: container_exited_port_conflict — container exited, gateway port held by foreign process.", + container_exited: "Failure layer: container_exited — container exited.", + gateway_unreachable: + "Failure layer: gateway_unreachable — container running but gateway API unresponsive.", +}; + +export function getLayerHeader(layer: GatewayFailureLayer): string { + return LAYER_HEADERS[layer]; +} diff --git a/src/lib/actions/sandbox/status.ts b/src/lib/actions/sandbox/status.ts index 970dae565b8..e654f5d9cbe 100644 --- a/src/lib/actions/sandbox/status.ts +++ b/src/lib/actions/sandbox/status.ts @@ -32,6 +32,7 @@ import { printGatewayLifecycleHint, printWrongGatewayActiveGuidance, } from "./gateway-state"; +import { classifyGatewayFailure, getLayerHeader } from "./gateway-failure-classifier"; import { isSandboxGatewayRunningForStatus, probeSandboxInferenceGatewayHealth, @@ -104,6 +105,11 @@ function maybeEnsureHermesToolGatewayBroker(sb: registry.SandboxEntry | null): v } } +async function printGatewayFailureLayerHeader(sandboxName: string): Promise { + const failure = await classifyGatewayFailure(sandboxName); + console.log(` ${getLayerHeader(failure.layer)}`); +} + // eslint-disable-next-line complexity export async function showSandboxStatus(sandboxName: string): Promise { const sb = registry.getSandbox(sandboxName); @@ -282,6 +288,7 @@ export async function showSandboxStatus(sandboxName: string): Promise { if (guard.state === "connected_other") { printWrongGatewayActiveGuidance(sandboxName, guard.activeGateway, console.log); } else { + await printGatewayFailureLayerHeader(sandboxName); printGatewayLifecycleHint(guard.status || "", sandboxName, console.log); } } else { @@ -315,6 +322,7 @@ export async function showSandboxStatus(sandboxName: string): Promise { process.exit(1); } else if (lookup.state === "gateway_unreachable_after_restart") { console.log(""); + await printGatewayFailureLayerHeader(sandboxName); console.log( ` Sandbox '${sandboxName}' may still exist, but the selected ${CLI_DISPLAY_NAME} gateway is still refusing connections after restart.`, ); @@ -330,6 +338,7 @@ export async function showSandboxStatus(sandboxName: string): Promise { process.exit(1); } else if (lookup.state === "gateway_missing_after_restart") { console.log(""); + await printGatewayFailureLayerHeader(sandboxName); console.log( ` Sandbox '${sandboxName}' may still exist locally, but the ${CLI_DISPLAY_NAME} gateway is no longer configured after restart/rebuild.`, ); @@ -349,6 +358,7 @@ export async function showSandboxStatus(sandboxName: string): Promise { if (lookup.output) { console.log(lookup.output); } + await printGatewayFailureLayerHeader(sandboxName); printGatewayLifecycleHint(lookup.output, sandboxName, console.log); process.exit(1); } diff --git a/test/gateway-failure-classifier.test.ts b/test/gateway-failure-classifier.test.ts new file mode 100644 index 00000000000..6420e3c97af --- /dev/null +++ b/test/gateway-failure-classifier.test.ts @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + classifyGatewayFailure, + getLayerHeader, + type GatewayFailureRunners, +} from "../dist/lib/actions/sandbox/gateway-failure-classifier.js"; + +function makeRunners(overrides: Partial = {}): GatewayFailureRunners { + return { + dockerInfo: () => true, + dockerIsRunning: () => true, + dockerExists: () => true, + portProbe: async () => false, + ...overrides, + }; +} + +describe("classifyGatewayFailure", () => { + it("returns docker_unreachable when docker info fails", async () => { + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ dockerInfo: () => false }), + }); + expect(result.layer).toBe("docker_unreachable"); + expect(result.detail).toContain("Docker daemon"); + }); + + it("returns gateway_unreachable when container is running but API is unresponsive", async () => { + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners(), + }); + expect(result.layer).toBe("gateway_unreachable"); + expect(result.detail).toContain("not responding"); + }); + + it("returns container_missing when container is not running AND `docker ps -a` does not list it", async () => { + // This is the gap CodeRabbit flagged on #3309: a removed/never-created + // container must not be mislabeled as exited. + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ + dockerIsRunning: () => false, + dockerExists: () => false, + }), + }); + expect(result.layer).toBe("container_missing"); + expect(result.detail).toContain("not present"); + }); + + it("returns container_exited_port_conflict when container exited AND port is held", async () => { + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ + dockerIsRunning: () => false, + dockerExists: () => true, + portProbe: async () => true, + }), + }); + expect(result.layer).toBe("container_exited_port_conflict"); + expect(result.detail).toContain("port"); + expect(result.detail).toContain("another process"); + }); + + it("returns container_exited when container exited AND port is free", async () => { + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ + dockerIsRunning: () => false, + dockerExists: () => true, + portProbe: async () => false, + }), + }); + expect(result.layer).toBe("container_exited"); + expect(result.detail).toContain("exited"); + }); + + it("does not call dockerIsRunning / dockerExists / portProbe when docker info fails", async () => { + let dockerIsRunningCalled = false; + let dockerExistsCalled = false; + let portProbeCalled = false; + await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ + dockerInfo: () => false, + dockerIsRunning: () => { + dockerIsRunningCalled = true; + return false; + }, + dockerExists: () => { + dockerExistsCalled = true; + return false; + }, + portProbe: async () => { + portProbeCalled = true; + return false; + }, + }), + }); + expect(dockerIsRunningCalled).toBe(false); + expect(dockerExistsCalled).toBe(false); + expect(portProbeCalled).toBe(false); + }); + + it("does not call portProbe when the container is missing", async () => { + // Existence check fails fast — we should not probe the port for a + // non-existent container, since port_conflict isn't a meaningful + // classification without a container to recover. + let portProbeCalled = false; + await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ + dockerIsRunning: () => false, + dockerExists: () => false, + portProbe: async () => { + portProbeCalled = true; + return false; + }, + }), + }); + expect(portProbeCalled).toBe(false); + }); +}); + +describe("getLayerHeader", () => { + it("returns a header naming each layer", () => { + expect(getLayerHeader("docker_unreachable")).toContain("docker_unreachable"); + expect(getLayerHeader("container_missing")).toContain("container_missing"); + expect(getLayerHeader("container_exited_port_conflict")).toContain( + "container_exited_port_conflict", + ); + expect(getLayerHeader("container_exited")).toContain("container_exited"); + expect(getLayerHeader("gateway_unreachable")).toContain("gateway_unreachable"); + }); +}); diff --git a/test/repro-2666-silent-list-status.test.ts b/test/repro-2666-silent-list-status.test.ts index f5a8c55eb29..11c54e96a01 100644 --- a/test/repro-2666-silent-list-status.test.ts +++ b/test/repro-2666-silent-list-status.test.ts @@ -245,6 +245,22 @@ describe("#2666 — subprocess regression: simulated (container-stopped + foreig }; } + function writeFakeDocker(lines: string[]): void { + fs.writeFileSync(path.join(binDir, "docker"), lines.join("\n"), { mode: 0o755 }); + } + + function writeFakeOpenshell(lines: string[]): void { + fs.writeFileSync(path.join(binDir, "openshell"), lines.join("\n"), { mode: 0o755 }); + } + + function expectLayerBefore(combined: string, layer: string, laterText: string): void { + const layerIndex = combined.indexOf(`Failure layer: ${layer}`); + const laterIndex = combined.indexOf(laterText); + expect(layerIndex).toBeGreaterThanOrEqual(0); + expect(laterIndex).toBeGreaterThanOrEqual(0); + expect(layerIndex).toBeLessThan(laterIndex); + } + it("nemoclaw list never produces silent empty output when openshell is broken", () => { const { code, stdout, stderr } = runCli(["list"]); const combined = `${stdout}\n${stderr}`; @@ -268,4 +284,142 @@ describe("#2666 — subprocess regression: simulated (container-stopped + foreig // — that's the contract a watchdog wrapping the command relies on. expect(code).not.toBe(0); }); + + it("nemoclaw status prints the classifier header before gateway_unreachable_after_restart guidance", () => { + writeFakeDocker([ + "#!/usr/bin/env bash", + "if [ \"$1\" = info ]; then echo 'Server Version: 24.0.0'; exit 0; fi", + "if [ \"$1\" = ps ]; then echo 'openshell-cluster-nemoclaw'; exit 0; fi", + "exit 0", + ]); + writeFakeOpenshell([ + "#!/usr/bin/env bash", + "case \"$*\" in", + ' "sandbox get my-assist")', + " echo 'Error: sandbox not found' >&2", + " exit 1", + " ;;", + " status)", + " cat <<'EOF'", + "Status: Disconnected", + " Gateway: nemoclaw", + " client error (Connect): tcp connect error: Connection refused (os error 61)", + "EOF", + " exit 1", + " ;;", + ' "gateway info -g nemoclaw")', + " echo 'Gateway: nemoclaw'", + " exit 0", + " ;;", + " *)", + " exit 0", + " ;;", + "esac", + ]); + + const { code, stdout, stderr } = runCli(["my-assist", "status"]); + const combined = `${stdout}\n${stderr}`; + expectLayerBefore(combined, "gateway_unreachable", "still refusing connections after restart"); + expect(code).not.toBe(0); + }); + + it("nemoclaw status prints the classifier header before gateway_missing_after_restart guidance", () => { + writeFakeDocker([ + "#!/usr/bin/env bash", + "if [ \"$1\" = info ]; then echo 'Server Version: 24.0.0'; exit 0; fi", + "if [ \"$1\" = ps ]; then exit 0; fi", + "exit 0", + ]); + writeFakeOpenshell([ + "#!/usr/bin/env bash", + "case \"$*\" in", + ' "sandbox get my-assist")', + " echo 'Error: sandbox not found' >&2", + " exit 1", + " ;;", + " status)", + " echo 'No gateway configured'", + " exit 1", + " ;;", + ' "gateway info -g nemoclaw")', + " echo 'No gateway configured'", + " exit 1", + " ;;", + " *)", + " exit 0", + " ;;", + "esac", + ]); + + const { code, stdout, stderr } = runCli(["my-assist", "status"]); + const combined = `${stdout}\n${stderr}`; + expectLayerBefore(combined, "container_missing", "gateway is no longer configured"); + expect(code).not.toBe(0); + }); + + it("nemoclaw status prints the container_exited_port_conflict layer header (#3271)", async () => { + // Simulate the AC #2 scenario from #3271: docker daemon up, container + // exists in `docker ps -a` but is NOT in `docker ps` (i.e. exited), AND + // a foreign process holds the gateway port. The classifier must label + // this exactly as container_exited_port_conflict. + const net = await import("node:net"); + + // Pick a free port, hold it, then point the classifier at it via + // NEMOCLAW_GATEWAY_PORT so the test never races a real gateway. + const listener = net.createServer(); + await new Promise((resolve) => listener.listen(0, "127.0.0.1", resolve)); + const port = (listener.address() as { port: number }).port; + + try { + // Fake docker: info OK, ps shows nothing running, ps -a shows the + // openshell-cluster-nemoclaw container (i.e. it exited cleanly). + writeFakeDocker([ + "#!/usr/bin/env bash", + "if [ \"$1\" = info ]; then echo 'Server Version: 24.0.0'; exit 0; fi", + "if [ \"$1\" = ps ] && [ \"$2\" = -a ]; then echo 'openshell-cluster-nemoclaw'; exit 0; fi", + "if [ \"$1\" = ps ]; then exit 0; fi", + "exit 0", + ]); + + // Override the default fake openshell with one that returns a generic + // unrecognized failure, so status.ts falls through to the final else + // branch where the classifier runs (rather than a recovery-hint branch). + writeFakeOpenshell([ + "#!/usr/bin/env bash", + "case \"$*\" in", + ' "sandbox get my-assist")', + " echo 'transport error: unexpected EOF' >&2", + " exit 1", + " ;;", + " status)", + " echo 'Status: Unknown'", + " exit 1", + " ;;", + " *)", + " exit 0", + " ;;", + "esac", + ]); + + const result = spawnSync(process.execPath, [CLI, "my-assist", "status"], { + encoding: "utf-8", + timeout: 30_000, + env: { + ...process.env, + HOME: home, + PATH: `${binDir}:${process.env.PATH || ""}`, + NEMOCLAW_HEALTH_POLL_COUNT: "1", + NEMOCLAW_HEALTH_POLL_INTERVAL: "0", + NEMOCLAW_STATUS_PROBE_TIMEOUT_MS: "2000", + NEMOCLAW_TEST_NO_SLEEP: "1", + NEMOCLAW_GATEWAY_PORT: String(port), + }, + }); + const combined = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + expect(combined).toContain("container_exited_port_conflict"); + expect(result.status).not.toBe(0); + } finally { + await new Promise((resolve) => listener.close(() => resolve())); + } + }); });