From d87d2c0b40092622428bda3947963267109ea06c Mon Sep 17 00:00:00 2001 From: kagura-agent Date: Sat, 9 May 2026 12:20:57 +0800 Subject: [PATCH 1/2] feat(status): classify failing layer when gateway probe fails (#3271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a gateway-failure-classifier that detects four distinct failure layers in the status command: 1. docker_unreachable — Docker daemon down or socket inaccessible 2. container_exited_port_conflict — container stopped, port held by foreign process 3. container_exited — container not running 4. gateway_unreachable — container running but gateway API unresponsive Each layer prints a clearly-named header before the existing recovery hints from printGatewayLifecycleHint, giving users immediate context about what went wrong. The classifier uses injectable runners (docker info, docker ps, net.connect port probe) for full unit testability. Added: - Unit tests for all 4 layers + short-circuit behavior - Subprocess regression test extending #2666 coverage Closes #3271 --- .../sandbox/gateway-failure-classifier.ts | 121 ++++++++++++++++++ src/lib/actions/sandbox/status.ts | 3 + test/gateway-failure-classifier.test.ts | 89 +++++++++++++ test/repro-2666-silent-list-status.test.ts | 53 ++++++++ 4 files changed, 266 insertions(+) create mode 100644 src/lib/actions/sandbox/gateway-failure-classifier.ts create mode 100644 test/gateway-failure-classifier.test.ts diff --git a/src/lib/actions/sandbox/gateway-failure-classifier.ts b/src/lib/actions/sandbox/gateway-failure-classifier.ts new file mode 100644 index 00000000000..9723b32789c --- /dev/null +++ b/src/lib/actions/sandbox/gateway-failure-classifier.ts @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import net from "node:net"; +import { execSync } from "node:child_process"; + +import { GATEWAY_PORT } from "../../core/ports"; + +const DEFAULT_CONTAINER = "openshell-cluster-nemoclaw"; +const DOCKER_TIMEOUT_MS = 3000; +const PORT_PROBE_TIMEOUT_MS = 2000; + +export type GatewayFailureLayer = + | "docker_unreachable" + | "container_exited_port_conflict" + | "container_exited" + | "gateway_unreachable"; + +export type GatewayFailureResult = { + layer: GatewayFailureLayer; + detail: string; +}; + +export type GatewayFailureRunners = { + dockerInfo: () => boolean; + dockerIsRunning: (container: string) => boolean; + portProbe: (port: number) => Promise; +}; + +function defaultDockerInfo(): boolean { + try { + execSync("docker info", { timeout: DOCKER_TIMEOUT_MS, stdio: "pipe" }); + return true; + } catch { + return false; + } +} + +function defaultDockerIsRunning(container: string): boolean { + try { + const out = execSync(`docker ps --filter name=${container} --format "{{.Names}}"`, { + timeout: DOCKER_TIMEOUT_MS, + stdio: "pipe", + encoding: "utf-8", + }); + return out.trim().split("\n").some((line) => line.trim() === container); + } catch { + return false; + } +} + +function defaultPortProbe(port: number): Promise { + return new Promise((resolve) => { + const sock = net.connect({ host: "127.0.0.1", port }, () => { + sock.destroy(); + resolve(true); + }); + sock.setTimeout(PORT_PROBE_TIMEOUT_MS); + sock.on("timeout", () => { + sock.destroy(); + resolve(false); + }); + sock.on("error", () => { + resolve(false); + }); + }); +} + +const defaultRunners: GatewayFailureRunners = { + dockerInfo: defaultDockerInfo, + dockerIsRunning: defaultDockerIsRunning, + portProbe: defaultPortProbe, +}; + +export async function classifyGatewayFailure( + _sandboxName: string, + opts?: { runners?: GatewayFailureRunners }, +): Promise { + const runners = opts?.runners ?? defaultRunners; + + if (!runners.dockerInfo()) { + return { + layer: "docker_unreachable", + detail: "Docker daemon is not reachable (docker info failed or timed out).", + }; + } + + const containerRunning = runners.dockerIsRunning(DEFAULT_CONTAINER); + + if (!containerRunning) { + const portInUse = await runners.portProbe(GATEWAY_PORT); + if (portInUse) { + return { + layer: "container_exited_port_conflict", + detail: `Container '${DEFAULT_CONTAINER}' is not running, but port ${GATEWAY_PORT} is held by another process.`, + }; + } + return { + layer: "container_exited", + detail: `Container '${DEFAULT_CONTAINER}' is not running.`, + }; + } + + return { + layer: "gateway_unreachable", + detail: `Container '${DEFAULT_CONTAINER}' is running but the gateway API is not responding.`, + }; +} + +const LAYER_HEADERS: Record = { + docker_unreachable: "Failure layer: docker_unreachable — Docker daemon is not reachable.", + container_exited_port_conflict: + "Failure layer: container_exited_port_conflict — container stopped, gateway port held by foreign process.", + container_exited: "Failure layer: container_exited — container is not running.", + gateway_unreachable: + "Failure layer: gateway_unreachable — container running but gateway API unresponsive.", +}; + +export function getLayerHeader(layer: GatewayFailureLayer): string { + return LAYER_HEADERS[layer]; +} diff --git a/src/lib/actions/sandbox/status.ts b/src/lib/actions/sandbox/status.ts index ea6fb9e602c..818ceb288f3 100644 --- a/src/lib/actions/sandbox/status.ts +++ b/src/lib/actions/sandbox/status.ts @@ -27,6 +27,7 @@ import { printGatewayLifecycleHint, printWrongGatewayActiveGuidance, } from "./gateway-state"; +import { classifyGatewayFailure, getLayerHeader } from "./gateway-failure-classifier"; import { isSandboxGatewayRunningForStatus, probeSandboxInferenceGatewayHealth, @@ -317,6 +318,8 @@ export async function showSandboxStatus(sandboxName: string): Promise { if (lookup.output) { console.log(lookup.output); } + const failure = await classifyGatewayFailure(sandboxName); + console.log(` ${YW}${getLayerHeader(failure.layer)}${R}`); printGatewayLifecycleHint(lookup.output, sandboxName, console.log); process.exit(1); } diff --git a/test/gateway-failure-classifier.test.ts b/test/gateway-failure-classifier.test.ts new file mode 100644 index 00000000000..923be344087 --- /dev/null +++ b/test/gateway-failure-classifier.test.ts @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + classifyGatewayFailure, + getLayerHeader, + type GatewayFailureRunners, +} from "../dist/lib/actions/sandbox/gateway-failure-classifier.js"; + +function makeRunners(overrides: Partial = {}): GatewayFailureRunners { + return { + dockerInfo: () => true, + dockerIsRunning: () => true, + portProbe: async () => false, + ...overrides, + }; +} + +describe("classifyGatewayFailure", () => { + it("returns docker_unreachable when docker info fails", async () => { + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ dockerInfo: () => false }), + }); + expect(result.layer).toBe("docker_unreachable"); + expect(result.detail).toContain("Docker daemon"); + }); + + it("returns container_exited_port_conflict when container is stopped and port is in use", async () => { + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ + dockerIsRunning: () => false, + portProbe: async () => true, + }), + }); + expect(result.layer).toBe("container_exited_port_conflict"); + expect(result.detail).toContain("port"); + expect(result.detail).toContain("another process"); + }); + + it("returns container_exited when container is stopped and port is free", async () => { + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ + dockerIsRunning: () => false, + portProbe: async () => false, + }), + }); + expect(result.layer).toBe("container_exited"); + expect(result.detail).toContain("not running"); + }); + + it("returns gateway_unreachable when container is running but gateway unresponsive", async () => { + const result = await classifyGatewayFailure("my-sandbox", { + runners: makeRunners(), + }); + expect(result.layer).toBe("gateway_unreachable"); + expect(result.detail).toContain("not responding"); + }); + + it("does not call dockerIsRunning or portProbe when docker info fails", async () => { + let dockerIsRunningCalled = false; + let portProbeCalled = false; + await classifyGatewayFailure("my-sandbox", { + runners: makeRunners({ + dockerInfo: () => false, + dockerIsRunning: () => { + dockerIsRunningCalled = true; + return false; + }, + portProbe: async () => { + portProbeCalled = true; + return false; + }, + }), + }); + expect(dockerIsRunningCalled).toBe(false); + expect(portProbeCalled).toBe(false); + }); +}); + +describe("getLayerHeader", () => { + it("returns a header containing the layer name", () => { + expect(getLayerHeader("docker_unreachable")).toContain("docker_unreachable"); + expect(getLayerHeader("container_exited_port_conflict")).toContain("container_exited_port_conflict"); + expect(getLayerHeader("container_exited")).toContain("container_exited"); + expect(getLayerHeader("gateway_unreachable")).toContain("gateway_unreachable"); + }); +}); diff --git a/test/repro-2666-silent-list-status.test.ts b/test/repro-2666-silent-list-status.test.ts index f5a8c55eb29..f3ebae6b3af 100644 --- a/test/repro-2666-silent-list-status.test.ts +++ b/test/repro-2666-silent-list-status.test.ts @@ -200,6 +200,28 @@ describe("#2666 — subprocess regression: simulated (container-stopped + foreig { mode: 0o755 }, ); + // Fake docker that simulates: daemon reachable, container stopped + fs.writeFileSync( + path.join(binDir, "docker"), + [ + "#!/usr/bin/env bash", + "case \"$1\" in", + " info)", + " echo 'Server Version: 24.0.0'", + " exit 0", + " ;;", + " ps)", + " echo ''", + " exit 0", + " ;;", + " *)", + " exit 0", + " ;;", + "esac", + ].join("\n"), + { mode: 0o755 }, + ); + const registryDir = path.join(home, ".nemoclaw"); fs.mkdirSync(registryDir, { recursive: true }); fs.writeFileSync( @@ -268,4 +290,35 @@ describe("#2666 — subprocess regression: simulated (container-stopped + foreig // — that's the contract a watchdog wrapping the command relies on. expect(code).not.toBe(0); }); + + it("nemoclaw status prints the failure layer when the gateway probe fails (#3271)", () => { + // Override the fake openshell with one that produces a generic gateway_error + // that does NOT match the recovery heuristics (no "Gateway: nemoclaw" etc.), + // so the status command falls through to the final else branch where the + // classifier runs. + fs.writeFileSync( + path.join(binDir, "openshell"), + [ + "#!/usr/bin/env bash", + "case \"$*\" in", + ' "sandbox get my-assist")', + " echo 'transport error: unexpected EOF' >&2", + " exit 1", + " ;;", + " status)", + " echo 'Status: Unknown'", + " exit 1", + " ;;", + " *)", + " exit 1", + " ;;", + "esac", + ].join("\n"), + { mode: 0o755 }, + ); + const { stdout, stderr } = runCli(["my-assist", "status"]); + const combined = `${stdout}\n${stderr}`; + expect(combined).toContain("Failure layer:"); + expect(combined).toMatch(/container_exited|docker_unreachable|gateway_unreachable/); + }); }); From fa8f93248673021c9881ef4ba11a29e5a37dc065 Mon Sep 17 00:00:00 2001 From: kagura-agent Date: Sat, 9 May 2026 12:27:09 +0800 Subject: [PATCH 2/2] test: include container_exited_port_conflict in layer assertion Address CodeRabbit review: the regex matching classifier output in the subprocess test omitted container_exited_port_conflict. Added word boundaries for precision. --- test/repro-2666-silent-list-status.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/repro-2666-silent-list-status.test.ts b/test/repro-2666-silent-list-status.test.ts index f3ebae6b3af..6805a4b7448 100644 --- a/test/repro-2666-silent-list-status.test.ts +++ b/test/repro-2666-silent-list-status.test.ts @@ -319,6 +319,8 @@ describe("#2666 — subprocess regression: simulated (container-stopped + foreig const { stdout, stderr } = runCli(["my-assist", "status"]); const combined = `${stdout}\n${stderr}`; expect(combined).toContain("Failure layer:"); - expect(combined).toMatch(/container_exited|docker_unreachable|gateway_unreachable/); + expect(combined).toMatch( + /\b(container_exited_port_conflict|container_exited|docker_unreachable|gateway_unreachable)\b/, + ); }); });