diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index ded31eecbf3..557d46c3adb 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -33,8 +33,9 @@ import { createSystemDeps as createSessionDeps, getActiveSandboxSessions, } from "../../state/sandbox-session"; +import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action"; import { runSetupDnsProxy } from "../dns"; -import { ensureLiveSandboxOrExit } from "./gateway-state"; +import { ensureLiveSandboxOrExit, printGatewayLifecycleHint } from "./gateway-state"; import { checkAndRecoverSandboxProcesses } from "./process-recovery"; import { applyOpenShellVmDnsMonkeypatch, @@ -52,6 +53,11 @@ type SpawnLikeResult = { signal?: NodeJS.Signals | null; }; +type SandboxListProbe = { + status: number | null; + output: string; +}; + type SandboxInferenceRouteProbe = { healthy: boolean; broken: boolean; @@ -166,6 +172,55 @@ function sleepSync(ms: number): void { }); } +function isBlockingGatewayLifecycle( + lifecycle: ReturnType, +): boolean { + if (lifecycle.state === "named_unreachable" || lifecycle.state === "named_unhealthy") { + return true; + } + return ( + lifecycle.state === "missing_named" && + /No gateway configured|No active gateway|Connection refused|Status:\s*Disconnected/i.test( + lifecycle.status || "", + ) + ); +} + +function failConnectReadinessGatewayUnavailable( + sandboxName: string, + detailOutput = "", +): never { + console.error(""); + console.error( + ` OpenShell gateway is not running or unreachable; cannot verify sandbox '${sandboxName}' readiness.`, + ); + if (detailOutput.trim()) { + console.error(detailOutput.trimEnd()); + printGatewayLifecycleHint(detailOutput, sandboxName, console.error); + } + console.error(" Recovery:"); + console.error(" 1. Run: openshell gateway start --name nemoclaw"); + console.error(` 2. If the gateway cannot be restarted, run: ${CLI_NAME} onboard`); + console.error(` 3. Retry: ${CLI_NAME} ${sandboxName} connect`); + process.exit(1); +} + +function outputShowsGatewayUnavailable(output = ""): boolean { + return /No gateway configured|No active gateway|Connection refused|client error \(Connect\)|tcp connect error|Status:\s*Disconnected/i.test( + output, + ); +} + +function failIfGatewayBlocksConnectReadiness(sandboxName: string): void { + const lifecycle = getNamedGatewayLifecycleState(); + if (isBlockingGatewayLifecycle(lifecycle)) { + failConnectReadinessGatewayUnavailable( + sandboxName, + lifecycle.status || lifecycle.gatewayInfo || "", + ); + } +} + function probeSandboxInferenceRoute( sandboxName: string, { attempts = 1, delayMs = 0 }: InferenceRouteProbeOptions = {}, @@ -646,15 +701,27 @@ export async function connectSandbox( const deadline = startedAt + timeout * 1000; const elapsedSec = () => Math.floor((Date.now() - startedAt) / 1000); const remainingMs = () => Math.max(1, deadline - Date.now()); - const runSandboxList = () => - captureOpenshell(["sandbox", "list"], { + const runSandboxList = (): SandboxListProbe => { + const result = captureOpenshell(["sandbox", "list"], { ignoreError: true, timeout: remainingMs(), - }).output; + }); + return { status: result.status, output: result.output }; + }; - const list = runSandboxList(); + const listProbe = runSandboxList(); + const listCommandFailed = listProbe.status !== 0; + if (listCommandFailed) { + if (outputShowsGatewayUnavailable(listProbe.output)) { + failConnectReadinessGatewayUnavailable(sandboxName, listProbe.output); + } + } + const list = listProbe.output; if (!isSandboxReady(list, sandboxName)) { const status = parseSandboxStatus(list, sandboxName); + if (!listCommandFailed && status && /^unknown$/i.test(status)) { + failIfGatewayBlocksConnectReadiness(sandboxName); + } const TERMINAL = new Set([ "Failed", "Error", @@ -678,13 +745,24 @@ export async function connectSandbox( const sleepFor = Math.min(interval, remainingMs() / 1000); if (sleepFor <= 0) break; spawnSync("sleep", [String(sleepFor)]); - const poll = runSandboxList(); + const pollProbe = runSandboxList(); + const pollCommandFailed = pollProbe.status !== 0; + if (pollCommandFailed) { + if (outputShowsGatewayUnavailable(pollProbe.output)) { + failConnectReadinessGatewayUnavailable(sandboxName, pollProbe.output); + } + } + const poll = pollProbe.output; const elapsed = elapsedSec(); if (isSandboxReady(poll, sandboxName)) { ready = true; break; } - const cur = parseSandboxStatus(poll, sandboxName) || "unknown"; + const parsedCur = parseSandboxStatus(poll, sandboxName); + const cur = parsedCur || "unknown"; + if (!pollCommandFailed && parsedCur && /^unknown$/i.test(parsedCur)) { + failIfGatewayBlocksConnectReadiness(sandboxName); + } if (cur !== "unknown") everSeen = true; if (TERMINAL.has(cur)) { console.error(""); diff --git a/test/cli.test.ts b/test/cli.test.ts index 1d69a4d045b..8467be0c55f 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -3835,6 +3835,95 @@ describe("CLI dispatch", () => { expect(calls).toContain("sandbox connect alpha"); }); + it( + "fails fast with gateway recovery guidance when connect readiness sees a disconnected gateway", + () => { + const home = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-cli-connect-gateway-down-"), + ); + const localBin = path.join(home, "bin"); + const registryDir = path.join(home, ".nemoclaw"); + const markerFile = path.join(home, "openshell-calls"); + fs.mkdirSync(localBin, { recursive: true }); + fs.mkdirSync(registryDir, { recursive: true }); + fs.writeFileSync( + path.join(registryDir, "sandboxes.json"), + JSON.stringify({ + sandboxes: { + alpha: { + name: "alpha", + model: "test-model", + provider: "nvidia-prod", + gpuEnabled: false, + policies: [], + }, + }, + defaultSandbox: "alpha", + }), + { mode: 0o600 }, + ); + fs.writeFileSync( + path.join(localBin, "openshell"), + [ + "#!/usr/bin/env bash", + `marker_file=${JSON.stringify(markerFile)}`, + 'printf \'%s\\n\' "$*" >> "$marker_file"', + 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', + " echo 'Sandbox:'", + " echo", + " echo ' Id: abc'", + " echo ' Name: alpha'", + " echo ' Namespace: openshell'", + " echo ' Phase: Pending'", + " exit 0", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', + " echo 'alpha unknown 103s ago'", + " exit 0", + "fi", + 'if [ "$1" = "status" ]; then', + " echo 'Server Status'", + " echo", + " echo ' Gateway: nemoclaw'", + " echo ' Status: Disconnected'", + " exit 0", + "fi", + 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', + " echo 'Gateway Info'", + " echo", + " echo ' Gateway: nemoclaw'", + " exit 0", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "connect" ] && [ "$3" = "alpha" ]; then', + " echo 'should-not-connect' >> \"$marker_file\"", + " exit 0", + "fi", + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + + const r = runWithEnv( + "alpha connect", + { + HOME: home, + NEMOCLAW_CONNECT_TIMEOUT: "1", + PATH: `${localBin}:${process.env.PATH || ""}`, + }, + execTimeout(10_000), + ); + + expect(r.code).toBe(1); + expect(r.out).toContain("OpenShell gateway is not running or unreachable"); + expect(r.out).toContain("nemoclaw onboard"); + expect(r.out).not.toContain("Timed out after 1s"); + const calls = fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean); + expect(calls).toContain("status"); + expect(calls).not.toContain("should-not-connect"); + }, + testTimeout(15_000), + ); + it("prints recovery guidance when readiness polling hits a terminal sandbox state", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-failed-")); const localBin = path.join(home, "bin");