diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f4d59c3f991..2ef70ffec7e 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -61,6 +61,10 @@ const { const { getSelectionDrift, }: typeof import("./onboard/selection-drift") = require("./onboard/selection-drift"); +const { + formatOllamaProxyUnreachableMessage, + probeOllamaProxySandboxReachability, +}: typeof import("./onboard/ollama-proxy-reachability") = require("./onboard/ollama-proxy-reachability"); const crypto = require("node:crypto"); const fs = require("fs"); const os = require("os"); @@ -7973,6 +7977,18 @@ async function setupInference( // Persist token now that ollama-local is confirmed as the provider. // Not persisted earlier in case the user backs out to a different provider. persistProxyToken(proxyToken); + // Probe sandbox → proxy connectivity before committing the inference + // route. Running before `inference set` ensures isInferenceRouteReady() + // stays false on failure, so a retry (including --resume) re-enters + // setupInference and re-runs this check rather than skipping it. + const reach = await probeOllamaProxySandboxReachability(); + if (!reach.ok) { + const msg = formatOllamaProxyUnreachableMessage(reach); + if (reach.reason === "tcp_failed") { + console.error(msg); + process.exit(1); + } + } } // Use a dedicated internal credential env (NEMOCLAW_OLLAMA_PROXY_TOKEN) // so the gateway never reads the user's host OPENAI_API_KEY for local diff --git a/src/lib/onboard/ollama-proxy-reachability.test.ts b/src/lib/onboard/ollama-proxy-reachability.test.ts new file mode 100644 index 00000000000..6d4ff7c08e9 --- /dev/null +++ b/src/lib/onboard/ollama-proxy-reachability.test.ts @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Unit tests for the sandbox-side Ollama auth proxy reachability probe. +// +// See: https://github.com/NVIDIA/NemoClaw/issues/3340 + +import { describe, expect, it, vi } from "vitest"; + +// Mock the docker adapter so the test never loads runner.ts (which requires +// the compiled ./platform artifact unavailable in the test environment). +vi.mock("../adapters/docker/run", () => ({ + dockerRun: vi.fn(), + dockerCapture: vi.fn(), +})); + +import { OLLAMA_PROXY_PORT } from "../core/ports"; +import { + DEFAULT_OLLAMA_PROBE_NETWORK, + formatOllamaProxyUnreachableMessage, + probeOllamaProxySandboxReachability, + __test, +} from "./ollama-proxy-reachability"; + +const { parseNetworkIpamConfig } = __test; + +// ── parseNetworkIpamConfig ─────────────────────────────────────────────────── + +describe("parseNetworkIpamConfig", () => { + it("parses a well-formed IPAM config with IPv4 gateway", () => { + const raw = JSON.stringify([ + { Subnet: "172.20.0.0/16", Gateway: "172.20.0.1" }, + ]); + expect(parseNetworkIpamConfig(raw)).toEqual({ + subnet: "172.20.0.0/16", + gatewayIp: "172.20.0.1", + }); + }); + + it("skips IPv6 entries and returns first IPv4 entry", () => { + const raw = JSON.stringify([ + { Subnet: "fd00::/64", Gateway: "fd00::1" }, + { Subnet: "10.0.0.0/8", Gateway: "10.0.0.1" }, + ]); + expect(parseNetworkIpamConfig(raw)).toEqual({ + subnet: "10.0.0.0/8", + gatewayIp: "10.0.0.1", + }); + }); + + it("returns undefined for empty string", () => { + expect(parseNetworkIpamConfig("")).toBeUndefined(); + }); + + it("returns undefined for Docker '' sentinel", () => { + expect(parseNetworkIpamConfig("")).toBeUndefined(); + }); + + it("returns undefined for invalid JSON", () => { + expect(parseNetworkIpamConfig("not-json")).toBeUndefined(); + }); + + it("returns undefined for non-array JSON", () => { + expect(parseNetworkIpamConfig('{"Subnet":"10.0.0.0/8"}')).toBeUndefined(); + }); + + it("returns undefined for empty array", () => { + expect(parseNetworkIpamConfig("[]")).toBeUndefined(); + }); + + it("returns undefined when all entries lack an IPv4 Gateway field", () => { + // No Gateway field in any entry — the loop finds no IPv4 gateway to return + const raw = JSON.stringify([{ Subnet: "192.168.0.0/20" }]); + expect(parseNetworkIpamConfig(raw)).toBeUndefined(); + }); +}); + +// ── probeOllamaProxySandboxReachability ────────────────────────────────────── + +function makeNetwork( + partial: { subnet?: string; gatewayIp?: string } = {}, +): { subnet?: string; gatewayIp?: string } { + return { subnet: "172.20.0.0/16", gatewayIp: "172.20.0.1", ...partial }; +} + +describe("probeOllamaProxySandboxReachability (#3340)", () => { + it("returns probe_unavailable when the Docker network does not exist", async () => { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => undefined, + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: 0 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("probe_unavailable"); + expect(result.detail).toMatch(/not found/i); + }); + + it("returns probe_unavailable when network has no IPv4 gateway (non-host-gateway mode)", async () => { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => makeNetwork({ gatewayIp: undefined }), + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: 0 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("probe_unavailable"); + expect(result.detail).toMatch(/no IPv4 gateway/i); + }); + + it("proceeds without gatewayIp in host-gateway mode", async () => { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => makeNetwork({ gatewayIp: undefined }), + usesHostGatewayRouteImpl: () => true, + runImpl: () => ({ status: 0 }), + }); + expect(result.ok).toBe(true); + expect(result.reason).toBe("ok"); + }); + + it("returns ok when nc exits with status 0", async () => { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => makeNetwork(), + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: 0 }), + }); + expect(result.ok).toBe(true); + expect(result.reason).toBe("ok"); + expect(result.subnet).toBe("172.20.0.0/16"); + expect(result.gatewayIp).toBe("172.20.0.1"); + expect(result.networkName).toBe(DEFAULT_OLLAMA_PROBE_NETWORK); + }); + + it("returns tcp_failed when nc exits with status 1 on Linux native", async () => { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => makeNetwork(), + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: 1, stderr: "nc: connect failed" }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("tcp_failed"); + expect(result.detail).toContain("host.openshell.internal"); + expect(result.detail).toContain(String(OLLAMA_PROXY_PORT)); + }); + + it("returns probe_unavailable when nc exits with status 1 in host-gateway mode (Docker Desktop)", async () => { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => makeNetwork(), + usesHostGatewayRouteImpl: () => true, + runImpl: () => ({ status: 1 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("probe_unavailable"); + }); + + it("returns probe_unavailable for unexpected non-0/non-1 exit codes", async () => { + for (const code of [2, 127, 255]) { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => makeNetwork(), + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: code }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("probe_unavailable"); + } + }); + + it("returns probe_unavailable when the container runner reports an error", async () => { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => makeNetwork(), + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: null, error: "docker: Cannot connect to the Docker daemon" }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("probe_unavailable"); + expect(result.detail).toContain("docker: Cannot connect"); + }); + + it("returns probe_unavailable on DNS resolution failures (bad address)", async () => { + const result = await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: () => makeNetwork(), + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ + status: 1, + stderr: "nc: bad address 'host.openshell.internal'", + }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("probe_unavailable"); + }); + + it("uses a custom networkName when OPENSHELL_DOCKER_NETWORK_NAME env var is set", async () => { + const original = process.env.OPENSHELL_DOCKER_NETWORK_NAME; + process.env.OPENSHELL_DOCKER_NETWORK_NAME = "my-custom-network"; + try { + let capturedArgs: readonly string[] = []; + await probeOllamaProxySandboxReachability({ + inspectNetworkImpl: (name) => { + if (name === "my-custom-network") return makeNetwork(); + return undefined; + }, + usesHostGatewayRouteImpl: () => false, + runImpl: (args) => { + capturedArgs = args; + return { status: 0 }; + }, + }); + expect(capturedArgs).toContain("my-custom-network"); + } finally { + if (original === undefined) { + delete process.env.OPENSHELL_DOCKER_NETWORK_NAME; + } else { + process.env.OPENSHELL_DOCKER_NETWORK_NAME = original; + } + } + }); + + it("passes the correct docker run arguments including --add-host and nc command", async () => { + let capturedArgs: readonly string[] = []; + await probeOllamaProxySandboxReachability({ + port: 11435, + networkName: "test-network", + timeoutSec: 3, + inspectNetworkImpl: () => makeNetwork({ gatewayIp: "172.99.0.1" }), + usesHostGatewayRouteImpl: () => false, + runImpl: (args) => { + capturedArgs = args; + return { status: 0 }; + }, + }); + expect(capturedArgs).toContain("--rm"); + expect(capturedArgs).toContain("test-network"); + expect(capturedArgs).toContain("host.openshell.internal:172.99.0.1"); + expect(capturedArgs).toContain("nc"); + expect(capturedArgs).toContain("-zw3"); + expect(capturedArgs).toContain("host.openshell.internal"); + expect(capturedArgs).toContain("11435"); + }); + + it("uses host-gateway alias in the --add-host flag when in host-gateway mode", async () => { + let capturedArgs: readonly string[] = []; + await probeOllamaProxySandboxReachability({ + networkName: "test-network", + inspectNetworkImpl: () => makeNetwork({ gatewayIp: "172.99.0.1" }), + usesHostGatewayRouteImpl: () => true, + runImpl: (args) => { + capturedArgs = args; + return { status: 0 }; + }, + }); + expect(capturedArgs).toContain("host.openshell.internal:host-gateway"); + expect(capturedArgs).not.toContain("host.openshell.internal:172.99.0.1"); + }); +}); + +// ── formatOllamaProxyUnreachableMessage ────────────────────────────────────── + +describe("formatOllamaProxyUnreachableMessage", () => { + it("returns empty string for ok result", () => { + expect( + formatOllamaProxyUnreachableMessage({ + ok: true, + reason: "ok", + networkName: "openshell-docker", + }), + ).toBe(""); + }); + + it("returns empty string for probe_unavailable result", () => { + expect( + formatOllamaProxyUnreachableMessage({ + ok: false, + reason: "probe_unavailable", + networkName: "openshell-docker", + }), + ).toBe(""); + }); + + it("includes subnet-specific ufw command when subnet is known", () => { + const msg = formatOllamaProxyUnreachableMessage({ + ok: false, + reason: "tcp_failed", + networkName: "openshell-docker", + subnet: "172.20.0.0/16", + }); + expect(msg).toContain("172.20.0.0/16"); + expect(msg).toContain(String(OLLAMA_PROXY_PORT)); + expect(msg).toContain("sudo ufw allow"); + expect(msg).toContain("host.openshell.internal"); + expect(msg).toContain("nemoclaw onboard"); + }); + + it("includes dynamic SUBNET= fallback when subnet is unknown", () => { + const msg = formatOllamaProxyUnreachableMessage({ + ok: false, + reason: "tcp_failed", + networkName: "openshell-docker", + }); + expect(msg).toContain("SUBNET="); + expect(msg).toContain("docker network inspect openshell-docker"); + expect(msg).toContain(String(OLLAMA_PROXY_PORT)); + expect(msg).toContain("sudo ufw allow"); + }); + + it("uses the custom port argument when provided", () => { + const msg = formatOllamaProxyUnreachableMessage( + { + ok: false, + reason: "tcp_failed", + networkName: "openshell-docker", + subnet: "10.0.0.0/8", + }, + 19999, + ); + expect(msg).toContain("19999"); + expect(msg).not.toContain(String(OLLAMA_PROXY_PORT)); + }); +}); diff --git a/src/lib/onboard/ollama-proxy-reachability.ts b/src/lib/onboard/ollama-proxy-reachability.ts new file mode 100644 index 00000000000..708fc77f680 --- /dev/null +++ b/src/lib/onboard/ollama-proxy-reachability.ts @@ -0,0 +1,256 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Sandbox-side reachability probe for the Ollama auth proxy (port 11435). + * + * Issue #3340: On Brev VMs (and any Linux host with UFW default-deny), the + * Ollama auth proxy on port 11435 is unreachable from the sandbox's Docker + * bridge network. Host-side validation cannot detect this because + * host.openshell.internal only resolves inside the sandbox network. This + * probe runs a short-lived container on the same Docker network OpenShell + * uses for sandboxes and performs a TCP connect to the proxy port, mirroring + * the exact route the real sandbox takes. A tcp_failed result means a host + * firewall is blocking the port; the caller can then surface an actionable + * ufw remediation before declaring onboard successful. + */ + +import { OLLAMA_PROXY_PORT } from "../core/ports"; +import { dockerCapture, dockerRun } from "../adapters/docker/run"; + +export const DEFAULT_OLLAMA_PROBE_NETWORK = "openshell-docker"; +const HOST_INTERNAL_NAME = "host.openshell.internal"; +// Pinned busybox digest — same image used by the gateway bridge probe so +// it is likely already pulled and avoids a redundant registry fetch. +const PROBE_IMAGE = + "busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662"; +const PROBE_TIMEOUT_SEC = 5; +const PROBE_OVERHEAD_MS = 10_000; + +export type OllamaProxyReachabilityReason = "ok" | "tcp_failed" | "probe_unavailable"; + +export interface OllamaProxyReachabilityResult { + ok: boolean; + reason: OllamaProxyReachabilityReason; + networkName: string; + subnet?: string; + gatewayIp?: string; + detail?: string; +} + +interface ProbeRunResult { + status: number | null; + signal?: NodeJS.Signals | null; + error?: string; + stderr?: string | Buffer | null; +} + +export interface OllamaProxyReachabilityOptions { + port?: number; + networkName?: string; + timeoutSec?: number; + probeImage?: string; + runImpl?: (args: readonly string[], timeoutMs: number) => ProbeRunResult; + inspectNetworkImpl?: (networkName: string) => { subnet?: string; gatewayIp?: string } | undefined; + usesHostGatewayRouteImpl?: () => boolean; +} + +function parseNetworkIpamConfig( + raw: string, +): { subnet?: string; gatewayIp?: string } | undefined { + const text = raw.trim(); + if (!text || text === "") return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return undefined; + } + if (!Array.isArray(parsed)) return undefined; + for (const entry of parsed) { + if (!entry || typeof entry !== "object") continue; + const r = entry as Record; + const subnet = typeof r.Subnet === "string" ? r.Subnet : undefined; + const gatewayIp = typeof r.Gateway === "string" ? r.Gateway : undefined; + // Skip IPv6-only entries (contain colons) + if (gatewayIp && !gatewayIp.includes(":")) return { subnet, gatewayIp }; + } + return undefined; +} + +function defaultInspectNetwork( + networkName: string, +): { subnet?: string; gatewayIp?: string } | undefined { + const raw = dockerCapture( + ["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName], + { ignoreError: true }, + ); + return parseNetworkIpamConfig(raw); +} + +// Docker Desktop and VM-backed Docker use a special host-gateway alias rather +// than a specific bridge IP. UFW is not relevant on those platforms, so we +// classify probes from those environments as probe_unavailable. +function defaultUsesHostGatewayRoute(): boolean { + if (process.platform !== "linux") return true; + const info = dockerCapture( + ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], + { ignoreError: true }, + ); + return /Docker Desktop|com\.docker\.desktop\./i.test(info); +} + +function defaultRunImpl(args: readonly string[], timeoutMs: number): ProbeRunResult { + const result = dockerRun(args, { + timeout: timeoutMs, + ignoreError: true, + suppressOutput: true, + stdio: ["ignore", "pipe", "pipe"], + }); + return { + status: result.status ?? null, + signal: result.signal, + error: result.error?.message, + stderr: result.stderr, + }; +} + +function outputTail(value: unknown): string | undefined { + const raw = + Buffer.isBuffer(value) + ? value.toString("utf8") + : value == null + ? "" + : String(value); + const text = raw.trim(); + return text ? text.slice(-400) : undefined; +} + +function isNameResolutionFailure(detail: string): boolean { + return /bad address|name or service not known|temporary failure in name resolution|could not resolve|getaddrinfo/i.test( + detail, + ); +} + +export async function probeOllamaProxySandboxReachability( + opts: OllamaProxyReachabilityOptions = {}, +): Promise { + const networkName = + opts.networkName ?? + process.env.OPENSHELL_DOCKER_NETWORK_NAME ?? + DEFAULT_OLLAMA_PROBE_NETWORK; + const port = opts.port ?? OLLAMA_PROXY_PORT; + const timeoutSec = opts.timeoutSec ?? PROBE_TIMEOUT_SEC; + const probeImage = opts.probeImage ?? PROBE_IMAGE; + const inspectNetwork = opts.inspectNetworkImpl ?? defaultInspectNetwork; + const usesHostGatewayRoute = opts.usesHostGatewayRouteImpl ?? defaultUsesHostGatewayRoute; + const runImpl = opts.runImpl ?? defaultRunImpl; + + const network = inspectNetwork(networkName); + if (!network) { + return { + ok: false, + reason: "probe_unavailable", + networkName, + detail: `Docker network "${networkName}" not found`, + }; + } + + const isHostGateway = usesHostGatewayRoute(); + + if (!isHostGateway && !network.gatewayIp) { + return { + ok: false, + reason: "probe_unavailable", + networkName, + subnet: network.subnet, + detail: `Docker network "${networkName}" has no IPv4 gateway`, + }; + } + + const hostInternalTarget = isHostGateway ? "host-gateway" : (network.gatewayIp as string); + + const probeArgs = [ + "run", + "--rm", + "--pull=missing", + "--network", + networkName, + "--add-host", + `${HOST_INTERNAL_NAME}:${hostInternalTarget}`, + probeImage, + "nc", + `-zw${timeoutSec}`, + HOST_INTERNAL_NAME, + String(port), + ]; + + const result = runImpl(probeArgs, timeoutSec * 1000 + PROBE_OVERHEAD_MS); + + if (result.status === 0) { + return { + ok: true, + reason: "ok", + networkName, + subnet: network.subnet, + gatewayIp: network.gatewayIp, + }; + } + + const detail = [ + result.error, + outputTail(result.stderr), + result.signal ? `signal ${result.signal}` : undefined, + result.status !== null ? `exit ${result.status}` : undefined, + ] + .filter((s): s is string => Boolean(s)) + .join(" | "); + + // Classify as probe_unavailable for: non-nc exit codes, DNS failures, + // or host-gateway mode (Docker Desktop / macOS — no UFW concern there). + if (result.status !== 1 || isNameResolutionFailure(detail) || isHostGateway) { + return { + ok: false, + reason: "probe_unavailable", + networkName, + subnet: network.subnet, + gatewayIp: network.gatewayIp, + detail: detail || "probe did not complete", + }; + } + + return { + ok: false, + reason: "tcp_failed", + networkName, + subnet: network.subnet, + gatewayIp: network.gatewayIp, + detail: `sandbox container on "${networkName}" could not reach ${HOST_INTERNAL_NAME}:${port}`, + }; +} + +export function formatOllamaProxyUnreachableMessage( + result: OllamaProxyReachabilityResult, + port: number = OLLAMA_PROXY_PORT, +): string { + if (result.ok || result.reason !== "tcp_failed") return ""; + + const allowCmd = result.subnet + ? ` sudo ufw allow from ${result.subnet} to any port ${port} proto tcp` + : [ + ` SUBNET=$(docker network inspect ${result.networkName ?? DEFAULT_OLLAMA_PROBE_NETWORK} --format '{{(index .IPAM.Config 0).Subnet}}')`, + ` sudo ufw allow from "$SUBNET" to any port ${port} proto tcp`, + ].join("\n"); + + return [ + ` ✗ Sandbox containers cannot reach the Ollama auth proxy at ${HOST_INTERNAL_NAME}:${port}.`, + " A host firewall may be blocking traffic from the OpenShell Docker bridge.", + " To allow it:", + allowCmd, + " Then re-run `nemoclaw onboard`.", + ].join("\n"); +} + +export const __test = { + parseNetworkIpamConfig, +};