diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 2946c46f29a..ba4e85cf45a 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1905,7 +1905,7 @@ async function preflight( !sandboxGpuConfig.sandboxGpuEnabled; assertCdiNvidiaGpuSpecPresent(host, optedOutGpuPassthrough, sandboxGpuConfig.hostGpuPlatform); - assertDockerBridgeAndContainerDnsHealthy(host); + assertDockerBridgeAndContainerDnsHealthy(host, isNonInteractive()); if (host.runtime !== "unknown") { console.log(` ✓ Container runtime: ${host.runtime}`); diff --git a/src/lib/onboard/bridge-dns-preflight.ts b/src/lib/onboard/bridge-dns-preflight.ts index 726ec410a98..e37cce3f779 100644 --- a/src/lib/onboard/bridge-dns-preflight.ts +++ b/src/lib/onboard/bridge-dns-preflight.ts @@ -77,13 +77,16 @@ function printDaemonJsonDnsPatch(opts: DaemonJsonDnsPatchOpts): void { } import { BUSYBOX_PROBE_IMAGE, + DEFAULT_HOST_DNS_PROBE_HOSTNAME, DOCKER_DESKTOP_WSL_INTEGRATION_HINT, type DockerBridgeContainerStartProbeResult, getDockerBridgeGatewayIp, type HostAssessment, isFatalContainerDnsProbeFailure, + isFatalHostDnsProbeFailure, probeContainerDns, probeDockerBridgeContainerStart, + probeHostDns, } from "./preflight"; type Host = HostAssessment; @@ -145,7 +148,7 @@ export function printDockerBridgeContainerStartFailure( * wall (mirroring the [[assertCdiNvidiaGpuSpecPresent]] resume backstop * pattern at #3152). */ -export function assertDockerBridgeAndContainerDnsHealthy(host: Host): void { +export function assertDockerBridgeAndContainerDnsHealthy(host: Host, nonInteractive = false): void { // A minimal bridge-backed container start catches Docker/kernel failures // (notably Jetson veth "operation not supported") before longer gateway or // sandbox build work starts. Only veth/timeout/killed/daemon-unreachable @@ -176,6 +179,15 @@ export function assertDockerBridgeAndContainerDnsHealthy(host: Host): void { console.warn(" Continuing to DNS probe for more specific diagnosis."); } + // Host-side DNS (#4784). The container DNS probe below only proves the + // docker network namespace can resolve names; a host OUTPUT chain that + // drops tcp/udp:53 still lets that pass while the CLI process itself + // cannot resolve the provider endpoint, so later NVIDIA Endpoints + // validation dies with `curl: (6) Could not resolve host: ...`. Catch + // that here, before provider validation, and keep it distinct from the + // container-DNS line. + assertHostDnsHealthy(host, { nonInteractive }); + // DNS resolution from inside containers (#2101). A corp firewall that // blocks outbound UDP:53 to public resolvers leaves the sandbox build // unable to resolve registry.npmjs.org; npm then retries for ~15 min and @@ -253,6 +265,182 @@ export function assertDockerBridgeAndContainerDnsHealthy(host: Host): void { process.exit(1); } +/** + * Environment escape hatch for the host DNS preflight. Air-gapped or + * local-inference-only hosts that intentionally cannot resolve public + * provider endpoints can set this to skip the check rather than abort. + */ +export const SKIP_HOST_DNS_PREFLIGHT_ENV = "NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT"; + +function hostDnsPreflightSkipped(env: NodeJS.ProcessEnv = process.env): boolean { + const value = env[SKIP_HOST_DNS_PREFLIGHT_ENV]; + return value === "1" || String(value || "").toLowerCase() === "true"; +} + +// `NEMOCLAW_PROVIDER` keys that resolve to NVIDIA-hosted endpoints +// (integrate.api.nvidia.com). Mirrors the aliases in +// `onboard/providers.ts::getNonInteractiveProvider`. Local/custom and +// other hosted providers (ollama, vllm, openai, anthropic, nim-local, …) +// do not need this host, so the NVIDIA host DNS probe must not gate them. +const NVIDIA_ENDPOINT_PROVIDER_KEYS = new Set(["build", "cloud", "routed"]); + +/** + * Whether onboarding's effective inference provider is NVIDIA Endpoints, + * so the `integrate.api.nvidia.com` host DNS probe is relevant. + * + * `NEMOCLAW_PROVIDER` is honored only in non-interactive mode (mirroring + * `getRequestedProviderHint`), where an unset value defaults to NVIDIA + * Endpoints. A fresh interactive run hits preflight *before* the provider + * menu, so it returns false and is never blocked on NVIDIA-domain DNS — an + * interactive user can still pick a local provider (codex #4784 P2). + */ +function usesNvidiaEndpointProvider( + env: NodeJS.ProcessEnv = process.env, + nonInteractive = false, +): boolean { + if (!nonInteractive) return false; + const envKey = String(env.NEMOCLAW_PROVIDER || "") + .trim() + .toLowerCase(); + if (!envKey) return true; + return NVIDIA_ENDPOINT_PROVIDER_KEYS.has(envKey); +} + +/** + * Whether the provider host is reached through a configured HTTPS/ALL + * proxy (honoring NO_PROXY). When it is, the later curl-based provider + * validation hands the hostname to the proxy and never resolves it + * locally, so a direct host DNS probe would false-fail valid corporate / + * proxied setups (codex #4784 P2). Skip the probe in that case. + */ +function providerHostProxied(env: NodeJS.ProcessEnv, hostname: string): boolean { + const proxy = env.HTTPS_PROXY || env.https_proxy || env.ALL_PROXY || env.all_proxy || ""; + if (!String(proxy).trim()) return false; + const noProxy = String(env.NO_PROXY || env.no_proxy || ""); + const host = hostname.trim().toLowerCase(); + const bypassed = noProxy + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter(Boolean) + .some((entry) => { + if (entry === "*") return true; + const e = entry.replace(/^\*?\./, ""); + return host === e || host.endsWith(`.${e}`); + }); + return !bypassed; +} + +export interface AssertHostDnsHealthyOpts { + /** Whether onboarding is running non-interactively (default NVIDIA path). */ + nonInteractive?: boolean; + /** Inject a host DNS probe result (test seam). */ + probeHostDnsImpl?: typeof probeHostDns; + /** Override the skip-env decision (test seam). */ + env?: NodeJS.ProcessEnv; + /** Override process exit (test seam). */ + exit?: (code: number) => void; +} + +/** + * Host DNS gate (#4784). Resolves the provider endpoint from the CLI + * process and prints a `Host DNS resolution` line distinct from the + * `Container DNS resolution` line. A fatal failure (resolver unreachable + * or the name cannot be resolved) exits early with host-DNS remediation; + * an inconclusive probe (could not even spawn) warns and continues so a + * probe-infrastructure hiccup never blocks onboarding. + */ +export function assertHostDnsHealthy(host: Host, opts: AssertHostDnsHealthyOpts = {}): void { + const env = opts.env ?? process.env; + const exit = opts.exit ?? ((code: number) => process.exit(code)); + const probe = opts.probeHostDnsImpl ?? probeHostDns; + + if (hostDnsPreflightSkipped(env)) { + console.log(` ⓘ Host DNS resolution check skipped (${SKIP_HOST_DNS_PREFLIGHT_ENV} is set)`); + return; + } + + // The probe resolves the NVIDIA Endpoints host, so only run it when that + // is the effective provider. A user who explicitly selected a local or + // non-NVIDIA provider — or who hasn't chosen one yet in interactive mode — + // must not be blocked by NVIDIA-domain DNS. + if (!usesNvidiaEndpointProvider(env, opts.nonInteractive ?? false)) { + return; + } + + // When the provider host is reached through a configured proxy, curl-based + // provider validation never resolves it locally, so a direct DNS probe + // would false-fail. Skip and let provider validation use the proxy route. + if (providerHostProxied(env, DEFAULT_HOST_DNS_PROBE_HOSTNAME)) { + console.log( + " ⓘ Host DNS resolution check skipped (HTTPS proxy configured; provider reached via proxy)", + ); + return; + } + + const result = probe(); + if (result.ok) { + console.log(" ✓ Host DNS resolution works"); + return; + } + if (!isFatalHostDnsProbeFailure(result)) { + console.warn(` ⚠ Host DNS probe inconclusive (reason: ${result.reason ?? "unknown"}).`); + if (result.details) { + console.warn(` ${String(result.details).trim()}`); + } + console.warn( + " Proceeding; provider validation will surface a clearer error if DNS is broken.", + ); + return; + } + + if (result.reason === "timeout" || result.reason === "killed") { + console.error(` ✗ Host DNS probe did not complete (could not resolve ${result.hostname}).`); + } else if (result.reason === "resolution_failed") { + console.error(` ✗ Host could not resolve ${result.hostname} (resolver answered, no record).`); + } else { + console.error(` ✗ Host DNS resolution failed (could not resolve ${result.hostname}).`); + } + if (result.details) { + console.error(` ${String(result.details).trim()}`); + } + console.error(""); + printHostDnsRemediation(host, result.hostname); + exit(1); +} + +export function printHostDnsRemediation( + host: Pick, + hostname: string = DEFAULT_HOST_DNS_PROBE_HOSTNAME, +): void { + console.error(` ${cliDisplayName()} could not resolve ${hostname} from this host.`); + console.error(" Container DNS may still look healthy, but the CLI itself cannot reach a DNS"); + console.error(" resolver, so provider validation will later fail with"); + console.error(` \`curl: (6) Could not resolve host: ${hostname}\`. See issue #4784.`); + console.error(""); + console.error(" Common causes and fixes:"); + console.error(""); + console.error(" 1. A firewall is dropping outbound DNS (tcp/udp port 53). On Linux, check the"); + console.error(" OUTPUT chain and remove any port-53 DROP/REJECT rules:"); + console.error(" sudo iptables -S OUTPUT | grep -- '--dport 53'"); + console.error(" sudo nft list ruleset | grep -- 53 # if you use nftables"); + console.error(" 2. A VPN or corporate split-DNS / kill-switch is blocking public DNS — connect"); + console.error(" to the VPN (or disconnect it) so the host can resolve provider endpoints."); + console.error(" 3. The system resolver is misconfigured — verify /etc/resolv.conf points at a"); + console.error(" reachable nameserver, then retry."); + if (host.platform === "win32" || host.isWsl) { + console.error(" 4. On Windows/WSL, check Windows Firewall and any VPN client DNS settings."); + } + console.error(""); + console.error(` If this host is intentionally air-gapped or uses only local inference, set`); + console.error(` ${SKIP_HOST_DNS_PREFLIGHT_ENV}=1 to skip this check.`); + console.error(""); + console.error(" Verify the fix worked:"); + console.error( + ` node -e "require('node:dns').resolve('${hostname}', (e,a)=>{if(e)throw e;console.log(a)})"`, + ); + console.error(` curl -sS https://${hostname}/v1/models -o /dev/null && echo reachable`); +} + export function printContainerDnsRemediation(host: Host): void { console.error(" The sandbox build runs `npm ci` inside a container and needs to resolve"); console.error(" registry.npmjs.org. On networks that block outbound UDP:53 to public DNS"); diff --git a/src/lib/onboard/host-dns-preflight.test.ts b/src/lib/onboard/host-dns-preflight.test.ts new file mode 100644 index 00000000000..c522875e0c7 --- /dev/null +++ b/src/lib/onboard/host-dns-preflight.test.ts @@ -0,0 +1,401 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Host DNS preflight (#4784): the CLI process must be able to resolve the +// provider endpoint over port 53. A host OUTPUT chain that drops tcp/udp:53 +// lets the container DNS probe pass while later provider validation dies with +// `curl: (6) Could not resolve host: integrate.api.nvidia.com`. These tests +// cover the host-side probe (preflight.ts) plus the gate and remediation that +// surface it before provider validation (bridge-dns-preflight.ts). + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + assertHostDnsHealthy, + printHostDnsRemediation, +} from "../../../dist/lib/onboard/bridge-dns-preflight"; +import { isFatalHostDnsProbeFailure, probeHostDns } from "../../../dist/lib/onboard/preflight"; + +describe("probeHostDns (#4784)", () => { + // The probe runs `node -e `. Inject a + // structured execution so the test never spawns node or touches DNS. + const exec = (over: Record) => () => ({ + stdout: "", + stderr: "", + exitCode: 0, + signal: null, + timedOut: false, + error: null, + ...over, + }); + + it("returns ok when the resolver answers (HOSTDNS_OK on stdout)", () => { + const result = probeHostDns({ + runProbeImpl: exec({ stdout: "HOSTDNS_OK 1.2.3.4,5.6.7.8", exitCode: 0 }), + }); + expect(result.ok).toBe(true); + expect(result.hostname).toBe("integrate.api.nvidia.com"); + expect(result.reason).toBeUndefined(); + expect(isFatalHostDnsProbeFailure(result)).toBe(false); + }); + + it("flags servers_unreachable on ECONNREFUSED — the host OUTPUT-chain DNS block (#4784)", () => { + const result = probeHostDns({ + runProbeImpl: exec({ stderr: "HOSTDNS_ERR ECONNREFUSED", exitCode: 3 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("servers_unreachable"); + expect(result.details).toContain("ECONNREFUSED"); + expect(isFatalHostDnsProbeFailure(result)).toBe(true); + }); + + it("flags servers_unreachable on ETIMEOUT (silently dropped UDP:53)", () => { + const result = probeHostDns({ + runProbeImpl: exec({ stderr: "HOSTDNS_ERR ETIMEOUT", exitCode: 3 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("servers_unreachable"); + expect(isFatalHostDnsProbeFailure(result)).toBe(true); + }); + + it("flags servers_unreachable on EAI_AGAIN — the getaddrinfo signature of a blocked resolver (#4784)", () => { + // `dns.lookup` (getaddrinfo) returns EAI_AGAIN when the stub/upstream + // resolver is unreachable, which is what the iptables OUTPUT-chain + // port-53 DROP produces in practice. + const result = probeHostDns({ + runProbeImpl: exec({ stderr: "HOSTDNS_ERR EAI_AGAIN", exitCode: 3 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("servers_unreachable"); + expect(isFatalHostDnsProbeFailure(result)).toBe(true); + }); + + it("flags resolution_failed (fatal) on ENOTFOUND for the always-present provider host", () => { + const result = probeHostDns({ + runProbeImpl: exec({ stderr: "HOSTDNS_ERR ENOTFOUND", exitCode: 3 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("resolution_failed"); + expect(isFatalHostDnsProbeFailure(result)).toBe(true); + }); + + it("flags timeout when the child prints the inner-timeout marker", () => { + const result = probeHostDns({ + runProbeImpl: exec({ stderr: "HOSTDNS_TIMEOUT", exitCode: 2 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("timeout"); + expect(isFatalHostDnsProbeFailure(result)).toBe(true); + }); + + it("flags timeout when the spawn itself times out (signal/ETIMEDOUT)", () => { + const result = probeHostDns({ + runProbeImpl: exec({ stderr: "", exitCode: null, signal: "SIGTERM", timedOut: true }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("timeout"); + expect(isFatalHostDnsProbeFailure(result)).toBe(true); + }); + + it("stays inconclusive (error, non-fatal) when node could not be spawned", () => { + const result = probeHostDns({ + runProbeImpl: exec({ + stderr: "spawn node ENOENT", + exitCode: null, + error: "spawn node ENOENT", + }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("error"); + // Inconclusive: never abort onboarding on a probe-infra failure. + expect(isFatalHostDnsProbeFailure(result)).toBe(false); + }); + + it("stays inconclusive on an unknown c-ares error code", () => { + const result = probeHostDns({ + runProbeImpl: exec({ stderr: "HOSTDNS_ERR ESOMETHINGWEIRD", exitCode: 3 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("error"); + expect(isFatalHostDnsProbeFailure(result)).toBe(false); + }); + + it("resolves a caller-provided hostname", () => { + const result = probeHostDns({ + hostname: "example.com", + runProbeImpl: exec({ stdout: "HOSTDNS_OK 93.184.216.34", exitCode: 0 }), + }); + expect(result.ok).toBe(true); + expect(result.hostname).toBe("example.com"); + }); + + it("rejects a hostname with shell/JS metacharacters", () => { + expect(() => probeHostDns({ hostname: "evil.com; rm -rf /" })).toThrow(/plain DNS name/); + }); +}); + +describe("printHostDnsRemediation (#4784)", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("explains the host-vs-container DNS distinction and the iptables OUTPUT cause", () => { + const messages: string[] = []; + vi.spyOn(console, "error").mockImplementation((arg?: unknown) => { + messages.push(String(arg ?? "")); + }); + printHostDnsRemediation({ platform: "linux", isWsl: false }, "integrate.api.nvidia.com"); + const blob = messages.join("\n"); + expect(blob).toContain("could not resolve integrate.api.nvidia.com"); + expect(blob).toContain("Container DNS may still look healthy"); + expect(blob).toContain("curl: (6) Could not resolve host: integrate.api.nvidia.com"); + expect(blob).toContain("--dport 53"); + expect(blob).toContain("NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT=1"); + expect(blob).toContain("#4784"); + }); + + it("adds a Windows/WSL firewall hint only on those platforms", () => { + const winMessages: string[] = []; + const errSpy = vi.spyOn(console, "error").mockImplementation((arg?: unknown) => { + winMessages.push(String(arg ?? "")); + }); + printHostDnsRemediation({ platform: "win32", isWsl: false }); + expect(winMessages.join("\n")).toContain("Windows Firewall"); + errSpy.mockRestore(); + + const linuxMessages: string[] = []; + vi.spyOn(console, "error").mockImplementation((arg?: unknown) => { + linuxMessages.push(String(arg ?? "")); + }); + printHostDnsRemediation({ platform: "linux", isWsl: false }); + expect(linuxMessages.join("\n")).not.toContain("Windows Firewall"); + }); +}); + +describe("assertHostDnsHealthy (#4784)", () => { + const host = { platform: "linux", isWsl: false } as Parameters[0]; + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("prints the success line and does not exit when host DNS resolves", () => { + const logs: string[] = []; + vi.spyOn(console, "log").mockImplementation((arg?: unknown) => logs.push(String(arg ?? ""))); + const exit = vi.fn(); + assertHostDnsHealthy(host, { + env: {}, + nonInteractive: true, + exit, + probeHostDnsImpl: () => ({ ok: true, hostname: "integrate.api.nvidia.com" }), + }); + expect(logs.join("\n")).toContain("✓ Host DNS resolution works"); + expect(exit).not.toHaveBeenCalled(); + }); + + it("aborts (exit 1) with remediation when host DNS is blocked but container DNS would pass (#4784)", () => { + const errors: string[] = []; + vi.spyOn(console, "error").mockImplementation((arg?: unknown) => + errors.push(String(arg ?? "")), + ); + const exit = vi.fn(); + assertHostDnsHealthy(host, { + env: {}, + nonInteractive: true, + exit, + probeHostDnsImpl: () => ({ + ok: false, + hostname: "integrate.api.nvidia.com", + reason: "servers_unreachable", + details: "dns.resolve integrate.api.nvidia.com: ECONNREFUSED", + }), + }); + expect(exit).toHaveBeenCalledWith(1); + const blob = errors.join("\n"); + expect(blob).toContain("✗ Host DNS resolution failed"); + expect(blob).toContain("could not resolve integrate.api.nvidia.com"); + expect(blob).toContain("--dport 53"); + }); + + it("aborts on resolution_failed with the resolver-answered wording", () => { + const errors: string[] = []; + vi.spyOn(console, "error").mockImplementation((arg?: unknown) => + errors.push(String(arg ?? "")), + ); + const exit = vi.fn(); + assertHostDnsHealthy(host, { + env: {}, + nonInteractive: true, + exit, + probeHostDnsImpl: () => ({ + ok: false, + hostname: "integrate.api.nvidia.com", + reason: "resolution_failed", + details: "dns.resolve integrate.api.nvidia.com: ENOTFOUND", + }), + }); + expect(exit).toHaveBeenCalledWith(1); + expect(errors.join("\n")).toContain("resolver answered, no record"); + }); + + it("warns and continues (no exit) when the probe is inconclusive", () => { + const warns: string[] = []; + vi.spyOn(console, "warn").mockImplementation((arg?: unknown) => warns.push(String(arg ?? ""))); + const exit = vi.fn(); + assertHostDnsHealthy(host, { + env: {}, + nonInteractive: true, + exit, + probeHostDnsImpl: () => ({ + ok: false, + hostname: "integrate.api.nvidia.com", + reason: "error", + details: "spawn node ENOENT", + }), + }); + expect(exit).not.toHaveBeenCalled(); + expect(warns.join("\n")).toContain("Host DNS probe inconclusive"); + }); + + it("skips the check (no exit, no probe) when NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT is set", () => { + const logs: string[] = []; + vi.spyOn(console, "log").mockImplementation((arg?: unknown) => logs.push(String(arg ?? ""))); + const exit = vi.fn(); + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + assertHostDnsHealthy(host, { + env: { NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT: "1" }, + exit, + probeHostDnsImpl: probe, + }); + expect(probe).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + expect(logs.join("\n")).toContain("Host DNS resolution check skipped"); + }); + + it("skips silently (no probe, no exit) when a non-NVIDIA provider is selected (codex P2)", () => { + const exit = vi.fn(); + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + // A user who picked a local/non-NVIDIA provider must not be blocked by + // NVIDIA-domain DNS even if their host cannot resolve it — including in + // non-interactive mode where the choice is explicit. + for (const provider of ["ollama", "openai", "anthropic", "vllm", "custom"]) { + assertHostDnsHealthy(host, { + env: { NEMOCLAW_PROVIDER: provider }, + nonInteractive: true, + exit, + probeHostDnsImpl: probe, + }); + } + expect(probe).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + }); + + it("skips an unset provider in interactive mode (provider not yet chosen — codex P2)", () => { + const exit = vi.fn(); + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + // Fresh interactive onboarding hits preflight before the provider menu; + // it may end up on Ollama/vLLM, so an NVIDIA-DNS block must not abort here. + assertHostDnsHealthy(host, { + env: {}, + nonInteractive: false, + exit, + probeHostDnsImpl: probe, + }); + expect(probe).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + }); + + it("runs for an unset provider only in non-interactive mode (NVIDIA Endpoints default)", () => { + const exit = vi.fn(); + vi.spyOn(console, "log").mockImplementation(() => {}); + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + assertHostDnsHealthy(host, { + env: {}, + nonInteractive: true, + exit, + probeHostDnsImpl: probe, + }); + expect(probe).toHaveBeenCalledTimes(1); + expect(exit).not.toHaveBeenCalled(); + }); + + it("runs for explicit NVIDIA Endpoints provider keys (build/cloud/routed) in non-interactive mode", () => { + const exit = vi.fn(); + vi.spyOn(console, "log").mockImplementation(() => {}); + for (const provider of ["build", "cloud", "routed"]) { + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + assertHostDnsHealthy(host, { + env: { NEMOCLAW_PROVIDER: provider }, + nonInteractive: true, + exit, + probeHostDnsImpl: probe, + }); + expect(probe).toHaveBeenCalledTimes(1); + } + expect(exit).not.toHaveBeenCalled(); + }); + + it("ignores NEMOCLAW_PROVIDER in interactive mode (onboard ignores it there too)", () => { + const exit = vi.fn(); + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + // Interactive onboarding ignores NEMOCLAW_PROVIDER and shows the menu, so + // we must not assume NVIDIA from it before the user has chosen. + assertHostDnsHealthy(host, { + env: { NEMOCLAW_PROVIDER: "build" }, + nonInteractive: false, + exit, + probeHostDnsImpl: probe, + }); + expect(probe).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + }); + + it("skips an explicit local NIM provider (nim-local) in non-interactive mode", () => { + const exit = vi.fn(); + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + // `nim-local` runs NIM locally and validates against localhost, not + // integrate.api.nvidia.com, so the NVIDIA host DNS probe must not gate it. + assertHostDnsHealthy(host, { + env: { NEMOCLAW_PROVIDER: "nim-local" }, + nonInteractive: true, + exit, + probeHostDnsImpl: probe, + }); + expect(probe).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + }); + + it("skips when the provider host is reached via a configured HTTPS proxy (codex P2 proxy)", () => { + const logs: string[] = []; + vi.spyOn(console, "log").mockImplementation((arg?: unknown) => logs.push(String(arg ?? ""))); + const exit = vi.fn(); + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + assertHostDnsHealthy(host, { + env: { NEMOCLAW_PROVIDER: "build", HTTPS_PROXY: "http://proxy.corp:3128" }, + nonInteractive: true, + exit, + probeHostDnsImpl: probe, + }); + expect(probe).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + expect(logs.join("\n")).toContain("HTTPS proxy configured"); + }); + + it("still runs when NO_PROXY exempts the provider host from the proxy", () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + const exit = vi.fn(); + const probe = vi.fn(() => ({ ok: true as const, hostname: "integrate.api.nvidia.com" })); + assertHostDnsHealthy(host, { + env: { + NEMOCLAW_PROVIDER: "build", + HTTPS_PROXY: "http://proxy.corp:3128", + NO_PROXY: ".nvidia.com", + }, + nonInteractive: true, + exit, + probeHostDnsImpl: probe, + }); + expect(probe).toHaveBeenCalledTimes(1); + expect(exit).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/preflight.ts b/src/lib/onboard/preflight.ts index b09e44ac283..e2bfaa35de1 100644 --- a/src/lib/onboard/preflight.ts +++ b/src/lib/onboard/preflight.ts @@ -1916,6 +1916,198 @@ export function isFatalContainerDnsProbeFailure(result: DnsProbeResult): boolean return result.reason === "image_pull_failed" && isRegistryResolutionFailure(result.details ?? ""); } +// ── Host DNS probe (#4784) ──────────────────────────────────────── +// `probeContainerDns` above only proves the *docker container* network +// namespace can reach a resolver. A host whose OUTPUT chain drops +// tcp/udp:53 (a corporate firewall, a VPN kill-switch, or a literal +// `iptables -I OUTPUT -p udp --dport 53 -j DROP`) can still pass the +// container probe — docker's embedded resolver at 127.0.0.11 and the +// bridge NAT take a different egress path — while the NemoClaw CLI +// process itself cannot resolve the provider endpoint. That gap let +// onboarding print "Container DNS resolution works" and then fail much +// later at NVIDIA Endpoints validation with the cryptic +// `curl: (6) Could not resolve host: integrate.api.nvidia.com`. This +// probe resolves the provider hostname from the host (CLI) process so +// the blocked-DNS condition surfaces up front, distinct from the +// container-DNS path. + +/** The NVIDIA Endpoints provider host onboarding validates by default. */ +export const DEFAULT_HOST_DNS_PROBE_HOSTNAME = "integrate.api.nvidia.com"; + +/** + * Host DNS probe budget (ms). Shorter than the container probe: there is + * no image pull, just a c-ares round-trip to the system resolver. + */ +const HOST_DNS_PROBE_TIMEOUT_MS = 10_000; + +export interface HostDnsProbeResult { + ok: boolean; + hostname: string; + reason?: "servers_unreachable" | "resolution_failed" | "timeout" | "killed" | "error"; + details?: string; +} + +export interface ProbeHostDnsOpts { + /** Hostname to resolve (default: the NVIDIA Endpoints provider host). */ + hostname?: string; + /** Override structured probe execution (test seam). */ + runProbeImpl?: RunProbeFn; + /** Override the node executable used to run the resolver (test seam). */ + nodeExecPath?: string; + /** Probe budget (ms). Defaults to HOST_DNS_PROBE_TIMEOUT_MS. */ + timeoutMs?: number; +} + +// getaddrinfo (and c-ares fallback) error codes meaning "the resolver +// could not be reached at all": the UDP/TCP:53 egress is blocked. This is +// the #4784 signature. `EAI_AGAIN` ("Temporary failure in name +// resolution") is what getaddrinfo returns when the stub/upstream resolver +// is unreachable. +const HOST_DNS_UNREACHABLE_CODES = new Set([ + "EAI_AGAIN", + "EAI_FAIL", + "ECONNREFUSED", + "ETIMEOUT", + "ETIMEDOUT", + "ESERVFAIL", + "ECONNRESET", + "EREFUSED", + "ENETUNREACH", + "EHOSTUNREACH", + "ECANCELLED", +]); + +// Error codes meaning "the resolver answered but there is no usable +// record". For a real, always-present provider host this still blocks +// onboarding (provider validation cannot resolve it), so it is fatal too. +const HOST_DNS_RESOLUTION_CODES = new Set([ + "ENOTFOUND", + "ENODATA", + "EAI_NONAME", + "EAI_NODATA", + "ENXDOMAIN", + "EBADNAME", + "EBADRESP", +]); + +/** + * Resolve `hostname` from the host (CLI) process to prove the host can + * reach a DNS resolver for the provider endpoint. Uses `dns.lookup` + * (getaddrinfo) — not `dns.resolve` — so it follows the *same* resolution + * path the later curl-based provider validation does: `/etc/hosts`, + * nsswitch, and the system resolver. That keeps it from false-failing + * hosts that reach the provider via an `/etc/hosts` entry. Runs `node -e` + * in a child process so the check stays synchronous like + * `probeContainerDns` and is fully injectable for tests. The hostname is + * passed as a process argument (never interpolated into the script body), + * so it cannot inject shell or JS tokens; we still validate it as a plain + * DNS name as defense in depth and to keep error output sane. + */ +export function probeHostDns(opts: ProbeHostDnsOpts = {}): HostDnsProbeResult { + const hostname = opts.hostname ?? DEFAULT_HOST_DNS_PROBE_HOSTNAME; + if (!/^[a-z0-9]([a-z0-9.-]{0,253})$/i.test(hostname)) { + throw new Error( + `hostname must be a plain DNS name (RFC 1035 label characters), got: ${JSON.stringify(hostname)}`, + ); + } + const timeoutMs = opts.timeoutMs ?? HOST_DNS_PROBE_TIMEOUT_MS; + // Bound the c-ares query inside the child too, so a silently dropped + // UDP:53 (no RST / ICMP unreachable) cannot outlive the spawn timeout: + // print a stable marker and exit non-zero a beat before the outer + // budget so the marker (not a SIGTERM) classifies the failure. + const innerTimeoutMs = Math.max(1_000, timeoutMs - 1_000); + const script = [ + "const dns=require('node:dns');", + "const host=process.argv[1];", + "let settled=false;", + `const t=setTimeout(()=>{if(settled)return;settled=true;process.stderr.write('HOSTDNS_TIMEOUT');process.exit(2);},${innerTimeoutMs});`, + "if(typeof t.unref==='function')t.unref();", + "dns.lookup(host,{all:false},(err,addr)=>{if(settled)return;settled=true;clearTimeout(t);", + "if(err){process.stderr.write('HOSTDNS_ERR '+(err.code||err.errno||err.message||'unknown'));process.exit(3);}", + "process.stdout.write('HOSTDNS_OK '+(addr||''));process.exit(0);});", + ].join(""); + const nodeExec = opts.nodeExecPath ?? process.execPath; + const command = [nodeExec, "-e", script, hostname]; + + const runProbe = opts.runProbeImpl ?? defaultRunProbe; + let execution: ReturnType; + try { + execution = normalizeProbeExecution(runProbe(command, { timeout: timeoutMs })); + } catch (e) { + return { ok: false, hostname, reason: "error", details: String((e as Error)?.message ?? e) }; + } + + const output = probeCombinedOutput(execution); + + // Outer spawn timeout / kill signal dominate the exit code, so check + // them before parsing the resolver markers. + if (execution.timedOut) { + return { + ok: false, + hostname, + reason: "timeout", + details: probeExecutionDetails("host DNS probe", execution, timeoutMs, output), + }; + } + if (execution.signal) { + return { + ok: false, + hostname, + reason: "killed", + details: probeExecutionDetails("host DNS probe", execution, timeoutMs, output), + }; + } + + if (execution.exitCode === 0 && /HOSTDNS_OK/.test(output)) { + return { ok: true, hostname }; + } + if (/HOSTDNS_TIMEOUT/.test(output)) { + return { + ok: false, + hostname, + reason: "timeout", + details: probeExecutionDetails("host DNS probe", execution, timeoutMs, output), + }; + } + const errMatch = output.match(/HOSTDNS_ERR\s+(\S+)/); + if (errMatch) { + const code = errMatch[1].toUpperCase(); + const details = `dns.lookup ${hostname}: ${errMatch[1]}`; + if (HOST_DNS_UNREACHABLE_CODES.has(code)) { + return { ok: false, hostname, reason: "servers_unreachable", details }; + } + if (HOST_DNS_RESOLUTION_CODES.has(code)) { + return { ok: false, hostname, reason: "resolution_failed", details }; + } + return { ok: false, hostname, reason: "error", details }; + } + // Couldn't run node / unexpected output — stay inconclusive so we + // never abort onboarding on a probe-infrastructure failure. + return { + ok: false, + hostname, + reason: "error", + details: output.trim() ? outputTail(output) : "host DNS probe produced no output", + }; +} + +/** + * A host DNS probe failure is fatal when the host genuinely cannot + * resolve the provider endpoint (resolver unreachable, NXDOMAIN/ENODATA, + * or the probe timed out / was killed mid-query). A generic `error` + * (could not even spawn the probe) stays inconclusive — the probe never + * proved host DNS is broken, so onboarding must not abort on it. + */ +export function isFatalHostDnsProbeFailure(result: HostDnsProbeResult): boolean { + if (result.ok) return false; + return ( + result.reason === "servers_unreachable" || + result.reason === "resolution_failed" || + result.reason === "timeout" || + result.reason === "killed" + ); +} + export function probeDockerBridgeContainerStart( opts: ProbeDockerBridgeContainerStartOpts = {}, ): DockerBridgeContainerStartProbeResult {