diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7e53a419a..7ad7ed5f80 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -200,7 +200,7 @@ repos: - id: pyright-check name: Pyright (nemoclaw-blueprint) - entry: bash -c 'cd nemoclaw-blueprint && uv run --with pyright pyright' + entry: bash -c 'cd nemoclaw-blueprint && uv run --with pyright --with pytest pyright' language: system pass_filenames: false always_run: true diff --git a/bin/lib/onboard.js b/bin/lib/onboard.js index 36e1cb7a98..6677750cc3 100644 --- a/bin/lib/onboard.js +++ b/bin/lib/onboard.js @@ -459,14 +459,15 @@ async function startGateway(gpu) { sleep(2); } - // CoreDNS fix — always run. k3s-inside-Docker has broken DNS on all platforms. + // CoreDNS fix — k3s-inside-Docker has broken DNS forwarding on all platforms. const runtime = getContainerRuntime(); if (shouldPatchCoredns(runtime)) { - console.log(" Patching CoreDNS for Colima..."); + console.log(" Patching CoreDNS DNS forwarding..."); run(`bash "${path.join(SCRIPTS, "fix-coredns.sh")}" nemoclaw 2>&1 || true`, { ignoreError: true }); } // Give DNS a moment to propagate sleep(5); + } // ── Step 3: Sandbox ────────────────────────────────────────────── @@ -613,6 +614,11 @@ async function createSandbox(gpu) { gpuEnabled: !!gpu, }); + // DNS proxy — run a forwarder in the sandbox pod so the isolated + // sandbox namespace can resolve DNS. Must run after sandbox is Ready. + console.log(" Setting up sandbox DNS proxy..."); + run(`bash "${path.join(SCRIPTS, "setup-dns-proxy.sh")}" nemoclaw "${sandboxName}" 2>&1 || true`, { ignoreError: true }); + console.log(` ✓ Sandbox '${sandboxName}' created`); return sandboxName; } diff --git a/bin/lib/platform.js b/bin/lib/platform.js index 67c31a3f3e..592b573ac8 100644 --- a/bin/lib/platform.js +++ b/bin/lib/platform.js @@ -36,7 +36,10 @@ function isUnsupportedMacosRuntime(runtime, opts = {}) { } function shouldPatchCoredns(runtime) { - return runtime === "colima"; + // k3s-inside-Docker has broken DNS forwarding on all platforms + // (systemd-resolved, Docker Desktop DNS, Colima DNS). + // Always patch CoreDNS to use a non-loopback upstream. + return runtime !== "unknown"; } function getColimaDockerSocketCandidates(opts = {}) { diff --git a/scripts/fix-coredns.sh b/scripts/fix-coredns.sh index 9b587ab339..0e2b787471 100755 --- a/scripts/fix-coredns.sh +++ b/scripts/fix-coredns.sh @@ -2,17 +2,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Fix CoreDNS on local OpenShell gateways running under Colima. +# Fix CoreDNS on local OpenShell gateways. # # Problem: k3s CoreDNS forwards to /etc/resolv.conf which inside the -# CoreDNS pod resolves to 127.0.0.11 (Docker's embedded DNS). That -# address is NOT reachable from k3s pods, causing DNS to fail and -# CoreDNS to CrashLoop. +# CoreDNS pod resolves to a loopback address (127.0.0.11 on Docker, +# 127.0.0.53 on systemd-resolved hosts). That address is NOT reachable +# from k3s pods, causing DNS to fail and CoreDNS to CrashLoop. # -# Fix: forward CoreDNS to the container's default gateway IP, which -# is reachable from pods and routes DNS through Docker to the host. +# Fix: forward CoreDNS to a non-loopback upstream — either the +# container's default gateway IP (routes through Docker to the host) +# or a public DNS server (8.8.8.8) as a last resort. # -# Run this after `openshell gateway start` on Colima setups. +# Run this after `openshell gateway start`. # # Usage: ./scripts/fix-coredns.sh [gateway-name] @@ -23,15 +24,11 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # shellcheck source=./lib/runtime.sh . "$SCRIPT_DIR/lib/runtime.sh" -COLIMA_SOCKET="$(find_colima_docker_socket || true)" - if [ -z "${DOCKER_HOST:-}" ]; then - if [ -n "$COLIMA_SOCKET" ]; then - export DOCKER_HOST="unix://$COLIMA_SOCKET" - else - echo "Skipping CoreDNS patch: Colima socket not found." - exit 0 + if docker_host="$(detect_docker_host)"; then + export DOCKER_HOST="$docker_host" fi + # If still unset, Docker CLI will use the default socket fi # Find the cluster container @@ -48,10 +45,17 @@ fi CONTAINER_RESOLV_CONF="$(docker exec "$CLUSTER" cat /etc/resolv.conf 2>/dev/null || true)" HOST_RESOLV_CONF="$(cat /etc/resolv.conf 2>/dev/null || true)" -UPSTREAM_DNS="$(resolve_coredns_upstream "$CONTAINER_RESOLV_CONF" "$HOST_RESOLV_CONF" "colima" || true)" + +# Detect runtime for Colima-specific DNS discovery paths +RUNTIME="unknown" +if [ -n "${DOCKER_HOST:-}" ]; then + RUNTIME="$(docker_host_runtime "$DOCKER_HOST" || echo "unknown")" +fi + +UPSTREAM_DNS="$(resolve_coredns_upstream "$CONTAINER_RESOLV_CONF" "$HOST_RESOLV_CONF" "$RUNTIME" || true)" if [ -z "$UPSTREAM_DNS" ]; then - echo "ERROR: Could not determine a non-loopback DNS upstream for Colima." + echo "ERROR: Could not determine a non-loopback DNS upstream." exit 1 fi diff --git a/scripts/lib/runtime.sh b/scripts/lib/runtime.sh index a6bba65f2f..19218e236c 100755 --- a/scripts/lib/runtime.sh +++ b/scripts/lib/runtime.sh @@ -163,7 +163,9 @@ resolve_coredns_upstream() { return 0 fi - return 1 + # Last resort: public DNS. Needed on hosts where all nameservers are + # loopback (e.g. systemd-resolved uses 127.0.0.53). + printf '8.8.8.8\n' } select_openshell_cluster_container() { diff --git a/scripts/setup-dns-proxy.sh b/scripts/setup-dns-proxy.sh new file mode 100755 index 0000000000..36c5349e25 --- /dev/null +++ b/scripts/setup-dns-proxy.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Fix sandbox DNS by running a lightweight DNS forwarder in the sandbox pod. +# +# Problem: The sandbox runs in an isolated network namespace (10.200.0.0/24). +# Its /etc/resolv.conf points to the k3s CoreDNS service IP (10.43.0.10), but +# DNS packets from the sandbox route through the pod namespace — where the +# CoreDNS service IP is not locally handled. The result: dns.lookup() fails +# with EAI_AGAIN for every outbound request. +# +# Fix: Run a Python DNS forwarder in the sandbox pod's namespace that: +# 1. Adds 10.43.0.10 as a local address on lo (so packets from the sandbox +# are delivered locally instead of forwarded) +# 2. Listens on 0.0.0.0:53 (UDP) and forwards to public DNS (8.8.8.8) +# +# The sandbox's existing resolv.conf (nameserver 10.43.0.10) works without +# modification — the forwarder intercepts the traffic transparently. +# +# The DNS proxy is launched via `docker exec -d` + `nsenter` from the gateway +# container, which keeps it alive as a persistent background process. +# +# Requires: sandbox must be in Ready state. Run after sandbox creation. +# +# Usage: ./scripts/setup-dns-proxy.sh [gateway-name] + +set -euo pipefail + +GATEWAY_NAME="${1:-}" +SANDBOX_NAME="${2:-}" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=./lib/runtime.sh +. "$SCRIPT_DIR/lib/runtime.sh" + +if [ -z "$SANDBOX_NAME" ]; then + echo "Usage: $0 [gateway-name] " + exit 1 +fi + +# CoreDNS service IP that the sandbox's /etc/resolv.conf points to +COREDNS_SERVICE_IP="10.43.0.10" +# DNS_UPSTREAM is set below after we discover the CoreDNS pod IP + +# ── Find the gateway container ────────────────────────────────────── + +if [ -z "${DOCKER_HOST:-}" ]; then + if docker_host="$(detect_docker_host)"; then + export DOCKER_HOST="$docker_host" + fi +fi + +CLUSTERS="$(docker ps --filter "name=openshell-cluster" --format '{{.Names}}' 2>/dev/null || true)" +CLUSTER="$(select_openshell_cluster_container "$GATEWAY_NAME" "$CLUSTERS" || true)" + +if [ -z "$CLUSTER" ]; then + if [ -n "$GATEWAY_NAME" ]; then + echo "ERROR: Could not find gateway container for '$GATEWAY_NAME'." + else + echo "ERROR: Could not find any openshell cluster container." + fi + exit 1 +fi + +# ── Helper: kubectl via gateway ───────────────────────────────────── + +kctl() { + docker exec "$CLUSTER" kubectl "$@" +} + +# ── Discover CoreDNS pod IP ───────────────────────────────────────── +# +# Forward to CoreDNS (not 8.8.8.8) so k8s-internal names like +# openshell-0.openshell.svc.cluster.local still resolve. CoreDNS +# handles both k8s names (kubernetes plugin) and external names +# (forward plugin, patched by fix-coredns.sh). + +DNS_UPSTREAM="$(kctl get endpoints kube-dns \ + -n kube-system -o jsonpath='{.subsets[0].addresses[0].ip}' 2>/dev/null || true)" + +if [ -z "$DNS_UPSTREAM" ]; then + echo "WARNING: Could not discover CoreDNS pod IP. Falling back to 8.8.8.8." + echo "WARNING: k8s-internal names (inference.local routing) will NOT work." + DNS_UPSTREAM="8.8.8.8" +fi + +# ── Find the sandbox pod and its PID ──────────────────────────────── + +POD="$(kctl get pods -n openshell -o name 2>/dev/null \ + | grep -- "$SANDBOX_NAME" | head -1 | sed 's|pod/||' || true)" + +if [ -z "$POD" ]; then + echo "ERROR: Could not find pod for sandbox '$SANDBOX_NAME'." + exit 1 +fi + +# Get the pod's init PID as seen from the gateway container (for nsenter) +POD_PID="$(docker exec "$CLUSTER" sh -c " + # Find PID that has the pod's hostname in its UTS namespace + for pid in /proc/[0-9]*/ns; do + p=\${pid%/ns}; p=\${p##*/} + if [ -f /proc/\$p/root/etc/hostname ] 2>/dev/null; then + hn=\$(cat /proc/\$p/root/etc/hostname 2>/dev/null) + if [ \"\$hn\" = \"$POD\" ]; then + echo \$p + break + fi + fi + done +" 2>/dev/null || true)" + +if [ -z "$POD_PID" ]; then + echo "WARNING: Could not find pod PID via hostname. Trying kubectl..." + # Fallback: use kubectl exec to find a PID we can nsenter into + POD_PID="$(kctl exec -n openshell "$POD" -- sh -c 'echo $$' 2>/dev/null || true)" +fi + +if [ -z "$POD_PID" ]; then + echo "ERROR: Could not determine pod PID for nsenter." + exit 1 +fi + +echo "Setting up DNS proxy in pod '$POD' (pid=$POD_PID, ${COREDNS_SERVICE_IP} → ${DNS_UPSTREAM})..." + +# ── Step 1: Add CoreDNS service IP as local address ───────────────── + +kctl exec -n openshell "$POD" -- \ + ip addr add "${COREDNS_SERVICE_IP}/32" dev lo 2>/dev/null || true + +# ── Step 2: Write DNS proxy script to the pod ─────────────────────── + +kctl exec -n openshell "$POD" -- sh -c "cat > /tmp/dns-proxy.py << DNSPROXY +import socket, threading, os + +UPSTREAM = ('${DNS_UPSTREAM}', 53) + +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +sock.bind(('10.43.0.10', 53)) + +with open('/tmp/dns-proxy.pid', 'w') as pf: + pf.write(str(os.getpid())) + +with open('/tmp/dns-proxy.log', 'w') as log: + log.write('dns-proxy: 10.43.0.10:53 -> {}:{} pid={}\n'.format( + UPSTREAM[0], UPSTREAM[1], os.getpid())) + +def forward(data, addr): + try: + f = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + f.settimeout(5) + f.sendto(data, UPSTREAM) + r, _ = f.recvfrom(4096) + sock.sendto(r, addr) + f.close() + except Exception: + pass + +while True: + d, a = sock.recvfrom(4096) + threading.Thread(target=forward, args=(d, a), daemon=True).start() +DNSPROXY" + +# ── Step 3: Kill any existing DNS proxy ───────────────────────────── + +OLD_PID="$(kctl exec -n openshell "$POD" -- cat /tmp/dns-proxy.pid 2>/dev/null || true)" +if [ -n "$OLD_PID" ]; then + kctl exec -n openshell "$POD" -- kill "$OLD_PID" 2>/dev/null || true + sleep 1 +fi + +# ── Step 4: Launch DNS proxy via docker exec -d (persistent) ──────── +# +# Using `docker exec -d` (detached) + `nsenter` to enter the pod's +# network and mount namespaces. This creates a persistent process that +# survives after the script exits — unlike kubectl exec which kills +# child processes on session end. + +docker exec -d "$CLUSTER" \ + nsenter -t "$POD_PID" -n -m -- \ + python3 -u /tmp/dns-proxy.py + +sleep 2 + +# ── Step 5: Verify ────────────────────────────────────────────────── + +LOG="$(kctl exec -n openshell "$POD" -- cat /tmp/dns-proxy.log 2>/dev/null || true)" +if echo "$LOG" | grep -q "dns-proxy:"; then + echo "DNS proxy started: $LOG" +else + echo "WARNING: DNS proxy may not have started. Log: $LOG" +fi diff --git a/scripts/setup.sh b/scripts/setup.sh index 6aeb680851..47d253a231 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -123,9 +123,9 @@ for i in 1 2 3 4 5; do done info "Gateway is healthy" -# 2. CoreDNS fix (Colima only) -if [ "$CONTAINER_RUNTIME" = "colima" ]; then - info "Patching CoreDNS for Colima..." +# 2. CoreDNS fix — k3s-inside-Docker has broken DNS forwarding on all platforms. +if [ "$CONTAINER_RUNTIME" != "unknown" ]; then + info "Patching CoreDNS DNS forwarding..." bash "$SCRIPT_DIR/fix-coredns.sh" nemoclaw 2>&1 || warn "CoreDNS patch failed (may not be needed)" fi @@ -230,6 +230,10 @@ if ! echo "$SANDBOX_LINE" | grep -q "Ready"; then fail "Sandbox created but not Ready (phase: ${SANDBOX_PHASE:-unknown}). Check 'openshell sandbox get ${SANDBOX_NAME}'." fi +# 5b. DNS proxy for sandbox — run after sandbox is Ready. +info "Setting up sandbox DNS proxy..." +bash "$SCRIPT_DIR/setup-dns-proxy.sh" nemoclaw "$SANDBOX_NAME" 2>&1 || warn "DNS proxy setup failed (may not be needed)" + # 6. Done echo "" info "Setup complete!" diff --git a/test/dns-proxy.test.js b/test/dns-proxy.test.js new file mode 100644 index 0000000000..462e62f84b --- /dev/null +++ b/test/dns-proxy.test.js @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, it, expect } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +const SETUP_DNS_PROXY = path.join(import.meta.dirname, "..", "scripts", "setup-dns-proxy.sh"); +const RUNTIME_SH = path.join(import.meta.dirname, "..", "scripts", "lib", "runtime.sh"); + +describe("setup-dns-proxy.sh", () => { + it("exists and is executable", () => { + const stat = fs.statSync(SETUP_DNS_PROXY); + expect(stat.isFile()).toBe(true); + // Check executable bit (owner) + expect(stat.mode & 0o100).toBeTruthy(); + }); + + it("sources runtime.sh successfully", () => { + const result = spawnSync("bash", ["-lc", `source "${RUNTIME_SH}"; echo ok`], { + encoding: "utf-8", + env: { ...process.env }, + }); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe("ok"); + }); + + it("exits with usage when no sandbox name provided", () => { + const result = spawnSync("bash", ["-lc", ` + bash "${SETUP_DNS_PROXY}" nemoclaw + `], { + encoding: "utf-8", + env: { ...process.env, SETUP_DNS_PROXY }, + }); + + expect(result.status).not.toBe(0); + expect(result.stderr + result.stdout).toMatch(/Usage:/i); + }); + + it("references CoreDNS service IP and upstream DNS", () => { + const content = fs.readFileSync(SETUP_DNS_PROXY, "utf-8"); + expect(content).toContain('COREDNS_SERVICE_IP="10.43.0.10"'); + expect(content).toContain('DNS_UPSTREAM="8.8.8.8"'); + }); + + it("adds CoreDNS service IP as local address in pod", () => { + const content = fs.readFileSync(SETUP_DNS_PROXY, "utf-8"); + expect(content).toContain("ip addr add"); + expect(content).toContain("COREDNS_SERVICE_IP"); + }); + + it("deploys a Python DNS forwarder to the pod", () => { + const content = fs.readFileSync(SETUP_DNS_PROXY, "utf-8"); + expect(content).toContain("dns-proxy.py"); + expect(content).toContain("socket.SOCK_DGRAM"); + expect(content).toContain("kctl exec"); + }); +}); diff --git a/test/platform.test.js b/test/platform.test.js index 0eb85277ee..0582e45450 100644 --- a/test/platform.test.js +++ b/test/platform.test.js @@ -134,10 +134,14 @@ describe("platform helpers", () => { }); describe("shouldPatchCoredns", () => { - it("patches CoreDNS for Colima only", () => { + it("patches CoreDNS for all known runtimes", () => { expect(shouldPatchCoredns("colima")).toBe(true); - expect(shouldPatchCoredns("docker-desktop")).toBe(false); - expect(shouldPatchCoredns("docker")).toBe(false); + expect(shouldPatchCoredns("docker-desktop")).toBe(true); + expect(shouldPatchCoredns("docker")).toBe(true); + }); + + it("skips patching when runtime is unknown", () => { + expect(shouldPatchCoredns("unknown")).toBe(false); }); }); }); diff --git a/test/runtime-shell.test.js b/test/runtime-shell.test.js index 2e5702f22f..7592fa9468 100644 --- a/test/runtime-shell.test.js +++ b/test/runtime-shell.test.js @@ -170,6 +170,17 @@ describe("shell runtime helpers", () => { expect(result.stdout.trim()).toBe("9.9.9.9"); }); + it("falls back to public DNS when all nameservers are loopback", () => { + const result = runShell( + `source "${RUNTIME_SH}"; + get_colima_vm_nameserver() { return 1; } + resolve_coredns_upstream $'nameserver 127.0.0.11' $'nameserver 127.0.0.53' unknown`, + ); + + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe("8.8.8.8"); + }); + it("does not consume installer stdin when reading the Colima VM nameserver", () => { const result = runShell( `function colima() { cat > /dev/null || true; printf 'nameserver 100.100.100.100\\n'; }